github.com/treeverse/lakefs@v1.24.1-0.20240520134607-95648127bfb0/pkg/block/factory/build.go (about)

     1  package factory
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  
     7  	"cloud.google.com/go/storage"
     8  	"github.com/aws/aws-sdk-go-v2/service/s3"
     9  	"github.com/treeverse/lakefs/pkg/block"
    10  	"github.com/treeverse/lakefs/pkg/block/azure"
    11  	"github.com/treeverse/lakefs/pkg/block/gs"
    12  	"github.com/treeverse/lakefs/pkg/block/local"
    13  	"github.com/treeverse/lakefs/pkg/block/mem"
    14  	"github.com/treeverse/lakefs/pkg/block/params"
    15  	s3a "github.com/treeverse/lakefs/pkg/block/s3"
    16  	"github.com/treeverse/lakefs/pkg/block/transient"
    17  	"github.com/treeverse/lakefs/pkg/logging"
    18  	"github.com/treeverse/lakefs/pkg/stats"
    19  	"golang.org/x/oauth2/google"
    20  	"google.golang.org/api/option"
    21  )
    22  
    23  const (
    24  	// googleAuthCloudPlatform - Cloud Storage authentication https://cloud.google.com/storage/docs/authentication
    25  	googleAuthCloudPlatform = "https://www.googleapis.com/auth/cloud-platform"
    26  )
    27  
    28  func BuildBlockAdapter(ctx context.Context, statsCollector stats.Collector, c params.AdapterConfig) (block.Adapter, error) {
    29  	blockstore := c.BlockstoreType()
    30  	logging.FromContext(ctx).
    31  		WithField("type", blockstore).
    32  		Info("initialize blockstore adapter")
    33  	switch blockstore {
    34  	case block.BlockstoreTypeLocal:
    35  		p, err := c.BlockstoreLocalParams()
    36  		if err != nil {
    37  			return nil, err
    38  		}
    39  		return buildLocalAdapter(ctx, p)
    40  	case block.BlockstoreTypeS3:
    41  		p, err := c.BlockstoreS3Params()
    42  		if err != nil {
    43  			return nil, err
    44  		}
    45  		return buildS3Adapter(ctx, statsCollector, p)
    46  	case block.BlockstoreTypeMem, "memory":
    47  		return mem.New(ctx), nil
    48  	case block.BlockstoreTypeTransient:
    49  		return transient.New(ctx), nil
    50  	case block.BlockstoreTypeGS:
    51  		p, err := c.BlockstoreGSParams()
    52  		if err != nil {
    53  			return nil, err
    54  		}
    55  		return buildGSAdapter(ctx, p)
    56  	case block.BlockstoreTypeAzure:
    57  		p, err := c.BlockstoreAzureParams()
    58  		if err != nil {
    59  			return nil, err
    60  		}
    61  		return azure.NewAdapter(ctx, p)
    62  	default:
    63  		return nil, fmt.Errorf("%w '%s' please choose one of %s",
    64  			block.ErrInvalidAddress, blockstore, []string{block.BlockstoreTypeLocal, block.BlockstoreTypeS3, block.BlockstoreTypeAzure, block.BlockstoreTypeMem, block.BlockstoreTypeTransient, block.BlockstoreTypeGS})
    65  	}
    66  }
    67  
    68  func buildLocalAdapter(ctx context.Context, params params.Local) (*local.Adapter, error) {
    69  	adapter, err := local.NewAdapter(params.Path,
    70  		local.WithAllowedExternalPrefixes(params.AllowedExternalPrefixes),
    71  		local.WithImportEnabled(params.ImportEnabled),
    72  	)
    73  	if err != nil {
    74  		return nil, fmt.Errorf("got error opening a local block adapter with path %s: %w", params.Path, err)
    75  	}
    76  	logging.FromContext(ctx).WithFields(logging.Fields{
    77  		"type": "local",
    78  		"path": params.Path,
    79  	}).Info("initialized blockstore adapter")
    80  	return adapter, nil
    81  }
    82  
    83  func BuildS3Client(ctx context.Context, params params.S3) (*s3.Client, error) {
    84  	cfg, err := s3a.LoadConfig(ctx, params)
    85  	if err != nil {
    86  		return nil, err
    87  	}
    88  
    89  	client := s3.NewFromConfig(cfg, s3a.WithClientParams(params))
    90  	return client, nil
    91  }
    92  
    93  func buildS3Adapter(ctx context.Context, statsCollector stats.Collector, params params.S3) (*s3a.Adapter, error) {
    94  	opts := []s3a.AdapterOption{
    95  		s3a.WithStatsCollector(statsCollector),
    96  		s3a.WithDiscoverBucketRegion(params.DiscoverBucketRegion),
    97  		s3a.WithPreSignedExpiry(params.PreSignedExpiry),
    98  		s3a.WithDisablePreSigned(params.DisablePreSigned),
    99  		s3a.WithDisablePreSignedUI(params.DisablePreSignedUI),
   100  		s3a.WithDisablePreSignedMultipart(params.DisablePreSignedMultipart),
   101  	}
   102  	if params.ServerSideEncryption != "" {
   103  		opts = append(opts, s3a.WithServerSideEncryption(params.ServerSideEncryption))
   104  	}
   105  	if params.ServerSideEncryptionKmsKeyID != "" {
   106  		opts = append(opts, s3a.WithServerSideEncryptionKmsKeyID(params.ServerSideEncryptionKmsKeyID))
   107  	}
   108  	adapter, err := s3a.NewAdapter(ctx, params, opts...)
   109  	if err != nil {
   110  		return nil, err
   111  	}
   112  	logging.FromContext(ctx).WithField("type", "s3").Info("initialized blockstore adapter")
   113  	return adapter, nil
   114  }
   115  
   116  func BuildGSClient(ctx context.Context, params params.GS) (*storage.Client, error) {
   117  	var opts []option.ClientOption
   118  	if params.CredentialsFile != "" {
   119  		opts = append(opts, option.WithCredentialsFile(params.CredentialsFile))
   120  	} else if params.CredentialsJSON != "" {
   121  		cred, err := google.CredentialsFromJSON(ctx, []byte(params.CredentialsJSON), googleAuthCloudPlatform)
   122  		if err != nil {
   123  			return nil, err
   124  		}
   125  		opts = append(opts, option.WithCredentials(cred))
   126  	}
   127  	return storage.NewClient(ctx, opts...)
   128  }
   129  
   130  func buildGSAdapter(ctx context.Context, params params.GS) (*gs.Adapter, error) {
   131  	client, err := BuildGSClient(ctx, params)
   132  	if err != nil {
   133  		return nil, err
   134  	}
   135  	adapter := gs.NewAdapter(client,
   136  		gs.WithPreSignedExpiry(params.PreSignedExpiry),
   137  		gs.WithDisablePreSigned(params.DisablePreSigned),
   138  		gs.WithDisablePreSignedUI(params.DisablePreSignedUI),
   139  	)
   140  	logging.FromContext(ctx).WithField("type", "gs").Info("initialized blockstore adapter")
   141  	return adapter, nil
   142  }