storj.io/uplink@v1.13.0/project.go (about)

     1  // Copyright (C) 2020 Storj Labs, Inc.
     2  // See LICENSE for copying information.
     3  
     4  package uplink
     5  
     6  import (
     7  	"context"
     8  
     9  	"github.com/zeebo/errs"
    10  
    11  	"storj.io/common/leak"
    12  	"storj.io/common/memory"
    13  	"storj.io/common/rpc"
    14  	"storj.io/common/storj"
    15  	"storj.io/uplink/private/ecclient"
    16  	"storj.io/uplink/private/metaclient"
    17  	"storj.io/uplink/private/storage/streams"
    18  	"storj.io/uplink/private/testuplink"
    19  	"storj.io/uplink/private/version"
    20  )
    21  
    22  // TODO we need find a way how to pass it from satellite to client.
    23  const maxInlineSize = 4096 // 4KiB
    24  
    25  // maxSegmentSize can be used to override max segment size with ldflags build parameter.
    26  // Example: go build -ldflags "-X 'storj.io/uplink.maxSegmentSize=1MiB'" storj.io/storj/cmd/uplink.
    27  var maxSegmentSize string
    28  
    29  // Project provides access to managing buckets and objects.
    30  type Project struct {
    31  	config                        Config
    32  	access                        *Access
    33  	satelliteDialer               rpc.Dialer
    34  	storagenodeDialer             rpc.Dialer
    35  	ec                            ecclient.Client
    36  	segmentSize                   int64
    37  	encryptionParameters          storj.EncryptionParameters
    38  	concurrentSegmentUploadConfig *testuplink.ConcurrentSegmentUploadsConfig
    39  
    40  	tracker leak.Ref
    41  }
    42  
    43  // OpenProject opens a project with the specific access grant.
    44  func OpenProject(ctx context.Context, access *Access) (*Project, error) {
    45  	return (Config{}).OpenProject(ctx, access)
    46  }
    47  
    48  // OpenProject opens a project with the specific access grant.
    49  func (config Config) OpenProject(ctx context.Context, access *Access) (project *Project, err error) {
    50  	defer mon.Task()(&ctx)(&err)
    51  
    52  	if access == nil {
    53  		return nil, packageError.New("access grant is nil")
    54  	}
    55  
    56  	switch {
    57  	case config.DialTimeout < 0:
    58  		config.DialTimeout = 0 // no timeout
    59  	case config.DialTimeout == 0:
    60  		config.DialTimeout = defaultDialTimeout
    61  	}
    62  
    63  	if err := config.validateUserAgent(ctx); err != nil {
    64  		return nil, packageError.New("invalid user agent: %w", err)
    65  	}
    66  
    67  	config.UserAgent, err = version.AppendVersionToUserAgent(config.UserAgent)
    68  	if err != nil {
    69  		return nil, packageError.Wrap(err)
    70  	}
    71  
    72  	storagenodeDialer, err := config.getDialerForPool(ctx, config.pool)
    73  	if err != nil {
    74  		return nil, packageError.Wrap(err)
    75  	}
    76  	satelliteDialer, err := config.getDialerForPool(ctx, config.satellitePool)
    77  	if err != nil {
    78  		return nil, packageError.Wrap(err)
    79  	}
    80  
    81  	// TODO: This should come from the EncryptionAccess. For now it's hardcoded to twice the
    82  	// stripe size of the default redundancy scheme on the satellite.
    83  	encBlockSize := 29 * 256 * memory.B.Int32()
    84  
    85  	encryptionParameters := storj.EncryptionParameters{
    86  		// N.B.: This is the ciphersuite we use for encrypting content keys,
    87  		// which should absolutely be encrypted, even if the access grant
    88  		// says EncNull.
    89  		CipherSuite: storj.EncAESGCM,
    90  		BlockSize:   encBlockSize,
    91  	}
    92  
    93  	// TODO: All these should be controlled by the satellite and not configured by the uplink.
    94  	// For now we need to have these hard coded values that match the satellite configuration
    95  	// to be able to create the underlying stream store.
    96  	var (
    97  		segmentsSize = 64 * memory.MiB.Int64()
    98  	)
    99  
   100  	if maxSegmentSize != "" {
   101  		segmentsSize, err = memory.ParseString(maxSegmentSize)
   102  		if err != nil {
   103  			return nil, packageError.Wrap(err)
   104  		}
   105  	} else {
   106  		s, ok := testuplink.GetMaxSegmentSize(ctx)
   107  		if ok {
   108  			segmentsSize = s.Int64()
   109  		}
   110  	}
   111  
   112  	ec := ecclient.New(storagenodeDialer, 0)
   113  
   114  	tracker := leak.FromContext(ctx)
   115  	if tracker == (leak.Ref{}) { // TODO: handle this check better
   116  		tracker = leak.Root(1)
   117  	}
   118  
   119  	return &Project{
   120  		config:                        config,
   121  		access:                        access,
   122  		satelliteDialer:               satelliteDialer,
   123  		storagenodeDialer:             storagenodeDialer,
   124  		ec:                            ec,
   125  		segmentSize:                   segmentsSize,
   126  		encryptionParameters:          encryptionParameters,
   127  		concurrentSegmentUploadConfig: testuplink.GetConcurrentSegmentUploadsConfig(ctx),
   128  
   129  		tracker: tracker,
   130  	}, nil
   131  }
   132  
   133  // Close closes the project and all associated resources.
   134  func (project *Project) Close() (err error) {
   135  	// only close the connection pools if it's created through OpenProject / getDialer()
   136  	if project.config.pool == nil {
   137  		err = errs.Combine(err, project.storagenodeDialer.Pool.Close())
   138  
   139  		if project.config.satellitePool == nil {
   140  			// if config.satellitePool is nil, but config.pool is not, it might be a second Close, but it's safe.
   141  			err = errs.Combine(err, project.satelliteDialer.Pool.Close())
   142  		}
   143  	}
   144  
   145  	return packageError.Wrap(errs.Combine(err, project.tracker.Close()))
   146  }
   147  
   148  func (project *Project) getStreamsStore(ctx context.Context) (_ *streams.Store, err error) {
   149  	defer mon.Task()(&ctx)(&err)
   150  
   151  	metainfoClient, err := project.dialMetainfoClient(ctx)
   152  	if err != nil {
   153  		return nil, packageError.Wrap(err)
   154  	}
   155  	defer func() {
   156  		if err != nil {
   157  			err = errs.Combine(err, metainfoClient.Close())
   158  		}
   159  	}()
   160  
   161  	var longTailMargin int
   162  	if project.concurrentSegmentUploadConfig != nil {
   163  		longTailMargin = project.concurrentSegmentUploadConfig.LongTailMargin
   164  	}
   165  
   166  	streamStore, err := streams.NewStreamStore(
   167  		metainfoClient,
   168  		project.ec,
   169  		project.segmentSize,
   170  		project.access.encAccess.Store,
   171  		project.encryptionParameters,
   172  		maxInlineSize,
   173  		longTailMargin)
   174  	if err != nil {
   175  		return nil, packageError.Wrap(err)
   176  	}
   177  
   178  	return streamStore, nil
   179  }
   180  
   181  func (project *Project) dialMetainfoDB(ctx context.Context) (_ *metaclient.DB, err error) {
   182  	defer mon.Task()(&ctx)(&err)
   183  
   184  	metainfoClient, err := project.dialMetainfoClient(ctx)
   185  	if err != nil {
   186  		return nil, packageError.Wrap(err)
   187  	}
   188  
   189  	return metaclient.New(metainfoClient, project.encryptionParameters, project.access.encAccess.Store), nil
   190  }
   191  
   192  func (project *Project) dialMetainfoClient(ctx context.Context) (_ *metaclient.Client, err error) {
   193  	defer mon.Task()(&ctx)(&err)
   194  
   195  	metainfoClient, err := metaclient.DialNodeURL(ctx,
   196  		project.satelliteDialer,
   197  		project.access.satelliteURL.String(),
   198  		project.access.apiKey,
   199  		project.config.UserAgent)
   200  	if err != nil {
   201  		return nil, packageError.Wrap(err)
   202  	}
   203  
   204  	return metainfoClient, nil
   205  }
   206  
   207  //nolint:deadcode
   208  //lint:ignore U1000 its used in private/object package
   209  func dialMetainfoDBWithProject(ctx context.Context, project *Project) (_ *metaclient.DB, err error) {
   210  	defer mon.Task()(&ctx)(&err)
   211  
   212  	return project.dialMetainfoDB(ctx)
   213  }
   214  
   215  //nolint:deadcode
   216  //lint:ignore U1000 its used in private/object package
   217  func getStreamsStoreWithProject(ctx context.Context, project *Project) (_ *streams.Store, err error) {
   218  	defer mon.Task()(&ctx)(&err)
   219  
   220  	return project.getStreamsStore(ctx)
   221  }