storj.io/uplink@v1.13.0/project.go (about) 1 // Copyright (C) 2020 Storj Labs, Inc. 2 // See LICENSE for copying information. 3 4 package uplink 5 6 import ( 7 "context" 8 9 "github.com/zeebo/errs" 10 11 "storj.io/common/leak" 12 "storj.io/common/memory" 13 "storj.io/common/rpc" 14 "storj.io/common/storj" 15 "storj.io/uplink/private/ecclient" 16 "storj.io/uplink/private/metaclient" 17 "storj.io/uplink/private/storage/streams" 18 "storj.io/uplink/private/testuplink" 19 "storj.io/uplink/private/version" 20 ) 21 22 // TODO we need find a way how to pass it from satellite to client. 23 const maxInlineSize = 4096 // 4KiB 24 25 // maxSegmentSize can be used to override max segment size with ldflags build parameter. 26 // Example: go build -ldflags "-X 'storj.io/uplink.maxSegmentSize=1MiB'" storj.io/storj/cmd/uplink. 27 var maxSegmentSize string 28 29 // Project provides access to managing buckets and objects. 30 type Project struct { 31 config Config 32 access *Access 33 satelliteDialer rpc.Dialer 34 storagenodeDialer rpc.Dialer 35 ec ecclient.Client 36 segmentSize int64 37 encryptionParameters storj.EncryptionParameters 38 concurrentSegmentUploadConfig *testuplink.ConcurrentSegmentUploadsConfig 39 40 tracker leak.Ref 41 } 42 43 // OpenProject opens a project with the specific access grant. 44 func OpenProject(ctx context.Context, access *Access) (*Project, error) { 45 return (Config{}).OpenProject(ctx, access) 46 } 47 48 // OpenProject opens a project with the specific access grant. 49 func (config Config) OpenProject(ctx context.Context, access *Access) (project *Project, err error) { 50 defer mon.Task()(&ctx)(&err) 51 52 if access == nil { 53 return nil, packageError.New("access grant is nil") 54 } 55 56 switch { 57 case config.DialTimeout < 0: 58 config.DialTimeout = 0 // no timeout 59 case config.DialTimeout == 0: 60 config.DialTimeout = defaultDialTimeout 61 } 62 63 if err := config.validateUserAgent(ctx); err != nil { 64 return nil, packageError.New("invalid user agent: %w", err) 65 } 66 67 config.UserAgent, err = version.AppendVersionToUserAgent(config.UserAgent) 68 if err != nil { 69 return nil, packageError.Wrap(err) 70 } 71 72 storagenodeDialer, err := config.getDialerForPool(ctx, config.pool) 73 if err != nil { 74 return nil, packageError.Wrap(err) 75 } 76 satelliteDialer, err := config.getDialerForPool(ctx, config.satellitePool) 77 if err != nil { 78 return nil, packageError.Wrap(err) 79 } 80 81 // TODO: This should come from the EncryptionAccess. For now it's hardcoded to twice the 82 // stripe size of the default redundancy scheme on the satellite. 83 encBlockSize := 29 * 256 * memory.B.Int32() 84 85 encryptionParameters := storj.EncryptionParameters{ 86 // N.B.: This is the ciphersuite we use for encrypting content keys, 87 // which should absolutely be encrypted, even if the access grant 88 // says EncNull. 89 CipherSuite: storj.EncAESGCM, 90 BlockSize: encBlockSize, 91 } 92 93 // TODO: All these should be controlled by the satellite and not configured by the uplink. 94 // For now we need to have these hard coded values that match the satellite configuration 95 // to be able to create the underlying stream store. 96 var ( 97 segmentsSize = 64 * memory.MiB.Int64() 98 ) 99 100 if maxSegmentSize != "" { 101 segmentsSize, err = memory.ParseString(maxSegmentSize) 102 if err != nil { 103 return nil, packageError.Wrap(err) 104 } 105 } else { 106 s, ok := testuplink.GetMaxSegmentSize(ctx) 107 if ok { 108 segmentsSize = s.Int64() 109 } 110 } 111 112 ec := ecclient.New(storagenodeDialer, 0) 113 114 tracker := leak.FromContext(ctx) 115 if tracker == (leak.Ref{}) { // TODO: handle this check better 116 tracker = leak.Root(1) 117 } 118 119 return &Project{ 120 config: config, 121 access: access, 122 satelliteDialer: satelliteDialer, 123 storagenodeDialer: storagenodeDialer, 124 ec: ec, 125 segmentSize: segmentsSize, 126 encryptionParameters: encryptionParameters, 127 concurrentSegmentUploadConfig: testuplink.GetConcurrentSegmentUploadsConfig(ctx), 128 129 tracker: tracker, 130 }, nil 131 } 132 133 // Close closes the project and all associated resources. 134 func (project *Project) Close() (err error) { 135 // only close the connection pools if it's created through OpenProject / getDialer() 136 if project.config.pool == nil { 137 err = errs.Combine(err, project.storagenodeDialer.Pool.Close()) 138 139 if project.config.satellitePool == nil { 140 // if config.satellitePool is nil, but config.pool is not, it might be a second Close, but it's safe. 141 err = errs.Combine(err, project.satelliteDialer.Pool.Close()) 142 } 143 } 144 145 return packageError.Wrap(errs.Combine(err, project.tracker.Close())) 146 } 147 148 func (project *Project) getStreamsStore(ctx context.Context) (_ *streams.Store, err error) { 149 defer mon.Task()(&ctx)(&err) 150 151 metainfoClient, err := project.dialMetainfoClient(ctx) 152 if err != nil { 153 return nil, packageError.Wrap(err) 154 } 155 defer func() { 156 if err != nil { 157 err = errs.Combine(err, metainfoClient.Close()) 158 } 159 }() 160 161 var longTailMargin int 162 if project.concurrentSegmentUploadConfig != nil { 163 longTailMargin = project.concurrentSegmentUploadConfig.LongTailMargin 164 } 165 166 streamStore, err := streams.NewStreamStore( 167 metainfoClient, 168 project.ec, 169 project.segmentSize, 170 project.access.encAccess.Store, 171 project.encryptionParameters, 172 maxInlineSize, 173 longTailMargin) 174 if err != nil { 175 return nil, packageError.Wrap(err) 176 } 177 178 return streamStore, nil 179 } 180 181 func (project *Project) dialMetainfoDB(ctx context.Context) (_ *metaclient.DB, err error) { 182 defer mon.Task()(&ctx)(&err) 183 184 metainfoClient, err := project.dialMetainfoClient(ctx) 185 if err != nil { 186 return nil, packageError.Wrap(err) 187 } 188 189 return metaclient.New(metainfoClient, project.encryptionParameters, project.access.encAccess.Store), nil 190 } 191 192 func (project *Project) dialMetainfoClient(ctx context.Context) (_ *metaclient.Client, err error) { 193 defer mon.Task()(&ctx)(&err) 194 195 metainfoClient, err := metaclient.DialNodeURL(ctx, 196 project.satelliteDialer, 197 project.access.satelliteURL.String(), 198 project.access.apiKey, 199 project.config.UserAgent) 200 if err != nil { 201 return nil, packageError.Wrap(err) 202 } 203 204 return metainfoClient, nil 205 } 206 207 //nolint:deadcode 208 //lint:ignore U1000 its used in private/object package 209 func dialMetainfoDBWithProject(ctx context.Context, project *Project) (_ *metaclient.DB, err error) { 210 defer mon.Task()(&ctx)(&err) 211 212 return project.dialMetainfoDB(ctx) 213 } 214 215 //nolint:deadcode 216 //lint:ignore U1000 its used in private/object package 217 func getStreamsStoreWithProject(ctx context.Context, project *Project) (_ *streams.Store, err error) { 218 defer mon.Task()(&ctx)(&err) 219 220 return project.getStreamsStore(ctx) 221 }