storj.io/uplink@v1.13.0/private/storage/streams/uploader.go (about) 1 // Copyright (C) 2023 Storj Labs, Inc. 2 // See LICENSE for copying information. 3 4 package streams 5 6 import ( 7 "context" 8 "crypto/rand" 9 "fmt" 10 "io" 11 "sync/atomic" 12 "time" 13 14 "github.com/zeebo/errs" 15 16 "storj.io/common/encryption" 17 "storj.io/common/paths" 18 "storj.io/common/pb" 19 "storj.io/common/storj" 20 "storj.io/uplink/private/metaclient" 21 "storj.io/uplink/private/storage/streams/pieceupload" 22 "storj.io/uplink/private/storage/streams/segmentupload" 23 "storj.io/uplink/private/storage/streams/splitter" 24 "storj.io/uplink/private/storage/streams/streamupload" 25 "storj.io/uplink/private/testuplink" 26 ) 27 28 // At a high level, uploads are composed of two pieces: a SegmentSource and an 29 // UploaderBackend. The SegmentSource is responsible for providing the UploaderBackend 30 // with encrypted segments to upload along with the metadata necessary to upload them, 31 // and the UploaderBackend is responsible for uploading the all of segments as fast and as 32 // reliably as possible. 33 // 34 // One main reason for this split is to create a "light upload" where a smaller client 35 // can be the SegmentSource, encrypting and splitting the data, and a third party server 36 // can be the UploaderBackend, performing the Reed-Solomon encoding and uploading the pieces 37 // to nodes and issuing the correct RPCs to the satellite. 38 // 39 // Concretely, the SegmentSource is implemented by the splitter package, and the 40 // UploaderBackend is implemented by the streamupload package. The Uploader in this package 41 // creates and combines those two to perform the uploads without the help of a third 42 // party server. 43 // 44 // The splitter package exports the Splitter type which implements SegmentSource and has 45 // these methods (only Next is part of the SegmentSource interface): 46 // 47 // * Write([]byte) (int, error): the standard io.Writer interface. 48 // * Finish(error): informs the splitter that no more writes are coming 49 // with a potential error if there was one. 50 // * Next(context.Context) (Segment, error): blocks until enough data has been 51 // written to know if the segment should 52 // be inline and then returns the next 53 // segment to upload. 54 // 55 // where the Segment type is an interface that mainly allows one to get a reader 56 // for the data with the `Reader() io.Reader` method and has many other methods 57 // for getting the metadata an UploaderBackend would need to upload the segment. 58 // This means the Segment is somewhat like a promise type, where not necessarily 59 // all of the data is available immediately, but it will be available eventually. 60 // 61 // The Next method on the SegmentSource is used by the UploaderBackend when it wants 62 // a new segment to upload, and the Write and Finish calls are used by the client 63 // providing the data to upload. The splitter.Splitter has the property that there 64 // is bounded write-ahead: Write calls will block until enough data has been read from 65 // the io.Reader returned by the Segment. This provides backpressure to the client 66 // avoiding the need for large buffers. 67 // 68 // The UploaderBackend ends up calling one of UploadObject or UploadPart 69 // in the streamupload package. Both of those end up dispatching to the 70 // same logic in uploadSegments which creates and uses many smaller helper 71 // types with small responsibilities. The main helper types are: 72 // 73 // * streambatcher.Batcher: Wraps the metaclient.Batch api to keep track of 74 // plain bytes uploaded and the stream id of the upload, 75 // because sometimes the stream id is not known until 76 // after a BeginObject call. 77 // * batchaggregator.Aggregator: Aggregates individual metaclient.BatchItems 78 // into larger batches, lazily flushing them 79 // to reduce round trips to the satellite. 80 // * segmenttracker.Tracker: Some data is only available after the final segment 81 // is uploaded, like the ETag. But, since segments can 82 // be uploaded in parallel and finish out of order, we 83 // cannot commit any segment until we are sure it is not 84 // the last segment. This type holds CommitSegment calls 85 // until it knows the ETag will be available and then 86 // flushes them when possible. 87 // 88 // The uploadSegments function simply calls Next on the SegmentSource, issuing 89 // any RPCs necessary to begin the segment and passes it to a provided SegmentUploader. 90 // multiple segments to be uploaded in parallel thanks to the SegmentUploader interface. 91 // It has a method to begin uploading a segment that blocks until enough resources are 92 // available, and returns a handle that can be Waited on that returns when the segment 93 // is finished uploading. The function thus calls Begin synchronously with the loop 94 // getting the next segment, and then calls Wait asynchronously in a goroutine. Thus, 95 // the function is limited on the input side by the Write calls to the source, and 96 // limited on the output side on the Begin call blocking for enough resources. 97 // Finally, when all of the segments are finished indicated by Next returning a nil 98 // segment, it issues some more necessary RPCs to finish off the upload, and returns the 99 // result. 100 // 101 // The SegmentUploader is responsible for uploading an individual segment and also has 102 // many smaller helper types: 103 // 104 // * pieceupload.LimitsExchanger: This allows the segment upload to request more 105 // nodes to upload to when some of the piece uploads 106 // fail, making segment uploads resilient. 107 // * pieceupload.Manager: Responsible for handing out pieces to upload and in charge 108 // of using the LimitsExchanger when necessary. It keeps track 109 // of which pieces have succeeded and which have failed for 110 // final inspection, ensuring enough pieces are successful for 111 // an upload and constructing a FinishSegment RPC telling the 112 // satellite which pieces were successful. 113 // * pieceupload.PiecePutter: This is the interface that actually performs an individual 114 // piece upload to a single node. It will grab a piece from 115 // the Manager and attempt to upload it, looping until it is 116 // either canceled or succeeds in uploading some piece to 117 // some node, both potentially different every iteration. 118 // * scheduler.Scheduler: This is how the SegmentUploader ensures that it operates 119 // within some amount of resources. At time of writing, it is 120 // a priority semaphore in the sense that each segment upload 121 // grabs a Handle that can be used to grab a Resource. The total 122 // number of Resources is limited, and when a Resource is put 123 // back, the earliest acquired Handle is given priority. This 124 // ensures that earlier started segments finish more quickly 125 // keeping overall resource usage low. 126 // 127 // The SegmentUploader simply acquires a Handle from the scheduler, and for each piece 128 // acquires a Resource from the Handle, launches a goroutine that attempts to upload 129 // a piece and returns the Resource when it is done, and returns a type that knows how 130 // to wait for all of those goroutines to finish and return the upload results. 131 132 // MetainfoUpload are the metainfo methods needed to upload a stream. 133 type MetainfoUpload interface { 134 metaclient.Batcher 135 RetryBeginSegmentPieces(ctx context.Context, params metaclient.RetryBeginSegmentPiecesParams) (metaclient.RetryBeginSegmentPiecesResponse, error) 136 io.Closer 137 } 138 139 // Uploader uploads object or part streams. 140 type Uploader struct { 141 metainfo MetainfoUpload 142 piecePutter pieceupload.PiecePutter 143 segmentSize int64 144 encStore *encryption.Store 145 encryptionParameters storj.EncryptionParameters 146 inlineThreshold int 147 longTailMargin int 148 149 // The backend is fixed to the real backend in production but is overridden 150 // for testing. 151 backend uploaderBackend 152 } 153 154 // NewUploader constructs a new stream putter. 155 func NewUploader(metainfo MetainfoUpload, piecePutter pieceupload.PiecePutter, segmentSize int64, encStore *encryption.Store, encryptionParameters storj.EncryptionParameters, inlineThreshold, longTailMargin int) (*Uploader, error) { 156 switch { 157 case segmentSize <= 0: 158 return nil, errs.New("segment size must be larger than 0") 159 case encryptionParameters.BlockSize <= 0: 160 return nil, errs.New("encryption block size must be larger than 0") 161 case inlineThreshold <= 0: 162 return nil, errs.New("inline threshold must be larger than 0") 163 } 164 return &Uploader{ 165 metainfo: metainfo, 166 piecePutter: piecePutter, 167 segmentSize: segmentSize, 168 encStore: encStore, 169 encryptionParameters: encryptionParameters, 170 inlineThreshold: inlineThreshold, 171 longTailMargin: longTailMargin, 172 backend: realUploaderBackend{}, 173 }, nil 174 } 175 176 // Close closes the underlying resources for the uploader. 177 func (u *Uploader) Close() error { 178 return u.metainfo.Close() 179 } 180 181 var uploadCounter int64 182 183 // UploadObject starts an upload of an object to the given location. The object 184 // contents can be written to the returned upload, which can then be committed. 185 func (u *Uploader) UploadObject(ctx context.Context, bucket, unencryptedKey string, metadata Metadata, expiration time.Time, sched segmentupload.Scheduler) (_ *Upload, err error) { 186 ctx = testuplink.WithLogWriterContext(ctx, "upload", fmt.Sprint(atomic.AddInt64(&uploadCounter, 1))) 187 testuplink.Log(ctx, "Starting upload") 188 defer testuplink.Log(ctx, "Done starting upload") 189 190 ctx, cancel := context.WithCancel(ctx) 191 defer func() { 192 if err != nil { 193 cancel() 194 } 195 }() 196 197 done := make(chan uploadResult, 1) 198 199 derivedKey, err := encryption.DeriveContentKey(bucket, paths.NewUnencrypted(unencryptedKey), u.encStore) 200 if err != nil { 201 return nil, errs.Wrap(err) 202 } 203 encPath, err := encryption.EncryptPathWithStoreCipher(bucket, paths.NewUnencrypted(unencryptedKey), u.encStore) 204 if err != nil { 205 return nil, errs.Wrap(err) 206 } 207 208 split, err := splitter.New(splitter.Options{ 209 Split: u.segmentSize, 210 Minimum: int64(u.inlineThreshold), 211 Params: u.encryptionParameters, 212 Key: derivedKey, 213 PartNumber: 0, 214 }) 215 if err != nil { 216 return nil, errs.Wrap(err) 217 } 218 go func() { 219 <-ctx.Done() 220 split.Finish(ctx.Err()) 221 }() 222 223 beginObject := &metaclient.BeginObjectParams{ 224 Bucket: []byte(bucket), 225 EncryptedObjectKey: []byte(encPath.Raw()), 226 ExpiresAt: expiration, 227 EncryptionParameters: u.encryptionParameters, 228 } 229 230 uploader := segmentUploader{metainfo: u.metainfo, piecePutter: u.piecePutter, sched: sched, longTailMargin: u.longTailMargin} 231 232 encMeta := u.newEncryptedMetadata(metadata, derivedKey) 233 234 go func() { 235 info, err := u.backend.UploadObject( 236 ctx, 237 split, 238 uploader, 239 u.metainfo, 240 beginObject, 241 encMeta, 242 ) 243 // On failure, we need to "finish" the splitter with an error so that 244 // outstanding writes to the splitter fail, otherwise the writes will 245 // block waiting for the upload to read the stream. 246 if err != nil { 247 split.Finish(err) 248 } 249 testuplink.Log(ctx, "Upload finished. err:", err) 250 done <- uploadResult{info: info, err: err} 251 }() 252 253 return &Upload{ 254 split: split, 255 done: done, 256 cancel: cancel, 257 }, nil 258 } 259 260 // UploadPart starts an upload of a part to the given location for the given 261 // multipart upload stream. The eTag is an optional channel is used to provide 262 // the eTag to be encrypted and included in the final segment of the part. The 263 // eTag should be sent on the channel only after the contents of the part have 264 // been fully written to the returned upload, but before calling Commit. 265 func (u *Uploader) UploadPart(ctx context.Context, bucket, unencryptedKey string, streamID storj.StreamID, partNumber int32, eTag <-chan []byte, sched segmentupload.Scheduler) (_ *Upload, err error) { 266 ctx = testuplink.WithLogWriterContext(ctx, 267 "upload", fmt.Sprint(atomic.AddInt64(&uploadCounter, 1)), 268 "part_number", fmt.Sprint(partNumber), 269 ) 270 testuplink.Log(ctx, "Starting upload") 271 defer testuplink.Log(ctx, "Done starting upload") 272 273 ctx, cancel := context.WithCancel(ctx) 274 defer func() { 275 if err != nil { 276 cancel() 277 } 278 }() 279 280 done := make(chan uploadResult, 1) 281 282 derivedKey, err := encryption.DeriveContentKey(bucket, paths.NewUnencrypted(unencryptedKey), u.encStore) 283 if err != nil { 284 return nil, errs.Wrap(err) 285 } 286 287 split, err := splitter.New(splitter.Options{ 288 Split: u.segmentSize, 289 Minimum: int64(u.inlineThreshold), 290 Params: u.encryptionParameters, 291 Key: derivedKey, 292 PartNumber: partNumber, 293 }) 294 if err != nil { 295 return nil, errs.Wrap(err) 296 } 297 go func() { 298 <-ctx.Done() 299 split.Finish(ctx.Err()) 300 }() 301 302 uploader := segmentUploader{metainfo: u.metainfo, piecePutter: u.piecePutter, sched: sched, longTailMargin: u.longTailMargin} 303 304 go func() { 305 info, err := u.backend.UploadPart( 306 ctx, 307 split, 308 uploader, 309 u.metainfo, 310 streamID, 311 eTag, 312 ) 313 // On failure, we need to "finish" the splitter with an error so that 314 // outstanding writes to the splitter fail, otherwise the writes will 315 // block waiting for the upload to read the stream. 316 if err != nil { 317 split.Finish(err) 318 } 319 testuplink.Log(ctx, "Upload finished. err:", err) 320 done <- uploadResult{info: info, err: err} 321 }() 322 323 return &Upload{ 324 split: split, 325 done: done, 326 cancel: cancel, 327 }, nil 328 } 329 330 func (u *Uploader) newEncryptedMetadata(metadata Metadata, derivedKey *storj.Key) streamupload.EncryptedMetadata { 331 return &encryptedMetadata{ 332 metadata: metadata, 333 segmentSize: u.segmentSize, 334 derivedKey: derivedKey, 335 cipherSuite: u.encryptionParameters.CipherSuite, 336 } 337 } 338 339 type segmentUploader struct { 340 metainfo MetainfoUpload 341 piecePutter pieceupload.PiecePutter 342 sched segmentupload.Scheduler 343 longTailMargin int 344 } 345 346 func (u segmentUploader) Begin(ctx context.Context, beginSegment *metaclient.BeginSegmentResponse, segment splitter.Segment) (streamupload.SegmentUpload, error) { 347 return segmentupload.Begin(ctx, beginSegment, segment, limitsExchanger{u.metainfo}, u.piecePutter, u.sched, u.longTailMargin) 348 } 349 350 type limitsExchanger struct { 351 metainfo MetainfoUpload 352 } 353 354 func (e limitsExchanger) ExchangeLimits(ctx context.Context, segmentID storj.SegmentID, pieceNumbers []int) (storj.SegmentID, []*pb.AddressedOrderLimit, error) { 355 resp, err := e.metainfo.RetryBeginSegmentPieces(ctx, metaclient.RetryBeginSegmentPiecesParams{ 356 SegmentID: segmentID, 357 RetryPieceNumbers: pieceNumbers, 358 }) 359 if err != nil { 360 return nil, nil, err 361 } 362 return resp.SegmentID, resp.Limits, nil 363 } 364 365 type encryptedMetadata struct { 366 metadata Metadata 367 segmentSize int64 368 derivedKey *storj.Key 369 cipherSuite storj.CipherSuite 370 } 371 372 func (e *encryptedMetadata) EncryptedMetadata(lastSegmentSize int64) (data []byte, encKey *storj.EncryptedPrivateKey, nonce *storj.Nonce, err error) { 373 metadataBytes, err := e.metadata.Metadata() 374 if err != nil { 375 return nil, nil, nil, err 376 } 377 378 streamInfo, err := pb.Marshal(&pb.StreamInfo{ 379 SegmentsSize: e.segmentSize, 380 LastSegmentSize: lastSegmentSize, 381 Metadata: metadataBytes, 382 }) 383 if err != nil { 384 return nil, nil, nil, err 385 } 386 387 var metadataKey storj.Key 388 if _, err := rand.Read(metadataKey[:]); err != nil { 389 return nil, nil, nil, err 390 } 391 392 var encryptedMetadataKeyNonce storj.Nonce 393 if _, err := rand.Read(encryptedMetadataKeyNonce[:]); err != nil { 394 return nil, nil, nil, err 395 } 396 397 // encrypt the metadata key with the derived key and the random encrypted key nonce 398 encryptedMetadataKey, err := encryption.EncryptKey(&metadataKey, e.cipherSuite, e.derivedKey, &encryptedMetadataKeyNonce) 399 if err != nil { 400 return nil, nil, nil, err 401 } 402 403 // encrypt the stream info with the metadata key and the zero nonce 404 encryptedStreamInfo, err := encryption.Encrypt(streamInfo, e.cipherSuite, &metadataKey, &storj.Nonce{}) 405 if err != nil { 406 return nil, nil, nil, err 407 } 408 409 streamMeta, err := pb.Marshal(&pb.StreamMeta{ 410 EncryptedStreamInfo: encryptedStreamInfo, 411 }) 412 if err != nil { 413 return nil, nil, nil, err 414 } 415 416 return streamMeta, &encryptedMetadataKey, &encryptedMetadataKeyNonce, nil 417 } 418 419 type uploaderBackend interface { 420 UploadObject(ctx context.Context, segmentSource streamupload.SegmentSource, segmentUploader streamupload.SegmentUploader, miBatcher metaclient.Batcher, beginObject *metaclient.BeginObjectParams, encMeta streamupload.EncryptedMetadata) (streamupload.Info, error) 421 UploadPart(ctx context.Context, segmentSource streamupload.SegmentSource, segmentUploader streamupload.SegmentUploader, miBatcher metaclient.Batcher, streamID storj.StreamID, eTagCh <-chan []byte) (streamupload.Info, error) 422 } 423 424 type realUploaderBackend struct{} 425 426 func (realUploaderBackend) UploadObject(ctx context.Context, segmentSource streamupload.SegmentSource, segmentUploader streamupload.SegmentUploader, miBatcher metaclient.Batcher, beginObject *metaclient.BeginObjectParams, encMeta streamupload.EncryptedMetadata) (streamupload.Info, error) { 427 return streamupload.UploadObject(ctx, segmentSource, segmentUploader, miBatcher, beginObject, encMeta) 428 } 429 430 func (realUploaderBackend) UploadPart(ctx context.Context, segmentSource streamupload.SegmentSource, segmentUploader streamupload.SegmentUploader, miBatcher metaclient.Batcher, streamID storj.StreamID, eTagCh <-chan []byte) (streamupload.Info, error) { 431 return streamupload.UploadPart(ctx, segmentSource, segmentUploader, miBatcher, streamID, eTagCh) 432 }