storj.io/uplink@v1.13.0/private/storage/streams/uploader.go (about)

     1  // Copyright (C) 2023 Storj Labs, Inc.
     2  // See LICENSE for copying information.
     3  
     4  package streams
     5  
     6  import (
     7  	"context"
     8  	"crypto/rand"
     9  	"fmt"
    10  	"io"
    11  	"sync/atomic"
    12  	"time"
    13  
    14  	"github.com/zeebo/errs"
    15  
    16  	"storj.io/common/encryption"
    17  	"storj.io/common/paths"
    18  	"storj.io/common/pb"
    19  	"storj.io/common/storj"
    20  	"storj.io/uplink/private/metaclient"
    21  	"storj.io/uplink/private/storage/streams/pieceupload"
    22  	"storj.io/uplink/private/storage/streams/segmentupload"
    23  	"storj.io/uplink/private/storage/streams/splitter"
    24  	"storj.io/uplink/private/storage/streams/streamupload"
    25  	"storj.io/uplink/private/testuplink"
    26  )
    27  
    28  // At a high level, uploads are composed of two pieces: a SegmentSource and an
    29  // UploaderBackend. The SegmentSource is responsible for providing the UploaderBackend
    30  // with encrypted segments to upload along with the metadata necessary to upload them,
    31  // and the UploaderBackend is responsible for uploading the all of segments as fast and as
    32  // reliably as possible.
    33  //
    34  // One main reason for this split is to create a "light upload" where a smaller client
    35  // can be the SegmentSource, encrypting and splitting the data, and a third party server
    36  // can be the UploaderBackend, performing the Reed-Solomon encoding and uploading the pieces
    37  // to nodes and issuing the correct RPCs to the satellite.
    38  //
    39  // Concretely, the SegmentSource is implemented by the splitter package, and the
    40  // UploaderBackend is implemented by the streamupload package. The Uploader in this package
    41  // creates and combines those two to perform the uploads without the help of a third
    42  // party server.
    43  //
    44  // The splitter package exports the Splitter type which implements SegmentSource and has
    45  // these methods (only Next is part of the SegmentSource interface):
    46  //
    47  //  * Write([]byte) (int, error): the standard io.Writer interface.
    48  //  * Finish(error): informs the splitter that no more writes are coming
    49  //                   with a potential error if there was one.
    50  //  * Next(context.Context) (Segment, error): blocks until enough data has been
    51  //                                            written to know if the segment should
    52  //                                            be inline and then returns the next
    53  //                                            segment to upload.
    54  //
    55  // where the Segment type is an interface that mainly allows one to get a reader
    56  // for the data with the `Reader() io.Reader` method and has many other methods
    57  // for getting the metadata an UploaderBackend would need to upload the segment.
    58  // This means the Segment is somewhat like a promise type, where not necessarily
    59  // all of the data is available immediately, but it will be available eventually.
    60  //
    61  // The Next method on the SegmentSource is used by the UploaderBackend when it wants
    62  // a new segment to upload, and the Write and Finish calls are used by the client
    63  // providing the data to upload. The splitter.Splitter has the property that there
    64  // is bounded write-ahead: Write calls will block until enough data has been read from
    65  // the io.Reader returned by the Segment. This provides backpressure to the client
    66  //  avoiding the need for large buffers.
    67  //
    68  // The UploaderBackend ends up calling one of UploadObject or UploadPart
    69  // in the streamupload package. Both of those end up dispatching to the
    70  // same logic in uploadSegments which creates and uses many smaller helper
    71  // types with small responsibilities. The main helper types are:
    72  //
    73  //  * streambatcher.Batcher: Wraps the metaclient.Batch api to keep track of
    74  //                           plain bytes uploaded and the stream id of the upload,
    75  //                           because sometimes the stream id is not known until
    76  //                           after a BeginObject call.
    77  //  * batchaggregator.Aggregator: Aggregates individual metaclient.BatchItems
    78  //                                into larger batches, lazily flushing them
    79  //                                to reduce round trips to the satellite.
    80  //  * segmenttracker.Tracker: Some data is only available after the final segment
    81  //                            is uploaded, like the ETag. But, since segments can
    82  //                            be uploaded in parallel and finish out of order, we
    83  //                            cannot commit any segment until we are sure it is not
    84  //                            the last segment. This type holds CommitSegment calls
    85  //                            until it knows the ETag will be available and then
    86  //                            flushes them when possible.
    87  //
    88  // The uploadSegments function simply calls Next on the SegmentSource, issuing
    89  // any RPCs necessary to begin the segment and passes it to a provided SegmentUploader.
    90  // multiple segments to be uploaded in parallel thanks to the SegmentUploader interface.
    91  // It has a method to begin uploading a segment that blocks until enough resources are
    92  // available, and returns a handle that can be Waited on that returns when the segment
    93  // is finished uploading. The function thus calls Begin synchronously with the loop
    94  // getting the next segment, and then calls Wait asynchronously in a goroutine. Thus,
    95  // the function is limited on the input side by the Write calls to the source, and
    96  // limited on the output side on the Begin call blocking for enough resources.
    97  // Finally, when all of the segments are finished indicated by Next returning a nil
    98  // segment, it issues some more necessary RPCs to finish off the upload, and returns the
    99  // result.
   100  //
   101  // The SegmentUploader is responsible for uploading an individual segment and also has
   102  // many smaller helper types:
   103  //
   104  //  * pieceupload.LimitsExchanger: This allows the segment upload to request more
   105  //                                 nodes to upload to when some of the piece uploads
   106  //                                 fail, making segment uploads resilient.
   107  //  * pieceupload.Manager: Responsible for handing out pieces to upload and in charge
   108  //                         of using the LimitsExchanger when necessary. It keeps track
   109  //                         of which pieces have succeeded and which have failed for
   110  //                         final inspection, ensuring enough pieces are successful for
   111  //                         an upload and constructing a FinishSegment RPC telling the
   112  //                         satellite which pieces were successful.
   113  //  * pieceupload.PiecePutter: This is the interface that actually performs an individual
   114  //                             piece upload to a single node. It will grab a piece from
   115  //                             the Manager and attempt to upload it, looping until it is
   116  //                             either canceled or succeeds in uploading some piece to
   117  //                             some node, both potentially different every iteration.
   118  //  * scheduler.Scheduler: This is how the SegmentUploader ensures that it operates
   119  //                         within some amount of resources. At time of writing, it is
   120  //                         a priority semaphore in the sense that each segment upload
   121  //                         grabs a Handle that can be used to grab a Resource. The total
   122  //                         number of Resources is limited, and when a Resource is put
   123  //                         back, the earliest acquired Handle is given priority. This
   124  //                         ensures that earlier started segments finish more quickly
   125  //                         keeping overall resource usage low.
   126  //
   127  // The SegmentUploader simply acquires a Handle from the scheduler, and for each piece
   128  // acquires a Resource from the Handle, launches a goroutine that attempts to upload
   129  // a piece and returns the Resource when it is done, and returns a type that knows how
   130  // to wait for all of those goroutines to finish and return the upload results.
   131  
   132  // MetainfoUpload are the metainfo methods needed to upload a stream.
   133  type MetainfoUpload interface {
   134  	metaclient.Batcher
   135  	RetryBeginSegmentPieces(ctx context.Context, params metaclient.RetryBeginSegmentPiecesParams) (metaclient.RetryBeginSegmentPiecesResponse, error)
   136  	io.Closer
   137  }
   138  
   139  // Uploader uploads object or part streams.
   140  type Uploader struct {
   141  	metainfo             MetainfoUpload
   142  	piecePutter          pieceupload.PiecePutter
   143  	segmentSize          int64
   144  	encStore             *encryption.Store
   145  	encryptionParameters storj.EncryptionParameters
   146  	inlineThreshold      int
   147  	longTailMargin       int
   148  
   149  	// The backend is fixed to the real backend in production but is overridden
   150  	// for testing.
   151  	backend uploaderBackend
   152  }
   153  
   154  // NewUploader constructs a new stream putter.
   155  func NewUploader(metainfo MetainfoUpload, piecePutter pieceupload.PiecePutter, segmentSize int64, encStore *encryption.Store, encryptionParameters storj.EncryptionParameters, inlineThreshold, longTailMargin int) (*Uploader, error) {
   156  	switch {
   157  	case segmentSize <= 0:
   158  		return nil, errs.New("segment size must be larger than 0")
   159  	case encryptionParameters.BlockSize <= 0:
   160  		return nil, errs.New("encryption block size must be larger than 0")
   161  	case inlineThreshold <= 0:
   162  		return nil, errs.New("inline threshold must be larger than 0")
   163  	}
   164  	return &Uploader{
   165  		metainfo:             metainfo,
   166  		piecePutter:          piecePutter,
   167  		segmentSize:          segmentSize,
   168  		encStore:             encStore,
   169  		encryptionParameters: encryptionParameters,
   170  		inlineThreshold:      inlineThreshold,
   171  		longTailMargin:       longTailMargin,
   172  		backend:              realUploaderBackend{},
   173  	}, nil
   174  }
   175  
   176  // Close closes the underlying resources for the uploader.
   177  func (u *Uploader) Close() error {
   178  	return u.metainfo.Close()
   179  }
   180  
   181  var uploadCounter int64
   182  
   183  // UploadObject starts an upload of an object to the given location. The object
   184  // contents can be written to the returned upload, which can then be committed.
   185  func (u *Uploader) UploadObject(ctx context.Context, bucket, unencryptedKey string, metadata Metadata, expiration time.Time, sched segmentupload.Scheduler) (_ *Upload, err error) {
   186  	ctx = testuplink.WithLogWriterContext(ctx, "upload", fmt.Sprint(atomic.AddInt64(&uploadCounter, 1)))
   187  	testuplink.Log(ctx, "Starting upload")
   188  	defer testuplink.Log(ctx, "Done starting upload")
   189  
   190  	ctx, cancel := context.WithCancel(ctx)
   191  	defer func() {
   192  		if err != nil {
   193  			cancel()
   194  		}
   195  	}()
   196  
   197  	done := make(chan uploadResult, 1)
   198  
   199  	derivedKey, err := encryption.DeriveContentKey(bucket, paths.NewUnencrypted(unencryptedKey), u.encStore)
   200  	if err != nil {
   201  		return nil, errs.Wrap(err)
   202  	}
   203  	encPath, err := encryption.EncryptPathWithStoreCipher(bucket, paths.NewUnencrypted(unencryptedKey), u.encStore)
   204  	if err != nil {
   205  		return nil, errs.Wrap(err)
   206  	}
   207  
   208  	split, err := splitter.New(splitter.Options{
   209  		Split:      u.segmentSize,
   210  		Minimum:    int64(u.inlineThreshold),
   211  		Params:     u.encryptionParameters,
   212  		Key:        derivedKey,
   213  		PartNumber: 0,
   214  	})
   215  	if err != nil {
   216  		return nil, errs.Wrap(err)
   217  	}
   218  	go func() {
   219  		<-ctx.Done()
   220  		split.Finish(ctx.Err())
   221  	}()
   222  
   223  	beginObject := &metaclient.BeginObjectParams{
   224  		Bucket:               []byte(bucket),
   225  		EncryptedObjectKey:   []byte(encPath.Raw()),
   226  		ExpiresAt:            expiration,
   227  		EncryptionParameters: u.encryptionParameters,
   228  	}
   229  
   230  	uploader := segmentUploader{metainfo: u.metainfo, piecePutter: u.piecePutter, sched: sched, longTailMargin: u.longTailMargin}
   231  
   232  	encMeta := u.newEncryptedMetadata(metadata, derivedKey)
   233  
   234  	go func() {
   235  		info, err := u.backend.UploadObject(
   236  			ctx,
   237  			split,
   238  			uploader,
   239  			u.metainfo,
   240  			beginObject,
   241  			encMeta,
   242  		)
   243  		// On failure, we need to "finish" the splitter with an error so that
   244  		// outstanding writes to the splitter fail, otherwise the writes will
   245  		// block waiting for the upload to read the stream.
   246  		if err != nil {
   247  			split.Finish(err)
   248  		}
   249  		testuplink.Log(ctx, "Upload finished. err:", err)
   250  		done <- uploadResult{info: info, err: err}
   251  	}()
   252  
   253  	return &Upload{
   254  		split:  split,
   255  		done:   done,
   256  		cancel: cancel,
   257  	}, nil
   258  }
   259  
   260  // UploadPart starts an upload of a part to the given location for the given
   261  // multipart upload stream. The eTag is an optional channel is used to provide
   262  // the eTag to be encrypted and included in the final segment of the part. The
   263  // eTag should be  sent on the channel only after the contents of the part have
   264  // been fully written to the returned upload, but before calling Commit.
   265  func (u *Uploader) UploadPart(ctx context.Context, bucket, unencryptedKey string, streamID storj.StreamID, partNumber int32, eTag <-chan []byte, sched segmentupload.Scheduler) (_ *Upload, err error) {
   266  	ctx = testuplink.WithLogWriterContext(ctx,
   267  		"upload", fmt.Sprint(atomic.AddInt64(&uploadCounter, 1)),
   268  		"part_number", fmt.Sprint(partNumber),
   269  	)
   270  	testuplink.Log(ctx, "Starting upload")
   271  	defer testuplink.Log(ctx, "Done starting upload")
   272  
   273  	ctx, cancel := context.WithCancel(ctx)
   274  	defer func() {
   275  		if err != nil {
   276  			cancel()
   277  		}
   278  	}()
   279  
   280  	done := make(chan uploadResult, 1)
   281  
   282  	derivedKey, err := encryption.DeriveContentKey(bucket, paths.NewUnencrypted(unencryptedKey), u.encStore)
   283  	if err != nil {
   284  		return nil, errs.Wrap(err)
   285  	}
   286  
   287  	split, err := splitter.New(splitter.Options{
   288  		Split:      u.segmentSize,
   289  		Minimum:    int64(u.inlineThreshold),
   290  		Params:     u.encryptionParameters,
   291  		Key:        derivedKey,
   292  		PartNumber: partNumber,
   293  	})
   294  	if err != nil {
   295  		return nil, errs.Wrap(err)
   296  	}
   297  	go func() {
   298  		<-ctx.Done()
   299  		split.Finish(ctx.Err())
   300  	}()
   301  
   302  	uploader := segmentUploader{metainfo: u.metainfo, piecePutter: u.piecePutter, sched: sched, longTailMargin: u.longTailMargin}
   303  
   304  	go func() {
   305  		info, err := u.backend.UploadPart(
   306  			ctx,
   307  			split,
   308  			uploader,
   309  			u.metainfo,
   310  			streamID,
   311  			eTag,
   312  		)
   313  		// On failure, we need to "finish" the splitter with an error so that
   314  		// outstanding writes to the splitter fail, otherwise the writes will
   315  		// block waiting for the upload to read the stream.
   316  		if err != nil {
   317  			split.Finish(err)
   318  		}
   319  		testuplink.Log(ctx, "Upload finished. err:", err)
   320  		done <- uploadResult{info: info, err: err}
   321  	}()
   322  
   323  	return &Upload{
   324  		split:  split,
   325  		done:   done,
   326  		cancel: cancel,
   327  	}, nil
   328  }
   329  
   330  func (u *Uploader) newEncryptedMetadata(metadata Metadata, derivedKey *storj.Key) streamupload.EncryptedMetadata {
   331  	return &encryptedMetadata{
   332  		metadata:    metadata,
   333  		segmentSize: u.segmentSize,
   334  		derivedKey:  derivedKey,
   335  		cipherSuite: u.encryptionParameters.CipherSuite,
   336  	}
   337  }
   338  
   339  type segmentUploader struct {
   340  	metainfo       MetainfoUpload
   341  	piecePutter    pieceupload.PiecePutter
   342  	sched          segmentupload.Scheduler
   343  	longTailMargin int
   344  }
   345  
   346  func (u segmentUploader) Begin(ctx context.Context, beginSegment *metaclient.BeginSegmentResponse, segment splitter.Segment) (streamupload.SegmentUpload, error) {
   347  	return segmentupload.Begin(ctx, beginSegment, segment, limitsExchanger{u.metainfo}, u.piecePutter, u.sched, u.longTailMargin)
   348  }
   349  
   350  type limitsExchanger struct {
   351  	metainfo MetainfoUpload
   352  }
   353  
   354  func (e limitsExchanger) ExchangeLimits(ctx context.Context, segmentID storj.SegmentID, pieceNumbers []int) (storj.SegmentID, []*pb.AddressedOrderLimit, error) {
   355  	resp, err := e.metainfo.RetryBeginSegmentPieces(ctx, metaclient.RetryBeginSegmentPiecesParams{
   356  		SegmentID:         segmentID,
   357  		RetryPieceNumbers: pieceNumbers,
   358  	})
   359  	if err != nil {
   360  		return nil, nil, err
   361  	}
   362  	return resp.SegmentID, resp.Limits, nil
   363  }
   364  
   365  type encryptedMetadata struct {
   366  	metadata    Metadata
   367  	segmentSize int64
   368  	derivedKey  *storj.Key
   369  	cipherSuite storj.CipherSuite
   370  }
   371  
   372  func (e *encryptedMetadata) EncryptedMetadata(lastSegmentSize int64) (data []byte, encKey *storj.EncryptedPrivateKey, nonce *storj.Nonce, err error) {
   373  	metadataBytes, err := e.metadata.Metadata()
   374  	if err != nil {
   375  		return nil, nil, nil, err
   376  	}
   377  
   378  	streamInfo, err := pb.Marshal(&pb.StreamInfo{
   379  		SegmentsSize:    e.segmentSize,
   380  		LastSegmentSize: lastSegmentSize,
   381  		Metadata:        metadataBytes,
   382  	})
   383  	if err != nil {
   384  		return nil, nil, nil, err
   385  	}
   386  
   387  	var metadataKey storj.Key
   388  	if _, err := rand.Read(metadataKey[:]); err != nil {
   389  		return nil, nil, nil, err
   390  	}
   391  
   392  	var encryptedMetadataKeyNonce storj.Nonce
   393  	if _, err := rand.Read(encryptedMetadataKeyNonce[:]); err != nil {
   394  		return nil, nil, nil, err
   395  	}
   396  
   397  	// encrypt the metadata key with the derived key and the random encrypted key nonce
   398  	encryptedMetadataKey, err := encryption.EncryptKey(&metadataKey, e.cipherSuite, e.derivedKey, &encryptedMetadataKeyNonce)
   399  	if err != nil {
   400  		return nil, nil, nil, err
   401  	}
   402  
   403  	// encrypt the stream info with the metadata key and the zero nonce
   404  	encryptedStreamInfo, err := encryption.Encrypt(streamInfo, e.cipherSuite, &metadataKey, &storj.Nonce{})
   405  	if err != nil {
   406  		return nil, nil, nil, err
   407  	}
   408  
   409  	streamMeta, err := pb.Marshal(&pb.StreamMeta{
   410  		EncryptedStreamInfo: encryptedStreamInfo,
   411  	})
   412  	if err != nil {
   413  		return nil, nil, nil, err
   414  	}
   415  
   416  	return streamMeta, &encryptedMetadataKey, &encryptedMetadataKeyNonce, nil
   417  }
   418  
   419  type uploaderBackend interface {
   420  	UploadObject(ctx context.Context, segmentSource streamupload.SegmentSource, segmentUploader streamupload.SegmentUploader, miBatcher metaclient.Batcher, beginObject *metaclient.BeginObjectParams, encMeta streamupload.EncryptedMetadata) (streamupload.Info, error)
   421  	UploadPart(ctx context.Context, segmentSource streamupload.SegmentSource, segmentUploader streamupload.SegmentUploader, miBatcher metaclient.Batcher, streamID storj.StreamID, eTagCh <-chan []byte) (streamupload.Info, error)
   422  }
   423  
   424  type realUploaderBackend struct{}
   425  
   426  func (realUploaderBackend) UploadObject(ctx context.Context, segmentSource streamupload.SegmentSource, segmentUploader streamupload.SegmentUploader, miBatcher metaclient.Batcher, beginObject *metaclient.BeginObjectParams, encMeta streamupload.EncryptedMetadata) (streamupload.Info, error) {
   427  	return streamupload.UploadObject(ctx, segmentSource, segmentUploader, miBatcher, beginObject, encMeta)
   428  }
   429  
   430  func (realUploaderBackend) UploadPart(ctx context.Context, segmentSource streamupload.SegmentSource, segmentUploader streamupload.SegmentUploader, miBatcher metaclient.Batcher, streamID storj.StreamID, eTagCh <-chan []byte) (streamupload.Info, error) {
   431  	return streamupload.UploadPart(ctx, segmentSource, segmentUploader, miBatcher, streamID, eTagCh)
   432  }