storj.io/uplink@v1.13.0/upload.go (about)

     1  // Copyright (C) 2020 Storj Labs, Inc.
     2  // See LICENSE for copying information.
     3  
     4  package uplink
     5  
     6  import (
     7  	"context"
     8  	"errors"
     9  	"io"
    10  	"runtime"
    11  	"sync"
    12  	"time"
    13  
    14  	"github.com/zeebo/errs"
    15  
    16  	"storj.io/common/leak"
    17  	"storj.io/common/pb"
    18  	"storj.io/eventkit"
    19  	"storj.io/uplink/private/eestream/scheduler"
    20  	"storj.io/uplink/private/storage/streams"
    21  	"storj.io/uplink/private/stream"
    22  )
    23  
    24  // ErrUploadDone is returned when either Abort or Commit has already been called.
    25  var ErrUploadDone = errors.New("upload done")
    26  
    27  // UploadOptions contains additional options for uploading.
    28  type UploadOptions struct {
    29  	// When Expires is zero, there is no expiration.
    30  	Expires time.Time
    31  }
    32  
    33  // UploadObject starts an upload to the specific key.
    34  //
    35  // It is not guaranteed that the uncommitted object is visible through ListUploads while uploading.
    36  func (project *Project) UploadObject(ctx context.Context, bucket, key string, options *UploadOptions) (_ *Upload, err error) {
    37  	upload := &Upload{
    38  		bucket: bucket,
    39  		stats:  newOperationStats(ctx, project.access.satelliteURL),
    40  	}
    41  	upload.task = mon.TaskNamed("Upload")(&ctx)
    42  	defer func() {
    43  		if err != nil {
    44  			upload.stats.flagFailure(err)
    45  			upload.emitEvent(false)
    46  		}
    47  	}()
    48  	defer upload.stats.trackWorking()()
    49  	defer mon.Task()(&ctx)(&err)
    50  
    51  	if bucket == "" {
    52  		return nil, errwrapf("%w (%q)", ErrBucketNameInvalid, bucket)
    53  	}
    54  	if key == "" {
    55  		return nil, errwrapf("%w (%q)", ErrObjectKeyInvalid, key)
    56  	}
    57  
    58  	if options == nil {
    59  		options = &UploadOptions{}
    60  	}
    61  
    62  	// N.B. we always call dbCleanup which closes the db because
    63  	// closing it earlier has the benefit of returning a connection to
    64  	// the pool, so we try to do that as early as possible.
    65  
    66  	db, err := project.dialMetainfoDB(ctx)
    67  	if err != nil {
    68  		return nil, convertKnownErrors(err, bucket, key)
    69  	}
    70  	defer func() { err = errs.Combine(err, db.Close()) }()
    71  
    72  	obj, err := db.CreateObject(ctx, bucket, key, nil)
    73  	if err != nil {
    74  		return nil, convertKnownErrors(err, bucket, key)
    75  	}
    76  
    77  	info := obj.Info()
    78  
    79  	ctx, cancel := context.WithCancel(ctx)
    80  
    81  	upload.cancel = cancel
    82  	upload.object = convertObject(&info)
    83  
    84  	meta := dynamicMetadata{upload.object}
    85  	mutableStream, err := obj.CreateDynamicStream(ctx, meta, options.Expires)
    86  	if err != nil {
    87  		return nil, convertKnownErrors(err, bucket, key)
    88  	}
    89  
    90  	// Return the connection to the pool as soon as we can.
    91  	if err := db.Close(); err != nil {
    92  		return nil, convertKnownErrors(err, bucket, key)
    93  	}
    94  
    95  	// TODO: don't calculate this twice.
    96  	if encPath, err := encryptPath(project, bucket, key); err == nil {
    97  		upload.stats.encPath = encPath
    98  	}
    99  
   100  	streams, err := project.getStreamsStore(ctx)
   101  	if err != nil {
   102  		return nil, convertKnownErrors(err, bucket, key)
   103  	}
   104  	upload.streams = streams
   105  
   106  	if project.concurrentSegmentUploadConfig == nil {
   107  		upload.upload = stream.NewUpload(ctx, mutableStream, streams)
   108  	} else {
   109  		sched := scheduler.New(project.concurrentSegmentUploadConfig.SchedulerOptions)
   110  		u, err := streams.UploadObject(ctx, mutableStream.BucketName(), mutableStream.Path(), mutableStream, mutableStream.Expires(), sched)
   111  		if err != nil {
   112  			return nil, convertKnownErrors(err, bucket, key)
   113  		}
   114  		upload.upload = u
   115  	}
   116  
   117  	upload.tracker = project.tracker.Child("upload", 1)
   118  	return upload, nil
   119  }
   120  
   121  type dynamicMetadata struct{ *Object }
   122  
   123  func (dyn dynamicMetadata) Metadata() ([]byte, error) {
   124  	return pb.Marshal(&pb.SerializableMeta{
   125  		UserDefined: dyn.Object.Custom.Clone(),
   126  	})
   127  }
   128  
   129  type streamUpload interface {
   130  	io.Writer
   131  	Commit() error
   132  	Abort() error
   133  	Meta() *streams.Meta
   134  }
   135  
   136  // Upload is an upload to Storj Network.
   137  type Upload struct {
   138  	mu      sync.Mutex
   139  	closed  bool
   140  	aborted bool
   141  	cancel  context.CancelFunc
   142  	upload  streamUpload
   143  	bucket  string
   144  	object  *Object
   145  	streams *streams.Store
   146  
   147  	stats operationStats
   148  	task  func(*error)
   149  
   150  	tracker leak.Ref
   151  }
   152  
   153  // Info returns the last information about the uploaded object.
   154  func (upload *Upload) Info() *Object {
   155  	meta := upload.upload.Meta()
   156  	if meta != nil {
   157  		upload.object.System.ContentLength = meta.Size
   158  		upload.object.System.Created = meta.Modified
   159  		upload.object.version = meta.Version
   160  	}
   161  	return upload.object
   162  }
   163  
   164  // Write uploads len(p) bytes from p to the object's data stream.
   165  // It returns the number of bytes written from p (0 <= n <= len(p))
   166  // and any error encountered that caused the write to stop early.
   167  func (upload *Upload) Write(p []byte) (n int, err error) {
   168  	track := upload.stats.trackWorking()
   169  	n, err = upload.upload.Write(p)
   170  	upload.mu.Lock()
   171  	upload.stats.bytes += int64(n)
   172  	upload.stats.flagFailure(err)
   173  	track()
   174  	upload.mu.Unlock()
   175  	return n, convertKnownErrors(err, upload.bucket, upload.object.Key)
   176  }
   177  
   178  // Commit commits data to the store.
   179  //
   180  // Returns ErrUploadDone when either Abort or Commit has already been called.
   181  func (upload *Upload) Commit() error {
   182  	track := upload.stats.trackWorking()
   183  	upload.mu.Lock()
   184  	defer upload.mu.Unlock()
   185  
   186  	if upload.aborted {
   187  		return errwrapf("%w: already aborted", ErrUploadDone)
   188  	}
   189  
   190  	if upload.closed {
   191  		return errwrapf("%w: already committed", ErrUploadDone)
   192  	}
   193  
   194  	upload.closed = true
   195  
   196  	err := errs.Combine(
   197  		upload.upload.Commit(),
   198  		upload.streams.Close(),
   199  		upload.tracker.Close(),
   200  	)
   201  	upload.stats.flagFailure(err)
   202  	track()
   203  	upload.emitEvent(false)
   204  
   205  	return convertKnownErrors(err, upload.bucket, upload.object.Key)
   206  }
   207  
   208  // Abort aborts the upload.
   209  //
   210  // Returns ErrUploadDone when either Abort or Commit has already been called.
   211  func (upload *Upload) Abort() error {
   212  	track := upload.stats.trackWorking()
   213  	upload.mu.Lock()
   214  	defer upload.mu.Unlock()
   215  
   216  	if upload.closed {
   217  		return errwrapf("%w: already committed", ErrUploadDone)
   218  	}
   219  
   220  	if upload.aborted {
   221  		return errwrapf("%w: already aborted", ErrUploadDone)
   222  	}
   223  
   224  	upload.aborted = true
   225  	upload.cancel()
   226  
   227  	err := errs.Combine(
   228  		upload.upload.Abort(),
   229  		upload.streams.Close(),
   230  		upload.tracker.Close(),
   231  	)
   232  
   233  	track()
   234  	upload.stats.flagFailure(err)
   235  	upload.emitEvent(true)
   236  
   237  	return convertKnownErrors(err, upload.bucket, upload.object.Key)
   238  }
   239  
   240  func (upload *Upload) emitEvent(aborted bool) {
   241  	message, err := upload.stats.err()
   242  	upload.task(&err)
   243  
   244  	expires := false
   245  	if upload.upload != nil {
   246  		meta := upload.upload.Meta()
   247  		if meta != nil && !meta.Expiration.IsZero() {
   248  			expires = true
   249  		}
   250  	}
   251  
   252  	evs.Event("upload",
   253  		eventkit.Int64("bytes", upload.stats.bytes),
   254  		eventkit.Duration("user-elapsed", time.Since(upload.stats.start)),
   255  		eventkit.Duration("working-elapsed", upload.stats.working),
   256  		eventkit.Bool("success", err == nil),
   257  		eventkit.String("error", message),
   258  		eventkit.Bool("aborted", aborted),
   259  		eventkit.String("arch", runtime.GOARCH),
   260  		eventkit.String("os", runtime.GOOS),
   261  		eventkit.Int64("cpus", int64(runtime.NumCPU())),
   262  		eventkit.Bool("expires", expires),
   263  		eventkit.Int64("quic-rollout", int64(upload.stats.quicRollout)),
   264  		eventkit.String("satellite", upload.stats.satellite),
   265  		eventkit.Bytes("path-checksum", pathChecksum(upload.stats.encPath)),
   266  		eventkit.Int64("noise-version", noiseVersion),
   267  		// upload.upload.Meta().Expiration
   268  		// segment count
   269  		// ram available
   270  	)
   271  }
   272  
   273  // SetCustomMetadata updates custom metadata to be included with the object.
   274  // If it is nil, it won't be modified.
   275  func (upload *Upload) SetCustomMetadata(ctx context.Context, custom CustomMetadata) error {
   276  	upload.mu.Lock()
   277  	defer upload.mu.Unlock()
   278  
   279  	if upload.aborted {
   280  		return errwrapf("%w: upload aborted", ErrUploadDone)
   281  	}
   282  	if upload.closed {
   283  		return errwrapf("%w: already committed", ErrUploadDone)
   284  	}
   285  	if upload.upload.Meta() != nil {
   286  		return errwrapf("%w: already committed", ErrUploadDone)
   287  	}
   288  
   289  	if custom != nil {
   290  		if err := custom.Verify(); err != nil {
   291  			return packageError.Wrap(err)
   292  		}
   293  		upload.object.Custom = custom.Clone()
   294  	}
   295  
   296  	return nil
   297  }