storj.io/uplink@v1.13.0/upload.go (about) 1 // Copyright (C) 2020 Storj Labs, Inc. 2 // See LICENSE for copying information. 3 4 package uplink 5 6 import ( 7 "context" 8 "errors" 9 "io" 10 "runtime" 11 "sync" 12 "time" 13 14 "github.com/zeebo/errs" 15 16 "storj.io/common/leak" 17 "storj.io/common/pb" 18 "storj.io/eventkit" 19 "storj.io/uplink/private/eestream/scheduler" 20 "storj.io/uplink/private/storage/streams" 21 "storj.io/uplink/private/stream" 22 ) 23 24 // ErrUploadDone is returned when either Abort or Commit has already been called. 25 var ErrUploadDone = errors.New("upload done") 26 27 // UploadOptions contains additional options for uploading. 28 type UploadOptions struct { 29 // When Expires is zero, there is no expiration. 30 Expires time.Time 31 } 32 33 // UploadObject starts an upload to the specific key. 34 // 35 // It is not guaranteed that the uncommitted object is visible through ListUploads while uploading. 36 func (project *Project) UploadObject(ctx context.Context, bucket, key string, options *UploadOptions) (_ *Upload, err error) { 37 upload := &Upload{ 38 bucket: bucket, 39 stats: newOperationStats(ctx, project.access.satelliteURL), 40 } 41 upload.task = mon.TaskNamed("Upload")(&ctx) 42 defer func() { 43 if err != nil { 44 upload.stats.flagFailure(err) 45 upload.emitEvent(false) 46 } 47 }() 48 defer upload.stats.trackWorking()() 49 defer mon.Task()(&ctx)(&err) 50 51 if bucket == "" { 52 return nil, errwrapf("%w (%q)", ErrBucketNameInvalid, bucket) 53 } 54 if key == "" { 55 return nil, errwrapf("%w (%q)", ErrObjectKeyInvalid, key) 56 } 57 58 if options == nil { 59 options = &UploadOptions{} 60 } 61 62 // N.B. we always call dbCleanup which closes the db because 63 // closing it earlier has the benefit of returning a connection to 64 // the pool, so we try to do that as early as possible. 65 66 db, err := project.dialMetainfoDB(ctx) 67 if err != nil { 68 return nil, convertKnownErrors(err, bucket, key) 69 } 70 defer func() { err = errs.Combine(err, db.Close()) }() 71 72 obj, err := db.CreateObject(ctx, bucket, key, nil) 73 if err != nil { 74 return nil, convertKnownErrors(err, bucket, key) 75 } 76 77 info := obj.Info() 78 79 ctx, cancel := context.WithCancel(ctx) 80 81 upload.cancel = cancel 82 upload.object = convertObject(&info) 83 84 meta := dynamicMetadata{upload.object} 85 mutableStream, err := obj.CreateDynamicStream(ctx, meta, options.Expires) 86 if err != nil { 87 return nil, convertKnownErrors(err, bucket, key) 88 } 89 90 // Return the connection to the pool as soon as we can. 91 if err := db.Close(); err != nil { 92 return nil, convertKnownErrors(err, bucket, key) 93 } 94 95 // TODO: don't calculate this twice. 96 if encPath, err := encryptPath(project, bucket, key); err == nil { 97 upload.stats.encPath = encPath 98 } 99 100 streams, err := project.getStreamsStore(ctx) 101 if err != nil { 102 return nil, convertKnownErrors(err, bucket, key) 103 } 104 upload.streams = streams 105 106 if project.concurrentSegmentUploadConfig == nil { 107 upload.upload = stream.NewUpload(ctx, mutableStream, streams) 108 } else { 109 sched := scheduler.New(project.concurrentSegmentUploadConfig.SchedulerOptions) 110 u, err := streams.UploadObject(ctx, mutableStream.BucketName(), mutableStream.Path(), mutableStream, mutableStream.Expires(), sched) 111 if err != nil { 112 return nil, convertKnownErrors(err, bucket, key) 113 } 114 upload.upload = u 115 } 116 117 upload.tracker = project.tracker.Child("upload", 1) 118 return upload, nil 119 } 120 121 type dynamicMetadata struct{ *Object } 122 123 func (dyn dynamicMetadata) Metadata() ([]byte, error) { 124 return pb.Marshal(&pb.SerializableMeta{ 125 UserDefined: dyn.Object.Custom.Clone(), 126 }) 127 } 128 129 type streamUpload interface { 130 io.Writer 131 Commit() error 132 Abort() error 133 Meta() *streams.Meta 134 } 135 136 // Upload is an upload to Storj Network. 137 type Upload struct { 138 mu sync.Mutex 139 closed bool 140 aborted bool 141 cancel context.CancelFunc 142 upload streamUpload 143 bucket string 144 object *Object 145 streams *streams.Store 146 147 stats operationStats 148 task func(*error) 149 150 tracker leak.Ref 151 } 152 153 // Info returns the last information about the uploaded object. 154 func (upload *Upload) Info() *Object { 155 meta := upload.upload.Meta() 156 if meta != nil { 157 upload.object.System.ContentLength = meta.Size 158 upload.object.System.Created = meta.Modified 159 upload.object.version = meta.Version 160 } 161 return upload.object 162 } 163 164 // Write uploads len(p) bytes from p to the object's data stream. 165 // It returns the number of bytes written from p (0 <= n <= len(p)) 166 // and any error encountered that caused the write to stop early. 167 func (upload *Upload) Write(p []byte) (n int, err error) { 168 track := upload.stats.trackWorking() 169 n, err = upload.upload.Write(p) 170 upload.mu.Lock() 171 upload.stats.bytes += int64(n) 172 upload.stats.flagFailure(err) 173 track() 174 upload.mu.Unlock() 175 return n, convertKnownErrors(err, upload.bucket, upload.object.Key) 176 } 177 178 // Commit commits data to the store. 179 // 180 // Returns ErrUploadDone when either Abort or Commit has already been called. 181 func (upload *Upload) Commit() error { 182 track := upload.stats.trackWorking() 183 upload.mu.Lock() 184 defer upload.mu.Unlock() 185 186 if upload.aborted { 187 return errwrapf("%w: already aborted", ErrUploadDone) 188 } 189 190 if upload.closed { 191 return errwrapf("%w: already committed", ErrUploadDone) 192 } 193 194 upload.closed = true 195 196 err := errs.Combine( 197 upload.upload.Commit(), 198 upload.streams.Close(), 199 upload.tracker.Close(), 200 ) 201 upload.stats.flagFailure(err) 202 track() 203 upload.emitEvent(false) 204 205 return convertKnownErrors(err, upload.bucket, upload.object.Key) 206 } 207 208 // Abort aborts the upload. 209 // 210 // Returns ErrUploadDone when either Abort or Commit has already been called. 211 func (upload *Upload) Abort() error { 212 track := upload.stats.trackWorking() 213 upload.mu.Lock() 214 defer upload.mu.Unlock() 215 216 if upload.closed { 217 return errwrapf("%w: already committed", ErrUploadDone) 218 } 219 220 if upload.aborted { 221 return errwrapf("%w: already aborted", ErrUploadDone) 222 } 223 224 upload.aborted = true 225 upload.cancel() 226 227 err := errs.Combine( 228 upload.upload.Abort(), 229 upload.streams.Close(), 230 upload.tracker.Close(), 231 ) 232 233 track() 234 upload.stats.flagFailure(err) 235 upload.emitEvent(true) 236 237 return convertKnownErrors(err, upload.bucket, upload.object.Key) 238 } 239 240 func (upload *Upload) emitEvent(aborted bool) { 241 message, err := upload.stats.err() 242 upload.task(&err) 243 244 expires := false 245 if upload.upload != nil { 246 meta := upload.upload.Meta() 247 if meta != nil && !meta.Expiration.IsZero() { 248 expires = true 249 } 250 } 251 252 evs.Event("upload", 253 eventkit.Int64("bytes", upload.stats.bytes), 254 eventkit.Duration("user-elapsed", time.Since(upload.stats.start)), 255 eventkit.Duration("working-elapsed", upload.stats.working), 256 eventkit.Bool("success", err == nil), 257 eventkit.String("error", message), 258 eventkit.Bool("aborted", aborted), 259 eventkit.String("arch", runtime.GOARCH), 260 eventkit.String("os", runtime.GOOS), 261 eventkit.Int64("cpus", int64(runtime.NumCPU())), 262 eventkit.Bool("expires", expires), 263 eventkit.Int64("quic-rollout", int64(upload.stats.quicRollout)), 264 eventkit.String("satellite", upload.stats.satellite), 265 eventkit.Bytes("path-checksum", pathChecksum(upload.stats.encPath)), 266 eventkit.Int64("noise-version", noiseVersion), 267 // upload.upload.Meta().Expiration 268 // segment count 269 // ram available 270 ) 271 } 272 273 // SetCustomMetadata updates custom metadata to be included with the object. 274 // If it is nil, it won't be modified. 275 func (upload *Upload) SetCustomMetadata(ctx context.Context, custom CustomMetadata) error { 276 upload.mu.Lock() 277 defer upload.mu.Unlock() 278 279 if upload.aborted { 280 return errwrapf("%w: upload aborted", ErrUploadDone) 281 } 282 if upload.closed { 283 return errwrapf("%w: already committed", ErrUploadDone) 284 } 285 if upload.upload.Meta() != nil { 286 return errwrapf("%w: already committed", ErrUploadDone) 287 } 288 289 if custom != nil { 290 if err := custom.Verify(); err != nil { 291 return packageError.Wrap(err) 292 } 293 upload.object.Custom = custom.Clone() 294 } 295 296 return nil 297 }