storj.io/uplink@v1.13.0/download.go (about) 1 // Copyright (C) 2020 Storj Labs, Inc. 2 // See LICENSE for copying information. 3 4 package uplink 5 6 import ( 7 "context" 8 "crypto/hmac" 9 "crypto/sha1" 10 "errors" 11 "io" 12 "runtime" 13 "sync" 14 "time" 15 _ "unsafe" // for go:linkname 16 17 "github.com/zeebo/errs" 18 19 "storj.io/common/leak" 20 "storj.io/common/paths" 21 "storj.io/eventkit" 22 "storj.io/uplink/private/metaclient" 23 "storj.io/uplink/private/storage/streams" 24 "storj.io/uplink/private/stream" 25 ) 26 27 // DownloadOptions contains additional options for downloading. 28 type DownloadOptions struct { 29 // When Offset is negative it will read the suffix of the blob. 30 // Combining negative offset and positive length is not supported. 31 Offset int64 32 // When Length is negative it will read until the end of the blob. 33 Length int64 34 } 35 36 // DownloadObject starts a download from the specific key. 37 func (project *Project) DownloadObject(ctx context.Context, bucket, key string, options *DownloadOptions) (_ *Download, err error) { 38 return project.downloadObjectWithVersion(ctx, bucket, key, nil, options) 39 } 40 41 func (project *Project) downloadObjectWithVersion(ctx context.Context, bucket, key string, version []byte, options *DownloadOptions) (_ *Download, err error) { 42 download := &Download{ 43 bucket: bucket, 44 stats: newOperationStats(ctx, project.access.satelliteURL), 45 } 46 download.task = mon.TaskNamed("Download")(&ctx) 47 defer func() { 48 if err != nil { 49 download.stats.flagFailure(err) 50 download.emitEvent() 51 } 52 }() 53 defer download.stats.trackWorking()() 54 defer mon.Task()(&ctx)(&err) 55 56 if bucket == "" { 57 return nil, errwrapf("%w (%q)", ErrBucketNameInvalid, bucket) 58 } 59 if key == "" { 60 return nil, errwrapf("%w (%q)", ErrObjectKeyInvalid, key) 61 } 62 63 var opts metaclient.DownloadOptions 64 switch { 65 case options == nil: 66 opts.Range = metaclient.StreamRange{ 67 Mode: metaclient.StreamRangeAll, 68 } 69 case options.Offset < 0: 70 if options.Length >= 0 { 71 return nil, packageError.New("suffix requires length to be negative, got %v", options.Length) 72 } 73 opts.Range = metaclient.StreamRange{ 74 Mode: metaclient.StreamRangeSuffix, 75 Suffix: -options.Offset, 76 } 77 case options.Length < 0: 78 opts.Range = metaclient.StreamRange{ 79 Mode: metaclient.StreamRangeStart, 80 Start: options.Offset, 81 } 82 83 default: 84 opts.Range = metaclient.StreamRange{ 85 Mode: metaclient.StreamRangeStartLimit, 86 Start: options.Offset, 87 Limit: options.Offset + options.Length, 88 } 89 } 90 91 // N.B. we always call dbCleanup which closes the db because 92 // closing it earlier has the benefit of returning a connection to 93 // the pool, so we try to do that as early as possible. 94 95 db, err := project.dialMetainfoDB(ctx) 96 if err != nil { 97 return nil, convertKnownErrors(err, bucket, key) 98 } 99 defer func() { err = errs.Combine(err, db.Close()) }() 100 101 objectDownload, err := db.DownloadObject(ctx, bucket, key, version, opts) 102 if err != nil { 103 return nil, convertKnownErrors(err, bucket, key) 104 } 105 106 download.stats.encPath = objectDownload.EncPath 107 108 // store this data so even failing events have the best chance of 109 // reporting this. 110 streamRange := objectDownload.Range 111 download.sizes.offset = streamRange.Start 112 download.sizes.length = streamRange.Limit - streamRange.Start 113 download.sizes.total = objectDownload.Object.Size 114 115 // Return the connection to the pool as soon as we can. 116 if err := db.Close(); err != nil { 117 return nil, convertKnownErrors(err, bucket, key) 118 } 119 120 streams, err := project.getStreamsStore(ctx) 121 if err != nil { 122 return nil, convertKnownErrors(err, bucket, key) 123 } 124 download.streams = streams 125 126 download.object = &objectDownload.Object 127 download.download = stream.NewDownloadRange(ctx, objectDownload, streams, streamRange.Start, streamRange.Limit-streamRange.Start) 128 download.tracker = project.tracker.Child("download", 1) 129 return download, nil 130 } 131 132 // Download is a download from Storj Network. 133 type Download struct { 134 mu sync.Mutex 135 download *stream.Download 136 object *metaclient.Object 137 bucket string 138 streams *streams.Store 139 140 sizes struct { 141 offset, length, total int64 142 } 143 ttfb time.Duration 144 stats operationStats 145 task func(*error) 146 147 tracker leak.Ref 148 } 149 150 // Info returns the last information about the object. 151 func (download *Download) Info() *Object { 152 return convertObject(download.object) 153 } 154 155 // Read downloads up to len(p) bytes into p from the object's data stream. 156 // It returns the number of bytes read (0 <= n <= len(p)) and any error encountered. 157 func (download *Download) Read(p []byte) (n int, err error) { 158 track := download.stats.trackWorking() 159 n, err = download.download.Read(p) 160 download.mu.Lock() 161 download.stats.bytes += int64(n) 162 if err != nil && !errors.Is(err, io.EOF) { 163 download.stats.flagFailure(err) 164 } 165 if download.ttfb == 0 && n > 0 { 166 download.ttfb = time.Since(download.stats.start) 167 } 168 track() 169 download.mu.Unlock() 170 return n, convertKnownErrors(err, download.bucket, download.object.Path) 171 } 172 173 // Close closes the reader of the download. 174 func (download *Download) Close() error { 175 track := download.stats.trackWorking() 176 err := errs.Combine( 177 download.download.Close(), 178 download.streams.Close(), 179 download.tracker.Close(), 180 ) 181 download.mu.Lock() 182 track() 183 download.stats.flagFailure(err) 184 download.emitEvent() 185 download.mu.Unlock() 186 return convertKnownErrors(err, download.bucket, download.object.Path) 187 } 188 189 func pathChecksum(encPath paths.Encrypted) []byte { 190 mac := hmac.New(sha1.New, []byte(encPath.Raw())) 191 _, err := mac.Write([]byte("event")) 192 if err != nil { 193 panic(err) 194 } 195 return mac.Sum(nil)[:16] 196 } 197 198 func (download *Download) emitEvent() { 199 message, err := download.stats.err() 200 download.task(&err) 201 202 evs.Event("download", 203 eventkit.Int64("bytes", download.stats.bytes), 204 eventkit.Int64("requested_bytes", download.sizes.length), 205 eventkit.Int64("offset", download.sizes.offset), 206 eventkit.Int64("object_size", download.sizes.total), 207 eventkit.Duration("user-elapsed", time.Since(download.stats.start)), 208 eventkit.Duration("working-elapsed", download.stats.working), 209 eventkit.Bool("success", err == nil), 210 eventkit.String("error", message), 211 eventkit.String("arch", runtime.GOARCH), 212 eventkit.String("os", runtime.GOOS), 213 eventkit.Int64("cpus", int64(runtime.NumCPU())), 214 eventkit.Int64("quic-rollout", int64(download.stats.quicRollout)), 215 eventkit.String("satellite", download.stats.satellite), 216 eventkit.Bytes("path-checksum", pathChecksum(download.stats.encPath)), 217 eventkit.Duration("ttfb", download.ttfb), 218 eventkit.Int64("noise-version", noiseVersion), 219 // TODO: segment count 220 // TODO: ram available 221 ) 222 } 223 224 // downloadObjectWithVersion is exposing project.downloadObjectWithVersion method. 225 // 226 // NB: this is used with linkname in private/object. 227 // It needs to be updated when this is updated. 228 // 229 //lint:ignore U1000, used with linkname 230 //nolint:deadcode,unused 231 //go:linkname downloadObjectWithVersion 232 func downloadObjectWithVersion(ctx context.Context, project *Project, bucket, key string, version []byte, options *DownloadOptions) (_ *Download, err error) { 233 return project.downloadObjectWithVersion(ctx, bucket, key, version, options) 234 } 235 236 // download_getMetaclientObject exposes the object downloaded from the metainfo database. 237 // 238 // NB: this is used with linkname in private/object. 239 // It needs to be updated when this is updated. 240 // 241 //lint:ignore U1000, used with linkname 242 //nolint:deadcode,unused 243 //go:linkname download_getMetaclientObject 244 func download_getMetaclientObject(dl *Download) *metaclient.Object { return dl.object }