storj.io/uplink@v1.13.0/download.go (about)

     1  // Copyright (C) 2020 Storj Labs, Inc.
     2  // See LICENSE for copying information.
     3  
     4  package uplink
     5  
     6  import (
     7  	"context"
     8  	"crypto/hmac"
     9  	"crypto/sha1"
    10  	"errors"
    11  	"io"
    12  	"runtime"
    13  	"sync"
    14  	"time"
    15  	_ "unsafe" // for go:linkname
    16  
    17  	"github.com/zeebo/errs"
    18  
    19  	"storj.io/common/leak"
    20  	"storj.io/common/paths"
    21  	"storj.io/eventkit"
    22  	"storj.io/uplink/private/metaclient"
    23  	"storj.io/uplink/private/storage/streams"
    24  	"storj.io/uplink/private/stream"
    25  )
    26  
    27  // DownloadOptions contains additional options for downloading.
    28  type DownloadOptions struct {
    29  	// When Offset is negative it will read the suffix of the blob.
    30  	// Combining negative offset and positive length is not supported.
    31  	Offset int64
    32  	// When Length is negative it will read until the end of the blob.
    33  	Length int64
    34  }
    35  
    36  // DownloadObject starts a download from the specific key.
    37  func (project *Project) DownloadObject(ctx context.Context, bucket, key string, options *DownloadOptions) (_ *Download, err error) {
    38  	return project.downloadObjectWithVersion(ctx, bucket, key, nil, options)
    39  }
    40  
    41  func (project *Project) downloadObjectWithVersion(ctx context.Context, bucket, key string, version []byte, options *DownloadOptions) (_ *Download, err error) {
    42  	download := &Download{
    43  		bucket: bucket,
    44  		stats:  newOperationStats(ctx, project.access.satelliteURL),
    45  	}
    46  	download.task = mon.TaskNamed("Download")(&ctx)
    47  	defer func() {
    48  		if err != nil {
    49  			download.stats.flagFailure(err)
    50  			download.emitEvent()
    51  		}
    52  	}()
    53  	defer download.stats.trackWorking()()
    54  	defer mon.Task()(&ctx)(&err)
    55  
    56  	if bucket == "" {
    57  		return nil, errwrapf("%w (%q)", ErrBucketNameInvalid, bucket)
    58  	}
    59  	if key == "" {
    60  		return nil, errwrapf("%w (%q)", ErrObjectKeyInvalid, key)
    61  	}
    62  
    63  	var opts metaclient.DownloadOptions
    64  	switch {
    65  	case options == nil:
    66  		opts.Range = metaclient.StreamRange{
    67  			Mode: metaclient.StreamRangeAll,
    68  		}
    69  	case options.Offset < 0:
    70  		if options.Length >= 0 {
    71  			return nil, packageError.New("suffix requires length to be negative, got %v", options.Length)
    72  		}
    73  		opts.Range = metaclient.StreamRange{
    74  			Mode:   metaclient.StreamRangeSuffix,
    75  			Suffix: -options.Offset,
    76  		}
    77  	case options.Length < 0:
    78  		opts.Range = metaclient.StreamRange{
    79  			Mode:  metaclient.StreamRangeStart,
    80  			Start: options.Offset,
    81  		}
    82  
    83  	default:
    84  		opts.Range = metaclient.StreamRange{
    85  			Mode:  metaclient.StreamRangeStartLimit,
    86  			Start: options.Offset,
    87  			Limit: options.Offset + options.Length,
    88  		}
    89  	}
    90  
    91  	// N.B. we always call dbCleanup which closes the db because
    92  	// closing it earlier has the benefit of returning a connection to
    93  	// the pool, so we try to do that as early as possible.
    94  
    95  	db, err := project.dialMetainfoDB(ctx)
    96  	if err != nil {
    97  		return nil, convertKnownErrors(err, bucket, key)
    98  	}
    99  	defer func() { err = errs.Combine(err, db.Close()) }()
   100  
   101  	objectDownload, err := db.DownloadObject(ctx, bucket, key, version, opts)
   102  	if err != nil {
   103  		return nil, convertKnownErrors(err, bucket, key)
   104  	}
   105  
   106  	download.stats.encPath = objectDownload.EncPath
   107  
   108  	// store this data so even failing events have the best chance of
   109  	// reporting this.
   110  	streamRange := objectDownload.Range
   111  	download.sizes.offset = streamRange.Start
   112  	download.sizes.length = streamRange.Limit - streamRange.Start
   113  	download.sizes.total = objectDownload.Object.Size
   114  
   115  	// Return the connection to the pool as soon as we can.
   116  	if err := db.Close(); err != nil {
   117  		return nil, convertKnownErrors(err, bucket, key)
   118  	}
   119  
   120  	streams, err := project.getStreamsStore(ctx)
   121  	if err != nil {
   122  		return nil, convertKnownErrors(err, bucket, key)
   123  	}
   124  	download.streams = streams
   125  
   126  	download.object = &objectDownload.Object
   127  	download.download = stream.NewDownloadRange(ctx, objectDownload, streams, streamRange.Start, streamRange.Limit-streamRange.Start)
   128  	download.tracker = project.tracker.Child("download", 1)
   129  	return download, nil
   130  }
   131  
   132  // Download is a download from Storj Network.
   133  type Download struct {
   134  	mu       sync.Mutex
   135  	download *stream.Download
   136  	object   *metaclient.Object
   137  	bucket   string
   138  	streams  *streams.Store
   139  
   140  	sizes struct {
   141  		offset, length, total int64
   142  	}
   143  	ttfb  time.Duration
   144  	stats operationStats
   145  	task  func(*error)
   146  
   147  	tracker leak.Ref
   148  }
   149  
   150  // Info returns the last information about the object.
   151  func (download *Download) Info() *Object {
   152  	return convertObject(download.object)
   153  }
   154  
   155  // Read downloads up to len(p) bytes into p from the object's data stream.
   156  // It returns the number of bytes read (0 <= n <= len(p)) and any error encountered.
   157  func (download *Download) Read(p []byte) (n int, err error) {
   158  	track := download.stats.trackWorking()
   159  	n, err = download.download.Read(p)
   160  	download.mu.Lock()
   161  	download.stats.bytes += int64(n)
   162  	if err != nil && !errors.Is(err, io.EOF) {
   163  		download.stats.flagFailure(err)
   164  	}
   165  	if download.ttfb == 0 && n > 0 {
   166  		download.ttfb = time.Since(download.stats.start)
   167  	}
   168  	track()
   169  	download.mu.Unlock()
   170  	return n, convertKnownErrors(err, download.bucket, download.object.Path)
   171  }
   172  
   173  // Close closes the reader of the download.
   174  func (download *Download) Close() error {
   175  	track := download.stats.trackWorking()
   176  	err := errs.Combine(
   177  		download.download.Close(),
   178  		download.streams.Close(),
   179  		download.tracker.Close(),
   180  	)
   181  	download.mu.Lock()
   182  	track()
   183  	download.stats.flagFailure(err)
   184  	download.emitEvent()
   185  	download.mu.Unlock()
   186  	return convertKnownErrors(err, download.bucket, download.object.Path)
   187  }
   188  
   189  func pathChecksum(encPath paths.Encrypted) []byte {
   190  	mac := hmac.New(sha1.New, []byte(encPath.Raw()))
   191  	_, err := mac.Write([]byte("event"))
   192  	if err != nil {
   193  		panic(err)
   194  	}
   195  	return mac.Sum(nil)[:16]
   196  }
   197  
   198  func (download *Download) emitEvent() {
   199  	message, err := download.stats.err()
   200  	download.task(&err)
   201  
   202  	evs.Event("download",
   203  		eventkit.Int64("bytes", download.stats.bytes),
   204  		eventkit.Int64("requested_bytes", download.sizes.length),
   205  		eventkit.Int64("offset", download.sizes.offset),
   206  		eventkit.Int64("object_size", download.sizes.total),
   207  		eventkit.Duration("user-elapsed", time.Since(download.stats.start)),
   208  		eventkit.Duration("working-elapsed", download.stats.working),
   209  		eventkit.Bool("success", err == nil),
   210  		eventkit.String("error", message),
   211  		eventkit.String("arch", runtime.GOARCH),
   212  		eventkit.String("os", runtime.GOOS),
   213  		eventkit.Int64("cpus", int64(runtime.NumCPU())),
   214  		eventkit.Int64("quic-rollout", int64(download.stats.quicRollout)),
   215  		eventkit.String("satellite", download.stats.satellite),
   216  		eventkit.Bytes("path-checksum", pathChecksum(download.stats.encPath)),
   217  		eventkit.Duration("ttfb", download.ttfb),
   218  		eventkit.Int64("noise-version", noiseVersion),
   219  		// TODO: segment count
   220  		// TODO: ram available
   221  	)
   222  }
   223  
   224  // downloadObjectWithVersion is exposing project.downloadObjectWithVersion method.
   225  //
   226  // NB: this is used with linkname in private/object.
   227  // It needs to be updated when this is updated.
   228  //
   229  //lint:ignore U1000, used with linkname
   230  //nolint:deadcode,unused
   231  //go:linkname downloadObjectWithVersion
   232  func downloadObjectWithVersion(ctx context.Context, project *Project, bucket, key string, version []byte, options *DownloadOptions) (_ *Download, err error) {
   233  	return project.downloadObjectWithVersion(ctx, bucket, key, version, options)
   234  }
   235  
   236  // download_getMetaclientObject exposes the object downloaded from the metainfo database.
   237  //
   238  // NB: this is used with linkname in private/object.
   239  // It needs to be updated when this is updated.
   240  //
   241  //lint:ignore U1000, used with linkname
   242  //nolint:deadcode,unused
   243  //go:linkname download_getMetaclientObject
   244  func download_getMetaclientObject(dl *Download) *metaclient.Object { return dl.object }