storj.io/uplink@v1.13.0/objects.go (about)

     1  // Copyright (C) 2020 Storj Labs, Inc.
     2  // See LICENSE for copying information.
     3  
     4  package uplink
     5  
     6  import (
     7  	"context"
     8  
     9  	"github.com/zeebo/errs"
    10  
    11  	"storj.io/uplink/private/metaclient"
    12  	"storj.io/uplink/private/testuplink"
    13  )
    14  
    15  // ListObjectsOptions defines object listing options.
    16  type ListObjectsOptions struct {
    17  	// Prefix allows to filter objects by a key prefix.
    18  	// If not empty, it must end with slash.
    19  	Prefix string
    20  	// Cursor sets the starting position of the iterator.
    21  	// The first item listed will be the one after the cursor.
    22  	// Cursor is relative to Prefix.
    23  	Cursor string
    24  	// Recursive iterates the objects without collapsing prefixes.
    25  	Recursive bool
    26  
    27  	// System includes SystemMetadata in the results.
    28  	System bool
    29  	// Custom includes CustomMetadata in the results.
    30  	Custom bool
    31  }
    32  
    33  // ListObjects returns an iterator over the objects.
    34  func (project *Project) ListObjects(ctx context.Context, bucket string, options *ListObjectsOptions) *ObjectIterator {
    35  	defer mon.Task()(&ctx)(nil)
    36  
    37  	b := metaclient.Bucket{Name: bucket}
    38  	opts := metaclient.ListOptions{
    39  		Direction: metaclient.After,
    40  	}
    41  
    42  	if options != nil {
    43  		opts.Prefix = options.Prefix
    44  		opts.Cursor = options.Cursor
    45  		opts.Recursive = options.Recursive
    46  		opts.IncludeCustomMetadata = options.Custom
    47  		opts.IncludeSystemMetadata = options.System
    48  	}
    49  
    50  	opts.Limit = testuplink.GetListLimit(ctx)
    51  
    52  	objects := ObjectIterator{
    53  		ctx:     ctx,
    54  		project: project,
    55  		bucket:  b,
    56  		options: opts,
    57  	}
    58  
    59  	if options != nil {
    60  		objects.objOptions = *options
    61  	}
    62  
    63  	return &objects
    64  }
    65  
    66  // ObjectIterator is an iterator over a collection of objects or prefixes.
    67  type ObjectIterator struct {
    68  	ctx        context.Context
    69  	project    *Project
    70  	bucket     metaclient.Bucket
    71  	options    metaclient.ListOptions
    72  	objOptions ListObjectsOptions
    73  	list       *metaclient.ObjectList
    74  	position   int
    75  	completed  bool
    76  	err        error
    77  }
    78  
    79  // Next prepares next Object for reading.
    80  // It returns false if the end of the iteration is reached and there are no more objects, or if there is an error.
    81  func (objects *ObjectIterator) Next() bool {
    82  	if objects.err != nil {
    83  		objects.completed = true
    84  		return false
    85  	}
    86  
    87  	if objects.list == nil {
    88  		more := objects.loadNext()
    89  		objects.completed = !more
    90  		return more
    91  	}
    92  
    93  	if objects.position >= len(objects.list.Items)-1 {
    94  		if !objects.list.More {
    95  			objects.completed = true
    96  			return false
    97  		}
    98  		more := objects.loadNext()
    99  		objects.completed = !more
   100  		return more
   101  	}
   102  
   103  	objects.position++
   104  
   105  	return true
   106  }
   107  
   108  func (objects *ObjectIterator) loadNext() bool {
   109  	ok, err := objects.tryLoadNext()
   110  	if err != nil {
   111  		objects.err = err
   112  		return false
   113  	}
   114  	return ok
   115  }
   116  
   117  func (objects *ObjectIterator) tryLoadNext() (ok bool, err error) {
   118  	db, err := objects.project.dialMetainfoDB(objects.ctx)
   119  	if err != nil {
   120  		return false, convertKnownErrors(err, objects.bucket.Name, "")
   121  	}
   122  	defer func() { err = errs.Combine(err, db.Close()) }()
   123  
   124  	list, err := db.ListObjects(objects.ctx, objects.bucket.Name, objects.options)
   125  	if err != nil {
   126  		return false, convertKnownErrors(err, objects.bucket.Name, "")
   127  	}
   128  	objects.list = &list
   129  	if list.More {
   130  		objects.options = objects.options.NextPage(list)
   131  	}
   132  	objects.position = 0
   133  	return len(list.Items) > 0, nil
   134  }
   135  
   136  // Err returns error, if one happened during iteration.
   137  func (objects *ObjectIterator) Err() error {
   138  	return packageError.Wrap(objects.err)
   139  }
   140  
   141  // Item returns the current object in the iterator.
   142  func (objects *ObjectIterator) Item() *Object {
   143  	item := objects.item()
   144  	if item == nil {
   145  		return nil
   146  	}
   147  
   148  	key := item.Path
   149  	if len(objects.options.Prefix) > 0 {
   150  		key = objects.options.Prefix + item.Path
   151  	}
   152  
   153  	obj := Object{
   154  		Key:      key,
   155  		IsPrefix: item.IsPrefix,
   156  	}
   157  
   158  	// TODO: Make this filtering on the satellite
   159  	if objects.objOptions.System {
   160  		obj.System = SystemMetadata{
   161  			Created:       item.Created,
   162  			Expires:       item.Expires,
   163  			ContentLength: item.Size,
   164  		}
   165  	}
   166  
   167  	// TODO: Make this filtering on the satellite
   168  	if objects.objOptions.Custom {
   169  		obj.Custom = item.Metadata
   170  	}
   171  
   172  	return &obj
   173  }
   174  
   175  func (objects *ObjectIterator) item() *metaclient.Object {
   176  	if objects.completed {
   177  		return nil
   178  	}
   179  
   180  	if objects.err != nil {
   181  		return nil
   182  	}
   183  
   184  	if objects.list == nil {
   185  		return nil
   186  	}
   187  
   188  	if len(objects.list.Items) == 0 {
   189  		return nil
   190  	}
   191  
   192  	return &objects.list.Items[objects.position]
   193  }