storj.io/uplink@v1.13.0/objects.go (about) 1 // Copyright (C) 2020 Storj Labs, Inc. 2 // See LICENSE for copying information. 3 4 package uplink 5 6 import ( 7 "context" 8 9 "github.com/zeebo/errs" 10 11 "storj.io/uplink/private/metaclient" 12 "storj.io/uplink/private/testuplink" 13 ) 14 15 // ListObjectsOptions defines object listing options. 16 type ListObjectsOptions struct { 17 // Prefix allows to filter objects by a key prefix. 18 // If not empty, it must end with slash. 19 Prefix string 20 // Cursor sets the starting position of the iterator. 21 // The first item listed will be the one after the cursor. 22 // Cursor is relative to Prefix. 23 Cursor string 24 // Recursive iterates the objects without collapsing prefixes. 25 Recursive bool 26 27 // System includes SystemMetadata in the results. 28 System bool 29 // Custom includes CustomMetadata in the results. 30 Custom bool 31 } 32 33 // ListObjects returns an iterator over the objects. 34 func (project *Project) ListObjects(ctx context.Context, bucket string, options *ListObjectsOptions) *ObjectIterator { 35 defer mon.Task()(&ctx)(nil) 36 37 b := metaclient.Bucket{Name: bucket} 38 opts := metaclient.ListOptions{ 39 Direction: metaclient.After, 40 } 41 42 if options != nil { 43 opts.Prefix = options.Prefix 44 opts.Cursor = options.Cursor 45 opts.Recursive = options.Recursive 46 opts.IncludeCustomMetadata = options.Custom 47 opts.IncludeSystemMetadata = options.System 48 } 49 50 opts.Limit = testuplink.GetListLimit(ctx) 51 52 objects := ObjectIterator{ 53 ctx: ctx, 54 project: project, 55 bucket: b, 56 options: opts, 57 } 58 59 if options != nil { 60 objects.objOptions = *options 61 } 62 63 return &objects 64 } 65 66 // ObjectIterator is an iterator over a collection of objects or prefixes. 67 type ObjectIterator struct { 68 ctx context.Context 69 project *Project 70 bucket metaclient.Bucket 71 options metaclient.ListOptions 72 objOptions ListObjectsOptions 73 list *metaclient.ObjectList 74 position int 75 completed bool 76 err error 77 } 78 79 // Next prepares next Object for reading. 80 // It returns false if the end of the iteration is reached and there are no more objects, or if there is an error. 81 func (objects *ObjectIterator) Next() bool { 82 if objects.err != nil { 83 objects.completed = true 84 return false 85 } 86 87 if objects.list == nil { 88 more := objects.loadNext() 89 objects.completed = !more 90 return more 91 } 92 93 if objects.position >= len(objects.list.Items)-1 { 94 if !objects.list.More { 95 objects.completed = true 96 return false 97 } 98 more := objects.loadNext() 99 objects.completed = !more 100 return more 101 } 102 103 objects.position++ 104 105 return true 106 } 107 108 func (objects *ObjectIterator) loadNext() bool { 109 ok, err := objects.tryLoadNext() 110 if err != nil { 111 objects.err = err 112 return false 113 } 114 return ok 115 } 116 117 func (objects *ObjectIterator) tryLoadNext() (ok bool, err error) { 118 db, err := objects.project.dialMetainfoDB(objects.ctx) 119 if err != nil { 120 return false, convertKnownErrors(err, objects.bucket.Name, "") 121 } 122 defer func() { err = errs.Combine(err, db.Close()) }() 123 124 list, err := db.ListObjects(objects.ctx, objects.bucket.Name, objects.options) 125 if err != nil { 126 return false, convertKnownErrors(err, objects.bucket.Name, "") 127 } 128 objects.list = &list 129 if list.More { 130 objects.options = objects.options.NextPage(list) 131 } 132 objects.position = 0 133 return len(list.Items) > 0, nil 134 } 135 136 // Err returns error, if one happened during iteration. 137 func (objects *ObjectIterator) Err() error { 138 return packageError.Wrap(objects.err) 139 } 140 141 // Item returns the current object in the iterator. 142 func (objects *ObjectIterator) Item() *Object { 143 item := objects.item() 144 if item == nil { 145 return nil 146 } 147 148 key := item.Path 149 if len(objects.options.Prefix) > 0 { 150 key = objects.options.Prefix + item.Path 151 } 152 153 obj := Object{ 154 Key: key, 155 IsPrefix: item.IsPrefix, 156 } 157 158 // TODO: Make this filtering on the satellite 159 if objects.objOptions.System { 160 obj.System = SystemMetadata{ 161 Created: item.Created, 162 Expires: item.Expires, 163 ContentLength: item.Size, 164 } 165 } 166 167 // TODO: Make this filtering on the satellite 168 if objects.objOptions.Custom { 169 obj.Custom = item.Metadata 170 } 171 172 return &obj 173 } 174 175 func (objects *ObjectIterator) item() *metaclient.Object { 176 if objects.completed { 177 return nil 178 } 179 180 if objects.err != nil { 181 return nil 182 } 183 184 if objects.list == nil { 185 return nil 186 } 187 188 if len(objects.list.Items) == 0 { 189 return nil 190 } 191 192 return &objects.list.Items[objects.position] 193 }