github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/backend/oracleobjectstorage/command.go (about) 1 //go:build !plan9 && !solaris && !js 2 3 package oracleobjectstorage 4 5 import ( 6 "context" 7 "fmt" 8 "sort" 9 "strconv" 10 "strings" 11 "sync" 12 "time" 13 14 "github.com/oracle/oci-go-sdk/v65/common" 15 "github.com/oracle/oci-go-sdk/v65/objectstorage" 16 "github.com/rclone/rclone/fs" 17 "github.com/rclone/rclone/fs/operations" 18 ) 19 20 // ------------------------------------------------------------ 21 // Command Interface Implementation 22 // ------------------------------------------------------------ 23 24 const ( 25 operationRename = "rename" 26 operationListMultiPart = "list-multipart-uploads" 27 operationCleanup = "cleanup" 28 operationRestore = "restore" 29 ) 30 31 var commandHelp = []fs.CommandHelp{{ 32 Name: operationRename, 33 Short: "change the name of an object", 34 Long: `This command can be used to rename a object. 35 36 Usage Examples: 37 38 rclone backend rename oos:bucket relative-object-path-under-bucket object-new-name 39 `, 40 Opts: nil, 41 }, { 42 Name: operationListMultiPart, 43 Short: "List the unfinished multipart uploads", 44 Long: `This command lists the unfinished multipart uploads in JSON format. 45 46 rclone backend list-multipart-uploads oos:bucket/path/to/object 47 48 It returns a dictionary of buckets with values as lists of unfinished 49 multipart uploads. 50 51 You can call it with no bucket in which case it lists all bucket, with 52 a bucket or with a bucket and path. 53 54 { 55 "test-bucket": [ 56 { 57 "namespace": "test-namespace", 58 "bucket": "test-bucket", 59 "object": "600m.bin", 60 "uploadId": "51dd8114-52a4-b2f2-c42f-5291f05eb3c8", 61 "timeCreated": "2022-07-29T06:21:16.595Z", 62 "storageTier": "Standard" 63 } 64 ] 65 `, 66 }, { 67 Name: operationCleanup, 68 Short: "Remove unfinished multipart uploads.", 69 Long: `This command removes unfinished multipart uploads of age greater than 70 max-age which defaults to 24 hours. 71 72 Note that you can use --interactive/-i or --dry-run with this command to see what 73 it would do. 74 75 rclone backend cleanup oos:bucket/path/to/object 76 rclone backend cleanup -o max-age=7w oos:bucket/path/to/object 77 78 Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc. 79 `, 80 Opts: map[string]string{ 81 "max-age": "Max age of upload to delete", 82 }, 83 }, { 84 Name: operationRestore, 85 Short: "Restore objects from Archive to Standard storage", 86 Long: `This command can be used to restore one or more objects from Archive to Standard storage. 87 88 Usage Examples: 89 90 rclone backend restore oos:bucket/path/to/directory -o hours=HOURS 91 rclone backend restore oos:bucket -o hours=HOURS 92 93 This flag also obeys the filters. Test first with --interactive/-i or --dry-run flags 94 95 rclone --interactive backend restore --include "*.txt" oos:bucket/path -o hours=72 96 97 All the objects shown will be marked for restore, then 98 99 rclone backend restore --include "*.txt" oos:bucket/path -o hours=72 100 101 It returns a list of status dictionaries with Object Name and Status 102 keys. The Status will be "RESTORED"" if it was successful or an error message 103 if not. 104 105 [ 106 { 107 "Object": "test.txt" 108 "Status": "RESTORED", 109 }, 110 { 111 "Object": "test/file4.txt" 112 "Status": "RESTORED", 113 } 114 ] 115 `, 116 Opts: map[string]string{ 117 "hours": "The number of hours for which this object will be restored. Default is 24 hrs.", 118 }, 119 }, 120 } 121 122 /* 123 Command the backend to run a named command 124 125 The command run is name 126 args may be used to read arguments from 127 opts may be used to read optional arguments from 128 129 The result should be capable of being JSON encoded 130 If it is a string or a []string it will be shown to the user 131 otherwise it will be JSON encoded and shown to the user like that 132 */ 133 func (f *Fs) Command(ctx context.Context, commandName string, args []string, 134 opt map[string]string) (result interface{}, err error) { 135 // fs.Debugf(f, "command %v, args: %v, opts:%v", commandName, args, opt) 136 switch commandName { 137 case operationRename: 138 if len(args) < 2 { 139 return nil, fmt.Errorf("path to object or its new name to rename is empty") 140 } 141 remote := args[0] 142 newName := args[1] 143 return f.rename(ctx, remote, newName) 144 case operationListMultiPart: 145 return f.listMultipartUploadsAll(ctx) 146 case operationCleanup: 147 maxAge := 24 * time.Hour 148 if opt["max-age"] != "" { 149 maxAge, err = fs.ParseDuration(opt["max-age"]) 150 if err != nil { 151 return nil, fmt.Errorf("bad max-age: %w", err) 152 } 153 } 154 return nil, f.cleanUp(ctx, maxAge) 155 case operationRestore: 156 return f.restore(ctx, opt) 157 default: 158 return nil, fs.ErrorCommandNotFound 159 } 160 } 161 162 func (f *Fs) rename(ctx context.Context, remote, newName string) (interface{}, error) { 163 if remote == "" { 164 return nil, fmt.Errorf("path to object file cannot be empty") 165 } 166 if newName == "" { 167 return nil, fmt.Errorf("the object's new name cannot be empty") 168 } 169 o := &Object{ 170 fs: f, 171 remote: remote, 172 } 173 bucketName, objectPath := o.split() 174 err := o.readMetaData(ctx) 175 if err != nil { 176 fs.Errorf(f, "failed to read object:%v %v ", objectPath, err) 177 if strings.HasPrefix(objectPath, bucketName) { 178 fs.Errorf(f, "warn: ensure object path: %v is relative to bucket:%v and doesn't include the bucket name", 179 objectPath, bucketName) 180 } 181 return nil, fs.ErrorNotAFile 182 } 183 details := objectstorage.RenameObjectDetails{ 184 SourceName: common.String(objectPath), 185 NewName: common.String(newName), 186 } 187 request := objectstorage.RenameObjectRequest{ 188 NamespaceName: common.String(f.opt.Namespace), 189 BucketName: common.String(bucketName), 190 RenameObjectDetails: details, 191 OpcClientRequestId: nil, 192 RequestMetadata: common.RequestMetadata{}, 193 } 194 var response objectstorage.RenameObjectResponse 195 err = f.pacer.Call(func() (bool, error) { 196 response, err = f.srv.RenameObject(ctx, request) 197 return shouldRetry(ctx, response.HTTPResponse(), err) 198 }) 199 if err != nil { 200 return nil, err 201 } 202 fs.Infof(f, "success: renamed object-path: %v to %v", objectPath, newName) 203 return "renamed successfully", nil 204 } 205 206 func (f *Fs) listMultipartUploadsAll(ctx context.Context) (uploadsMap map[string][]*objectstorage.MultipartUpload, 207 err error) { 208 uploadsMap = make(map[string][]*objectstorage.MultipartUpload) 209 bucket, directory := f.split("") 210 if bucket != "" { 211 uploads, err := f.listMultipartUploads(ctx, bucket, directory) 212 if err != nil { 213 return uploadsMap, err 214 } 215 uploadsMap[bucket] = uploads 216 return uploadsMap, nil 217 } 218 entries, err := f.listBuckets(ctx) 219 if err != nil { 220 return uploadsMap, err 221 } 222 for _, entry := range entries { 223 bucket := entry.Remote() 224 uploads, listErr := f.listMultipartUploads(ctx, bucket, "") 225 if listErr != nil { 226 err = listErr 227 fs.Errorf(f, "%v", err) 228 } 229 uploadsMap[bucket] = uploads 230 } 231 return uploadsMap, err 232 } 233 234 // listMultipartUploads lists all outstanding multipart uploads for (bucket, key) 235 // 236 // Note that rather lazily we treat key as a prefix, so it matches 237 // directories and objects. This could surprise the user if they ask 238 // for "dir" and it returns "dirKey" 239 func (f *Fs) listMultipartUploads(ctx context.Context, bucketName, directory string) ( 240 uploads []*objectstorage.MultipartUpload, err error) { 241 return f.listMultipartUploadsObject(ctx, bucketName, directory, false) 242 } 243 244 // listMultipartUploads finds first outstanding multipart uploads for (bucket, key) 245 // 246 // Note that rather lazily we treat key as a prefix, so it matches 247 // directories and objects. This could surprise the user if they ask 248 // for "dir" and it returns "dirKey" 249 func (f *Fs) findLatestMultipartUpload(ctx context.Context, bucketName, directory string) ( 250 uploads []*objectstorage.MultipartUpload, err error) { 251 pastUploads, err := f.listMultipartUploadsObject(ctx, bucketName, directory, true) 252 if err != nil { 253 return nil, err 254 } 255 256 if len(pastUploads) > 0 { 257 sort.Slice(pastUploads, func(i, j int) bool { 258 return pastUploads[i].TimeCreated.After(pastUploads[j].TimeCreated.Time) 259 }) 260 return pastUploads[:1], nil 261 } 262 return nil, err 263 } 264 265 func (f *Fs) listMultipartUploadsObject(ctx context.Context, bucketName, directory string, exact bool) ( 266 uploads []*objectstorage.MultipartUpload, err error) { 267 268 uploads = []*objectstorage.MultipartUpload{} 269 req := objectstorage.ListMultipartUploadsRequest{ 270 NamespaceName: common.String(f.opt.Namespace), 271 BucketName: common.String(bucketName), 272 } 273 274 var response objectstorage.ListMultipartUploadsResponse 275 for { 276 err = f.pacer.Call(func() (bool, error) { 277 response, err = f.srv.ListMultipartUploads(ctx, req) 278 return shouldRetry(ctx, response.HTTPResponse(), err) 279 }) 280 if err != nil { 281 // fs.Debugf(f, "failed to list multi part uploads %v", err) 282 return uploads, err 283 } 284 for index, item := range response.Items { 285 if directory != "" && item.Object != nil && !strings.HasPrefix(*item.Object, directory) { 286 continue 287 } 288 if exact { 289 if *item.Object == directory { 290 uploads = append(uploads, &response.Items[index]) 291 } 292 } else { 293 uploads = append(uploads, &response.Items[index]) 294 } 295 } 296 if response.OpcNextPage == nil { 297 break 298 } 299 req.Page = response.OpcNextPage 300 } 301 return uploads, nil 302 } 303 304 func (f *Fs) listMultipartUploadParts(ctx context.Context, bucketName, bucketPath string, uploadID string) ( 305 uploadedParts map[int]objectstorage.MultipartUploadPartSummary, err error) { 306 uploadedParts = make(map[int]objectstorage.MultipartUploadPartSummary) 307 req := objectstorage.ListMultipartUploadPartsRequest{ 308 NamespaceName: common.String(f.opt.Namespace), 309 BucketName: common.String(bucketName), 310 ObjectName: common.String(bucketPath), 311 UploadId: common.String(uploadID), 312 Limit: common.Int(1000), 313 } 314 315 var response objectstorage.ListMultipartUploadPartsResponse 316 for { 317 err = f.pacer.Call(func() (bool, error) { 318 response, err = f.srv.ListMultipartUploadParts(ctx, req) 319 return shouldRetry(ctx, response.HTTPResponse(), err) 320 }) 321 if err != nil { 322 return uploadedParts, err 323 } 324 for _, item := range response.Items { 325 uploadedParts[*item.PartNumber] = item 326 } 327 if response.OpcNextPage == nil { 328 break 329 } 330 req.Page = response.OpcNextPage 331 } 332 return uploadedParts, nil 333 } 334 335 func (f *Fs) restore(ctx context.Context, opt map[string]string) (interface{}, error) { 336 req := objectstorage.RestoreObjectsRequest{ 337 NamespaceName: common.String(f.opt.Namespace), 338 RestoreObjectsDetails: objectstorage.RestoreObjectsDetails{}, 339 } 340 if hours := opt["hours"]; hours != "" { 341 ihours, err := strconv.Atoi(hours) 342 if err != nil { 343 return nil, fmt.Errorf("bad value for hours: %w", err) 344 } 345 req.RestoreObjectsDetails.Hours = &ihours 346 } 347 type status struct { 348 Object string 349 Status string 350 } 351 var ( 352 outMu sync.Mutex 353 out = []status{} 354 err error 355 ) 356 err = operations.ListFn(ctx, f, func(obj fs.Object) { 357 // Remember this is run --checkers times concurrently 358 o, ok := obj.(*Object) 359 st := status{Object: obj.Remote(), Status: "RESTORED"} 360 defer func() { 361 outMu.Lock() 362 out = append(out, st) 363 outMu.Unlock() 364 }() 365 if !ok { 366 st.Status = "Not an OCI Object Storage object" 367 return 368 } 369 if o.storageTier == nil || (*o.storageTier != "archive") { 370 st.Status = "Object not in Archive storage tier" 371 return 372 } 373 if operations.SkipDestructive(ctx, obj, "restore") { 374 return 375 } 376 bucket, bucketPath := o.split() 377 reqCopy := req 378 reqCopy.BucketName = &bucket 379 reqCopy.ObjectName = &bucketPath 380 var response objectstorage.RestoreObjectsResponse 381 err = f.pacer.Call(func() (bool, error) { 382 response, err = f.srv.RestoreObjects(ctx, reqCopy) 383 return shouldRetry(ctx, response.HTTPResponse(), err) 384 }) 385 if err != nil { 386 st.Status = err.Error() 387 } 388 }) 389 if err != nil { 390 return out, err 391 } 392 return out, nil 393 }