github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/backend/oracleobjectstorage/command.go (about)

     1  //go:build !plan9 && !solaris && !js
     2  
     3  package oracleobjectstorage
     4  
     5  import (
     6  	"context"
     7  	"fmt"
     8  	"sort"
     9  	"strconv"
    10  	"strings"
    11  	"sync"
    12  	"time"
    13  
    14  	"github.com/oracle/oci-go-sdk/v65/common"
    15  	"github.com/oracle/oci-go-sdk/v65/objectstorage"
    16  	"github.com/rclone/rclone/fs"
    17  	"github.com/rclone/rclone/fs/operations"
    18  )
    19  
    20  // ------------------------------------------------------------
    21  // Command Interface Implementation
    22  // ------------------------------------------------------------
    23  
    24  const (
    25  	operationRename        = "rename"
    26  	operationListMultiPart = "list-multipart-uploads"
    27  	operationCleanup       = "cleanup"
    28  	operationRestore       = "restore"
    29  )
    30  
    31  var commandHelp = []fs.CommandHelp{{
    32  	Name:  operationRename,
    33  	Short: "change the name of an object",
    34  	Long: `This command can be used to rename a object.
    35  
    36  Usage Examples:
    37  
    38      rclone backend rename oos:bucket relative-object-path-under-bucket object-new-name
    39  `,
    40  	Opts: nil,
    41  }, {
    42  	Name:  operationListMultiPart,
    43  	Short: "List the unfinished multipart uploads",
    44  	Long: `This command lists the unfinished multipart uploads in JSON format.
    45  
    46      rclone backend list-multipart-uploads oos:bucket/path/to/object
    47  
    48  It returns a dictionary of buckets with values as lists of unfinished
    49  multipart uploads.
    50  
    51  You can call it with no bucket in which case it lists all bucket, with
    52  a bucket or with a bucket and path.
    53  
    54      {
    55        "test-bucket": [
    56                  {
    57                          "namespace": "test-namespace",
    58                          "bucket": "test-bucket",
    59                          "object": "600m.bin",
    60                          "uploadId": "51dd8114-52a4-b2f2-c42f-5291f05eb3c8",
    61                          "timeCreated": "2022-07-29T06:21:16.595Z",
    62                          "storageTier": "Standard"
    63                  }
    64          ]
    65  `,
    66  }, {
    67  	Name:  operationCleanup,
    68  	Short: "Remove unfinished multipart uploads.",
    69  	Long: `This command removes unfinished multipart uploads of age greater than
    70  max-age which defaults to 24 hours.
    71  
    72  Note that you can use --interactive/-i or --dry-run with this command to see what
    73  it would do.
    74  
    75      rclone backend cleanup oos:bucket/path/to/object
    76      rclone backend cleanup -o max-age=7w oos:bucket/path/to/object
    77  
    78  Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc.
    79  `,
    80  	Opts: map[string]string{
    81  		"max-age": "Max age of upload to delete",
    82  	},
    83  }, {
    84  	Name:  operationRestore,
    85  	Short: "Restore objects from Archive to Standard storage",
    86  	Long: `This command can be used to restore one or more objects from Archive to Standard storage.
    87  
    88  	Usage Examples:
    89  
    90      rclone backend restore oos:bucket/path/to/directory -o hours=HOURS
    91      rclone backend restore oos:bucket -o hours=HOURS
    92  
    93  This flag also obeys the filters. Test first with --interactive/-i or --dry-run flags
    94  
    95  	rclone --interactive backend restore --include "*.txt" oos:bucket/path -o hours=72
    96  
    97  All the objects shown will be marked for restore, then
    98  
    99  	rclone backend restore --include "*.txt" oos:bucket/path -o hours=72
   100  
   101  	It returns a list of status dictionaries with Object Name and Status
   102  	keys. The Status will be "RESTORED"" if it was successful or an error message
   103  	if not.
   104  
   105  	[
   106  		{
   107  			"Object": "test.txt"
   108  			"Status": "RESTORED",
   109  		},
   110  		{
   111  			"Object": "test/file4.txt"
   112  			"Status": "RESTORED",
   113  		}
   114  	]
   115  `,
   116  	Opts: map[string]string{
   117  		"hours": "The number of hours for which this object will be restored. Default is 24 hrs.",
   118  	},
   119  },
   120  }
   121  
   122  /*
   123  Command the backend to run a named command
   124  
   125  The command run is name
   126  args may be used to read arguments from
   127  opts may be used to read optional arguments from
   128  
   129  The result should be capable of being JSON encoded
   130  If it is a string or a []string it will be shown to the user
   131  otherwise it will be JSON encoded and shown to the user like that
   132  */
   133  func (f *Fs) Command(ctx context.Context, commandName string, args []string,
   134  	opt map[string]string) (result interface{}, err error) {
   135  	// fs.Debugf(f, "command %v, args: %v, opts:%v", commandName, args, opt)
   136  	switch commandName {
   137  	case operationRename:
   138  		if len(args) < 2 {
   139  			return nil, fmt.Errorf("path to object or its new name to rename is empty")
   140  		}
   141  		remote := args[0]
   142  		newName := args[1]
   143  		return f.rename(ctx, remote, newName)
   144  	case operationListMultiPart:
   145  		return f.listMultipartUploadsAll(ctx)
   146  	case operationCleanup:
   147  		maxAge := 24 * time.Hour
   148  		if opt["max-age"] != "" {
   149  			maxAge, err = fs.ParseDuration(opt["max-age"])
   150  			if err != nil {
   151  				return nil, fmt.Errorf("bad max-age: %w", err)
   152  			}
   153  		}
   154  		return nil, f.cleanUp(ctx, maxAge)
   155  	case operationRestore:
   156  		return f.restore(ctx, opt)
   157  	default:
   158  		return nil, fs.ErrorCommandNotFound
   159  	}
   160  }
   161  
   162  func (f *Fs) rename(ctx context.Context, remote, newName string) (interface{}, error) {
   163  	if remote == "" {
   164  		return nil, fmt.Errorf("path to object file cannot be empty")
   165  	}
   166  	if newName == "" {
   167  		return nil, fmt.Errorf("the object's new name cannot be empty")
   168  	}
   169  	o := &Object{
   170  		fs:     f,
   171  		remote: remote,
   172  	}
   173  	bucketName, objectPath := o.split()
   174  	err := o.readMetaData(ctx)
   175  	if err != nil {
   176  		fs.Errorf(f, "failed to read object:%v %v ", objectPath, err)
   177  		if strings.HasPrefix(objectPath, bucketName) {
   178  			fs.Errorf(f, "warn: ensure object path: %v is relative to bucket:%v and doesn't include the bucket name",
   179  				objectPath, bucketName)
   180  		}
   181  		return nil, fs.ErrorNotAFile
   182  	}
   183  	details := objectstorage.RenameObjectDetails{
   184  		SourceName: common.String(objectPath),
   185  		NewName:    common.String(newName),
   186  	}
   187  	request := objectstorage.RenameObjectRequest{
   188  		NamespaceName:       common.String(f.opt.Namespace),
   189  		BucketName:          common.String(bucketName),
   190  		RenameObjectDetails: details,
   191  		OpcClientRequestId:  nil,
   192  		RequestMetadata:     common.RequestMetadata{},
   193  	}
   194  	var response objectstorage.RenameObjectResponse
   195  	err = f.pacer.Call(func() (bool, error) {
   196  		response, err = f.srv.RenameObject(ctx, request)
   197  		return shouldRetry(ctx, response.HTTPResponse(), err)
   198  	})
   199  	if err != nil {
   200  		return nil, err
   201  	}
   202  	fs.Infof(f, "success: renamed object-path: %v to %v", objectPath, newName)
   203  	return "renamed successfully", nil
   204  }
   205  
   206  func (f *Fs) listMultipartUploadsAll(ctx context.Context) (uploadsMap map[string][]*objectstorage.MultipartUpload,
   207  	err error) {
   208  	uploadsMap = make(map[string][]*objectstorage.MultipartUpload)
   209  	bucket, directory := f.split("")
   210  	if bucket != "" {
   211  		uploads, err := f.listMultipartUploads(ctx, bucket, directory)
   212  		if err != nil {
   213  			return uploadsMap, err
   214  		}
   215  		uploadsMap[bucket] = uploads
   216  		return uploadsMap, nil
   217  	}
   218  	entries, err := f.listBuckets(ctx)
   219  	if err != nil {
   220  		return uploadsMap, err
   221  	}
   222  	for _, entry := range entries {
   223  		bucket := entry.Remote()
   224  		uploads, listErr := f.listMultipartUploads(ctx, bucket, "")
   225  		if listErr != nil {
   226  			err = listErr
   227  			fs.Errorf(f, "%v", err)
   228  		}
   229  		uploadsMap[bucket] = uploads
   230  	}
   231  	return uploadsMap, err
   232  }
   233  
   234  // listMultipartUploads lists all outstanding multipart uploads for (bucket, key)
   235  //
   236  // Note that rather lazily we treat key as a prefix, so it matches
   237  // directories and objects. This could surprise the user if they ask
   238  // for "dir" and it returns "dirKey"
   239  func (f *Fs) listMultipartUploads(ctx context.Context, bucketName, directory string) (
   240  	uploads []*objectstorage.MultipartUpload, err error) {
   241  	return f.listMultipartUploadsObject(ctx, bucketName, directory, false)
   242  }
   243  
   244  // listMultipartUploads finds first outstanding multipart uploads for (bucket, key)
   245  //
   246  // Note that rather lazily we treat key as a prefix, so it matches
   247  // directories and objects. This could surprise the user if they ask
   248  // for "dir" and it returns "dirKey"
   249  func (f *Fs) findLatestMultipartUpload(ctx context.Context, bucketName, directory string) (
   250  	uploads []*objectstorage.MultipartUpload, err error) {
   251  	pastUploads, err := f.listMultipartUploadsObject(ctx, bucketName, directory, true)
   252  	if err != nil {
   253  		return nil, err
   254  	}
   255  
   256  	if len(pastUploads) > 0 {
   257  		sort.Slice(pastUploads, func(i, j int) bool {
   258  			return pastUploads[i].TimeCreated.After(pastUploads[j].TimeCreated.Time)
   259  		})
   260  		return pastUploads[:1], nil
   261  	}
   262  	return nil, err
   263  }
   264  
   265  func (f *Fs) listMultipartUploadsObject(ctx context.Context, bucketName, directory string, exact bool) (
   266  	uploads []*objectstorage.MultipartUpload, err error) {
   267  
   268  	uploads = []*objectstorage.MultipartUpload{}
   269  	req := objectstorage.ListMultipartUploadsRequest{
   270  		NamespaceName: common.String(f.opt.Namespace),
   271  		BucketName:    common.String(bucketName),
   272  	}
   273  
   274  	var response objectstorage.ListMultipartUploadsResponse
   275  	for {
   276  		err = f.pacer.Call(func() (bool, error) {
   277  			response, err = f.srv.ListMultipartUploads(ctx, req)
   278  			return shouldRetry(ctx, response.HTTPResponse(), err)
   279  		})
   280  		if err != nil {
   281  			// fs.Debugf(f, "failed to list multi part uploads %v", err)
   282  			return uploads, err
   283  		}
   284  		for index, item := range response.Items {
   285  			if directory != "" && item.Object != nil && !strings.HasPrefix(*item.Object, directory) {
   286  				continue
   287  			}
   288  			if exact {
   289  				if *item.Object == directory {
   290  					uploads = append(uploads, &response.Items[index])
   291  				}
   292  			} else {
   293  				uploads = append(uploads, &response.Items[index])
   294  			}
   295  		}
   296  		if response.OpcNextPage == nil {
   297  			break
   298  		}
   299  		req.Page = response.OpcNextPage
   300  	}
   301  	return uploads, nil
   302  }
   303  
   304  func (f *Fs) listMultipartUploadParts(ctx context.Context, bucketName, bucketPath string, uploadID string) (
   305  	uploadedParts map[int]objectstorage.MultipartUploadPartSummary, err error) {
   306  	uploadedParts = make(map[int]objectstorage.MultipartUploadPartSummary)
   307  	req := objectstorage.ListMultipartUploadPartsRequest{
   308  		NamespaceName: common.String(f.opt.Namespace),
   309  		BucketName:    common.String(bucketName),
   310  		ObjectName:    common.String(bucketPath),
   311  		UploadId:      common.String(uploadID),
   312  		Limit:         common.Int(1000),
   313  	}
   314  
   315  	var response objectstorage.ListMultipartUploadPartsResponse
   316  	for {
   317  		err = f.pacer.Call(func() (bool, error) {
   318  			response, err = f.srv.ListMultipartUploadParts(ctx, req)
   319  			return shouldRetry(ctx, response.HTTPResponse(), err)
   320  		})
   321  		if err != nil {
   322  			return uploadedParts, err
   323  		}
   324  		for _, item := range response.Items {
   325  			uploadedParts[*item.PartNumber] = item
   326  		}
   327  		if response.OpcNextPage == nil {
   328  			break
   329  		}
   330  		req.Page = response.OpcNextPage
   331  	}
   332  	return uploadedParts, nil
   333  }
   334  
   335  func (f *Fs) restore(ctx context.Context, opt map[string]string) (interface{}, error) {
   336  	req := objectstorage.RestoreObjectsRequest{
   337  		NamespaceName:         common.String(f.opt.Namespace),
   338  		RestoreObjectsDetails: objectstorage.RestoreObjectsDetails{},
   339  	}
   340  	if hours := opt["hours"]; hours != "" {
   341  		ihours, err := strconv.Atoi(hours)
   342  		if err != nil {
   343  			return nil, fmt.Errorf("bad value for hours: %w", err)
   344  		}
   345  		req.RestoreObjectsDetails.Hours = &ihours
   346  	}
   347  	type status struct {
   348  		Object string
   349  		Status string
   350  	}
   351  	var (
   352  		outMu sync.Mutex
   353  		out   = []status{}
   354  		err   error
   355  	)
   356  	err = operations.ListFn(ctx, f, func(obj fs.Object) {
   357  		// Remember this is run --checkers times concurrently
   358  		o, ok := obj.(*Object)
   359  		st := status{Object: obj.Remote(), Status: "RESTORED"}
   360  		defer func() {
   361  			outMu.Lock()
   362  			out = append(out, st)
   363  			outMu.Unlock()
   364  		}()
   365  		if !ok {
   366  			st.Status = "Not an OCI Object Storage object"
   367  			return
   368  		}
   369  		if o.storageTier == nil || (*o.storageTier != "archive") {
   370  			st.Status = "Object not in Archive storage tier"
   371  			return
   372  		}
   373  		if operations.SkipDestructive(ctx, obj, "restore") {
   374  			return
   375  		}
   376  		bucket, bucketPath := o.split()
   377  		reqCopy := req
   378  		reqCopy.BucketName = &bucket
   379  		reqCopy.ObjectName = &bucketPath
   380  		var response objectstorage.RestoreObjectsResponse
   381  		err = f.pacer.Call(func() (bool, error) {
   382  			response, err = f.srv.RestoreObjects(ctx, reqCopy)
   383  			return shouldRetry(ctx, response.HTTPResponse(), err)
   384  		})
   385  		if err != nil {
   386  			st.Status = err.Error()
   387  		}
   388  	})
   389  	if err != nil {
   390  		return out, err
   391  	}
   392  	return out, nil
   393  }