github.com/artpar/rclone@v1.67.3/backend/premiumizeme/premiumizeme.go (about)

     1  // Package premiumizeme provides an interface to the premiumize.me
     2  // object storage system.
     3  package premiumizeme
     4  
     5  /*
     6  Run of rclone info
     7  stringNeedsEscaping = []rune{
     8  	0x00, 0x0A, 0x0D, 0x22, 0x2F, 0x5C, 0xBF, 0xFE
     9  	0x00, 0x0A, 0x0D, '"',  '/',  '\\', 0xBF, 0xFE
    10  }
    11  maxFileLength = 255
    12  canWriteUnnormalized = true
    13  canReadUnnormalized   = true
    14  canReadRenormalized   = false
    15  canStream = false
    16  */
    17  
    18  import (
    19  	"context"
    20  	"encoding/json"
    21  	"errors"
    22  	"fmt"
    23  	"io"
    24  	"net"
    25  	"net/http"
    26  	"net/url"
    27  	"path"
    28  	"strings"
    29  	"time"
    30  
    31  	"github.com/artpar/rclone/backend/premiumizeme/api"
    32  	"github.com/artpar/rclone/fs"
    33  	"github.com/artpar/rclone/fs/config"
    34  	"github.com/artpar/rclone/fs/config/configmap"
    35  	"github.com/artpar/rclone/fs/config/configstruct"
    36  	"github.com/artpar/rclone/fs/config/obscure"
    37  	"github.com/artpar/rclone/fs/fserrors"
    38  	"github.com/artpar/rclone/fs/fshttp"
    39  	"github.com/artpar/rclone/fs/hash"
    40  	"github.com/artpar/rclone/lib/dircache"
    41  	"github.com/artpar/rclone/lib/encoder"
    42  	"github.com/artpar/rclone/lib/oauthutil"
    43  	"github.com/artpar/rclone/lib/pacer"
    44  	"github.com/artpar/rclone/lib/random"
    45  	"github.com/artpar/rclone/lib/rest"
    46  	"golang.org/x/oauth2"
    47  )
    48  
    49  const (
    50  	rcloneClientID              = "658922194"
    51  	rcloneEncryptedClientSecret = "B5YIvQoRIhcpAYs8HYeyjb9gK-ftmZEbqdh_gNfc4RgO9Q"
    52  	minSleep                    = 10 * time.Millisecond
    53  	maxSleep                    = 2 * time.Second
    54  	decayConstant               = 2   // bigger for slower decay, exponential
    55  	rootID                      = "0" // ID of root folder is always this
    56  	rootURL                     = "https://www.premiumize.me/api"
    57  )
    58  
    59  // Globals
    60  var (
    61  	// Description of how to auth for this app
    62  	oauthConfig = &oauth2.Config{
    63  		Scopes: nil,
    64  		Endpoint: oauth2.Endpoint{
    65  			AuthURL:  "https://www.premiumize.me/authorize",
    66  			TokenURL: "https://www.premiumize.me/token",
    67  		},
    68  		ClientID:     rcloneClientID,
    69  		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
    70  		RedirectURL:  oauthutil.RedirectURL,
    71  	}
    72  )
    73  
    74  // Register with Fs
    75  func init() {
    76  	fs.Register(&fs.RegInfo{
    77  		Name:        "premiumizeme",
    78  		Description: "premiumize.me",
    79  		NewFs:       NewFs,
    80  		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
    81  			return oauthutil.ConfigOut("", &oauthutil.Options{
    82  				OAuth2Config: oauthConfig,
    83  			})
    84  		},
    85  		Options: append(oauthutil.SharedOptions, []fs.Option{{
    86  			Name: "api_key",
    87  			Help: `API Key.
    88  
    89  This is not normally used - use oauth instead.
    90  `,
    91  			Hide:      fs.OptionHideBoth,
    92  			Default:   "",
    93  			Sensitive: true,
    94  		}, {
    95  			Name:     config.ConfigEncoding,
    96  			Help:     config.ConfigEncodingHelp,
    97  			Advanced: true,
    98  			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
    99  			Default: (encoder.Display |
   100  				encoder.EncodeBackSlash |
   101  				encoder.EncodeDoubleQuote |
   102  				encoder.EncodeInvalidUtf8),
   103  		}}...),
   104  	})
   105  }
   106  
   107  // Options defines the configuration for this backend
   108  type Options struct {
   109  	APIKey string               `config:"api_key"`
   110  	Enc    encoder.MultiEncoder `config:"encoding"`
   111  }
   112  
   113  // Fs represents a remote cloud storage system
   114  type Fs struct {
   115  	name         string             // name of this remote
   116  	root         string             // the path we are working on
   117  	opt          Options            // parsed options
   118  	features     *fs.Features       // optional features
   119  	srv          *rest.Client       // the connection to the server
   120  	dirCache     *dircache.DirCache // Map of directory path to directory id
   121  	pacer        *fs.Pacer          // pacer for API calls
   122  	tokenRenewer *oauthutil.Renew   // renew the token on expiry
   123  }
   124  
   125  // Object describes a file
   126  type Object struct {
   127  	fs          *Fs       // what this object is part of
   128  	remote      string    // The remote path
   129  	hasMetaData bool      // metadata is present and correct
   130  	size        int64     // size of the object
   131  	modTime     time.Time // modification time of the object
   132  	id          string    // ID of the object
   133  	parentID    string    // ID of parent directory
   134  	mimeType    string    // Mime type of object
   135  	url         string    // URL to download file
   136  }
   137  
   138  // ------------------------------------------------------------
   139  
   140  // Name of the remote (as passed into NewFs)
   141  func (f *Fs) Name() string {
   142  	return f.name
   143  }
   144  
   145  // Root of the remote (as passed into NewFs)
   146  func (f *Fs) Root() string {
   147  	return f.root
   148  }
   149  
   150  // String converts this Fs to a string
   151  func (f *Fs) String() string {
   152  	return fmt.Sprintf("premiumize.me root '%s'", f.root)
   153  }
   154  
   155  // Features returns the optional features of this Fs
   156  func (f *Fs) Features() *fs.Features {
   157  	return f.features
   158  }
   159  
   160  // parsePath parses a premiumize.me 'url'
   161  func parsePath(path string) (root string) {
   162  	root = strings.Trim(path, "/")
   163  	return
   164  }
   165  
   166  // retryErrorCodes is a slice of error codes that we will retry
   167  var retryErrorCodes = []int{
   168  	429, // Too Many Requests.
   169  	500, // Internal Server Error
   170  	502, // Bad Gateway
   171  	503, // Service Unavailable
   172  	504, // Gateway Timeout
   173  	509, // Bandwidth Limit Exceeded
   174  }
   175  
   176  // shouldRetry returns a boolean as to whether this resp and err
   177  // deserve to be retried.  It returns the err as a convenience
   178  func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
   179  	if fserrors.ContextError(ctx, &err) {
   180  		return false, err
   181  	}
   182  	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
   183  }
   184  
   185  // readMetaDataForPath reads the metadata from the path
   186  func (f *Fs) readMetaDataForPath(ctx context.Context, path string, directoriesOnly bool, filesOnly bool) (info *api.Item, err error) {
   187  	// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)
   188  	leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)
   189  	if err != nil {
   190  		if err == fs.ErrorDirNotFound {
   191  			return nil, fs.ErrorObjectNotFound
   192  		}
   193  		return nil, err
   194  	}
   195  
   196  	lcLeaf := strings.ToLower(leaf)
   197  	_, found, err := f.listAll(ctx, directoryID, directoriesOnly, filesOnly, func(item *api.Item) bool {
   198  		if strings.ToLower(item.Name) == lcLeaf {
   199  			info = item
   200  			return true
   201  		}
   202  		return false
   203  	})
   204  	if err != nil {
   205  		return nil, err
   206  	}
   207  	if !found {
   208  		return nil, fs.ErrorObjectNotFound
   209  	}
   210  	return info, nil
   211  }
   212  
   213  // errorHandler parses a non 2xx error response into an error
   214  func errorHandler(resp *http.Response) error {
   215  	body, err := rest.ReadBody(resp)
   216  	if err != nil {
   217  		body = nil
   218  	}
   219  	var e = api.Response{
   220  		Message: string(body),
   221  		Status:  fmt.Sprintf("%s (%d)", resp.Status, resp.StatusCode),
   222  	}
   223  	if body != nil {
   224  		_ = json.Unmarshal(body, &e)
   225  	}
   226  	return &e
   227  }
   228  
   229  // Return a url.Values with the api key in
   230  func (f *Fs) baseParams() url.Values {
   231  	params := url.Values{}
   232  	if f.opt.APIKey != "" {
   233  		params.Add("apikey", f.opt.APIKey)
   234  	}
   235  	return params
   236  }
   237  
   238  // NewFs constructs an Fs from the path, container:path
   239  func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
   240  	// Parse config into Options struct
   241  	opt := new(Options)
   242  	err := configstruct.Set(m, opt)
   243  	if err != nil {
   244  		return nil, err
   245  	}
   246  
   247  	root = parsePath(root)
   248  
   249  	var client *http.Client
   250  	var ts *oauthutil.TokenSource
   251  	if opt.APIKey == "" {
   252  		client, ts, err = oauthutil.NewClient(ctx, name, m, oauthConfig)
   253  		if err != nil {
   254  			return nil, fmt.Errorf("failed to configure premiumize.me: %w", err)
   255  		}
   256  	} else {
   257  		client = fshttp.NewClient(ctx)
   258  	}
   259  
   260  	f := &Fs{
   261  		name:  name,
   262  		root:  root,
   263  		opt:   *opt,
   264  		srv:   rest.NewClient(client).SetRoot(rootURL),
   265  		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
   266  	}
   267  	f.features = (&fs.Features{
   268  		CaseInsensitive:         true,
   269  		CanHaveEmptyDirectories: true,
   270  		ReadMimeType:            true,
   271  	}).Fill(ctx, f)
   272  	f.srv.SetErrorHandler(errorHandler)
   273  
   274  	// Renew the token in the background
   275  	if ts != nil {
   276  		f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
   277  			_, err := f.About(ctx)
   278  			return err
   279  		})
   280  	}
   281  
   282  	// Get rootID
   283  	f.dirCache = dircache.New(root, rootID, f)
   284  
   285  	// Find the current root
   286  	err = f.dirCache.FindRoot(ctx, false)
   287  	if err != nil {
   288  		// Assume it is a file
   289  		newRoot, remote := dircache.SplitPath(root)
   290  		tempF := *f
   291  		tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
   292  		tempF.root = newRoot
   293  		// Make new Fs which is the parent
   294  		err = tempF.dirCache.FindRoot(ctx, false)
   295  		if err != nil {
   296  			// No root so return old f
   297  			return f, nil
   298  		}
   299  		_, err := tempF.newObjectWithInfo(ctx, remote, nil)
   300  		if err != nil {
   301  			if err == fs.ErrorObjectNotFound {
   302  				// File doesn't exist so return old f
   303  				return f, nil
   304  			}
   305  			return nil, err
   306  		}
   307  		f.features.Fill(ctx, &tempF)
   308  		// XXX: update the old f here instead of returning tempF, since
   309  		// `features` were already filled with functions having *f as a receiver.
   310  		// See https://github.com/artpar/rclone/issues/2182
   311  		f.dirCache = tempF.dirCache
   312  		f.root = tempF.root
   313  		// return an error with an fs which points to the parent
   314  		return f, fs.ErrorIsFile
   315  	}
   316  	return f, nil
   317  }
   318  
   319  // Return an Object from a path
   320  //
   321  // If it can't be found it returns the error fs.ErrorObjectNotFound.
   322  func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.Item) (fs.Object, error) {
   323  	o := &Object{
   324  		fs:     f,
   325  		remote: remote,
   326  	}
   327  	var err error
   328  	if info != nil {
   329  		// Set info
   330  		err = o.setMetaData(info)
   331  	} else {
   332  		err = o.readMetaData(ctx) // reads info and meta, returning an error
   333  	}
   334  	if err != nil {
   335  		return nil, err
   336  	}
   337  	return o, nil
   338  }
   339  
   340  // NewObject finds the Object at remote.  If it can't be found
   341  // it returns the error fs.ErrorObjectNotFound.
   342  func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
   343  	return f.newObjectWithInfo(ctx, remote, nil)
   344  }
   345  
   346  // FindLeaf finds a directory of name leaf in the folder with ID pathID
   347  func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
   348  	// Find the leaf in pathID
   349  	var newDirID string
   350  	newDirID, found, err = f.listAll(ctx, pathID, true, false, func(item *api.Item) bool {
   351  		if strings.EqualFold(item.Name, leaf) {
   352  			pathIDOut = item.ID
   353  			return true
   354  		}
   355  		return false
   356  	})
   357  	// Update the Root directory ID to its actual value
   358  	if pathID == rootID {
   359  		f.dirCache.SetRootIDAlias(newDirID)
   360  	}
   361  	return pathIDOut, found, err
   362  }
   363  
   364  // CreateDir makes a directory with pathID as parent and name leaf
   365  func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
   366  	// fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf)
   367  	var resp *http.Response
   368  	var info api.FolderCreateResponse
   369  	opts := rest.Opts{
   370  		Method:     "POST",
   371  		Path:       "/folder/create",
   372  		Parameters: f.baseParams(),
   373  		MultipartParams: url.Values{
   374  			"name":      {f.opt.Enc.FromStandardName(leaf)},
   375  			"parent_id": {pathID},
   376  		},
   377  	}
   378  	err = f.pacer.Call(func() (bool, error) {
   379  		resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
   380  		return shouldRetry(ctx, resp, err)
   381  	})
   382  	if err != nil {
   383  		//fmt.Printf("...Error %v\n", err)
   384  		return "", fmt.Errorf("CreateDir http: %w", err)
   385  	}
   386  	if err = info.AsErr(); err != nil {
   387  		return "", fmt.Errorf("CreateDir: %w", err)
   388  	}
   389  	// fmt.Printf("...Id %q\n", *info.Id)
   390  	return info.ID, nil
   391  }
   392  
   393  // list the objects into the function supplied
   394  //
   395  // If directories is set it only sends directories
   396  // User function to process a File item from listAll
   397  //
   398  // Should return true to finish processing
   399  type listAllFn func(*api.Item) bool
   400  
   401  // Lists the directory required calling the user function on each item found
   402  //
   403  // If the user fn ever returns true then it early exits with found = true
   404  //
   405  // It returns a newDirID which is what the system returned as the directory ID
   406  func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (newDirID string, found bool, err error) {
   407  	opts := rest.Opts{
   408  		Method:     "GET",
   409  		Path:       "/folder/list",
   410  		Parameters: f.baseParams(),
   411  	}
   412  	if dirID != rootID {
   413  		opts.Parameters.Set("id", dirID)
   414  	}
   415  	opts.Parameters.Set("includebreadcrumbs", "false")
   416  
   417  	var result api.FolderListResponse
   418  	var resp *http.Response
   419  	err = f.pacer.Call(func() (bool, error) {
   420  		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
   421  		return shouldRetry(ctx, resp, err)
   422  	})
   423  	if err != nil {
   424  		return newDirID, found, fmt.Errorf("couldn't list files: %w", err)
   425  	}
   426  	if err = result.AsErr(); err != nil {
   427  		return newDirID, found, fmt.Errorf("error while listing: %w", err)
   428  	}
   429  	newDirID = result.FolderID
   430  	for i := range result.Content {
   431  		item := &result.Content[i]
   432  		if item.Type == api.ItemTypeFolder {
   433  			if filesOnly {
   434  				continue
   435  			}
   436  		} else if item.Type == api.ItemTypeFile {
   437  			if directoriesOnly {
   438  				continue
   439  			}
   440  		} else {
   441  			fs.Debugf(f, "Ignoring %q - unknown type %q", item.Name, item.Type)
   442  			continue
   443  		}
   444  		item.Name = f.opt.Enc.ToStandardName(item.Name)
   445  		if fn(item) {
   446  			found = true
   447  			break
   448  		}
   449  	}
   450  	return
   451  }
   452  
   453  // List the objects and directories in dir into entries.  The
   454  // entries can be returned in any order but should be for a
   455  // complete directory.
   456  //
   457  // dir should be "" to list the root, and should not have
   458  // trailing slashes.
   459  //
   460  // This should return ErrDirNotFound if the directory isn't
   461  // found.
   462  func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
   463  	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
   464  	if err != nil {
   465  		return nil, err
   466  	}
   467  	var iErr error
   468  	_, _, err = f.listAll(ctx, directoryID, false, false, func(info *api.Item) bool {
   469  		remote := path.Join(dir, info.Name)
   470  		if info.Type == api.ItemTypeFolder {
   471  			// cache the directory ID for later lookups
   472  			f.dirCache.Put(remote, info.ID)
   473  			d := fs.NewDir(remote, time.Unix(info.CreatedAt, 0)).SetID(info.ID)
   474  			entries = append(entries, d)
   475  		} else if info.Type == api.ItemTypeFile {
   476  			o, err := f.newObjectWithInfo(ctx, remote, info)
   477  			if err != nil {
   478  				iErr = err
   479  				return true
   480  			}
   481  			entries = append(entries, o)
   482  		}
   483  		return false
   484  	})
   485  	if err != nil {
   486  		return nil, err
   487  	}
   488  	if iErr != nil {
   489  		return nil, iErr
   490  	}
   491  	return entries, nil
   492  }
   493  
   494  // Creates from the parameters passed in a half finished Object which
   495  // must have setMetaData called on it
   496  //
   497  // Returns the object, leaf, directoryID and error.
   498  //
   499  // Used to create new objects
   500  func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
   501  	// Create the directory for the object if it doesn't exist
   502  	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
   503  	if err != nil {
   504  		return
   505  	}
   506  	// Temporary Object under construction
   507  	o = &Object{
   508  		fs:     f,
   509  		remote: remote,
   510  	}
   511  	return o, leaf, directoryID, nil
   512  }
   513  
   514  // Put the object
   515  //
   516  // Copy the reader in to the new object which is returned.
   517  //
   518  // The new object may have been created if an error is returned
   519  func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
   520  	existingObj, err := f.newObjectWithInfo(ctx, src.Remote(), nil)
   521  	switch err {
   522  	case nil:
   523  		return existingObj, existingObj.Update(ctx, in, src, options...)
   524  	case fs.ErrorObjectNotFound:
   525  		// Not found so create it
   526  		return f.PutUnchecked(ctx, in, src, options...)
   527  	default:
   528  		return nil, err
   529  	}
   530  }
   531  
   532  // PutUnchecked the object into the container
   533  //
   534  // This will produce an error if the object already exists.
   535  //
   536  // Copy the reader in to the new object which is returned.
   537  //
   538  // The new object may have been created if an error is returned
   539  func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
   540  	remote := src.Remote()
   541  	size := src.Size()
   542  	modTime := src.ModTime(ctx)
   543  
   544  	o, _, _, err := f.createObject(ctx, remote, modTime, size)
   545  	if err != nil {
   546  		return nil, err
   547  	}
   548  	return o, o.Update(ctx, in, src, options...)
   549  }
   550  
   551  // Mkdir creates the container if it doesn't exist
   552  func (f *Fs) Mkdir(ctx context.Context, dir string) error {
   553  	_, err := f.dirCache.FindDir(ctx, dir, true)
   554  	return err
   555  }
   556  
   557  // purgeCheck removes the root directory, if check is set then it
   558  // refuses to do so if it has anything in
   559  func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
   560  	root := path.Join(f.root, dir)
   561  	if root == "" {
   562  		return errors.New("can't purge root directory")
   563  	}
   564  	dc := f.dirCache
   565  	rootID, err := dc.FindDir(ctx, dir, false)
   566  	if err != nil {
   567  		return err
   568  	}
   569  
   570  	// need to check if empty as it will delete recursively by default
   571  	if check {
   572  		_, found, err := f.listAll(ctx, rootID, false, false, func(item *api.Item) bool {
   573  			return true
   574  		})
   575  		if err != nil {
   576  			return fmt.Errorf("purgeCheck: %w", err)
   577  		}
   578  		if found {
   579  			return fs.ErrorDirectoryNotEmpty
   580  		}
   581  	}
   582  
   583  	opts := rest.Opts{
   584  		Method: "POST",
   585  		Path:   "/folder/delete",
   586  		MultipartParams: url.Values{
   587  			"id": {rootID},
   588  		},
   589  		Parameters: f.baseParams(),
   590  	}
   591  	var resp *http.Response
   592  	var result api.Response
   593  	err = f.pacer.Call(func() (bool, error) {
   594  		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
   595  		return shouldRetry(ctx, resp, err)
   596  	})
   597  	if err != nil {
   598  		return fmt.Errorf("rmdir failed: %w", err)
   599  	}
   600  	if err = result.AsErr(); err != nil {
   601  		return fmt.Errorf("rmdir: %w", err)
   602  	}
   603  	f.dirCache.FlushDir(dir)
   604  	if err != nil {
   605  		return err
   606  	}
   607  	return nil
   608  }
   609  
   610  // Rmdir deletes the root folder
   611  //
   612  // Returns an error if it isn't empty
   613  func (f *Fs) Rmdir(ctx context.Context, dir string) error {
   614  	return f.purgeCheck(ctx, dir, true)
   615  }
   616  
   617  // Precision return the precision of this Fs
   618  func (f *Fs) Precision() time.Duration {
   619  	return fs.ModTimeNotSupported
   620  }
   621  
   622  // Purge deletes all the files in the directory
   623  //
   624  // Optional interface: Only implement this if you have a way of
   625  // deleting all the files quicker than just running Remove() on the
   626  // result of List()
   627  func (f *Fs) Purge(ctx context.Context, dir string) error {
   628  	return f.purgeCheck(ctx, dir, false)
   629  }
   630  
   631  // move a file or folder
   632  //
   633  // This is complicated by the fact that there is an API to move files
   634  // between directories and a separate one to rename them.  We try to
   635  // call the minimum number of API calls.
   636  func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDirectoryID, newDirectoryID string) (err error) {
   637  	newLeaf = f.opt.Enc.FromStandardName(newLeaf)
   638  	oldLeaf = f.opt.Enc.FromStandardName(oldLeaf)
   639  	doRenameLeaf := oldLeaf != newLeaf
   640  	doMove := oldDirectoryID != newDirectoryID
   641  
   642  	// Now rename the leaf to a temporary name if we are moving to
   643  	// another directory to make sure we don't overwrite something
   644  	// in the destination directory by accident
   645  	if doRenameLeaf && doMove {
   646  		tmpLeaf := newLeaf + "." + random.String(8)
   647  		err = f.renameLeaf(ctx, isFile, id, tmpLeaf)
   648  		if err != nil {
   649  			return fmt.Errorf("Move rename leaf: %w", err)
   650  		}
   651  	}
   652  
   653  	// Move the object to a new directory (with the existing name)
   654  	// if required
   655  	if doMove {
   656  		opts := rest.Opts{
   657  			Method:     "POST",
   658  			Path:       "/folder/paste",
   659  			Parameters: f.baseParams(),
   660  			MultipartParams: url.Values{
   661  				"id": {newDirectoryID},
   662  			},
   663  		}
   664  		opts.MultipartParams.Set("items[0][id]", id)
   665  		if isFile {
   666  			opts.MultipartParams.Set("items[0][type]", "file")
   667  		} else {
   668  			opts.MultipartParams.Set("items[0][type]", "folder")
   669  		}
   670  		//replacedLeaf := enc.FromStandardName(leaf)
   671  		var resp *http.Response
   672  		var result api.Response
   673  		err = f.pacer.Call(func() (bool, error) {
   674  			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
   675  			return shouldRetry(ctx, resp, err)
   676  		})
   677  		if err != nil {
   678  			return fmt.Errorf("Move http: %w", err)
   679  		}
   680  		if err = result.AsErr(); err != nil {
   681  			return fmt.Errorf("Move: %w", err)
   682  		}
   683  	}
   684  
   685  	// Rename the leaf to its final name if required
   686  	if doRenameLeaf {
   687  		err = f.renameLeaf(ctx, isFile, id, newLeaf)
   688  		if err != nil {
   689  			return fmt.Errorf("Move rename leaf: %w", err)
   690  		}
   691  	}
   692  
   693  	return nil
   694  }
   695  
   696  // Move src to this remote using server-side move operations.
   697  //
   698  // This is stored with the remote path given.
   699  //
   700  // It returns the destination Object and a possible error.
   701  //
   702  // Will only be called if src.Fs().Name() == f.Name()
   703  //
   704  // If it isn't possible then return fs.ErrorCantMove
   705  func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
   706  	srcObj, ok := src.(*Object)
   707  	if !ok {
   708  		fs.Debugf(src, "Can't move - not same remote type")
   709  		return nil, fs.ErrorCantMove
   710  	}
   711  
   712  	// Create temporary object
   713  	dstObj, leaf, directoryID, err := f.createObject(ctx, remote, srcObj.modTime, srcObj.size)
   714  	if err != nil {
   715  		return nil, err
   716  	}
   717  
   718  	// Do the move
   719  	err = f.move(ctx, true, srcObj.id, path.Base(srcObj.remote), leaf, srcObj.parentID, directoryID)
   720  	if err != nil {
   721  		return nil, err
   722  	}
   723  
   724  	err = dstObj.readMetaData(ctx)
   725  	if err != nil {
   726  		return nil, err
   727  	}
   728  	return dstObj, nil
   729  }
   730  
   731  // DirMove moves src, srcRemote to this remote at dstRemote
   732  // using server-side move operations.
   733  //
   734  // Will only be called if src.Fs().Name() == f.Name()
   735  //
   736  // If it isn't possible then return fs.ErrorCantDirMove
   737  //
   738  // If destination exists then return fs.ErrorDirExists
   739  func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
   740  	srcFs, ok := src.(*Fs)
   741  	if !ok {
   742  		fs.Debugf(srcFs, "Can't move directory - not same remote type")
   743  		return fs.ErrorCantDirMove
   744  	}
   745  
   746  	srcID, srcDirectoryID, srcLeaf, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
   747  	if err != nil {
   748  		return err
   749  	}
   750  
   751  	// Do the move
   752  	err = f.move(ctx, false, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID)
   753  	if err != nil {
   754  		return err
   755  	}
   756  	srcFs.dirCache.FlushDir(srcRemote)
   757  	return nil
   758  }
   759  
   760  // PublicLink adds a "readable by anyone with link" permission on the given file or folder.
   761  func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
   762  	_, err := f.dirCache.FindDir(ctx, remote, false)
   763  	if err == nil {
   764  		return "", fs.ErrorCantShareDirectories
   765  	}
   766  	o, err := f.NewObject(ctx, remote)
   767  	if err != nil {
   768  		return "", err
   769  	}
   770  	return o.(*Object).url, nil
   771  }
   772  
   773  // Shutdown shutdown the fs
   774  func (f *Fs) Shutdown(ctx context.Context) error {
   775  	f.tokenRenewer.Shutdown()
   776  	return nil
   777  }
   778  
   779  // About gets quota information
   780  func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
   781  	var resp *http.Response
   782  	var info api.AccountInfoResponse
   783  	opts := rest.Opts{
   784  		Method:     "POST",
   785  		Path:       "/account/info",
   786  		Parameters: f.baseParams(),
   787  	}
   788  	err = f.pacer.Call(func() (bool, error) {
   789  		resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
   790  		return shouldRetry(ctx, resp, err)
   791  	})
   792  	if err != nil {
   793  		return nil, err
   794  	}
   795  	if err = info.AsErr(); err != nil {
   796  		return nil, err
   797  	}
   798  	usage = &fs.Usage{
   799  		Used: fs.NewUsageValue(int64(info.SpaceUsed)),
   800  	}
   801  	return usage, nil
   802  }
   803  
   804  // DirCacheFlush resets the directory cache - used in testing as an
   805  // optional interface
   806  func (f *Fs) DirCacheFlush() {
   807  	f.dirCache.ResetRoot()
   808  }
   809  
   810  // Hashes returns the supported hash sets.
   811  func (f *Fs) Hashes() hash.Set {
   812  	return hash.Set(hash.None)
   813  }
   814  
   815  // ------------------------------------------------------------
   816  
   817  // Fs returns the parent Fs
   818  func (o *Object) Fs() fs.Info {
   819  	return o.fs
   820  }
   821  
   822  // Return a string version
   823  func (o *Object) String() string {
   824  	if o == nil {
   825  		return "<nil>"
   826  	}
   827  	return o.remote
   828  }
   829  
   830  // Remote returns the remote path
   831  func (o *Object) Remote() string {
   832  	return o.remote
   833  }
   834  
   835  // Hash returns the SHA-1 of an object returning a lowercase hex string
   836  func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
   837  	return "", hash.ErrUnsupported
   838  }
   839  
   840  // Size returns the size of an object in bytes
   841  func (o *Object) Size() int64 {
   842  	err := o.readMetaData(context.TODO())
   843  	if err != nil {
   844  		fs.Logf(o, "Failed to read metadata: %v", err)
   845  		return 0
   846  	}
   847  	return o.size
   848  }
   849  
   850  // setMetaData sets the metadata from info
   851  func (o *Object) setMetaData(info *api.Item) (err error) {
   852  	if info.Type != "file" {
   853  		return fmt.Errorf("%q is %q: %w", o.remote, info.Type, fs.ErrorNotAFile)
   854  	}
   855  	o.hasMetaData = true
   856  	o.size = info.Size
   857  	o.modTime = time.Unix(info.CreatedAt, 0)
   858  	o.id = info.ID
   859  	o.mimeType = info.MimeType
   860  	o.url = info.Link
   861  	return nil
   862  }
   863  
   864  // readMetaData gets the metadata if it hasn't already been fetched
   865  //
   866  // it also sets the info
   867  func (o *Object) readMetaData(ctx context.Context) (err error) {
   868  	if o.hasMetaData {
   869  		return nil
   870  	}
   871  	info, err := o.fs.readMetaDataForPath(ctx, o.remote, false, true)
   872  	if err != nil {
   873  		return err
   874  	}
   875  	return o.setMetaData(info)
   876  }
   877  
   878  // ModTime returns the modification time of the object
   879  //
   880  // It attempts to read the objects mtime and if that isn't present the
   881  // LastModified returned in the http headers
   882  func (o *Object) ModTime(ctx context.Context) time.Time {
   883  	err := o.readMetaData(ctx)
   884  	if err != nil {
   885  		fs.Logf(o, "Failed to read metadata: %v", err)
   886  		return time.Now()
   887  	}
   888  	return o.modTime
   889  }
   890  
   891  // SetModTime sets the modification time of the local fs object
   892  func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
   893  	return fs.ErrorCantSetModTime
   894  }
   895  
   896  // Storable returns a boolean showing whether this object storable
   897  func (o *Object) Storable() bool {
   898  	return true
   899  }
   900  
   901  // Open an object for read
   902  func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
   903  	if o.url == "" {
   904  		return nil, errors.New("can't download - no URL")
   905  	}
   906  	fs.FixRangeOption(options, o.size)
   907  	var resp *http.Response
   908  	opts := rest.Opts{
   909  		Path:    "",
   910  		RootURL: o.url,
   911  		Method:  "GET",
   912  		Options: options,
   913  	}
   914  	err = o.fs.pacer.Call(func() (bool, error) {
   915  		resp, err = o.fs.srv.Call(ctx, &opts)
   916  		return shouldRetry(ctx, resp, err)
   917  	})
   918  	if err != nil {
   919  		return nil, err
   920  	}
   921  	return resp.Body, err
   922  }
   923  
   924  // Update the object with the contents of the io.Reader, modTime and size
   925  //
   926  // If existing is set then it updates the object rather than creating a new one.
   927  //
   928  // The new object may have been created if an error is returned
   929  func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
   930  	remote := o.Remote()
   931  	size := src.Size()
   932  
   933  	// Create the directory for the object if it doesn't exist
   934  	leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true)
   935  	if err != nil {
   936  		return err
   937  	}
   938  	leaf = o.fs.opt.Enc.FromStandardName(leaf)
   939  
   940  	var resp *http.Response
   941  	var info api.FolderUploadinfoResponse
   942  	opts := rest.Opts{
   943  		Method:     "POST",
   944  		Path:       "/folder/uploadinfo",
   945  		Parameters: o.fs.baseParams(),
   946  		Options:    options,
   947  		MultipartParams: url.Values{
   948  			"id": {directoryID},
   949  		},
   950  	}
   951  	err = o.fs.pacer.Call(func() (bool, error) {
   952  		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info)
   953  		if err != nil {
   954  			return shouldRetry(ctx, resp, err)
   955  		}
   956  		// Just check the download URL resolves - sometimes
   957  		// the URLs returned by premiumize.me don't resolve so
   958  		// this needs a retry.
   959  		var u *url.URL
   960  		u, err = url.Parse(info.URL)
   961  		if err != nil {
   962  			return true, fmt.Errorf("failed to parse download URL: %w", err)
   963  		}
   964  		_, err = net.LookupIP(u.Hostname())
   965  		if err != nil {
   966  			return true, fmt.Errorf("failed to resolve download URL: %w", err)
   967  		}
   968  		return false, nil
   969  	})
   970  	if err != nil {
   971  		return fmt.Errorf("upload get URL http: %w", err)
   972  	}
   973  	if err = info.AsErr(); err != nil {
   974  		return fmt.Errorf("upload get URL: %w", err)
   975  	}
   976  
   977  	// if file exists then rename it out the way otherwise uploads can fail
   978  	uploaded := false
   979  	var oldID = o.id
   980  	if o.hasMetaData {
   981  		newLeaf := leaf + "." + random.String(8)
   982  		fs.Debugf(o, "Moving old file out the way to %q", newLeaf)
   983  		err = o.fs.renameLeaf(ctx, true, oldID, newLeaf)
   984  		if err != nil {
   985  			return fmt.Errorf("upload rename old file: %w", err)
   986  		}
   987  		defer func() {
   988  			// on failed upload rename old file back
   989  			if !uploaded {
   990  				fs.Debugf(o, "Renaming old file back (from %q to %q) since upload failed", leaf, newLeaf)
   991  				newErr := o.fs.renameLeaf(ctx, true, oldID, leaf)
   992  				if newErr != nil && err == nil {
   993  					err = fmt.Errorf("upload renaming old file back: %w", newErr)
   994  				}
   995  			}
   996  		}()
   997  	}
   998  
   999  	opts = rest.Opts{
  1000  		Method:  "POST",
  1001  		RootURL: info.URL,
  1002  		Body:    in,
  1003  		MultipartParams: url.Values{
  1004  			"token": {info.Token},
  1005  		},
  1006  		MultipartContentName: "file", // ..name of the parameter which is the attached file
  1007  		MultipartFileName:    leaf,   // ..name of the file for the attached file
  1008  		ContentLength:        &size,
  1009  	}
  1010  	var result api.Response
  1011  	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
  1012  		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result)
  1013  		return shouldRetry(ctx, resp, err)
  1014  	})
  1015  	if err != nil {
  1016  		return fmt.Errorf("upload file http: %w", err)
  1017  	}
  1018  	if err = result.AsErr(); err != nil {
  1019  		return fmt.Errorf("upload file: %w", err)
  1020  	}
  1021  
  1022  	// on successful upload, remove old file if it exists
  1023  	uploaded = true
  1024  	if o.hasMetaData {
  1025  		fs.Debugf(o, "Removing old file")
  1026  		err := o.fs.remove(ctx, oldID)
  1027  		if err != nil {
  1028  			return fmt.Errorf("upload remove old file: %w", err)
  1029  		}
  1030  	}
  1031  
  1032  	o.hasMetaData = false
  1033  	return o.readMetaData(ctx)
  1034  }
  1035  
  1036  // Rename the leaf of a file or directory in a directory
  1037  func (f *Fs) renameLeaf(ctx context.Context, isFile bool, id string, newLeaf string) (err error) {
  1038  	opts := rest.Opts{
  1039  		Method: "POST",
  1040  		MultipartParams: url.Values{
  1041  			"id":   {id},
  1042  			"name": {newLeaf},
  1043  		},
  1044  		Parameters: f.baseParams(),
  1045  	}
  1046  	if isFile {
  1047  		opts.Path = "/item/rename"
  1048  	} else {
  1049  		opts.Path = "/folder/rename"
  1050  	}
  1051  	var resp *http.Response
  1052  	var result api.Response
  1053  	err = f.pacer.Call(func() (bool, error) {
  1054  		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
  1055  		return shouldRetry(ctx, resp, err)
  1056  	})
  1057  	if err != nil {
  1058  		return fmt.Errorf("rename http: %w", err)
  1059  	}
  1060  	if err = result.AsErr(); err != nil {
  1061  		return fmt.Errorf("rename: %w", err)
  1062  	}
  1063  	return nil
  1064  }
  1065  
  1066  // Remove an object by ID
  1067  func (f *Fs) remove(ctx context.Context, id string) (err error) {
  1068  	opts := rest.Opts{
  1069  		Method: "POST",
  1070  		Path:   "/item/delete",
  1071  		MultipartParams: url.Values{
  1072  			"id": {id},
  1073  		},
  1074  		Parameters: f.baseParams(),
  1075  	}
  1076  	var resp *http.Response
  1077  	var result api.Response
  1078  	err = f.pacer.Call(func() (bool, error) {
  1079  		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
  1080  		return shouldRetry(ctx, resp, err)
  1081  	})
  1082  	if err != nil {
  1083  		return fmt.Errorf("remove http: %w", err)
  1084  	}
  1085  	if err = result.AsErr(); err != nil {
  1086  		return fmt.Errorf("remove: %w", err)
  1087  	}
  1088  	return nil
  1089  }
  1090  
  1091  // Remove an object
  1092  func (o *Object) Remove(ctx context.Context) error {
  1093  	err := o.readMetaData(ctx)
  1094  	if err != nil {
  1095  		return fmt.Errorf("Remove: Failed to read metadata: %w", err)
  1096  	}
  1097  	return o.fs.remove(ctx, o.id)
  1098  }
  1099  
  1100  // MimeType of an Object if known, "" otherwise
  1101  func (o *Object) MimeType(ctx context.Context) string {
  1102  	return o.mimeType
  1103  }
  1104  
  1105  // ID returns the ID of the Object if known, or "" if not
  1106  func (o *Object) ID() string {
  1107  	return o.id
  1108  }
  1109  
  1110  // Check the interfaces are satisfied
  1111  var (
  1112  	_ fs.Fs              = (*Fs)(nil)
  1113  	_ fs.Purger          = (*Fs)(nil)
  1114  	_ fs.Mover           = (*Fs)(nil)
  1115  	_ fs.DirMover        = (*Fs)(nil)
  1116  	_ fs.DirCacheFlusher = (*Fs)(nil)
  1117  	_ fs.Abouter         = (*Fs)(nil)
  1118  	_ fs.PublicLinker    = (*Fs)(nil)
  1119  	_ fs.Shutdowner      = (*Fs)(nil)
  1120  	_ fs.Object          = (*Object)(nil)
  1121  	_ fs.MimeTyper       = (*Object)(nil)
  1122  	_ fs.IDer            = (*Object)(nil)
  1123  )