github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/backend/zoho/zoho.go (about)

     1  // Package zoho provides an interface to the Zoho Workdrive
     2  // storage system.
     3  package zoho
     4  
     5  import (
     6  	"context"
     7  	"errors"
     8  	"fmt"
     9  	"io"
    10  	"net/http"
    11  	"net/url"
    12  	"path"
    13  	"strconv"
    14  	"strings"
    15  	"time"
    16  
    17  	"github.com/rclone/rclone/lib/encoder"
    18  	"github.com/rclone/rclone/lib/pacer"
    19  	"github.com/rclone/rclone/lib/random"
    20  
    21  	"github.com/rclone/rclone/backend/zoho/api"
    22  	"github.com/rclone/rclone/fs"
    23  	"github.com/rclone/rclone/fs/config"
    24  	"github.com/rclone/rclone/fs/config/configmap"
    25  	"github.com/rclone/rclone/fs/config/configstruct"
    26  	"github.com/rclone/rclone/fs/config/obscure"
    27  	"github.com/rclone/rclone/fs/fserrors"
    28  	"github.com/rclone/rclone/fs/hash"
    29  	"github.com/rclone/rclone/lib/dircache"
    30  	"github.com/rclone/rclone/lib/oauthutil"
    31  	"github.com/rclone/rclone/lib/rest"
    32  	"golang.org/x/oauth2"
    33  )
    34  
    35  const (
    36  	rcloneClientID              = "1000.46MXF275FM2XV7QCHX5A7K3LGME66B"
    37  	rcloneEncryptedClientSecret = "U-2gxclZQBcOG9NPhjiXAhj-f0uQ137D0zar8YyNHXHkQZlTeSpIOQfmCb4oSpvosJp_SJLXmLLeUA"
    38  	minSleep                    = 10 * time.Millisecond
    39  	maxSleep                    = 2 * time.Second
    40  	decayConstant               = 2 // bigger for slower decay, exponential
    41  	configRootID                = "root_folder_id"
    42  )
    43  
    44  // Globals
    45  var (
    46  	// Description of how to auth for this app
    47  	oauthConfig = &oauth2.Config{
    48  		Scopes: []string{
    49  			"aaaserver.profile.read",
    50  			"WorkDrive.team.READ",
    51  			"WorkDrive.workspace.READ",
    52  			"WorkDrive.files.ALL",
    53  		},
    54  		Endpoint: oauth2.Endpoint{
    55  			AuthURL:   "https://accounts.zoho.eu/oauth/v2/auth",
    56  			TokenURL:  "https://accounts.zoho.eu/oauth/v2/token",
    57  			AuthStyle: oauth2.AuthStyleInParams,
    58  		},
    59  		ClientID:     rcloneClientID,
    60  		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
    61  		RedirectURL:  oauthutil.RedirectLocalhostURL,
    62  	}
    63  	rootURL     = "https://workdrive.zoho.eu/api/v1"
    64  	accountsURL = "https://accounts.zoho.eu"
    65  )
    66  
    67  // Register with Fs
    68  func init() {
    69  	fs.Register(&fs.RegInfo{
    70  		Name:        "zoho",
    71  		Description: "Zoho",
    72  		NewFs:       NewFs,
    73  		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
    74  			// Need to setup region before configuring oauth
    75  			err := setupRegion(m)
    76  			if err != nil {
    77  				return nil, err
    78  			}
    79  			getSrvs := func() (authSrv, apiSrv *rest.Client, err error) {
    80  				oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
    81  				if err != nil {
    82  					return nil, nil, fmt.Errorf("failed to load oAuthClient: %w", err)
    83  				}
    84  				authSrv = rest.NewClient(oAuthClient).SetRoot(accountsURL)
    85  				apiSrv = rest.NewClient(oAuthClient).SetRoot(rootURL)
    86  				return authSrv, apiSrv, nil
    87  			}
    88  
    89  			switch config.State {
    90  			case "":
    91  				return oauthutil.ConfigOut("teams", &oauthutil.Options{
    92  					OAuth2Config: oauthConfig,
    93  					// No refresh token unless ApprovalForce is set
    94  					OAuth2Opts: []oauth2.AuthCodeOption{oauth2.ApprovalForce},
    95  				})
    96  			case "teams":
    97  				// We need to rewrite the token type to "Zoho-oauthtoken" because Zoho wants
    98  				// it's own custom type
    99  				token, err := oauthutil.GetToken(name, m)
   100  				if err != nil {
   101  					return nil, fmt.Errorf("failed to read token: %w", err)
   102  				}
   103  				if token.TokenType != "Zoho-oauthtoken" {
   104  					token.TokenType = "Zoho-oauthtoken"
   105  					err = oauthutil.PutToken(name, m, token, false)
   106  					if err != nil {
   107  						return nil, fmt.Errorf("failed to configure token: %w", err)
   108  					}
   109  				}
   110  
   111  				authSrv, apiSrv, err := getSrvs()
   112  				if err != nil {
   113  					return nil, err
   114  				}
   115  
   116  				// Get the user Info
   117  				opts := rest.Opts{
   118  					Method: "GET",
   119  					Path:   "/oauth/user/info",
   120  				}
   121  				var user api.User
   122  				_, err = authSrv.CallJSON(ctx, &opts, nil, &user)
   123  				if err != nil {
   124  					return nil, err
   125  				}
   126  
   127  				// Get the teams
   128  				teams, err := listTeams(ctx, user.ZUID, apiSrv)
   129  				if err != nil {
   130  					return nil, err
   131  				}
   132  				return fs.ConfigChoose("workspace", "config_team_drive_id", "Team Drive ID", len(teams), func(i int) (string, string) {
   133  					team := teams[i]
   134  					return team.ID, team.Attributes.Name
   135  				})
   136  			case "workspace":
   137  				_, apiSrv, err := getSrvs()
   138  				if err != nil {
   139  					return nil, err
   140  				}
   141  				teamID := config.Result
   142  				workspaces, err := listWorkspaces(ctx, teamID, apiSrv)
   143  				if err != nil {
   144  					return nil, err
   145  				}
   146  				return fs.ConfigChoose("workspace_end", "config_workspace", "Workspace ID", len(workspaces), func(i int) (string, string) {
   147  					workspace := workspaces[i]
   148  					return workspace.ID, workspace.Attributes.Name
   149  				})
   150  			case "workspace_end":
   151  				workspaceID := config.Result
   152  				m.Set(configRootID, workspaceID)
   153  				return nil, nil
   154  			}
   155  			return nil, fmt.Errorf("unknown state %q", config.State)
   156  		},
   157  		Options: append(oauthutil.SharedOptions, []fs.Option{{
   158  			Name: "region",
   159  			Help: `Zoho region to connect to.
   160  
   161  You'll have to use the region your organization is registered in. If
   162  not sure use the same top level domain as you connect to in your
   163  browser.`,
   164  			Examples: []fs.OptionExample{{
   165  				Value: "com",
   166  				Help:  "United states / Global",
   167  			}, {
   168  				Value: "eu",
   169  				Help:  "Europe",
   170  			}, {
   171  				Value: "in",
   172  				Help:  "India",
   173  			}, {
   174  				Value: "jp",
   175  				Help:  "Japan",
   176  			}, {
   177  				Value: "com.cn",
   178  				Help:  "China",
   179  			}, {
   180  				Value: "com.au",
   181  				Help:  "Australia",
   182  			}}}, {
   183  			Name:     config.ConfigEncoding,
   184  			Help:     config.ConfigEncodingHelp,
   185  			Advanced: true,
   186  			Default: (encoder.EncodeZero |
   187  				encoder.EncodeCtl |
   188  				encoder.EncodeDel |
   189  				encoder.EncodeInvalidUtf8),
   190  		}}...),
   191  	})
   192  }
   193  
   194  // Options defines the configuration for this backend
   195  type Options struct {
   196  	RootFolderID string               `config:"root_folder_id"`
   197  	Region       string               `config:"region"`
   198  	Enc          encoder.MultiEncoder `config:"encoding"`
   199  }
   200  
   201  // Fs represents a remote workdrive
   202  type Fs struct {
   203  	name     string             // name of this remote
   204  	root     string             // the path we are working on
   205  	opt      Options            // parsed options
   206  	features *fs.Features       // optional features
   207  	srv      *rest.Client       // the connection to the server
   208  	dirCache *dircache.DirCache // Map of directory path to directory id
   209  	pacer    *fs.Pacer          // pacer for API calls
   210  }
   211  
   212  // Object describes a Zoho WorkDrive object
   213  //
   214  // Will definitely have info but maybe not meta
   215  type Object struct {
   216  	fs          *Fs       // what this object is part of
   217  	remote      string    // The remote path
   218  	hasMetaData bool      // whether info below has been set
   219  	size        int64     // size of the object
   220  	modTime     time.Time // modification time of the object
   221  	id          string    // ID of the object
   222  }
   223  
   224  // ------------------------------------------------------------
   225  
   226  func setupRegion(m configmap.Mapper) error {
   227  	region, ok := m.Get("region")
   228  	if !ok || region == "" {
   229  		return errors.New("no region set")
   230  	}
   231  	rootURL = fmt.Sprintf("https://workdrive.zoho.%s/api/v1", region)
   232  	accountsURL = fmt.Sprintf("https://accounts.zoho.%s", region)
   233  	oauthConfig.Endpoint.AuthURL = fmt.Sprintf("https://accounts.zoho.%s/oauth/v2/auth", region)
   234  	oauthConfig.Endpoint.TokenURL = fmt.Sprintf("https://accounts.zoho.%s/oauth/v2/token", region)
   235  	return nil
   236  }
   237  
   238  // ------------------------------------------------------------
   239  
   240  func listTeams(ctx context.Context, uid int64, srv *rest.Client) ([]api.TeamWorkspace, error) {
   241  	var teamList api.TeamWorkspaceResponse
   242  	opts := rest.Opts{
   243  		Method:       "GET",
   244  		Path:         "/users/" + strconv.FormatInt(uid, 10) + "/teams",
   245  		ExtraHeaders: map[string]string{"Accept": "application/vnd.api+json"},
   246  	}
   247  	_, err := srv.CallJSON(ctx, &opts, nil, &teamList)
   248  	if err != nil {
   249  		return nil, err
   250  	}
   251  	return teamList.TeamWorkspace, nil
   252  }
   253  
   254  func listWorkspaces(ctx context.Context, teamID string, srv *rest.Client) ([]api.TeamWorkspace, error) {
   255  	var workspaceList api.TeamWorkspaceResponse
   256  	opts := rest.Opts{
   257  		Method:       "GET",
   258  		Path:         "/teams/" + teamID + "/workspaces",
   259  		ExtraHeaders: map[string]string{"Accept": "application/vnd.api+json"},
   260  	}
   261  	_, err := srv.CallJSON(ctx, &opts, nil, &workspaceList)
   262  	if err != nil {
   263  		return nil, err
   264  	}
   265  	return workspaceList.TeamWorkspace, nil
   266  }
   267  
   268  // --------------------------------------------------------------
   269  
   270  // retryErrorCodes is a slice of error codes that we will retry
   271  var retryErrorCodes = []int{
   272  	429, // Too Many Requests.
   273  	500, // Internal Server Error
   274  	502, // Bad Gateway
   275  	503, // Service Unavailable
   276  	504, // Gateway Timeout
   277  	509, // Bandwidth Limit Exceeded
   278  }
   279  
   280  // shouldRetry returns a boolean as to whether this resp and err
   281  // deserve to be retried.  It returns the err as a convenience
   282  func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
   283  	if fserrors.ContextError(ctx, &err) {
   284  		return false, err
   285  	}
   286  	authRetry := false
   287  
   288  	if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Contains(resp.Header["Www-Authenticate"][0], "expired_token") {
   289  		authRetry = true
   290  		fs.Debugf(nil, "Should retry: %v", err)
   291  	}
   292  	return authRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
   293  }
   294  
   295  // --------------------------------------------------------------
   296  
   297  // Name of the remote (as passed into NewFs)
   298  func (f *Fs) Name() string {
   299  	return f.name
   300  }
   301  
   302  // Root of the remote (as passed into NewFs)
   303  func (f *Fs) Root() string {
   304  	return f.root
   305  }
   306  
   307  // String converts this Fs to a string
   308  func (f *Fs) String() string {
   309  	return fmt.Sprintf("zoho root '%s'", f.root)
   310  }
   311  
   312  // Precision return the precision of this Fs
   313  func (f *Fs) Precision() time.Duration {
   314  	return fs.ModTimeNotSupported
   315  }
   316  
   317  // Hashes returns the supported hash sets.
   318  func (f *Fs) Hashes() hash.Set {
   319  	return hash.Set(hash.None)
   320  }
   321  
   322  // Features returns the optional features of this Fs
   323  func (f *Fs) Features() *fs.Features {
   324  	return f.features
   325  }
   326  
   327  // parsePath parses a zoho 'url'
   328  func parsePath(path string) (root string) {
   329  	root = strings.Trim(path, "/")
   330  	return
   331  }
   332  
   333  // readMetaDataForPath reads the metadata from the path
   334  func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Item, err error) {
   335  	// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)
   336  	leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)
   337  	if err != nil {
   338  		if err == fs.ErrorDirNotFound {
   339  			return nil, fs.ErrorObjectNotFound
   340  		}
   341  		return nil, err
   342  	}
   343  
   344  	found, err := f.listAll(ctx, directoryID, false, true, func(item *api.Item) bool {
   345  		if item.Attributes.Name == leaf {
   346  			info = item
   347  			return true
   348  		}
   349  		return false
   350  	})
   351  	if err != nil {
   352  		return nil, err
   353  	}
   354  	if !found {
   355  		return nil, fs.ErrorObjectNotFound
   356  	}
   357  	return info, nil
   358  }
   359  
   360  // readMetaDataForID reads the metadata for the object with given ID
   361  func (f *Fs) readMetaDataForID(ctx context.Context, id string) (*api.Item, error) {
   362  	opts := rest.Opts{
   363  		Method:       "GET",
   364  		Path:         "/files/" + id,
   365  		ExtraHeaders: map[string]string{"Accept": "application/vnd.api+json"},
   366  		Parameters:   url.Values{},
   367  	}
   368  	var result *api.ItemInfo
   369  	var resp *http.Response
   370  	var err error
   371  	err = f.pacer.Call(func() (bool, error) {
   372  		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
   373  		return shouldRetry(ctx, resp, err)
   374  	})
   375  	if err != nil {
   376  		return nil, err
   377  	}
   378  	return &result.Item, nil
   379  }
   380  
   381  // NewFs constructs an Fs from the path, container:path
   382  func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
   383  	// Parse config into Options struct
   384  	opt := new(Options)
   385  	if err := configstruct.Set(m, opt); err != nil {
   386  		return nil, err
   387  	}
   388  	err := setupRegion(m)
   389  	if err != nil {
   390  		return nil, err
   391  	}
   392  
   393  	root = parsePath(root)
   394  	oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
   395  	if err != nil {
   396  		return nil, err
   397  	}
   398  
   399  	f := &Fs{
   400  		name:  name,
   401  		root:  root,
   402  		opt:   *opt,
   403  		srv:   rest.NewClient(oAuthClient).SetRoot(rootURL),
   404  		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
   405  	}
   406  	f.features = (&fs.Features{
   407  		CanHaveEmptyDirectories: true,
   408  	}).Fill(ctx, f)
   409  
   410  	// Get rootFolderID
   411  	rootID := f.opt.RootFolderID
   412  	f.dirCache = dircache.New(root, rootID, f)
   413  
   414  	// Find the current root
   415  	err = f.dirCache.FindRoot(ctx, false)
   416  	if err != nil {
   417  		// Assume it is a file
   418  		newRoot, remote := dircache.SplitPath(root)
   419  		tempF := *f
   420  		tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
   421  		tempF.root = newRoot
   422  		// Make new Fs which is the parent
   423  		err = tempF.dirCache.FindRoot(ctx, false)
   424  		if err != nil {
   425  			// No root so return old f
   426  			return f, nil
   427  		}
   428  		_, err := tempF.newObjectWithInfo(ctx, remote, nil)
   429  		if err != nil {
   430  			if err == fs.ErrorObjectNotFound {
   431  				// File doesn't exist so return old f
   432  				return f, nil
   433  			}
   434  			return nil, err
   435  		}
   436  		f.features.Fill(ctx, &tempF)
   437  		f.dirCache = tempF.dirCache
   438  		f.root = tempF.root
   439  		// return an error with an fs which points to the parent
   440  		return f, fs.ErrorIsFile
   441  	}
   442  	return f, nil
   443  }
   444  
   445  // list the objects into the function supplied
   446  //
   447  // If directories is set it only sends directories
   448  // User function to process a File item from listAll
   449  //
   450  // Should return true to finish processing
   451  type listAllFn func(*api.Item) bool
   452  
   453  // Lists the directory required calling the user function on each item found
   454  //
   455  // If the user fn ever returns true then it early exits with found = true
   456  func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
   457  	opts := rest.Opts{
   458  		Method:       "GET",
   459  		Path:         "/files/" + dirID + "/files",
   460  		ExtraHeaders: map[string]string{"Accept": "application/vnd.api+json"},
   461  		Parameters:   url.Values{},
   462  	}
   463  	opts.Parameters.Set("page[limit]", strconv.Itoa(10))
   464  	offset := 0
   465  OUTER:
   466  	for {
   467  		opts.Parameters.Set("page[offset]", strconv.Itoa(offset))
   468  
   469  		var result api.ItemList
   470  		var resp *http.Response
   471  		err = f.pacer.Call(func() (bool, error) {
   472  			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
   473  			return shouldRetry(ctx, resp, err)
   474  		})
   475  		if err != nil {
   476  			return found, fmt.Errorf("couldn't list files: %w", err)
   477  		}
   478  		if len(result.Items) == 0 {
   479  			break
   480  		}
   481  		for i := range result.Items {
   482  			item := &result.Items[i]
   483  			if item.Attributes.IsFolder {
   484  				if filesOnly {
   485  					continue
   486  				}
   487  			} else {
   488  				if directoriesOnly {
   489  					continue
   490  				}
   491  			}
   492  			item.Attributes.Name = f.opt.Enc.ToStandardName(item.Attributes.Name)
   493  			if fn(item) {
   494  				found = true
   495  				break OUTER
   496  			}
   497  		}
   498  		offset += 10
   499  	}
   500  	return
   501  }
   502  
   503  // List the objects and directories in dir into entries.  The
   504  // entries can be returned in any order but should be for a
   505  // complete directory.
   506  //
   507  // dir should be "" to list the root, and should not have
   508  // trailing slashes.
   509  //
   510  // This should return ErrDirNotFound if the directory isn't
   511  // found.
   512  func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
   513  	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
   514  	if err != nil {
   515  		return nil, err
   516  	}
   517  	var iErr error
   518  	_, err = f.listAll(ctx, directoryID, false, false, func(info *api.Item) bool {
   519  		remote := path.Join(dir, info.Attributes.Name)
   520  		if info.Attributes.IsFolder {
   521  			// cache the directory ID for later lookups
   522  			f.dirCache.Put(remote, info.ID)
   523  			d := fs.NewDir(remote, time.Time(info.Attributes.ModifiedTime)).SetID(info.ID)
   524  			entries = append(entries, d)
   525  		} else {
   526  			o, err := f.newObjectWithInfo(ctx, remote, info)
   527  			if err != nil {
   528  				iErr = err
   529  				return true
   530  			}
   531  			entries = append(entries, o)
   532  		}
   533  		return false
   534  	})
   535  	if err != nil {
   536  		return nil, err
   537  	}
   538  	if iErr != nil {
   539  		return nil, iErr
   540  	}
   541  	return entries, nil
   542  }
   543  
   544  // FindLeaf finds a directory of name leaf in the folder with ID pathID
   545  func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
   546  	// Find the leaf in pathID
   547  	found, err = f.listAll(ctx, pathID, true, false, func(item *api.Item) bool {
   548  		if item.Attributes.Name == leaf {
   549  			pathIDOut = item.ID
   550  			return true
   551  		}
   552  		return false
   553  	})
   554  	return pathIDOut, found, err
   555  }
   556  
   557  // CreateDir makes a directory with pathID as parent and name leaf
   558  func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
   559  	//fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf)
   560  	var resp *http.Response
   561  	var info *api.ItemInfo
   562  	opts := rest.Opts{
   563  		Method:       "POST",
   564  		Path:         "/files",
   565  		ExtraHeaders: map[string]string{"Accept": "application/vnd.api+json"},
   566  	}
   567  	mkdir := api.WriteMetadataRequest{
   568  		Data: api.WriteMetadata{
   569  			Attributes: api.WriteAttributes{
   570  				Name:     f.opt.Enc.FromStandardName(leaf),
   571  				ParentID: pathID,
   572  			},
   573  			Type: "files",
   574  		},
   575  	}
   576  	err = f.pacer.Call(func() (bool, error) {
   577  		resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info)
   578  		return shouldRetry(ctx, resp, err)
   579  	})
   580  	if err != nil {
   581  		//fmt.Printf("...Error %v\n", err)
   582  		return "", err
   583  	}
   584  	// fmt.Printf("...Id %q\n", *info.Id)
   585  	return info.Item.ID, nil
   586  }
   587  
   588  // Return an Object from a path
   589  //
   590  // If it can't be found it returns the error fs.ErrorObjectNotFound.
   591  func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.Item) (fs.Object, error) {
   592  	o := &Object{
   593  		fs:     f,
   594  		remote: remote,
   595  	}
   596  	var err error
   597  	if info != nil {
   598  		// Set info
   599  		err = o.setMetaData(info)
   600  	} else {
   601  		err = o.readMetaData(ctx) // reads info and meta, returning an error
   602  	}
   603  	if err != nil {
   604  		return nil, err
   605  	}
   606  	return o, nil
   607  }
   608  
   609  // NewObject finds the Object at remote.  If it can't be found
   610  // it returns the error fs.ErrorObjectNotFound.
   611  func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
   612  	return f.newObjectWithInfo(ctx, remote, nil)
   613  }
   614  
   615  // Creates from the parameters passed in a half finished Object which
   616  // must have setMetaData called on it
   617  //
   618  // Used to create new objects
   619  func (f *Fs) createObject(ctx context.Context, remote string, size int64, modTime time.Time) (o *Object, leaf string, directoryID string, err error) {
   620  	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
   621  	if err != nil {
   622  		return
   623  	}
   624  	// Temporary Object under construction
   625  	o = &Object{
   626  		fs:      f,
   627  		remote:  remote,
   628  		size:    size,
   629  		modTime: modTime,
   630  	}
   631  	return
   632  }
   633  
   634  // Put the object
   635  //
   636  // Copy the reader in to the new object which is returned.
   637  //
   638  // The new object may have been created if an error is returned
   639  func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
   640  	existingObj, err := f.newObjectWithInfo(ctx, src.Remote(), nil)
   641  	switch err {
   642  	case nil:
   643  		return existingObj, existingObj.Update(ctx, in, src, options...)
   644  	case fs.ErrorObjectNotFound:
   645  		// Not found so create it
   646  		return f.PutUnchecked(ctx, in, src)
   647  	default:
   648  		return nil, err
   649  	}
   650  }
   651  
   652  func isSimpleName(s string) bool {
   653  	for _, r := range s {
   654  		if (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') && (r != '.') {
   655  			return false
   656  		}
   657  	}
   658  	return true
   659  }
   660  
   661  func (f *Fs) upload(ctx context.Context, name string, parent string, size int64, in io.Reader, options ...fs.OpenOption) (*api.Item, error) {
   662  	params := url.Values{}
   663  	params.Set("filename", name)
   664  	params.Set("parent_id", parent)
   665  	params.Set("override-name-exist", strconv.FormatBool(true))
   666  	formReader, contentType, overhead, err := rest.MultipartUpload(ctx, in, nil, "content", name)
   667  	if err != nil {
   668  		return nil, fmt.Errorf("failed to make multipart upload: %w", err)
   669  	}
   670  
   671  	contentLength := overhead + size
   672  	opts := rest.Opts{
   673  		Method:           "POST",
   674  		Path:             "/upload",
   675  		Body:             formReader,
   676  		ContentType:      contentType,
   677  		ContentLength:    &contentLength,
   678  		Options:          options,
   679  		Parameters:       params,
   680  		TransferEncoding: []string{"identity"},
   681  	}
   682  
   683  	var resp *http.Response
   684  	var uploadResponse *api.UploadResponse
   685  	err = f.pacer.CallNoRetry(func() (bool, error) {
   686  		resp, err = f.srv.CallJSON(ctx, &opts, nil, &uploadResponse)
   687  		return shouldRetry(ctx, resp, err)
   688  	})
   689  	if err != nil {
   690  		return nil, fmt.Errorf("upload error: %w", err)
   691  	}
   692  	if len(uploadResponse.Uploads) != 1 {
   693  		return nil, errors.New("upload: invalid response")
   694  	}
   695  	// Received meta data is missing size so we have to read it again.
   696  	info, err := f.readMetaDataForID(ctx, uploadResponse.Uploads[0].Attributes.RessourceID)
   697  	if err != nil {
   698  		return nil, err
   699  	}
   700  
   701  	return info, nil
   702  }
   703  
   704  // PutUnchecked the object into the container
   705  //
   706  // This will produce an error if the object already exists.
   707  //
   708  // Copy the reader in to the new object which is returned.
   709  //
   710  // The new object may have been created if an error is returned
   711  func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
   712  	size := src.Size()
   713  	remote := src.Remote()
   714  
   715  	// Create the directory for the object if it doesn't exist
   716  	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
   717  	if err != nil {
   718  		return nil, err
   719  	}
   720  
   721  	if isSimpleName(leaf) {
   722  		info, err := f.upload(ctx, f.opt.Enc.FromStandardName(leaf), directoryID, size, in, options...)
   723  		if err != nil {
   724  			return nil, err
   725  		}
   726  		return f.newObjectWithInfo(ctx, remote, info)
   727  	}
   728  
   729  	tempName := "rcloneTemp" + random.String(8)
   730  	info, err := f.upload(ctx, tempName, directoryID, size, in, options...)
   731  	if err != nil {
   732  		return nil, err
   733  	}
   734  
   735  	o, err := f.newObjectWithInfo(ctx, remote, info)
   736  	if err != nil {
   737  		return nil, err
   738  	}
   739  	return o, o.(*Object).rename(ctx, leaf)
   740  }
   741  
   742  // Mkdir creates the container if it doesn't exist
   743  func (f *Fs) Mkdir(ctx context.Context, dir string) error {
   744  	_, err := f.dirCache.FindDir(ctx, dir, true)
   745  	return err
   746  }
   747  
   748  // deleteObject removes an object by ID
   749  func (f *Fs) deleteObject(ctx context.Context, id string) (err error) {
   750  	var resp *http.Response
   751  	opts := rest.Opts{
   752  		Method:       "PATCH",
   753  		Path:         "/files",
   754  		ExtraHeaders: map[string]string{"Accept": "application/vnd.api+json"},
   755  	}
   756  	delete := api.WriteMultiMetadataRequest{
   757  		Meta: []api.WriteMetadata{
   758  			{
   759  				Attributes: api.WriteAttributes{
   760  					Status: "51", // Status "51" is deleted
   761  				},
   762  				ID:   id,
   763  				Type: "files",
   764  			},
   765  		},
   766  	}
   767  	err = f.pacer.Call(func() (bool, error) {
   768  		resp, err = f.srv.CallJSON(ctx, &opts, &delete, nil)
   769  		return shouldRetry(ctx, resp, err)
   770  	})
   771  	if err != nil {
   772  		return fmt.Errorf("delete object failed: %w", err)
   773  	}
   774  	return nil
   775  }
   776  
   777  // purgeCheck removes the root directory, if check is set then it
   778  // refuses to do so if it has anything in
   779  func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
   780  	root := path.Join(f.root, dir)
   781  	if root == "" {
   782  		return errors.New("can't purge root directory")
   783  	}
   784  	rootID, err := f.dirCache.FindDir(ctx, dir, false)
   785  	if err != nil {
   786  		return err
   787  	}
   788  
   789  	info, err := f.readMetaDataForID(ctx, rootID)
   790  	if err != nil {
   791  		return err
   792  	}
   793  	if check && info.Attributes.StorageInfo.Size > 0 {
   794  		return fs.ErrorDirectoryNotEmpty
   795  	}
   796  
   797  	err = f.deleteObject(ctx, rootID)
   798  	if err != nil {
   799  		return fmt.Errorf("rmdir failed: %w", err)
   800  	}
   801  	f.dirCache.FlushDir(dir)
   802  	return nil
   803  }
   804  
   805  // Rmdir deletes the root folder
   806  //
   807  // Returns an error if it isn't empty
   808  func (f *Fs) Rmdir(ctx context.Context, dir string) error {
   809  	return f.purgeCheck(ctx, dir, true)
   810  }
   811  
   812  // Purge deletes all the files and the container
   813  //
   814  // Optional interface: Only implement this if you have a way of
   815  // deleting all the files quicker than just running Remove() on the
   816  // result of List()
   817  func (f *Fs) Purge(ctx context.Context, dir string) error {
   818  	return f.purgeCheck(ctx, dir, false)
   819  }
   820  
   821  func (f *Fs) rename(ctx context.Context, id, name string) (item *api.Item, err error) {
   822  	var resp *http.Response
   823  	opts := rest.Opts{
   824  		Method:       "PATCH",
   825  		Path:         "/files/" + id,
   826  		ExtraHeaders: map[string]string{"Accept": "application/vnd.api+json"},
   827  	}
   828  	rename := api.WriteMetadataRequest{
   829  		Data: api.WriteMetadata{
   830  			Attributes: api.WriteAttributes{
   831  				Name: f.opt.Enc.FromStandardName(name),
   832  			},
   833  			Type: "files",
   834  		},
   835  	}
   836  	var result *api.ItemInfo
   837  	err = f.pacer.Call(func() (bool, error) {
   838  		resp, err = f.srv.CallJSON(ctx, &opts, &rename, &result)
   839  		return shouldRetry(ctx, resp, err)
   840  	})
   841  	if err != nil {
   842  		return nil, fmt.Errorf("rename failed: %w", err)
   843  	}
   844  	return &result.Item, nil
   845  }
   846  
   847  // Copy src to this remote using server side copy operations.
   848  //
   849  // This is stored with the remote path given.
   850  //
   851  // It returns the destination Object and a possible error.
   852  //
   853  // Will only be called if src.Fs().Name() == f.Name()
   854  //
   855  // If it isn't possible then return fs.ErrorCantCopy
   856  func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
   857  	srcObj, ok := src.(*Object)
   858  	if !ok {
   859  		fs.Debugf(src, "Can't copy - not same remote type")
   860  		return nil, fs.ErrorCantCopy
   861  	}
   862  	err := srcObj.readMetaData(ctx)
   863  	if err != nil {
   864  		return nil, err
   865  	}
   866  
   867  	// Create temporary object
   868  	dstObject, leaf, directoryID, err := f.createObject(ctx, remote, srcObj.size, srcObj.modTime)
   869  	if err != nil {
   870  		return nil, err
   871  	}
   872  	// Copy the object
   873  	opts := rest.Opts{
   874  		Method:       "POST",
   875  		Path:         "/files/" + directoryID + "/copy",
   876  		ExtraHeaders: map[string]string{"Accept": "application/vnd.api+json"},
   877  	}
   878  	copyFile := api.WriteMultiMetadataRequest{
   879  		Meta: []api.WriteMetadata{
   880  			{
   881  				Attributes: api.WriteAttributes{
   882  					RessourceID: srcObj.id,
   883  				},
   884  				Type: "files",
   885  			},
   886  		},
   887  	}
   888  	var resp *http.Response
   889  	var result *api.ItemList
   890  	err = f.pacer.Call(func() (bool, error) {
   891  		resp, err = f.srv.CallJSON(ctx, &opts, &copyFile, &result)
   892  		return shouldRetry(ctx, resp, err)
   893  	})
   894  	if err != nil {
   895  		return nil, fmt.Errorf("couldn't copy file: %w", err)
   896  	}
   897  	// Server acts weird some times make sure we actually got
   898  	// an item
   899  	if len(result.Items) != 1 {
   900  		return nil, errors.New("couldn't copy file: invalid response")
   901  	}
   902  	// Only set ID here because response is not complete Item struct
   903  	dstObject.id = result.Items[0].ID
   904  
   905  	// Can't copy and change name in one step so we have to check if we have
   906  	// the correct name after copy
   907  	if f.opt.Enc.ToStandardName(result.Items[0].Attributes.Name) != leaf {
   908  		if err = dstObject.rename(ctx, leaf); err != nil {
   909  			return nil, fmt.Errorf("copy: couldn't rename copied file: %w", err)
   910  		}
   911  	}
   912  	return dstObject, nil
   913  }
   914  
   915  func (f *Fs) move(ctx context.Context, srcID, parentID string) (item *api.Item, err error) {
   916  	// Move the object
   917  	opts := rest.Opts{
   918  		Method:       "PATCH",
   919  		Path:         "/files",
   920  		ExtraHeaders: map[string]string{"Accept": "application/vnd.api+json"},
   921  	}
   922  	moveFile := api.WriteMultiMetadataRequest{
   923  		Meta: []api.WriteMetadata{
   924  			{
   925  				Attributes: api.WriteAttributes{
   926  					ParentID: parentID,
   927  				},
   928  				ID:   srcID,
   929  				Type: "files",
   930  			},
   931  		},
   932  	}
   933  	var resp *http.Response
   934  	var result *api.ItemList
   935  	err = f.pacer.Call(func() (bool, error) {
   936  		resp, err = f.srv.CallJSON(ctx, &opts, &moveFile, &result)
   937  		return shouldRetry(ctx, resp, err)
   938  	})
   939  	if err != nil {
   940  		return nil, fmt.Errorf("move failed: %w", err)
   941  	}
   942  	// Server acts weird some times make sure our array actually contains
   943  	// a file
   944  	if len(result.Items) != 1 {
   945  		return nil, errors.New("move failed: invalid response")
   946  	}
   947  	return &result.Items[0], nil
   948  }
   949  
   950  // Move src to this remote using server side move operations.
   951  //
   952  // This is stored with the remote path given.
   953  //
   954  // It returns the destination Object and a possible error.
   955  //
   956  // Will only be called if src.Fs().Name() == f.Name()
   957  //
   958  // If it isn't possible then return fs.ErrorCantMove
   959  func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
   960  	srcObj, ok := src.(*Object)
   961  	if !ok {
   962  		fs.Debugf(src, "Can't move - not same remote type")
   963  		return nil, fs.ErrorCantMove
   964  	}
   965  	err := srcObj.readMetaData(ctx)
   966  	if err != nil {
   967  		return nil, err
   968  	}
   969  
   970  	srcLeaf, srcParentID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false)
   971  	if err != nil {
   972  		return nil, err
   973  	}
   974  
   975  	// Create temporary object
   976  	dstObject, dstLeaf, directoryID, err := f.createObject(ctx, remote, srcObj.size, srcObj.modTime)
   977  	if err != nil {
   978  		return nil, err
   979  	}
   980  
   981  	needRename := srcLeaf != dstLeaf
   982  	needMove := srcParentID != directoryID
   983  
   984  	// rename the leaf to a temporary name if we are moving to
   985  	// another directory to make sure we don't overwrite something
   986  	// in the source directory by accident
   987  	if needRename && needMove {
   988  		tmpLeaf := "rcloneTemp" + random.String(8)
   989  		if err = srcObj.rename(ctx, tmpLeaf); err != nil {
   990  			return nil, fmt.Errorf("move: pre move rename failed: %w", err)
   991  		}
   992  	}
   993  
   994  	// do the move if required
   995  	if needMove {
   996  		item, err := f.move(ctx, srcObj.id, directoryID)
   997  		if err != nil {
   998  			return nil, err
   999  		}
  1000  		// Only set ID here because response is not complete Item struct
  1001  		dstObject.id = item.ID
  1002  	} else {
  1003  		// same parent only need to rename
  1004  		dstObject.id = srcObj.id
  1005  	}
  1006  
  1007  	// rename the leaf to its final name
  1008  	if needRename {
  1009  		if err = dstObject.rename(ctx, dstLeaf); err != nil {
  1010  			return nil, fmt.Errorf("move: couldn't rename moved file: %w", err)
  1011  		}
  1012  	}
  1013  	return dstObject, nil
  1014  }
  1015  
  1016  // DirMove moves src, srcRemote to this remote at dstRemote
  1017  // using server side move operations.
  1018  //
  1019  // Will only be called if src.Fs().Name() == f.Name()
  1020  //
  1021  // If it isn't possible then return fs.ErrorCantDirMove
  1022  //
  1023  // If destination exists then return fs.ErrorDirExists
  1024  func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
  1025  	srcFs, ok := src.(*Fs)
  1026  	if !ok {
  1027  		fs.Debugf(srcFs, "Can't move directory - not same remote type")
  1028  		return fs.ErrorCantDirMove
  1029  	}
  1030  
  1031  	srcID, srcDirectoryID, srcLeaf, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
  1032  	if err != nil {
  1033  		return err
  1034  	}
  1035  	// same parent only need to rename
  1036  	if srcDirectoryID == dstDirectoryID {
  1037  		_, err = f.rename(ctx, srcID, dstLeaf)
  1038  		return err
  1039  	}
  1040  
  1041  	// do the move
  1042  	_, err = f.move(ctx, srcID, dstDirectoryID)
  1043  	if err != nil {
  1044  		return fmt.Errorf("couldn't dir move: %w", err)
  1045  	}
  1046  
  1047  	// Can't copy and change name in one step so we have to check if we have
  1048  	// the correct name after copy
  1049  	if srcLeaf != dstLeaf {
  1050  		_, err = f.rename(ctx, srcID, dstLeaf)
  1051  		if err != nil {
  1052  			return fmt.Errorf("dirmove: couldn't rename moved dir: %w", err)
  1053  		}
  1054  	}
  1055  	srcFs.dirCache.FlushDir(srcRemote)
  1056  	return nil
  1057  }
  1058  
  1059  // About gets quota information
  1060  func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
  1061  	info, err := f.readMetaDataForID(ctx, f.opt.RootFolderID)
  1062  	if err != nil {
  1063  		return nil, err
  1064  	}
  1065  	usage = &fs.Usage{
  1066  		Used: fs.NewUsageValue(info.Attributes.StorageInfo.Size),
  1067  	}
  1068  	return usage, nil
  1069  }
  1070  
  1071  // DirCacheFlush resets the directory cache - used in testing as an
  1072  // optional interface
  1073  func (f *Fs) DirCacheFlush() {
  1074  	f.dirCache.ResetRoot()
  1075  }
  1076  
  1077  // ------------------------------------------------------------
  1078  
  1079  // Fs returns the parent Fs
  1080  func (o *Object) Fs() fs.Info {
  1081  	return o.fs
  1082  }
  1083  
  1084  // Return a string version
  1085  func (o *Object) String() string {
  1086  	if o == nil {
  1087  		return "<nil>"
  1088  	}
  1089  	return o.remote
  1090  }
  1091  
  1092  // Remote returns the remote path
  1093  func (o *Object) Remote() string {
  1094  	return o.remote
  1095  }
  1096  
  1097  // Storable returns a boolean showing whether this object storable
  1098  func (o *Object) Storable() bool {
  1099  	return true
  1100  }
  1101  
  1102  // Hash returns the SHA-1 of an object returning a lowercase hex string
  1103  func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
  1104  	return "", nil
  1105  }
  1106  
  1107  // Size returns the size of an object in bytes
  1108  func (o *Object) Size() int64 {
  1109  	if err := o.readMetaData(context.TODO()); err != nil {
  1110  		fs.Logf(o, "Failed to read metadata: %v", err)
  1111  		return 0
  1112  	}
  1113  	return o.size
  1114  }
  1115  
  1116  // setMetaData sets the metadata from info
  1117  func (o *Object) setMetaData(info *api.Item) (err error) {
  1118  	if info.Attributes.IsFolder {
  1119  		return fs.ErrorIsDir
  1120  	}
  1121  	o.hasMetaData = true
  1122  	o.size = info.Attributes.StorageInfo.Size
  1123  	o.modTime = time.Time(info.Attributes.ModifiedTime)
  1124  	o.id = info.ID
  1125  	return nil
  1126  }
  1127  
  1128  // readMetaData gets the metadata if it hasn't already been fetched
  1129  //
  1130  // it also sets the info
  1131  func (o *Object) readMetaData(ctx context.Context) (err error) {
  1132  	if o.hasMetaData {
  1133  		return nil
  1134  	}
  1135  	info, err := o.fs.readMetaDataForPath(ctx, o.remote)
  1136  	if err != nil {
  1137  		return err
  1138  	}
  1139  	return o.setMetaData(info)
  1140  }
  1141  
  1142  // ModTime returns the modification time of the object
  1143  //
  1144  // It attempts to read the objects mtime and if that isn't present the
  1145  // LastModified returned in the http headers
  1146  func (o *Object) ModTime(ctx context.Context) time.Time {
  1147  	return o.modTime
  1148  }
  1149  
  1150  // SetModTime sets the modification time of the local fs object
  1151  func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
  1152  	return fs.ErrorCantSetModTime
  1153  }
  1154  
  1155  // rename renames an object in place
  1156  //
  1157  // this a separate api call then move with zoho
  1158  func (o *Object) rename(ctx context.Context, name string) (err error) {
  1159  	item, err := o.fs.rename(ctx, o.id, name)
  1160  	if err != nil {
  1161  		return err
  1162  	}
  1163  	return o.setMetaData(item)
  1164  }
  1165  
  1166  // Open an object for read
  1167  func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
  1168  	if o.id == "" {
  1169  		return nil, errors.New("can't download - no id")
  1170  	}
  1171  	var resp *http.Response
  1172  	fs.FixRangeOption(options, o.size)
  1173  	opts := rest.Opts{
  1174  		Method:  "GET",
  1175  		Path:    "/download/" + o.id,
  1176  		Options: options,
  1177  	}
  1178  	err = o.fs.pacer.Call(func() (bool, error) {
  1179  		resp, err = o.fs.srv.Call(ctx, &opts)
  1180  		return shouldRetry(ctx, resp, err)
  1181  	})
  1182  	if err != nil {
  1183  		return nil, err
  1184  	}
  1185  	return resp.Body, nil
  1186  }
  1187  
  1188  // Update the object with the contents of the io.Reader, modTime and size
  1189  //
  1190  // If existing is set then it updates the object rather than creating a new one.
  1191  //
  1192  // The new object may have been created if an error is returned
  1193  func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
  1194  	size := src.Size()
  1195  	remote := o.Remote()
  1196  
  1197  	// Create the directory for the object if it doesn't exist
  1198  	leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true)
  1199  	if err != nil {
  1200  		return err
  1201  	}
  1202  
  1203  	if isSimpleName(leaf) {
  1204  		// Simple name we can just overwrite the old file
  1205  		info, err := o.fs.upload(ctx, o.fs.opt.Enc.FromStandardName(leaf), directoryID, size, in, options...)
  1206  		if err != nil {
  1207  			return err
  1208  		}
  1209  		return o.setMetaData(info)
  1210  	}
  1211  
  1212  	// We have to fall back to upload + rename
  1213  	tempName := "rcloneTemp" + random.String(8)
  1214  	info, err := o.fs.upload(ctx, tempName, directoryID, size, in, options...)
  1215  	if err != nil {
  1216  		return err
  1217  	}
  1218  
  1219  	// upload was successful, need to delete old object before rename
  1220  	if err = o.Remove(ctx); err != nil {
  1221  		return fmt.Errorf("failed to remove old object: %w", err)
  1222  	}
  1223  	if err = o.setMetaData(info); err != nil {
  1224  		return err
  1225  	}
  1226  
  1227  	// rename also updates metadata
  1228  	return o.rename(ctx, leaf)
  1229  }
  1230  
  1231  // Remove an object
  1232  func (o *Object) Remove(ctx context.Context) error {
  1233  	return o.fs.deleteObject(ctx, o.id)
  1234  }
  1235  
  1236  // ID returns the ID of the Object if known, or "" if not
  1237  func (o *Object) ID() string {
  1238  	return o.id
  1239  }
  1240  
  1241  // Check the interfaces are satisfied
  1242  var (
  1243  	_ fs.Fs              = (*Fs)(nil)
  1244  	_ fs.Purger          = (*Fs)(nil)
  1245  	_ fs.Copier          = (*Fs)(nil)
  1246  	_ fs.Abouter         = (*Fs)(nil)
  1247  	_ fs.Mover           = (*Fs)(nil)
  1248  	_ fs.DirMover        = (*Fs)(nil)
  1249  	_ fs.DirCacheFlusher = (*Fs)(nil)
  1250  	_ fs.Object          = (*Object)(nil)
  1251  	_ fs.IDer            = (*Object)(nil)
  1252  )