github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/backend/zoho/zoho.go (about) 1 // Package zoho provides an interface to the Zoho Workdrive 2 // storage system. 3 package zoho 4 5 import ( 6 "context" 7 "errors" 8 "fmt" 9 "io" 10 "net/http" 11 "net/url" 12 "path" 13 "strconv" 14 "strings" 15 "time" 16 17 "github.com/rclone/rclone/lib/encoder" 18 "github.com/rclone/rclone/lib/pacer" 19 "github.com/rclone/rclone/lib/random" 20 21 "github.com/rclone/rclone/backend/zoho/api" 22 "github.com/rclone/rclone/fs" 23 "github.com/rclone/rclone/fs/config" 24 "github.com/rclone/rclone/fs/config/configmap" 25 "github.com/rclone/rclone/fs/config/configstruct" 26 "github.com/rclone/rclone/fs/config/obscure" 27 "github.com/rclone/rclone/fs/fserrors" 28 "github.com/rclone/rclone/fs/hash" 29 "github.com/rclone/rclone/lib/dircache" 30 "github.com/rclone/rclone/lib/oauthutil" 31 "github.com/rclone/rclone/lib/rest" 32 "golang.org/x/oauth2" 33 ) 34 35 const ( 36 rcloneClientID = "1000.46MXF275FM2XV7QCHX5A7K3LGME66B" 37 rcloneEncryptedClientSecret = "U-2gxclZQBcOG9NPhjiXAhj-f0uQ137D0zar8YyNHXHkQZlTeSpIOQfmCb4oSpvosJp_SJLXmLLeUA" 38 minSleep = 10 * time.Millisecond 39 maxSleep = 2 * time.Second 40 decayConstant = 2 // bigger for slower decay, exponential 41 configRootID = "root_folder_id" 42 ) 43 44 // Globals 45 var ( 46 // Description of how to auth for this app 47 oauthConfig = &oauth2.Config{ 48 Scopes: []string{ 49 "aaaserver.profile.read", 50 "WorkDrive.team.READ", 51 "WorkDrive.workspace.READ", 52 "WorkDrive.files.ALL", 53 }, 54 Endpoint: oauth2.Endpoint{ 55 AuthURL: "https://accounts.zoho.eu/oauth/v2/auth", 56 TokenURL: "https://accounts.zoho.eu/oauth/v2/token", 57 AuthStyle: oauth2.AuthStyleInParams, 58 }, 59 ClientID: rcloneClientID, 60 ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), 61 RedirectURL: oauthutil.RedirectLocalhostURL, 62 } 63 rootURL = "https://workdrive.zoho.eu/api/v1" 64 accountsURL = "https://accounts.zoho.eu" 65 ) 66 67 // Register with Fs 68 func init() { 69 fs.Register(&fs.RegInfo{ 70 Name: "zoho", 71 Description: "Zoho", 72 NewFs: NewFs, 73 Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) { 74 // Need to setup region before configuring oauth 75 err := setupRegion(m) 76 if err != nil { 77 return nil, err 78 } 79 getSrvs := func() (authSrv, apiSrv *rest.Client, err error) { 80 oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig) 81 if err != nil { 82 return nil, nil, fmt.Errorf("failed to load oAuthClient: %w", err) 83 } 84 authSrv = rest.NewClient(oAuthClient).SetRoot(accountsURL) 85 apiSrv = rest.NewClient(oAuthClient).SetRoot(rootURL) 86 return authSrv, apiSrv, nil 87 } 88 89 switch config.State { 90 case "": 91 return oauthutil.ConfigOut("teams", &oauthutil.Options{ 92 OAuth2Config: oauthConfig, 93 // No refresh token unless ApprovalForce is set 94 OAuth2Opts: []oauth2.AuthCodeOption{oauth2.ApprovalForce}, 95 }) 96 case "teams": 97 // We need to rewrite the token type to "Zoho-oauthtoken" because Zoho wants 98 // it's own custom type 99 token, err := oauthutil.GetToken(name, m) 100 if err != nil { 101 return nil, fmt.Errorf("failed to read token: %w", err) 102 } 103 if token.TokenType != "Zoho-oauthtoken" { 104 token.TokenType = "Zoho-oauthtoken" 105 err = oauthutil.PutToken(name, m, token, false) 106 if err != nil { 107 return nil, fmt.Errorf("failed to configure token: %w", err) 108 } 109 } 110 111 authSrv, apiSrv, err := getSrvs() 112 if err != nil { 113 return nil, err 114 } 115 116 // Get the user Info 117 opts := rest.Opts{ 118 Method: "GET", 119 Path: "/oauth/user/info", 120 } 121 var user api.User 122 _, err = authSrv.CallJSON(ctx, &opts, nil, &user) 123 if err != nil { 124 return nil, err 125 } 126 127 // Get the teams 128 teams, err := listTeams(ctx, user.ZUID, apiSrv) 129 if err != nil { 130 return nil, err 131 } 132 return fs.ConfigChoose("workspace", "config_team_drive_id", "Team Drive ID", len(teams), func(i int) (string, string) { 133 team := teams[i] 134 return team.ID, team.Attributes.Name 135 }) 136 case "workspace": 137 _, apiSrv, err := getSrvs() 138 if err != nil { 139 return nil, err 140 } 141 teamID := config.Result 142 workspaces, err := listWorkspaces(ctx, teamID, apiSrv) 143 if err != nil { 144 return nil, err 145 } 146 return fs.ConfigChoose("workspace_end", "config_workspace", "Workspace ID", len(workspaces), func(i int) (string, string) { 147 workspace := workspaces[i] 148 return workspace.ID, workspace.Attributes.Name 149 }) 150 case "workspace_end": 151 workspaceID := config.Result 152 m.Set(configRootID, workspaceID) 153 return nil, nil 154 } 155 return nil, fmt.Errorf("unknown state %q", config.State) 156 }, 157 Options: append(oauthutil.SharedOptions, []fs.Option{{ 158 Name: "region", 159 Help: `Zoho region to connect to. 160 161 You'll have to use the region your organization is registered in. If 162 not sure use the same top level domain as you connect to in your 163 browser.`, 164 Examples: []fs.OptionExample{{ 165 Value: "com", 166 Help: "United states / Global", 167 }, { 168 Value: "eu", 169 Help: "Europe", 170 }, { 171 Value: "in", 172 Help: "India", 173 }, { 174 Value: "jp", 175 Help: "Japan", 176 }, { 177 Value: "com.cn", 178 Help: "China", 179 }, { 180 Value: "com.au", 181 Help: "Australia", 182 }}}, { 183 Name: config.ConfigEncoding, 184 Help: config.ConfigEncodingHelp, 185 Advanced: true, 186 Default: (encoder.EncodeZero | 187 encoder.EncodeCtl | 188 encoder.EncodeDel | 189 encoder.EncodeInvalidUtf8), 190 }}...), 191 }) 192 } 193 194 // Options defines the configuration for this backend 195 type Options struct { 196 RootFolderID string `config:"root_folder_id"` 197 Region string `config:"region"` 198 Enc encoder.MultiEncoder `config:"encoding"` 199 } 200 201 // Fs represents a remote workdrive 202 type Fs struct { 203 name string // name of this remote 204 root string // the path we are working on 205 opt Options // parsed options 206 features *fs.Features // optional features 207 srv *rest.Client // the connection to the server 208 dirCache *dircache.DirCache // Map of directory path to directory id 209 pacer *fs.Pacer // pacer for API calls 210 } 211 212 // Object describes a Zoho WorkDrive object 213 // 214 // Will definitely have info but maybe not meta 215 type Object struct { 216 fs *Fs // what this object is part of 217 remote string // The remote path 218 hasMetaData bool // whether info below has been set 219 size int64 // size of the object 220 modTime time.Time // modification time of the object 221 id string // ID of the object 222 } 223 224 // ------------------------------------------------------------ 225 226 func setupRegion(m configmap.Mapper) error { 227 region, ok := m.Get("region") 228 if !ok || region == "" { 229 return errors.New("no region set") 230 } 231 rootURL = fmt.Sprintf("https://workdrive.zoho.%s/api/v1", region) 232 accountsURL = fmt.Sprintf("https://accounts.zoho.%s", region) 233 oauthConfig.Endpoint.AuthURL = fmt.Sprintf("https://accounts.zoho.%s/oauth/v2/auth", region) 234 oauthConfig.Endpoint.TokenURL = fmt.Sprintf("https://accounts.zoho.%s/oauth/v2/token", region) 235 return nil 236 } 237 238 // ------------------------------------------------------------ 239 240 func listTeams(ctx context.Context, uid int64, srv *rest.Client) ([]api.TeamWorkspace, error) { 241 var teamList api.TeamWorkspaceResponse 242 opts := rest.Opts{ 243 Method: "GET", 244 Path: "/users/" + strconv.FormatInt(uid, 10) + "/teams", 245 ExtraHeaders: map[string]string{"Accept": "application/vnd.api+json"}, 246 } 247 _, err := srv.CallJSON(ctx, &opts, nil, &teamList) 248 if err != nil { 249 return nil, err 250 } 251 return teamList.TeamWorkspace, nil 252 } 253 254 func listWorkspaces(ctx context.Context, teamID string, srv *rest.Client) ([]api.TeamWorkspace, error) { 255 var workspaceList api.TeamWorkspaceResponse 256 opts := rest.Opts{ 257 Method: "GET", 258 Path: "/teams/" + teamID + "/workspaces", 259 ExtraHeaders: map[string]string{"Accept": "application/vnd.api+json"}, 260 } 261 _, err := srv.CallJSON(ctx, &opts, nil, &workspaceList) 262 if err != nil { 263 return nil, err 264 } 265 return workspaceList.TeamWorkspace, nil 266 } 267 268 // -------------------------------------------------------------- 269 270 // retryErrorCodes is a slice of error codes that we will retry 271 var retryErrorCodes = []int{ 272 429, // Too Many Requests. 273 500, // Internal Server Error 274 502, // Bad Gateway 275 503, // Service Unavailable 276 504, // Gateway Timeout 277 509, // Bandwidth Limit Exceeded 278 } 279 280 // shouldRetry returns a boolean as to whether this resp and err 281 // deserve to be retried. It returns the err as a convenience 282 func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { 283 if fserrors.ContextError(ctx, &err) { 284 return false, err 285 } 286 authRetry := false 287 288 if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Contains(resp.Header["Www-Authenticate"][0], "expired_token") { 289 authRetry = true 290 fs.Debugf(nil, "Should retry: %v", err) 291 } 292 return authRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err 293 } 294 295 // -------------------------------------------------------------- 296 297 // Name of the remote (as passed into NewFs) 298 func (f *Fs) Name() string { 299 return f.name 300 } 301 302 // Root of the remote (as passed into NewFs) 303 func (f *Fs) Root() string { 304 return f.root 305 } 306 307 // String converts this Fs to a string 308 func (f *Fs) String() string { 309 return fmt.Sprintf("zoho root '%s'", f.root) 310 } 311 312 // Precision return the precision of this Fs 313 func (f *Fs) Precision() time.Duration { 314 return fs.ModTimeNotSupported 315 } 316 317 // Hashes returns the supported hash sets. 318 func (f *Fs) Hashes() hash.Set { 319 return hash.Set(hash.None) 320 } 321 322 // Features returns the optional features of this Fs 323 func (f *Fs) Features() *fs.Features { 324 return f.features 325 } 326 327 // parsePath parses a zoho 'url' 328 func parsePath(path string) (root string) { 329 root = strings.Trim(path, "/") 330 return 331 } 332 333 // readMetaDataForPath reads the metadata from the path 334 func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Item, err error) { 335 // defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err) 336 leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false) 337 if err != nil { 338 if err == fs.ErrorDirNotFound { 339 return nil, fs.ErrorObjectNotFound 340 } 341 return nil, err 342 } 343 344 found, err := f.listAll(ctx, directoryID, false, true, func(item *api.Item) bool { 345 if item.Attributes.Name == leaf { 346 info = item 347 return true 348 } 349 return false 350 }) 351 if err != nil { 352 return nil, err 353 } 354 if !found { 355 return nil, fs.ErrorObjectNotFound 356 } 357 return info, nil 358 } 359 360 // readMetaDataForID reads the metadata for the object with given ID 361 func (f *Fs) readMetaDataForID(ctx context.Context, id string) (*api.Item, error) { 362 opts := rest.Opts{ 363 Method: "GET", 364 Path: "/files/" + id, 365 ExtraHeaders: map[string]string{"Accept": "application/vnd.api+json"}, 366 Parameters: url.Values{}, 367 } 368 var result *api.ItemInfo 369 var resp *http.Response 370 var err error 371 err = f.pacer.Call(func() (bool, error) { 372 resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) 373 return shouldRetry(ctx, resp, err) 374 }) 375 if err != nil { 376 return nil, err 377 } 378 return &result.Item, nil 379 } 380 381 // NewFs constructs an Fs from the path, container:path 382 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { 383 // Parse config into Options struct 384 opt := new(Options) 385 if err := configstruct.Set(m, opt); err != nil { 386 return nil, err 387 } 388 err := setupRegion(m) 389 if err != nil { 390 return nil, err 391 } 392 393 root = parsePath(root) 394 oAuthClient, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig) 395 if err != nil { 396 return nil, err 397 } 398 399 f := &Fs{ 400 name: name, 401 root: root, 402 opt: *opt, 403 srv: rest.NewClient(oAuthClient).SetRoot(rootURL), 404 pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))), 405 } 406 f.features = (&fs.Features{ 407 CanHaveEmptyDirectories: true, 408 }).Fill(ctx, f) 409 410 // Get rootFolderID 411 rootID := f.opt.RootFolderID 412 f.dirCache = dircache.New(root, rootID, f) 413 414 // Find the current root 415 err = f.dirCache.FindRoot(ctx, false) 416 if err != nil { 417 // Assume it is a file 418 newRoot, remote := dircache.SplitPath(root) 419 tempF := *f 420 tempF.dirCache = dircache.New(newRoot, rootID, &tempF) 421 tempF.root = newRoot 422 // Make new Fs which is the parent 423 err = tempF.dirCache.FindRoot(ctx, false) 424 if err != nil { 425 // No root so return old f 426 return f, nil 427 } 428 _, err := tempF.newObjectWithInfo(ctx, remote, nil) 429 if err != nil { 430 if err == fs.ErrorObjectNotFound { 431 // File doesn't exist so return old f 432 return f, nil 433 } 434 return nil, err 435 } 436 f.features.Fill(ctx, &tempF) 437 f.dirCache = tempF.dirCache 438 f.root = tempF.root 439 // return an error with an fs which points to the parent 440 return f, fs.ErrorIsFile 441 } 442 return f, nil 443 } 444 445 // list the objects into the function supplied 446 // 447 // If directories is set it only sends directories 448 // User function to process a File item from listAll 449 // 450 // Should return true to finish processing 451 type listAllFn func(*api.Item) bool 452 453 // Lists the directory required calling the user function on each item found 454 // 455 // If the user fn ever returns true then it early exits with found = true 456 func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) { 457 opts := rest.Opts{ 458 Method: "GET", 459 Path: "/files/" + dirID + "/files", 460 ExtraHeaders: map[string]string{"Accept": "application/vnd.api+json"}, 461 Parameters: url.Values{}, 462 } 463 opts.Parameters.Set("page[limit]", strconv.Itoa(10)) 464 offset := 0 465 OUTER: 466 for { 467 opts.Parameters.Set("page[offset]", strconv.Itoa(offset)) 468 469 var result api.ItemList 470 var resp *http.Response 471 err = f.pacer.Call(func() (bool, error) { 472 resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) 473 return shouldRetry(ctx, resp, err) 474 }) 475 if err != nil { 476 return found, fmt.Errorf("couldn't list files: %w", err) 477 } 478 if len(result.Items) == 0 { 479 break 480 } 481 for i := range result.Items { 482 item := &result.Items[i] 483 if item.Attributes.IsFolder { 484 if filesOnly { 485 continue 486 } 487 } else { 488 if directoriesOnly { 489 continue 490 } 491 } 492 item.Attributes.Name = f.opt.Enc.ToStandardName(item.Attributes.Name) 493 if fn(item) { 494 found = true 495 break OUTER 496 } 497 } 498 offset += 10 499 } 500 return 501 } 502 503 // List the objects and directories in dir into entries. The 504 // entries can be returned in any order but should be for a 505 // complete directory. 506 // 507 // dir should be "" to list the root, and should not have 508 // trailing slashes. 509 // 510 // This should return ErrDirNotFound if the directory isn't 511 // found. 512 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { 513 directoryID, err := f.dirCache.FindDir(ctx, dir, false) 514 if err != nil { 515 return nil, err 516 } 517 var iErr error 518 _, err = f.listAll(ctx, directoryID, false, false, func(info *api.Item) bool { 519 remote := path.Join(dir, info.Attributes.Name) 520 if info.Attributes.IsFolder { 521 // cache the directory ID for later lookups 522 f.dirCache.Put(remote, info.ID) 523 d := fs.NewDir(remote, time.Time(info.Attributes.ModifiedTime)).SetID(info.ID) 524 entries = append(entries, d) 525 } else { 526 o, err := f.newObjectWithInfo(ctx, remote, info) 527 if err != nil { 528 iErr = err 529 return true 530 } 531 entries = append(entries, o) 532 } 533 return false 534 }) 535 if err != nil { 536 return nil, err 537 } 538 if iErr != nil { 539 return nil, iErr 540 } 541 return entries, nil 542 } 543 544 // FindLeaf finds a directory of name leaf in the folder with ID pathID 545 func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) { 546 // Find the leaf in pathID 547 found, err = f.listAll(ctx, pathID, true, false, func(item *api.Item) bool { 548 if item.Attributes.Name == leaf { 549 pathIDOut = item.ID 550 return true 551 } 552 return false 553 }) 554 return pathIDOut, found, err 555 } 556 557 // CreateDir makes a directory with pathID as parent and name leaf 558 func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) { 559 //fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf) 560 var resp *http.Response 561 var info *api.ItemInfo 562 opts := rest.Opts{ 563 Method: "POST", 564 Path: "/files", 565 ExtraHeaders: map[string]string{"Accept": "application/vnd.api+json"}, 566 } 567 mkdir := api.WriteMetadataRequest{ 568 Data: api.WriteMetadata{ 569 Attributes: api.WriteAttributes{ 570 Name: f.opt.Enc.FromStandardName(leaf), 571 ParentID: pathID, 572 }, 573 Type: "files", 574 }, 575 } 576 err = f.pacer.Call(func() (bool, error) { 577 resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info) 578 return shouldRetry(ctx, resp, err) 579 }) 580 if err != nil { 581 //fmt.Printf("...Error %v\n", err) 582 return "", err 583 } 584 // fmt.Printf("...Id %q\n", *info.Id) 585 return info.Item.ID, nil 586 } 587 588 // Return an Object from a path 589 // 590 // If it can't be found it returns the error fs.ErrorObjectNotFound. 591 func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.Item) (fs.Object, error) { 592 o := &Object{ 593 fs: f, 594 remote: remote, 595 } 596 var err error 597 if info != nil { 598 // Set info 599 err = o.setMetaData(info) 600 } else { 601 err = o.readMetaData(ctx) // reads info and meta, returning an error 602 } 603 if err != nil { 604 return nil, err 605 } 606 return o, nil 607 } 608 609 // NewObject finds the Object at remote. If it can't be found 610 // it returns the error fs.ErrorObjectNotFound. 611 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { 612 return f.newObjectWithInfo(ctx, remote, nil) 613 } 614 615 // Creates from the parameters passed in a half finished Object which 616 // must have setMetaData called on it 617 // 618 // Used to create new objects 619 func (f *Fs) createObject(ctx context.Context, remote string, size int64, modTime time.Time) (o *Object, leaf string, directoryID string, err error) { 620 leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true) 621 if err != nil { 622 return 623 } 624 // Temporary Object under construction 625 o = &Object{ 626 fs: f, 627 remote: remote, 628 size: size, 629 modTime: modTime, 630 } 631 return 632 } 633 634 // Put the object 635 // 636 // Copy the reader in to the new object which is returned. 637 // 638 // The new object may have been created if an error is returned 639 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 640 existingObj, err := f.newObjectWithInfo(ctx, src.Remote(), nil) 641 switch err { 642 case nil: 643 return existingObj, existingObj.Update(ctx, in, src, options...) 644 case fs.ErrorObjectNotFound: 645 // Not found so create it 646 return f.PutUnchecked(ctx, in, src) 647 default: 648 return nil, err 649 } 650 } 651 652 func isSimpleName(s string) bool { 653 for _, r := range s { 654 if (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') && (r != '.') { 655 return false 656 } 657 } 658 return true 659 } 660 661 func (f *Fs) upload(ctx context.Context, name string, parent string, size int64, in io.Reader, options ...fs.OpenOption) (*api.Item, error) { 662 params := url.Values{} 663 params.Set("filename", name) 664 params.Set("parent_id", parent) 665 params.Set("override-name-exist", strconv.FormatBool(true)) 666 formReader, contentType, overhead, err := rest.MultipartUpload(ctx, in, nil, "content", name) 667 if err != nil { 668 return nil, fmt.Errorf("failed to make multipart upload: %w", err) 669 } 670 671 contentLength := overhead + size 672 opts := rest.Opts{ 673 Method: "POST", 674 Path: "/upload", 675 Body: formReader, 676 ContentType: contentType, 677 ContentLength: &contentLength, 678 Options: options, 679 Parameters: params, 680 TransferEncoding: []string{"identity"}, 681 } 682 683 var resp *http.Response 684 var uploadResponse *api.UploadResponse 685 err = f.pacer.CallNoRetry(func() (bool, error) { 686 resp, err = f.srv.CallJSON(ctx, &opts, nil, &uploadResponse) 687 return shouldRetry(ctx, resp, err) 688 }) 689 if err != nil { 690 return nil, fmt.Errorf("upload error: %w", err) 691 } 692 if len(uploadResponse.Uploads) != 1 { 693 return nil, errors.New("upload: invalid response") 694 } 695 // Received meta data is missing size so we have to read it again. 696 info, err := f.readMetaDataForID(ctx, uploadResponse.Uploads[0].Attributes.RessourceID) 697 if err != nil { 698 return nil, err 699 } 700 701 return info, nil 702 } 703 704 // PutUnchecked the object into the container 705 // 706 // This will produce an error if the object already exists. 707 // 708 // Copy the reader in to the new object which is returned. 709 // 710 // The new object may have been created if an error is returned 711 func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 712 size := src.Size() 713 remote := src.Remote() 714 715 // Create the directory for the object if it doesn't exist 716 leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true) 717 if err != nil { 718 return nil, err 719 } 720 721 if isSimpleName(leaf) { 722 info, err := f.upload(ctx, f.opt.Enc.FromStandardName(leaf), directoryID, size, in, options...) 723 if err != nil { 724 return nil, err 725 } 726 return f.newObjectWithInfo(ctx, remote, info) 727 } 728 729 tempName := "rcloneTemp" + random.String(8) 730 info, err := f.upload(ctx, tempName, directoryID, size, in, options...) 731 if err != nil { 732 return nil, err 733 } 734 735 o, err := f.newObjectWithInfo(ctx, remote, info) 736 if err != nil { 737 return nil, err 738 } 739 return o, o.(*Object).rename(ctx, leaf) 740 } 741 742 // Mkdir creates the container if it doesn't exist 743 func (f *Fs) Mkdir(ctx context.Context, dir string) error { 744 _, err := f.dirCache.FindDir(ctx, dir, true) 745 return err 746 } 747 748 // deleteObject removes an object by ID 749 func (f *Fs) deleteObject(ctx context.Context, id string) (err error) { 750 var resp *http.Response 751 opts := rest.Opts{ 752 Method: "PATCH", 753 Path: "/files", 754 ExtraHeaders: map[string]string{"Accept": "application/vnd.api+json"}, 755 } 756 delete := api.WriteMultiMetadataRequest{ 757 Meta: []api.WriteMetadata{ 758 { 759 Attributes: api.WriteAttributes{ 760 Status: "51", // Status "51" is deleted 761 }, 762 ID: id, 763 Type: "files", 764 }, 765 }, 766 } 767 err = f.pacer.Call(func() (bool, error) { 768 resp, err = f.srv.CallJSON(ctx, &opts, &delete, nil) 769 return shouldRetry(ctx, resp, err) 770 }) 771 if err != nil { 772 return fmt.Errorf("delete object failed: %w", err) 773 } 774 return nil 775 } 776 777 // purgeCheck removes the root directory, if check is set then it 778 // refuses to do so if it has anything in 779 func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { 780 root := path.Join(f.root, dir) 781 if root == "" { 782 return errors.New("can't purge root directory") 783 } 784 rootID, err := f.dirCache.FindDir(ctx, dir, false) 785 if err != nil { 786 return err 787 } 788 789 info, err := f.readMetaDataForID(ctx, rootID) 790 if err != nil { 791 return err 792 } 793 if check && info.Attributes.StorageInfo.Size > 0 { 794 return fs.ErrorDirectoryNotEmpty 795 } 796 797 err = f.deleteObject(ctx, rootID) 798 if err != nil { 799 return fmt.Errorf("rmdir failed: %w", err) 800 } 801 f.dirCache.FlushDir(dir) 802 return nil 803 } 804 805 // Rmdir deletes the root folder 806 // 807 // Returns an error if it isn't empty 808 func (f *Fs) Rmdir(ctx context.Context, dir string) error { 809 return f.purgeCheck(ctx, dir, true) 810 } 811 812 // Purge deletes all the files and the container 813 // 814 // Optional interface: Only implement this if you have a way of 815 // deleting all the files quicker than just running Remove() on the 816 // result of List() 817 func (f *Fs) Purge(ctx context.Context, dir string) error { 818 return f.purgeCheck(ctx, dir, false) 819 } 820 821 func (f *Fs) rename(ctx context.Context, id, name string) (item *api.Item, err error) { 822 var resp *http.Response 823 opts := rest.Opts{ 824 Method: "PATCH", 825 Path: "/files/" + id, 826 ExtraHeaders: map[string]string{"Accept": "application/vnd.api+json"}, 827 } 828 rename := api.WriteMetadataRequest{ 829 Data: api.WriteMetadata{ 830 Attributes: api.WriteAttributes{ 831 Name: f.opt.Enc.FromStandardName(name), 832 }, 833 Type: "files", 834 }, 835 } 836 var result *api.ItemInfo 837 err = f.pacer.Call(func() (bool, error) { 838 resp, err = f.srv.CallJSON(ctx, &opts, &rename, &result) 839 return shouldRetry(ctx, resp, err) 840 }) 841 if err != nil { 842 return nil, fmt.Errorf("rename failed: %w", err) 843 } 844 return &result.Item, nil 845 } 846 847 // Copy src to this remote using server side copy operations. 848 // 849 // This is stored with the remote path given. 850 // 851 // It returns the destination Object and a possible error. 852 // 853 // Will only be called if src.Fs().Name() == f.Name() 854 // 855 // If it isn't possible then return fs.ErrorCantCopy 856 func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { 857 srcObj, ok := src.(*Object) 858 if !ok { 859 fs.Debugf(src, "Can't copy - not same remote type") 860 return nil, fs.ErrorCantCopy 861 } 862 err := srcObj.readMetaData(ctx) 863 if err != nil { 864 return nil, err 865 } 866 867 // Create temporary object 868 dstObject, leaf, directoryID, err := f.createObject(ctx, remote, srcObj.size, srcObj.modTime) 869 if err != nil { 870 return nil, err 871 } 872 // Copy the object 873 opts := rest.Opts{ 874 Method: "POST", 875 Path: "/files/" + directoryID + "/copy", 876 ExtraHeaders: map[string]string{"Accept": "application/vnd.api+json"}, 877 } 878 copyFile := api.WriteMultiMetadataRequest{ 879 Meta: []api.WriteMetadata{ 880 { 881 Attributes: api.WriteAttributes{ 882 RessourceID: srcObj.id, 883 }, 884 Type: "files", 885 }, 886 }, 887 } 888 var resp *http.Response 889 var result *api.ItemList 890 err = f.pacer.Call(func() (bool, error) { 891 resp, err = f.srv.CallJSON(ctx, &opts, ©File, &result) 892 return shouldRetry(ctx, resp, err) 893 }) 894 if err != nil { 895 return nil, fmt.Errorf("couldn't copy file: %w", err) 896 } 897 // Server acts weird some times make sure we actually got 898 // an item 899 if len(result.Items) != 1 { 900 return nil, errors.New("couldn't copy file: invalid response") 901 } 902 // Only set ID here because response is not complete Item struct 903 dstObject.id = result.Items[0].ID 904 905 // Can't copy and change name in one step so we have to check if we have 906 // the correct name after copy 907 if f.opt.Enc.ToStandardName(result.Items[0].Attributes.Name) != leaf { 908 if err = dstObject.rename(ctx, leaf); err != nil { 909 return nil, fmt.Errorf("copy: couldn't rename copied file: %w", err) 910 } 911 } 912 return dstObject, nil 913 } 914 915 func (f *Fs) move(ctx context.Context, srcID, parentID string) (item *api.Item, err error) { 916 // Move the object 917 opts := rest.Opts{ 918 Method: "PATCH", 919 Path: "/files", 920 ExtraHeaders: map[string]string{"Accept": "application/vnd.api+json"}, 921 } 922 moveFile := api.WriteMultiMetadataRequest{ 923 Meta: []api.WriteMetadata{ 924 { 925 Attributes: api.WriteAttributes{ 926 ParentID: parentID, 927 }, 928 ID: srcID, 929 Type: "files", 930 }, 931 }, 932 } 933 var resp *http.Response 934 var result *api.ItemList 935 err = f.pacer.Call(func() (bool, error) { 936 resp, err = f.srv.CallJSON(ctx, &opts, &moveFile, &result) 937 return shouldRetry(ctx, resp, err) 938 }) 939 if err != nil { 940 return nil, fmt.Errorf("move failed: %w", err) 941 } 942 // Server acts weird some times make sure our array actually contains 943 // a file 944 if len(result.Items) != 1 { 945 return nil, errors.New("move failed: invalid response") 946 } 947 return &result.Items[0], nil 948 } 949 950 // Move src to this remote using server side move operations. 951 // 952 // This is stored with the remote path given. 953 // 954 // It returns the destination Object and a possible error. 955 // 956 // Will only be called if src.Fs().Name() == f.Name() 957 // 958 // If it isn't possible then return fs.ErrorCantMove 959 func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { 960 srcObj, ok := src.(*Object) 961 if !ok { 962 fs.Debugf(src, "Can't move - not same remote type") 963 return nil, fs.ErrorCantMove 964 } 965 err := srcObj.readMetaData(ctx) 966 if err != nil { 967 return nil, err 968 } 969 970 srcLeaf, srcParentID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false) 971 if err != nil { 972 return nil, err 973 } 974 975 // Create temporary object 976 dstObject, dstLeaf, directoryID, err := f.createObject(ctx, remote, srcObj.size, srcObj.modTime) 977 if err != nil { 978 return nil, err 979 } 980 981 needRename := srcLeaf != dstLeaf 982 needMove := srcParentID != directoryID 983 984 // rename the leaf to a temporary name if we are moving to 985 // another directory to make sure we don't overwrite something 986 // in the source directory by accident 987 if needRename && needMove { 988 tmpLeaf := "rcloneTemp" + random.String(8) 989 if err = srcObj.rename(ctx, tmpLeaf); err != nil { 990 return nil, fmt.Errorf("move: pre move rename failed: %w", err) 991 } 992 } 993 994 // do the move if required 995 if needMove { 996 item, err := f.move(ctx, srcObj.id, directoryID) 997 if err != nil { 998 return nil, err 999 } 1000 // Only set ID here because response is not complete Item struct 1001 dstObject.id = item.ID 1002 } else { 1003 // same parent only need to rename 1004 dstObject.id = srcObj.id 1005 } 1006 1007 // rename the leaf to its final name 1008 if needRename { 1009 if err = dstObject.rename(ctx, dstLeaf); err != nil { 1010 return nil, fmt.Errorf("move: couldn't rename moved file: %w", err) 1011 } 1012 } 1013 return dstObject, nil 1014 } 1015 1016 // DirMove moves src, srcRemote to this remote at dstRemote 1017 // using server side move operations. 1018 // 1019 // Will only be called if src.Fs().Name() == f.Name() 1020 // 1021 // If it isn't possible then return fs.ErrorCantDirMove 1022 // 1023 // If destination exists then return fs.ErrorDirExists 1024 func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { 1025 srcFs, ok := src.(*Fs) 1026 if !ok { 1027 fs.Debugf(srcFs, "Can't move directory - not same remote type") 1028 return fs.ErrorCantDirMove 1029 } 1030 1031 srcID, srcDirectoryID, srcLeaf, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote) 1032 if err != nil { 1033 return err 1034 } 1035 // same parent only need to rename 1036 if srcDirectoryID == dstDirectoryID { 1037 _, err = f.rename(ctx, srcID, dstLeaf) 1038 return err 1039 } 1040 1041 // do the move 1042 _, err = f.move(ctx, srcID, dstDirectoryID) 1043 if err != nil { 1044 return fmt.Errorf("couldn't dir move: %w", err) 1045 } 1046 1047 // Can't copy and change name in one step so we have to check if we have 1048 // the correct name after copy 1049 if srcLeaf != dstLeaf { 1050 _, err = f.rename(ctx, srcID, dstLeaf) 1051 if err != nil { 1052 return fmt.Errorf("dirmove: couldn't rename moved dir: %w", err) 1053 } 1054 } 1055 srcFs.dirCache.FlushDir(srcRemote) 1056 return nil 1057 } 1058 1059 // About gets quota information 1060 func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) { 1061 info, err := f.readMetaDataForID(ctx, f.opt.RootFolderID) 1062 if err != nil { 1063 return nil, err 1064 } 1065 usage = &fs.Usage{ 1066 Used: fs.NewUsageValue(info.Attributes.StorageInfo.Size), 1067 } 1068 return usage, nil 1069 } 1070 1071 // DirCacheFlush resets the directory cache - used in testing as an 1072 // optional interface 1073 func (f *Fs) DirCacheFlush() { 1074 f.dirCache.ResetRoot() 1075 } 1076 1077 // ------------------------------------------------------------ 1078 1079 // Fs returns the parent Fs 1080 func (o *Object) Fs() fs.Info { 1081 return o.fs 1082 } 1083 1084 // Return a string version 1085 func (o *Object) String() string { 1086 if o == nil { 1087 return "<nil>" 1088 } 1089 return o.remote 1090 } 1091 1092 // Remote returns the remote path 1093 func (o *Object) Remote() string { 1094 return o.remote 1095 } 1096 1097 // Storable returns a boolean showing whether this object storable 1098 func (o *Object) Storable() bool { 1099 return true 1100 } 1101 1102 // Hash returns the SHA-1 of an object returning a lowercase hex string 1103 func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { 1104 return "", nil 1105 } 1106 1107 // Size returns the size of an object in bytes 1108 func (o *Object) Size() int64 { 1109 if err := o.readMetaData(context.TODO()); err != nil { 1110 fs.Logf(o, "Failed to read metadata: %v", err) 1111 return 0 1112 } 1113 return o.size 1114 } 1115 1116 // setMetaData sets the metadata from info 1117 func (o *Object) setMetaData(info *api.Item) (err error) { 1118 if info.Attributes.IsFolder { 1119 return fs.ErrorIsDir 1120 } 1121 o.hasMetaData = true 1122 o.size = info.Attributes.StorageInfo.Size 1123 o.modTime = time.Time(info.Attributes.ModifiedTime) 1124 o.id = info.ID 1125 return nil 1126 } 1127 1128 // readMetaData gets the metadata if it hasn't already been fetched 1129 // 1130 // it also sets the info 1131 func (o *Object) readMetaData(ctx context.Context) (err error) { 1132 if o.hasMetaData { 1133 return nil 1134 } 1135 info, err := o.fs.readMetaDataForPath(ctx, o.remote) 1136 if err != nil { 1137 return err 1138 } 1139 return o.setMetaData(info) 1140 } 1141 1142 // ModTime returns the modification time of the object 1143 // 1144 // It attempts to read the objects mtime and if that isn't present the 1145 // LastModified returned in the http headers 1146 func (o *Object) ModTime(ctx context.Context) time.Time { 1147 return o.modTime 1148 } 1149 1150 // SetModTime sets the modification time of the local fs object 1151 func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { 1152 return fs.ErrorCantSetModTime 1153 } 1154 1155 // rename renames an object in place 1156 // 1157 // this a separate api call then move with zoho 1158 func (o *Object) rename(ctx context.Context, name string) (err error) { 1159 item, err := o.fs.rename(ctx, o.id, name) 1160 if err != nil { 1161 return err 1162 } 1163 return o.setMetaData(item) 1164 } 1165 1166 // Open an object for read 1167 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { 1168 if o.id == "" { 1169 return nil, errors.New("can't download - no id") 1170 } 1171 var resp *http.Response 1172 fs.FixRangeOption(options, o.size) 1173 opts := rest.Opts{ 1174 Method: "GET", 1175 Path: "/download/" + o.id, 1176 Options: options, 1177 } 1178 err = o.fs.pacer.Call(func() (bool, error) { 1179 resp, err = o.fs.srv.Call(ctx, &opts) 1180 return shouldRetry(ctx, resp, err) 1181 }) 1182 if err != nil { 1183 return nil, err 1184 } 1185 return resp.Body, nil 1186 } 1187 1188 // Update the object with the contents of the io.Reader, modTime and size 1189 // 1190 // If existing is set then it updates the object rather than creating a new one. 1191 // 1192 // The new object may have been created if an error is returned 1193 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { 1194 size := src.Size() 1195 remote := o.Remote() 1196 1197 // Create the directory for the object if it doesn't exist 1198 leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true) 1199 if err != nil { 1200 return err 1201 } 1202 1203 if isSimpleName(leaf) { 1204 // Simple name we can just overwrite the old file 1205 info, err := o.fs.upload(ctx, o.fs.opt.Enc.FromStandardName(leaf), directoryID, size, in, options...) 1206 if err != nil { 1207 return err 1208 } 1209 return o.setMetaData(info) 1210 } 1211 1212 // We have to fall back to upload + rename 1213 tempName := "rcloneTemp" + random.String(8) 1214 info, err := o.fs.upload(ctx, tempName, directoryID, size, in, options...) 1215 if err != nil { 1216 return err 1217 } 1218 1219 // upload was successful, need to delete old object before rename 1220 if err = o.Remove(ctx); err != nil { 1221 return fmt.Errorf("failed to remove old object: %w", err) 1222 } 1223 if err = o.setMetaData(info); err != nil { 1224 return err 1225 } 1226 1227 // rename also updates metadata 1228 return o.rename(ctx, leaf) 1229 } 1230 1231 // Remove an object 1232 func (o *Object) Remove(ctx context.Context) error { 1233 return o.fs.deleteObject(ctx, o.id) 1234 } 1235 1236 // ID returns the ID of the Object if known, or "" if not 1237 func (o *Object) ID() string { 1238 return o.id 1239 } 1240 1241 // Check the interfaces are satisfied 1242 var ( 1243 _ fs.Fs = (*Fs)(nil) 1244 _ fs.Purger = (*Fs)(nil) 1245 _ fs.Copier = (*Fs)(nil) 1246 _ fs.Abouter = (*Fs)(nil) 1247 _ fs.Mover = (*Fs)(nil) 1248 _ fs.DirMover = (*Fs)(nil) 1249 _ fs.DirCacheFlusher = (*Fs)(nil) 1250 _ fs.Object = (*Object)(nil) 1251 _ fs.IDer = (*Object)(nil) 1252 )