github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/backend/quatrix/quatrix.go (about) 1 // Package quatrix provides an interface to the Quatrix by Maytech 2 // object storage system. 3 package quatrix 4 5 // FIXME Quatrix only supports file names of 255 characters or less. Names 6 // that will not be supported are those that contain non-printable 7 // ascii, / or \, names with trailing spaces, and the special names 8 // “.” and “..”. 9 10 import ( 11 "context" 12 "errors" 13 "fmt" 14 "io" 15 "net/http" 16 "net/url" 17 "path" 18 "strconv" 19 "strings" 20 "time" 21 22 "github.com/rclone/rclone/backend/quatrix/api" 23 "github.com/rclone/rclone/fs" 24 "github.com/rclone/rclone/fs/config" 25 "github.com/rclone/rclone/fs/config/configmap" 26 "github.com/rclone/rclone/fs/config/configstruct" 27 "github.com/rclone/rclone/fs/fserrors" 28 "github.com/rclone/rclone/fs/fshttp" 29 "github.com/rclone/rclone/fs/hash" 30 "github.com/rclone/rclone/lib/dircache" 31 "github.com/rclone/rclone/lib/encoder" 32 "github.com/rclone/rclone/lib/multipart" 33 "github.com/rclone/rclone/lib/pacer" 34 "github.com/rclone/rclone/lib/rest" 35 ) 36 37 const ( 38 minSleep = 10 * time.Millisecond 39 maxSleep = 2 * time.Second 40 decayConstant = 2 // bigger for slower decay, exponential 41 rootURL = "https://%s/api/1.0/" 42 uploadURL = "https://%s/upload/chunked/" 43 44 unlimitedUserQuota = -1 45 ) 46 47 func init() { 48 fs.Register(&fs.RegInfo{ 49 Name: "quatrix", 50 Description: "Quatrix by Maytech", 51 NewFs: NewFs, 52 Options: fs.Options{ 53 { 54 Name: "api_key", 55 Help: "API key for accessing Quatrix account", 56 Required: true, 57 Sensitive: true, 58 }, 59 { 60 Name: "host", 61 Help: "Host name of Quatrix account", 62 Required: true, 63 }, 64 { 65 Name: config.ConfigEncoding, 66 Help: config.ConfigEncodingHelp, 67 Advanced: true, 68 Default: encoder.Standard | 69 encoder.EncodeBackSlash | 70 encoder.EncodeInvalidUtf8, 71 }, 72 { 73 Name: "effective_upload_time", 74 Help: "Wanted upload time for one chunk", 75 Advanced: true, 76 Default: "4s", 77 }, 78 { 79 Name: "minimal_chunk_size", 80 Help: "The minimal size for one chunk", 81 Advanced: true, 82 Default: fs.SizeSuffix(10_000_000), 83 }, 84 { 85 Name: "maximal_summary_chunk_size", 86 Help: "The maximal summary for all chunks. It should not be less than 'transfers'*'minimal_chunk_size'", 87 Advanced: true, 88 Default: fs.SizeSuffix(100_000_000), 89 }, 90 { 91 Name: "hard_delete", 92 Help: "Delete files permanently rather than putting them into the trash", 93 Advanced: true, 94 Default: false, 95 }, 96 { 97 Name: "skip_project_folders", 98 Help: "Skip project folders in operations", 99 Advanced: true, 100 Default: false, 101 }, 102 }, 103 }) 104 } 105 106 // Options defines the configuration for Quatrix backend 107 type Options struct { 108 APIKey string `config:"api_key"` 109 Host string `config:"host"` 110 Enc encoder.MultiEncoder `config:"encoding"` 111 EffectiveUploadTime fs.Duration `config:"effective_upload_time"` 112 MinimalChunkSize fs.SizeSuffix `config:"minimal_chunk_size"` 113 MaximalSummaryChunkSize fs.SizeSuffix `config:"maximal_summary_chunk_size"` 114 HardDelete bool `config:"hard_delete"` 115 SkipProjectFolders bool `config:"skip_project_folders"` 116 } 117 118 // Fs represents remote Quatrix fs 119 type Fs struct { 120 name string 121 root string 122 description string 123 features *fs.Features 124 opt Options 125 ci *fs.ConfigInfo 126 srv *rest.Client // the connection to the quatrix server 127 pacer *fs.Pacer // pacer for API calls 128 dirCache *dircache.DirCache 129 uploadMemoryManager *UploadMemoryManager 130 } 131 132 // Object describes a quatrix object 133 type Object struct { 134 fs *Fs 135 remote string 136 size int64 137 modTime time.Time 138 id string 139 hasMetaData bool 140 obType string 141 } 142 143 // trimPath trims redundant slashes from quatrix 'url' 144 func trimPath(path string) (root string) { 145 root = strings.Trim(path, "/") 146 return 147 } 148 149 // retryErrorCodes is a slice of error codes that we will retry 150 var retryErrorCodes = []int{ 151 429, // Too Many Requests. 152 500, // Internal Server Error 153 502, // Bad Gateway 154 503, // Service Unavailable 155 504, // Gateway Timeout 156 509, // Bandwidth Limit Exceeded 157 } 158 159 // shouldRetry returns a boolean as to whether this resp and err 160 // deserve to be retried. It returns the err as a convenience 161 func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { 162 if fserrors.ContextError(ctx, &err) { 163 return false, err 164 } 165 166 return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err 167 } 168 169 // NewFs constructs an Fs from the path, container:path 170 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { 171 // Parse config into Options struct 172 opt := new(Options) 173 174 err := configstruct.Set(m, opt) 175 if err != nil { 176 return nil, err 177 } 178 179 // http client 180 client := fshttp.NewClient(ctx) 181 182 // since transport is a global variable that is initialized only once (due to sync.Once) 183 // we need to reset it to have correct transport per each client (with proper values extracted from rclone config) 184 client.Transport = fshttp.NewTransportCustom(ctx, nil) 185 186 root = trimPath(root) 187 188 ci := fs.GetConfig(ctx) 189 190 f := &Fs{ 191 name: name, 192 description: "Quatrix FS for account " + opt.Host, 193 root: root, 194 opt: *opt, 195 ci: ci, 196 srv: rest.NewClient(client).SetRoot(fmt.Sprintf(rootURL, opt.Host)), 197 pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))), 198 } 199 200 f.features = (&fs.Features{ 201 CaseInsensitive: false, 202 CanHaveEmptyDirectories: true, 203 PartialUploads: true, 204 }).Fill(ctx, f) 205 206 if f.opt.APIKey != "" { 207 f.srv.SetHeader("Authorization", "Bearer "+f.opt.APIKey) 208 } 209 210 f.uploadMemoryManager = NewUploadMemoryManager(f.ci, &f.opt) 211 212 // get quatrix root(home) id 213 rootID, found, err := f.fileID(ctx, "", "") 214 if err != nil { 215 return nil, err 216 } 217 218 if !found { 219 return nil, errors.New("root not found") 220 } 221 222 f.dirCache = dircache.New(root, rootID.FileID, f) 223 224 err = f.dirCache.FindRoot(ctx, false) 225 if err != nil { 226 fileID, found, err := f.fileID(ctx, "", root) 227 if err != nil { 228 return nil, fmt.Errorf("find root %s: %w", root, err) 229 } 230 231 if !found { 232 return f, nil 233 } 234 235 if fileID.IsFile() { 236 root, _ = dircache.SplitPath(root) 237 f.dirCache = dircache.New(root, rootID.FileID, f) 238 // Correct root if definitely pointing to a file 239 f.root = path.Dir(f.root) 240 if f.root == "." || f.root == "/" { 241 f.root = "" 242 } 243 244 return f, fs.ErrorIsFile 245 } 246 } 247 248 return f, nil 249 } 250 251 // fileID gets id, parent and type of path in given parentID 252 func (f *Fs) fileID(ctx context.Context, parentID, path string) (result *api.FileInfo, found bool, err error) { 253 opts := rest.Opts{ 254 Method: "POST", 255 Path: "file/id", 256 IgnoreStatus: true, 257 } 258 259 payload := api.FileInfoParams{ 260 Path: f.opt.Enc.FromStandardPath(path), 261 ParentID: parentID, 262 } 263 264 result = &api.FileInfo{} 265 266 err = f.pacer.Call(func() (bool, error) { 267 resp, err := f.srv.CallJSON(ctx, &opts, payload, result) 268 if resp != nil && resp.StatusCode == http.StatusNotFound { 269 return false, nil 270 } 271 return shouldRetry(ctx, resp, err) 272 }) 273 if err != nil { 274 return nil, false, fmt.Errorf("failed to get file id: %w", err) 275 } 276 277 if result.FileID == "" { 278 return nil, false, nil 279 } 280 281 return result, true, nil 282 } 283 284 // FindLeaf finds a directory of name leaf in the folder with ID pathID 285 func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (folderID string, found bool, err error) { 286 result, found, err := f.fileID(ctx, pathID, leaf) 287 if err != nil { 288 return "", false, fmt.Errorf("find leaf: %w", err) 289 } 290 291 if !found { 292 return "", false, nil 293 } 294 295 if result.IsFile() { 296 return "", false, nil 297 } 298 299 return result.FileID, true, nil 300 } 301 302 // createDir creates directory in pathID with name leaf 303 // 304 // resolve - if true will resolve name conflict on server side, if false - will return error if object with this name exists 305 func (f *Fs) createDir(ctx context.Context, pathID, leaf string, resolve bool) (newDir *api.File, err error) { 306 opts := rest.Opts{ 307 Method: "POST", 308 Path: "file/makedir", 309 } 310 311 payload := api.CreateDirParams{ 312 Name: f.opt.Enc.FromStandardName(leaf), 313 Target: pathID, 314 Resolve: resolve, 315 } 316 317 newDir = &api.File{} 318 319 err = f.pacer.Call(func() (bool, error) { 320 resp, err := f.srv.CallJSON(ctx, &opts, payload, newDir) 321 return shouldRetry(ctx, resp, err) 322 }) 323 if err != nil { 324 return nil, fmt.Errorf("failed to create directory: %w", err) 325 } 326 327 return 328 } 329 330 // CreateDir makes a directory with pathID as parent and name leaf 331 func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (dirID string, err error) { 332 dir, err := f.createDir(ctx, pathID, leaf, false) 333 if err != nil { 334 return "", err 335 } 336 337 return dir.ID, nil 338 } 339 340 // Name of the remote (as passed into NewFs) 341 func (f *Fs) Name() string { 342 return f.name 343 } 344 345 // Root of the remote (as passed into NewFs) 346 func (f *Fs) Root() string { 347 return f.root 348 } 349 350 // String converts this Fs to a string 351 func (f *Fs) String() string { 352 return f.description + " at " + f.root 353 } 354 355 // Precision return the precision of this Fs 356 func (f *Fs) Precision() time.Duration { 357 return time.Microsecond 358 } 359 360 // Hashes returns the supported hash sets. 361 func (f *Fs) Hashes() hash.Set { 362 return 0 363 } 364 365 // Features returns the optional features of this Fs 366 func (f *Fs) Features() *fs.Features { 367 return f.features 368 } 369 370 // List the objects and directories in dir into entries. The 371 // entries can be returned in any order but should be for a 372 // complete directory. 373 // 374 // dir should be "" to list the root, and should not have 375 // trailing slashes. 376 // 377 // This should return ErrDirNotFound if the directory isn't 378 // found. 379 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { 380 directoryID, err := f.dirCache.FindDir(ctx, dir, false) 381 if err != nil { 382 return nil, err 383 } 384 385 folder, err := f.metadata(ctx, directoryID, true) 386 if err != nil { 387 return nil, err 388 } 389 390 for _, file := range folder.Content { 391 if f.skipFile(&file) { 392 continue 393 } 394 395 remote := path.Join(dir, f.opt.Enc.ToStandardName(file.Name)) 396 if file.IsDir() { 397 f.dirCache.Put(remote, file.ID) 398 399 d := fs.NewDir(remote, time.Time(file.Modified)).SetID(file.ID).SetItems(file.Size) 400 // FIXME more info from dir? 401 entries = append(entries, d) 402 } else { 403 o := &Object{ 404 fs: f, 405 remote: remote, 406 } 407 408 err = o.setMetaData(&file) 409 if err != nil { 410 fs.Debugf(file, "failed to set object metadata: %s", err) 411 } 412 413 entries = append(entries, o) 414 } 415 } 416 417 return entries, nil 418 } 419 420 func (f *Fs) skipFile(file *api.File) bool { 421 return f.opt.SkipProjectFolders && file.IsProjectFolder() 422 } 423 424 // NewObject finds the Object at remote. If it can't be found 425 // it returns the error fs.ErrorObjectNotFound. 426 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { 427 return f.newObjectWithInfo(ctx, remote, nil) 428 } 429 430 // Creates from the parameters passed in a half finished Object which 431 // must have setMetaData called on it 432 // 433 // Returns the object, leaf, directoryID and error. 434 // 435 // Used to create new objects 436 func (f *Fs) createObject(ctx context.Context, remote string) (o *Object, leaf string, directoryID string, err error) { 437 // Create the directory for the object if it doesn't exist 438 leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true) 439 if err != nil { 440 return 441 } 442 // Temporary Object under construction 443 o = &Object{ 444 fs: f, 445 remote: remote, 446 } 447 return o, leaf, directoryID, nil 448 } 449 450 // Put the object into the container 451 // 452 // Copy the reader in to the new object which is returned. 453 // 454 // The new object may have been created if an error is returned 455 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 456 remote := src.Remote() 457 size := src.Size() 458 mtime := src.ModTime(ctx) 459 460 o := &Object{ 461 fs: f, 462 remote: remote, 463 size: size, 464 modTime: mtime, 465 } 466 467 return o, o.Update(ctx, in, src, options...) 468 } 469 470 func (f *Fs) rootSlash() string { 471 if f.root == "" { 472 return f.root 473 } 474 return f.root + "/" 475 } 476 477 func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.File) (fs.Object, error) { 478 o := &Object{ 479 fs: f, 480 remote: remote, 481 } 482 var err error 483 if info != nil { 484 // Set info 485 err = o.setMetaData(info) 486 } else { 487 err = o.readMetaData(ctx) // reads info and meta, returning an error 488 } 489 if err != nil { 490 return nil, err 491 } 492 return o, nil 493 } 494 495 // setMetaData sets the metadata from info 496 func (o *Object) setMetaData(info *api.File) (err error) { 497 if info.IsDir() { 498 fs.Debugf(o, "%q is %q", o.remote, info.Type) 499 return fs.ErrorIsDir 500 } 501 502 if !info.IsFile() { 503 fs.Debugf(o, "%q is %q", o.remote, info.Type) 504 return fmt.Errorf("%q is %q: %w", o.remote, info.Type, fs.ErrorNotAFile) 505 } 506 507 o.size = info.Size 508 o.modTime = time.Time(info.ModifiedMS) 509 o.id = info.ID 510 o.hasMetaData = true 511 o.obType = info.Type 512 513 return nil 514 } 515 516 func (o *Object) readMetaData(ctx context.Context) (err error) { 517 if o.hasMetaData { 518 return nil 519 } 520 521 leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false) 522 if err != nil { 523 if err == fs.ErrorDirNotFound { 524 return fs.ErrorObjectNotFound 525 } 526 return err 527 } 528 529 file, found, err := o.fs.fileID(ctx, directoryID, leaf) 530 if err != nil { 531 return fmt.Errorf("read metadata: fileID: %w", err) 532 } 533 534 if !found { 535 fs.Debugf(nil, "object not found: remote %s: directory %s: leaf %s", o.remote, directoryID, leaf) 536 return fs.ErrorObjectNotFound 537 } 538 539 result, err := o.fs.metadata(ctx, file.FileID, false) 540 if err != nil { 541 return fmt.Errorf("get file metadata: %w", err) 542 } 543 544 return o.setMetaData(result) 545 } 546 547 // Mkdir creates the container if it doesn't exist 548 func (f *Fs) Mkdir(ctx context.Context, dir string) error { 549 _, err := f.dirCache.FindDir(ctx, dir, true) 550 return err 551 } 552 553 // Rmdir deletes the root folder 554 // 555 // Returns an error if it isn't empty 556 func (f *Fs) Rmdir(ctx context.Context, dir string) error { 557 return f.purgeCheck(ctx, dir, true) 558 } 559 560 // DirCacheFlush resets the directory cache - used in testing as an 561 // optional interface 562 func (f *Fs) DirCacheFlush() { 563 f.dirCache.ResetRoot() 564 } 565 566 func (f *Fs) metadata(ctx context.Context, id string, withContent bool) (result *api.File, err error) { 567 parameters := url.Values{} 568 if !withContent { 569 parameters.Add("content", "0") 570 } 571 572 opts := rest.Opts{ 573 Method: "GET", 574 Path: path.Join("file/metadata", id), 575 Parameters: parameters, 576 } 577 578 result = &api.File{} 579 580 var resp *http.Response 581 err = f.pacer.Call(func() (bool, error) { 582 resp, err = f.srv.CallJSON(ctx, &opts, nil, result) 583 return shouldRetry(ctx, resp, err) 584 }) 585 if err != nil { 586 if resp != nil && resp.StatusCode == http.StatusNotFound { 587 return nil, fs.ErrorObjectNotFound 588 } 589 590 return nil, fmt.Errorf("failed to get file metadata: %w", err) 591 } 592 593 return result, nil 594 } 595 596 func (f *Fs) setMTime(ctx context.Context, id string, t time.Time) (result *api.File, err error) { 597 opts := rest.Opts{ 598 Method: "POST", 599 Path: "file/metadata", 600 } 601 602 params := &api.SetMTimeParams{ 603 ID: id, 604 MTime: api.JSONTime(t), 605 } 606 607 result = &api.File{} 608 609 var resp *http.Response 610 err = f.pacer.Call(func() (bool, error) { 611 resp, err = f.srv.CallJSON(ctx, &opts, params, result) 612 return shouldRetry(ctx, resp, err) 613 }) 614 if err != nil { 615 if resp != nil && resp.StatusCode == http.StatusNotFound { 616 return nil, fs.ErrorObjectNotFound 617 } 618 619 return nil, fmt.Errorf("failed to set file metadata: %w", err) 620 } 621 622 return result, nil 623 } 624 625 func (f *Fs) deleteObject(ctx context.Context, id string) error { 626 payload := &api.DeleteParams{ 627 IDs: []string{id}, 628 DeletePermanently: f.opt.HardDelete, 629 } 630 631 result := &api.IDList{} 632 633 opts := rest.Opts{ 634 Method: "POST", 635 Path: "file/delete", 636 } 637 638 err := f.pacer.Call(func() (bool, error) { 639 resp, err := f.srv.CallJSON(ctx, &opts, payload, result) 640 return shouldRetry(ctx, resp, err) 641 }) 642 if err != nil { 643 return err 644 } 645 646 for _, removedID := range result.IDs { 647 if removedID == id { 648 return nil 649 } 650 } 651 652 return fmt.Errorf("file %s was not deleted successfully", id) 653 } 654 655 func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { 656 root := path.Join(f.root, dir) 657 if root == "" { 658 return errors.New("can't purge root directory") 659 } 660 661 rootID, err := f.dirCache.FindDir(ctx, dir, false) 662 if err != nil { 663 return err 664 } 665 666 if check { 667 file, err := f.metadata(ctx, rootID, false) 668 if err != nil { 669 return err 670 } 671 672 if file.IsFile() { 673 return fs.ErrorIsFile 674 } 675 676 if file.Size != 0 { 677 return fs.ErrorDirectoryNotEmpty 678 } 679 } 680 681 err = f.deleteObject(ctx, rootID) 682 if err != nil { 683 return err 684 } 685 686 f.dirCache.FlushDir(dir) 687 688 return nil 689 } 690 691 // Purge deletes all the files in the directory 692 // 693 // Optional interface: Only implement this if you have a way of 694 // deleting all the files quicker than just running Remove() on the 695 // result of List() 696 func (f *Fs) Purge(ctx context.Context, dir string) error { 697 return f.purgeCheck(ctx, dir, false) 698 } 699 700 // Copy src to this remote using server-side copy operations. 701 // 702 // This is stored with the remote path given. 703 // 704 // It returns the destination Object and a possible error. 705 // 706 // Will only be called if src.Fs().Name() == f.Name() 707 // 708 // If it isn't possible then return fs.ErrorCantCopy 709 func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { 710 srcObj, ok := src.(*Object) 711 if !ok { 712 fs.Debugf(src, "Can't copy - not same remote type") 713 return nil, fs.ErrorCantCopy 714 } 715 716 if srcObj.fs == f { 717 srcPath := srcObj.rootPath() 718 dstPath := f.rootPath(remote) 719 if srcPath == dstPath { 720 return nil, fmt.Errorf("can't copy %q -> %q as they are same", srcPath, dstPath) 721 } 722 } 723 724 err := srcObj.readMetaData(ctx) 725 if err != nil { 726 fs.Debugf(srcObj, "read metadata for %s: %s", srcObj.rootPath(), err) 727 return nil, err 728 } 729 730 _, _, err = srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false) 731 if err != nil { 732 return nil, err 733 } 734 735 dstObj, dstLeaf, directoryID, err := f.createObject(ctx, remote) 736 if err != nil { 737 fs.Debugf(srcObj, "create empty object for %s: %s", dstObj.rootPath(), err) 738 return nil, err 739 } 740 741 opts := rest.Opts{ 742 Method: "POST", 743 Path: "file/copyone", 744 } 745 746 params := &api.FileCopyMoveOneParams{ 747 ID: srcObj.id, 748 Target: directoryID, 749 Resolve: true, 750 MTime: api.JSONTime(srcObj.ModTime(ctx)), 751 Name: dstLeaf, 752 ResolveMode: api.OverwriteMode, 753 } 754 755 result := &api.File{} 756 757 var resp *http.Response 758 759 err = f.pacer.Call(func() (bool, error) { 760 resp, err = f.srv.CallJSON(ctx, &opts, params, result) 761 return shouldRetry(ctx, resp, err) 762 }) 763 if err != nil { 764 if resp != nil && resp.StatusCode == http.StatusNotFound { 765 return nil, fs.ErrorObjectNotFound 766 } 767 768 return nil, fmt.Errorf("failed to copy: %w", err) 769 } 770 771 err = dstObj.setMetaData(result) 772 if err != nil { 773 return nil, err 774 } 775 776 return dstObj, nil 777 } 778 779 // Move src to this remote using server-side move operations. 780 // 781 // This is stored with the remote path given. 782 // 783 // It returns the destination Object and a possible error. 784 // 785 // Will only be called if src.Fs().Name() == f.Name() 786 // 787 // If it isn't possible then return fs.ErrorCantMove 788 func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { 789 srcObj, ok := src.(*Object) 790 if !ok { 791 fs.Debugf(src, "Can't move - not same remote type") 792 return nil, fs.ErrorCantMove 793 } 794 795 _, _, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false) 796 if err != nil { 797 return nil, err 798 } 799 800 // Create temporary object 801 dstObj, dstLeaf, directoryID, err := f.createObject(ctx, remote) 802 if err != nil { 803 return nil, err 804 } 805 806 opts := rest.Opts{ 807 Method: "POST", 808 Path: "file/moveone", 809 } 810 811 params := &api.FileCopyMoveOneParams{ 812 ID: srcObj.id, 813 Target: directoryID, 814 Resolve: true, 815 MTime: api.JSONTime(srcObj.ModTime(ctx)), 816 Name: dstLeaf, 817 ResolveMode: api.OverwriteMode, 818 } 819 820 var resp *http.Response 821 result := &api.File{} 822 823 err = f.pacer.Call(func() (bool, error) { 824 resp, err = f.srv.CallJSON(ctx, &opts, params, result) 825 return shouldRetry(ctx, resp, err) 826 }) 827 if err != nil { 828 if resp != nil && resp.StatusCode == http.StatusNotFound { 829 return nil, fs.ErrorObjectNotFound 830 } 831 832 return nil, fmt.Errorf("failed to move: %w", err) 833 } 834 835 err = dstObj.setMetaData(result) 836 if err != nil { 837 return nil, err 838 } 839 840 return dstObj, nil 841 } 842 843 // DirMove moves src, srcRemote to this remote at dstRemote 844 // using server-side move operations. 845 // 846 // Will only be called if src.Fs().Name() == f.Name() 847 // 848 // If it isn't possible then return fs.ErrorCantDirMove 849 // 850 // If destination exists then return fs.ErrorDirExists 851 func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { 852 srcFs, ok := src.(*Fs) 853 if !ok { 854 fs.Debugf(srcFs, "Can't move directory - not same remote type") 855 return fs.ErrorCantDirMove 856 } 857 858 srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote) 859 if err != nil { 860 return err 861 } 862 863 srcInfo, err := f.metadata(ctx, srcID, false) 864 if err != nil { 865 return err 866 } 867 868 opts := rest.Opts{ 869 Method: "POST", 870 Path: "file/moveone", 871 } 872 873 params := &api.FileCopyMoveOneParams{ 874 ID: srcID, 875 Target: dstDirectoryID, 876 Resolve: false, 877 MTime: srcInfo.ModifiedMS, 878 Name: dstLeaf, 879 } 880 881 var resp *http.Response 882 result := &api.File{} 883 884 err = f.pacer.Call(func() (bool, error) { 885 resp, err = f.srv.CallJSON(ctx, &opts, params, result) 886 return shouldRetry(ctx, resp, err) 887 }) 888 if err != nil { 889 if resp != nil && resp.StatusCode == http.StatusNotFound { 890 return fs.ErrorObjectNotFound 891 } 892 893 return fmt.Errorf("failed to move dir: %w", err) 894 } 895 896 srcFs.dirCache.FlushDir(srcRemote) 897 898 return nil 899 } 900 901 // About gets quota information 902 func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) { 903 opts := rest.Opts{ 904 Method: "GET", 905 Path: "profile/info", 906 } 907 var ( 908 user api.ProfileInfo 909 resp *http.Response 910 ) 911 912 err = f.pacer.Call(func() (bool, error) { 913 resp, err = f.srv.CallJSON(ctx, &opts, nil, &user) 914 return shouldRetry(ctx, resp, err) 915 }) 916 if err != nil { 917 return nil, fmt.Errorf("failed to read profile info: %w", err) 918 } 919 920 free := user.AccLimit - user.UserUsed 921 922 if user.UserLimit > unlimitedUserQuota { 923 free = user.UserLimit - user.UserUsed 924 } 925 926 usage = &fs.Usage{ 927 Used: fs.NewUsageValue(user.UserUsed), // bytes in use 928 Total: fs.NewUsageValue(user.AccLimit), // bytes total 929 Free: fs.NewUsageValue(free), // bytes free 930 } 931 932 return usage, nil 933 } 934 935 // Fs return the parent Fs 936 func (o *Object) Fs() fs.Info { 937 return o.fs 938 } 939 940 // String returns object remote path 941 func (o *Object) String() string { 942 if o == nil { 943 return "<nil>" 944 } 945 return o.remote 946 } 947 948 // Remote returns the remote path 949 func (o *Object) Remote() string { 950 return o.remote 951 } 952 953 // rootPath returns a path for use in server given a remote 954 func (f *Fs) rootPath(remote string) string { 955 return f.rootSlash() + remote 956 } 957 958 // rootPath returns a path for use in local functions 959 func (o *Object) rootPath() string { 960 return o.fs.rootPath(o.remote) 961 } 962 963 // Size returns the size of an object in bytes 964 func (o *Object) Size() int64 { 965 err := o.readMetaData(context.TODO()) 966 if err != nil { 967 fs.Logf(o, "Failed to read metadata: %v", err) 968 return 0 969 } 970 971 return o.size 972 } 973 974 // ModTime returns the modification time of the object 975 func (o *Object) ModTime(ctx context.Context) time.Time { 976 err := o.readMetaData(ctx) 977 if err != nil { 978 fs.Logf(o, "Failed to read metadata: %v", err) 979 return time.Now() 980 } 981 982 return o.modTime 983 } 984 985 // Storable returns a boolean showing whether this object storable 986 func (o *Object) Storable() bool { 987 return true 988 } 989 990 // ID returns the ID of the Object if known, or "" if not 991 func (o *Object) ID() string { 992 return o.id 993 } 994 995 // Hash returns the SHA-1 of an object. Not supported yet. 996 func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) { 997 return "", nil 998 } 999 1000 // Remove an object 1001 func (o *Object) Remove(ctx context.Context) error { 1002 err := o.fs.deleteObject(ctx, o.id) 1003 if err != nil { 1004 return err 1005 } 1006 1007 if o.obType != "F" { 1008 o.fs.dirCache.FlushDir(o.remote) 1009 } 1010 1011 return nil 1012 } 1013 1014 // Open an object for read 1015 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { 1016 if o.id == "" { 1017 return nil, errors.New("can't download - no id") 1018 } 1019 1020 linkID, err := o.fs.downloadLink(ctx, o.id) 1021 if err != nil { 1022 return nil, err 1023 } 1024 1025 fs.FixRangeOption(options, o.size) 1026 1027 opts := rest.Opts{ 1028 Method: "GET", 1029 Path: "/file/download/" + linkID, 1030 Options: options, 1031 } 1032 1033 var resp *http.Response 1034 1035 err = o.fs.pacer.Call(func() (bool, error) { 1036 resp, err = o.fs.srv.Call(ctx, &opts) 1037 return shouldRetry(ctx, resp, err) 1038 }) 1039 if err != nil { 1040 return nil, err 1041 } 1042 return resp.Body, err 1043 } 1044 1045 func (f *Fs) downloadLink(ctx context.Context, id string) (linkID string, err error) { 1046 linkParams := &api.IDList{ 1047 IDs: []string{id}, 1048 } 1049 opts := rest.Opts{ 1050 Method: "POST", 1051 Path: "file/download-link", 1052 } 1053 1054 var resp *http.Response 1055 link := &api.DownloadLinkResponse{} 1056 1057 err = f.pacer.Call(func() (bool, error) { 1058 resp, err = f.srv.CallJSON(ctx, &opts, linkParams, &link) 1059 return shouldRetry(ctx, resp, err) 1060 }) 1061 if err != nil { 1062 return "", err 1063 } 1064 return link.ID, nil 1065 } 1066 1067 // SetModTime sets the modification time of the local fs object 1068 func (o *Object) SetModTime(ctx context.Context, t time.Time) error { 1069 file, err := o.fs.setMTime(ctx, o.id, t) 1070 if err != nil { 1071 return fmt.Errorf("set mtime: %w", err) 1072 } 1073 1074 return o.setMetaData(file) 1075 } 1076 1077 // Update the object with the contents of the io.Reader, modTime and size 1078 // 1079 // The new object may have been created if an error is returned 1080 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { 1081 size := src.Size() 1082 modTime := src.ModTime(ctx) 1083 remote := o.Remote() 1084 1085 // Create the directory for the object if it doesn't exist 1086 leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true) 1087 if err != nil { 1088 return err 1089 } 1090 1091 uploadSession, err := o.uploadSession(ctx, directoryID, leaf) 1092 if err != nil { 1093 return fmt.Errorf("object update: %w", err) 1094 } 1095 1096 o.id = uploadSession.FileID 1097 1098 defer func() { 1099 if err == nil { 1100 return 1101 } 1102 1103 deleteErr := o.fs.deleteObject(ctx, o.id) 1104 if deleteErr != nil { 1105 fs.Logf(o.remote, "remove: %s", deleteErr) 1106 } 1107 }() 1108 1109 return o.dynamicUpload(ctx, size, modTime, in, uploadSession, options...) 1110 } 1111 1112 // dynamicUpload uploads object in chunks, which are being dynamically recalculated on each iteration 1113 // depending on upload speed in order to make upload faster 1114 func (o *Object) dynamicUpload(ctx context.Context, size int64, modTime time.Time, in io.Reader, 1115 uploadSession *api.UploadLinkResponse, options ...fs.OpenOption) error { 1116 var ( 1117 speed float64 1118 localChunk int64 1119 ) 1120 1121 defer o.fs.uploadMemoryManager.Return(o.id) 1122 1123 for offset := int64(0); offset < size; offset += localChunk { 1124 localChunk = o.fs.uploadMemoryManager.Consume(o.id, size-offset, speed) 1125 1126 rw := multipart.NewRW() 1127 1128 _, err := io.CopyN(rw, in, localChunk) 1129 if err != nil { 1130 return fmt.Errorf("read chunk with offset %d size %d: %w", offset, localChunk, err) 1131 } 1132 1133 start := time.Now() 1134 1135 err = o.upload(ctx, uploadSession.UploadKey, rw, size, offset, localChunk, options...) 1136 if err != nil { 1137 return fmt.Errorf("upload chunk with offset %d size %d: %w", offset, localChunk, err) 1138 } 1139 1140 speed = float64(localChunk) / (float64(time.Since(start)) / 1e9) 1141 } 1142 1143 o.fs.uploadMemoryManager.Return(o.id) 1144 1145 finalizeResult, err := o.finalize(ctx, uploadSession.UploadKey, modTime) 1146 if err != nil { 1147 return fmt.Errorf("upload %s finalize: %w", uploadSession.UploadKey, err) 1148 } 1149 1150 if size >= 0 && finalizeResult.FileSize != size { 1151 return fmt.Errorf("expected size %d, got %d", size, finalizeResult.FileSize) 1152 } 1153 1154 o.size = size 1155 o.modTime = modTime 1156 1157 return nil 1158 } 1159 1160 func (f *Fs) uploadLink(ctx context.Context, parentID, name string) (upload *api.UploadLinkResponse, err error) { 1161 opts := rest.Opts{ 1162 Method: "POST", 1163 Path: "upload/link", 1164 } 1165 1166 payload := api.UploadLinkParams{ 1167 Name: name, 1168 ParentID: parentID, 1169 Resolve: false, 1170 } 1171 1172 err = f.pacer.Call(func() (bool, error) { 1173 resp, err := f.srv.CallJSON(ctx, &opts, &payload, &upload) 1174 return shouldRetry(ctx, resp, err) 1175 }) 1176 if err != nil { 1177 return nil, fmt.Errorf("failed to get upload link: %w", err) 1178 } 1179 1180 return upload, nil 1181 } 1182 1183 func (f *Fs) modifyLink(ctx context.Context, fileID string) (upload *api.UploadLinkResponse, err error) { 1184 opts := rest.Opts{ 1185 Method: "POST", 1186 Path: "file/modify", 1187 } 1188 1189 payload := api.FileModifyParams{ 1190 ID: fileID, 1191 Truncate: 0, 1192 } 1193 1194 err = f.pacer.Call(func() (bool, error) { 1195 resp, err := f.srv.CallJSON(ctx, &opts, &payload, &upload) 1196 return shouldRetry(ctx, resp, err) 1197 }) 1198 if err != nil { 1199 return nil, fmt.Errorf("failed to get modify link: %w", err) 1200 } 1201 1202 return upload, nil 1203 } 1204 1205 func (o *Object) uploadSession(ctx context.Context, parentID, name string) (upload *api.UploadLinkResponse, err error) { 1206 encName := o.fs.opt.Enc.FromStandardName(name) 1207 fileID, found, err := o.fs.fileID(ctx, parentID, encName) 1208 if err != nil { 1209 return nil, fmt.Errorf("get file_id: %w", err) 1210 } 1211 1212 if found { 1213 return o.fs.modifyLink(ctx, fileID.FileID) 1214 } 1215 1216 return o.fs.uploadLink(ctx, parentID, encName) 1217 } 1218 1219 func (o *Object) upload(ctx context.Context, uploadKey string, chunk io.Reader, fullSize int64, offset int64, chunkSize int64, options ...fs.OpenOption) (err error) { 1220 opts := rest.Opts{ 1221 Method: "POST", 1222 RootURL: fmt.Sprintf(uploadURL, o.fs.opt.Host) + uploadKey, 1223 Body: chunk, 1224 ContentLength: &chunkSize, 1225 ContentRange: fmt.Sprintf("bytes %d-%d/%d", offset, offset+chunkSize-1, fullSize), 1226 Options: options, 1227 } 1228 1229 var fileID string 1230 1231 err = o.fs.pacer.Call(func() (bool, error) { 1232 resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &fileID) 1233 return shouldRetry(ctx, resp, err) 1234 }) 1235 if err != nil { 1236 return fmt.Errorf("failed to get upload chunk: %w", err) 1237 } 1238 1239 return nil 1240 } 1241 1242 func (o *Object) finalize(ctx context.Context, uploadKey string, mtime time.Time) (result *api.UploadFinalizeResponse, err error) { 1243 queryParams := url.Values{} 1244 queryParams.Add("mtime", strconv.FormatFloat(float64(mtime.UTC().UnixNano())/1e9, 'f', 6, 64)) 1245 1246 opts := rest.Opts{ 1247 Method: "GET", 1248 Path: path.Join("upload/finalize", uploadKey), 1249 Parameters: queryParams, 1250 } 1251 1252 result = &api.UploadFinalizeResponse{} 1253 1254 err = o.fs.pacer.Call(func() (bool, error) { 1255 resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, result) 1256 return shouldRetry(ctx, resp, err) 1257 }) 1258 if err != nil { 1259 return nil, fmt.Errorf("failed to finalize: %w", err) 1260 } 1261 1262 return result, nil 1263 } 1264 1265 // Check the interfaces are satisfied 1266 var ( 1267 _ fs.Fs = (*Fs)(nil) 1268 _ fs.Purger = (*Fs)(nil) 1269 _ fs.Copier = (*Fs)(nil) 1270 _ fs.Abouter = (*Fs)(nil) 1271 _ fs.Mover = (*Fs)(nil) 1272 _ fs.DirMover = (*Fs)(nil) 1273 _ dircache.DirCacher = (*Fs)(nil) 1274 _ fs.DirCacheFlusher = (*Fs)(nil) 1275 _ fs.Object = (*Object)(nil) 1276 _ fs.IDer = (*Object)(nil) 1277 )