github.com/artpar/rclone@v1.67.3/backend/premiumizeme/premiumizeme.go (about) 1 // Package premiumizeme provides an interface to the premiumize.me 2 // object storage system. 3 package premiumizeme 4 5 /* 6 Run of rclone info 7 stringNeedsEscaping = []rune{ 8 0x00, 0x0A, 0x0D, 0x22, 0x2F, 0x5C, 0xBF, 0xFE 9 0x00, 0x0A, 0x0D, '"', '/', '\\', 0xBF, 0xFE 10 } 11 maxFileLength = 255 12 canWriteUnnormalized = true 13 canReadUnnormalized = true 14 canReadRenormalized = false 15 canStream = false 16 */ 17 18 import ( 19 "context" 20 "encoding/json" 21 "errors" 22 "fmt" 23 "io" 24 "net" 25 "net/http" 26 "net/url" 27 "path" 28 "strings" 29 "time" 30 31 "github.com/artpar/rclone/backend/premiumizeme/api" 32 "github.com/artpar/rclone/fs" 33 "github.com/artpar/rclone/fs/config" 34 "github.com/artpar/rclone/fs/config/configmap" 35 "github.com/artpar/rclone/fs/config/configstruct" 36 "github.com/artpar/rclone/fs/config/obscure" 37 "github.com/artpar/rclone/fs/fserrors" 38 "github.com/artpar/rclone/fs/fshttp" 39 "github.com/artpar/rclone/fs/hash" 40 "github.com/artpar/rclone/lib/dircache" 41 "github.com/artpar/rclone/lib/encoder" 42 "github.com/artpar/rclone/lib/oauthutil" 43 "github.com/artpar/rclone/lib/pacer" 44 "github.com/artpar/rclone/lib/random" 45 "github.com/artpar/rclone/lib/rest" 46 "golang.org/x/oauth2" 47 ) 48 49 const ( 50 rcloneClientID = "658922194" 51 rcloneEncryptedClientSecret = "B5YIvQoRIhcpAYs8HYeyjb9gK-ftmZEbqdh_gNfc4RgO9Q" 52 minSleep = 10 * time.Millisecond 53 maxSleep = 2 * time.Second 54 decayConstant = 2 // bigger for slower decay, exponential 55 rootID = "0" // ID of root folder is always this 56 rootURL = "https://www.premiumize.me/api" 57 ) 58 59 // Globals 60 var ( 61 // Description of how to auth for this app 62 oauthConfig = &oauth2.Config{ 63 Scopes: nil, 64 Endpoint: oauth2.Endpoint{ 65 AuthURL: "https://www.premiumize.me/authorize", 66 TokenURL: "https://www.premiumize.me/token", 67 }, 68 ClientID: rcloneClientID, 69 ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), 70 RedirectURL: oauthutil.RedirectURL, 71 } 72 ) 73 74 // Register with Fs 75 func init() { 76 fs.Register(&fs.RegInfo{ 77 Name: "premiumizeme", 78 Description: "premiumize.me", 79 NewFs: NewFs, 80 Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) { 81 return oauthutil.ConfigOut("", &oauthutil.Options{ 82 OAuth2Config: oauthConfig, 83 }) 84 }, 85 Options: append(oauthutil.SharedOptions, []fs.Option{{ 86 Name: "api_key", 87 Help: `API Key. 88 89 This is not normally used - use oauth instead. 90 `, 91 Hide: fs.OptionHideBoth, 92 Default: "", 93 Sensitive: true, 94 }, { 95 Name: config.ConfigEncoding, 96 Help: config.ConfigEncodingHelp, 97 Advanced: true, 98 // Encode invalid UTF-8 bytes as json doesn't handle them properly. 99 Default: (encoder.Display | 100 encoder.EncodeBackSlash | 101 encoder.EncodeDoubleQuote | 102 encoder.EncodeInvalidUtf8), 103 }}...), 104 }) 105 } 106 107 // Options defines the configuration for this backend 108 type Options struct { 109 APIKey string `config:"api_key"` 110 Enc encoder.MultiEncoder `config:"encoding"` 111 } 112 113 // Fs represents a remote cloud storage system 114 type Fs struct { 115 name string // name of this remote 116 root string // the path we are working on 117 opt Options // parsed options 118 features *fs.Features // optional features 119 srv *rest.Client // the connection to the server 120 dirCache *dircache.DirCache // Map of directory path to directory id 121 pacer *fs.Pacer // pacer for API calls 122 tokenRenewer *oauthutil.Renew // renew the token on expiry 123 } 124 125 // Object describes a file 126 type Object struct { 127 fs *Fs // what this object is part of 128 remote string // The remote path 129 hasMetaData bool // metadata is present and correct 130 size int64 // size of the object 131 modTime time.Time // modification time of the object 132 id string // ID of the object 133 parentID string // ID of parent directory 134 mimeType string // Mime type of object 135 url string // URL to download file 136 } 137 138 // ------------------------------------------------------------ 139 140 // Name of the remote (as passed into NewFs) 141 func (f *Fs) Name() string { 142 return f.name 143 } 144 145 // Root of the remote (as passed into NewFs) 146 func (f *Fs) Root() string { 147 return f.root 148 } 149 150 // String converts this Fs to a string 151 func (f *Fs) String() string { 152 return fmt.Sprintf("premiumize.me root '%s'", f.root) 153 } 154 155 // Features returns the optional features of this Fs 156 func (f *Fs) Features() *fs.Features { 157 return f.features 158 } 159 160 // parsePath parses a premiumize.me 'url' 161 func parsePath(path string) (root string) { 162 root = strings.Trim(path, "/") 163 return 164 } 165 166 // retryErrorCodes is a slice of error codes that we will retry 167 var retryErrorCodes = []int{ 168 429, // Too Many Requests. 169 500, // Internal Server Error 170 502, // Bad Gateway 171 503, // Service Unavailable 172 504, // Gateway Timeout 173 509, // Bandwidth Limit Exceeded 174 } 175 176 // shouldRetry returns a boolean as to whether this resp and err 177 // deserve to be retried. It returns the err as a convenience 178 func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { 179 if fserrors.ContextError(ctx, &err) { 180 return false, err 181 } 182 return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err 183 } 184 185 // readMetaDataForPath reads the metadata from the path 186 func (f *Fs) readMetaDataForPath(ctx context.Context, path string, directoriesOnly bool, filesOnly bool) (info *api.Item, err error) { 187 // defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err) 188 leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false) 189 if err != nil { 190 if err == fs.ErrorDirNotFound { 191 return nil, fs.ErrorObjectNotFound 192 } 193 return nil, err 194 } 195 196 lcLeaf := strings.ToLower(leaf) 197 _, found, err := f.listAll(ctx, directoryID, directoriesOnly, filesOnly, func(item *api.Item) bool { 198 if strings.ToLower(item.Name) == lcLeaf { 199 info = item 200 return true 201 } 202 return false 203 }) 204 if err != nil { 205 return nil, err 206 } 207 if !found { 208 return nil, fs.ErrorObjectNotFound 209 } 210 return info, nil 211 } 212 213 // errorHandler parses a non 2xx error response into an error 214 func errorHandler(resp *http.Response) error { 215 body, err := rest.ReadBody(resp) 216 if err != nil { 217 body = nil 218 } 219 var e = api.Response{ 220 Message: string(body), 221 Status: fmt.Sprintf("%s (%d)", resp.Status, resp.StatusCode), 222 } 223 if body != nil { 224 _ = json.Unmarshal(body, &e) 225 } 226 return &e 227 } 228 229 // Return a url.Values with the api key in 230 func (f *Fs) baseParams() url.Values { 231 params := url.Values{} 232 if f.opt.APIKey != "" { 233 params.Add("apikey", f.opt.APIKey) 234 } 235 return params 236 } 237 238 // NewFs constructs an Fs from the path, container:path 239 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { 240 // Parse config into Options struct 241 opt := new(Options) 242 err := configstruct.Set(m, opt) 243 if err != nil { 244 return nil, err 245 } 246 247 root = parsePath(root) 248 249 var client *http.Client 250 var ts *oauthutil.TokenSource 251 if opt.APIKey == "" { 252 client, ts, err = oauthutil.NewClient(ctx, name, m, oauthConfig) 253 if err != nil { 254 return nil, fmt.Errorf("failed to configure premiumize.me: %w", err) 255 } 256 } else { 257 client = fshttp.NewClient(ctx) 258 } 259 260 f := &Fs{ 261 name: name, 262 root: root, 263 opt: *opt, 264 srv: rest.NewClient(client).SetRoot(rootURL), 265 pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))), 266 } 267 f.features = (&fs.Features{ 268 CaseInsensitive: true, 269 CanHaveEmptyDirectories: true, 270 ReadMimeType: true, 271 }).Fill(ctx, f) 272 f.srv.SetErrorHandler(errorHandler) 273 274 // Renew the token in the background 275 if ts != nil { 276 f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error { 277 _, err := f.About(ctx) 278 return err 279 }) 280 } 281 282 // Get rootID 283 f.dirCache = dircache.New(root, rootID, f) 284 285 // Find the current root 286 err = f.dirCache.FindRoot(ctx, false) 287 if err != nil { 288 // Assume it is a file 289 newRoot, remote := dircache.SplitPath(root) 290 tempF := *f 291 tempF.dirCache = dircache.New(newRoot, rootID, &tempF) 292 tempF.root = newRoot 293 // Make new Fs which is the parent 294 err = tempF.dirCache.FindRoot(ctx, false) 295 if err != nil { 296 // No root so return old f 297 return f, nil 298 } 299 _, err := tempF.newObjectWithInfo(ctx, remote, nil) 300 if err != nil { 301 if err == fs.ErrorObjectNotFound { 302 // File doesn't exist so return old f 303 return f, nil 304 } 305 return nil, err 306 } 307 f.features.Fill(ctx, &tempF) 308 // XXX: update the old f here instead of returning tempF, since 309 // `features` were already filled with functions having *f as a receiver. 310 // See https://github.com/artpar/rclone/issues/2182 311 f.dirCache = tempF.dirCache 312 f.root = tempF.root 313 // return an error with an fs which points to the parent 314 return f, fs.ErrorIsFile 315 } 316 return f, nil 317 } 318 319 // Return an Object from a path 320 // 321 // If it can't be found it returns the error fs.ErrorObjectNotFound. 322 func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.Item) (fs.Object, error) { 323 o := &Object{ 324 fs: f, 325 remote: remote, 326 } 327 var err error 328 if info != nil { 329 // Set info 330 err = o.setMetaData(info) 331 } else { 332 err = o.readMetaData(ctx) // reads info and meta, returning an error 333 } 334 if err != nil { 335 return nil, err 336 } 337 return o, nil 338 } 339 340 // NewObject finds the Object at remote. If it can't be found 341 // it returns the error fs.ErrorObjectNotFound. 342 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { 343 return f.newObjectWithInfo(ctx, remote, nil) 344 } 345 346 // FindLeaf finds a directory of name leaf in the folder with ID pathID 347 func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) { 348 // Find the leaf in pathID 349 var newDirID string 350 newDirID, found, err = f.listAll(ctx, pathID, true, false, func(item *api.Item) bool { 351 if strings.EqualFold(item.Name, leaf) { 352 pathIDOut = item.ID 353 return true 354 } 355 return false 356 }) 357 // Update the Root directory ID to its actual value 358 if pathID == rootID { 359 f.dirCache.SetRootIDAlias(newDirID) 360 } 361 return pathIDOut, found, err 362 } 363 364 // CreateDir makes a directory with pathID as parent and name leaf 365 func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) { 366 // fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf) 367 var resp *http.Response 368 var info api.FolderCreateResponse 369 opts := rest.Opts{ 370 Method: "POST", 371 Path: "/folder/create", 372 Parameters: f.baseParams(), 373 MultipartParams: url.Values{ 374 "name": {f.opt.Enc.FromStandardName(leaf)}, 375 "parent_id": {pathID}, 376 }, 377 } 378 err = f.pacer.Call(func() (bool, error) { 379 resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) 380 return shouldRetry(ctx, resp, err) 381 }) 382 if err != nil { 383 //fmt.Printf("...Error %v\n", err) 384 return "", fmt.Errorf("CreateDir http: %w", err) 385 } 386 if err = info.AsErr(); err != nil { 387 return "", fmt.Errorf("CreateDir: %w", err) 388 } 389 // fmt.Printf("...Id %q\n", *info.Id) 390 return info.ID, nil 391 } 392 393 // list the objects into the function supplied 394 // 395 // If directories is set it only sends directories 396 // User function to process a File item from listAll 397 // 398 // Should return true to finish processing 399 type listAllFn func(*api.Item) bool 400 401 // Lists the directory required calling the user function on each item found 402 // 403 // If the user fn ever returns true then it early exits with found = true 404 // 405 // It returns a newDirID which is what the system returned as the directory ID 406 func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (newDirID string, found bool, err error) { 407 opts := rest.Opts{ 408 Method: "GET", 409 Path: "/folder/list", 410 Parameters: f.baseParams(), 411 } 412 if dirID != rootID { 413 opts.Parameters.Set("id", dirID) 414 } 415 opts.Parameters.Set("includebreadcrumbs", "false") 416 417 var result api.FolderListResponse 418 var resp *http.Response 419 err = f.pacer.Call(func() (bool, error) { 420 resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) 421 return shouldRetry(ctx, resp, err) 422 }) 423 if err != nil { 424 return newDirID, found, fmt.Errorf("couldn't list files: %w", err) 425 } 426 if err = result.AsErr(); err != nil { 427 return newDirID, found, fmt.Errorf("error while listing: %w", err) 428 } 429 newDirID = result.FolderID 430 for i := range result.Content { 431 item := &result.Content[i] 432 if item.Type == api.ItemTypeFolder { 433 if filesOnly { 434 continue 435 } 436 } else if item.Type == api.ItemTypeFile { 437 if directoriesOnly { 438 continue 439 } 440 } else { 441 fs.Debugf(f, "Ignoring %q - unknown type %q", item.Name, item.Type) 442 continue 443 } 444 item.Name = f.opt.Enc.ToStandardName(item.Name) 445 if fn(item) { 446 found = true 447 break 448 } 449 } 450 return 451 } 452 453 // List the objects and directories in dir into entries. The 454 // entries can be returned in any order but should be for a 455 // complete directory. 456 // 457 // dir should be "" to list the root, and should not have 458 // trailing slashes. 459 // 460 // This should return ErrDirNotFound if the directory isn't 461 // found. 462 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { 463 directoryID, err := f.dirCache.FindDir(ctx, dir, false) 464 if err != nil { 465 return nil, err 466 } 467 var iErr error 468 _, _, err = f.listAll(ctx, directoryID, false, false, func(info *api.Item) bool { 469 remote := path.Join(dir, info.Name) 470 if info.Type == api.ItemTypeFolder { 471 // cache the directory ID for later lookups 472 f.dirCache.Put(remote, info.ID) 473 d := fs.NewDir(remote, time.Unix(info.CreatedAt, 0)).SetID(info.ID) 474 entries = append(entries, d) 475 } else if info.Type == api.ItemTypeFile { 476 o, err := f.newObjectWithInfo(ctx, remote, info) 477 if err != nil { 478 iErr = err 479 return true 480 } 481 entries = append(entries, o) 482 } 483 return false 484 }) 485 if err != nil { 486 return nil, err 487 } 488 if iErr != nil { 489 return nil, iErr 490 } 491 return entries, nil 492 } 493 494 // Creates from the parameters passed in a half finished Object which 495 // must have setMetaData called on it 496 // 497 // Returns the object, leaf, directoryID and error. 498 // 499 // Used to create new objects 500 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) { 501 // Create the directory for the object if it doesn't exist 502 leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true) 503 if err != nil { 504 return 505 } 506 // Temporary Object under construction 507 o = &Object{ 508 fs: f, 509 remote: remote, 510 } 511 return o, leaf, directoryID, nil 512 } 513 514 // Put the object 515 // 516 // Copy the reader in to the new object which is returned. 517 // 518 // The new object may have been created if an error is returned 519 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 520 existingObj, err := f.newObjectWithInfo(ctx, src.Remote(), nil) 521 switch err { 522 case nil: 523 return existingObj, existingObj.Update(ctx, in, src, options...) 524 case fs.ErrorObjectNotFound: 525 // Not found so create it 526 return f.PutUnchecked(ctx, in, src, options...) 527 default: 528 return nil, err 529 } 530 } 531 532 // PutUnchecked the object into the container 533 // 534 // This will produce an error if the object already exists. 535 // 536 // Copy the reader in to the new object which is returned. 537 // 538 // The new object may have been created if an error is returned 539 func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 540 remote := src.Remote() 541 size := src.Size() 542 modTime := src.ModTime(ctx) 543 544 o, _, _, err := f.createObject(ctx, remote, modTime, size) 545 if err != nil { 546 return nil, err 547 } 548 return o, o.Update(ctx, in, src, options...) 549 } 550 551 // Mkdir creates the container if it doesn't exist 552 func (f *Fs) Mkdir(ctx context.Context, dir string) error { 553 _, err := f.dirCache.FindDir(ctx, dir, true) 554 return err 555 } 556 557 // purgeCheck removes the root directory, if check is set then it 558 // refuses to do so if it has anything in 559 func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { 560 root := path.Join(f.root, dir) 561 if root == "" { 562 return errors.New("can't purge root directory") 563 } 564 dc := f.dirCache 565 rootID, err := dc.FindDir(ctx, dir, false) 566 if err != nil { 567 return err 568 } 569 570 // need to check if empty as it will delete recursively by default 571 if check { 572 _, found, err := f.listAll(ctx, rootID, false, false, func(item *api.Item) bool { 573 return true 574 }) 575 if err != nil { 576 return fmt.Errorf("purgeCheck: %w", err) 577 } 578 if found { 579 return fs.ErrorDirectoryNotEmpty 580 } 581 } 582 583 opts := rest.Opts{ 584 Method: "POST", 585 Path: "/folder/delete", 586 MultipartParams: url.Values{ 587 "id": {rootID}, 588 }, 589 Parameters: f.baseParams(), 590 } 591 var resp *http.Response 592 var result api.Response 593 err = f.pacer.Call(func() (bool, error) { 594 resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) 595 return shouldRetry(ctx, resp, err) 596 }) 597 if err != nil { 598 return fmt.Errorf("rmdir failed: %w", err) 599 } 600 if err = result.AsErr(); err != nil { 601 return fmt.Errorf("rmdir: %w", err) 602 } 603 f.dirCache.FlushDir(dir) 604 if err != nil { 605 return err 606 } 607 return nil 608 } 609 610 // Rmdir deletes the root folder 611 // 612 // Returns an error if it isn't empty 613 func (f *Fs) Rmdir(ctx context.Context, dir string) error { 614 return f.purgeCheck(ctx, dir, true) 615 } 616 617 // Precision return the precision of this Fs 618 func (f *Fs) Precision() time.Duration { 619 return fs.ModTimeNotSupported 620 } 621 622 // Purge deletes all the files in the directory 623 // 624 // Optional interface: Only implement this if you have a way of 625 // deleting all the files quicker than just running Remove() on the 626 // result of List() 627 func (f *Fs) Purge(ctx context.Context, dir string) error { 628 return f.purgeCheck(ctx, dir, false) 629 } 630 631 // move a file or folder 632 // 633 // This is complicated by the fact that there is an API to move files 634 // between directories and a separate one to rename them. We try to 635 // call the minimum number of API calls. 636 func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDirectoryID, newDirectoryID string) (err error) { 637 newLeaf = f.opt.Enc.FromStandardName(newLeaf) 638 oldLeaf = f.opt.Enc.FromStandardName(oldLeaf) 639 doRenameLeaf := oldLeaf != newLeaf 640 doMove := oldDirectoryID != newDirectoryID 641 642 // Now rename the leaf to a temporary name if we are moving to 643 // another directory to make sure we don't overwrite something 644 // in the destination directory by accident 645 if doRenameLeaf && doMove { 646 tmpLeaf := newLeaf + "." + random.String(8) 647 err = f.renameLeaf(ctx, isFile, id, tmpLeaf) 648 if err != nil { 649 return fmt.Errorf("Move rename leaf: %w", err) 650 } 651 } 652 653 // Move the object to a new directory (with the existing name) 654 // if required 655 if doMove { 656 opts := rest.Opts{ 657 Method: "POST", 658 Path: "/folder/paste", 659 Parameters: f.baseParams(), 660 MultipartParams: url.Values{ 661 "id": {newDirectoryID}, 662 }, 663 } 664 opts.MultipartParams.Set("items[0][id]", id) 665 if isFile { 666 opts.MultipartParams.Set("items[0][type]", "file") 667 } else { 668 opts.MultipartParams.Set("items[0][type]", "folder") 669 } 670 //replacedLeaf := enc.FromStandardName(leaf) 671 var resp *http.Response 672 var result api.Response 673 err = f.pacer.Call(func() (bool, error) { 674 resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) 675 return shouldRetry(ctx, resp, err) 676 }) 677 if err != nil { 678 return fmt.Errorf("Move http: %w", err) 679 } 680 if err = result.AsErr(); err != nil { 681 return fmt.Errorf("Move: %w", err) 682 } 683 } 684 685 // Rename the leaf to its final name if required 686 if doRenameLeaf { 687 err = f.renameLeaf(ctx, isFile, id, newLeaf) 688 if err != nil { 689 return fmt.Errorf("Move rename leaf: %w", err) 690 } 691 } 692 693 return nil 694 } 695 696 // Move src to this remote using server-side move operations. 697 // 698 // This is stored with the remote path given. 699 // 700 // It returns the destination Object and a possible error. 701 // 702 // Will only be called if src.Fs().Name() == f.Name() 703 // 704 // If it isn't possible then return fs.ErrorCantMove 705 func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { 706 srcObj, ok := src.(*Object) 707 if !ok { 708 fs.Debugf(src, "Can't move - not same remote type") 709 return nil, fs.ErrorCantMove 710 } 711 712 // Create temporary object 713 dstObj, leaf, directoryID, err := f.createObject(ctx, remote, srcObj.modTime, srcObj.size) 714 if err != nil { 715 return nil, err 716 } 717 718 // Do the move 719 err = f.move(ctx, true, srcObj.id, path.Base(srcObj.remote), leaf, srcObj.parentID, directoryID) 720 if err != nil { 721 return nil, err 722 } 723 724 err = dstObj.readMetaData(ctx) 725 if err != nil { 726 return nil, err 727 } 728 return dstObj, nil 729 } 730 731 // DirMove moves src, srcRemote to this remote at dstRemote 732 // using server-side move operations. 733 // 734 // Will only be called if src.Fs().Name() == f.Name() 735 // 736 // If it isn't possible then return fs.ErrorCantDirMove 737 // 738 // If destination exists then return fs.ErrorDirExists 739 func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { 740 srcFs, ok := src.(*Fs) 741 if !ok { 742 fs.Debugf(srcFs, "Can't move directory - not same remote type") 743 return fs.ErrorCantDirMove 744 } 745 746 srcID, srcDirectoryID, srcLeaf, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote) 747 if err != nil { 748 return err 749 } 750 751 // Do the move 752 err = f.move(ctx, false, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID) 753 if err != nil { 754 return err 755 } 756 srcFs.dirCache.FlushDir(srcRemote) 757 return nil 758 } 759 760 // PublicLink adds a "readable by anyone with link" permission on the given file or folder. 761 func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) { 762 _, err := f.dirCache.FindDir(ctx, remote, false) 763 if err == nil { 764 return "", fs.ErrorCantShareDirectories 765 } 766 o, err := f.NewObject(ctx, remote) 767 if err != nil { 768 return "", err 769 } 770 return o.(*Object).url, nil 771 } 772 773 // Shutdown shutdown the fs 774 func (f *Fs) Shutdown(ctx context.Context) error { 775 f.tokenRenewer.Shutdown() 776 return nil 777 } 778 779 // About gets quota information 780 func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) { 781 var resp *http.Response 782 var info api.AccountInfoResponse 783 opts := rest.Opts{ 784 Method: "POST", 785 Path: "/account/info", 786 Parameters: f.baseParams(), 787 } 788 err = f.pacer.Call(func() (bool, error) { 789 resp, err = f.srv.CallJSON(ctx, &opts, nil, &info) 790 return shouldRetry(ctx, resp, err) 791 }) 792 if err != nil { 793 return nil, err 794 } 795 if err = info.AsErr(); err != nil { 796 return nil, err 797 } 798 usage = &fs.Usage{ 799 Used: fs.NewUsageValue(int64(info.SpaceUsed)), 800 } 801 return usage, nil 802 } 803 804 // DirCacheFlush resets the directory cache - used in testing as an 805 // optional interface 806 func (f *Fs) DirCacheFlush() { 807 f.dirCache.ResetRoot() 808 } 809 810 // Hashes returns the supported hash sets. 811 func (f *Fs) Hashes() hash.Set { 812 return hash.Set(hash.None) 813 } 814 815 // ------------------------------------------------------------ 816 817 // Fs returns the parent Fs 818 func (o *Object) Fs() fs.Info { 819 return o.fs 820 } 821 822 // Return a string version 823 func (o *Object) String() string { 824 if o == nil { 825 return "<nil>" 826 } 827 return o.remote 828 } 829 830 // Remote returns the remote path 831 func (o *Object) Remote() string { 832 return o.remote 833 } 834 835 // Hash returns the SHA-1 of an object returning a lowercase hex string 836 func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { 837 return "", hash.ErrUnsupported 838 } 839 840 // Size returns the size of an object in bytes 841 func (o *Object) Size() int64 { 842 err := o.readMetaData(context.TODO()) 843 if err != nil { 844 fs.Logf(o, "Failed to read metadata: %v", err) 845 return 0 846 } 847 return o.size 848 } 849 850 // setMetaData sets the metadata from info 851 func (o *Object) setMetaData(info *api.Item) (err error) { 852 if info.Type != "file" { 853 return fmt.Errorf("%q is %q: %w", o.remote, info.Type, fs.ErrorNotAFile) 854 } 855 o.hasMetaData = true 856 o.size = info.Size 857 o.modTime = time.Unix(info.CreatedAt, 0) 858 o.id = info.ID 859 o.mimeType = info.MimeType 860 o.url = info.Link 861 return nil 862 } 863 864 // readMetaData gets the metadata if it hasn't already been fetched 865 // 866 // it also sets the info 867 func (o *Object) readMetaData(ctx context.Context) (err error) { 868 if o.hasMetaData { 869 return nil 870 } 871 info, err := o.fs.readMetaDataForPath(ctx, o.remote, false, true) 872 if err != nil { 873 return err 874 } 875 return o.setMetaData(info) 876 } 877 878 // ModTime returns the modification time of the object 879 // 880 // It attempts to read the objects mtime and if that isn't present the 881 // LastModified returned in the http headers 882 func (o *Object) ModTime(ctx context.Context) time.Time { 883 err := o.readMetaData(ctx) 884 if err != nil { 885 fs.Logf(o, "Failed to read metadata: %v", err) 886 return time.Now() 887 } 888 return o.modTime 889 } 890 891 // SetModTime sets the modification time of the local fs object 892 func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { 893 return fs.ErrorCantSetModTime 894 } 895 896 // Storable returns a boolean showing whether this object storable 897 func (o *Object) Storable() bool { 898 return true 899 } 900 901 // Open an object for read 902 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { 903 if o.url == "" { 904 return nil, errors.New("can't download - no URL") 905 } 906 fs.FixRangeOption(options, o.size) 907 var resp *http.Response 908 opts := rest.Opts{ 909 Path: "", 910 RootURL: o.url, 911 Method: "GET", 912 Options: options, 913 } 914 err = o.fs.pacer.Call(func() (bool, error) { 915 resp, err = o.fs.srv.Call(ctx, &opts) 916 return shouldRetry(ctx, resp, err) 917 }) 918 if err != nil { 919 return nil, err 920 } 921 return resp.Body, err 922 } 923 924 // Update the object with the contents of the io.Reader, modTime and size 925 // 926 // If existing is set then it updates the object rather than creating a new one. 927 // 928 // The new object may have been created if an error is returned 929 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { 930 remote := o.Remote() 931 size := src.Size() 932 933 // Create the directory for the object if it doesn't exist 934 leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true) 935 if err != nil { 936 return err 937 } 938 leaf = o.fs.opt.Enc.FromStandardName(leaf) 939 940 var resp *http.Response 941 var info api.FolderUploadinfoResponse 942 opts := rest.Opts{ 943 Method: "POST", 944 Path: "/folder/uploadinfo", 945 Parameters: o.fs.baseParams(), 946 Options: options, 947 MultipartParams: url.Values{ 948 "id": {directoryID}, 949 }, 950 } 951 err = o.fs.pacer.Call(func() (bool, error) { 952 resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info) 953 if err != nil { 954 return shouldRetry(ctx, resp, err) 955 } 956 // Just check the download URL resolves - sometimes 957 // the URLs returned by premiumize.me don't resolve so 958 // this needs a retry. 959 var u *url.URL 960 u, err = url.Parse(info.URL) 961 if err != nil { 962 return true, fmt.Errorf("failed to parse download URL: %w", err) 963 } 964 _, err = net.LookupIP(u.Hostname()) 965 if err != nil { 966 return true, fmt.Errorf("failed to resolve download URL: %w", err) 967 } 968 return false, nil 969 }) 970 if err != nil { 971 return fmt.Errorf("upload get URL http: %w", err) 972 } 973 if err = info.AsErr(); err != nil { 974 return fmt.Errorf("upload get URL: %w", err) 975 } 976 977 // if file exists then rename it out the way otherwise uploads can fail 978 uploaded := false 979 var oldID = o.id 980 if o.hasMetaData { 981 newLeaf := leaf + "." + random.String(8) 982 fs.Debugf(o, "Moving old file out the way to %q", newLeaf) 983 err = o.fs.renameLeaf(ctx, true, oldID, newLeaf) 984 if err != nil { 985 return fmt.Errorf("upload rename old file: %w", err) 986 } 987 defer func() { 988 // on failed upload rename old file back 989 if !uploaded { 990 fs.Debugf(o, "Renaming old file back (from %q to %q) since upload failed", leaf, newLeaf) 991 newErr := o.fs.renameLeaf(ctx, true, oldID, leaf) 992 if newErr != nil && err == nil { 993 err = fmt.Errorf("upload renaming old file back: %w", newErr) 994 } 995 } 996 }() 997 } 998 999 opts = rest.Opts{ 1000 Method: "POST", 1001 RootURL: info.URL, 1002 Body: in, 1003 MultipartParams: url.Values{ 1004 "token": {info.Token}, 1005 }, 1006 MultipartContentName: "file", // ..name of the parameter which is the attached file 1007 MultipartFileName: leaf, // ..name of the file for the attached file 1008 ContentLength: &size, 1009 } 1010 var result api.Response 1011 err = o.fs.pacer.CallNoRetry(func() (bool, error) { 1012 resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &result) 1013 return shouldRetry(ctx, resp, err) 1014 }) 1015 if err != nil { 1016 return fmt.Errorf("upload file http: %w", err) 1017 } 1018 if err = result.AsErr(); err != nil { 1019 return fmt.Errorf("upload file: %w", err) 1020 } 1021 1022 // on successful upload, remove old file if it exists 1023 uploaded = true 1024 if o.hasMetaData { 1025 fs.Debugf(o, "Removing old file") 1026 err := o.fs.remove(ctx, oldID) 1027 if err != nil { 1028 return fmt.Errorf("upload remove old file: %w", err) 1029 } 1030 } 1031 1032 o.hasMetaData = false 1033 return o.readMetaData(ctx) 1034 } 1035 1036 // Rename the leaf of a file or directory in a directory 1037 func (f *Fs) renameLeaf(ctx context.Context, isFile bool, id string, newLeaf string) (err error) { 1038 opts := rest.Opts{ 1039 Method: "POST", 1040 MultipartParams: url.Values{ 1041 "id": {id}, 1042 "name": {newLeaf}, 1043 }, 1044 Parameters: f.baseParams(), 1045 } 1046 if isFile { 1047 opts.Path = "/item/rename" 1048 } else { 1049 opts.Path = "/folder/rename" 1050 } 1051 var resp *http.Response 1052 var result api.Response 1053 err = f.pacer.Call(func() (bool, error) { 1054 resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) 1055 return shouldRetry(ctx, resp, err) 1056 }) 1057 if err != nil { 1058 return fmt.Errorf("rename http: %w", err) 1059 } 1060 if err = result.AsErr(); err != nil { 1061 return fmt.Errorf("rename: %w", err) 1062 } 1063 return nil 1064 } 1065 1066 // Remove an object by ID 1067 func (f *Fs) remove(ctx context.Context, id string) (err error) { 1068 opts := rest.Opts{ 1069 Method: "POST", 1070 Path: "/item/delete", 1071 MultipartParams: url.Values{ 1072 "id": {id}, 1073 }, 1074 Parameters: f.baseParams(), 1075 } 1076 var resp *http.Response 1077 var result api.Response 1078 err = f.pacer.Call(func() (bool, error) { 1079 resp, err = f.srv.CallJSON(ctx, &opts, nil, &result) 1080 return shouldRetry(ctx, resp, err) 1081 }) 1082 if err != nil { 1083 return fmt.Errorf("remove http: %w", err) 1084 } 1085 if err = result.AsErr(); err != nil { 1086 return fmt.Errorf("remove: %w", err) 1087 } 1088 return nil 1089 } 1090 1091 // Remove an object 1092 func (o *Object) Remove(ctx context.Context) error { 1093 err := o.readMetaData(ctx) 1094 if err != nil { 1095 return fmt.Errorf("Remove: Failed to read metadata: %w", err) 1096 } 1097 return o.fs.remove(ctx, o.id) 1098 } 1099 1100 // MimeType of an Object if known, "" otherwise 1101 func (o *Object) MimeType(ctx context.Context) string { 1102 return o.mimeType 1103 } 1104 1105 // ID returns the ID of the Object if known, or "" if not 1106 func (o *Object) ID() string { 1107 return o.id 1108 } 1109 1110 // Check the interfaces are satisfied 1111 var ( 1112 _ fs.Fs = (*Fs)(nil) 1113 _ fs.Purger = (*Fs)(nil) 1114 _ fs.Mover = (*Fs)(nil) 1115 _ fs.DirMover = (*Fs)(nil) 1116 _ fs.DirCacheFlusher = (*Fs)(nil) 1117 _ fs.Abouter = (*Fs)(nil) 1118 _ fs.PublicLinker = (*Fs)(nil) 1119 _ fs.Shutdowner = (*Fs)(nil) 1120 _ fs.Object = (*Object)(nil) 1121 _ fs.MimeTyper = (*Object)(nil) 1122 _ fs.IDer = (*Object)(nil) 1123 )