github.com/artpar/rclone@v1.67.3/backend/sugarsync/sugarsync.go (about) 1 // Package sugarsync provides an interface to the Sugarsync 2 // object storage system. 3 package sugarsync 4 5 /* FIXME 6 7 DirMove tests fails with: Can not move sync folder. 8 9 go test -v -short -run TestIntegration/FsMkdir/FsPutFiles/FsDirMove -verbose -dump-bodies 10 11 To work around this we use the remote "TestSugarSync:Test" to test with. 12 13 */ 14 15 import ( 16 "context" 17 "errors" 18 "fmt" 19 "io" 20 "net/http" 21 "net/url" 22 "path" 23 "regexp" 24 "strconv" 25 "strings" 26 "sync" 27 "time" 28 29 "github.com/artpar/rclone/backend/sugarsync/api" 30 "github.com/artpar/rclone/fs" 31 "github.com/artpar/rclone/fs/config" 32 "github.com/artpar/rclone/fs/config/configmap" 33 "github.com/artpar/rclone/fs/config/configstruct" 34 "github.com/artpar/rclone/fs/config/obscure" 35 "github.com/artpar/rclone/fs/fserrors" 36 "github.com/artpar/rclone/fs/fshttp" 37 "github.com/artpar/rclone/fs/hash" 38 "github.com/artpar/rclone/lib/dircache" 39 "github.com/artpar/rclone/lib/encoder" 40 "github.com/artpar/rclone/lib/pacer" 41 "github.com/artpar/rclone/lib/rest" 42 ) 43 44 /* 45 maxFileLength = 16383 46 canWriteUnnormalized = true 47 canReadUnnormalized = true 48 canReadRenormalized = false 49 canStream = true 50 */ 51 52 const ( 53 appID = "/sc/9068489/215_1736969337" 54 accessKeyID = "OTA2ODQ4OTE1NzEzNDAwNTI4Njc" 55 encryptedPrivateAccessKey = "JONdXuRLNSRI5ue2Cr-vn-5m_YxyMNq9yHRKUQevqo8uaZjH502Z-x1axhyqOa8cDyldGq08RfFxozo" 56 minSleep = 10 * time.Millisecond 57 maxSleep = 2 * time.Second 58 decayConstant = 2 // bigger for slower decay, exponential 59 rootURL = "https://api.sugarsync.com" 60 listChunks = 500 // chunk size to read directory listings 61 expiryLeeway = 5 * time.Minute // time before the token expires to renew 62 ) 63 64 // withDefault returns value but if value is "" then it returns defaultValue 65 func withDefault(key, defaultValue string) (value string) { 66 if value == "" { 67 value = defaultValue 68 } 69 return value 70 } 71 72 // Register with Fs 73 func init() { 74 fs.Register(&fs.RegInfo{ 75 Name: "sugarsync", 76 Description: "Sugarsync", 77 NewFs: NewFs, 78 Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) { 79 opt := new(Options) 80 err := configstruct.Set(m, opt) 81 if err != nil { 82 return nil, fmt.Errorf("failed to read options: %w", err) 83 } 84 85 switch config.State { 86 case "": 87 if opt.RefreshToken == "" { 88 return fs.ConfigGoto("username") 89 } 90 return fs.ConfigConfirm("refresh", true, "config_refresh", "Already have a token - refresh?") 91 case "refresh": 92 if config.Result == "false" { 93 return nil, nil 94 } 95 return fs.ConfigGoto("username") 96 case "username": 97 return fs.ConfigInput("password", "config_username", "username (email address)") 98 case "password": 99 m.Set("username", config.Result) 100 return fs.ConfigPassword("auth", "config_password", "Your Sugarsync password.\n\nOnly required during setup and will not be stored.") 101 case "auth": 102 username, _ := m.Get("username") 103 m.Set("username", "") 104 password := config.Result 105 106 authRequest := api.AppAuthorization{ 107 Username: username, 108 Password: obscure.MustReveal(password), 109 Application: withDefault(opt.AppID, appID), 110 AccessKeyID: withDefault(opt.AccessKeyID, accessKeyID), 111 PrivateAccessKey: withDefault(opt.PrivateAccessKey, obscure.MustReveal(encryptedPrivateAccessKey)), 112 } 113 114 var resp *http.Response 115 opts := rest.Opts{ 116 Method: "POST", 117 Path: "/app-authorization", 118 } 119 srv := rest.NewClient(fshttp.NewClient(ctx)).SetRoot(rootURL) // FIXME 120 121 // FIXME 122 //err = f.pacer.Call(func() (bool, error) { 123 resp, err = srv.CallXML(context.Background(), &opts, &authRequest, nil) 124 // return shouldRetry(ctx, resp, err) 125 //}) 126 if err != nil { 127 return nil, fmt.Errorf("failed to get token: %w", err) 128 } 129 opt.RefreshToken = resp.Header.Get("Location") 130 m.Set("refresh_token", opt.RefreshToken) 131 return nil, nil 132 } 133 return nil, fmt.Errorf("unknown state %q", config.State) 134 }, Options: []fs.Option{{ 135 Name: "app_id", 136 Help: "Sugarsync App ID.\n\nLeave blank to use rclone's.", 137 Sensitive: true, 138 }, { 139 Name: "access_key_id", 140 Help: "Sugarsync Access Key ID.\n\nLeave blank to use rclone's.", 141 Sensitive: true, 142 }, { 143 Name: "private_access_key", 144 Help: "Sugarsync Private Access Key.\n\nLeave blank to use rclone's.", 145 Sensitive: true, 146 }, { 147 Name: "hard_delete", 148 Help: "Permanently delete files if true\notherwise put them in the deleted files.", 149 Default: false, 150 }, { 151 Name: "refresh_token", 152 Help: "Sugarsync refresh token.\n\nLeave blank normally, will be auto configured by rclone.", 153 Advanced: true, 154 Sensitive: true, 155 }, { 156 Name: "authorization", 157 Help: "Sugarsync authorization.\n\nLeave blank normally, will be auto configured by rclone.", 158 Advanced: true, 159 Sensitive: true, 160 }, { 161 Name: "authorization_expiry", 162 Help: "Sugarsync authorization expiry.\n\nLeave blank normally, will be auto configured by rclone.", 163 Advanced: true, 164 }, { 165 Name: "user", 166 Help: "Sugarsync user.\n\nLeave blank normally, will be auto configured by rclone.", 167 Advanced: true, 168 Sensitive: true, 169 }, { 170 Name: "root_id", 171 Help: "Sugarsync root id.\n\nLeave blank normally, will be auto configured by rclone.", 172 Advanced: true, 173 Sensitive: true, 174 }, { 175 Name: "deleted_id", 176 Help: "Sugarsync deleted folder id.\n\nLeave blank normally, will be auto configured by rclone.", 177 Advanced: true, 178 Sensitive: true, 179 }, { 180 Name: config.ConfigEncoding, 181 Help: config.ConfigEncodingHelp, 182 Advanced: true, 183 Default: (encoder.Base | 184 encoder.EncodeCtl | 185 encoder.EncodeInvalidUtf8), 186 }}, 187 }) 188 } 189 190 // Options defines the configuration for this backend 191 type Options struct { 192 AppID string `config:"app_id"` 193 AccessKeyID string `config:"access_key_id"` 194 PrivateAccessKey string `config:"private_access_key"` 195 HardDelete bool `config:"hard_delete"` 196 RefreshToken string `config:"refresh_token"` 197 Authorization string `config:"authorization"` 198 AuthorizationExpiry string `config:"authorization_expiry"` 199 User string `config:"user"` 200 RootID string `config:"root_id"` 201 DeletedID string `config:"deleted_id"` 202 Enc encoder.MultiEncoder `config:"encoding"` 203 } 204 205 // Fs represents a remote sugarsync 206 type Fs struct { 207 name string // name of this remote 208 root string // the path we are working on 209 opt Options // parsed options 210 features *fs.Features // optional features 211 srv *rest.Client // the connection to the server 212 dirCache *dircache.DirCache // Map of directory path to directory id 213 pacer *fs.Pacer // pacer for API calls 214 m configmap.Mapper // config file access 215 authMu sync.Mutex // used when doing authorization 216 authExpiry time.Time // time the authorization expires 217 } 218 219 // Object describes a sugarsync object 220 // 221 // Will definitely have info but maybe not meta 222 type Object struct { 223 fs *Fs // what this object is part of 224 remote string // The remote path 225 hasMetaData bool // whether info below has been set 226 size int64 // size of the object 227 modTime time.Time // modification time of the object 228 id string // ID of the object 229 } 230 231 // ------------------------------------------------------------ 232 233 // Name of the remote (as passed into NewFs) 234 func (f *Fs) Name() string { 235 return f.name 236 } 237 238 // Root of the remote (as passed into NewFs) 239 func (f *Fs) Root() string { 240 return f.root 241 } 242 243 // String converts this Fs to a string 244 func (f *Fs) String() string { 245 return fmt.Sprintf("sugarsync root '%s'", f.root) 246 } 247 248 // Features returns the optional features of this Fs 249 func (f *Fs) Features() *fs.Features { 250 return f.features 251 } 252 253 // parsePath parses a sugarsync 'url' 254 func parsePath(path string) (root string) { 255 root = strings.Trim(path, "/") 256 return 257 } 258 259 // retryErrorCodes is a slice of error codes that we will retry 260 var retryErrorCodes = []int{ 261 429, // Too Many Requests. 262 500, // Internal Server Error 263 502, // Bad Gateway 264 503, // Service Unavailable 265 504, // Gateway Timeout 266 509, // Bandwidth Limit Exceeded 267 } 268 269 // shouldRetry returns a boolean as to whether this resp and err 270 // deserve to be retried. It returns the err as a convenience 271 func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { 272 if fserrors.ContextError(ctx, &err) { 273 return false, err 274 } 275 return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err 276 } 277 278 // readMetaDataForPath reads the metadata from the path 279 func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.File, err error) { 280 // defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err) 281 leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false) 282 if err != nil { 283 if err == fs.ErrorDirNotFound { 284 return nil, fs.ErrorObjectNotFound 285 } 286 return nil, err 287 } 288 289 found, err := f.listAll(ctx, directoryID, func(item *api.File) bool { 290 if strings.EqualFold(item.Name, leaf) { 291 info = item 292 return true 293 } 294 return false 295 }, nil) 296 if err != nil { 297 return nil, err 298 } 299 if !found { 300 return nil, fs.ErrorObjectNotFound 301 } 302 return info, nil 303 } 304 305 // readMetaDataForID reads the metadata for a file from the ID 306 func (f *Fs) readMetaDataForID(ctx context.Context, ID string) (info *api.File, err error) { 307 var resp *http.Response 308 opts := rest.Opts{ 309 Method: "GET", 310 RootURL: ID, 311 } 312 err = f.pacer.Call(func() (bool, error) { 313 resp, err = f.srv.CallXML(ctx, &opts, nil, &info) 314 return shouldRetry(ctx, resp, err) 315 }) 316 if err != nil { 317 if resp != nil && resp.StatusCode == http.StatusNotFound { 318 return nil, fs.ErrorObjectNotFound 319 } 320 return nil, fmt.Errorf("failed to get authorization: %w", err) 321 } 322 return info, nil 323 } 324 325 // getAuthToken gets an Auth token from the refresh token 326 func (f *Fs) getAuthToken(ctx context.Context) error { 327 fs.Debugf(f, "Renewing token") 328 329 var authRequest = api.TokenAuthRequest{ 330 AccessKeyID: withDefault(f.opt.AccessKeyID, accessKeyID), 331 PrivateAccessKey: withDefault(f.opt.PrivateAccessKey, obscure.MustReveal(encryptedPrivateAccessKey)), 332 RefreshToken: f.opt.RefreshToken, 333 } 334 335 if authRequest.RefreshToken == "" { 336 return errors.New("no refresh token found - run `rclone config reconnect`") 337 } 338 339 var authResponse api.Authorization 340 var err error 341 var resp *http.Response 342 opts := rest.Opts{ 343 Method: "POST", 344 Path: "/authorization", 345 ExtraHeaders: map[string]string{ 346 "Authorization": "", // unset Authorization 347 }, 348 } 349 err = f.pacer.Call(func() (bool, error) { 350 resp, err = f.srv.CallXML(ctx, &opts, &authRequest, &authResponse) 351 return shouldRetry(ctx, resp, err) 352 }) 353 if err != nil { 354 return fmt.Errorf("failed to get authorization: %w", err) 355 } 356 f.opt.Authorization = resp.Header.Get("Location") 357 f.authExpiry = authResponse.Expiration 358 f.opt.User = authResponse.User 359 360 // Cache the results 361 f.m.Set("authorization", f.opt.Authorization) 362 f.m.Set("authorization_expiry", f.authExpiry.Format(time.RFC3339)) 363 f.m.Set("user", f.opt.User) 364 return nil 365 } 366 367 // Read the auth from the config file and refresh it if it is expired, setting it in srv 368 func (f *Fs) getAuth(req *http.Request) (err error) { 369 f.authMu.Lock() 370 defer f.authMu.Unlock() 371 ctx := req.Context() 372 373 // if have auth, check it is in date 374 if f.opt.Authorization == "" || f.opt.User == "" || f.authExpiry.IsZero() || time.Until(f.authExpiry) < expiryLeeway { 375 // Get the auth token 376 f.srv.SetSigner(nil) // temporarily remove the signer so we don't infinitely recurse 377 err = f.getAuthToken(ctx) 378 f.srv.SetSigner(f.getAuth) // replace signer 379 if err != nil { 380 return err 381 } 382 } 383 384 // Set Authorization header 385 req.Header.Set("Authorization", f.opt.Authorization) 386 387 return nil 388 } 389 390 // Read the user info into f 391 func (f *Fs) getUser(ctx context.Context) (user *api.User, err error) { 392 var resp *http.Response 393 opts := rest.Opts{ 394 Method: "GET", 395 Path: "/user", 396 } 397 err = f.pacer.Call(func() (bool, error) { 398 resp, err = f.srv.CallXML(ctx, &opts, nil, &user) 399 return shouldRetry(ctx, resp, err) 400 }) 401 if err != nil { 402 return nil, fmt.Errorf("failed to get user: %w", err) 403 } 404 return user, nil 405 } 406 407 // Read the expiry time from a string 408 func parseExpiry(expiryString string) time.Time { 409 if expiryString == "" { 410 return time.Time{} 411 } 412 expiry, err := time.Parse(time.RFC3339, expiryString) 413 if err != nil { 414 fs.Debugf("sugarsync", "Invalid expiry time %q read from config", expiryString) 415 return time.Time{} 416 } 417 return expiry 418 } 419 420 // NewFs constructs an Fs from the path, container:path 421 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { 422 opt := new(Options) 423 err := configstruct.Set(m, opt) 424 if err != nil { 425 return nil, err 426 } 427 428 root = parsePath(root) 429 client := fshttp.NewClient(ctx) 430 f := &Fs{ 431 name: name, 432 root: root, 433 opt: *opt, 434 srv: rest.NewClient(client).SetRoot(rootURL), 435 pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))), 436 m: m, 437 authExpiry: parseExpiry(opt.AuthorizationExpiry), 438 } 439 f.features = (&fs.Features{ 440 CaseInsensitive: true, 441 CanHaveEmptyDirectories: true, 442 }).Fill(ctx, f) 443 f.srv.SetSigner(f.getAuth) // use signing hook to get the auth 444 f.srv.SetErrorHandler(errorHandler) 445 446 // Get rootID 447 if f.opt.RootID == "" { 448 user, err := f.getUser(ctx) 449 if err != nil { 450 return nil, err 451 } 452 f.opt.RootID = user.SyncFolders 453 if strings.HasSuffix(f.opt.RootID, "/contents") { 454 f.opt.RootID = f.opt.RootID[:len(f.opt.RootID)-9] 455 } else { 456 return nil, fmt.Errorf("unexpected rootID %q", f.opt.RootID) 457 } 458 // Cache the results 459 f.m.Set("root_id", f.opt.RootID) 460 f.opt.DeletedID = user.Deleted 461 f.m.Set("deleted_id", f.opt.DeletedID) 462 } 463 f.dirCache = dircache.New(root, f.opt.RootID, f) 464 465 // Find the current root 466 err = f.dirCache.FindRoot(ctx, false) 467 if err != nil { 468 // Assume it is a file 469 newRoot, remote := dircache.SplitPath(root) 470 oldDirCache := f.dirCache 471 f.dirCache = dircache.New(newRoot, f.opt.RootID, f) 472 f.root = newRoot 473 resetF := func() { 474 f.dirCache = oldDirCache 475 f.root = root 476 } 477 // Make new Fs which is the parent 478 err = f.dirCache.FindRoot(ctx, false) 479 if err != nil { 480 // No root so return old f 481 resetF() 482 return f, nil 483 } 484 _, err := f.newObjectWithInfo(ctx, remote, nil) 485 if err != nil { 486 if err == fs.ErrorObjectNotFound { 487 // File doesn't exist so return old f 488 resetF() 489 return f, nil 490 } 491 return nil, err 492 } 493 // return an error with an fs which points to the parent 494 return f, fs.ErrorIsFile 495 } 496 return f, nil 497 } 498 499 var findError = regexp.MustCompile(`<h3>(.*?)</h3>`) 500 501 // errorHandler parses errors from the body 502 // 503 // Errors seem to be HTML with <h3> containing the error text 504 // <h3>Can not move sync folder.</h3> 505 func errorHandler(resp *http.Response) (err error) { 506 body, err := rest.ReadBody(resp) 507 if err != nil { 508 return fmt.Errorf("error reading error out of body: %w", err) 509 } 510 match := findError.FindSubmatch(body) 511 if match == nil || len(match) < 2 || len(match[1]) == 0 { 512 return fmt.Errorf("HTTP error %v (%v) returned body: %q", resp.StatusCode, resp.Status, body) 513 } 514 return fmt.Errorf("HTTP error %v (%v): %s", resp.StatusCode, resp.Status, match[1]) 515 } 516 517 // rootSlash returns root with a slash on if it is empty, otherwise empty string 518 func (f *Fs) rootSlash() string { 519 if f.root == "" { 520 return f.root 521 } 522 return f.root + "/" 523 } 524 525 // Return an Object from a path 526 // 527 // If it can't be found it returns the error fs.ErrorObjectNotFound. 528 func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *api.File) (fs.Object, error) { 529 o := &Object{ 530 fs: f, 531 remote: remote, 532 } 533 var err error 534 if info != nil { 535 // Set info 536 err = o.setMetaData(info) 537 } else { 538 err = o.readMetaData(ctx) // reads info and meta, returning an error 539 } 540 if err != nil { 541 return nil, err 542 } 543 return o, nil 544 } 545 546 // NewObject finds the Object at remote. If it can't be found 547 // it returns the error fs.ErrorObjectNotFound. 548 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { 549 return f.newObjectWithInfo(ctx, remote, nil) 550 } 551 552 // FindLeaf finds a directory of name leaf in the folder with ID pathID 553 func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) { 554 //fs.Debugf(f, "FindLeaf(%q, %q)", pathID, leaf) 555 // Find the leaf in pathID 556 found, err = f.listAll(ctx, pathID, nil, func(item *api.Collection) bool { 557 if strings.EqualFold(item.Name, leaf) { 558 pathIDOut = item.Ref 559 return true 560 } 561 return false 562 }) 563 // fs.Debugf(f, ">FindLeaf %q, %v, %v", pathIDOut, found, err) 564 return pathIDOut, found, err 565 } 566 567 // CreateDir makes a directory with pathID as parent and name leaf 568 func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) { 569 // fs.Debugf(f, "CreateDir(%q, %q)\n", pathID, leaf) 570 var resp *http.Response 571 opts := rest.Opts{ 572 Method: "POST", 573 RootURL: pathID, 574 NoResponse: true, 575 } 576 var mkdir interface{} 577 if pathID == f.opt.RootID { 578 // folders at the root are syncFolders 579 mkdir = &api.CreateSyncFolder{ 580 Name: f.opt.Enc.FromStandardName(leaf), 581 } 582 opts.ExtraHeaders = map[string]string{ 583 "*X-SugarSync-API-Version": "1.5", // non canonical header 584 } 585 586 } else { 587 mkdir = &api.CreateFolder{ 588 Name: f.opt.Enc.FromStandardName(leaf), 589 } 590 } 591 err = f.pacer.Call(func() (bool, error) { 592 resp, err = f.srv.CallXML(ctx, &opts, mkdir, nil) 593 return shouldRetry(ctx, resp, err) 594 }) 595 if err != nil { 596 return "", err 597 } 598 newID = resp.Header.Get("Location") 599 if newID == "" { 600 // look up ID if not returned (e.g. for syncFolder) 601 var found bool 602 newID, found, err = f.FindLeaf(ctx, pathID, leaf) 603 if err != nil { 604 return "", err 605 } 606 if !found { 607 return "", fmt.Errorf("couldn't find ID for newly created directory %q", leaf) 608 } 609 610 } 611 return newID, nil 612 } 613 614 // list the objects into the function supplied 615 // 616 // Should return true to finish processing 617 type listAllFileFn func(*api.File) bool 618 619 // list the folders into the function supplied 620 // 621 // Should return true to finish processing 622 type listAllFolderFn func(*api.Collection) bool 623 624 // Lists the directory required calling the user function on each item found 625 // 626 // If the user fn ever returns true then it early exits with found = true 627 func (f *Fs) listAll(ctx context.Context, dirID string, fileFn listAllFileFn, folderFn listAllFolderFn) (found bool, err error) { 628 opts := rest.Opts{ 629 Method: "GET", 630 RootURL: dirID, 631 Path: "/contents", 632 Parameters: url.Values{}, 633 } 634 opts.Parameters.Set("max", strconv.Itoa(listChunks)) 635 start := 0 636 OUTER: 637 for { 638 opts.Parameters.Set("start", strconv.Itoa(start)) 639 640 var result api.CollectionContents 641 var resp *http.Response 642 err = f.pacer.Call(func() (bool, error) { 643 resp, err = f.srv.CallXML(ctx, &opts, nil, &result) 644 return shouldRetry(ctx, resp, err) 645 }) 646 if err != nil { 647 return found, fmt.Errorf("couldn't list files: %w", err) 648 } 649 if fileFn != nil { 650 for i := range result.Files { 651 item := &result.Files[i] 652 item.Name = f.opt.Enc.ToStandardName(item.Name) 653 if fileFn(item) { 654 found = true 655 break OUTER 656 } 657 } 658 } 659 if folderFn != nil { 660 for i := range result.Collections { 661 item := &result.Collections[i] 662 item.Name = f.opt.Enc.ToStandardName(item.Name) 663 if folderFn(item) { 664 found = true 665 break OUTER 666 } 667 } 668 } 669 if !result.HasMore { 670 break 671 } 672 start = result.End + 1 673 } 674 return 675 } 676 677 // List the objects and directories in dir into entries. The 678 // entries can be returned in any order but should be for a 679 // complete directory. 680 // 681 // dir should be "" to list the root, and should not have 682 // trailing slashes. 683 // 684 // This should return ErrDirNotFound if the directory isn't 685 // found. 686 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { 687 directoryID, err := f.dirCache.FindDir(ctx, dir, false) 688 if err != nil { 689 return nil, err 690 } 691 var iErr error 692 _, err = f.listAll(ctx, directoryID, 693 func(info *api.File) bool { 694 remote := path.Join(dir, info.Name) 695 o, err := f.newObjectWithInfo(ctx, remote, info) 696 if err != nil { 697 iErr = err 698 return true 699 } 700 entries = append(entries, o) 701 return false 702 }, 703 func(info *api.Collection) bool { 704 remote := path.Join(dir, info.Name) 705 id := info.Ref 706 // cache the directory ID for later lookups 707 f.dirCache.Put(remote, id) 708 d := fs.NewDir(remote, info.TimeCreated).SetID(id) 709 entries = append(entries, d) 710 return false 711 }) 712 if err != nil { 713 return nil, err 714 } 715 if iErr != nil { 716 return nil, iErr 717 } 718 return entries, nil 719 } 720 721 // Creates from the parameters passed in a half finished Object which 722 // must have setMetaData called on it 723 // 724 // Returns the object, leaf, directoryID and error. 725 // 726 // Used to create new objects 727 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) { 728 // Create the directory for the object if it doesn't exist 729 leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true) 730 if err != nil { 731 return 732 } 733 // Temporary Object under construction 734 o = &Object{ 735 fs: f, 736 remote: remote, 737 } 738 return o, leaf, directoryID, nil 739 } 740 741 // Put the object 742 // 743 // Copy the reader in to the new object which is returned. 744 // 745 // The new object may have been created if an error is returned 746 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 747 existingObj, err := f.newObjectWithInfo(ctx, src.Remote(), nil) 748 switch err { 749 case nil: 750 return existingObj, existingObj.Update(ctx, in, src, options...) 751 case fs.ErrorObjectNotFound: 752 // Not found so create it 753 return f.PutUnchecked(ctx, in, src, options...) 754 default: 755 return nil, err 756 } 757 } 758 759 // PutStream uploads to the remote path with the modTime given of indeterminate size 760 func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 761 return f.Put(ctx, in, src, options...) 762 } 763 764 // PutUnchecked the object into the container 765 // 766 // This will produce an error if the object already exists. 767 // 768 // Copy the reader in to the new object which is returned. 769 // 770 // The new object may have been created if an error is returned 771 func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 772 remote := src.Remote() 773 size := src.Size() 774 modTime := src.ModTime(ctx) 775 776 o, _, _, err := f.createObject(ctx, remote, modTime, size) 777 if err != nil { 778 return nil, err 779 } 780 return o, o.Update(ctx, in, src, options...) 781 } 782 783 // Mkdir creates the container if it doesn't exist 784 func (f *Fs) Mkdir(ctx context.Context, dir string) error { 785 _, err := f.dirCache.FindDir(ctx, dir, true) 786 return err 787 } 788 789 // delete removes an object or directory by ID either putting it 790 // in the Deleted files or deleting it permanently 791 func (f *Fs) delete(ctx context.Context, isFile bool, id string, remote string, hardDelete bool) (err error) { 792 if hardDelete { 793 opts := rest.Opts{ 794 Method: "DELETE", 795 RootURL: id, 796 NoResponse: true, 797 } 798 return f.pacer.Call(func() (bool, error) { 799 resp, err := f.srv.Call(ctx, &opts) 800 return shouldRetry(ctx, resp, err) 801 }) 802 } 803 // Move file/dir to deleted files if not hard delete 804 leaf := path.Base(remote) 805 if isFile { 806 _, err = f.moveFile(ctx, id, leaf, f.opt.DeletedID) 807 } else { 808 err = f.moveDir(ctx, id, leaf, f.opt.DeletedID) 809 } 810 return err 811 } 812 813 // purgeCheck removes the root directory, if check is set then it 814 // refuses to do so if it has anything in 815 func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { 816 root := path.Join(f.root, dir) 817 if root == "" { 818 return errors.New("can't purge root directory") 819 } 820 dc := f.dirCache 821 directoryID, err := dc.FindDir(ctx, dir, false) 822 if err != nil { 823 return err 824 } 825 826 if check { 827 found, err := f.listAll(ctx, directoryID, func(item *api.File) bool { 828 return true 829 }, func(item *api.Collection) bool { 830 return true 831 }) 832 if err != nil { 833 return err 834 } 835 if found { 836 return fs.ErrorDirectoryNotEmpty 837 } 838 } 839 840 err = f.delete(ctx, false, directoryID, root, f.opt.HardDelete || check) 841 if err != nil { 842 return err 843 } 844 845 f.dirCache.FlushDir(dir) 846 return nil 847 } 848 849 // Rmdir deletes the root folder 850 // 851 // Returns an error if it isn't empty 852 func (f *Fs) Rmdir(ctx context.Context, dir string) error { 853 return f.purgeCheck(ctx, dir, true) 854 } 855 856 // Precision return the precision of this Fs 857 func (f *Fs) Precision() time.Duration { 858 return fs.ModTimeNotSupported 859 } 860 861 // Copy src to this remote using server-side copy operations. 862 // 863 // This is stored with the remote path given. 864 // 865 // It returns the destination Object and a possible error. 866 // 867 // Will only be called if src.Fs().Name() == f.Name() 868 // 869 // If it isn't possible then return fs.ErrorCantCopy 870 func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { 871 srcObj, ok := src.(*Object) 872 if !ok { 873 fs.Debugf(src, "Can't copy - not same remote type") 874 return nil, fs.ErrorCantCopy 875 } 876 err := srcObj.readMetaData(ctx) 877 if err != nil { 878 return nil, err 879 } 880 881 srcPath := srcObj.fs.rootSlash() + srcObj.remote 882 dstPath := f.rootSlash() + remote 883 if strings.EqualFold(srcPath, dstPath) { 884 return nil, fmt.Errorf("can't copy %q -> %q as are same name when lowercase", srcPath, dstPath) 885 } 886 887 // Create temporary object 888 dstObj, leaf, directoryID, err := f.createObject(ctx, remote, srcObj.modTime, srcObj.size) 889 if err != nil { 890 return nil, err 891 } 892 893 // Copy the object 894 opts := rest.Opts{ 895 Method: "POST", 896 RootURL: directoryID, 897 NoResponse: true, 898 } 899 copyFile := api.CopyFile{ 900 Name: f.opt.Enc.FromStandardName(leaf), 901 Source: srcObj.id, 902 } 903 var resp *http.Response 904 err = f.pacer.Call(func() (bool, error) { 905 resp, err = f.srv.CallXML(ctx, &opts, ©File, nil) 906 return shouldRetry(ctx, resp, err) 907 }) 908 if err != nil { 909 return nil, err 910 } 911 dstObj.id = resp.Header.Get("Location") 912 err = dstObj.readMetaData(ctx) 913 if err != nil { 914 return nil, err 915 } 916 return dstObj, nil 917 } 918 919 // Purge deletes all the files in the directory 920 // 921 // Optional interface: Only implement this if you have a way of 922 // deleting all the files quicker than just running Remove() on the 923 // result of List() 924 func (f *Fs) Purge(ctx context.Context, dir string) error { 925 // Caution: Deleting a folder may orphan objects. It's important 926 // to remove the contents of the folder before you delete the 927 // folder. That's because removing a folder using DELETE does not 928 // remove the objects contained within the folder. If you delete 929 // a folder without first deleting its contents, the contents may 930 // be rendered inaccessible. 931 // 932 // An alternative to permanently deleting a folder is moving it to the 933 // Deleted Files folder. A folder (and all its contents) in the 934 // Deleted Files folder can be recovered. Your app can retrieve the 935 // link to the user's Deleted Files folder from the <deleted> element 936 // in the user resource representation. Your application can then move 937 // a folder to the Deleted Files folder by issuing an HTTP PUT request 938 // to the URL that represents the file resource and provide as input, 939 // XML that specifies in the <parent> element the link to the Deleted 940 // Files folder. 941 if f.opt.HardDelete { 942 return fs.ErrorCantPurge 943 } 944 return f.purgeCheck(ctx, dir, false) 945 } 946 947 // moveFile moves a file server-side 948 func (f *Fs) moveFile(ctx context.Context, id, leaf, directoryID string) (info *api.File, err error) { 949 opts := rest.Opts{ 950 Method: "PUT", 951 RootURL: id, 952 } 953 move := api.MoveFile{ 954 Name: f.opt.Enc.FromStandardName(leaf), 955 Parent: directoryID, 956 } 957 var resp *http.Response 958 err = f.pacer.Call(func() (bool, error) { 959 resp, err = f.srv.CallXML(ctx, &opts, &move, &info) 960 return shouldRetry(ctx, resp, err) 961 }) 962 if err != nil { 963 return nil, err 964 } 965 // The docs say that there is nothing returned but apparently 966 // there is... however it doesn't have Ref 967 // 968 // If ref not set, assume it hasn't changed 969 if info.Ref == "" { 970 info.Ref = id 971 } 972 return info, nil 973 } 974 975 // moveDir moves a folder server-side 976 func (f *Fs) moveDir(ctx context.Context, id, leaf, directoryID string) (err error) { 977 // Move the object 978 opts := rest.Opts{ 979 Method: "PUT", 980 RootURL: id, 981 NoResponse: true, 982 } 983 move := api.MoveFolder{ 984 Name: f.opt.Enc.FromStandardName(leaf), 985 Parent: directoryID, 986 } 987 var resp *http.Response 988 return f.pacer.Call(func() (bool, error) { 989 resp, err = f.srv.CallXML(ctx, &opts, &move, nil) 990 return shouldRetry(ctx, resp, err) 991 }) 992 } 993 994 // Move src to this remote using server-side move operations. 995 // 996 // This is stored with the remote path given. 997 // 998 // It returns the destination Object and a possible error. 999 // 1000 // Will only be called if src.Fs().Name() == f.Name() 1001 // 1002 // If it isn't possible then return fs.ErrorCantMove 1003 func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { 1004 srcObj, ok := src.(*Object) 1005 if !ok { 1006 fs.Debugf(src, "Can't move - not same remote type") 1007 return nil, fs.ErrorCantMove 1008 } 1009 1010 // Create temporary object 1011 dstObj, leaf, directoryID, err := f.createObject(ctx, remote, srcObj.modTime, srcObj.size) 1012 if err != nil { 1013 return nil, err 1014 } 1015 1016 // Do the move 1017 info, err := f.moveFile(ctx, srcObj.id, leaf, directoryID) 1018 if err != nil { 1019 return nil, err 1020 } 1021 1022 err = dstObj.setMetaData(info) 1023 if err != nil { 1024 return nil, err 1025 } 1026 return dstObj, nil 1027 } 1028 1029 // DirMove moves src, srcRemote to this remote at dstRemote 1030 // using server-side move operations. 1031 // 1032 // Will only be called if src.Fs().Name() == f.Name() 1033 // 1034 // If it isn't possible then return fs.ErrorCantDirMove 1035 // 1036 // If destination exists then return fs.ErrorDirExists 1037 func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { 1038 srcFs, ok := src.(*Fs) 1039 if !ok { 1040 fs.Debugf(srcFs, "Can't move directory - not same remote type") 1041 return fs.ErrorCantDirMove 1042 } 1043 1044 srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote) 1045 if err != nil { 1046 return err 1047 } 1048 1049 // Do the move 1050 err = f.moveDir(ctx, srcID, dstLeaf, dstDirectoryID) 1051 if err != nil { 1052 return err 1053 } 1054 srcFs.dirCache.FlushDir(srcRemote) 1055 return nil 1056 } 1057 1058 // PublicLink adds a "readable by anyone with link" permission on the given file or folder. 1059 func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) { 1060 obj, err := f.NewObject(ctx, remote) 1061 if err != nil { 1062 return "", err 1063 } 1064 o, ok := obj.(*Object) 1065 if !ok { 1066 return "", errors.New("internal error: not an Object") 1067 } 1068 opts := rest.Opts{ 1069 Method: "PUT", 1070 RootURL: o.id, 1071 } 1072 linkFile := api.SetPublicLink{ 1073 PublicLink: api.PublicLink{Enabled: true}, 1074 } 1075 var resp *http.Response 1076 var info *api.File 1077 err = f.pacer.Call(func() (bool, error) { 1078 resp, err = f.srv.CallXML(ctx, &opts, &linkFile, &info) 1079 return shouldRetry(ctx, resp, err) 1080 }) 1081 if err != nil { 1082 return "", err 1083 } 1084 return info.PublicLink.URL, err 1085 } 1086 1087 // DirCacheFlush resets the directory cache - used in testing as an 1088 // optional interface 1089 func (f *Fs) DirCacheFlush() { 1090 f.dirCache.ResetRoot() 1091 } 1092 1093 // Hashes returns the supported hash sets. 1094 func (f *Fs) Hashes() hash.Set { 1095 return hash.Set(hash.None) 1096 } 1097 1098 // ------------------------------------------------------------ 1099 1100 // Fs returns the parent Fs 1101 func (o *Object) Fs() fs.Info { 1102 return o.fs 1103 } 1104 1105 // Return a string version 1106 func (o *Object) String() string { 1107 if o == nil { 1108 return "<nil>" 1109 } 1110 return o.remote 1111 } 1112 1113 // Remote returns the remote path 1114 func (o *Object) Remote() string { 1115 return o.remote 1116 } 1117 1118 // Hash returns the SHA-1 of an object returning a lowercase hex string 1119 func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { 1120 return "", hash.ErrUnsupported 1121 } 1122 1123 // Size returns the size of an object in bytes 1124 func (o *Object) Size() int64 { 1125 err := o.readMetaData(context.TODO()) 1126 if err != nil { 1127 fs.Logf(o, "Failed to read metadata: %v", err) 1128 return 0 1129 } 1130 return o.size 1131 } 1132 1133 // setMetaData sets the metadata from info 1134 func (o *Object) setMetaData(info *api.File) (err error) { 1135 o.hasMetaData = true 1136 o.size = info.Size 1137 o.modTime = info.LastModified 1138 if info.Ref != "" { 1139 o.id = info.Ref 1140 } else if o.id == "" { 1141 return errors.New("no ID found in response") 1142 } 1143 return nil 1144 } 1145 1146 // readMetaData gets the metadata if it hasn't already been fetched 1147 // 1148 // it also sets the info 1149 func (o *Object) readMetaData(ctx context.Context) (err error) { 1150 if o.hasMetaData { 1151 return nil 1152 } 1153 var info *api.File 1154 if o.id != "" { 1155 info, err = o.fs.readMetaDataForID(ctx, o.id) 1156 } else { 1157 info, err = o.fs.readMetaDataForPath(ctx, o.remote) 1158 } 1159 if err != nil { 1160 return err 1161 } 1162 return o.setMetaData(info) 1163 } 1164 1165 // ModTime returns the modification time of the object 1166 // 1167 // It attempts to read the objects mtime and if that isn't present the 1168 // LastModified returned in the http headers 1169 func (o *Object) ModTime(ctx context.Context) time.Time { 1170 err := o.readMetaData(ctx) 1171 if err != nil { 1172 fs.Logf(o, "Failed to read metadata: %v", err) 1173 return time.Now() 1174 } 1175 return o.modTime 1176 } 1177 1178 // SetModTime sets the modification time of the local fs object 1179 func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { 1180 // Sugarsync doesn't support setting the mod time. 1181 // 1182 // In theory (but not in the docs) you could patch the object, 1183 // however it doesn't work. 1184 return fs.ErrorCantSetModTime 1185 } 1186 1187 // Storable returns a boolean showing whether this object storable 1188 func (o *Object) Storable() bool { 1189 return true 1190 } 1191 1192 // Open an object for read 1193 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { 1194 if o.id == "" { 1195 return nil, errors.New("can't download - no id") 1196 } 1197 fs.FixRangeOption(options, o.size) 1198 var resp *http.Response 1199 opts := rest.Opts{ 1200 Method: "GET", 1201 RootURL: o.id, 1202 Path: "/data", 1203 Options: options, 1204 } 1205 err = o.fs.pacer.Call(func() (bool, error) { 1206 resp, err = o.fs.srv.Call(ctx, &opts) 1207 return shouldRetry(ctx, resp, err) 1208 }) 1209 if err != nil { 1210 return nil, err 1211 } 1212 return resp.Body, err 1213 } 1214 1215 // createFile makes an (empty) file with pathID as parent and name leaf and returns the ID 1216 func (f *Fs) createFile(ctx context.Context, pathID, leaf, mimeType string) (newID string, err error) { 1217 var resp *http.Response 1218 opts := rest.Opts{ 1219 Method: "POST", 1220 RootURL: pathID, 1221 NoResponse: true, 1222 } 1223 mkdir := api.CreateFile{ 1224 Name: f.opt.Enc.FromStandardName(leaf), 1225 MediaType: mimeType, 1226 } 1227 err = f.pacer.Call(func() (bool, error) { 1228 resp, err = f.srv.CallXML(ctx, &opts, &mkdir, nil) 1229 return shouldRetry(ctx, resp, err) 1230 }) 1231 if err != nil { 1232 return "", err 1233 } 1234 return resp.Header.Get("Location"), nil 1235 } 1236 1237 // Update the object with the contents of the io.Reader, modTime and size 1238 // 1239 // If existing is set then it updates the object rather than creating a new one. 1240 // 1241 // The new object may have been created if an error is returned 1242 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { 1243 size := src.Size() 1244 // modTime := src.ModTime(ctx) 1245 remote := o.Remote() 1246 1247 // Create the directory for the object if it doesn't exist 1248 leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true) 1249 if err != nil { 1250 return err 1251 } 1252 1253 // if file doesn't exist, create it 1254 if o.id == "" { 1255 o.id, err = o.fs.createFile(ctx, directoryID, leaf, fs.MimeType(ctx, src)) 1256 if err != nil { 1257 return fmt.Errorf("failed to create file: %w", err) 1258 } 1259 if o.id == "" { 1260 return errors.New("failed to create file: no ID") 1261 } 1262 // if created the file and returning an error then delete the file 1263 defer func() { 1264 if err != nil { 1265 delErr := o.fs.delete(ctx, true, o.id, remote, o.fs.opt.HardDelete) 1266 if delErr != nil { 1267 fs.Errorf(o, "failed to remove failed upload: %v", delErr) 1268 } 1269 } 1270 }() 1271 } 1272 1273 var resp *http.Response 1274 opts := rest.Opts{ 1275 Method: "PUT", 1276 RootURL: o.id, 1277 Path: "/data", 1278 NoResponse: true, 1279 Options: options, 1280 Body: in, 1281 } 1282 if size >= 0 { 1283 opts.ContentLength = &size 1284 } 1285 err = o.fs.pacer.CallNoRetry(func() (bool, error) { 1286 resp, err = o.fs.srv.Call(ctx, &opts) 1287 return shouldRetry(ctx, resp, err) 1288 }) 1289 if err != nil { 1290 return fmt.Errorf("failed to upload file: %w", err) 1291 } 1292 1293 o.hasMetaData = false 1294 return o.readMetaData(ctx) 1295 } 1296 1297 // Remove an object 1298 func (o *Object) Remove(ctx context.Context) error { 1299 return o.fs.delete(ctx, true, o.id, o.remote, o.fs.opt.HardDelete) 1300 } 1301 1302 // ID returns the ID of the Object if known, or "" if not 1303 func (o *Object) ID() string { 1304 return o.id 1305 } 1306 1307 // Check the interfaces are satisfied 1308 var ( 1309 _ fs.Fs = (*Fs)(nil) 1310 _ fs.Purger = (*Fs)(nil) 1311 _ fs.PutStreamer = (*Fs)(nil) 1312 _ fs.Copier = (*Fs)(nil) 1313 _ fs.Mover = (*Fs)(nil) 1314 _ fs.DirMover = (*Fs)(nil) 1315 _ fs.DirCacheFlusher = (*Fs)(nil) 1316 _ fs.PublicLinker = (*Fs)(nil) 1317 _ fs.Object = (*Object)(nil) 1318 _ fs.IDer = (*Object)(nil) 1319 )