github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/backend/combine/combine.go (about) 1 // Package combine implements a backend to combine multiple remotes in a directory tree 2 package combine 3 4 /* 5 Have API to add/remove branches in the combine 6 */ 7 8 import ( 9 "context" 10 "errors" 11 "fmt" 12 "io" 13 "path" 14 "strings" 15 "sync" 16 "time" 17 18 "github.com/rclone/rclone/fs" 19 "github.com/rclone/rclone/fs/cache" 20 "github.com/rclone/rclone/fs/config/configmap" 21 "github.com/rclone/rclone/fs/config/configstruct" 22 "github.com/rclone/rclone/fs/hash" 23 "github.com/rclone/rclone/fs/operations" 24 "github.com/rclone/rclone/fs/walk" 25 "golang.org/x/sync/errgroup" 26 ) 27 28 // Register with Fs 29 func init() { 30 fsi := &fs.RegInfo{ 31 Name: "combine", 32 Description: "Combine several remotes into one", 33 NewFs: NewFs, 34 MetadataInfo: &fs.MetadataInfo{ 35 Help: `Any metadata supported by the underlying remote is read and written.`, 36 }, 37 Options: []fs.Option{{ 38 Name: "upstreams", 39 Help: `Upstreams for combining 40 41 These should be in the form 42 43 dir=remote:path dir2=remote2:path 44 45 Where before the = is specified the root directory and after is the remote to 46 put there. 47 48 Embedded spaces can be added using quotes 49 50 "dir=remote:path with space" "dir2=remote2:path with space" 51 52 `, 53 Required: true, 54 Default: fs.SpaceSepList(nil), 55 }}, 56 } 57 fs.Register(fsi) 58 } 59 60 // Options defines the configuration for this backend 61 type Options struct { 62 Upstreams fs.SpaceSepList `config:"upstreams"` 63 } 64 65 // Fs represents a combine of upstreams 66 type Fs struct { 67 name string // name of this remote 68 features *fs.Features // optional features 69 opt Options // options for this Fs 70 root string // the path we are working on 71 hashSet hash.Set // common hashes 72 when time.Time // directory times 73 upstreams map[string]*upstream // map of upstreams 74 } 75 76 // adjustment stores the info to add a prefix to a path or chop characters off 77 type adjustment struct { 78 root string 79 rootSlash string 80 mountpoint string 81 mountpointSlash string 82 } 83 84 // newAdjustment makes a new path adjustment adjusting between mountpoint and root 85 // 86 // mountpoint is the point the upstream is mounted and root is the combine root 87 func newAdjustment(root, mountpoint string) (a adjustment) { 88 return adjustment{ 89 root: root, 90 rootSlash: root + "/", 91 mountpoint: mountpoint, 92 mountpointSlash: mountpoint + "/", 93 } 94 } 95 96 var errNotUnderRoot = errors.New("file not under root") 97 98 // do makes the adjustment on s, mapping an upstream path into a combine path 99 func (a *adjustment) do(s string) (string, error) { 100 absPath := join(a.mountpoint, s) 101 if a.root == "" { 102 return absPath, nil 103 } 104 if absPath == a.root { 105 return "", nil 106 } 107 if !strings.HasPrefix(absPath, a.rootSlash) { 108 return "", errNotUnderRoot 109 } 110 return absPath[len(a.rootSlash):], nil 111 } 112 113 // undo makes the adjustment on s, mapping a combine path into an upstream path 114 func (a *adjustment) undo(s string) (string, error) { 115 absPath := join(a.root, s) 116 if absPath == a.mountpoint { 117 return "", nil 118 } 119 if !strings.HasPrefix(absPath, a.mountpointSlash) { 120 return "", errNotUnderRoot 121 } 122 return absPath[len(a.mountpointSlash):], nil 123 } 124 125 // upstream represents an upstream Fs 126 type upstream struct { 127 f fs.Fs 128 parent *Fs 129 dir string // directory the upstream is mounted 130 pathAdjustment adjustment // how to fiddle with the path 131 } 132 133 // Create an upstream from the directory it is mounted on and the remote 134 func (f *Fs) newUpstream(ctx context.Context, dir, remote string) (*upstream, error) { 135 uFs, err := cache.Get(ctx, remote) 136 if err == fs.ErrorIsFile { 137 return nil, fmt.Errorf("can't combine files yet, only directories %q: %w", remote, err) 138 } 139 if err != nil { 140 return nil, fmt.Errorf("failed to create upstream %q: %w", remote, err) 141 } 142 u := &upstream{ 143 f: uFs, 144 parent: f, 145 dir: dir, 146 pathAdjustment: newAdjustment(f.root, dir), 147 } 148 cache.PinUntilFinalized(u.f, u) 149 return u, nil 150 } 151 152 // NewFs constructs an Fs from the path. 153 // 154 // The returned Fs is the actual Fs, referenced by remote in the config 155 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs.Fs, err error) { 156 // defer log.Trace(nil, "name=%q, root=%q, m=%v", name, root, m)("f=%+v, err=%v", &outFs, &err) 157 // Parse config into Options struct 158 opt := new(Options) 159 err = configstruct.Set(m, opt) 160 if err != nil { 161 return nil, err 162 } 163 // Backward compatible to old config 164 if len(opt.Upstreams) == 0 { 165 return nil, errors.New("combine can't point to an empty upstream - check the value of the upstreams setting") 166 } 167 for _, u := range opt.Upstreams { 168 if strings.HasPrefix(u, name+":") { 169 return nil, errors.New("can't point combine remote at itself - check the value of the upstreams setting") 170 } 171 } 172 isDir := false 173 for strings.HasSuffix(root, "/") { 174 root = root[:len(root)-1] 175 isDir = true 176 } 177 178 f := &Fs{ 179 name: name, 180 root: root, 181 opt: *opt, 182 upstreams: make(map[string]*upstream, len(opt.Upstreams)), 183 when: time.Now(), 184 } 185 186 g, gCtx := errgroup.WithContext(ctx) 187 var mu sync.Mutex 188 for _, upstream := range opt.Upstreams { 189 upstream := upstream 190 g.Go(func() (err error) { 191 equal := strings.IndexRune(upstream, '=') 192 if equal < 0 { 193 return fmt.Errorf("no \"=\" in upstream definition %q", upstream) 194 } 195 dir, remote := upstream[:equal], upstream[equal+1:] 196 if dir == "" { 197 return fmt.Errorf("empty dir in upstream definition %q", upstream) 198 } 199 if remote == "" { 200 return fmt.Errorf("empty remote in upstream definition %q", upstream) 201 } 202 if strings.ContainsRune(dir, '/') { 203 return fmt.Errorf("dirs can't contain / (yet): %q", dir) 204 } 205 u, err := f.newUpstream(gCtx, dir, remote) 206 if err != nil { 207 return err 208 } 209 mu.Lock() 210 if _, found := f.upstreams[dir]; found { 211 err = fmt.Errorf("duplicate directory name %q", dir) 212 } else { 213 f.upstreams[dir] = u 214 } 215 mu.Unlock() 216 return err 217 }) 218 } 219 err = g.Wait() 220 if err != nil { 221 return nil, err 222 } 223 // check features 224 var features = (&fs.Features{ 225 CaseInsensitive: true, 226 DuplicateFiles: false, 227 ReadMimeType: true, 228 WriteMimeType: true, 229 CanHaveEmptyDirectories: true, 230 BucketBased: true, 231 SetTier: true, 232 GetTier: true, 233 ReadMetadata: true, 234 WriteMetadata: true, 235 UserMetadata: true, 236 ReadDirMetadata: true, 237 WriteDirMetadata: true, 238 WriteDirSetModTime: true, 239 UserDirMetadata: true, 240 DirModTimeUpdatesOnWrite: true, 241 PartialUploads: true, 242 }).Fill(ctx, f) 243 canMove := true 244 for _, u := range f.upstreams { 245 features = features.Mask(ctx, u.f) // Mask all upstream fs 246 if !operations.CanServerSideMove(u.f) { 247 canMove = false 248 } 249 } 250 // We can move if all remotes support Move or Copy 251 if canMove { 252 features.Move = f.Move 253 } 254 255 // Enable ListR when upstreams either support ListR or is local 256 // But not when all upstreams are local 257 if features.ListR == nil { 258 for _, u := range f.upstreams { 259 if u.f.Features().ListR != nil { 260 features.ListR = f.ListR 261 } else if !u.f.Features().IsLocal { 262 features.ListR = nil 263 break 264 } 265 } 266 } 267 268 // Enable Purge when any upstreams support it 269 if features.Purge == nil { 270 for _, u := range f.upstreams { 271 if u.f.Features().Purge != nil { 272 features.Purge = f.Purge 273 break 274 } 275 } 276 } 277 278 // Enable Shutdown when any upstreams support it 279 if features.Shutdown == nil { 280 for _, u := range f.upstreams { 281 if u.f.Features().Shutdown != nil { 282 features.Shutdown = f.Shutdown 283 break 284 } 285 } 286 } 287 288 // Enable DirCacheFlush when any upstreams support it 289 if features.DirCacheFlush == nil { 290 for _, u := range f.upstreams { 291 if u.f.Features().DirCacheFlush != nil { 292 features.DirCacheFlush = f.DirCacheFlush 293 break 294 } 295 } 296 } 297 298 // Enable CleanUp when any upstreams support it 299 if features.CleanUp == nil { 300 for _, u := range f.upstreams { 301 if u.f.Features().CleanUp != nil { 302 features.CleanUp = f.CleanUp 303 break 304 } 305 } 306 } 307 308 // Enable ChangeNotify when any upstreams support it 309 if features.ChangeNotify == nil { 310 for _, u := range f.upstreams { 311 if u.f.Features().ChangeNotify != nil { 312 features.ChangeNotify = f.ChangeNotify 313 break 314 } 315 } 316 } 317 318 // show that we wrap other backends 319 features.Overlay = true 320 321 f.features = features 322 323 // Get common intersection of hashes 324 var hashSet hash.Set 325 var first = true 326 for _, u := range f.upstreams { 327 if first { 328 hashSet = u.f.Hashes() 329 first = false 330 } else { 331 hashSet = hashSet.Overlap(u.f.Hashes()) 332 } 333 } 334 f.hashSet = hashSet 335 336 // Check to see if the root is actually a file 337 if f.root != "" && !isDir { 338 _, err := f.NewObject(ctx, "") 339 if err != nil { 340 if err == fs.ErrorObjectNotFound || err == fs.ErrorNotAFile || err == fs.ErrorIsDir { 341 // File doesn't exist or is a directory so return old f 342 return f, nil 343 } 344 return nil, err 345 } 346 347 // Check to see if the root path is actually an existing file 348 f.root = path.Dir(f.root) 349 if f.root == "." { 350 f.root = "" 351 } 352 // Adjust path adjustment to remove leaf 353 for _, u := range f.upstreams { 354 u.pathAdjustment = newAdjustment(f.root, u.dir) 355 } 356 return f, fs.ErrorIsFile 357 } 358 return f, nil 359 } 360 361 // Run a function over all the upstreams in parallel 362 func (f *Fs) multithread(ctx context.Context, fn func(context.Context, *upstream) error) error { 363 g, gCtx := errgroup.WithContext(ctx) 364 for _, u := range f.upstreams { 365 u := u 366 g.Go(func() (err error) { 367 return fn(gCtx, u) 368 }) 369 } 370 return g.Wait() 371 } 372 373 // join the elements together but unlike path.Join return empty string 374 func join(elem ...string) string { 375 result := path.Join(elem...) 376 if result == "." { 377 return "" 378 } 379 if len(result) > 0 && result[0] == '/' { 380 result = result[1:] 381 } 382 return result 383 } 384 385 // find the upstream for the remote passed in, returning the upstream and the adjusted path 386 func (f *Fs) findUpstream(remote string) (u *upstream, uRemote string, err error) { 387 // defer log.Trace(remote, "")("f=%v, uRemote=%q, err=%v", &u, &uRemote, &err) 388 for _, u := range f.upstreams { 389 uRemote, err = u.pathAdjustment.undo(remote) 390 if err == nil { 391 return u, uRemote, nil 392 } 393 } 394 return nil, "", fmt.Errorf("combine for remote %q: %w", remote, fs.ErrorDirNotFound) 395 } 396 397 // Name of the remote (as passed into NewFs) 398 func (f *Fs) Name() string { 399 return f.name 400 } 401 402 // Root of the remote (as passed into NewFs) 403 func (f *Fs) Root() string { 404 return f.root 405 } 406 407 // String converts this Fs to a string 408 func (f *Fs) String() string { 409 return fmt.Sprintf("combine root '%s'", f.root) 410 } 411 412 // Features returns the optional features of this Fs 413 func (f *Fs) Features() *fs.Features { 414 return f.features 415 } 416 417 // Rmdir removes the root directory of the Fs object 418 func (f *Fs) Rmdir(ctx context.Context, dir string) error { 419 // The root always exists 420 if f.root == "" && dir == "" { 421 return nil 422 } 423 u, uRemote, err := f.findUpstream(dir) 424 if err != nil { 425 return err 426 } 427 return u.f.Rmdir(ctx, uRemote) 428 } 429 430 // Hashes returns hash.HashNone to indicate remote hashing is unavailable 431 func (f *Fs) Hashes() hash.Set { 432 return f.hashSet 433 } 434 435 // Mkdir makes the root directory of the Fs object 436 func (f *Fs) Mkdir(ctx context.Context, dir string) error { 437 // The root always exists 438 if f.root == "" && dir == "" { 439 return nil 440 } 441 u, uRemote, err := f.findUpstream(dir) 442 if err != nil { 443 return err 444 } 445 return u.f.Mkdir(ctx, uRemote) 446 } 447 448 // MkdirMetadata makes the root directory of the Fs object 449 func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) { 450 u, uRemote, err := f.findUpstream(dir) 451 if err != nil { 452 return nil, err 453 } 454 do := u.f.Features().MkdirMetadata 455 if do == nil { 456 return nil, fs.ErrorNotImplemented 457 } 458 newDir, err := do(ctx, uRemote, metadata) 459 if err != nil { 460 return nil, err 461 } 462 entries := fs.DirEntries{newDir} 463 entries, err = u.wrapEntries(ctx, entries) 464 if err != nil { 465 return nil, err 466 } 467 newDir, ok := entries[0].(fs.Directory) 468 if !ok { 469 return nil, fmt.Errorf("internal error: expecting %T to be fs.Directory", entries[0]) 470 } 471 return newDir, nil 472 } 473 474 // purge the upstream or fallback to a slow way 475 func (u *upstream) purge(ctx context.Context, dir string) (err error) { 476 if do := u.f.Features().Purge; do != nil { 477 err = do(ctx, dir) 478 } else { 479 err = operations.Purge(ctx, u.f, dir) 480 } 481 return err 482 } 483 484 // Purge all files in the directory 485 // 486 // Implement this if you have a way of deleting all the files 487 // quicker than just running Remove() on the result of List() 488 // 489 // Return an error if it doesn't exist 490 func (f *Fs) Purge(ctx context.Context, dir string) error { 491 if f.root == "" && dir == "" { 492 return f.multithread(ctx, func(ctx context.Context, u *upstream) error { 493 return u.purge(ctx, "") 494 }) 495 } 496 u, uRemote, err := f.findUpstream(dir) 497 if err != nil { 498 return err 499 } 500 return u.purge(ctx, uRemote) 501 } 502 503 // Copy src to this remote using server-side copy operations. 504 // 505 // This is stored with the remote path given. 506 // 507 // It returns the destination Object and a possible error. 508 // 509 // Will only be called if src.Fs().Name() == f.Name() 510 // 511 // If it isn't possible then return fs.ErrorCantCopy 512 func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { 513 srcObj, ok := src.(*Object) 514 if !ok { 515 fs.Debugf(src, "Can't copy - not same remote type") 516 return nil, fs.ErrorCantCopy 517 } 518 519 dstU, dstRemote, err := f.findUpstream(remote) 520 if err != nil { 521 return nil, err 522 } 523 524 do := dstU.f.Features().Copy 525 if do == nil { 526 return nil, fs.ErrorCantCopy 527 } 528 529 o, err := do(ctx, srcObj.Object, dstRemote) 530 if err != nil { 531 return nil, err 532 } 533 534 return dstU.newObject(o), nil 535 } 536 537 // Move src to this remote using server-side move operations. 538 // 539 // This is stored with the remote path given. 540 // 541 // It returns the destination Object and a possible error. 542 // 543 // Will only be called if src.Fs().Name() == f.Name() 544 // 545 // If it isn't possible then return fs.ErrorCantMove 546 func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { 547 srcObj, ok := src.(*Object) 548 if !ok { 549 fs.Debugf(src, "Can't move - not same remote type") 550 return nil, fs.ErrorCantMove 551 } 552 553 dstU, dstRemote, err := f.findUpstream(remote) 554 if err != nil { 555 return nil, err 556 } 557 558 do := dstU.f.Features().Move 559 useCopy := false 560 if do == nil { 561 do = dstU.f.Features().Copy 562 if do == nil { 563 return nil, fs.ErrorCantMove 564 } 565 useCopy = true 566 } 567 568 o, err := do(ctx, srcObj.Object, dstRemote) 569 if err != nil { 570 return nil, err 571 } 572 573 // If did Copy then remove the source object 574 if useCopy { 575 err = srcObj.Remove(ctx) 576 if err != nil { 577 return nil, err 578 } 579 } 580 581 return dstU.newObject(o), nil 582 } 583 584 // DirMove moves src, srcRemote to this remote at dstRemote 585 // using server-side move operations. 586 // 587 // Will only be called if src.Fs().Name() == f.Name() 588 // 589 // If it isn't possible then return fs.ErrorCantDirMove 590 // 591 // If destination exists then return fs.ErrorDirExists 592 func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) (err error) { 593 // defer log.Trace(f, "src=%v, srcRemote=%q, dstRemote=%q", src, srcRemote, dstRemote)("err=%v", &err) 594 srcFs, ok := src.(*Fs) 595 if !ok { 596 fs.Debugf(src, "Can't move directory - not same remote type") 597 return fs.ErrorCantDirMove 598 } 599 600 dstU, dstURemote, err := f.findUpstream(dstRemote) 601 if err != nil { 602 return err 603 } 604 605 srcU, srcURemote, err := srcFs.findUpstream(srcRemote) 606 if err != nil { 607 return err 608 } 609 610 do := dstU.f.Features().DirMove 611 if do == nil { 612 return fs.ErrorCantDirMove 613 } 614 615 fs.Logf(dstU.f, "srcU.f=%v, srcURemote=%q, dstURemote=%q", srcU.f, srcURemote, dstURemote) 616 return do(ctx, srcU.f, srcURemote, dstURemote) 617 } 618 619 // ChangeNotify calls the passed function with a path 620 // that has had changes. If the implementation 621 // uses polling, it should adhere to the given interval. 622 // At least one value will be written to the channel, 623 // specifying the initial value and updated values might 624 // follow. A 0 Duration should pause the polling. 625 // The ChangeNotify implementation must empty the channel 626 // regularly. When the channel gets closed, the implementation 627 // should stop polling and release resources. 628 func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), ch <-chan time.Duration) { 629 var uChans []chan time.Duration 630 631 for _, u := range f.upstreams { 632 u := u 633 if do := u.f.Features().ChangeNotify; do != nil { 634 ch := make(chan time.Duration) 635 uChans = append(uChans, ch) 636 wrappedNotifyFunc := func(path string, entryType fs.EntryType) { 637 newPath, err := u.pathAdjustment.do(path) 638 if err != nil { 639 fs.Logf(f, "ChangeNotify: unable to process %q: %s", path, err) 640 return 641 } 642 fs.Debugf(f, "ChangeNotify: path %q entryType %d", newPath, entryType) 643 notifyFunc(newPath, entryType) 644 } 645 do(ctx, wrappedNotifyFunc, ch) 646 } 647 } 648 649 go func() { 650 for i := range ch { 651 for _, c := range uChans { 652 c <- i 653 } 654 } 655 for _, c := range uChans { 656 close(c) 657 } 658 }() 659 } 660 661 // DirCacheFlush resets the directory cache - used in testing 662 // as an optional interface 663 func (f *Fs) DirCacheFlush() { 664 ctx := context.Background() 665 _ = f.multithread(ctx, func(ctx context.Context, u *upstream) error { 666 if do := u.f.Features().DirCacheFlush; do != nil { 667 do() 668 } 669 return nil 670 }) 671 } 672 673 func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bool, options ...fs.OpenOption) (fs.Object, error) { 674 srcPath := src.Remote() 675 u, uRemote, err := f.findUpstream(srcPath) 676 if err != nil { 677 return nil, err 678 } 679 uSrc := fs.NewOverrideRemote(src, uRemote) 680 var o fs.Object 681 if stream { 682 o, err = u.f.Features().PutStream(ctx, in, uSrc, options...) 683 } else { 684 o, err = u.f.Put(ctx, in, uSrc, options...) 685 } 686 if err != nil { 687 return nil, err 688 } 689 return u.newObject(o), nil 690 } 691 692 // Put in to the remote path with the modTime given of the given size 693 // 694 // May create the object even if it returns an error - if so 695 // will return the object and the error, otherwise will return 696 // nil and the error 697 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 698 o, err := f.NewObject(ctx, src.Remote()) 699 switch err { 700 case nil: 701 return o, o.Update(ctx, in, src, options...) 702 case fs.ErrorObjectNotFound: 703 return f.put(ctx, in, src, false, options...) 704 default: 705 return nil, err 706 } 707 } 708 709 // PutStream uploads to the remote path with the modTime given of indeterminate size 710 // 711 // May create the object even if it returns an error - if so 712 // will return the object and the error, otherwise will return 713 // nil and the error 714 func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 715 o, err := f.NewObject(ctx, src.Remote()) 716 switch err { 717 case nil: 718 return o, o.Update(ctx, in, src, options...) 719 case fs.ErrorObjectNotFound: 720 return f.put(ctx, in, src, true, options...) 721 default: 722 return nil, err 723 } 724 } 725 726 // About gets quota information from the Fs 727 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) { 728 usage := &fs.Usage{ 729 Total: new(int64), 730 Used: new(int64), 731 Trashed: new(int64), 732 Other: new(int64), 733 Free: new(int64), 734 Objects: new(int64), 735 } 736 for _, u := range f.upstreams { 737 doAbout := u.f.Features().About 738 if doAbout == nil { 739 continue 740 } 741 usg, err := doAbout(ctx) 742 if errors.Is(err, fs.ErrorDirNotFound) { 743 continue 744 } 745 if err != nil { 746 return nil, err 747 } 748 if usg.Total != nil && usage.Total != nil { 749 *usage.Total += *usg.Total 750 } else { 751 usage.Total = nil 752 } 753 if usg.Used != nil && usage.Used != nil { 754 *usage.Used += *usg.Used 755 } else { 756 usage.Used = nil 757 } 758 if usg.Trashed != nil && usage.Trashed != nil { 759 *usage.Trashed += *usg.Trashed 760 } else { 761 usage.Trashed = nil 762 } 763 if usg.Other != nil && usage.Other != nil { 764 *usage.Other += *usg.Other 765 } else { 766 usage.Other = nil 767 } 768 if usg.Free != nil && usage.Free != nil { 769 *usage.Free += *usg.Free 770 } else { 771 usage.Free = nil 772 } 773 if usg.Objects != nil && usage.Objects != nil { 774 *usage.Objects += *usg.Objects 775 } else { 776 usage.Objects = nil 777 } 778 } 779 return usage, nil 780 } 781 782 // Wraps entries for this upstream 783 func (u *upstream) wrapEntries(ctx context.Context, entries fs.DirEntries) (fs.DirEntries, error) { 784 for i, entry := range entries { 785 switch x := entry.(type) { 786 case fs.Object: 787 entries[i] = u.newObject(x) 788 case fs.Directory: 789 newPath, err := u.pathAdjustment.do(x.Remote()) 790 if err != nil { 791 return nil, err 792 } 793 newDir := fs.NewDirWrapper(newPath, x) 794 entries[i] = newDir 795 default: 796 return nil, fmt.Errorf("unknown entry type %T", entry) 797 } 798 } 799 return entries, nil 800 } 801 802 // List the objects and directories in dir into entries. The 803 // entries can be returned in any order but should be for a 804 // complete directory. 805 // 806 // dir should be "" to list the root, and should not have 807 // trailing slashes. 808 // 809 // This should return ErrDirNotFound if the directory isn't 810 // found. 811 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { 812 // defer log.Trace(f, "dir=%q", dir)("entries = %v, err=%v", &entries, &err) 813 if f.root == "" && dir == "" { 814 entries = make(fs.DirEntries, 0, len(f.upstreams)) 815 for combineDir := range f.upstreams { 816 d := fs.NewLimitedDirWrapper(combineDir, fs.NewDir(combineDir, f.when)) 817 entries = append(entries, d) 818 } 819 return entries, nil 820 } 821 u, uRemote, err := f.findUpstream(dir) 822 if err != nil { 823 return nil, err 824 } 825 entries, err = u.f.List(ctx, uRemote) 826 if err != nil { 827 return nil, err 828 } 829 return u.wrapEntries(ctx, entries) 830 } 831 832 // ListR lists the objects and directories of the Fs starting 833 // from dir recursively into out. 834 // 835 // dir should be "" to start from the root, and should not 836 // have trailing slashes. 837 // 838 // This should return ErrDirNotFound if the directory isn't 839 // found. 840 // 841 // It should call callback for each tranche of entries read. 842 // These need not be returned in any particular order. If 843 // callback returns an error then the listing will stop 844 // immediately. 845 // 846 // Don't implement this unless you have a more efficient way 847 // of listing recursively that doing a directory traversal. 848 func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) { 849 // defer log.Trace(f, "dir=%q, callback=%v", dir, callback)("err=%v", &err) 850 if f.root == "" && dir == "" { 851 rootEntries, err := f.List(ctx, "") 852 if err != nil { 853 return err 854 } 855 err = callback(rootEntries) 856 if err != nil { 857 return err 858 } 859 var mu sync.Mutex 860 syncCallback := func(entries fs.DirEntries) error { 861 mu.Lock() 862 defer mu.Unlock() 863 return callback(entries) 864 } 865 err = f.multithread(ctx, func(ctx context.Context, u *upstream) error { 866 return f.ListR(ctx, u.dir, syncCallback) 867 }) 868 if err != nil { 869 return err 870 } 871 return nil 872 } 873 u, uRemote, err := f.findUpstream(dir) 874 if err != nil { 875 return err 876 } 877 wrapCallback := func(entries fs.DirEntries) error { 878 entries, err := u.wrapEntries(ctx, entries) 879 if err != nil { 880 return err 881 } 882 return callback(entries) 883 } 884 if do := u.f.Features().ListR; do != nil { 885 err = do(ctx, uRemote, wrapCallback) 886 } else { 887 err = walk.ListR(ctx, u.f, uRemote, true, -1, walk.ListAll, wrapCallback) 888 } 889 if err == fs.ErrorDirNotFound { 890 err = nil 891 } 892 return err 893 } 894 895 // NewObject creates a new remote combine file object 896 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { 897 u, uRemote, err := f.findUpstream(remote) 898 if err != nil { 899 return nil, err 900 } 901 if uRemote == "" || strings.HasSuffix(uRemote, "/") { 902 return nil, fs.ErrorIsDir 903 } 904 o, err := u.f.NewObject(ctx, uRemote) 905 if err != nil { 906 return nil, err 907 } 908 return u.newObject(o), nil 909 } 910 911 // Precision is the greatest Precision of all upstreams 912 func (f *Fs) Precision() time.Duration { 913 var greatestPrecision time.Duration 914 for _, u := range f.upstreams { 915 uPrecision := u.f.Precision() 916 if uPrecision > greatestPrecision { 917 greatestPrecision = uPrecision 918 } 919 } 920 return greatestPrecision 921 } 922 923 // Shutdown the backend, closing any background tasks and any 924 // cached connections. 925 func (f *Fs) Shutdown(ctx context.Context) error { 926 return f.multithread(ctx, func(ctx context.Context, u *upstream) error { 927 if do := u.f.Features().Shutdown; do != nil { 928 return do(ctx) 929 } 930 return nil 931 }) 932 } 933 934 // PublicLink generates a public link to the remote path (usually readable by anyone) 935 func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) { 936 u, uRemote, err := f.findUpstream(remote) 937 if err != nil { 938 return "", err 939 } 940 do := u.f.Features().PublicLink 941 if do == nil { 942 return "", fs.ErrorNotImplemented 943 } 944 return do(ctx, uRemote, expire, unlink) 945 } 946 947 // PutUnchecked in to the remote path with the modTime given of the given size 948 // 949 // May create the object even if it returns an error - if so 950 // will return the object and the error, otherwise will return 951 // nil and the error 952 // 953 // May create duplicates or return errors if src already 954 // exists. 955 func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 956 srcPath := src.Remote() 957 u, uRemote, err := f.findUpstream(srcPath) 958 if err != nil { 959 return nil, err 960 } 961 do := u.f.Features().PutUnchecked 962 if do == nil { 963 return nil, fs.ErrorNotImplemented 964 } 965 uSrc := fs.NewOverrideRemote(src, uRemote) 966 return do(ctx, in, uSrc, options...) 967 } 968 969 // MergeDirs merges the contents of all the directories passed 970 // in into the first one and rmdirs the other directories. 971 func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error { 972 if len(dirs) == 0 { 973 return nil 974 } 975 var ( 976 u *upstream 977 uDirs []fs.Directory 978 ) 979 for _, dir := range dirs { 980 uNew, uDir, err := f.findUpstream(dir.Remote()) 981 if err != nil { 982 return err 983 } 984 if u == nil { 985 u = uNew 986 } else if u != uNew { 987 return fmt.Errorf("can't merge directories from different upstreams") 988 } 989 uDirs = append(uDirs, fs.NewOverrideDirectory(dir, uDir)) 990 } 991 do := u.f.Features().MergeDirs 992 if do == nil { 993 return fs.ErrorNotImplemented 994 } 995 return do(ctx, uDirs) 996 } 997 998 // DirSetModTime sets the directory modtime for dir 999 func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error { 1000 u, uDir, err := f.findUpstream(dir) 1001 if err != nil { 1002 return err 1003 } 1004 if uDir == "" { 1005 fs.Debugf(dir, "Can't set modtime on upstream root. skipping.") 1006 return nil 1007 } 1008 if do := u.f.Features().DirSetModTime; do != nil { 1009 return do(ctx, uDir, modTime) 1010 } 1011 return fs.ErrorNotImplemented 1012 } 1013 1014 // CleanUp the trash in the Fs 1015 // 1016 // Implement this if you have a way of emptying the trash or 1017 // otherwise cleaning up old versions of files. 1018 func (f *Fs) CleanUp(ctx context.Context) error { 1019 return f.multithread(ctx, func(ctx context.Context, u *upstream) error { 1020 if do := u.f.Features().CleanUp; do != nil { 1021 return do(ctx) 1022 } 1023 return nil 1024 }) 1025 } 1026 1027 // OpenWriterAt opens with a handle for random access writes 1028 // 1029 // Pass in the remote desired and the size if known. 1030 // 1031 // It truncates any existing object 1032 func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) { 1033 u, uRemote, err := f.findUpstream(remote) 1034 if err != nil { 1035 return nil, err 1036 } 1037 do := u.f.Features().OpenWriterAt 1038 if do == nil { 1039 return nil, fs.ErrorNotImplemented 1040 } 1041 return do(ctx, uRemote, size) 1042 } 1043 1044 // Object describes a wrapped Object 1045 // 1046 // This is a wrapped Object which knows its path prefix 1047 type Object struct { 1048 fs.Object 1049 u *upstream 1050 } 1051 1052 func (u *upstream) newObject(o fs.Object) *Object { 1053 return &Object{ 1054 Object: o, 1055 u: u, 1056 } 1057 } 1058 1059 // Fs returns read only access to the Fs that this object is part of 1060 func (o *Object) Fs() fs.Info { 1061 return o.u.parent 1062 } 1063 1064 // String returns the remote path 1065 func (o *Object) String() string { 1066 return o.Remote() 1067 } 1068 1069 // Remote returns the remote path 1070 func (o *Object) Remote() string { 1071 newPath, err := o.u.pathAdjustment.do(o.Object.String()) 1072 if err != nil { 1073 fs.Errorf(o.Object, "Bad object: %v", err) 1074 return err.Error() 1075 } 1076 return newPath 1077 } 1078 1079 // MimeType returns the content type of the Object if known 1080 func (o *Object) MimeType(ctx context.Context) (mimeType string) { 1081 if do, ok := o.Object.(fs.MimeTyper); ok { 1082 mimeType = do.MimeType(ctx) 1083 } 1084 return mimeType 1085 } 1086 1087 // UnWrap returns the Object that this Object is wrapping or 1088 // nil if it isn't wrapping anything 1089 func (o *Object) UnWrap() fs.Object { 1090 return o.Object 1091 } 1092 1093 // GetTier returns storage tier or class of the Object 1094 func (o *Object) GetTier() string { 1095 do, ok := o.Object.(fs.GetTierer) 1096 if !ok { 1097 return "" 1098 } 1099 return do.GetTier() 1100 } 1101 1102 // ID returns the ID of the Object if known, or "" if not 1103 func (o *Object) ID() string { 1104 do, ok := o.Object.(fs.IDer) 1105 if !ok { 1106 return "" 1107 } 1108 return do.ID() 1109 } 1110 1111 // Metadata returns metadata for an object 1112 // 1113 // It should return nil if there is no Metadata 1114 func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) { 1115 do, ok := o.Object.(fs.Metadataer) 1116 if !ok { 1117 return nil, nil 1118 } 1119 return do.Metadata(ctx) 1120 } 1121 1122 // SetMetadata sets metadata for an Object 1123 // 1124 // It should return fs.ErrorNotImplemented if it can't set metadata 1125 func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error { 1126 do, ok := o.Object.(fs.SetMetadataer) 1127 if !ok { 1128 return fs.ErrorNotImplemented 1129 } 1130 return do.SetMetadata(ctx, metadata) 1131 } 1132 1133 // SetTier performs changing storage tier of the Object if 1134 // multiple storage classes supported 1135 func (o *Object) SetTier(tier string) error { 1136 do, ok := o.Object.(fs.SetTierer) 1137 if !ok { 1138 return errors.New("underlying remote does not support SetTier") 1139 } 1140 return do.SetTier(tier) 1141 } 1142 1143 // Check the interfaces are satisfied 1144 var ( 1145 _ fs.Fs = (*Fs)(nil) 1146 _ fs.Purger = (*Fs)(nil) 1147 _ fs.PutStreamer = (*Fs)(nil) 1148 _ fs.Copier = (*Fs)(nil) 1149 _ fs.Mover = (*Fs)(nil) 1150 _ fs.DirMover = (*Fs)(nil) 1151 _ fs.DirCacheFlusher = (*Fs)(nil) 1152 _ fs.ChangeNotifier = (*Fs)(nil) 1153 _ fs.Abouter = (*Fs)(nil) 1154 _ fs.ListRer = (*Fs)(nil) 1155 _ fs.Shutdowner = (*Fs)(nil) 1156 _ fs.PublicLinker = (*Fs)(nil) 1157 _ fs.PutUncheckeder = (*Fs)(nil) 1158 _ fs.MergeDirser = (*Fs)(nil) 1159 _ fs.DirSetModTimer = (*Fs)(nil) 1160 _ fs.MkdirMetadataer = (*Fs)(nil) 1161 _ fs.CleanUpper = (*Fs)(nil) 1162 _ fs.OpenWriterAter = (*Fs)(nil) 1163 _ fs.FullObject = (*Object)(nil) 1164 )