github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/backend/union/union.go (about) 1 // Package union implements a virtual provider to join existing remotes. 2 package union 3 4 import ( 5 "bufio" 6 "context" 7 "errors" 8 "fmt" 9 "io" 10 "path" 11 "path/filepath" 12 "strings" 13 "sync" 14 "time" 15 16 "github.com/rclone/rclone/backend/union/common" 17 "github.com/rclone/rclone/backend/union/policy" 18 "github.com/rclone/rclone/backend/union/upstream" 19 "github.com/rclone/rclone/fs" 20 "github.com/rclone/rclone/fs/config/configmap" 21 "github.com/rclone/rclone/fs/config/configstruct" 22 "github.com/rclone/rclone/fs/hash" 23 "github.com/rclone/rclone/fs/operations" 24 "github.com/rclone/rclone/fs/walk" 25 ) 26 27 // Register with Fs 28 func init() { 29 fsi := &fs.RegInfo{ 30 Name: "union", 31 Description: "Union merges the contents of several upstream fs", 32 NewFs: NewFs, 33 MetadataInfo: &fs.MetadataInfo{ 34 Help: `Any metadata supported by the underlying remote is read and written.`, 35 }, 36 Options: []fs.Option{{ 37 Name: "upstreams", 38 Help: "List of space separated upstreams.\n\nCan be 'upstreama:test/dir upstreamb:', '\"upstreama:test/space:ro dir\" upstreamb:', etc.", 39 Required: true, 40 }, { 41 Name: "action_policy", 42 Help: "Policy to choose upstream on ACTION category.", 43 Default: "epall", 44 }, { 45 Name: "create_policy", 46 Help: "Policy to choose upstream on CREATE category.", 47 Default: "epmfs", 48 }, { 49 Name: "search_policy", 50 Help: "Policy to choose upstream on SEARCH category.", 51 Default: "ff", 52 }, { 53 Name: "cache_time", 54 Help: "Cache time of usage and free space (in seconds).\n\nThis option is only useful when a path preserving policy is used.", 55 Default: 120, 56 }, { 57 Name: "min_free_space", 58 Help: `Minimum viable free space for lfs/eplfs policies. 59 60 If a remote has less than this much free space then it won't be 61 considered for use in lfs or eplfs policies.`, 62 Advanced: true, 63 Default: fs.Gibi, 64 }}, 65 } 66 fs.Register(fsi) 67 } 68 69 // Fs represents a union of upstreams 70 type Fs struct { 71 name string // name of this remote 72 features *fs.Features // optional features 73 opt common.Options // options for this Fs 74 root string // the path we are working on 75 upstreams []*upstream.Fs // slice of upstreams 76 hashSet hash.Set // intersection of hash types 77 actionPolicy policy.Policy // policy for ACTION 78 createPolicy policy.Policy // policy for CREATE 79 searchPolicy policy.Policy // policy for SEARCH 80 } 81 82 // Wrap candidate objects in to a union Object 83 func (f *Fs) wrapEntries(entries ...upstream.Entry) (entry, error) { 84 e, err := f.searchEntries(entries...) 85 if err != nil { 86 return nil, err 87 } 88 switch e := e.(type) { 89 case *upstream.Object: 90 return &Object{ 91 Object: e, 92 fs: f, 93 co: entries, 94 }, nil 95 case *upstream.Directory: 96 return &Directory{ 97 Directory: e, 98 fs: f, 99 cd: entries, 100 }, nil 101 default: 102 return nil, fmt.Errorf("unknown object type %T", e) 103 } 104 } 105 106 // Name of the remote (as passed into NewFs) 107 func (f *Fs) Name() string { 108 return f.name 109 } 110 111 // Root of the remote (as passed into NewFs) 112 func (f *Fs) Root() string { 113 return f.root 114 } 115 116 // String converts this Fs to a string 117 func (f *Fs) String() string { 118 return fmt.Sprintf("union root '%s'", f.root) 119 } 120 121 // Features returns the optional features of this Fs 122 func (f *Fs) Features() *fs.Features { 123 return f.features 124 } 125 126 // Rmdir removes the root directory of the Fs object 127 func (f *Fs) Rmdir(ctx context.Context, dir string) error { 128 upstreams, err := f.action(ctx, dir) 129 if err != nil { 130 // If none of the backends can have empty directories then 131 // don't complain about directories not being found 132 if !f.features.CanHaveEmptyDirectories && err == fs.ErrorObjectNotFound { 133 return nil 134 } 135 return err 136 } 137 errs := Errors(make([]error, len(upstreams))) 138 multithread(len(upstreams), func(i int) { 139 err := upstreams[i].Rmdir(ctx, dir) 140 if err != nil { 141 errs[i] = fmt.Errorf("%s: %w", upstreams[i].Name(), err) 142 } 143 }) 144 return errs.Err() 145 } 146 147 // Hashes returns hash.HashNone to indicate remote hashing is unavailable 148 func (f *Fs) Hashes() hash.Set { 149 return f.hashSet 150 } 151 152 // mkdir makes the directory passed in and returns the upstreams used 153 func (f *Fs) mkdir(ctx context.Context, dir string) ([]*upstream.Fs, error) { 154 upstreams, err := f.create(ctx, dir) 155 if err == fs.ErrorObjectNotFound { 156 parent := parentDir(dir) 157 if dir != parent { 158 upstreams, err = f.mkdir(ctx, parent) 159 } else if dir == "" { 160 // If root dirs not created then create them 161 upstreams, err = f.upstreams, nil 162 } 163 } 164 if err != nil { 165 return nil, err 166 } 167 errs := Errors(make([]error, len(upstreams))) 168 multithread(len(upstreams), func(i int) { 169 err := upstreams[i].Mkdir(ctx, dir) 170 if err != nil { 171 errs[i] = fmt.Errorf("%s: %w", upstreams[i].Name(), err) 172 } 173 }) 174 err = errs.Err() 175 if err != nil { 176 return nil, err 177 } 178 // If created roots then choose one 179 if dir == "" { 180 upstreams, err = f.create(ctx, dir) 181 } 182 return upstreams, err 183 } 184 185 // Mkdir makes the root directory of the Fs object 186 func (f *Fs) Mkdir(ctx context.Context, dir string) error { 187 _, err := f.mkdir(ctx, dir) 188 return err 189 } 190 191 // MkdirMetadata makes the root directory of the Fs object 192 func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) { 193 upstreams, err := f.create(ctx, dir) 194 if err != nil { 195 return nil, err 196 } 197 errs := Errors(make([]error, len(upstreams))) 198 entries := make([]upstream.Entry, len(upstreams)) 199 multithread(len(upstreams), func(i int) { 200 u := upstreams[i] 201 if do := u.Features().MkdirMetadata; do != nil { 202 newDir, err := do(ctx, dir, metadata) 203 if err != nil { 204 errs[i] = fmt.Errorf("%s: %w", upstreams[i].Name(), err) 205 } else { 206 entries[i], err = u.WrapEntry(newDir) 207 if err != nil { 208 errs[i] = fmt.Errorf("%s: %w", upstreams[i].Name(), err) 209 } 210 } 211 212 } else { 213 // Just do Mkdir on upstreams which don't support MkdirMetadata 214 err := u.Mkdir(ctx, dir) 215 if err != nil { 216 errs[i] = fmt.Errorf("%s: %w", upstreams[i].Name(), err) 217 } 218 } 219 }) 220 err = errs.Err() 221 if err != nil { 222 return nil, err 223 } 224 225 entry, err := f.wrapEntries(entries...) 226 if err != nil { 227 return nil, err 228 } 229 newDir, ok := entry.(fs.Directory) 230 if !ok { 231 return nil, fmt.Errorf("internal error: expecting %T to be an fs.Directory", entry) 232 } 233 return newDir, nil 234 } 235 236 // Purge all files in the directory 237 // 238 // Implement this if you have a way of deleting all the files 239 // quicker than just running Remove() on the result of List() 240 // 241 // Return an error if it doesn't exist 242 func (f *Fs) Purge(ctx context.Context, dir string) error { 243 for _, r := range f.upstreams { 244 if r.Features().Purge == nil { 245 return fs.ErrorCantPurge 246 } 247 } 248 upstreams, err := f.action(ctx, "") 249 if err != nil { 250 return err 251 } 252 errs := Errors(make([]error, len(upstreams))) 253 multithread(len(upstreams), func(i int) { 254 err := upstreams[i].Features().Purge(ctx, dir) 255 if errors.Is(err, fs.ErrorDirNotFound) { 256 err = nil 257 } 258 if err != nil { 259 errs[i] = fmt.Errorf("%s: %w", upstreams[i].Name(), err) 260 } 261 }) 262 return errs.Err() 263 } 264 265 // Copy src to this remote using server-side copy operations. 266 // 267 // This is stored with the remote path given. 268 // 269 // It returns the destination Object and a possible error. 270 // 271 // Will only be called if src.Fs().Name() == f.Name() 272 // 273 // If it isn't possible then return fs.ErrorCantCopy 274 func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { 275 srcObj, ok := src.(*Object) 276 if !ok { 277 fs.Debugf(src, "Can't copy - not same remote type") 278 return nil, fs.ErrorCantCopy 279 } 280 o := srcObj.UnWrapUpstream() 281 su := o.UpstreamFs() 282 if su.Features().Copy == nil { 283 return nil, fs.ErrorCantCopy 284 } 285 var du *upstream.Fs 286 for _, u := range f.upstreams { 287 if operations.Same(u.RootFs, su.RootFs) { 288 du = u 289 } 290 } 291 if du == nil { 292 return nil, fs.ErrorCantCopy 293 } 294 if !du.IsCreatable() { 295 return nil, fs.ErrorPermissionDenied 296 } 297 co, err := du.Features().Copy(ctx, o, remote) 298 if err != nil || co == nil { 299 return nil, err 300 } 301 wo, err := f.wrapEntries(du.WrapObject(co)) 302 return wo.(*Object), err 303 } 304 305 // Move src to this remote using server-side move operations. 306 // 307 // This is stored with the remote path given. 308 // 309 // It returns the destination Object and a possible error. 310 // 311 // Will only be called if src.Fs().Name() == f.Name() 312 // 313 // If it isn't possible then return fs.ErrorCantMove 314 func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { 315 o, ok := src.(*Object) 316 if !ok { 317 fs.Debugf(src, "Can't move - not same remote type") 318 return nil, fs.ErrorCantMove 319 } 320 entries, err := f.actionEntries(o.candidates()...) 321 if err != nil { 322 return nil, err 323 } 324 for _, e := range entries { 325 if !operations.CanServerSideMove(e.UpstreamFs()) { 326 return nil, fs.ErrorCantMove 327 } 328 } 329 objs := make([]*upstream.Object, len(entries)) 330 errs := Errors(make([]error, len(entries))) 331 multithread(len(entries), func(i int) { 332 su := entries[i].UpstreamFs() 333 o, ok := entries[i].(*upstream.Object) 334 if !ok { 335 errs[i] = fmt.Errorf("%s: %w", su.Name(), fs.ErrorNotAFile) 336 return 337 } 338 var du *upstream.Fs 339 for _, u := range f.upstreams { 340 if operations.Same(u.RootFs, su.RootFs) { 341 du = u 342 } 343 } 344 if du == nil { 345 errs[i] = fmt.Errorf("%s: %s: %w", su.Name(), remote, fs.ErrorCantMove) 346 return 347 } 348 srcObj := o.UnWrap() 349 duFeatures := du.Features() 350 do := duFeatures.Move 351 if duFeatures.Move == nil { 352 do = duFeatures.Copy 353 } 354 // Do the Move or Copy 355 dstObj, err := do(ctx, srcObj, remote) 356 if err != nil { 357 errs[i] = fmt.Errorf("%s: %w", su.Name(), err) 358 return 359 } 360 if dstObj == nil { 361 errs[i] = fmt.Errorf("%s: destination object not found", su.Name()) 362 return 363 } 364 objs[i] = du.WrapObject(dstObj) 365 // Delete the source object if Copy 366 if duFeatures.Move == nil { 367 err = srcObj.Remove(ctx) 368 if err != nil { 369 errs[i] = fmt.Errorf("%s: %w", su.Name(), err) 370 return 371 } 372 } 373 }) 374 var en []upstream.Entry 375 for _, o := range objs { 376 if o != nil { 377 en = append(en, o) 378 } 379 } 380 e, err := f.wrapEntries(en...) 381 if err != nil { 382 return nil, err 383 } 384 return e.(*Object), errs.Err() 385 } 386 387 // DirMove moves src, srcRemote to this remote at dstRemote 388 // using server-side move operations. 389 // 390 // Will only be called if src.Fs().Name() == f.Name() 391 // 392 // If it isn't possible then return fs.ErrorCantDirMove 393 // 394 // If destination exists then return fs.ErrorDirExists 395 func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { 396 sfs, ok := src.(*Fs) 397 if !ok { 398 fs.Debugf(src, "Can't move directory - not same remote type") 399 return fs.ErrorCantDirMove 400 } 401 upstreams, err := sfs.action(ctx, srcRemote) 402 if err != nil { 403 return err 404 } 405 for _, u := range upstreams { 406 if u.Features().DirMove == nil { 407 return fs.ErrorCantDirMove 408 } 409 } 410 errs := Errors(make([]error, len(upstreams))) 411 multithread(len(upstreams), func(i int) { 412 su := upstreams[i] 413 var du *upstream.Fs 414 for _, u := range f.upstreams { 415 if operations.Same(u.RootFs, su.RootFs) { 416 du = u 417 } 418 } 419 if du == nil { 420 errs[i] = fmt.Errorf("%s: %s: %w", su.Name(), su.Root(), fs.ErrorCantDirMove) 421 return 422 } 423 err := du.Features().DirMove(ctx, su.Fs, srcRemote, dstRemote) 424 if err != nil { 425 errs[i] = fmt.Errorf("%s: %w", du.Name()+":"+du.Root(), err) 426 } 427 }) 428 errs = errs.FilterNil() 429 if len(errs) == 0 { 430 return nil 431 } 432 for _, e := range errs { 433 if !errors.Is(e, fs.ErrorDirExists) { 434 return errs 435 } 436 } 437 return fs.ErrorDirExists 438 } 439 440 // DirSetModTime sets the directory modtime for dir 441 func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error { 442 upstreams, err := f.action(ctx, dir) 443 if err != nil { 444 return err 445 } 446 errs := Errors(make([]error, len(upstreams))) 447 multithread(len(upstreams), func(i int) { 448 u := upstreams[i] 449 // ignore DirSetModTime on upstreams which don't support it 450 if do := u.Features().DirSetModTime; do != nil { 451 err := do(ctx, dir, modTime) 452 if err != nil { 453 errs[i] = fmt.Errorf("%s: %w", upstreams[i].Name(), err) 454 } 455 } 456 }) 457 return errs.Err() 458 } 459 460 // ChangeNotify calls the passed function with a path 461 // that has had changes. If the implementation 462 // uses polling, it should adhere to the given interval. 463 // At least one value will be written to the channel, 464 // specifying the initial value and updated values might 465 // follow. A 0 Duration should pause the polling. 466 // The ChangeNotify implementation must empty the channel 467 // regularly. When the channel gets closed, the implementation 468 // should stop polling and release resources. 469 func (f *Fs) ChangeNotify(ctx context.Context, fn func(string, fs.EntryType), ch <-chan time.Duration) { 470 var uChans []chan time.Duration 471 472 for _, u := range f.upstreams { 473 if ChangeNotify := u.Features().ChangeNotify; ChangeNotify != nil { 474 ch := make(chan time.Duration) 475 uChans = append(uChans, ch) 476 ChangeNotify(ctx, fn, ch) 477 } 478 } 479 480 go func() { 481 for i := range ch { 482 for _, c := range uChans { 483 c <- i 484 } 485 } 486 for _, c := range uChans { 487 close(c) 488 } 489 }() 490 } 491 492 // DirCacheFlush resets the directory cache - used in testing 493 // as an optional interface 494 func (f *Fs) DirCacheFlush() { 495 multithread(len(f.upstreams), func(i int) { 496 if do := f.upstreams[i].Features().DirCacheFlush; do != nil { 497 do() 498 } 499 }) 500 } 501 502 // Tee in into n outputs 503 // 504 // When finished read the error from the channel 505 func multiReader(n int, in io.Reader) ([]io.Reader, <-chan error) { 506 readers := make([]io.Reader, n) 507 pipeWriters := make([]*io.PipeWriter, n) 508 writers := make([]io.Writer, n) 509 errChan := make(chan error, 1) 510 for i := range writers { 511 r, w := io.Pipe() 512 bw := bufio.NewWriter(w) 513 readers[i], pipeWriters[i], writers[i] = r, w, bw 514 } 515 go func() { 516 mw := io.MultiWriter(writers...) 517 es := make([]error, 2*n+1) 518 _, copyErr := io.Copy(mw, in) 519 es[2*n] = copyErr 520 // Flush the buffers 521 for i, bw := range writers { 522 es[i] = bw.(*bufio.Writer).Flush() 523 } 524 // Close the underlying pipes 525 for i, pw := range pipeWriters { 526 es[2*i] = pw.CloseWithError(copyErr) 527 } 528 errChan <- Errors(es).Err() 529 }() 530 return readers, errChan 531 } 532 533 func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bool, options ...fs.OpenOption) (fs.Object, error) { 534 srcPath := src.Remote() 535 upstreams, err := f.create(ctx, srcPath) 536 if err == fs.ErrorObjectNotFound { 537 upstreams, err = f.mkdir(ctx, parentDir(srcPath)) 538 } 539 if err != nil { 540 return nil, err 541 } 542 if len(upstreams) == 1 { 543 u := upstreams[0] 544 var o fs.Object 545 var err error 546 if stream { 547 o, err = u.Features().PutStream(ctx, in, src, options...) 548 } else { 549 o, err = u.Put(ctx, in, src, options...) 550 } 551 if err != nil { 552 return nil, err 553 } 554 e, err := f.wrapEntries(u.WrapObject(o)) 555 return e.(*Object), err 556 } 557 // Multi-threading 558 readers, errChan := multiReader(len(upstreams), in) 559 errs := Errors(make([]error, len(upstreams)+1)) 560 objs := make([]upstream.Entry, len(upstreams)) 561 multithread(len(upstreams), func(i int) { 562 u := upstreams[i] 563 var o fs.Object 564 var err error 565 if stream { 566 o, err = u.Features().PutStream(ctx, readers[i], src, options...) 567 } else { 568 o, err = u.Put(ctx, readers[i], src, options...) 569 } 570 if err != nil { 571 errs[i] = fmt.Errorf("%s: %w", u.Name(), err) 572 if len(upstreams) > 1 { 573 // Drain the input buffer to allow other uploads to continue 574 _, _ = io.Copy(io.Discard, readers[i]) 575 } 576 return 577 } 578 objs[i] = u.WrapObject(o) 579 }) 580 errs[len(upstreams)] = <-errChan 581 err = errs.Err() 582 if err != nil { 583 return nil, err 584 } 585 e, err := f.wrapEntries(objs...) 586 return e.(*Object), err 587 } 588 589 // Put in to the remote path with the modTime given of the given size 590 // 591 // May create the object even if it returns an error - if so 592 // will return the object and the error, otherwise will return 593 // nil and the error 594 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 595 o, err := f.NewObject(ctx, src.Remote()) 596 switch err { 597 case nil: 598 return o, o.Update(ctx, in, src, options...) 599 case fs.ErrorObjectNotFound: 600 return f.put(ctx, in, src, false, options...) 601 default: 602 return nil, err 603 } 604 } 605 606 // PutStream uploads to the remote path with the modTime given of indeterminate size 607 // 608 // May create the object even if it returns an error - if so 609 // will return the object and the error, otherwise will return 610 // nil and the error 611 func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 612 o, err := f.NewObject(ctx, src.Remote()) 613 switch err { 614 case nil: 615 return o, o.Update(ctx, in, src, options...) 616 case fs.ErrorObjectNotFound: 617 return f.put(ctx, in, src, true, options...) 618 default: 619 return nil, err 620 } 621 } 622 623 // About gets quota information from the Fs 624 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) { 625 usage := &fs.Usage{ 626 Total: new(int64), 627 Used: new(int64), 628 Trashed: new(int64), 629 Other: new(int64), 630 Free: new(int64), 631 Objects: new(int64), 632 } 633 for _, u := range f.upstreams { 634 usg, err := u.About(ctx) 635 if errors.Is(err, fs.ErrorDirNotFound) { 636 continue 637 } 638 if err != nil { 639 return nil, err 640 } 641 if usg.Total != nil && usage.Total != nil { 642 *usage.Total += *usg.Total 643 } else { 644 usage.Total = nil 645 } 646 if usg.Used != nil && usage.Used != nil { 647 *usage.Used += *usg.Used 648 } else { 649 usage.Used = nil 650 } 651 if usg.Trashed != nil && usage.Trashed != nil { 652 *usage.Trashed += *usg.Trashed 653 } else { 654 usage.Trashed = nil 655 } 656 if usg.Other != nil && usage.Other != nil { 657 *usage.Other += *usg.Other 658 } else { 659 usage.Other = nil 660 } 661 if usg.Free != nil && usage.Free != nil { 662 *usage.Free += *usg.Free 663 } else { 664 usage.Free = nil 665 } 666 if usg.Objects != nil && usage.Objects != nil { 667 *usage.Objects += *usg.Objects 668 } else { 669 usage.Objects = nil 670 } 671 } 672 return usage, nil 673 } 674 675 // List the objects and directories in dir into entries. The 676 // entries can be returned in any order but should be for a 677 // complete directory. 678 // 679 // dir should be "" to list the root, and should not have 680 // trailing slashes. 681 // 682 // This should return ErrDirNotFound if the directory isn't 683 // found. 684 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { 685 entriesList := make([][]upstream.Entry, len(f.upstreams)) 686 errs := Errors(make([]error, len(f.upstreams))) 687 multithread(len(f.upstreams), func(i int) { 688 u := f.upstreams[i] 689 entries, err := u.List(ctx, dir) 690 if err != nil { 691 errs[i] = fmt.Errorf("%s: %w", u.Name(), err) 692 return 693 } 694 uEntries := make([]upstream.Entry, len(entries)) 695 for j, e := range entries { 696 uEntries[j], _ = u.WrapEntry(e) 697 } 698 entriesList[i] = uEntries 699 }) 700 if len(errs) == len(errs.FilterNil()) { 701 errs = errs.Map(func(e error) error { 702 if errors.Is(e, fs.ErrorDirNotFound) { 703 return nil 704 } 705 return e 706 }) 707 if len(errs) == 0 { 708 return nil, fs.ErrorDirNotFound 709 } 710 return nil, errs.Err() 711 } 712 return f.mergeDirEntries(entriesList) 713 } 714 715 // ListR lists the objects and directories of the Fs starting 716 // from dir recursively into out. 717 // 718 // dir should be "" to start from the root, and should not 719 // have trailing slashes. 720 // 721 // This should return ErrDirNotFound if the directory isn't 722 // found. 723 // 724 // It should call callback for each tranche of entries read. 725 // These need not be returned in any particular order. If 726 // callback returns an error then the listing will stop 727 // immediately. 728 // 729 // Don't implement this unless you have a more efficient way 730 // of listing recursively that doing a directory traversal. 731 func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) { 732 var entriesList [][]upstream.Entry 733 errs := Errors(make([]error, len(f.upstreams))) 734 var mutex sync.Mutex 735 multithread(len(f.upstreams), func(i int) { 736 u := f.upstreams[i] 737 var err error 738 callback := func(entries fs.DirEntries) error { 739 uEntries := make([]upstream.Entry, len(entries)) 740 for j, e := range entries { 741 uEntries[j], _ = u.WrapEntry(e) 742 } 743 mutex.Lock() 744 entriesList = append(entriesList, uEntries) 745 mutex.Unlock() 746 return nil 747 } 748 do := u.Features().ListR 749 if do != nil { 750 err = do(ctx, dir, callback) 751 } else { 752 err = walk.ListR(ctx, u, dir, true, -1, walk.ListAll, callback) 753 } 754 if err != nil { 755 errs[i] = fmt.Errorf("%s: %w", u.Name(), err) 756 return 757 } 758 }) 759 if len(errs) == len(errs.FilterNil()) { 760 errs = errs.Map(func(e error) error { 761 if errors.Is(e, fs.ErrorDirNotFound) { 762 return nil 763 } 764 return e 765 }) 766 if len(errs) == 0 { 767 return fs.ErrorDirNotFound 768 } 769 return errs.Err() 770 } 771 entries, err := f.mergeDirEntries(entriesList) 772 if err != nil { 773 return err 774 } 775 return callback(entries) 776 } 777 778 // NewObject creates a new remote union file object 779 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { 780 objs := make([]*upstream.Object, len(f.upstreams)) 781 errs := Errors(make([]error, len(f.upstreams))) 782 multithread(len(f.upstreams), func(i int) { 783 u := f.upstreams[i] 784 o, err := u.NewObject(ctx, remote) 785 if err != nil && err != fs.ErrorObjectNotFound { 786 errs[i] = fmt.Errorf("%s: %w", u.Name(), err) 787 return 788 } 789 objs[i] = u.WrapObject(o) 790 }) 791 var entries []upstream.Entry 792 for _, o := range objs { 793 if o != nil { 794 entries = append(entries, o) 795 } 796 } 797 if len(entries) == 0 { 798 return nil, fs.ErrorObjectNotFound 799 } 800 e, err := f.wrapEntries(entries...) 801 if err != nil { 802 return nil, err 803 } 804 return e.(*Object), errs.Err() 805 } 806 807 // Precision is the greatest Precision of all upstreams 808 func (f *Fs) Precision() time.Duration { 809 var greatestPrecision time.Duration 810 for _, u := range f.upstreams { 811 if u.Precision() > greatestPrecision { 812 greatestPrecision = u.Precision() 813 } 814 } 815 return greatestPrecision 816 } 817 818 func (f *Fs) action(ctx context.Context, path string) ([]*upstream.Fs, error) { 819 return f.actionPolicy.Action(ctx, f.upstreams, path) 820 } 821 822 func (f *Fs) actionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) { 823 return f.actionPolicy.ActionEntries(entries...) 824 } 825 826 func (f *Fs) create(ctx context.Context, path string) ([]*upstream.Fs, error) { 827 return f.createPolicy.Create(ctx, f.upstreams, path) 828 } 829 830 func (f *Fs) searchEntries(entries ...upstream.Entry) (upstream.Entry, error) { 831 return f.searchPolicy.SearchEntries(entries...) 832 } 833 834 func (f *Fs) mergeDirEntries(entriesList [][]upstream.Entry) (fs.DirEntries, error) { 835 entryMap := make(map[string]([]upstream.Entry)) 836 for _, en := range entriesList { 837 if en == nil { 838 continue 839 } 840 for _, entry := range en { 841 remote := entry.Remote() 842 if f.Features().CaseInsensitive { 843 remote = strings.ToLower(remote) 844 } 845 entryMap[remote] = append(entryMap[remote], entry) 846 } 847 } 848 var entries fs.DirEntries 849 for path := range entryMap { 850 e, err := f.wrapEntries(entryMap[path]...) 851 if err != nil { 852 return nil, err 853 } 854 entries = append(entries, e) 855 } 856 return entries, nil 857 } 858 859 // Shutdown the backend, closing any background tasks and any 860 // cached connections. 861 func (f *Fs) Shutdown(ctx context.Context) error { 862 errs := Errors(make([]error, len(f.upstreams))) 863 multithread(len(f.upstreams), func(i int) { 864 u := f.upstreams[i] 865 if do := u.Features().Shutdown; do != nil { 866 err := do(ctx) 867 if err != nil { 868 errs[i] = fmt.Errorf("%s: %w", u.Name(), err) 869 } 870 } 871 }) 872 return errs.Err() 873 } 874 875 // CleanUp the trash in the Fs 876 // 877 // Implement this if you have a way of emptying the trash or 878 // otherwise cleaning up old versions of files. 879 func (f *Fs) CleanUp(ctx context.Context) error { 880 errs := Errors(make([]error, len(f.upstreams))) 881 multithread(len(f.upstreams), func(i int) { 882 u := f.upstreams[i] 883 if do := u.Features().CleanUp; do != nil { 884 err := do(ctx) 885 if err != nil { 886 errs[i] = fmt.Errorf("%s: %w", u.Name(), err) 887 } 888 } 889 }) 890 return errs.Err() 891 } 892 893 // NewFs constructs an Fs from the path. 894 // 895 // The returned Fs is the actual Fs, referenced by remote in the config 896 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { 897 // Parse config into Options struct 898 opt := new(common.Options) 899 err := configstruct.Set(m, opt) 900 if err != nil { 901 return nil, err 902 } 903 // Backward compatible to old config 904 if len(opt.Upstreams) == 0 && len(opt.Remotes) > 0 { 905 for i := 0; i < len(opt.Remotes)-1; i++ { 906 opt.Remotes[i] = opt.Remotes[i] + ":ro" 907 } 908 opt.Upstreams = opt.Remotes 909 } 910 if len(opt.Upstreams) == 0 { 911 return nil, errors.New("union can't point to an empty upstream - check the value of the upstreams setting") 912 } 913 if len(opt.Upstreams) == 1 { 914 return nil, errors.New("union can't point to a single upstream - check the value of the upstreams setting") 915 } 916 for _, u := range opt.Upstreams { 917 if strings.HasPrefix(u, name+":") { 918 return nil, errors.New("can't point union remote at itself - check the value of the upstreams setting") 919 } 920 } 921 922 root = strings.Trim(root, "/") 923 upstreams := make([]*upstream.Fs, len(opt.Upstreams)) 924 errs := Errors(make([]error, len(opt.Upstreams))) 925 multithread(len(opt.Upstreams), func(i int) { 926 u := opt.Upstreams[i] 927 upstreams[i], errs[i] = upstream.New(ctx, u, root, opt) 928 }) 929 var usedUpstreams []*upstream.Fs 930 var fserr error 931 for i, err := range errs { 932 if err != nil && err != fs.ErrorIsFile { 933 return nil, err 934 } 935 // Only the upstreams returns ErrorIsFile would be used if any 936 if err == fs.ErrorIsFile { 937 usedUpstreams = append(usedUpstreams, upstreams[i]) 938 fserr = fs.ErrorIsFile 939 } 940 } 941 if fserr == nil { 942 usedUpstreams = upstreams 943 } 944 945 f := &Fs{ 946 name: name, 947 root: root, 948 opt: *opt, 949 upstreams: usedUpstreams, 950 } 951 // Correct root if definitely pointing to a file 952 if fserr == fs.ErrorIsFile { 953 f.root = path.Dir(f.root) 954 if f.root == "." || f.root == "/" { 955 f.root = "" 956 } 957 } 958 err = upstream.Prepare(f.upstreams) 959 if err != nil { 960 return nil, err 961 } 962 f.actionPolicy, err = policy.Get(opt.ActionPolicy) 963 if err != nil { 964 return nil, err 965 } 966 f.createPolicy, err = policy.Get(opt.CreatePolicy) 967 if err != nil { 968 return nil, err 969 } 970 f.searchPolicy, err = policy.Get(opt.SearchPolicy) 971 if err != nil { 972 return nil, err 973 } 974 fs.Debugf(f, "actionPolicy = %T, createPolicy = %T, searchPolicy = %T", f.actionPolicy, f.createPolicy, f.searchPolicy) 975 var features = (&fs.Features{ 976 CaseInsensitive: true, 977 DuplicateFiles: false, 978 ReadMimeType: true, 979 WriteMimeType: true, 980 CanHaveEmptyDirectories: true, 981 BucketBased: true, 982 SetTier: true, 983 GetTier: true, 984 ReadMetadata: true, 985 WriteMetadata: true, 986 UserMetadata: true, 987 ReadDirMetadata: true, 988 WriteDirMetadata: true, 989 WriteDirSetModTime: true, 990 UserDirMetadata: true, 991 DirModTimeUpdatesOnWrite: true, 992 PartialUploads: true, 993 }).Fill(ctx, f) 994 canMove, slowHash := true, false 995 for _, f := range upstreams { 996 features = features.Mask(ctx, f) // Mask all upstream fs 997 if !operations.CanServerSideMove(f) { 998 canMove = false 999 } 1000 slowHash = slowHash || f.Features().SlowHash 1001 } 1002 // We can move if all remotes support Move or Copy 1003 if canMove { 1004 features.Move = f.Move 1005 } 1006 1007 // If any of upstreams are SlowHash, propagate it 1008 features.SlowHash = slowHash 1009 1010 // Enable ListR when upstreams either support ListR or is local 1011 // But not when all upstreams are local 1012 if features.ListR == nil { 1013 for _, u := range upstreams { 1014 if u.Features().ListR != nil { 1015 features.ListR = f.ListR 1016 } else if !u.Features().IsLocal { 1017 features.ListR = nil 1018 break 1019 } 1020 } 1021 } 1022 1023 // show that we wrap other backends 1024 features.Overlay = true 1025 1026 f.features = features 1027 1028 // Get common intersection of hashes 1029 hashSet := f.upstreams[0].Hashes() 1030 for _, u := range f.upstreams[1:] { 1031 hashSet = hashSet.Overlap(u.Hashes()) 1032 } 1033 f.hashSet = hashSet 1034 1035 return f, fserr 1036 } 1037 1038 func parentDir(absPath string) string { 1039 parent := path.Dir(strings.TrimRight(filepath.ToSlash(absPath), "/")) 1040 if parent == "." { 1041 parent = "" 1042 } 1043 return parent 1044 } 1045 1046 func multithread(num int, fn func(int)) { 1047 var wg sync.WaitGroup 1048 for i := 0; i < num; i++ { 1049 wg.Add(1) 1050 i := i 1051 go func() { 1052 defer wg.Done() 1053 fn(i) 1054 }() 1055 } 1056 wg.Wait() 1057 } 1058 1059 // Check the interfaces are satisfied 1060 var ( 1061 _ fs.Fs = (*Fs)(nil) 1062 _ fs.Purger = (*Fs)(nil) 1063 _ fs.PutStreamer = (*Fs)(nil) 1064 _ fs.Copier = (*Fs)(nil) 1065 _ fs.Mover = (*Fs)(nil) 1066 _ fs.DirMover = (*Fs)(nil) 1067 _ fs.DirSetModTimer = (*Fs)(nil) 1068 _ fs.MkdirMetadataer = (*Fs)(nil) 1069 _ fs.DirCacheFlusher = (*Fs)(nil) 1070 _ fs.ChangeNotifier = (*Fs)(nil) 1071 _ fs.Abouter = (*Fs)(nil) 1072 _ fs.ListRer = (*Fs)(nil) 1073 _ fs.Shutdowner = (*Fs)(nil) 1074 _ fs.CleanUpper = (*Fs)(nil) 1075 )