github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/backend/combine/combine.go (about)

     1  // Package combine implements a backend to combine multiple remotes in a directory tree
     2  package combine
     3  
     4  /*
     5     Have API to add/remove branches in the combine
     6  */
     7  
     8  import (
     9  	"context"
    10  	"errors"
    11  	"fmt"
    12  	"io"
    13  	"path"
    14  	"strings"
    15  	"sync"
    16  	"time"
    17  
    18  	"github.com/rclone/rclone/fs"
    19  	"github.com/rclone/rclone/fs/cache"
    20  	"github.com/rclone/rclone/fs/config/configmap"
    21  	"github.com/rclone/rclone/fs/config/configstruct"
    22  	"github.com/rclone/rclone/fs/hash"
    23  	"github.com/rclone/rclone/fs/operations"
    24  	"github.com/rclone/rclone/fs/walk"
    25  	"golang.org/x/sync/errgroup"
    26  )
    27  
    28  // Register with Fs
    29  func init() {
    30  	fsi := &fs.RegInfo{
    31  		Name:        "combine",
    32  		Description: "Combine several remotes into one",
    33  		NewFs:       NewFs,
    34  		MetadataInfo: &fs.MetadataInfo{
    35  			Help: `Any metadata supported by the underlying remote is read and written.`,
    36  		},
    37  		Options: []fs.Option{{
    38  			Name: "upstreams",
    39  			Help: `Upstreams for combining
    40  
    41  These should be in the form
    42  
    43      dir=remote:path dir2=remote2:path
    44  
    45  Where before the = is specified the root directory and after is the remote to
    46  put there.
    47  
    48  Embedded spaces can be added using quotes
    49  
    50      "dir=remote:path with space" "dir2=remote2:path with space"
    51  
    52  `,
    53  			Required: true,
    54  			Default:  fs.SpaceSepList(nil),
    55  		}},
    56  	}
    57  	fs.Register(fsi)
    58  }
    59  
    60  // Options defines the configuration for this backend
    61  type Options struct {
    62  	Upstreams fs.SpaceSepList `config:"upstreams"`
    63  }
    64  
    65  // Fs represents a combine of upstreams
    66  type Fs struct {
    67  	name      string               // name of this remote
    68  	features  *fs.Features         // optional features
    69  	opt       Options              // options for this Fs
    70  	root      string               // the path we are working on
    71  	hashSet   hash.Set             // common hashes
    72  	when      time.Time            // directory times
    73  	upstreams map[string]*upstream // map of upstreams
    74  }
    75  
    76  // adjustment stores the info to add a prefix to a path or chop characters off
    77  type adjustment struct {
    78  	root            string
    79  	rootSlash       string
    80  	mountpoint      string
    81  	mountpointSlash string
    82  }
    83  
    84  // newAdjustment makes a new path adjustment adjusting between mountpoint and root
    85  //
    86  // mountpoint is the point the upstream is mounted and root is the combine root
    87  func newAdjustment(root, mountpoint string) (a adjustment) {
    88  	return adjustment{
    89  		root:            root,
    90  		rootSlash:       root + "/",
    91  		mountpoint:      mountpoint,
    92  		mountpointSlash: mountpoint + "/",
    93  	}
    94  }
    95  
    96  var errNotUnderRoot = errors.New("file not under root")
    97  
    98  // do makes the adjustment on s, mapping an upstream path into a combine path
    99  func (a *adjustment) do(s string) (string, error) {
   100  	absPath := join(a.mountpoint, s)
   101  	if a.root == "" {
   102  		return absPath, nil
   103  	}
   104  	if absPath == a.root {
   105  		return "", nil
   106  	}
   107  	if !strings.HasPrefix(absPath, a.rootSlash) {
   108  		return "", errNotUnderRoot
   109  	}
   110  	return absPath[len(a.rootSlash):], nil
   111  }
   112  
   113  // undo makes the adjustment on s, mapping a combine path into an upstream path
   114  func (a *adjustment) undo(s string) (string, error) {
   115  	absPath := join(a.root, s)
   116  	if absPath == a.mountpoint {
   117  		return "", nil
   118  	}
   119  	if !strings.HasPrefix(absPath, a.mountpointSlash) {
   120  		return "", errNotUnderRoot
   121  	}
   122  	return absPath[len(a.mountpointSlash):], nil
   123  }
   124  
   125  // upstream represents an upstream Fs
   126  type upstream struct {
   127  	f              fs.Fs
   128  	parent         *Fs
   129  	dir            string     // directory the upstream is mounted
   130  	pathAdjustment adjustment // how to fiddle with the path
   131  }
   132  
   133  // Create an upstream from the directory it is mounted on and the remote
   134  func (f *Fs) newUpstream(ctx context.Context, dir, remote string) (*upstream, error) {
   135  	uFs, err := cache.Get(ctx, remote)
   136  	if err == fs.ErrorIsFile {
   137  		return nil, fmt.Errorf("can't combine files yet, only directories %q: %w", remote, err)
   138  	}
   139  	if err != nil {
   140  		return nil, fmt.Errorf("failed to create upstream %q: %w", remote, err)
   141  	}
   142  	u := &upstream{
   143  		f:              uFs,
   144  		parent:         f,
   145  		dir:            dir,
   146  		pathAdjustment: newAdjustment(f.root, dir),
   147  	}
   148  	cache.PinUntilFinalized(u.f, u)
   149  	return u, nil
   150  }
   151  
   152  // NewFs constructs an Fs from the path.
   153  //
   154  // The returned Fs is the actual Fs, referenced by remote in the config
   155  func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs.Fs, err error) {
   156  	// defer log.Trace(nil, "name=%q, root=%q, m=%v", name, root, m)("f=%+v, err=%v", &outFs, &err)
   157  	// Parse config into Options struct
   158  	opt := new(Options)
   159  	err = configstruct.Set(m, opt)
   160  	if err != nil {
   161  		return nil, err
   162  	}
   163  	// Backward compatible to old config
   164  	if len(opt.Upstreams) == 0 {
   165  		return nil, errors.New("combine can't point to an empty upstream - check the value of the upstreams setting")
   166  	}
   167  	for _, u := range opt.Upstreams {
   168  		if strings.HasPrefix(u, name+":") {
   169  			return nil, errors.New("can't point combine remote at itself - check the value of the upstreams setting")
   170  		}
   171  	}
   172  	isDir := false
   173  	for strings.HasSuffix(root, "/") {
   174  		root = root[:len(root)-1]
   175  		isDir = true
   176  	}
   177  
   178  	f := &Fs{
   179  		name:      name,
   180  		root:      root,
   181  		opt:       *opt,
   182  		upstreams: make(map[string]*upstream, len(opt.Upstreams)),
   183  		when:      time.Now(),
   184  	}
   185  
   186  	g, gCtx := errgroup.WithContext(ctx)
   187  	var mu sync.Mutex
   188  	for _, upstream := range opt.Upstreams {
   189  		upstream := upstream
   190  		g.Go(func() (err error) {
   191  			equal := strings.IndexRune(upstream, '=')
   192  			if equal < 0 {
   193  				return fmt.Errorf("no \"=\" in upstream definition %q", upstream)
   194  			}
   195  			dir, remote := upstream[:equal], upstream[equal+1:]
   196  			if dir == "" {
   197  				return fmt.Errorf("empty dir in upstream definition %q", upstream)
   198  			}
   199  			if remote == "" {
   200  				return fmt.Errorf("empty remote in upstream definition %q", upstream)
   201  			}
   202  			if strings.ContainsRune(dir, '/') {
   203  				return fmt.Errorf("dirs can't contain / (yet): %q", dir)
   204  			}
   205  			u, err := f.newUpstream(gCtx, dir, remote)
   206  			if err != nil {
   207  				return err
   208  			}
   209  			mu.Lock()
   210  			if _, found := f.upstreams[dir]; found {
   211  				err = fmt.Errorf("duplicate directory name %q", dir)
   212  			} else {
   213  				f.upstreams[dir] = u
   214  			}
   215  			mu.Unlock()
   216  			return err
   217  		})
   218  	}
   219  	err = g.Wait()
   220  	if err != nil {
   221  		return nil, err
   222  	}
   223  	// check features
   224  	var features = (&fs.Features{
   225  		CaseInsensitive:          true,
   226  		DuplicateFiles:           false,
   227  		ReadMimeType:             true,
   228  		WriteMimeType:            true,
   229  		CanHaveEmptyDirectories:  true,
   230  		BucketBased:              true,
   231  		SetTier:                  true,
   232  		GetTier:                  true,
   233  		ReadMetadata:             true,
   234  		WriteMetadata:            true,
   235  		UserMetadata:             true,
   236  		ReadDirMetadata:          true,
   237  		WriteDirMetadata:         true,
   238  		WriteDirSetModTime:       true,
   239  		UserDirMetadata:          true,
   240  		DirModTimeUpdatesOnWrite: true,
   241  		PartialUploads:           true,
   242  	}).Fill(ctx, f)
   243  	canMove := true
   244  	for _, u := range f.upstreams {
   245  		features = features.Mask(ctx, u.f) // Mask all upstream fs
   246  		if !operations.CanServerSideMove(u.f) {
   247  			canMove = false
   248  		}
   249  	}
   250  	// We can move if all remotes support Move or Copy
   251  	if canMove {
   252  		features.Move = f.Move
   253  	}
   254  
   255  	// Enable ListR when upstreams either support ListR or is local
   256  	// But not when all upstreams are local
   257  	if features.ListR == nil {
   258  		for _, u := range f.upstreams {
   259  			if u.f.Features().ListR != nil {
   260  				features.ListR = f.ListR
   261  			} else if !u.f.Features().IsLocal {
   262  				features.ListR = nil
   263  				break
   264  			}
   265  		}
   266  	}
   267  
   268  	// Enable Purge when any upstreams support it
   269  	if features.Purge == nil {
   270  		for _, u := range f.upstreams {
   271  			if u.f.Features().Purge != nil {
   272  				features.Purge = f.Purge
   273  				break
   274  			}
   275  		}
   276  	}
   277  
   278  	// Enable Shutdown when any upstreams support it
   279  	if features.Shutdown == nil {
   280  		for _, u := range f.upstreams {
   281  			if u.f.Features().Shutdown != nil {
   282  				features.Shutdown = f.Shutdown
   283  				break
   284  			}
   285  		}
   286  	}
   287  
   288  	// Enable DirCacheFlush when any upstreams support it
   289  	if features.DirCacheFlush == nil {
   290  		for _, u := range f.upstreams {
   291  			if u.f.Features().DirCacheFlush != nil {
   292  				features.DirCacheFlush = f.DirCacheFlush
   293  				break
   294  			}
   295  		}
   296  	}
   297  
   298  	// Enable CleanUp when any upstreams support it
   299  	if features.CleanUp == nil {
   300  		for _, u := range f.upstreams {
   301  			if u.f.Features().CleanUp != nil {
   302  				features.CleanUp = f.CleanUp
   303  				break
   304  			}
   305  		}
   306  	}
   307  
   308  	// Enable ChangeNotify when any upstreams support it
   309  	if features.ChangeNotify == nil {
   310  		for _, u := range f.upstreams {
   311  			if u.f.Features().ChangeNotify != nil {
   312  				features.ChangeNotify = f.ChangeNotify
   313  				break
   314  			}
   315  		}
   316  	}
   317  
   318  	// show that we wrap other backends
   319  	features.Overlay = true
   320  
   321  	f.features = features
   322  
   323  	// Get common intersection of hashes
   324  	var hashSet hash.Set
   325  	var first = true
   326  	for _, u := range f.upstreams {
   327  		if first {
   328  			hashSet = u.f.Hashes()
   329  			first = false
   330  		} else {
   331  			hashSet = hashSet.Overlap(u.f.Hashes())
   332  		}
   333  	}
   334  	f.hashSet = hashSet
   335  
   336  	// Check to see if the root is actually a file
   337  	if f.root != "" && !isDir {
   338  		_, err := f.NewObject(ctx, "")
   339  		if err != nil {
   340  			if err == fs.ErrorObjectNotFound || err == fs.ErrorNotAFile || err == fs.ErrorIsDir {
   341  				// File doesn't exist or is a directory so return old f
   342  				return f, nil
   343  			}
   344  			return nil, err
   345  		}
   346  
   347  		// Check to see if the root path is actually an existing file
   348  		f.root = path.Dir(f.root)
   349  		if f.root == "." {
   350  			f.root = ""
   351  		}
   352  		// Adjust path adjustment to remove leaf
   353  		for _, u := range f.upstreams {
   354  			u.pathAdjustment = newAdjustment(f.root, u.dir)
   355  		}
   356  		return f, fs.ErrorIsFile
   357  	}
   358  	return f, nil
   359  }
   360  
   361  // Run a function over all the upstreams in parallel
   362  func (f *Fs) multithread(ctx context.Context, fn func(context.Context, *upstream) error) error {
   363  	g, gCtx := errgroup.WithContext(ctx)
   364  	for _, u := range f.upstreams {
   365  		u := u
   366  		g.Go(func() (err error) {
   367  			return fn(gCtx, u)
   368  		})
   369  	}
   370  	return g.Wait()
   371  }
   372  
   373  // join the elements together but unlike path.Join return empty string
   374  func join(elem ...string) string {
   375  	result := path.Join(elem...)
   376  	if result == "." {
   377  		return ""
   378  	}
   379  	if len(result) > 0 && result[0] == '/' {
   380  		result = result[1:]
   381  	}
   382  	return result
   383  }
   384  
   385  // find the upstream for the remote passed in, returning the upstream and the adjusted path
   386  func (f *Fs) findUpstream(remote string) (u *upstream, uRemote string, err error) {
   387  	// defer log.Trace(remote, "")("f=%v, uRemote=%q, err=%v", &u, &uRemote, &err)
   388  	for _, u := range f.upstreams {
   389  		uRemote, err = u.pathAdjustment.undo(remote)
   390  		if err == nil {
   391  			return u, uRemote, nil
   392  		}
   393  	}
   394  	return nil, "", fmt.Errorf("combine for remote %q: %w", remote, fs.ErrorDirNotFound)
   395  }
   396  
   397  // Name of the remote (as passed into NewFs)
   398  func (f *Fs) Name() string {
   399  	return f.name
   400  }
   401  
   402  // Root of the remote (as passed into NewFs)
   403  func (f *Fs) Root() string {
   404  	return f.root
   405  }
   406  
   407  // String converts this Fs to a string
   408  func (f *Fs) String() string {
   409  	return fmt.Sprintf("combine root '%s'", f.root)
   410  }
   411  
   412  // Features returns the optional features of this Fs
   413  func (f *Fs) Features() *fs.Features {
   414  	return f.features
   415  }
   416  
   417  // Rmdir removes the root directory of the Fs object
   418  func (f *Fs) Rmdir(ctx context.Context, dir string) error {
   419  	// The root always exists
   420  	if f.root == "" && dir == "" {
   421  		return nil
   422  	}
   423  	u, uRemote, err := f.findUpstream(dir)
   424  	if err != nil {
   425  		return err
   426  	}
   427  	return u.f.Rmdir(ctx, uRemote)
   428  }
   429  
   430  // Hashes returns hash.HashNone to indicate remote hashing is unavailable
   431  func (f *Fs) Hashes() hash.Set {
   432  	return f.hashSet
   433  }
   434  
   435  // Mkdir makes the root directory of the Fs object
   436  func (f *Fs) Mkdir(ctx context.Context, dir string) error {
   437  	// The root always exists
   438  	if f.root == "" && dir == "" {
   439  		return nil
   440  	}
   441  	u, uRemote, err := f.findUpstream(dir)
   442  	if err != nil {
   443  		return err
   444  	}
   445  	return u.f.Mkdir(ctx, uRemote)
   446  }
   447  
   448  // MkdirMetadata makes the root directory of the Fs object
   449  func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) {
   450  	u, uRemote, err := f.findUpstream(dir)
   451  	if err != nil {
   452  		return nil, err
   453  	}
   454  	do := u.f.Features().MkdirMetadata
   455  	if do == nil {
   456  		return nil, fs.ErrorNotImplemented
   457  	}
   458  	newDir, err := do(ctx, uRemote, metadata)
   459  	if err != nil {
   460  		return nil, err
   461  	}
   462  	entries := fs.DirEntries{newDir}
   463  	entries, err = u.wrapEntries(ctx, entries)
   464  	if err != nil {
   465  		return nil, err
   466  	}
   467  	newDir, ok := entries[0].(fs.Directory)
   468  	if !ok {
   469  		return nil, fmt.Errorf("internal error: expecting %T to be fs.Directory", entries[0])
   470  	}
   471  	return newDir, nil
   472  }
   473  
   474  // purge the upstream or fallback to a slow way
   475  func (u *upstream) purge(ctx context.Context, dir string) (err error) {
   476  	if do := u.f.Features().Purge; do != nil {
   477  		err = do(ctx, dir)
   478  	} else {
   479  		err = operations.Purge(ctx, u.f, dir)
   480  	}
   481  	return err
   482  }
   483  
   484  // Purge all files in the directory
   485  //
   486  // Implement this if you have a way of deleting all the files
   487  // quicker than just running Remove() on the result of List()
   488  //
   489  // Return an error if it doesn't exist
   490  func (f *Fs) Purge(ctx context.Context, dir string) error {
   491  	if f.root == "" && dir == "" {
   492  		return f.multithread(ctx, func(ctx context.Context, u *upstream) error {
   493  			return u.purge(ctx, "")
   494  		})
   495  	}
   496  	u, uRemote, err := f.findUpstream(dir)
   497  	if err != nil {
   498  		return err
   499  	}
   500  	return u.purge(ctx, uRemote)
   501  }
   502  
   503  // Copy src to this remote using server-side copy operations.
   504  //
   505  // This is stored with the remote path given.
   506  //
   507  // It returns the destination Object and a possible error.
   508  //
   509  // Will only be called if src.Fs().Name() == f.Name()
   510  //
   511  // If it isn't possible then return fs.ErrorCantCopy
   512  func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
   513  	srcObj, ok := src.(*Object)
   514  	if !ok {
   515  		fs.Debugf(src, "Can't copy - not same remote type")
   516  		return nil, fs.ErrorCantCopy
   517  	}
   518  
   519  	dstU, dstRemote, err := f.findUpstream(remote)
   520  	if err != nil {
   521  		return nil, err
   522  	}
   523  
   524  	do := dstU.f.Features().Copy
   525  	if do == nil {
   526  		return nil, fs.ErrorCantCopy
   527  	}
   528  
   529  	o, err := do(ctx, srcObj.Object, dstRemote)
   530  	if err != nil {
   531  		return nil, err
   532  	}
   533  
   534  	return dstU.newObject(o), nil
   535  }
   536  
   537  // Move src to this remote using server-side move operations.
   538  //
   539  // This is stored with the remote path given.
   540  //
   541  // It returns the destination Object and a possible error.
   542  //
   543  // Will only be called if src.Fs().Name() == f.Name()
   544  //
   545  // If it isn't possible then return fs.ErrorCantMove
   546  func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
   547  	srcObj, ok := src.(*Object)
   548  	if !ok {
   549  		fs.Debugf(src, "Can't move - not same remote type")
   550  		return nil, fs.ErrorCantMove
   551  	}
   552  
   553  	dstU, dstRemote, err := f.findUpstream(remote)
   554  	if err != nil {
   555  		return nil, err
   556  	}
   557  
   558  	do := dstU.f.Features().Move
   559  	useCopy := false
   560  	if do == nil {
   561  		do = dstU.f.Features().Copy
   562  		if do == nil {
   563  			return nil, fs.ErrorCantMove
   564  		}
   565  		useCopy = true
   566  	}
   567  
   568  	o, err := do(ctx, srcObj.Object, dstRemote)
   569  	if err != nil {
   570  		return nil, err
   571  	}
   572  
   573  	// If did Copy then remove the source object
   574  	if useCopy {
   575  		err = srcObj.Remove(ctx)
   576  		if err != nil {
   577  			return nil, err
   578  		}
   579  	}
   580  
   581  	return dstU.newObject(o), nil
   582  }
   583  
   584  // DirMove moves src, srcRemote to this remote at dstRemote
   585  // using server-side move operations.
   586  //
   587  // Will only be called if src.Fs().Name() == f.Name()
   588  //
   589  // If it isn't possible then return fs.ErrorCantDirMove
   590  //
   591  // If destination exists then return fs.ErrorDirExists
   592  func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) (err error) {
   593  	// defer log.Trace(f, "src=%v, srcRemote=%q, dstRemote=%q", src, srcRemote, dstRemote)("err=%v", &err)
   594  	srcFs, ok := src.(*Fs)
   595  	if !ok {
   596  		fs.Debugf(src, "Can't move directory - not same remote type")
   597  		return fs.ErrorCantDirMove
   598  	}
   599  
   600  	dstU, dstURemote, err := f.findUpstream(dstRemote)
   601  	if err != nil {
   602  		return err
   603  	}
   604  
   605  	srcU, srcURemote, err := srcFs.findUpstream(srcRemote)
   606  	if err != nil {
   607  		return err
   608  	}
   609  
   610  	do := dstU.f.Features().DirMove
   611  	if do == nil {
   612  		return fs.ErrorCantDirMove
   613  	}
   614  
   615  	fs.Logf(dstU.f, "srcU.f=%v, srcURemote=%q, dstURemote=%q", srcU.f, srcURemote, dstURemote)
   616  	return do(ctx, srcU.f, srcURemote, dstURemote)
   617  }
   618  
   619  // ChangeNotify calls the passed function with a path
   620  // that has had changes. If the implementation
   621  // uses polling, it should adhere to the given interval.
   622  // At least one value will be written to the channel,
   623  // specifying the initial value and updated values might
   624  // follow. A 0 Duration should pause the polling.
   625  // The ChangeNotify implementation must empty the channel
   626  // regularly. When the channel gets closed, the implementation
   627  // should stop polling and release resources.
   628  func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), ch <-chan time.Duration) {
   629  	var uChans []chan time.Duration
   630  
   631  	for _, u := range f.upstreams {
   632  		u := u
   633  		if do := u.f.Features().ChangeNotify; do != nil {
   634  			ch := make(chan time.Duration)
   635  			uChans = append(uChans, ch)
   636  			wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
   637  				newPath, err := u.pathAdjustment.do(path)
   638  				if err != nil {
   639  					fs.Logf(f, "ChangeNotify: unable to process %q: %s", path, err)
   640  					return
   641  				}
   642  				fs.Debugf(f, "ChangeNotify: path %q entryType %d", newPath, entryType)
   643  				notifyFunc(newPath, entryType)
   644  			}
   645  			do(ctx, wrappedNotifyFunc, ch)
   646  		}
   647  	}
   648  
   649  	go func() {
   650  		for i := range ch {
   651  			for _, c := range uChans {
   652  				c <- i
   653  			}
   654  		}
   655  		for _, c := range uChans {
   656  			close(c)
   657  		}
   658  	}()
   659  }
   660  
   661  // DirCacheFlush resets the directory cache - used in testing
   662  // as an optional interface
   663  func (f *Fs) DirCacheFlush() {
   664  	ctx := context.Background()
   665  	_ = f.multithread(ctx, func(ctx context.Context, u *upstream) error {
   666  		if do := u.f.Features().DirCacheFlush; do != nil {
   667  			do()
   668  		}
   669  		return nil
   670  	})
   671  }
   672  
   673  func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bool, options ...fs.OpenOption) (fs.Object, error) {
   674  	srcPath := src.Remote()
   675  	u, uRemote, err := f.findUpstream(srcPath)
   676  	if err != nil {
   677  		return nil, err
   678  	}
   679  	uSrc := fs.NewOverrideRemote(src, uRemote)
   680  	var o fs.Object
   681  	if stream {
   682  		o, err = u.f.Features().PutStream(ctx, in, uSrc, options...)
   683  	} else {
   684  		o, err = u.f.Put(ctx, in, uSrc, options...)
   685  	}
   686  	if err != nil {
   687  		return nil, err
   688  	}
   689  	return u.newObject(o), nil
   690  }
   691  
   692  // Put in to the remote path with the modTime given of the given size
   693  //
   694  // May create the object even if it returns an error - if so
   695  // will return the object and the error, otherwise will return
   696  // nil and the error
   697  func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
   698  	o, err := f.NewObject(ctx, src.Remote())
   699  	switch err {
   700  	case nil:
   701  		return o, o.Update(ctx, in, src, options...)
   702  	case fs.ErrorObjectNotFound:
   703  		return f.put(ctx, in, src, false, options...)
   704  	default:
   705  		return nil, err
   706  	}
   707  }
   708  
   709  // PutStream uploads to the remote path with the modTime given of indeterminate size
   710  //
   711  // May create the object even if it returns an error - if so
   712  // will return the object and the error, otherwise will return
   713  // nil and the error
   714  func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
   715  	o, err := f.NewObject(ctx, src.Remote())
   716  	switch err {
   717  	case nil:
   718  		return o, o.Update(ctx, in, src, options...)
   719  	case fs.ErrorObjectNotFound:
   720  		return f.put(ctx, in, src, true, options...)
   721  	default:
   722  		return nil, err
   723  	}
   724  }
   725  
   726  // About gets quota information from the Fs
   727  func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
   728  	usage := &fs.Usage{
   729  		Total:   new(int64),
   730  		Used:    new(int64),
   731  		Trashed: new(int64),
   732  		Other:   new(int64),
   733  		Free:    new(int64),
   734  		Objects: new(int64),
   735  	}
   736  	for _, u := range f.upstreams {
   737  		doAbout := u.f.Features().About
   738  		if doAbout == nil {
   739  			continue
   740  		}
   741  		usg, err := doAbout(ctx)
   742  		if errors.Is(err, fs.ErrorDirNotFound) {
   743  			continue
   744  		}
   745  		if err != nil {
   746  			return nil, err
   747  		}
   748  		if usg.Total != nil && usage.Total != nil {
   749  			*usage.Total += *usg.Total
   750  		} else {
   751  			usage.Total = nil
   752  		}
   753  		if usg.Used != nil && usage.Used != nil {
   754  			*usage.Used += *usg.Used
   755  		} else {
   756  			usage.Used = nil
   757  		}
   758  		if usg.Trashed != nil && usage.Trashed != nil {
   759  			*usage.Trashed += *usg.Trashed
   760  		} else {
   761  			usage.Trashed = nil
   762  		}
   763  		if usg.Other != nil && usage.Other != nil {
   764  			*usage.Other += *usg.Other
   765  		} else {
   766  			usage.Other = nil
   767  		}
   768  		if usg.Free != nil && usage.Free != nil {
   769  			*usage.Free += *usg.Free
   770  		} else {
   771  			usage.Free = nil
   772  		}
   773  		if usg.Objects != nil && usage.Objects != nil {
   774  			*usage.Objects += *usg.Objects
   775  		} else {
   776  			usage.Objects = nil
   777  		}
   778  	}
   779  	return usage, nil
   780  }
   781  
   782  // Wraps entries for this upstream
   783  func (u *upstream) wrapEntries(ctx context.Context, entries fs.DirEntries) (fs.DirEntries, error) {
   784  	for i, entry := range entries {
   785  		switch x := entry.(type) {
   786  		case fs.Object:
   787  			entries[i] = u.newObject(x)
   788  		case fs.Directory:
   789  			newPath, err := u.pathAdjustment.do(x.Remote())
   790  			if err != nil {
   791  				return nil, err
   792  			}
   793  			newDir := fs.NewDirWrapper(newPath, x)
   794  			entries[i] = newDir
   795  		default:
   796  			return nil, fmt.Errorf("unknown entry type %T", entry)
   797  		}
   798  	}
   799  	return entries, nil
   800  }
   801  
   802  // List the objects and directories in dir into entries.  The
   803  // entries can be returned in any order but should be for a
   804  // complete directory.
   805  //
   806  // dir should be "" to list the root, and should not have
   807  // trailing slashes.
   808  //
   809  // This should return ErrDirNotFound if the directory isn't
   810  // found.
   811  func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
   812  	// defer log.Trace(f, "dir=%q", dir)("entries = %v, err=%v", &entries, &err)
   813  	if f.root == "" && dir == "" {
   814  		entries = make(fs.DirEntries, 0, len(f.upstreams))
   815  		for combineDir := range f.upstreams {
   816  			d := fs.NewLimitedDirWrapper(combineDir, fs.NewDir(combineDir, f.when))
   817  			entries = append(entries, d)
   818  		}
   819  		return entries, nil
   820  	}
   821  	u, uRemote, err := f.findUpstream(dir)
   822  	if err != nil {
   823  		return nil, err
   824  	}
   825  	entries, err = u.f.List(ctx, uRemote)
   826  	if err != nil {
   827  		return nil, err
   828  	}
   829  	return u.wrapEntries(ctx, entries)
   830  }
   831  
   832  // ListR lists the objects and directories of the Fs starting
   833  // from dir recursively into out.
   834  //
   835  // dir should be "" to start from the root, and should not
   836  // have trailing slashes.
   837  //
   838  // This should return ErrDirNotFound if the directory isn't
   839  // found.
   840  //
   841  // It should call callback for each tranche of entries read.
   842  // These need not be returned in any particular order.  If
   843  // callback returns an error then the listing will stop
   844  // immediately.
   845  //
   846  // Don't implement this unless you have a more efficient way
   847  // of listing recursively that doing a directory traversal.
   848  func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
   849  	// defer log.Trace(f, "dir=%q, callback=%v", dir, callback)("err=%v", &err)
   850  	if f.root == "" && dir == "" {
   851  		rootEntries, err := f.List(ctx, "")
   852  		if err != nil {
   853  			return err
   854  		}
   855  		err = callback(rootEntries)
   856  		if err != nil {
   857  			return err
   858  		}
   859  		var mu sync.Mutex
   860  		syncCallback := func(entries fs.DirEntries) error {
   861  			mu.Lock()
   862  			defer mu.Unlock()
   863  			return callback(entries)
   864  		}
   865  		err = f.multithread(ctx, func(ctx context.Context, u *upstream) error {
   866  			return f.ListR(ctx, u.dir, syncCallback)
   867  		})
   868  		if err != nil {
   869  			return err
   870  		}
   871  		return nil
   872  	}
   873  	u, uRemote, err := f.findUpstream(dir)
   874  	if err != nil {
   875  		return err
   876  	}
   877  	wrapCallback := func(entries fs.DirEntries) error {
   878  		entries, err := u.wrapEntries(ctx, entries)
   879  		if err != nil {
   880  			return err
   881  		}
   882  		return callback(entries)
   883  	}
   884  	if do := u.f.Features().ListR; do != nil {
   885  		err = do(ctx, uRemote, wrapCallback)
   886  	} else {
   887  		err = walk.ListR(ctx, u.f, uRemote, true, -1, walk.ListAll, wrapCallback)
   888  	}
   889  	if err == fs.ErrorDirNotFound {
   890  		err = nil
   891  	}
   892  	return err
   893  }
   894  
   895  // NewObject creates a new remote combine file object
   896  func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
   897  	u, uRemote, err := f.findUpstream(remote)
   898  	if err != nil {
   899  		return nil, err
   900  	}
   901  	if uRemote == "" || strings.HasSuffix(uRemote, "/") {
   902  		return nil, fs.ErrorIsDir
   903  	}
   904  	o, err := u.f.NewObject(ctx, uRemote)
   905  	if err != nil {
   906  		return nil, err
   907  	}
   908  	return u.newObject(o), nil
   909  }
   910  
   911  // Precision is the greatest Precision of all upstreams
   912  func (f *Fs) Precision() time.Duration {
   913  	var greatestPrecision time.Duration
   914  	for _, u := range f.upstreams {
   915  		uPrecision := u.f.Precision()
   916  		if uPrecision > greatestPrecision {
   917  			greatestPrecision = uPrecision
   918  		}
   919  	}
   920  	return greatestPrecision
   921  }
   922  
   923  // Shutdown the backend, closing any background tasks and any
   924  // cached connections.
   925  func (f *Fs) Shutdown(ctx context.Context) error {
   926  	return f.multithread(ctx, func(ctx context.Context, u *upstream) error {
   927  		if do := u.f.Features().Shutdown; do != nil {
   928  			return do(ctx)
   929  		}
   930  		return nil
   931  	})
   932  }
   933  
   934  // PublicLink generates a public link to the remote path (usually readable by anyone)
   935  func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
   936  	u, uRemote, err := f.findUpstream(remote)
   937  	if err != nil {
   938  		return "", err
   939  	}
   940  	do := u.f.Features().PublicLink
   941  	if do == nil {
   942  		return "", fs.ErrorNotImplemented
   943  	}
   944  	return do(ctx, uRemote, expire, unlink)
   945  }
   946  
   947  // PutUnchecked in to the remote path with the modTime given of the given size
   948  //
   949  // May create the object even if it returns an error - if so
   950  // will return the object and the error, otherwise will return
   951  // nil and the error
   952  //
   953  // May create duplicates or return errors if src already
   954  // exists.
   955  func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
   956  	srcPath := src.Remote()
   957  	u, uRemote, err := f.findUpstream(srcPath)
   958  	if err != nil {
   959  		return nil, err
   960  	}
   961  	do := u.f.Features().PutUnchecked
   962  	if do == nil {
   963  		return nil, fs.ErrorNotImplemented
   964  	}
   965  	uSrc := fs.NewOverrideRemote(src, uRemote)
   966  	return do(ctx, in, uSrc, options...)
   967  }
   968  
   969  // MergeDirs merges the contents of all the directories passed
   970  // in into the first one and rmdirs the other directories.
   971  func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
   972  	if len(dirs) == 0 {
   973  		return nil
   974  	}
   975  	var (
   976  		u     *upstream
   977  		uDirs []fs.Directory
   978  	)
   979  	for _, dir := range dirs {
   980  		uNew, uDir, err := f.findUpstream(dir.Remote())
   981  		if err != nil {
   982  			return err
   983  		}
   984  		if u == nil {
   985  			u = uNew
   986  		} else if u != uNew {
   987  			return fmt.Errorf("can't merge directories from different upstreams")
   988  		}
   989  		uDirs = append(uDirs, fs.NewOverrideDirectory(dir, uDir))
   990  	}
   991  	do := u.f.Features().MergeDirs
   992  	if do == nil {
   993  		return fs.ErrorNotImplemented
   994  	}
   995  	return do(ctx, uDirs)
   996  }
   997  
   998  // DirSetModTime sets the directory modtime for dir
   999  func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
  1000  	u, uDir, err := f.findUpstream(dir)
  1001  	if err != nil {
  1002  		return err
  1003  	}
  1004  	if uDir == "" {
  1005  		fs.Debugf(dir, "Can't set modtime on upstream root. skipping.")
  1006  		return nil
  1007  	}
  1008  	if do := u.f.Features().DirSetModTime; do != nil {
  1009  		return do(ctx, uDir, modTime)
  1010  	}
  1011  	return fs.ErrorNotImplemented
  1012  }
  1013  
  1014  // CleanUp the trash in the Fs
  1015  //
  1016  // Implement this if you have a way of emptying the trash or
  1017  // otherwise cleaning up old versions of files.
  1018  func (f *Fs) CleanUp(ctx context.Context) error {
  1019  	return f.multithread(ctx, func(ctx context.Context, u *upstream) error {
  1020  		if do := u.f.Features().CleanUp; do != nil {
  1021  			return do(ctx)
  1022  		}
  1023  		return nil
  1024  	})
  1025  }
  1026  
  1027  // OpenWriterAt opens with a handle for random access writes
  1028  //
  1029  // Pass in the remote desired and the size if known.
  1030  //
  1031  // It truncates any existing object
  1032  func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) {
  1033  	u, uRemote, err := f.findUpstream(remote)
  1034  	if err != nil {
  1035  		return nil, err
  1036  	}
  1037  	do := u.f.Features().OpenWriterAt
  1038  	if do == nil {
  1039  		return nil, fs.ErrorNotImplemented
  1040  	}
  1041  	return do(ctx, uRemote, size)
  1042  }
  1043  
  1044  // Object describes a wrapped Object
  1045  //
  1046  // This is a wrapped Object which knows its path prefix
  1047  type Object struct {
  1048  	fs.Object
  1049  	u *upstream
  1050  }
  1051  
  1052  func (u *upstream) newObject(o fs.Object) *Object {
  1053  	return &Object{
  1054  		Object: o,
  1055  		u:      u,
  1056  	}
  1057  }
  1058  
  1059  // Fs returns read only access to the Fs that this object is part of
  1060  func (o *Object) Fs() fs.Info {
  1061  	return o.u.parent
  1062  }
  1063  
  1064  // String returns the remote path
  1065  func (o *Object) String() string {
  1066  	return o.Remote()
  1067  }
  1068  
  1069  // Remote returns the remote path
  1070  func (o *Object) Remote() string {
  1071  	newPath, err := o.u.pathAdjustment.do(o.Object.String())
  1072  	if err != nil {
  1073  		fs.Errorf(o.Object, "Bad object: %v", err)
  1074  		return err.Error()
  1075  	}
  1076  	return newPath
  1077  }
  1078  
  1079  // MimeType returns the content type of the Object if known
  1080  func (o *Object) MimeType(ctx context.Context) (mimeType string) {
  1081  	if do, ok := o.Object.(fs.MimeTyper); ok {
  1082  		mimeType = do.MimeType(ctx)
  1083  	}
  1084  	return mimeType
  1085  }
  1086  
  1087  // UnWrap returns the Object that this Object is wrapping or
  1088  // nil if it isn't wrapping anything
  1089  func (o *Object) UnWrap() fs.Object {
  1090  	return o.Object
  1091  }
  1092  
  1093  // GetTier returns storage tier or class of the Object
  1094  func (o *Object) GetTier() string {
  1095  	do, ok := o.Object.(fs.GetTierer)
  1096  	if !ok {
  1097  		return ""
  1098  	}
  1099  	return do.GetTier()
  1100  }
  1101  
  1102  // ID returns the ID of the Object if known, or "" if not
  1103  func (o *Object) ID() string {
  1104  	do, ok := o.Object.(fs.IDer)
  1105  	if !ok {
  1106  		return ""
  1107  	}
  1108  	return do.ID()
  1109  }
  1110  
  1111  // Metadata returns metadata for an object
  1112  //
  1113  // It should return nil if there is no Metadata
  1114  func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) {
  1115  	do, ok := o.Object.(fs.Metadataer)
  1116  	if !ok {
  1117  		return nil, nil
  1118  	}
  1119  	return do.Metadata(ctx)
  1120  }
  1121  
  1122  // SetMetadata sets metadata for an Object
  1123  //
  1124  // It should return fs.ErrorNotImplemented if it can't set metadata
  1125  func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
  1126  	do, ok := o.Object.(fs.SetMetadataer)
  1127  	if !ok {
  1128  		return fs.ErrorNotImplemented
  1129  	}
  1130  	return do.SetMetadata(ctx, metadata)
  1131  }
  1132  
  1133  // SetTier performs changing storage tier of the Object if
  1134  // multiple storage classes supported
  1135  func (o *Object) SetTier(tier string) error {
  1136  	do, ok := o.Object.(fs.SetTierer)
  1137  	if !ok {
  1138  		return errors.New("underlying remote does not support SetTier")
  1139  	}
  1140  	return do.SetTier(tier)
  1141  }
  1142  
  1143  // Check the interfaces are satisfied
  1144  var (
  1145  	_ fs.Fs              = (*Fs)(nil)
  1146  	_ fs.Purger          = (*Fs)(nil)
  1147  	_ fs.PutStreamer     = (*Fs)(nil)
  1148  	_ fs.Copier          = (*Fs)(nil)
  1149  	_ fs.Mover           = (*Fs)(nil)
  1150  	_ fs.DirMover        = (*Fs)(nil)
  1151  	_ fs.DirCacheFlusher = (*Fs)(nil)
  1152  	_ fs.ChangeNotifier  = (*Fs)(nil)
  1153  	_ fs.Abouter         = (*Fs)(nil)
  1154  	_ fs.ListRer         = (*Fs)(nil)
  1155  	_ fs.Shutdowner      = (*Fs)(nil)
  1156  	_ fs.PublicLinker    = (*Fs)(nil)
  1157  	_ fs.PutUncheckeder  = (*Fs)(nil)
  1158  	_ fs.MergeDirser     = (*Fs)(nil)
  1159  	_ fs.DirSetModTimer  = (*Fs)(nil)
  1160  	_ fs.MkdirMetadataer = (*Fs)(nil)
  1161  	_ fs.CleanUpper      = (*Fs)(nil)
  1162  	_ fs.OpenWriterAter  = (*Fs)(nil)
  1163  	_ fs.FullObject      = (*Object)(nil)
  1164  )