github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/backend/local/local.go (about)

     1  // Package local provides a filesystem interface
     2  package local
     3  
     4  import (
     5  	"bytes"
     6  	"context"
     7  	"errors"
     8  	"fmt"
     9  	"io"
    10  	"os"
    11  	"path"
    12  	"path/filepath"
    13  	"runtime"
    14  	"strings"
    15  	"sync"
    16  	"sync/atomic"
    17  	"time"
    18  	"unicode/utf8"
    19  
    20  	"github.com/rclone/rclone/fs"
    21  	"github.com/rclone/rclone/fs/accounting"
    22  	"github.com/rclone/rclone/fs/config"
    23  	"github.com/rclone/rclone/fs/config/configmap"
    24  	"github.com/rclone/rclone/fs/config/configstruct"
    25  	"github.com/rclone/rclone/fs/filter"
    26  	"github.com/rclone/rclone/fs/fserrors"
    27  	"github.com/rclone/rclone/fs/hash"
    28  	"github.com/rclone/rclone/lib/encoder"
    29  	"github.com/rclone/rclone/lib/file"
    30  	"github.com/rclone/rclone/lib/readers"
    31  	"golang.org/x/text/unicode/norm"
    32  )
    33  
    34  // Constants
    35  const devUnset = 0xdeadbeefcafebabe                                       // a device id meaning it is unset
    36  const linkSuffix = ".rclonelink"                                          // The suffix added to a translated symbolic link
    37  const useReadDir = (runtime.GOOS == "windows" || runtime.GOOS == "plan9") // these OSes read FileInfos directly
    38  
    39  // timeType allows the user to choose what exactly ModTime() returns
    40  type timeType = fs.Enum[timeTypeChoices]
    41  
    42  const (
    43  	mTime timeType = iota
    44  	aTime
    45  	bTime
    46  	cTime
    47  )
    48  
    49  type timeTypeChoices struct{}
    50  
    51  func (timeTypeChoices) Choices() []string {
    52  	return []string{
    53  		mTime: "mtime",
    54  		aTime: "atime",
    55  		bTime: "btime",
    56  		cTime: "ctime",
    57  	}
    58  }
    59  
    60  // Register with Fs
    61  func init() {
    62  	fsi := &fs.RegInfo{
    63  		Name:        "local",
    64  		Description: "Local Disk",
    65  		NewFs:       NewFs,
    66  		CommandHelp: commandHelp,
    67  		MetadataInfo: &fs.MetadataInfo{
    68  			System: systemMetadataInfo,
    69  			Help: `Depending on which OS is in use the local backend may return only some
    70  of the system metadata. Setting system metadata is supported on all
    71  OSes but setting user metadata is only supported on linux, freebsd,
    72  netbsd, macOS and Solaris. It is **not** supported on Windows yet
    73  ([see pkg/attrs#47](https://github.com/pkg/xattr/issues/47)).
    74  
    75  User metadata is stored as extended attributes (which may not be
    76  supported by all file systems) under the "user.*" prefix.
    77  
    78  Metadata is supported on files and directories.
    79  `,
    80  		},
    81  		Options: []fs.Option{{
    82  			Name:     "nounc",
    83  			Help:     "Disable UNC (long path names) conversion on Windows.",
    84  			Default:  false,
    85  			Advanced: runtime.GOOS != "windows",
    86  			Examples: []fs.OptionExample{{
    87  				Value: "true",
    88  				Help:  "Disables long file names.",
    89  			}},
    90  		}, {
    91  			Name:     "copy_links",
    92  			Help:     "Follow symlinks and copy the pointed to item.",
    93  			Default:  false,
    94  			NoPrefix: true,
    95  			ShortOpt: "L",
    96  			Advanced: true,
    97  		}, {
    98  			Name:     "links",
    99  			Help:     "Translate symlinks to/from regular files with a '" + linkSuffix + "' extension.",
   100  			Default:  false,
   101  			NoPrefix: true,
   102  			ShortOpt: "l",
   103  			Advanced: true,
   104  		}, {
   105  			Name: "skip_links",
   106  			Help: `Don't warn about skipped symlinks.
   107  
   108  This flag disables warning messages on skipped symlinks or junction
   109  points, as you explicitly acknowledge that they should be skipped.`,
   110  			Default:  false,
   111  			NoPrefix: true,
   112  			Advanced: true,
   113  		}, {
   114  			Name: "zero_size_links",
   115  			Help: `Assume the Stat size of links is zero (and read them instead) (deprecated).
   116  
   117  Rclone used to use the Stat size of links as the link size, but this fails in quite a few places:
   118  
   119  - Windows
   120  - On some virtual filesystems (such ash LucidLink)
   121  - Android
   122  
   123  So rclone now always reads the link.
   124  `,
   125  			Default:  false,
   126  			Advanced: true,
   127  		}, {
   128  			Name: "unicode_normalization",
   129  			Help: `Apply unicode NFC normalization to paths and filenames.
   130  
   131  This flag can be used to normalize file names into unicode NFC form
   132  that are read from the local filesystem.
   133  
   134  Rclone does not normally touch the encoding of file names it reads from
   135  the file system.
   136  
   137  This can be useful when using macOS as it normally provides decomposed (NFD)
   138  unicode which in some language (eg Korean) doesn't display properly on
   139  some OSes.
   140  
   141  Note that rclone compares filenames with unicode normalization in the sync
   142  routine so this flag shouldn't normally be used.`,
   143  			Default:  false,
   144  			Advanced: true,
   145  		}, {
   146  			Name: "no_check_updated",
   147  			Help: `Don't check to see if the files change during upload.
   148  
   149  Normally rclone checks the size and modification time of files as they
   150  are being uploaded and aborts with a message which starts "can't copy -
   151  source file is being updated" if the file changes during upload.
   152  
   153  However on some file systems this modification time check may fail (e.g.
   154  [Glusterfs #2206](https://github.com/rclone/rclone/issues/2206)) so this
   155  check can be disabled with this flag.
   156  
   157  If this flag is set, rclone will use its best efforts to transfer a
   158  file which is being updated. If the file is only having things
   159  appended to it (e.g. a log) then rclone will transfer the log file with
   160  the size it had the first time rclone saw it.
   161  
   162  If the file is being modified throughout (not just appended to) then
   163  the transfer may fail with a hash check failure.
   164  
   165  In detail, once the file has had stat() called on it for the first
   166  time we:
   167  
   168  - Only transfer the size that stat gave
   169  - Only checksum the size that stat gave
   170  - Don't update the stat info for the file
   171  
   172  **NB** do not use this flag on a Windows Volume Shadow (VSS). For some
   173  unknown reason, files in a VSS sometimes show different sizes from the
   174  directory listing (where the initial stat value comes from on Windows)
   175  and when stat is called on them directly. Other copy tools always use
   176  the direct stat value and setting this flag will disable that.
   177  `,
   178  			Default:  false,
   179  			Advanced: true,
   180  		}, {
   181  			Name:     "one_file_system",
   182  			Help:     "Don't cross filesystem boundaries (unix/macOS only).",
   183  			Default:  false,
   184  			NoPrefix: true,
   185  			ShortOpt: "x",
   186  			Advanced: true,
   187  		}, {
   188  			Name: "case_sensitive",
   189  			Help: `Force the filesystem to report itself as case sensitive.
   190  
   191  Normally the local backend declares itself as case insensitive on
   192  Windows/macOS and case sensitive for everything else.  Use this flag
   193  to override the default choice.`,
   194  			Default:  false,
   195  			Advanced: true,
   196  		}, {
   197  			Name: "case_insensitive",
   198  			Help: `Force the filesystem to report itself as case insensitive.
   199  
   200  Normally the local backend declares itself as case insensitive on
   201  Windows/macOS and case sensitive for everything else.  Use this flag
   202  to override the default choice.`,
   203  			Default:  false,
   204  			Advanced: true,
   205  		}, {
   206  			Name: "no_preallocate",
   207  			Help: `Disable preallocation of disk space for transferred files.
   208  
   209  Preallocation of disk space helps prevent filesystem fragmentation.
   210  However, some virtual filesystem layers (such as Google Drive File
   211  Stream) may incorrectly set the actual file size equal to the
   212  preallocated space, causing checksum and file size checks to fail.
   213  Use this flag to disable preallocation.`,
   214  			Default:  false,
   215  			Advanced: true,
   216  		}, {
   217  			Name: "no_sparse",
   218  			Help: `Disable sparse files for multi-thread downloads.
   219  
   220  On Windows platforms rclone will make sparse files when doing
   221  multi-thread downloads. This avoids long pauses on large files where
   222  the OS zeros the file. However sparse files may be undesirable as they
   223  cause disk fragmentation and can be slow to work with.`,
   224  			Default:  false,
   225  			Advanced: true,
   226  		}, {
   227  			Name: "no_set_modtime",
   228  			Help: `Disable setting modtime.
   229  
   230  Normally rclone updates modification time of files after they are done
   231  uploading. This can cause permissions issues on Linux platforms when 
   232  the user rclone is running as does not own the file uploaded, such as
   233  when copying to a CIFS mount owned by another user. If this option is 
   234  enabled, rclone will no longer update the modtime after copying a file.`,
   235  			Default:  false,
   236  			Advanced: true,
   237  		}, {
   238  			Name: "time_type",
   239  			Help: `Set what kind of time is returned.
   240  
   241  Normally rclone does all operations on the mtime or Modification time.
   242  
   243  If you set this flag then rclone will return the Modified time as whatever
   244  you set here. So if you use "rclone lsl --local-time-type ctime" then
   245  you will see ctimes in the listing.
   246  
   247  If the OS doesn't support returning the time_type specified then rclone
   248  will silently replace it with the modification time which all OSes support.
   249  
   250  - mtime is supported by all OSes
   251  - atime is supported on all OSes except: plan9, js
   252  - btime is only supported on: Windows, macOS, freebsd, netbsd
   253  - ctime is supported on all Oses except: Windows, plan9, js
   254  
   255  Note that setting the time will still set the modified time so this is
   256  only useful for reading.
   257  `,
   258  			Default:  mTime,
   259  			Advanced: true,
   260  			Examples: []fs.OptionExample{{
   261  				Value: mTime.String(),
   262  				Help:  "The last modification time.",
   263  			}, {
   264  				Value: aTime.String(),
   265  				Help:  "The last access time.",
   266  			}, {
   267  				Value: bTime.String(),
   268  				Help:  "The creation time.",
   269  			}, {
   270  				Value: cTime.String(),
   271  				Help:  "The last status change time.",
   272  			}},
   273  		}, {
   274  			Name:     config.ConfigEncoding,
   275  			Help:     config.ConfigEncodingHelp,
   276  			Advanced: true,
   277  			Default:  encoder.OS,
   278  		}},
   279  	}
   280  	fs.Register(fsi)
   281  }
   282  
   283  // Options defines the configuration for this backend
   284  type Options struct {
   285  	FollowSymlinks    bool                 `config:"copy_links"`
   286  	TranslateSymlinks bool                 `config:"links"`
   287  	SkipSymlinks      bool                 `config:"skip_links"`
   288  	UTFNorm           bool                 `config:"unicode_normalization"`
   289  	NoCheckUpdated    bool                 `config:"no_check_updated"`
   290  	NoUNC             bool                 `config:"nounc"`
   291  	OneFileSystem     bool                 `config:"one_file_system"`
   292  	CaseSensitive     bool                 `config:"case_sensitive"`
   293  	CaseInsensitive   bool                 `config:"case_insensitive"`
   294  	NoPreAllocate     bool                 `config:"no_preallocate"`
   295  	NoSparse          bool                 `config:"no_sparse"`
   296  	NoSetModTime      bool                 `config:"no_set_modtime"`
   297  	TimeType          timeType             `config:"time_type"`
   298  	Enc               encoder.MultiEncoder `config:"encoding"`
   299  }
   300  
   301  // Fs represents a local filesystem rooted at root
   302  type Fs struct {
   303  	name           string              // the name of the remote
   304  	root           string              // The root directory (OS path)
   305  	opt            Options             // parsed config options
   306  	features       *fs.Features        // optional features
   307  	dev            uint64              // device number of root node
   308  	precisionOk    sync.Once           // Whether we need to read the precision
   309  	precision      time.Duration       // precision of local filesystem
   310  	warnedMu       sync.Mutex          // used for locking access to 'warned'.
   311  	warned         map[string]struct{} // whether we have warned about this string
   312  	xattrSupported atomic.Int32        // whether xattrs are supported
   313  
   314  	// do os.Lstat or os.Stat
   315  	lstat        func(name string) (os.FileInfo, error)
   316  	objectMetaMu sync.RWMutex // global lock for Object metadata
   317  }
   318  
   319  // Object represents a local filesystem object
   320  type Object struct {
   321  	fs     *Fs    // The Fs this object is part of
   322  	remote string // The remote path (encoded path)
   323  	path   string // The local path (OS path)
   324  	// When using these items the fs.objectMetaMu must be held
   325  	size    int64 // file metadata - always present
   326  	mode    os.FileMode
   327  	modTime time.Time
   328  	hashes  map[hash.Type]string // Hashes
   329  	// these are read only and don't need the mutex held
   330  	translatedLink bool // Is this object a translated link
   331  }
   332  
   333  // Directory represents a local filesystem directory
   334  type Directory struct {
   335  	Object
   336  }
   337  
   338  // ------------------------------------------------------------
   339  
   340  var (
   341  	errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
   342  	errLinksNeedsSuffix  = errors.New("need \"" + linkSuffix + "\" suffix to refer to symlink when using -l/--links")
   343  )
   344  
   345  // NewFs constructs an Fs from the path
   346  func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
   347  	// Parse config into Options struct
   348  	opt := new(Options)
   349  	err := configstruct.Set(m, opt)
   350  	if err != nil {
   351  		return nil, err
   352  	}
   353  	if opt.TranslateSymlinks && opt.FollowSymlinks {
   354  		return nil, errLinksAndCopyLinks
   355  	}
   356  
   357  	f := &Fs{
   358  		name:   name,
   359  		opt:    *opt,
   360  		warned: make(map[string]struct{}),
   361  		dev:    devUnset,
   362  		lstat:  os.Lstat,
   363  	}
   364  	if xattrSupported {
   365  		f.xattrSupported.Store(1)
   366  	}
   367  	f.root = cleanRootPath(root, f.opt.NoUNC, f.opt.Enc)
   368  	f.features = (&fs.Features{
   369  		CaseInsensitive:          f.caseInsensitive(),
   370  		CanHaveEmptyDirectories:  true,
   371  		IsLocal:                  true,
   372  		SlowHash:                 true,
   373  		ReadMetadata:             true,
   374  		WriteMetadata:            true,
   375  		ReadDirMetadata:          true,
   376  		WriteDirMetadata:         true,
   377  		WriteDirSetModTime:       true,
   378  		UserDirMetadata:          xattrSupported, // can only R/W general purpose metadata if xattrs are supported
   379  		DirModTimeUpdatesOnWrite: true,
   380  		UserMetadata:             xattrSupported, // can only R/W general purpose metadata if xattrs are supported
   381  		FilterAware:              true,
   382  		PartialUploads:           true,
   383  	}).Fill(ctx, f)
   384  	if opt.FollowSymlinks {
   385  		f.lstat = os.Stat
   386  	}
   387  
   388  	// Check to see if this points to a file
   389  	fi, err := f.lstat(f.root)
   390  	if err == nil {
   391  		f.dev = readDevice(fi, f.opt.OneFileSystem)
   392  	}
   393  	// Check to see if this is a .rclonelink if not found
   394  	hasLinkSuffix := strings.HasSuffix(f.root, linkSuffix)
   395  	if hasLinkSuffix && opt.TranslateSymlinks && os.IsNotExist(err) {
   396  		fi, err = f.lstat(strings.TrimSuffix(f.root, linkSuffix))
   397  	}
   398  	if err == nil && f.isRegular(fi.Mode()) {
   399  		// Handle the odd case, that a symlink was specified by name without the link suffix
   400  		if !hasLinkSuffix && opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 {
   401  			return nil, errLinksNeedsSuffix
   402  		}
   403  		// It is a file, so use the parent as the root
   404  		f.root = filepath.Dir(f.root)
   405  		// return an error with an fs which points to the parent
   406  		return f, fs.ErrorIsFile
   407  	}
   408  	return f, nil
   409  }
   410  
   411  // Determine whether a file is a 'regular' file,
   412  // Symlinks are regular files, only if the TranslateSymlink
   413  // option is in-effect
   414  func (f *Fs) isRegular(mode os.FileMode) bool {
   415  	if !f.opt.TranslateSymlinks {
   416  		return mode.IsRegular()
   417  	}
   418  
   419  	// fi.Mode().IsRegular() tests that all mode bits are zero
   420  	// Since symlinks are accepted, test that all other bits are zero,
   421  	// except the symlink bit
   422  	return mode&os.ModeType&^os.ModeSymlink == 0
   423  }
   424  
   425  // Name of the remote (as passed into NewFs)
   426  func (f *Fs) Name() string {
   427  	return f.name
   428  }
   429  
   430  // Root of the remote (as passed into NewFs)
   431  func (f *Fs) Root() string {
   432  	return f.opt.Enc.ToStandardPath(filepath.ToSlash(f.root))
   433  }
   434  
   435  // String converts this Fs to a string
   436  func (f *Fs) String() string {
   437  	return fmt.Sprintf("Local file system at %s", f.Root())
   438  }
   439  
   440  // Features returns the optional features of this Fs
   441  func (f *Fs) Features() *fs.Features {
   442  	return f.features
   443  }
   444  
   445  // caseInsensitive returns whether the remote is case insensitive or not
   446  func (f *Fs) caseInsensitive() bool {
   447  	if f.opt.CaseSensitive {
   448  		return false
   449  	}
   450  	if f.opt.CaseInsensitive {
   451  		return true
   452  	}
   453  	// FIXME not entirely accurate since you can have case
   454  	// sensitive Fses on darwin and case insensitive Fses on linux.
   455  	// Should probably check but that would involve creating a
   456  	// file in the remote to be most accurate which probably isn't
   457  	// desirable.
   458  	return runtime.GOOS == "windows" || runtime.GOOS == "darwin"
   459  }
   460  
   461  // translateLink checks whether the remote is a translated link
   462  // and returns a new path, removing the suffix as needed,
   463  // It also returns whether this is a translated link at all
   464  //
   465  // for regular files, localPath is returned unchanged
   466  func translateLink(remote, localPath string) (newLocalPath string, isTranslatedLink bool) {
   467  	isTranslatedLink = strings.HasSuffix(remote, linkSuffix)
   468  	newLocalPath = strings.TrimSuffix(localPath, linkSuffix)
   469  	return newLocalPath, isTranslatedLink
   470  }
   471  
   472  // newObject makes a half completed Object
   473  func (f *Fs) newObject(remote string) *Object {
   474  	translatedLink := false
   475  	localPath := f.localPath(remote)
   476  
   477  	if f.opt.TranslateSymlinks {
   478  		// Possibly receive a new name for localPath
   479  		localPath, translatedLink = translateLink(remote, localPath)
   480  	}
   481  
   482  	return &Object{
   483  		fs:             f,
   484  		remote:         remote,
   485  		path:           localPath,
   486  		translatedLink: translatedLink,
   487  	}
   488  }
   489  
   490  // Return an Object from a path
   491  //
   492  // May return nil if an error occurred
   493  func (f *Fs) newObjectWithInfo(remote string, info os.FileInfo) (fs.Object, error) {
   494  	o := f.newObject(remote)
   495  	if info != nil {
   496  		o.setMetadata(info)
   497  	} else {
   498  		err := o.lstat()
   499  		if err != nil {
   500  			if os.IsNotExist(err) {
   501  				return nil, fs.ErrorObjectNotFound
   502  			}
   503  			if os.IsPermission(err) {
   504  				return nil, fs.ErrorPermissionDenied
   505  			}
   506  			return nil, err
   507  		}
   508  		// Handle the odd case, that a symlink was specified by name without the link suffix
   509  		if o.fs.opt.TranslateSymlinks && o.mode&os.ModeSymlink != 0 && !o.translatedLink {
   510  			return nil, fs.ErrorObjectNotFound
   511  		}
   512  
   513  	}
   514  	if o.mode.IsDir() {
   515  		return nil, fs.ErrorIsDir
   516  	}
   517  	return o, nil
   518  }
   519  
   520  // NewObject finds the Object at remote.  If it can't be found
   521  // it returns the error ErrorObjectNotFound.
   522  func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
   523  	return f.newObjectWithInfo(remote, nil)
   524  }
   525  
   526  // Create new directory object from the info passed in
   527  func (f *Fs) newDirectory(dir string, fi os.FileInfo) *Directory {
   528  	o := f.newObject(dir)
   529  	o.setMetadata(fi)
   530  	return &Directory{
   531  		Object: *o,
   532  	}
   533  }
   534  
   535  // List the objects and directories in dir into entries.  The
   536  // entries can be returned in any order but should be for a
   537  // complete directory.
   538  //
   539  // dir should be "" to list the root, and should not have
   540  // trailing slashes.
   541  //
   542  // This should return ErrDirNotFound if the directory isn't
   543  // found.
   544  func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
   545  	filter, useFilter := filter.GetConfig(ctx), filter.GetUseFilter(ctx)
   546  
   547  	fsDirPath := f.localPath(dir)
   548  	_, err = os.Stat(fsDirPath)
   549  	if err != nil {
   550  		return nil, fs.ErrorDirNotFound
   551  	}
   552  
   553  	fd, err := os.Open(fsDirPath)
   554  	if err != nil {
   555  		isPerm := os.IsPermission(err)
   556  		err = fmt.Errorf("failed to open directory %q: %w", dir, err)
   557  		fs.Errorf(dir, "%v", err)
   558  		if isPerm {
   559  			_ = accounting.Stats(ctx).Error(fserrors.NoRetryError(err))
   560  			err = nil // ignore error but fail sync
   561  		}
   562  		return nil, err
   563  	}
   564  	defer func() {
   565  		cerr := fd.Close()
   566  		if cerr != nil && err == nil {
   567  			err = fmt.Errorf("failed to close directory %q:: %w", dir, cerr)
   568  		}
   569  	}()
   570  
   571  	for {
   572  		var fis []os.FileInfo
   573  		if useReadDir {
   574  			// Windows and Plan9 read the directory entries with the stat information in which
   575  			// shouldn't fail because of unreadable entries.
   576  			fis, err = fd.Readdir(1024)
   577  			if err == io.EOF && len(fis) == 0 {
   578  				break
   579  			}
   580  		} else {
   581  			// For other OSes we read the names only (which shouldn't fail) then stat the
   582  			// individual ourselves so we can log errors but not fail the directory read.
   583  			var names []string
   584  			names, err = fd.Readdirnames(1024)
   585  			if err == io.EOF && len(names) == 0 {
   586  				break
   587  			}
   588  			if err == nil {
   589  				for _, name := range names {
   590  					namepath := filepath.Join(fsDirPath, name)
   591  					fi, fierr := os.Lstat(namepath)
   592  					if os.IsNotExist(fierr) {
   593  						// skip entry removed by a concurrent goroutine
   594  						continue
   595  					}
   596  					if fierr != nil {
   597  						// Don't report errors on any file names that are excluded
   598  						if useFilter {
   599  							newRemote := f.cleanRemote(dir, name)
   600  							if !filter.IncludeRemote(newRemote) {
   601  								continue
   602  							}
   603  						}
   604  						fierr = fmt.Errorf("failed to get info about directory entry %q: %w", namepath, fierr)
   605  						fs.Errorf(dir, "%v", fierr)
   606  						_ = accounting.Stats(ctx).Error(fserrors.NoRetryError(fierr)) // fail the sync
   607  						continue
   608  					}
   609  					fis = append(fis, fi)
   610  				}
   611  			}
   612  		}
   613  		if err != nil {
   614  			return nil, fmt.Errorf("failed to read directory entry: %w", err)
   615  		}
   616  
   617  		for _, fi := range fis {
   618  			name := fi.Name()
   619  			mode := fi.Mode()
   620  			newRemote := f.cleanRemote(dir, name)
   621  			// Follow symlinks if required
   622  			if f.opt.FollowSymlinks && (mode&os.ModeSymlink) != 0 {
   623  				localPath := filepath.Join(fsDirPath, name)
   624  				fi, err = os.Stat(localPath)
   625  				// Quietly skip errors on excluded files and directories
   626  				if err != nil && useFilter && !filter.IncludeRemote(newRemote) {
   627  					continue
   628  				}
   629  				if os.IsNotExist(err) || isCircularSymlinkError(err) {
   630  					// Skip bad symlinks and circular symlinks
   631  					err = fserrors.NoRetryError(fmt.Errorf("symlink: %w", err))
   632  					fs.Errorf(newRemote, "Listing error: %v", err)
   633  					err = accounting.Stats(ctx).Error(err)
   634  					continue
   635  				}
   636  				if err != nil {
   637  					return nil, err
   638  				}
   639  				mode = fi.Mode()
   640  			}
   641  			if fi.IsDir() {
   642  				// Ignore directories which are symlinks.  These are junction points under windows which
   643  				// are kind of a souped up symlink. Unix doesn't have directories which are symlinks.
   644  				if (mode&os.ModeSymlink) == 0 && f.dev == readDevice(fi, f.opt.OneFileSystem) {
   645  					d := f.newDirectory(newRemote, fi)
   646  					entries = append(entries, d)
   647  				}
   648  			} else {
   649  				// Check whether this link should be translated
   650  				if f.opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 {
   651  					newRemote += linkSuffix
   652  				}
   653  				// Don't include non directory if not included
   654  				// we leave directory filtering to the layer above
   655  				if useFilter && !filter.IncludeRemote(newRemote) {
   656  					continue
   657  				}
   658  				fso, err := f.newObjectWithInfo(newRemote, fi)
   659  				if err != nil {
   660  					return nil, err
   661  				}
   662  				if fso.Storable() {
   663  					entries = append(entries, fso)
   664  				}
   665  			}
   666  		}
   667  	}
   668  	return entries, nil
   669  }
   670  
   671  func (f *Fs) cleanRemote(dir, filename string) (remote string) {
   672  	if f.opt.UTFNorm {
   673  		filename = norm.NFC.String(filename)
   674  	}
   675  	remote = path.Join(dir, f.opt.Enc.ToStandardName(filename))
   676  
   677  	if !utf8.ValidString(filename) {
   678  		f.warnedMu.Lock()
   679  		if _, ok := f.warned[remote]; !ok {
   680  			fs.Logf(f, "Replacing invalid UTF-8 characters in %q", remote)
   681  			f.warned[remote] = struct{}{}
   682  		}
   683  		f.warnedMu.Unlock()
   684  	}
   685  	return
   686  }
   687  
   688  func (f *Fs) localPath(name string) string {
   689  	return filepath.Join(f.root, filepath.FromSlash(f.opt.Enc.FromStandardPath(name)))
   690  }
   691  
   692  // Put the Object to the local filesystem
   693  func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
   694  	// Temporary Object under construction - info filled in by Update()
   695  	o := f.newObject(src.Remote())
   696  	err := o.Update(ctx, in, src, options...)
   697  	if err != nil {
   698  		return nil, err
   699  	}
   700  	return o, nil
   701  }
   702  
   703  // PutStream uploads to the remote path with the modTime given of indeterminate size
   704  func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
   705  	return f.Put(ctx, in, src, options...)
   706  }
   707  
   708  // Mkdir creates the directory if it doesn't exist
   709  func (f *Fs) Mkdir(ctx context.Context, dir string) error {
   710  	localPath := f.localPath(dir)
   711  	err := file.MkdirAll(localPath, 0777)
   712  	if err != nil {
   713  		return err
   714  	}
   715  	if dir == "" {
   716  		fi, err := f.lstat(localPath)
   717  		if err != nil {
   718  			return err
   719  		}
   720  		f.dev = readDevice(fi, f.opt.OneFileSystem)
   721  	}
   722  	return nil
   723  }
   724  
   725  // DirSetModTime sets the directory modtime for dir
   726  func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
   727  	o := Object{
   728  		fs:     f,
   729  		remote: dir,
   730  		path:   f.localPath(dir),
   731  	}
   732  	return o.SetModTime(ctx, modTime)
   733  }
   734  
   735  // MkdirMetadata makes the directory passed in as dir.
   736  //
   737  // It shouldn't return an error if it already exists.
   738  //
   739  // If the metadata is not nil it is set.
   740  //
   741  // It returns the directory that was created.
   742  func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) {
   743  	// Find and or create the directory
   744  	localPath := f.localPath(dir)
   745  	fi, err := f.lstat(localPath)
   746  	if errors.Is(err, os.ErrNotExist) {
   747  		err := f.Mkdir(ctx, dir)
   748  		if err != nil {
   749  			return nil, fmt.Errorf("mkdir metadata: failed make directory: %w", err)
   750  		}
   751  		fi, err = f.lstat(localPath)
   752  		if err != nil {
   753  			return nil, fmt.Errorf("mkdir metadata: failed to read info: %w", err)
   754  		}
   755  	} else if err != nil {
   756  		return nil, err
   757  	}
   758  
   759  	// Create directory object
   760  	d := f.newDirectory(dir, fi)
   761  
   762  	// Set metadata on the directory object if provided
   763  	if metadata != nil {
   764  		err = d.writeMetadata(metadata)
   765  		if err != nil {
   766  			return nil, fmt.Errorf("failed to set metadata on directory: %w", err)
   767  		}
   768  		// Re-read info now we have finished setting stuff
   769  		err = d.lstat()
   770  		if err != nil {
   771  			return nil, fmt.Errorf("mkdir metadata: failed to re-read info: %w", err)
   772  		}
   773  	}
   774  	return d, nil
   775  }
   776  
   777  // Rmdir removes the directory
   778  //
   779  // If it isn't empty it will return an error
   780  func (f *Fs) Rmdir(ctx context.Context, dir string) error {
   781  	localPath := f.localPath(dir)
   782  	if fi, err := os.Stat(localPath); err != nil {
   783  		return err
   784  	} else if !fi.IsDir() {
   785  		return fs.ErrorIsFile
   786  	}
   787  	return os.Remove(localPath)
   788  }
   789  
   790  // Precision of the file system
   791  func (f *Fs) Precision() (precision time.Duration) {
   792  	if f.opt.NoSetModTime {
   793  		return fs.ModTimeNotSupported
   794  	}
   795  
   796  	f.precisionOk.Do(func() {
   797  		f.precision = f.readPrecision()
   798  	})
   799  	return f.precision
   800  }
   801  
   802  // Read the precision
   803  func (f *Fs) readPrecision() (precision time.Duration) {
   804  	// Default precision of 1s
   805  	precision = time.Second
   806  
   807  	// Create temporary file and test it
   808  	fd, err := os.CreateTemp("", "rclone")
   809  	if err != nil {
   810  		// If failed return 1s
   811  		// fmt.Println("Failed to create temp file", err)
   812  		return time.Second
   813  	}
   814  	path := fd.Name()
   815  	// fmt.Println("Created temp file", path)
   816  	err = fd.Close()
   817  	if err != nil {
   818  		return time.Second
   819  	}
   820  
   821  	// Delete it on return
   822  	defer func() {
   823  		// fmt.Println("Remove temp file")
   824  		_ = os.Remove(path) // ignore error
   825  	}()
   826  
   827  	// Find the minimum duration we can detect
   828  	for duration := time.Duration(1); duration < time.Second; duration *= 10 {
   829  		// Current time with delta
   830  		t := time.Unix(time.Now().Unix(), int64(duration))
   831  		err := os.Chtimes(path, t, t)
   832  		if err != nil {
   833  			// fmt.Println("Failed to Chtimes", err)
   834  			break
   835  		}
   836  
   837  		// Read the actual time back
   838  		fi, err := os.Stat(path)
   839  		if err != nil {
   840  			// fmt.Println("Failed to Stat", err)
   841  			break
   842  		}
   843  
   844  		// If it matches - have found the precision
   845  		// fmt.Println("compare", fi.ModTime(ctx), t)
   846  		if fi.ModTime().Equal(t) {
   847  			// fmt.Println("Precision detected as", duration)
   848  			return duration
   849  		}
   850  	}
   851  	return
   852  }
   853  
   854  // Move src to this remote using server-side move operations.
   855  //
   856  // This is stored with the remote path given.
   857  //
   858  // It returns the destination Object and a possible error.
   859  //
   860  // Will only be called if src.Fs().Name() == f.Name()
   861  //
   862  // If it isn't possible then return fs.ErrorCantMove
   863  func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
   864  	srcObj, ok := src.(*Object)
   865  	if !ok {
   866  		fs.Debugf(src, "Can't move - not same remote type")
   867  		return nil, fs.ErrorCantMove
   868  	}
   869  
   870  	// Temporary Object under construction
   871  	dstObj := f.newObject(remote)
   872  	dstObj.fs.objectMetaMu.RLock()
   873  	dstObjMode := dstObj.mode
   874  	dstObj.fs.objectMetaMu.RUnlock()
   875  
   876  	// Check it is a file if it exists
   877  	err := dstObj.lstat()
   878  	if os.IsNotExist(err) {
   879  		// OK
   880  	} else if err != nil {
   881  		return nil, err
   882  	} else if !dstObj.fs.isRegular(dstObjMode) {
   883  		// It isn't a file
   884  		return nil, errors.New("can't move file onto non-file")
   885  	}
   886  
   887  	// Create destination
   888  	err = dstObj.mkdirAll()
   889  	if err != nil {
   890  		return nil, err
   891  	}
   892  
   893  	// Fetch metadata if --metadata is in use
   894  	meta, err := fs.GetMetadataOptions(ctx, f, src, fs.MetadataAsOpenOptions(ctx))
   895  	if err != nil {
   896  		return nil, fmt.Errorf("move: failed to read metadata: %w", err)
   897  	}
   898  
   899  	// Do the move
   900  	err = os.Rename(srcObj.path, dstObj.path)
   901  	if os.IsNotExist(err) {
   902  		// race condition, source was deleted in the meantime
   903  		return nil, err
   904  	} else if os.IsPermission(err) {
   905  		// not enough rights to write to dst
   906  		return nil, err
   907  	} else if err != nil {
   908  		// not quite clear, but probably trying to move a file across file system
   909  		// boundaries. Copying might still work.
   910  		fs.Debugf(src, "Can't move: %v: trying copy", err)
   911  		return nil, fs.ErrorCantMove
   912  	}
   913  
   914  	// Set metadata if --metadata is in use
   915  	err = dstObj.writeMetadata(meta)
   916  	if err != nil {
   917  		return nil, fmt.Errorf("move: failed to set metadata: %w", err)
   918  	}
   919  
   920  	// Update the info
   921  	err = dstObj.lstat()
   922  	if err != nil {
   923  		return nil, err
   924  	}
   925  
   926  	return dstObj, nil
   927  }
   928  
   929  // DirMove moves src, srcRemote to this remote at dstRemote
   930  // using server-side move operations.
   931  //
   932  // Will only be called if src.Fs().Name() == f.Name()
   933  //
   934  // If it isn't possible then return fs.ErrorCantDirMove
   935  //
   936  // If destination exists then return fs.ErrorDirExists
   937  func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
   938  	srcFs, ok := src.(*Fs)
   939  	if !ok {
   940  		fs.Debugf(srcFs, "Can't move directory - not same remote type")
   941  		return fs.ErrorCantDirMove
   942  	}
   943  	srcPath := srcFs.localPath(srcRemote)
   944  	dstPath := f.localPath(dstRemote)
   945  
   946  	// Check if destination exists
   947  	_, err := os.Lstat(dstPath)
   948  	if !os.IsNotExist(err) {
   949  		return fs.ErrorDirExists
   950  	}
   951  
   952  	// Create parent of destination
   953  	dstParentPath := filepath.Dir(dstPath)
   954  	err = file.MkdirAll(dstParentPath, 0777)
   955  	if err != nil {
   956  		return err
   957  	}
   958  
   959  	// Do the move
   960  	err = os.Rename(srcPath, dstPath)
   961  	if os.IsNotExist(err) {
   962  		// race condition, source was deleted in the meantime
   963  		return err
   964  	} else if os.IsPermission(err) {
   965  		// not enough rights to write to dst
   966  		return err
   967  	} else if err != nil {
   968  		// not quite clear, but probably trying to move directory across file system
   969  		// boundaries. Copying might still work.
   970  		fs.Debugf(src, "Can't move dir: %v: trying copy", err)
   971  		return fs.ErrorCantDirMove
   972  	}
   973  	return nil
   974  }
   975  
   976  // Hashes returns the supported hash sets.
   977  func (f *Fs) Hashes() hash.Set {
   978  	return hash.Supported()
   979  }
   980  
   981  var commandHelp = []fs.CommandHelp{
   982  	{
   983  		Name:  "noop",
   984  		Short: "A null operation for testing backend commands",
   985  		Long: `This is a test command which has some options
   986  you can try to change the output.`,
   987  		Opts: map[string]string{
   988  			"echo":  "echo the input arguments",
   989  			"error": "return an error based on option value",
   990  		},
   991  	},
   992  }
   993  
   994  // Command the backend to run a named command
   995  //
   996  // The command run is name
   997  // args may be used to read arguments from
   998  // opts may be used to read optional arguments from
   999  //
  1000  // The result should be capable of being JSON encoded
  1001  // If it is a string or a []string it will be shown to the user
  1002  // otherwise it will be JSON encoded and shown to the user like that
  1003  func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error) {
  1004  	switch name {
  1005  	case "noop":
  1006  		if txt, ok := opt["error"]; ok {
  1007  			if txt == "" {
  1008  				txt = "unspecified error"
  1009  			}
  1010  			return nil, errors.New(txt)
  1011  		}
  1012  		if _, ok := opt["echo"]; ok {
  1013  			out := map[string]interface{}{}
  1014  			out["name"] = name
  1015  			out["arg"] = arg
  1016  			out["opt"] = opt
  1017  			return out, nil
  1018  		}
  1019  		return nil, nil
  1020  	default:
  1021  		return nil, fs.ErrorCommandNotFound
  1022  	}
  1023  }
  1024  
  1025  // ------------------------------------------------------------
  1026  
  1027  // Fs returns the parent Fs
  1028  func (o *Object) Fs() fs.Info {
  1029  	return o.fs
  1030  }
  1031  
  1032  // Return a string version
  1033  func (o *Object) String() string {
  1034  	if o == nil {
  1035  		return "<nil>"
  1036  	}
  1037  	return o.remote
  1038  }
  1039  
  1040  // Remote returns the remote path
  1041  func (o *Object) Remote() string {
  1042  	return o.remote
  1043  }
  1044  
  1045  // Hash returns the requested hash of a file as a lowercase hex string
  1046  func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
  1047  	// Check that the underlying file hasn't changed
  1048  	o.fs.objectMetaMu.RLock()
  1049  	oldtime := o.modTime
  1050  	oldsize := o.size
  1051  	o.fs.objectMetaMu.RUnlock()
  1052  	err := o.lstat()
  1053  	var changed bool
  1054  	if err != nil {
  1055  		if errors.Is(err, os.ErrNotExist) {
  1056  			// If file not found then we assume any accumulated
  1057  			// hashes are OK - this will error on Open
  1058  			changed = true
  1059  		} else {
  1060  			return "", fmt.Errorf("hash: failed to stat: %w", err)
  1061  		}
  1062  	} else {
  1063  		o.fs.objectMetaMu.RLock()
  1064  		changed = !o.modTime.Equal(oldtime) || oldsize != o.size
  1065  		o.fs.objectMetaMu.RUnlock()
  1066  	}
  1067  
  1068  	o.fs.objectMetaMu.RLock()
  1069  	hashValue, hashFound := o.hashes[r]
  1070  	o.fs.objectMetaMu.RUnlock()
  1071  
  1072  	if changed || !hashFound {
  1073  		var in io.ReadCloser
  1074  
  1075  		if !o.translatedLink {
  1076  			var fd *os.File
  1077  			fd, err = file.Open(o.path)
  1078  			if fd != nil {
  1079  				in = newFadviseReadCloser(o, fd, 0, 0)
  1080  			}
  1081  		} else {
  1082  			in, err = o.openTranslatedLink(0, -1)
  1083  		}
  1084  		// If not checking for updates, only read size given
  1085  		if o.fs.opt.NoCheckUpdated {
  1086  			in = readers.NewLimitedReadCloser(in, o.size)
  1087  		}
  1088  		if err != nil {
  1089  			return "", fmt.Errorf("hash: failed to open: %w", err)
  1090  		}
  1091  		var hashes map[hash.Type]string
  1092  		hashes, err = hash.StreamTypes(readers.NewContextReader(ctx, in), hash.NewHashSet(r))
  1093  		closeErr := in.Close()
  1094  		if err != nil {
  1095  			return "", fmt.Errorf("hash: failed to read: %w", err)
  1096  		}
  1097  		if closeErr != nil {
  1098  			return "", fmt.Errorf("hash: failed to close: %w", closeErr)
  1099  		}
  1100  		hashValue = hashes[r]
  1101  		o.fs.objectMetaMu.Lock()
  1102  		if o.hashes == nil {
  1103  			o.hashes = hashes
  1104  		} else {
  1105  			o.hashes[r] = hashValue
  1106  		}
  1107  		o.fs.objectMetaMu.Unlock()
  1108  	}
  1109  	return hashValue, nil
  1110  }
  1111  
  1112  // Size returns the size of an object in bytes
  1113  func (o *Object) Size() int64 {
  1114  	o.fs.objectMetaMu.RLock()
  1115  	defer o.fs.objectMetaMu.RUnlock()
  1116  	return o.size
  1117  }
  1118  
  1119  // ModTime returns the modification time of the object
  1120  func (o *Object) ModTime(ctx context.Context) time.Time {
  1121  	o.fs.objectMetaMu.RLock()
  1122  	defer o.fs.objectMetaMu.RUnlock()
  1123  	return o.modTime
  1124  }
  1125  
  1126  // Set the atime and ltime of the object
  1127  func (o *Object) setTimes(atime, mtime time.Time) (err error) {
  1128  	if o.translatedLink {
  1129  		err = lChtimes(o.path, atime, mtime)
  1130  	} else {
  1131  		err = os.Chtimes(o.path, atime, mtime)
  1132  	}
  1133  	return err
  1134  }
  1135  
  1136  // SetModTime sets the modification time of the local fs object
  1137  func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
  1138  	if o.fs.opt.NoSetModTime {
  1139  		return nil
  1140  	}
  1141  	err := o.setTimes(modTime, modTime)
  1142  	if err != nil {
  1143  		return err
  1144  	}
  1145  	// Re-read metadata
  1146  	return o.lstat()
  1147  }
  1148  
  1149  // Storable returns a boolean showing if this object is storable
  1150  func (o *Object) Storable() bool {
  1151  	o.fs.objectMetaMu.RLock()
  1152  	mode := o.mode
  1153  	o.fs.objectMetaMu.RUnlock()
  1154  	if mode&os.ModeSymlink != 0 && !o.fs.opt.TranslateSymlinks {
  1155  		if !o.fs.opt.SkipSymlinks {
  1156  			fs.Logf(o, "Can't follow symlink without -L/--copy-links")
  1157  		}
  1158  		return false
  1159  	} else if mode&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
  1160  		fs.Logf(o, "Can't transfer non file/directory")
  1161  		return false
  1162  	} else if mode&os.ModeDir != 0 {
  1163  		// fs.Debugf(o, "Skipping directory")
  1164  		return false
  1165  	}
  1166  	return true
  1167  }
  1168  
  1169  // localOpenFile wraps an io.ReadCloser and updates the md5sum of the
  1170  // object that is read
  1171  type localOpenFile struct {
  1172  	o    *Object           // object that is open
  1173  	in   io.ReadCloser     // handle we are wrapping
  1174  	hash *hash.MultiHasher // currently accumulating hashes
  1175  	fd   *os.File          // file object reference
  1176  }
  1177  
  1178  // Read bytes from the object - see io.Reader
  1179  func (file *localOpenFile) Read(p []byte) (n int, err error) {
  1180  	if !file.o.fs.opt.NoCheckUpdated {
  1181  		// Check if file has the same size and modTime
  1182  		fi, err := file.fd.Stat()
  1183  		if err != nil {
  1184  			return 0, fmt.Errorf("can't read status of source file while transferring: %w", err)
  1185  		}
  1186  		file.o.fs.objectMetaMu.RLock()
  1187  		oldtime := file.o.modTime
  1188  		oldsize := file.o.size
  1189  		file.o.fs.objectMetaMu.RUnlock()
  1190  		if oldsize != fi.Size() {
  1191  			return 0, fserrors.NoLowLevelRetryError(fmt.Errorf("can't copy - source file is being updated (size changed from %d to %d)", oldsize, fi.Size()))
  1192  		}
  1193  		if !oldtime.Equal(readTime(file.o.fs.opt.TimeType, fi)) {
  1194  			return 0, fserrors.NoLowLevelRetryError(fmt.Errorf("can't copy - source file is being updated (mod time changed from %v to %v)", oldtime, fi.ModTime()))
  1195  		}
  1196  	}
  1197  
  1198  	n, err = file.in.Read(p)
  1199  	if n > 0 {
  1200  		// Hash routines never return an error
  1201  		_, _ = file.hash.Write(p[:n])
  1202  	}
  1203  	return
  1204  }
  1205  
  1206  // Close the object and update the hashes
  1207  func (file *localOpenFile) Close() (err error) {
  1208  	err = file.in.Close()
  1209  	if err == nil {
  1210  		if file.hash.Size() == file.o.Size() {
  1211  			file.o.fs.objectMetaMu.Lock()
  1212  			file.o.hashes = file.hash.Sums()
  1213  			file.o.fs.objectMetaMu.Unlock()
  1214  		}
  1215  	}
  1216  	return err
  1217  }
  1218  
  1219  // Returns a ReadCloser() object that contains the contents of a symbolic link
  1220  func (o *Object) openTranslatedLink(offset, limit int64) (lrc io.ReadCloser, err error) {
  1221  	// Read the link and return the destination  it as the contents of the object
  1222  	linkdst, err := os.Readlink(o.path)
  1223  	if err != nil {
  1224  		return nil, err
  1225  	}
  1226  	return readers.NewLimitedReadCloser(io.NopCloser(strings.NewReader(linkdst[offset:])), limit), nil
  1227  }
  1228  
  1229  // Open an object for read
  1230  func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
  1231  	var offset, limit int64 = 0, -1
  1232  	var hasher *hash.MultiHasher
  1233  	for _, option := range options {
  1234  		switch x := option.(type) {
  1235  		case *fs.SeekOption:
  1236  			offset = x.Offset
  1237  		case *fs.RangeOption:
  1238  			offset, limit = x.Decode(o.Size())
  1239  		case *fs.HashesOption:
  1240  			if x.Hashes.Count() > 0 {
  1241  				hasher, err = hash.NewMultiHasherTypes(x.Hashes)
  1242  				if err != nil {
  1243  					return nil, err
  1244  				}
  1245  			}
  1246  		default:
  1247  			if option.Mandatory() {
  1248  				fs.Logf(o, "Unsupported mandatory option: %v", option)
  1249  			}
  1250  		}
  1251  	}
  1252  
  1253  	// Update the file info before we start reading
  1254  	err = o.lstat()
  1255  	if err != nil {
  1256  		return nil, err
  1257  	}
  1258  
  1259  	// If not checking updated then limit to current size.  This means if
  1260  	// file is being extended, readers will read a o.Size() bytes rather
  1261  	// than the new size making for a consistent upload.
  1262  	if limit < 0 && o.fs.opt.NoCheckUpdated {
  1263  		limit = o.size
  1264  	}
  1265  
  1266  	// Handle a translated link
  1267  	if o.translatedLink {
  1268  		return o.openTranslatedLink(offset, limit)
  1269  	}
  1270  
  1271  	fd, err := file.Open(o.path)
  1272  	if err != nil {
  1273  		return
  1274  	}
  1275  	wrappedFd := readers.NewLimitedReadCloser(newFadviseReadCloser(o, fd, offset, limit), limit)
  1276  	if offset != 0 {
  1277  		// seek the object
  1278  		_, err = fd.Seek(offset, io.SeekStart)
  1279  		// don't attempt to make checksums
  1280  		return wrappedFd, err
  1281  	}
  1282  	if hasher == nil {
  1283  		// no need to wrap since we don't need checksums
  1284  		return wrappedFd, nil
  1285  	}
  1286  	// Update the hashes as we go along
  1287  	in = &localOpenFile{
  1288  		o:    o,
  1289  		in:   wrappedFd,
  1290  		hash: hasher,
  1291  		fd:   fd,
  1292  	}
  1293  	return in, nil
  1294  }
  1295  
  1296  // mkdirAll makes all the directories needed to store the object
  1297  func (o *Object) mkdirAll() error {
  1298  	dir := filepath.Dir(o.path)
  1299  	return file.MkdirAll(dir, 0777)
  1300  }
  1301  
  1302  type nopWriterCloser struct {
  1303  	*bytes.Buffer
  1304  }
  1305  
  1306  func (nwc nopWriterCloser) Close() error {
  1307  	// noop
  1308  	return nil
  1309  }
  1310  
  1311  // Update the object from in with modTime and size
  1312  func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
  1313  	var out io.WriteCloser
  1314  	var hasher *hash.MultiHasher
  1315  
  1316  	for _, option := range options {
  1317  		switch x := option.(type) {
  1318  		case *fs.HashesOption:
  1319  			if x.Hashes.Count() > 0 {
  1320  				hasher, err = hash.NewMultiHasherTypes(x.Hashes)
  1321  				if err != nil {
  1322  					return err
  1323  				}
  1324  			}
  1325  		}
  1326  	}
  1327  
  1328  	err = o.mkdirAll()
  1329  	if err != nil {
  1330  		return err
  1331  	}
  1332  
  1333  	// Wipe hashes before update
  1334  	o.clearHashCache()
  1335  
  1336  	var symlinkData bytes.Buffer
  1337  	// If the object is a regular file, create it.
  1338  	// If it is a translated link, just read in the contents, and
  1339  	// then create a symlink
  1340  	if !o.translatedLink {
  1341  		f, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
  1342  		if err != nil {
  1343  			if runtime.GOOS == "windows" && os.IsPermission(err) {
  1344  				// If permission denied on Windows might be trying to update a
  1345  				// hidden file, in which case try opening without CREATE
  1346  				// See: https://stackoverflow.com/questions/13215716/ioerror-errno-13-permission-denied-when-trying-to-open-hidden-file-in-w-mod
  1347  				f, err = file.OpenFile(o.path, os.O_WRONLY|os.O_TRUNC, 0666)
  1348  				if err != nil {
  1349  					return err
  1350  				}
  1351  			} else {
  1352  				return err
  1353  			}
  1354  		}
  1355  		if !o.fs.opt.NoPreAllocate {
  1356  			// Pre-allocate the file for performance reasons
  1357  			err = file.PreAllocate(src.Size(), f)
  1358  			if err != nil {
  1359  				fs.Debugf(o, "Failed to pre-allocate: %v", err)
  1360  				if err == file.ErrDiskFull {
  1361  					_ = f.Close()
  1362  					return err
  1363  				}
  1364  			}
  1365  		}
  1366  		out = f
  1367  	} else {
  1368  		out = nopWriterCloser{&symlinkData}
  1369  	}
  1370  
  1371  	// Calculate the hash of the object we are reading as we go along
  1372  	if hasher != nil {
  1373  		in = io.TeeReader(in, hasher)
  1374  	}
  1375  
  1376  	_, err = io.Copy(out, in)
  1377  	closeErr := out.Close()
  1378  	if err == nil {
  1379  		err = closeErr
  1380  	}
  1381  
  1382  	if o.translatedLink {
  1383  		if err == nil {
  1384  			// Remove any current symlink or file, if one exists
  1385  			if _, err := os.Lstat(o.path); err == nil {
  1386  				if removeErr := os.Remove(o.path); removeErr != nil {
  1387  					fs.Errorf(o, "Failed to remove previous file: %v", removeErr)
  1388  					return removeErr
  1389  				}
  1390  			}
  1391  			// Use the contents for the copied object to create a symlink
  1392  			err = os.Symlink(symlinkData.String(), o.path)
  1393  		}
  1394  
  1395  		// only continue if symlink creation succeeded
  1396  		if err != nil {
  1397  			return err
  1398  		}
  1399  	}
  1400  
  1401  	if err != nil {
  1402  		fs.Logf(o, "Removing partially written file on error: %v", err)
  1403  		if removeErr := os.Remove(o.path); removeErr != nil {
  1404  			fs.Errorf(o, "Failed to remove partially written file: %v", removeErr)
  1405  		}
  1406  		return err
  1407  	}
  1408  
  1409  	// All successful so update the hashes
  1410  	if hasher != nil {
  1411  		o.fs.objectMetaMu.Lock()
  1412  		o.hashes = hasher.Sums()
  1413  		o.fs.objectMetaMu.Unlock()
  1414  	}
  1415  
  1416  	// Set the mtime
  1417  	err = o.SetModTime(ctx, src.ModTime(ctx))
  1418  	if err != nil {
  1419  		return err
  1420  	}
  1421  
  1422  	// Fetch and set metadata if --metadata is in use
  1423  	meta, err := fs.GetMetadataOptions(ctx, o.fs, src, options)
  1424  	if err != nil {
  1425  		return fmt.Errorf("failed to read metadata from source object: %w", err)
  1426  	}
  1427  	err = o.writeMetadata(meta)
  1428  	if err != nil {
  1429  		return fmt.Errorf("failed to set metadata: %w", err)
  1430  	}
  1431  
  1432  	// ReRead info now that we have finished
  1433  	return o.lstat()
  1434  }
  1435  
  1436  var sparseWarning sync.Once
  1437  
  1438  // OpenWriterAt opens with a handle for random access writes
  1439  //
  1440  // Pass in the remote desired and the size if known.
  1441  //
  1442  // It truncates any existing object
  1443  func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) {
  1444  	// Temporary Object under construction
  1445  	o := f.newObject(remote)
  1446  
  1447  	err := o.mkdirAll()
  1448  	if err != nil {
  1449  		return nil, err
  1450  	}
  1451  
  1452  	if o.translatedLink {
  1453  		return nil, errors.New("can't open a symlink for random writing")
  1454  	}
  1455  
  1456  	out, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
  1457  	if err != nil {
  1458  		return nil, err
  1459  	}
  1460  	// Pre-allocate the file for performance reasons
  1461  	if !f.opt.NoPreAllocate {
  1462  		err = file.PreAllocate(size, out)
  1463  		if err != nil {
  1464  			fs.Debugf(o, "Failed to pre-allocate: %v", err)
  1465  		}
  1466  	}
  1467  	if !f.opt.NoSparse && file.SetSparseImplemented {
  1468  		sparseWarning.Do(func() {
  1469  			fs.Infof(nil, "Writing sparse files: use --local-no-sparse or --multi-thread-streams 0 to disable")
  1470  		})
  1471  		// Set the file to be a sparse file (important on Windows)
  1472  		err = file.SetSparse(out)
  1473  		if err != nil {
  1474  			fs.Errorf(o, "Failed to set sparse: %v", err)
  1475  		}
  1476  	}
  1477  
  1478  	return out, nil
  1479  }
  1480  
  1481  // setMetadata sets the file info from the os.FileInfo passed in
  1482  func (o *Object) setMetadata(info os.FileInfo) {
  1483  	// if not checking updated then don't update the stat
  1484  	if o.fs.opt.NoCheckUpdated && !o.modTime.IsZero() {
  1485  		return
  1486  	}
  1487  	o.fs.objectMetaMu.Lock()
  1488  	o.size = info.Size()
  1489  	o.modTime = readTime(o.fs.opt.TimeType, info)
  1490  	o.mode = info.Mode()
  1491  	o.fs.objectMetaMu.Unlock()
  1492  	// Read the size of the link.
  1493  	//
  1494  	// The value in info.Size() is not always correct
  1495  	// - Windows links read as 0 size
  1496  	// - Some virtual filesystems (such ash LucidLink) links read as 0 size
  1497  	// - Android - some versions the links are larger than readlink suggests
  1498  	if o.translatedLink {
  1499  		linkdst, err := os.Readlink(o.path)
  1500  		if err != nil {
  1501  			fs.Errorf(o, "Failed to read link size: %v", err)
  1502  		} else {
  1503  			o.size = int64(len(linkdst))
  1504  		}
  1505  	}
  1506  }
  1507  
  1508  // clearHashCache wipes any cached hashes for the object
  1509  func (o *Object) clearHashCache() {
  1510  	o.fs.objectMetaMu.Lock()
  1511  	o.hashes = nil
  1512  	o.fs.objectMetaMu.Unlock()
  1513  }
  1514  
  1515  // Stat an Object into info
  1516  func (o *Object) lstat() error {
  1517  	info, err := o.fs.lstat(o.path)
  1518  	if err == nil {
  1519  		o.setMetadata(info)
  1520  	}
  1521  	return err
  1522  }
  1523  
  1524  // Remove an object
  1525  func (o *Object) Remove(ctx context.Context) error {
  1526  	o.clearHashCache()
  1527  	return remove(o.path)
  1528  }
  1529  
  1530  // Metadata returns metadata for an object
  1531  //
  1532  // It should return nil if there is no Metadata
  1533  func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) {
  1534  	metadata, err = o.getXattr()
  1535  	if err != nil {
  1536  		return nil, err
  1537  	}
  1538  	err = o.readMetadataFromFile(&metadata)
  1539  	if err != nil {
  1540  		return nil, err
  1541  	}
  1542  	return metadata, nil
  1543  }
  1544  
  1545  // Write the metadata on the object
  1546  func (o *Object) writeMetadata(metadata fs.Metadata) (err error) {
  1547  	err = o.setXattr(metadata)
  1548  	if err != nil {
  1549  		return err
  1550  	}
  1551  	err = o.writeMetadataToFile(metadata)
  1552  	if err != nil {
  1553  		return err
  1554  	}
  1555  	return err
  1556  }
  1557  
  1558  // SetMetadata sets metadata for an Object
  1559  //
  1560  // It should return fs.ErrorNotImplemented if it can't set metadata
  1561  func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
  1562  	err := o.writeMetadata(metadata)
  1563  	if err != nil {
  1564  		return fmt.Errorf("SetMetadata failed on Object: %w", err)
  1565  	}
  1566  	// Re-read info now we have finished setting stuff
  1567  	return o.lstat()
  1568  }
  1569  
  1570  func cleanRootPath(s string, noUNC bool, enc encoder.MultiEncoder) string {
  1571  	if runtime.GOOS != "windows" || !strings.HasPrefix(s, "\\") {
  1572  		if !filepath.IsAbs(s) {
  1573  			s2, err := filepath.Abs(s)
  1574  			if err == nil {
  1575  				s = s2
  1576  			}
  1577  		} else {
  1578  			s = filepath.Clean(s)
  1579  		}
  1580  	}
  1581  	if runtime.GOOS == "windows" {
  1582  		s = filepath.ToSlash(s)
  1583  		vol := filepath.VolumeName(s)
  1584  		if vol == `\\?` && len(s) >= 6 {
  1585  			// `\\?\C:`
  1586  			vol = s[:6]
  1587  		}
  1588  		s = vol + enc.FromStandardPath(s[len(vol):])
  1589  		s = filepath.FromSlash(s)
  1590  		if !noUNC {
  1591  			// Convert to UNC
  1592  			s = file.UNCPath(s)
  1593  		}
  1594  		return s
  1595  	}
  1596  	s = enc.FromStandardPath(s)
  1597  	return s
  1598  }
  1599  
  1600  // Items returns the count of items in this directory or this
  1601  // directory and subdirectories if known, -1 for unknown
  1602  func (d *Directory) Items() int64 {
  1603  	return -1
  1604  }
  1605  
  1606  // ID returns the internal ID of this directory if known, or
  1607  // "" otherwise
  1608  func (d *Directory) ID() string {
  1609  	return ""
  1610  }
  1611  
  1612  // SetMetadata sets metadata for a Directory
  1613  //
  1614  // It should return fs.ErrorNotImplemented if it can't set metadata
  1615  func (d *Directory) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
  1616  	err := d.writeMetadata(metadata)
  1617  	if err != nil {
  1618  		return fmt.Errorf("SetMetadata failed on Directory: %w", err)
  1619  	}
  1620  	// Re-read info now we have finished setting stuff
  1621  	return d.lstat()
  1622  }
  1623  
  1624  // Hash does nothing on a directory
  1625  //
  1626  // This method is implemented with the incorrect type signature to
  1627  // stop the Directory type asserting to fs.Object or fs.ObjectInfo
  1628  func (d *Directory) Hash() {
  1629  	// Does nothing
  1630  }
  1631  
  1632  // Check the interfaces are satisfied
  1633  var (
  1634  	_ fs.Fs              = &Fs{}
  1635  	_ fs.PutStreamer     = &Fs{}
  1636  	_ fs.Mover           = &Fs{}
  1637  	_ fs.DirMover        = &Fs{}
  1638  	_ fs.Commander       = &Fs{}
  1639  	_ fs.OpenWriterAter  = &Fs{}
  1640  	_ fs.DirSetModTimer  = &Fs{}
  1641  	_ fs.MkdirMetadataer = &Fs{}
  1642  	_ fs.Object          = &Object{}
  1643  	_ fs.Metadataer      = &Object{}
  1644  	_ fs.SetMetadataer   = &Object{}
  1645  	_ fs.Directory       = &Directory{}
  1646  	_ fs.SetModTimer     = &Directory{}
  1647  	_ fs.SetMetadataer   = &Directory{}
  1648  )