github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/backend/local/local.go (about) 1 // Package local provides a filesystem interface 2 package local 3 4 import ( 5 "bytes" 6 "context" 7 "errors" 8 "fmt" 9 "io" 10 "os" 11 "path" 12 "path/filepath" 13 "runtime" 14 "strings" 15 "sync" 16 "sync/atomic" 17 "time" 18 "unicode/utf8" 19 20 "github.com/rclone/rclone/fs" 21 "github.com/rclone/rclone/fs/accounting" 22 "github.com/rclone/rclone/fs/config" 23 "github.com/rclone/rclone/fs/config/configmap" 24 "github.com/rclone/rclone/fs/config/configstruct" 25 "github.com/rclone/rclone/fs/filter" 26 "github.com/rclone/rclone/fs/fserrors" 27 "github.com/rclone/rclone/fs/hash" 28 "github.com/rclone/rclone/lib/encoder" 29 "github.com/rclone/rclone/lib/file" 30 "github.com/rclone/rclone/lib/readers" 31 "golang.org/x/text/unicode/norm" 32 ) 33 34 // Constants 35 const devUnset = 0xdeadbeefcafebabe // a device id meaning it is unset 36 const linkSuffix = ".rclonelink" // The suffix added to a translated symbolic link 37 const useReadDir = (runtime.GOOS == "windows" || runtime.GOOS == "plan9") // these OSes read FileInfos directly 38 39 // timeType allows the user to choose what exactly ModTime() returns 40 type timeType = fs.Enum[timeTypeChoices] 41 42 const ( 43 mTime timeType = iota 44 aTime 45 bTime 46 cTime 47 ) 48 49 type timeTypeChoices struct{} 50 51 func (timeTypeChoices) Choices() []string { 52 return []string{ 53 mTime: "mtime", 54 aTime: "atime", 55 bTime: "btime", 56 cTime: "ctime", 57 } 58 } 59 60 // Register with Fs 61 func init() { 62 fsi := &fs.RegInfo{ 63 Name: "local", 64 Description: "Local Disk", 65 NewFs: NewFs, 66 CommandHelp: commandHelp, 67 MetadataInfo: &fs.MetadataInfo{ 68 System: systemMetadataInfo, 69 Help: `Depending on which OS is in use the local backend may return only some 70 of the system metadata. Setting system metadata is supported on all 71 OSes but setting user metadata is only supported on linux, freebsd, 72 netbsd, macOS and Solaris. It is **not** supported on Windows yet 73 ([see pkg/attrs#47](https://github.com/pkg/xattr/issues/47)). 74 75 User metadata is stored as extended attributes (which may not be 76 supported by all file systems) under the "user.*" prefix. 77 78 Metadata is supported on files and directories. 79 `, 80 }, 81 Options: []fs.Option{{ 82 Name: "nounc", 83 Help: "Disable UNC (long path names) conversion on Windows.", 84 Default: false, 85 Advanced: runtime.GOOS != "windows", 86 Examples: []fs.OptionExample{{ 87 Value: "true", 88 Help: "Disables long file names.", 89 }}, 90 }, { 91 Name: "copy_links", 92 Help: "Follow symlinks and copy the pointed to item.", 93 Default: false, 94 NoPrefix: true, 95 ShortOpt: "L", 96 Advanced: true, 97 }, { 98 Name: "links", 99 Help: "Translate symlinks to/from regular files with a '" + linkSuffix + "' extension.", 100 Default: false, 101 NoPrefix: true, 102 ShortOpt: "l", 103 Advanced: true, 104 }, { 105 Name: "skip_links", 106 Help: `Don't warn about skipped symlinks. 107 108 This flag disables warning messages on skipped symlinks or junction 109 points, as you explicitly acknowledge that they should be skipped.`, 110 Default: false, 111 NoPrefix: true, 112 Advanced: true, 113 }, { 114 Name: "zero_size_links", 115 Help: `Assume the Stat size of links is zero (and read them instead) (deprecated). 116 117 Rclone used to use the Stat size of links as the link size, but this fails in quite a few places: 118 119 - Windows 120 - On some virtual filesystems (such ash LucidLink) 121 - Android 122 123 So rclone now always reads the link. 124 `, 125 Default: false, 126 Advanced: true, 127 }, { 128 Name: "unicode_normalization", 129 Help: `Apply unicode NFC normalization to paths and filenames. 130 131 This flag can be used to normalize file names into unicode NFC form 132 that are read from the local filesystem. 133 134 Rclone does not normally touch the encoding of file names it reads from 135 the file system. 136 137 This can be useful when using macOS as it normally provides decomposed (NFD) 138 unicode which in some language (eg Korean) doesn't display properly on 139 some OSes. 140 141 Note that rclone compares filenames with unicode normalization in the sync 142 routine so this flag shouldn't normally be used.`, 143 Default: false, 144 Advanced: true, 145 }, { 146 Name: "no_check_updated", 147 Help: `Don't check to see if the files change during upload. 148 149 Normally rclone checks the size and modification time of files as they 150 are being uploaded and aborts with a message which starts "can't copy - 151 source file is being updated" if the file changes during upload. 152 153 However on some file systems this modification time check may fail (e.g. 154 [Glusterfs #2206](https://github.com/rclone/rclone/issues/2206)) so this 155 check can be disabled with this flag. 156 157 If this flag is set, rclone will use its best efforts to transfer a 158 file which is being updated. If the file is only having things 159 appended to it (e.g. a log) then rclone will transfer the log file with 160 the size it had the first time rclone saw it. 161 162 If the file is being modified throughout (not just appended to) then 163 the transfer may fail with a hash check failure. 164 165 In detail, once the file has had stat() called on it for the first 166 time we: 167 168 - Only transfer the size that stat gave 169 - Only checksum the size that stat gave 170 - Don't update the stat info for the file 171 172 **NB** do not use this flag on a Windows Volume Shadow (VSS). For some 173 unknown reason, files in a VSS sometimes show different sizes from the 174 directory listing (where the initial stat value comes from on Windows) 175 and when stat is called on them directly. Other copy tools always use 176 the direct stat value and setting this flag will disable that. 177 `, 178 Default: false, 179 Advanced: true, 180 }, { 181 Name: "one_file_system", 182 Help: "Don't cross filesystem boundaries (unix/macOS only).", 183 Default: false, 184 NoPrefix: true, 185 ShortOpt: "x", 186 Advanced: true, 187 }, { 188 Name: "case_sensitive", 189 Help: `Force the filesystem to report itself as case sensitive. 190 191 Normally the local backend declares itself as case insensitive on 192 Windows/macOS and case sensitive for everything else. Use this flag 193 to override the default choice.`, 194 Default: false, 195 Advanced: true, 196 }, { 197 Name: "case_insensitive", 198 Help: `Force the filesystem to report itself as case insensitive. 199 200 Normally the local backend declares itself as case insensitive on 201 Windows/macOS and case sensitive for everything else. Use this flag 202 to override the default choice.`, 203 Default: false, 204 Advanced: true, 205 }, { 206 Name: "no_preallocate", 207 Help: `Disable preallocation of disk space for transferred files. 208 209 Preallocation of disk space helps prevent filesystem fragmentation. 210 However, some virtual filesystem layers (such as Google Drive File 211 Stream) may incorrectly set the actual file size equal to the 212 preallocated space, causing checksum and file size checks to fail. 213 Use this flag to disable preallocation.`, 214 Default: false, 215 Advanced: true, 216 }, { 217 Name: "no_sparse", 218 Help: `Disable sparse files for multi-thread downloads. 219 220 On Windows platforms rclone will make sparse files when doing 221 multi-thread downloads. This avoids long pauses on large files where 222 the OS zeros the file. However sparse files may be undesirable as they 223 cause disk fragmentation and can be slow to work with.`, 224 Default: false, 225 Advanced: true, 226 }, { 227 Name: "no_set_modtime", 228 Help: `Disable setting modtime. 229 230 Normally rclone updates modification time of files after they are done 231 uploading. This can cause permissions issues on Linux platforms when 232 the user rclone is running as does not own the file uploaded, such as 233 when copying to a CIFS mount owned by another user. If this option is 234 enabled, rclone will no longer update the modtime after copying a file.`, 235 Default: false, 236 Advanced: true, 237 }, { 238 Name: "time_type", 239 Help: `Set what kind of time is returned. 240 241 Normally rclone does all operations on the mtime or Modification time. 242 243 If you set this flag then rclone will return the Modified time as whatever 244 you set here. So if you use "rclone lsl --local-time-type ctime" then 245 you will see ctimes in the listing. 246 247 If the OS doesn't support returning the time_type specified then rclone 248 will silently replace it with the modification time which all OSes support. 249 250 - mtime is supported by all OSes 251 - atime is supported on all OSes except: plan9, js 252 - btime is only supported on: Windows, macOS, freebsd, netbsd 253 - ctime is supported on all Oses except: Windows, plan9, js 254 255 Note that setting the time will still set the modified time so this is 256 only useful for reading. 257 `, 258 Default: mTime, 259 Advanced: true, 260 Examples: []fs.OptionExample{{ 261 Value: mTime.String(), 262 Help: "The last modification time.", 263 }, { 264 Value: aTime.String(), 265 Help: "The last access time.", 266 }, { 267 Value: bTime.String(), 268 Help: "The creation time.", 269 }, { 270 Value: cTime.String(), 271 Help: "The last status change time.", 272 }}, 273 }, { 274 Name: config.ConfigEncoding, 275 Help: config.ConfigEncodingHelp, 276 Advanced: true, 277 Default: encoder.OS, 278 }}, 279 } 280 fs.Register(fsi) 281 } 282 283 // Options defines the configuration for this backend 284 type Options struct { 285 FollowSymlinks bool `config:"copy_links"` 286 TranslateSymlinks bool `config:"links"` 287 SkipSymlinks bool `config:"skip_links"` 288 UTFNorm bool `config:"unicode_normalization"` 289 NoCheckUpdated bool `config:"no_check_updated"` 290 NoUNC bool `config:"nounc"` 291 OneFileSystem bool `config:"one_file_system"` 292 CaseSensitive bool `config:"case_sensitive"` 293 CaseInsensitive bool `config:"case_insensitive"` 294 NoPreAllocate bool `config:"no_preallocate"` 295 NoSparse bool `config:"no_sparse"` 296 NoSetModTime bool `config:"no_set_modtime"` 297 TimeType timeType `config:"time_type"` 298 Enc encoder.MultiEncoder `config:"encoding"` 299 } 300 301 // Fs represents a local filesystem rooted at root 302 type Fs struct { 303 name string // the name of the remote 304 root string // The root directory (OS path) 305 opt Options // parsed config options 306 features *fs.Features // optional features 307 dev uint64 // device number of root node 308 precisionOk sync.Once // Whether we need to read the precision 309 precision time.Duration // precision of local filesystem 310 warnedMu sync.Mutex // used for locking access to 'warned'. 311 warned map[string]struct{} // whether we have warned about this string 312 xattrSupported atomic.Int32 // whether xattrs are supported 313 314 // do os.Lstat or os.Stat 315 lstat func(name string) (os.FileInfo, error) 316 objectMetaMu sync.RWMutex // global lock for Object metadata 317 } 318 319 // Object represents a local filesystem object 320 type Object struct { 321 fs *Fs // The Fs this object is part of 322 remote string // The remote path (encoded path) 323 path string // The local path (OS path) 324 // When using these items the fs.objectMetaMu must be held 325 size int64 // file metadata - always present 326 mode os.FileMode 327 modTime time.Time 328 hashes map[hash.Type]string // Hashes 329 // these are read only and don't need the mutex held 330 translatedLink bool // Is this object a translated link 331 } 332 333 // Directory represents a local filesystem directory 334 type Directory struct { 335 Object 336 } 337 338 // ------------------------------------------------------------ 339 340 var ( 341 errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links") 342 errLinksNeedsSuffix = errors.New("need \"" + linkSuffix + "\" suffix to refer to symlink when using -l/--links") 343 ) 344 345 // NewFs constructs an Fs from the path 346 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { 347 // Parse config into Options struct 348 opt := new(Options) 349 err := configstruct.Set(m, opt) 350 if err != nil { 351 return nil, err 352 } 353 if opt.TranslateSymlinks && opt.FollowSymlinks { 354 return nil, errLinksAndCopyLinks 355 } 356 357 f := &Fs{ 358 name: name, 359 opt: *opt, 360 warned: make(map[string]struct{}), 361 dev: devUnset, 362 lstat: os.Lstat, 363 } 364 if xattrSupported { 365 f.xattrSupported.Store(1) 366 } 367 f.root = cleanRootPath(root, f.opt.NoUNC, f.opt.Enc) 368 f.features = (&fs.Features{ 369 CaseInsensitive: f.caseInsensitive(), 370 CanHaveEmptyDirectories: true, 371 IsLocal: true, 372 SlowHash: true, 373 ReadMetadata: true, 374 WriteMetadata: true, 375 ReadDirMetadata: true, 376 WriteDirMetadata: true, 377 WriteDirSetModTime: true, 378 UserDirMetadata: xattrSupported, // can only R/W general purpose metadata if xattrs are supported 379 DirModTimeUpdatesOnWrite: true, 380 UserMetadata: xattrSupported, // can only R/W general purpose metadata if xattrs are supported 381 FilterAware: true, 382 PartialUploads: true, 383 }).Fill(ctx, f) 384 if opt.FollowSymlinks { 385 f.lstat = os.Stat 386 } 387 388 // Check to see if this points to a file 389 fi, err := f.lstat(f.root) 390 if err == nil { 391 f.dev = readDevice(fi, f.opt.OneFileSystem) 392 } 393 // Check to see if this is a .rclonelink if not found 394 hasLinkSuffix := strings.HasSuffix(f.root, linkSuffix) 395 if hasLinkSuffix && opt.TranslateSymlinks && os.IsNotExist(err) { 396 fi, err = f.lstat(strings.TrimSuffix(f.root, linkSuffix)) 397 } 398 if err == nil && f.isRegular(fi.Mode()) { 399 // Handle the odd case, that a symlink was specified by name without the link suffix 400 if !hasLinkSuffix && opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 { 401 return nil, errLinksNeedsSuffix 402 } 403 // It is a file, so use the parent as the root 404 f.root = filepath.Dir(f.root) 405 // return an error with an fs which points to the parent 406 return f, fs.ErrorIsFile 407 } 408 return f, nil 409 } 410 411 // Determine whether a file is a 'regular' file, 412 // Symlinks are regular files, only if the TranslateSymlink 413 // option is in-effect 414 func (f *Fs) isRegular(mode os.FileMode) bool { 415 if !f.opt.TranslateSymlinks { 416 return mode.IsRegular() 417 } 418 419 // fi.Mode().IsRegular() tests that all mode bits are zero 420 // Since symlinks are accepted, test that all other bits are zero, 421 // except the symlink bit 422 return mode&os.ModeType&^os.ModeSymlink == 0 423 } 424 425 // Name of the remote (as passed into NewFs) 426 func (f *Fs) Name() string { 427 return f.name 428 } 429 430 // Root of the remote (as passed into NewFs) 431 func (f *Fs) Root() string { 432 return f.opt.Enc.ToStandardPath(filepath.ToSlash(f.root)) 433 } 434 435 // String converts this Fs to a string 436 func (f *Fs) String() string { 437 return fmt.Sprintf("Local file system at %s", f.Root()) 438 } 439 440 // Features returns the optional features of this Fs 441 func (f *Fs) Features() *fs.Features { 442 return f.features 443 } 444 445 // caseInsensitive returns whether the remote is case insensitive or not 446 func (f *Fs) caseInsensitive() bool { 447 if f.opt.CaseSensitive { 448 return false 449 } 450 if f.opt.CaseInsensitive { 451 return true 452 } 453 // FIXME not entirely accurate since you can have case 454 // sensitive Fses on darwin and case insensitive Fses on linux. 455 // Should probably check but that would involve creating a 456 // file in the remote to be most accurate which probably isn't 457 // desirable. 458 return runtime.GOOS == "windows" || runtime.GOOS == "darwin" 459 } 460 461 // translateLink checks whether the remote is a translated link 462 // and returns a new path, removing the suffix as needed, 463 // It also returns whether this is a translated link at all 464 // 465 // for regular files, localPath is returned unchanged 466 func translateLink(remote, localPath string) (newLocalPath string, isTranslatedLink bool) { 467 isTranslatedLink = strings.HasSuffix(remote, linkSuffix) 468 newLocalPath = strings.TrimSuffix(localPath, linkSuffix) 469 return newLocalPath, isTranslatedLink 470 } 471 472 // newObject makes a half completed Object 473 func (f *Fs) newObject(remote string) *Object { 474 translatedLink := false 475 localPath := f.localPath(remote) 476 477 if f.opt.TranslateSymlinks { 478 // Possibly receive a new name for localPath 479 localPath, translatedLink = translateLink(remote, localPath) 480 } 481 482 return &Object{ 483 fs: f, 484 remote: remote, 485 path: localPath, 486 translatedLink: translatedLink, 487 } 488 } 489 490 // Return an Object from a path 491 // 492 // May return nil if an error occurred 493 func (f *Fs) newObjectWithInfo(remote string, info os.FileInfo) (fs.Object, error) { 494 o := f.newObject(remote) 495 if info != nil { 496 o.setMetadata(info) 497 } else { 498 err := o.lstat() 499 if err != nil { 500 if os.IsNotExist(err) { 501 return nil, fs.ErrorObjectNotFound 502 } 503 if os.IsPermission(err) { 504 return nil, fs.ErrorPermissionDenied 505 } 506 return nil, err 507 } 508 // Handle the odd case, that a symlink was specified by name without the link suffix 509 if o.fs.opt.TranslateSymlinks && o.mode&os.ModeSymlink != 0 && !o.translatedLink { 510 return nil, fs.ErrorObjectNotFound 511 } 512 513 } 514 if o.mode.IsDir() { 515 return nil, fs.ErrorIsDir 516 } 517 return o, nil 518 } 519 520 // NewObject finds the Object at remote. If it can't be found 521 // it returns the error ErrorObjectNotFound. 522 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { 523 return f.newObjectWithInfo(remote, nil) 524 } 525 526 // Create new directory object from the info passed in 527 func (f *Fs) newDirectory(dir string, fi os.FileInfo) *Directory { 528 o := f.newObject(dir) 529 o.setMetadata(fi) 530 return &Directory{ 531 Object: *o, 532 } 533 } 534 535 // List the objects and directories in dir into entries. The 536 // entries can be returned in any order but should be for a 537 // complete directory. 538 // 539 // dir should be "" to list the root, and should not have 540 // trailing slashes. 541 // 542 // This should return ErrDirNotFound if the directory isn't 543 // found. 544 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { 545 filter, useFilter := filter.GetConfig(ctx), filter.GetUseFilter(ctx) 546 547 fsDirPath := f.localPath(dir) 548 _, err = os.Stat(fsDirPath) 549 if err != nil { 550 return nil, fs.ErrorDirNotFound 551 } 552 553 fd, err := os.Open(fsDirPath) 554 if err != nil { 555 isPerm := os.IsPermission(err) 556 err = fmt.Errorf("failed to open directory %q: %w", dir, err) 557 fs.Errorf(dir, "%v", err) 558 if isPerm { 559 _ = accounting.Stats(ctx).Error(fserrors.NoRetryError(err)) 560 err = nil // ignore error but fail sync 561 } 562 return nil, err 563 } 564 defer func() { 565 cerr := fd.Close() 566 if cerr != nil && err == nil { 567 err = fmt.Errorf("failed to close directory %q:: %w", dir, cerr) 568 } 569 }() 570 571 for { 572 var fis []os.FileInfo 573 if useReadDir { 574 // Windows and Plan9 read the directory entries with the stat information in which 575 // shouldn't fail because of unreadable entries. 576 fis, err = fd.Readdir(1024) 577 if err == io.EOF && len(fis) == 0 { 578 break 579 } 580 } else { 581 // For other OSes we read the names only (which shouldn't fail) then stat the 582 // individual ourselves so we can log errors but not fail the directory read. 583 var names []string 584 names, err = fd.Readdirnames(1024) 585 if err == io.EOF && len(names) == 0 { 586 break 587 } 588 if err == nil { 589 for _, name := range names { 590 namepath := filepath.Join(fsDirPath, name) 591 fi, fierr := os.Lstat(namepath) 592 if os.IsNotExist(fierr) { 593 // skip entry removed by a concurrent goroutine 594 continue 595 } 596 if fierr != nil { 597 // Don't report errors on any file names that are excluded 598 if useFilter { 599 newRemote := f.cleanRemote(dir, name) 600 if !filter.IncludeRemote(newRemote) { 601 continue 602 } 603 } 604 fierr = fmt.Errorf("failed to get info about directory entry %q: %w", namepath, fierr) 605 fs.Errorf(dir, "%v", fierr) 606 _ = accounting.Stats(ctx).Error(fserrors.NoRetryError(fierr)) // fail the sync 607 continue 608 } 609 fis = append(fis, fi) 610 } 611 } 612 } 613 if err != nil { 614 return nil, fmt.Errorf("failed to read directory entry: %w", err) 615 } 616 617 for _, fi := range fis { 618 name := fi.Name() 619 mode := fi.Mode() 620 newRemote := f.cleanRemote(dir, name) 621 // Follow symlinks if required 622 if f.opt.FollowSymlinks && (mode&os.ModeSymlink) != 0 { 623 localPath := filepath.Join(fsDirPath, name) 624 fi, err = os.Stat(localPath) 625 // Quietly skip errors on excluded files and directories 626 if err != nil && useFilter && !filter.IncludeRemote(newRemote) { 627 continue 628 } 629 if os.IsNotExist(err) || isCircularSymlinkError(err) { 630 // Skip bad symlinks and circular symlinks 631 err = fserrors.NoRetryError(fmt.Errorf("symlink: %w", err)) 632 fs.Errorf(newRemote, "Listing error: %v", err) 633 err = accounting.Stats(ctx).Error(err) 634 continue 635 } 636 if err != nil { 637 return nil, err 638 } 639 mode = fi.Mode() 640 } 641 if fi.IsDir() { 642 // Ignore directories which are symlinks. These are junction points under windows which 643 // are kind of a souped up symlink. Unix doesn't have directories which are symlinks. 644 if (mode&os.ModeSymlink) == 0 && f.dev == readDevice(fi, f.opt.OneFileSystem) { 645 d := f.newDirectory(newRemote, fi) 646 entries = append(entries, d) 647 } 648 } else { 649 // Check whether this link should be translated 650 if f.opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 { 651 newRemote += linkSuffix 652 } 653 // Don't include non directory if not included 654 // we leave directory filtering to the layer above 655 if useFilter && !filter.IncludeRemote(newRemote) { 656 continue 657 } 658 fso, err := f.newObjectWithInfo(newRemote, fi) 659 if err != nil { 660 return nil, err 661 } 662 if fso.Storable() { 663 entries = append(entries, fso) 664 } 665 } 666 } 667 } 668 return entries, nil 669 } 670 671 func (f *Fs) cleanRemote(dir, filename string) (remote string) { 672 if f.opt.UTFNorm { 673 filename = norm.NFC.String(filename) 674 } 675 remote = path.Join(dir, f.opt.Enc.ToStandardName(filename)) 676 677 if !utf8.ValidString(filename) { 678 f.warnedMu.Lock() 679 if _, ok := f.warned[remote]; !ok { 680 fs.Logf(f, "Replacing invalid UTF-8 characters in %q", remote) 681 f.warned[remote] = struct{}{} 682 } 683 f.warnedMu.Unlock() 684 } 685 return 686 } 687 688 func (f *Fs) localPath(name string) string { 689 return filepath.Join(f.root, filepath.FromSlash(f.opt.Enc.FromStandardPath(name))) 690 } 691 692 // Put the Object to the local filesystem 693 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 694 // Temporary Object under construction - info filled in by Update() 695 o := f.newObject(src.Remote()) 696 err := o.Update(ctx, in, src, options...) 697 if err != nil { 698 return nil, err 699 } 700 return o, nil 701 } 702 703 // PutStream uploads to the remote path with the modTime given of indeterminate size 704 func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { 705 return f.Put(ctx, in, src, options...) 706 } 707 708 // Mkdir creates the directory if it doesn't exist 709 func (f *Fs) Mkdir(ctx context.Context, dir string) error { 710 localPath := f.localPath(dir) 711 err := file.MkdirAll(localPath, 0777) 712 if err != nil { 713 return err 714 } 715 if dir == "" { 716 fi, err := f.lstat(localPath) 717 if err != nil { 718 return err 719 } 720 f.dev = readDevice(fi, f.opt.OneFileSystem) 721 } 722 return nil 723 } 724 725 // DirSetModTime sets the directory modtime for dir 726 func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error { 727 o := Object{ 728 fs: f, 729 remote: dir, 730 path: f.localPath(dir), 731 } 732 return o.SetModTime(ctx, modTime) 733 } 734 735 // MkdirMetadata makes the directory passed in as dir. 736 // 737 // It shouldn't return an error if it already exists. 738 // 739 // If the metadata is not nil it is set. 740 // 741 // It returns the directory that was created. 742 func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) { 743 // Find and or create the directory 744 localPath := f.localPath(dir) 745 fi, err := f.lstat(localPath) 746 if errors.Is(err, os.ErrNotExist) { 747 err := f.Mkdir(ctx, dir) 748 if err != nil { 749 return nil, fmt.Errorf("mkdir metadata: failed make directory: %w", err) 750 } 751 fi, err = f.lstat(localPath) 752 if err != nil { 753 return nil, fmt.Errorf("mkdir metadata: failed to read info: %w", err) 754 } 755 } else if err != nil { 756 return nil, err 757 } 758 759 // Create directory object 760 d := f.newDirectory(dir, fi) 761 762 // Set metadata on the directory object if provided 763 if metadata != nil { 764 err = d.writeMetadata(metadata) 765 if err != nil { 766 return nil, fmt.Errorf("failed to set metadata on directory: %w", err) 767 } 768 // Re-read info now we have finished setting stuff 769 err = d.lstat() 770 if err != nil { 771 return nil, fmt.Errorf("mkdir metadata: failed to re-read info: %w", err) 772 } 773 } 774 return d, nil 775 } 776 777 // Rmdir removes the directory 778 // 779 // If it isn't empty it will return an error 780 func (f *Fs) Rmdir(ctx context.Context, dir string) error { 781 localPath := f.localPath(dir) 782 if fi, err := os.Stat(localPath); err != nil { 783 return err 784 } else if !fi.IsDir() { 785 return fs.ErrorIsFile 786 } 787 return os.Remove(localPath) 788 } 789 790 // Precision of the file system 791 func (f *Fs) Precision() (precision time.Duration) { 792 if f.opt.NoSetModTime { 793 return fs.ModTimeNotSupported 794 } 795 796 f.precisionOk.Do(func() { 797 f.precision = f.readPrecision() 798 }) 799 return f.precision 800 } 801 802 // Read the precision 803 func (f *Fs) readPrecision() (precision time.Duration) { 804 // Default precision of 1s 805 precision = time.Second 806 807 // Create temporary file and test it 808 fd, err := os.CreateTemp("", "rclone") 809 if err != nil { 810 // If failed return 1s 811 // fmt.Println("Failed to create temp file", err) 812 return time.Second 813 } 814 path := fd.Name() 815 // fmt.Println("Created temp file", path) 816 err = fd.Close() 817 if err != nil { 818 return time.Second 819 } 820 821 // Delete it on return 822 defer func() { 823 // fmt.Println("Remove temp file") 824 _ = os.Remove(path) // ignore error 825 }() 826 827 // Find the minimum duration we can detect 828 for duration := time.Duration(1); duration < time.Second; duration *= 10 { 829 // Current time with delta 830 t := time.Unix(time.Now().Unix(), int64(duration)) 831 err := os.Chtimes(path, t, t) 832 if err != nil { 833 // fmt.Println("Failed to Chtimes", err) 834 break 835 } 836 837 // Read the actual time back 838 fi, err := os.Stat(path) 839 if err != nil { 840 // fmt.Println("Failed to Stat", err) 841 break 842 } 843 844 // If it matches - have found the precision 845 // fmt.Println("compare", fi.ModTime(ctx), t) 846 if fi.ModTime().Equal(t) { 847 // fmt.Println("Precision detected as", duration) 848 return duration 849 } 850 } 851 return 852 } 853 854 // Move src to this remote using server-side move operations. 855 // 856 // This is stored with the remote path given. 857 // 858 // It returns the destination Object and a possible error. 859 // 860 // Will only be called if src.Fs().Name() == f.Name() 861 // 862 // If it isn't possible then return fs.ErrorCantMove 863 func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { 864 srcObj, ok := src.(*Object) 865 if !ok { 866 fs.Debugf(src, "Can't move - not same remote type") 867 return nil, fs.ErrorCantMove 868 } 869 870 // Temporary Object under construction 871 dstObj := f.newObject(remote) 872 dstObj.fs.objectMetaMu.RLock() 873 dstObjMode := dstObj.mode 874 dstObj.fs.objectMetaMu.RUnlock() 875 876 // Check it is a file if it exists 877 err := dstObj.lstat() 878 if os.IsNotExist(err) { 879 // OK 880 } else if err != nil { 881 return nil, err 882 } else if !dstObj.fs.isRegular(dstObjMode) { 883 // It isn't a file 884 return nil, errors.New("can't move file onto non-file") 885 } 886 887 // Create destination 888 err = dstObj.mkdirAll() 889 if err != nil { 890 return nil, err 891 } 892 893 // Fetch metadata if --metadata is in use 894 meta, err := fs.GetMetadataOptions(ctx, f, src, fs.MetadataAsOpenOptions(ctx)) 895 if err != nil { 896 return nil, fmt.Errorf("move: failed to read metadata: %w", err) 897 } 898 899 // Do the move 900 err = os.Rename(srcObj.path, dstObj.path) 901 if os.IsNotExist(err) { 902 // race condition, source was deleted in the meantime 903 return nil, err 904 } else if os.IsPermission(err) { 905 // not enough rights to write to dst 906 return nil, err 907 } else if err != nil { 908 // not quite clear, but probably trying to move a file across file system 909 // boundaries. Copying might still work. 910 fs.Debugf(src, "Can't move: %v: trying copy", err) 911 return nil, fs.ErrorCantMove 912 } 913 914 // Set metadata if --metadata is in use 915 err = dstObj.writeMetadata(meta) 916 if err != nil { 917 return nil, fmt.Errorf("move: failed to set metadata: %w", err) 918 } 919 920 // Update the info 921 err = dstObj.lstat() 922 if err != nil { 923 return nil, err 924 } 925 926 return dstObj, nil 927 } 928 929 // DirMove moves src, srcRemote to this remote at dstRemote 930 // using server-side move operations. 931 // 932 // Will only be called if src.Fs().Name() == f.Name() 933 // 934 // If it isn't possible then return fs.ErrorCantDirMove 935 // 936 // If destination exists then return fs.ErrorDirExists 937 func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error { 938 srcFs, ok := src.(*Fs) 939 if !ok { 940 fs.Debugf(srcFs, "Can't move directory - not same remote type") 941 return fs.ErrorCantDirMove 942 } 943 srcPath := srcFs.localPath(srcRemote) 944 dstPath := f.localPath(dstRemote) 945 946 // Check if destination exists 947 _, err := os.Lstat(dstPath) 948 if !os.IsNotExist(err) { 949 return fs.ErrorDirExists 950 } 951 952 // Create parent of destination 953 dstParentPath := filepath.Dir(dstPath) 954 err = file.MkdirAll(dstParentPath, 0777) 955 if err != nil { 956 return err 957 } 958 959 // Do the move 960 err = os.Rename(srcPath, dstPath) 961 if os.IsNotExist(err) { 962 // race condition, source was deleted in the meantime 963 return err 964 } else if os.IsPermission(err) { 965 // not enough rights to write to dst 966 return err 967 } else if err != nil { 968 // not quite clear, but probably trying to move directory across file system 969 // boundaries. Copying might still work. 970 fs.Debugf(src, "Can't move dir: %v: trying copy", err) 971 return fs.ErrorCantDirMove 972 } 973 return nil 974 } 975 976 // Hashes returns the supported hash sets. 977 func (f *Fs) Hashes() hash.Set { 978 return hash.Supported() 979 } 980 981 var commandHelp = []fs.CommandHelp{ 982 { 983 Name: "noop", 984 Short: "A null operation for testing backend commands", 985 Long: `This is a test command which has some options 986 you can try to change the output.`, 987 Opts: map[string]string{ 988 "echo": "echo the input arguments", 989 "error": "return an error based on option value", 990 }, 991 }, 992 } 993 994 // Command the backend to run a named command 995 // 996 // The command run is name 997 // args may be used to read arguments from 998 // opts may be used to read optional arguments from 999 // 1000 // The result should be capable of being JSON encoded 1001 // If it is a string or a []string it will be shown to the user 1002 // otherwise it will be JSON encoded and shown to the user like that 1003 func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error) { 1004 switch name { 1005 case "noop": 1006 if txt, ok := opt["error"]; ok { 1007 if txt == "" { 1008 txt = "unspecified error" 1009 } 1010 return nil, errors.New(txt) 1011 } 1012 if _, ok := opt["echo"]; ok { 1013 out := map[string]interface{}{} 1014 out["name"] = name 1015 out["arg"] = arg 1016 out["opt"] = opt 1017 return out, nil 1018 } 1019 return nil, nil 1020 default: 1021 return nil, fs.ErrorCommandNotFound 1022 } 1023 } 1024 1025 // ------------------------------------------------------------ 1026 1027 // Fs returns the parent Fs 1028 func (o *Object) Fs() fs.Info { 1029 return o.fs 1030 } 1031 1032 // Return a string version 1033 func (o *Object) String() string { 1034 if o == nil { 1035 return "<nil>" 1036 } 1037 return o.remote 1038 } 1039 1040 // Remote returns the remote path 1041 func (o *Object) Remote() string { 1042 return o.remote 1043 } 1044 1045 // Hash returns the requested hash of a file as a lowercase hex string 1046 func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) { 1047 // Check that the underlying file hasn't changed 1048 o.fs.objectMetaMu.RLock() 1049 oldtime := o.modTime 1050 oldsize := o.size 1051 o.fs.objectMetaMu.RUnlock() 1052 err := o.lstat() 1053 var changed bool 1054 if err != nil { 1055 if errors.Is(err, os.ErrNotExist) { 1056 // If file not found then we assume any accumulated 1057 // hashes are OK - this will error on Open 1058 changed = true 1059 } else { 1060 return "", fmt.Errorf("hash: failed to stat: %w", err) 1061 } 1062 } else { 1063 o.fs.objectMetaMu.RLock() 1064 changed = !o.modTime.Equal(oldtime) || oldsize != o.size 1065 o.fs.objectMetaMu.RUnlock() 1066 } 1067 1068 o.fs.objectMetaMu.RLock() 1069 hashValue, hashFound := o.hashes[r] 1070 o.fs.objectMetaMu.RUnlock() 1071 1072 if changed || !hashFound { 1073 var in io.ReadCloser 1074 1075 if !o.translatedLink { 1076 var fd *os.File 1077 fd, err = file.Open(o.path) 1078 if fd != nil { 1079 in = newFadviseReadCloser(o, fd, 0, 0) 1080 } 1081 } else { 1082 in, err = o.openTranslatedLink(0, -1) 1083 } 1084 // If not checking for updates, only read size given 1085 if o.fs.opt.NoCheckUpdated { 1086 in = readers.NewLimitedReadCloser(in, o.size) 1087 } 1088 if err != nil { 1089 return "", fmt.Errorf("hash: failed to open: %w", err) 1090 } 1091 var hashes map[hash.Type]string 1092 hashes, err = hash.StreamTypes(readers.NewContextReader(ctx, in), hash.NewHashSet(r)) 1093 closeErr := in.Close() 1094 if err != nil { 1095 return "", fmt.Errorf("hash: failed to read: %w", err) 1096 } 1097 if closeErr != nil { 1098 return "", fmt.Errorf("hash: failed to close: %w", closeErr) 1099 } 1100 hashValue = hashes[r] 1101 o.fs.objectMetaMu.Lock() 1102 if o.hashes == nil { 1103 o.hashes = hashes 1104 } else { 1105 o.hashes[r] = hashValue 1106 } 1107 o.fs.objectMetaMu.Unlock() 1108 } 1109 return hashValue, nil 1110 } 1111 1112 // Size returns the size of an object in bytes 1113 func (o *Object) Size() int64 { 1114 o.fs.objectMetaMu.RLock() 1115 defer o.fs.objectMetaMu.RUnlock() 1116 return o.size 1117 } 1118 1119 // ModTime returns the modification time of the object 1120 func (o *Object) ModTime(ctx context.Context) time.Time { 1121 o.fs.objectMetaMu.RLock() 1122 defer o.fs.objectMetaMu.RUnlock() 1123 return o.modTime 1124 } 1125 1126 // Set the atime and ltime of the object 1127 func (o *Object) setTimes(atime, mtime time.Time) (err error) { 1128 if o.translatedLink { 1129 err = lChtimes(o.path, atime, mtime) 1130 } else { 1131 err = os.Chtimes(o.path, atime, mtime) 1132 } 1133 return err 1134 } 1135 1136 // SetModTime sets the modification time of the local fs object 1137 func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { 1138 if o.fs.opt.NoSetModTime { 1139 return nil 1140 } 1141 err := o.setTimes(modTime, modTime) 1142 if err != nil { 1143 return err 1144 } 1145 // Re-read metadata 1146 return o.lstat() 1147 } 1148 1149 // Storable returns a boolean showing if this object is storable 1150 func (o *Object) Storable() bool { 1151 o.fs.objectMetaMu.RLock() 1152 mode := o.mode 1153 o.fs.objectMetaMu.RUnlock() 1154 if mode&os.ModeSymlink != 0 && !o.fs.opt.TranslateSymlinks { 1155 if !o.fs.opt.SkipSymlinks { 1156 fs.Logf(o, "Can't follow symlink without -L/--copy-links") 1157 } 1158 return false 1159 } else if mode&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 { 1160 fs.Logf(o, "Can't transfer non file/directory") 1161 return false 1162 } else if mode&os.ModeDir != 0 { 1163 // fs.Debugf(o, "Skipping directory") 1164 return false 1165 } 1166 return true 1167 } 1168 1169 // localOpenFile wraps an io.ReadCloser and updates the md5sum of the 1170 // object that is read 1171 type localOpenFile struct { 1172 o *Object // object that is open 1173 in io.ReadCloser // handle we are wrapping 1174 hash *hash.MultiHasher // currently accumulating hashes 1175 fd *os.File // file object reference 1176 } 1177 1178 // Read bytes from the object - see io.Reader 1179 func (file *localOpenFile) Read(p []byte) (n int, err error) { 1180 if !file.o.fs.opt.NoCheckUpdated { 1181 // Check if file has the same size and modTime 1182 fi, err := file.fd.Stat() 1183 if err != nil { 1184 return 0, fmt.Errorf("can't read status of source file while transferring: %w", err) 1185 } 1186 file.o.fs.objectMetaMu.RLock() 1187 oldtime := file.o.modTime 1188 oldsize := file.o.size 1189 file.o.fs.objectMetaMu.RUnlock() 1190 if oldsize != fi.Size() { 1191 return 0, fserrors.NoLowLevelRetryError(fmt.Errorf("can't copy - source file is being updated (size changed from %d to %d)", oldsize, fi.Size())) 1192 } 1193 if !oldtime.Equal(readTime(file.o.fs.opt.TimeType, fi)) { 1194 return 0, fserrors.NoLowLevelRetryError(fmt.Errorf("can't copy - source file is being updated (mod time changed from %v to %v)", oldtime, fi.ModTime())) 1195 } 1196 } 1197 1198 n, err = file.in.Read(p) 1199 if n > 0 { 1200 // Hash routines never return an error 1201 _, _ = file.hash.Write(p[:n]) 1202 } 1203 return 1204 } 1205 1206 // Close the object and update the hashes 1207 func (file *localOpenFile) Close() (err error) { 1208 err = file.in.Close() 1209 if err == nil { 1210 if file.hash.Size() == file.o.Size() { 1211 file.o.fs.objectMetaMu.Lock() 1212 file.o.hashes = file.hash.Sums() 1213 file.o.fs.objectMetaMu.Unlock() 1214 } 1215 } 1216 return err 1217 } 1218 1219 // Returns a ReadCloser() object that contains the contents of a symbolic link 1220 func (o *Object) openTranslatedLink(offset, limit int64) (lrc io.ReadCloser, err error) { 1221 // Read the link and return the destination it as the contents of the object 1222 linkdst, err := os.Readlink(o.path) 1223 if err != nil { 1224 return nil, err 1225 } 1226 return readers.NewLimitedReadCloser(io.NopCloser(strings.NewReader(linkdst[offset:])), limit), nil 1227 } 1228 1229 // Open an object for read 1230 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { 1231 var offset, limit int64 = 0, -1 1232 var hasher *hash.MultiHasher 1233 for _, option := range options { 1234 switch x := option.(type) { 1235 case *fs.SeekOption: 1236 offset = x.Offset 1237 case *fs.RangeOption: 1238 offset, limit = x.Decode(o.Size()) 1239 case *fs.HashesOption: 1240 if x.Hashes.Count() > 0 { 1241 hasher, err = hash.NewMultiHasherTypes(x.Hashes) 1242 if err != nil { 1243 return nil, err 1244 } 1245 } 1246 default: 1247 if option.Mandatory() { 1248 fs.Logf(o, "Unsupported mandatory option: %v", option) 1249 } 1250 } 1251 } 1252 1253 // Update the file info before we start reading 1254 err = o.lstat() 1255 if err != nil { 1256 return nil, err 1257 } 1258 1259 // If not checking updated then limit to current size. This means if 1260 // file is being extended, readers will read a o.Size() bytes rather 1261 // than the new size making for a consistent upload. 1262 if limit < 0 && o.fs.opt.NoCheckUpdated { 1263 limit = o.size 1264 } 1265 1266 // Handle a translated link 1267 if o.translatedLink { 1268 return o.openTranslatedLink(offset, limit) 1269 } 1270 1271 fd, err := file.Open(o.path) 1272 if err != nil { 1273 return 1274 } 1275 wrappedFd := readers.NewLimitedReadCloser(newFadviseReadCloser(o, fd, offset, limit), limit) 1276 if offset != 0 { 1277 // seek the object 1278 _, err = fd.Seek(offset, io.SeekStart) 1279 // don't attempt to make checksums 1280 return wrappedFd, err 1281 } 1282 if hasher == nil { 1283 // no need to wrap since we don't need checksums 1284 return wrappedFd, nil 1285 } 1286 // Update the hashes as we go along 1287 in = &localOpenFile{ 1288 o: o, 1289 in: wrappedFd, 1290 hash: hasher, 1291 fd: fd, 1292 } 1293 return in, nil 1294 } 1295 1296 // mkdirAll makes all the directories needed to store the object 1297 func (o *Object) mkdirAll() error { 1298 dir := filepath.Dir(o.path) 1299 return file.MkdirAll(dir, 0777) 1300 } 1301 1302 type nopWriterCloser struct { 1303 *bytes.Buffer 1304 } 1305 1306 func (nwc nopWriterCloser) Close() error { 1307 // noop 1308 return nil 1309 } 1310 1311 // Update the object from in with modTime and size 1312 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { 1313 var out io.WriteCloser 1314 var hasher *hash.MultiHasher 1315 1316 for _, option := range options { 1317 switch x := option.(type) { 1318 case *fs.HashesOption: 1319 if x.Hashes.Count() > 0 { 1320 hasher, err = hash.NewMultiHasherTypes(x.Hashes) 1321 if err != nil { 1322 return err 1323 } 1324 } 1325 } 1326 } 1327 1328 err = o.mkdirAll() 1329 if err != nil { 1330 return err 1331 } 1332 1333 // Wipe hashes before update 1334 o.clearHashCache() 1335 1336 var symlinkData bytes.Buffer 1337 // If the object is a regular file, create it. 1338 // If it is a translated link, just read in the contents, and 1339 // then create a symlink 1340 if !o.translatedLink { 1341 f, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) 1342 if err != nil { 1343 if runtime.GOOS == "windows" && os.IsPermission(err) { 1344 // If permission denied on Windows might be trying to update a 1345 // hidden file, in which case try opening without CREATE 1346 // See: https://stackoverflow.com/questions/13215716/ioerror-errno-13-permission-denied-when-trying-to-open-hidden-file-in-w-mod 1347 f, err = file.OpenFile(o.path, os.O_WRONLY|os.O_TRUNC, 0666) 1348 if err != nil { 1349 return err 1350 } 1351 } else { 1352 return err 1353 } 1354 } 1355 if !o.fs.opt.NoPreAllocate { 1356 // Pre-allocate the file for performance reasons 1357 err = file.PreAllocate(src.Size(), f) 1358 if err != nil { 1359 fs.Debugf(o, "Failed to pre-allocate: %v", err) 1360 if err == file.ErrDiskFull { 1361 _ = f.Close() 1362 return err 1363 } 1364 } 1365 } 1366 out = f 1367 } else { 1368 out = nopWriterCloser{&symlinkData} 1369 } 1370 1371 // Calculate the hash of the object we are reading as we go along 1372 if hasher != nil { 1373 in = io.TeeReader(in, hasher) 1374 } 1375 1376 _, err = io.Copy(out, in) 1377 closeErr := out.Close() 1378 if err == nil { 1379 err = closeErr 1380 } 1381 1382 if o.translatedLink { 1383 if err == nil { 1384 // Remove any current symlink or file, if one exists 1385 if _, err := os.Lstat(o.path); err == nil { 1386 if removeErr := os.Remove(o.path); removeErr != nil { 1387 fs.Errorf(o, "Failed to remove previous file: %v", removeErr) 1388 return removeErr 1389 } 1390 } 1391 // Use the contents for the copied object to create a symlink 1392 err = os.Symlink(symlinkData.String(), o.path) 1393 } 1394 1395 // only continue if symlink creation succeeded 1396 if err != nil { 1397 return err 1398 } 1399 } 1400 1401 if err != nil { 1402 fs.Logf(o, "Removing partially written file on error: %v", err) 1403 if removeErr := os.Remove(o.path); removeErr != nil { 1404 fs.Errorf(o, "Failed to remove partially written file: %v", removeErr) 1405 } 1406 return err 1407 } 1408 1409 // All successful so update the hashes 1410 if hasher != nil { 1411 o.fs.objectMetaMu.Lock() 1412 o.hashes = hasher.Sums() 1413 o.fs.objectMetaMu.Unlock() 1414 } 1415 1416 // Set the mtime 1417 err = o.SetModTime(ctx, src.ModTime(ctx)) 1418 if err != nil { 1419 return err 1420 } 1421 1422 // Fetch and set metadata if --metadata is in use 1423 meta, err := fs.GetMetadataOptions(ctx, o.fs, src, options) 1424 if err != nil { 1425 return fmt.Errorf("failed to read metadata from source object: %w", err) 1426 } 1427 err = o.writeMetadata(meta) 1428 if err != nil { 1429 return fmt.Errorf("failed to set metadata: %w", err) 1430 } 1431 1432 // ReRead info now that we have finished 1433 return o.lstat() 1434 } 1435 1436 var sparseWarning sync.Once 1437 1438 // OpenWriterAt opens with a handle for random access writes 1439 // 1440 // Pass in the remote desired and the size if known. 1441 // 1442 // It truncates any existing object 1443 func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) { 1444 // Temporary Object under construction 1445 o := f.newObject(remote) 1446 1447 err := o.mkdirAll() 1448 if err != nil { 1449 return nil, err 1450 } 1451 1452 if o.translatedLink { 1453 return nil, errors.New("can't open a symlink for random writing") 1454 } 1455 1456 out, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) 1457 if err != nil { 1458 return nil, err 1459 } 1460 // Pre-allocate the file for performance reasons 1461 if !f.opt.NoPreAllocate { 1462 err = file.PreAllocate(size, out) 1463 if err != nil { 1464 fs.Debugf(o, "Failed to pre-allocate: %v", err) 1465 } 1466 } 1467 if !f.opt.NoSparse && file.SetSparseImplemented { 1468 sparseWarning.Do(func() { 1469 fs.Infof(nil, "Writing sparse files: use --local-no-sparse or --multi-thread-streams 0 to disable") 1470 }) 1471 // Set the file to be a sparse file (important on Windows) 1472 err = file.SetSparse(out) 1473 if err != nil { 1474 fs.Errorf(o, "Failed to set sparse: %v", err) 1475 } 1476 } 1477 1478 return out, nil 1479 } 1480 1481 // setMetadata sets the file info from the os.FileInfo passed in 1482 func (o *Object) setMetadata(info os.FileInfo) { 1483 // if not checking updated then don't update the stat 1484 if o.fs.opt.NoCheckUpdated && !o.modTime.IsZero() { 1485 return 1486 } 1487 o.fs.objectMetaMu.Lock() 1488 o.size = info.Size() 1489 o.modTime = readTime(o.fs.opt.TimeType, info) 1490 o.mode = info.Mode() 1491 o.fs.objectMetaMu.Unlock() 1492 // Read the size of the link. 1493 // 1494 // The value in info.Size() is not always correct 1495 // - Windows links read as 0 size 1496 // - Some virtual filesystems (such ash LucidLink) links read as 0 size 1497 // - Android - some versions the links are larger than readlink suggests 1498 if o.translatedLink { 1499 linkdst, err := os.Readlink(o.path) 1500 if err != nil { 1501 fs.Errorf(o, "Failed to read link size: %v", err) 1502 } else { 1503 o.size = int64(len(linkdst)) 1504 } 1505 } 1506 } 1507 1508 // clearHashCache wipes any cached hashes for the object 1509 func (o *Object) clearHashCache() { 1510 o.fs.objectMetaMu.Lock() 1511 o.hashes = nil 1512 o.fs.objectMetaMu.Unlock() 1513 } 1514 1515 // Stat an Object into info 1516 func (o *Object) lstat() error { 1517 info, err := o.fs.lstat(o.path) 1518 if err == nil { 1519 o.setMetadata(info) 1520 } 1521 return err 1522 } 1523 1524 // Remove an object 1525 func (o *Object) Remove(ctx context.Context) error { 1526 o.clearHashCache() 1527 return remove(o.path) 1528 } 1529 1530 // Metadata returns metadata for an object 1531 // 1532 // It should return nil if there is no Metadata 1533 func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) { 1534 metadata, err = o.getXattr() 1535 if err != nil { 1536 return nil, err 1537 } 1538 err = o.readMetadataFromFile(&metadata) 1539 if err != nil { 1540 return nil, err 1541 } 1542 return metadata, nil 1543 } 1544 1545 // Write the metadata on the object 1546 func (o *Object) writeMetadata(metadata fs.Metadata) (err error) { 1547 err = o.setXattr(metadata) 1548 if err != nil { 1549 return err 1550 } 1551 err = o.writeMetadataToFile(metadata) 1552 if err != nil { 1553 return err 1554 } 1555 return err 1556 } 1557 1558 // SetMetadata sets metadata for an Object 1559 // 1560 // It should return fs.ErrorNotImplemented if it can't set metadata 1561 func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error { 1562 err := o.writeMetadata(metadata) 1563 if err != nil { 1564 return fmt.Errorf("SetMetadata failed on Object: %w", err) 1565 } 1566 // Re-read info now we have finished setting stuff 1567 return o.lstat() 1568 } 1569 1570 func cleanRootPath(s string, noUNC bool, enc encoder.MultiEncoder) string { 1571 if runtime.GOOS != "windows" || !strings.HasPrefix(s, "\\") { 1572 if !filepath.IsAbs(s) { 1573 s2, err := filepath.Abs(s) 1574 if err == nil { 1575 s = s2 1576 } 1577 } else { 1578 s = filepath.Clean(s) 1579 } 1580 } 1581 if runtime.GOOS == "windows" { 1582 s = filepath.ToSlash(s) 1583 vol := filepath.VolumeName(s) 1584 if vol == `\\?` && len(s) >= 6 { 1585 // `\\?\C:` 1586 vol = s[:6] 1587 } 1588 s = vol + enc.FromStandardPath(s[len(vol):]) 1589 s = filepath.FromSlash(s) 1590 if !noUNC { 1591 // Convert to UNC 1592 s = file.UNCPath(s) 1593 } 1594 return s 1595 } 1596 s = enc.FromStandardPath(s) 1597 return s 1598 } 1599 1600 // Items returns the count of items in this directory or this 1601 // directory and subdirectories if known, -1 for unknown 1602 func (d *Directory) Items() int64 { 1603 return -1 1604 } 1605 1606 // ID returns the internal ID of this directory if known, or 1607 // "" otherwise 1608 func (d *Directory) ID() string { 1609 return "" 1610 } 1611 1612 // SetMetadata sets metadata for a Directory 1613 // 1614 // It should return fs.ErrorNotImplemented if it can't set metadata 1615 func (d *Directory) SetMetadata(ctx context.Context, metadata fs.Metadata) error { 1616 err := d.writeMetadata(metadata) 1617 if err != nil { 1618 return fmt.Errorf("SetMetadata failed on Directory: %w", err) 1619 } 1620 // Re-read info now we have finished setting stuff 1621 return d.lstat() 1622 } 1623 1624 // Hash does nothing on a directory 1625 // 1626 // This method is implemented with the incorrect type signature to 1627 // stop the Directory type asserting to fs.Object or fs.ObjectInfo 1628 func (d *Directory) Hash() { 1629 // Does nothing 1630 } 1631 1632 // Check the interfaces are satisfied 1633 var ( 1634 _ fs.Fs = &Fs{} 1635 _ fs.PutStreamer = &Fs{} 1636 _ fs.Mover = &Fs{} 1637 _ fs.DirMover = &Fs{} 1638 _ fs.Commander = &Fs{} 1639 _ fs.OpenWriterAter = &Fs{} 1640 _ fs.DirSetModTimer = &Fs{} 1641 _ fs.MkdirMetadataer = &Fs{} 1642 _ fs.Object = &Object{} 1643 _ fs.Metadataer = &Object{} 1644 _ fs.SetMetadataer = &Object{} 1645 _ fs.Directory = &Directory{} 1646 _ fs.SetModTimer = &Directory{} 1647 _ fs.SetMetadataer = &Directory{} 1648 )