github.com/10XDev/rclone@v1.52.3-0.20200626220027-16af9ab76b2a/vfs/dir.go (about) 1 package vfs 2 3 import ( 4 "context" 5 "os" 6 "path" 7 "sort" 8 "strings" 9 "sync" 10 "time" 11 12 "github.com/pkg/errors" 13 "github.com/rclone/rclone/fs" 14 "github.com/rclone/rclone/fs/dirtree" 15 "github.com/rclone/rclone/fs/list" 16 "github.com/rclone/rclone/fs/log" 17 "github.com/rclone/rclone/fs/operations" 18 "github.com/rclone/rclone/fs/walk" 19 "github.com/rclone/rclone/vfs/vfscommon" 20 ) 21 22 // Dir represents a directory entry 23 type Dir struct { 24 vfs *VFS // read only 25 inode uint64 // read only: inode number 26 f fs.Fs // read only 27 28 mu sync.RWMutex // protects the following 29 parent *Dir // parent, nil for root 30 path string 31 modTime time.Time 32 entry fs.Directory 33 read time.Time // time directory entry last read 34 items map[string]Node // directory entries - can be empty but not nil 35 sys interface{} // user defined info to be attached here 36 } 37 38 func newDir(vfs *VFS, f fs.Fs, parent *Dir, fsDir fs.Directory) *Dir { 39 return &Dir{ 40 vfs: vfs, 41 f: f, 42 parent: parent, 43 entry: fsDir, 44 path: fsDir.Remote(), 45 modTime: fsDir.ModTime(context.TODO()), 46 inode: newInode(), 47 items: make(map[string]Node), 48 } 49 } 50 51 // String converts it to printablee 52 func (d *Dir) String() string { 53 if d == nil { 54 return "<nil *Dir>" 55 } 56 d.mu.RLock() 57 defer d.mu.RUnlock() 58 return d.path + "/" 59 } 60 61 // IsFile returns false for Dir - satisfies Node interface 62 func (d *Dir) IsFile() bool { 63 return false 64 } 65 66 // IsDir returns true for Dir - satisfies Node interface 67 func (d *Dir) IsDir() bool { 68 return true 69 } 70 71 // Mode bits of the directory - satisfies Node interface 72 func (d *Dir) Mode() (mode os.FileMode) { 73 return d.vfs.Opt.DirPerms 74 } 75 76 // Name (base) of the directory - satisfies Node interface 77 func (d *Dir) Name() (name string) { 78 d.mu.RLock() 79 name = path.Base(d.path) 80 d.mu.RUnlock() 81 if name == "." { 82 name = "/" 83 } 84 return name 85 } 86 87 // Path of the directory - satisfies Node interface 88 func (d *Dir) Path() (name string) { 89 d.mu.RLock() 90 defer d.mu.RUnlock() 91 return d.path 92 } 93 94 // Sys returns underlying data source (can be nil) - satisfies Node interface 95 func (d *Dir) Sys() interface{} { 96 d.mu.RLock() 97 defer d.mu.RUnlock() 98 return d.sys 99 } 100 101 // SetSys sets the underlying data source (can be nil) - satisfies Node interface 102 func (d *Dir) SetSys(x interface{}) { 103 d.mu.Lock() 104 d.sys = x 105 d.mu.Unlock() 106 } 107 108 // Inode returns the inode number - satisfies Node interface 109 func (d *Dir) Inode() uint64 { 110 return d.inode 111 } 112 113 // Node returns the Node assocuated with this - satisfies Noder interface 114 func (d *Dir) Node() Node { 115 return d 116 } 117 118 // forgetDirPath clears the cache for itself and all subdirectories if 119 // they match the given path. The path is specified relative from the 120 // directory it is called from. 121 // 122 // It does not invalidate or clear the cache of the parent directory. 123 func (d *Dir) forgetDirPath(relativePath string) { 124 if dir := d.cachedDir(relativePath); dir != nil { 125 dir.walk(func(dir *Dir) { 126 // this is called with the mutex held 127 fs.Debugf(dir.path, "forgetting directory cache") 128 dir.read = time.Time{} 129 dir.items = make(map[string]Node) 130 }) 131 } 132 } 133 134 // ForgetAll ensures the directory and all its children are purged 135 // from the cache. 136 // 137 // It does not invalidate or clear the cache of the parent directory. 138 func (d *Dir) ForgetAll() { 139 d.forgetDirPath("") 140 } 141 142 // invalidateDir invalidates the directory cache for absPath relative to this dir 143 func (d *Dir) invalidateDir(absPath string) { 144 node := d.vfs.root.cachedNode(absPath) 145 if dir, ok := node.(*Dir); ok { 146 dir.mu.Lock() 147 if !dir.read.IsZero() { 148 fs.Debugf(dir.path, "invalidating directory cache") 149 dir.read = time.Time{} 150 } 151 dir.mu.Unlock() 152 } 153 } 154 155 // changeNotify invalidates the directory cache for the relativePath 156 // passed in. 157 // 158 // if entryType is a directory it invalidates the parent of the directory too. 159 func (d *Dir) changeNotify(relativePath string, entryType fs.EntryType) { 160 defer log.Trace(d.path, "relativePath=%q, type=%v", relativePath, entryType)("") 161 d.mu.RLock() 162 absPath := path.Join(d.path, relativePath) 163 d.mu.RUnlock() 164 d.invalidateDir(vfscommon.FindParent(absPath)) 165 if entryType == fs.EntryDirectory { 166 d.invalidateDir(absPath) 167 } 168 } 169 170 // ForgetPath clears the cache for itself and all subdirectories if 171 // they match the given path. The path is specified relative from the 172 // directory it is called from. The cache of the parent directory is 173 // marked as stale, but not cleared otherwise. 174 // It is not possible to traverse the directory tree upwards, i.e. 175 // you cannot clear the cache for the Dir's ancestors or siblings. 176 func (d *Dir) ForgetPath(relativePath string, entryType fs.EntryType) { 177 defer log.Trace(d.path, "relativePath=%q, type=%v", relativePath, entryType)("") 178 d.mu.RLock() 179 absPath := path.Join(d.path, relativePath) 180 d.mu.RUnlock() 181 if absPath != "" { 182 d.invalidateDir(vfscommon.FindParent(absPath)) 183 } 184 if entryType == fs.EntryDirectory { 185 d.forgetDirPath(relativePath) 186 } 187 } 188 189 // walk runs a function on all cached directories. It will be called 190 // on a directory's children first. 191 // 192 // The mutex will be held for the directory when fun is called 193 func (d *Dir) walk(fun func(*Dir)) { 194 d.mu.Lock() 195 defer d.mu.Unlock() 196 for _, node := range d.items { 197 if dir, ok := node.(*Dir); ok { 198 dir.walk(fun) 199 } 200 } 201 202 fun(d) 203 } 204 205 // countActiveWriters returns the number of writers active in this 206 // directory and any subdirectories. 207 func (d *Dir) countActiveWriters() (writers int) { 208 d.walk(func(d *Dir) { 209 // NB d.mu is held by walk() here 210 fs.Debugf(d.path, "Looking for writers") 211 for leaf, item := range d.items { 212 fs.Debugf(leaf, "reading active writers") 213 if file, ok := item.(*File); ok { 214 n := file.activeWriters() 215 if n != 0 { 216 fs.Debugf(file, "active writers %d", n) 217 } 218 writers += n 219 } 220 } 221 }) 222 return writers 223 } 224 225 // age returns the duration since the last time the directory contents 226 // was read and the content is cosidered stale. age will be 0 and 227 // stale true if the last read time is empty. 228 // age must be called with d.mu held. 229 func (d *Dir) _age(when time.Time) (age time.Duration, stale bool) { 230 if d.read.IsZero() { 231 return age, true 232 } 233 age = when.Sub(d.read) 234 stale = age > d.vfs.Opt.DirCacheTime 235 return 236 } 237 238 // rename should be called after the directory is renamed 239 // 240 // Reset the directory to new state, discarding all the objects and 241 // reading everything again 242 func (d *Dir) rename(newParent *Dir, fsDir fs.Directory) { 243 d.ForgetAll() 244 d.mu.Lock() 245 d.parent = newParent 246 d.entry = fsDir 247 d.path = fsDir.Remote() 248 d.modTime = fsDir.ModTime(context.TODO()) 249 d.read = time.Time{} 250 d.mu.Unlock() 251 } 252 253 // addObject adds a new object or directory to the directory 254 // 255 // note that we add new objects rather than updating old ones 256 func (d *Dir) addObject(node Node) { 257 d.mu.Lock() 258 d.items[node.Name()] = node 259 d.mu.Unlock() 260 } 261 262 // delObject removes an object from the directory 263 func (d *Dir) delObject(leaf string) { 264 d.mu.Lock() 265 delete(d.items, leaf) 266 d.mu.Unlock() 267 } 268 269 // read the directory and sets d.items - must be called with the lock held 270 func (d *Dir) _readDir() error { 271 when := time.Now() 272 if age, stale := d._age(when); stale { 273 if age != 0 { 274 fs.Debugf(d.path, "Re-reading directory (%v old)", age) 275 } 276 } else { 277 return nil 278 } 279 entries, err := list.DirSorted(context.TODO(), d.f, false, d.path) 280 if err == fs.ErrorDirNotFound { 281 // We treat directory not found as empty because we 282 // create directories on the fly 283 } else if err != nil { 284 return err 285 } 286 287 err = d._readDirFromEntries(entries, nil, time.Time{}) 288 if err != nil { 289 return err 290 } 291 292 d.read = when 293 return nil 294 } 295 296 // update d.items for each dir in the DirTree below this one and 297 // set the last read time - must be called with the lock held 298 func (d *Dir) _readDirFromDirTree(dirTree dirtree.DirTree, when time.Time) error { 299 return d._readDirFromEntries(dirTree[d.path], dirTree, when) 300 } 301 302 // update d.items and if dirTree is not nil update each dir in the DirTree below this one and 303 // set the last read time - must be called with the lock held 304 func (d *Dir) _readDirFromEntries(entries fs.DirEntries, dirTree dirtree.DirTree, when time.Time) error { 305 var err error 306 // Cache the items by name 307 found := make(map[string]struct{}) 308 for _, entry := range entries { 309 name := path.Base(entry.Remote()) 310 if name == "." || name == ".." { 311 continue 312 } 313 node := d.items[name] 314 found[name] = struct{}{} 315 switch item := entry.(type) { 316 case fs.Object: 317 obj := item 318 // Reuse old file value if it exists 319 if file, ok := node.(*File); node != nil && ok { 320 file.setObjectNoUpdate(obj) 321 } else { 322 node = newFile(d, d.path, obj, name) 323 } 324 case fs.Directory: 325 // Reuse old dir value if it exists 326 if node == nil || !node.IsDir() { 327 node = newDir(d.vfs, d.f, d, item) 328 } 329 if dirTree != nil { 330 dir := node.(*Dir) 331 dir.mu.Lock() 332 err = dir._readDirFromDirTree(dirTree, when) 333 if err != nil { 334 dir.read = time.Time{} 335 } else { 336 dir.read = when 337 } 338 dir.mu.Unlock() 339 if err != nil { 340 return err 341 } 342 } 343 default: 344 err = errors.Errorf("unknown type %T", item) 345 fs.Errorf(d, "readDir error: %v", err) 346 return err 347 } 348 d.items[name] = node 349 } 350 // delete unused entries 351 for name := range d.items { 352 if _, ok := found[name]; !ok { 353 delete(d.items, name) 354 } 355 } 356 return nil 357 } 358 359 // readDirTree forces a refresh of the complete directory tree 360 func (d *Dir) readDirTree() error { 361 d.mu.RLock() 362 f, path := d.f, d.path 363 d.mu.RUnlock() 364 when := time.Now() 365 fs.Debugf(path, "Reading directory tree") 366 dt, err := walk.NewDirTree(context.TODO(), f, path, false, -1) 367 if err != nil { 368 return err 369 } 370 d.mu.Lock() 371 defer d.mu.Unlock() 372 d.read = time.Time{} 373 err = d._readDirFromDirTree(dt, when) 374 if err != nil { 375 return err 376 } 377 fs.Debugf(d.path, "Reading directory tree done in %s", time.Since(when)) 378 d.read = when 379 return nil 380 } 381 382 // readDir forces a refresh of the directory 383 func (d *Dir) readDir() error { 384 d.mu.Lock() 385 defer d.mu.Unlock() 386 d.read = time.Time{} 387 return d._readDir() 388 } 389 390 // stat a single item in the directory 391 // 392 // returns ENOENT if not found. 393 // returns a custom error if directory on a case-insensitive file system 394 // contains files with names that differ only by case. 395 func (d *Dir) stat(leaf string) (Node, error) { 396 d.mu.Lock() 397 defer d.mu.Unlock() 398 err := d._readDir() 399 if err != nil { 400 return nil, err 401 } 402 item, ok := d.items[leaf] 403 404 if !ok && d.vfs.Opt.CaseInsensitive { 405 leafLower := strings.ToLower(leaf) 406 for name, node := range d.items { 407 if strings.ToLower(name) == leafLower { 408 if ok { 409 // duplicate case insensitive match is an error 410 return nil, errors.Errorf("duplicate filename %q detected with --vfs-case-insensitive set", leaf) 411 } 412 // found a case insenstive match 413 ok = true 414 item = node 415 } 416 } 417 } 418 419 if !ok { 420 return nil, ENOENT 421 } 422 return item, nil 423 } 424 425 // Check to see if a directory is empty 426 func (d *Dir) isEmpty() (bool, error) { 427 d.mu.Lock() 428 defer d.mu.Unlock() 429 err := d._readDir() 430 if err != nil { 431 return false, err 432 } 433 return len(d.items) == 0, nil 434 } 435 436 // ModTime returns the modification time of the directory 437 func (d *Dir) ModTime() time.Time { 438 d.mu.RLock() 439 defer d.mu.RUnlock() 440 // fs.Debugf(d.path, "Dir.ModTime %v", d.modTime) 441 return d.modTime 442 } 443 444 // Size of the directory 445 func (d *Dir) Size() int64 { 446 return 0 447 } 448 449 // SetModTime sets the modTime for this dir 450 func (d *Dir) SetModTime(modTime time.Time) error { 451 if d.vfs.Opt.ReadOnly { 452 return EROFS 453 } 454 d.mu.Lock() 455 d.modTime = modTime 456 d.mu.Unlock() 457 return nil 458 } 459 460 func (d *Dir) cachedDir(relativePath string) (dir *Dir) { 461 dir, _ = d.cachedNode(relativePath).(*Dir) 462 return 463 } 464 465 func (d *Dir) cachedNode(relativePath string) Node { 466 segments := strings.Split(strings.Trim(relativePath, "/"), "/") 467 var node Node = d 468 for _, s := range segments { 469 if s == "" { 470 continue 471 } 472 if dir, ok := node.(*Dir); ok { 473 dir.mu.Lock() 474 node = dir.items[s] 475 dir.mu.Unlock() 476 477 if node != nil { 478 continue 479 } 480 } 481 return nil 482 } 483 484 return node 485 } 486 487 // Stat looks up a specific entry in the receiver. 488 // 489 // Stat should return a Node corresponding to the entry. If the 490 // name does not exist in the directory, Stat should return ENOENT. 491 // 492 // Stat need not to handle the names "." and "..". 493 func (d *Dir) Stat(name string) (node Node, err error) { 494 // fs.Debugf(path, "Dir.Stat") 495 node, err = d.stat(name) 496 if err != nil { 497 if err != ENOENT { 498 fs.Errorf(d, "Dir.Stat error: %v", err) 499 } 500 return nil, err 501 } 502 // fs.Debugf(path, "Dir.Stat OK") 503 return node, nil 504 } 505 506 // ReadDirAll reads the contents of the directory sorted 507 func (d *Dir) ReadDirAll() (items Nodes, err error) { 508 // fs.Debugf(d.path, "Dir.ReadDirAll") 509 d.mu.Lock() 510 err = d._readDir() 511 if err != nil { 512 fs.Debugf(d.path, "Dir.ReadDirAll error: %v", err) 513 d.mu.Unlock() 514 return nil, err 515 } 516 for _, item := range d.items { 517 items = append(items, item) 518 } 519 d.mu.Unlock() 520 sort.Sort(items) 521 // fs.Debugf(d.path, "Dir.ReadDirAll OK with %d entries", len(items)) 522 return items, nil 523 } 524 525 // accessModeMask masks off the read modes from the flags 526 const accessModeMask = (os.O_RDONLY | os.O_WRONLY | os.O_RDWR) 527 528 // Open the directory according to the flags provided 529 func (d *Dir) Open(flags int) (fd Handle, err error) { 530 rdwrMode := flags & accessModeMask 531 if rdwrMode != os.O_RDONLY { 532 fs.Errorf(d, "Can only open directories read only") 533 return nil, EPERM 534 } 535 return newDirHandle(d), nil 536 } 537 538 // Create makes a new file node 539 func (d *Dir) Create(name string, flags int) (*File, error) { 540 // fs.Debugf(path, "Dir.Create") 541 if d.vfs.Opt.ReadOnly { 542 return nil, EROFS 543 } 544 // This gets added to the directory when the file is opened for write 545 return newFile(d, d.Path(), nil, name), nil 546 } 547 548 // Mkdir creates a new directory 549 func (d *Dir) Mkdir(name string) (*Dir, error) { 550 if d.vfs.Opt.ReadOnly { 551 return nil, EROFS 552 } 553 path := path.Join(d.path, name) 554 node, err := d.stat(name) 555 switch err { 556 case ENOENT: 557 // not found, carry on 558 case nil: 559 // found so check what it is 560 if node.IsDir() { 561 return node.(*Dir), err 562 } 563 return nil, EEXIST 564 default: 565 // a different error - report 566 fs.Errorf(d, "Dir.Mkdir failed to read directory: %v", err) 567 return nil, err 568 } 569 // fs.Debugf(path, "Dir.Mkdir") 570 err = d.f.Mkdir(context.TODO(), path) 571 if err != nil { 572 fs.Errorf(d, "Dir.Mkdir failed to create directory: %v", err) 573 return nil, err 574 } 575 fsDir := fs.NewDir(path, time.Now()) 576 dir := newDir(d.vfs, d.f, d, fsDir) 577 d.addObject(dir) 578 // fs.Debugf(path, "Dir.Mkdir OK") 579 return dir, nil 580 } 581 582 // Remove the directory 583 func (d *Dir) Remove() error { 584 if d.vfs.Opt.ReadOnly { 585 return EROFS 586 } 587 // Check directory is empty first 588 empty, err := d.isEmpty() 589 if err != nil { 590 fs.Errorf(d, "Dir.Remove dir error: %v", err) 591 return err 592 } 593 if !empty { 594 fs.Errorf(d, "Dir.Remove not empty") 595 return ENOTEMPTY 596 } 597 // remove directory 598 err = d.f.Rmdir(context.TODO(), d.path) 599 if err != nil { 600 fs.Errorf(d, "Dir.Remove failed to remove directory: %v", err) 601 return err 602 } 603 // Remove the item from the parent directory listing 604 if d.parent != nil { 605 d.parent.delObject(d.Name()) 606 } 607 return nil 608 } 609 610 // RemoveAll removes the directory and any contents recursively 611 func (d *Dir) RemoveAll() error { 612 if d.vfs.Opt.ReadOnly { 613 return EROFS 614 } 615 // Remove contents of the directory 616 nodes, err := d.ReadDirAll() 617 if err != nil { 618 fs.Errorf(d, "Dir.RemoveAll failed to read directory: %v", err) 619 return err 620 } 621 for _, node := range nodes { 622 err = node.RemoveAll() 623 if err != nil { 624 fs.Errorf(node.Path(), "Dir.RemoveAll failed to remove: %v", err) 625 return err 626 } 627 } 628 return d.Remove() 629 } 630 631 // DirEntry returns the underlying fs.DirEntry 632 func (d *Dir) DirEntry() (entry fs.DirEntry) { 633 return d.entry 634 } 635 636 // RemoveName removes the entry with the given name from the receiver, 637 // which must be a directory. The entry to be removed may correspond 638 // to a file (unlink) or to a directory (rmdir). 639 func (d *Dir) RemoveName(name string) error { 640 if d.vfs.Opt.ReadOnly { 641 return EROFS 642 } 643 // fs.Debugf(path, "Dir.Remove") 644 node, err := d.stat(name) 645 if err != nil { 646 fs.Errorf(d, "Dir.Remove error: %v", err) 647 return err 648 } 649 return node.Remove() 650 } 651 652 // Rename the file 653 func (d *Dir) Rename(oldName, newName string, destDir *Dir) error { 654 if d.vfs.Opt.ReadOnly { 655 return EROFS 656 } 657 oldPath := path.Join(d.path, oldName) 658 newPath := path.Join(destDir.path, newName) 659 // fs.Debugf(oldPath, "Dir.Rename to %q", newPath) 660 oldNode, err := d.stat(oldName) 661 if err != nil { 662 fs.Errorf(oldPath, "Dir.Rename error: %v", err) 663 return err 664 } 665 switch x := oldNode.DirEntry().(type) { 666 case nil: 667 if oldFile, ok := oldNode.(*File); ok { 668 if err = oldFile.rename(context.TODO(), destDir, newName); err != nil { 669 fs.Errorf(oldPath, "Dir.Rename error: %v", err) 670 return err 671 } 672 } else { 673 fs.Errorf(oldPath, "Dir.Rename can't rename open file that is not a vfs.File") 674 return EPERM 675 } 676 case fs.Object: 677 if oldFile, ok := oldNode.(*File); ok { 678 if err = oldFile.rename(context.TODO(), destDir, newName); err != nil { 679 fs.Errorf(oldPath, "Dir.Rename error: %v", err) 680 return err 681 } 682 } else { 683 err := errors.Errorf("Fs %q can't rename file that is not a vfs.File", d.f) 684 fs.Errorf(oldPath, "Dir.Rename error: %v", err) 685 return err 686 } 687 case fs.Directory: 688 features := d.f.Features() 689 if features.DirMove == nil && features.Move == nil && features.Copy == nil { 690 err := errors.Errorf("Fs %q can't rename directories (no DirMove, Move or Copy)", d.f) 691 fs.Errorf(oldPath, "Dir.Rename error: %v", err) 692 return err 693 } 694 srcRemote := x.Remote() 695 dstRemote := newPath 696 err = operations.DirMove(context.TODO(), d.f, srcRemote, dstRemote) 697 if err != nil { 698 fs.Errorf(oldPath, "Dir.Rename error: %v", err) 699 return err 700 } 701 newDir := fs.NewDirCopy(context.TODO(), x).SetRemote(newPath) 702 // Update the node with the new details 703 if oldNode != nil { 704 if oldDir, ok := oldNode.(*Dir); ok { 705 fs.Debugf(x, "Updating dir with %v %p", newDir, oldDir) 706 oldDir.rename(destDir, newDir) 707 } 708 } 709 default: 710 err = errors.Errorf("unknown type %T", oldNode) 711 fs.Errorf(d.path, "Dir.Rename error: %v", err) 712 return err 713 } 714 715 // Show moved - delete from old dir and add to new 716 d.delObject(oldName) 717 destDir.addObject(oldNode) 718 719 // fs.Debugf(newPath, "Dir.Rename renamed from %q", oldPath) 720 return nil 721 } 722 723 // Sync the directory 724 // 725 // Note that we don't do anything except return OK 726 func (d *Dir) Sync() error { 727 return nil 728 } 729 730 // VFS returns the instance of the VFS 731 func (d *Dir) VFS() *VFS { 732 // No locking required 733 return d.vfs 734 } 735 736 // Fs returns the Fs that the Dir is on 737 func (d *Dir) Fs() fs.Fs { 738 // No locking required 739 return d.f 740 } 741 742 // Truncate changes the size of the named file. 743 func (d *Dir) Truncate(size int64) error { 744 return ENOSYS 745 }