github.com/xhghs/rclone@v1.51.1-0.20200430155106-e186a28cced8/lib/dircache/dircache.go (about)

     1  // Package dircache provides a simple cache for caching directory to path lookups
     2  package dircache
     3  
     4  // _methods are called without the lock
     5  
     6  import (
     7  	"bytes"
     8  	"context"
     9  	"fmt"
    10  	"log"
    11  	"strings"
    12  	"sync"
    13  
    14  	"github.com/pkg/errors"
    15  	"github.com/rclone/rclone/fs"
    16  )
    17  
    18  // DirCache caches paths to directory IDs and vice versa
    19  type DirCache struct {
    20  	cacheMu      sync.RWMutex
    21  	cache        map[string]string
    22  	invCache     map[string]string
    23  	mu           sync.Mutex
    24  	fs           DirCacher // Interface to find and make stuff
    25  	trueRootID   string    // ID of the absolute root
    26  	root         string    // the path we are working on
    27  	rootID       string    // ID of the root directory
    28  	rootParentID string    // ID of the root's parent directory
    29  	foundRoot    bool      // Whether we have found the root or not
    30  }
    31  
    32  // DirCacher describes an interface for doing the low level directory work
    33  type DirCacher interface {
    34  	FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error)
    35  	CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error)
    36  }
    37  
    38  // New makes a DirCache
    39  //
    40  // The cache is safe for concurrent use
    41  func New(root string, trueRootID string, fs DirCacher) *DirCache {
    42  	d := &DirCache{
    43  		trueRootID: trueRootID,
    44  		root:       root,
    45  		fs:         fs,
    46  	}
    47  	d.Flush()
    48  	d.ResetRoot()
    49  	return d
    50  }
    51  
    52  // String returns the directory cache in string form for debugging
    53  func (dc *DirCache) String() string {
    54  	dc.cacheMu.RLock()
    55  	defer dc.cacheMu.RUnlock()
    56  	var buf bytes.Buffer
    57  	_, _ = buf.WriteString("DirCache{\n")
    58  	_, _ = fmt.Fprintf(&buf, "\ttrueRootID: %q,\n", dc.trueRootID)
    59  	_, _ = fmt.Fprintf(&buf, "\troot: %q,\n", dc.root)
    60  	_, _ = fmt.Fprintf(&buf, "\trootID: %q,\n", dc.rootID)
    61  	_, _ = fmt.Fprintf(&buf, "\trootParentID: %q,\n", dc.rootParentID)
    62  	_, _ = fmt.Fprintf(&buf, "\tfoundRoot: %v,\n", dc.foundRoot)
    63  	_, _ = buf.WriteString("\tcache: {\n")
    64  	for k, v := range dc.cache {
    65  		_, _ = fmt.Fprintf(&buf, "\t\t%q: %q,\n", k, v)
    66  	}
    67  	_, _ = buf.WriteString("\t},\n")
    68  	_, _ = buf.WriteString("\tinvCache: {\n")
    69  	for k, v := range dc.invCache {
    70  		_, _ = fmt.Fprintf(&buf, "\t\t%q: %q,\n", k, v)
    71  	}
    72  	_, _ = buf.WriteString("\t},\n")
    73  	_, _ = buf.WriteString("}\n")
    74  	return buf.String()
    75  }
    76  
    77  // Get an ID given a path
    78  func (dc *DirCache) Get(path string) (id string, ok bool) {
    79  	dc.cacheMu.RLock()
    80  	id, ok = dc.cache[path]
    81  	dc.cacheMu.RUnlock()
    82  	return
    83  }
    84  
    85  // GetInv gets a path given an ID
    86  func (dc *DirCache) GetInv(id string) (path string, ok bool) {
    87  	dc.cacheMu.RLock()
    88  	path, ok = dc.invCache[id]
    89  	dc.cacheMu.RUnlock()
    90  	return
    91  }
    92  
    93  // Put a path, id into the map
    94  func (dc *DirCache) Put(path, id string) {
    95  	dc.cacheMu.Lock()
    96  	dc.cache[path] = id
    97  	dc.invCache[id] = path
    98  	dc.cacheMu.Unlock()
    99  }
   100  
   101  // Flush the map of all data
   102  func (dc *DirCache) Flush() {
   103  	dc.cacheMu.Lock()
   104  	dc.cache = make(map[string]string)
   105  	dc.invCache = make(map[string]string)
   106  	dc.cacheMu.Unlock()
   107  }
   108  
   109  // FlushDir flushes the map of all data starting with dir
   110  //
   111  // If dir is empty then this is equivalent to calling ResetRoot
   112  func (dc *DirCache) FlushDir(dir string) {
   113  	if dir == "" {
   114  		dc.ResetRoot()
   115  		return
   116  	}
   117  	dc.cacheMu.Lock()
   118  
   119  	// Delete the root dir
   120  	ID, ok := dc.cache[dir]
   121  	if ok {
   122  		delete(dc.cache, dir)
   123  		delete(dc.invCache, ID)
   124  	}
   125  
   126  	// And any sub directories
   127  	dir += "/"
   128  	for key, ID := range dc.cache {
   129  		if strings.HasPrefix(key, dir) {
   130  			delete(dc.cache, key)
   131  			delete(dc.invCache, ID)
   132  		}
   133  	}
   134  
   135  	dc.cacheMu.Unlock()
   136  }
   137  
   138  // SplitPath splits a path into directory, leaf
   139  //
   140  // Path shouldn't start or end with a /
   141  //
   142  // If there are no slashes then directory will be "" and leaf = path
   143  func SplitPath(path string) (directory, leaf string) {
   144  	lastSlash := strings.LastIndex(path, "/")
   145  	if lastSlash >= 0 {
   146  		directory = path[:lastSlash]
   147  		leaf = path[lastSlash+1:]
   148  	} else {
   149  		directory = ""
   150  		leaf = path
   151  	}
   152  	return
   153  }
   154  
   155  // FindDir finds the directory passed in returning the directory ID
   156  // starting from pathID
   157  //
   158  // Path shouldn't start or end with a /
   159  //
   160  // If create is set it will make the directory if not found
   161  //
   162  // Algorithm:
   163  //  Look in the cache for the path, if found return the pathID
   164  //  If not found strip the last path off the path and recurse
   165  //  Now have a parent directory id, so look in the parent for self and return it
   166  func (dc *DirCache) FindDir(ctx context.Context, path string, create bool) (pathID string, err error) {
   167  	dc.mu.Lock()
   168  	defer dc.mu.Unlock()
   169  	return dc._findDir(ctx, path, create)
   170  }
   171  
   172  // Look for the root and in the cache - safe to call without the mu
   173  func (dc *DirCache) _findDirInCache(path string) string {
   174  	// fmt.Println("Finding",path,"create",create,"cache",cache)
   175  	// If it is the root, then return it
   176  	if path == "" {
   177  		// fmt.Println("Root")
   178  		return dc.rootID
   179  	}
   180  
   181  	// If it is in the cache then return it
   182  	pathID, ok := dc.Get(path)
   183  	if ok {
   184  		// fmt.Println("Cache hit on", path)
   185  		return pathID
   186  	}
   187  
   188  	return ""
   189  }
   190  
   191  // Unlocked findDir - must have mu
   192  func (dc *DirCache) _findDir(ctx context.Context, path string, create bool) (pathID string, err error) {
   193  	pathID = dc._findDirInCache(path)
   194  	if pathID != "" {
   195  		return pathID, nil
   196  	}
   197  
   198  	// Split the path into directory, leaf
   199  	directory, leaf := SplitPath(path)
   200  
   201  	// Recurse and find pathID for parent directory
   202  	parentPathID, err := dc._findDir(ctx, directory, create)
   203  	if err != nil {
   204  		return "", err
   205  
   206  	}
   207  
   208  	// Find the leaf in parentPathID
   209  	pathID, found, err := dc.fs.FindLeaf(ctx, parentPathID, leaf)
   210  	if err != nil {
   211  		return "", err
   212  	}
   213  
   214  	// If not found create the directory if required or return an error
   215  	if !found {
   216  		if create {
   217  			pathID, err = dc.fs.CreateDir(ctx, parentPathID, leaf)
   218  			if err != nil {
   219  				return "", errors.Wrap(err, "failed to make directory")
   220  			}
   221  		} else {
   222  			return "", fs.ErrorDirNotFound
   223  		}
   224  	}
   225  
   226  	// Store the leaf directory in the cache
   227  	dc.Put(path, pathID)
   228  
   229  	// fmt.Println("Dir", path, "is", pathID)
   230  	return pathID, nil
   231  }
   232  
   233  // FindPath finds the leaf and directoryID from a path
   234  //
   235  // Do not call FindPath with the root directory - it will return an error
   236  //
   237  // If create is set parent directories will be created if they don't exist
   238  func (dc *DirCache) FindPath(ctx context.Context, path string, create bool) (leaf, directoryID string, err error) {
   239  	if path == "" {
   240  		err = errors.New("internal error: can't call FindPath with root directory")
   241  		return
   242  	}
   243  	dc.mu.Lock()
   244  	defer dc.mu.Unlock()
   245  	directory, leaf := SplitPath(path)
   246  	directoryID, err = dc._findDir(ctx, directory, create)
   247  	return
   248  }
   249  
   250  // FindRoot finds the root directory if not already found
   251  //
   252  // Resets the root directory
   253  //
   254  // If create is set it will make the directory if not found
   255  func (dc *DirCache) FindRoot(ctx context.Context, create bool) error {
   256  	dc.mu.Lock()
   257  	defer dc.mu.Unlock()
   258  	if dc.foundRoot {
   259  		return nil
   260  	}
   261  	rootID, err := dc._findDir(ctx, dc.root, create)
   262  	if err != nil {
   263  		return err
   264  	}
   265  	dc.foundRoot = true
   266  	dc.rootID = rootID
   267  
   268  	// Find the parent of the root while we still have the root
   269  	// directory tree cached
   270  	rootParentPath, _ := SplitPath(dc.root)
   271  	dc.rootParentID, _ = dc.Get(rootParentPath)
   272  
   273  	// Reset the tree based on dc.root
   274  	dc.Flush()
   275  	// Put the root directory in
   276  	dc.Put("", dc.rootID)
   277  	return nil
   278  }
   279  
   280  // FindRootAndPath finds the root first if not found then finds leaf and directoryID from a path
   281  //
   282  // If create is set parent directories will be created if they don't exist
   283  func (dc *DirCache) FindRootAndPath(ctx context.Context, path string, create bool) (leaf, directoryID string, err error) {
   284  	err = dc.FindRoot(ctx, create)
   285  	if err != nil {
   286  		return
   287  	}
   288  	return dc.FindPath(ctx, path, create)
   289  }
   290  
   291  // FoundRoot returns whether the root directory has been found yet
   292  //
   293  // Call this from FindLeaf or CreateDir only
   294  func (dc *DirCache) FoundRoot() bool {
   295  	return dc.foundRoot
   296  }
   297  
   298  // RootID returns the ID of the root directory
   299  //
   300  // This should be called after FindRoot
   301  func (dc *DirCache) RootID() string {
   302  	dc.mu.Lock()
   303  	defer dc.mu.Unlock()
   304  	if !dc.foundRoot {
   305  		log.Fatalf("Internal Error: RootID() called before FindRoot")
   306  	}
   307  	return dc.rootID
   308  }
   309  
   310  // RootParentID returns the ID of the parent of the root directory
   311  //
   312  // This should be called after FindRoot
   313  func (dc *DirCache) RootParentID() (string, error) {
   314  	dc.mu.Lock()
   315  	defer dc.mu.Unlock()
   316  	if !dc.foundRoot {
   317  		return "", errors.New("internal error: RootID() called before FindRoot")
   318  	}
   319  	if dc.rootParentID == "" {
   320  		return "", errors.New("internal error: didn't find rootParentID")
   321  	}
   322  	if dc.rootID == dc.trueRootID {
   323  		return "", errors.New("is root directory")
   324  	}
   325  	return dc.rootParentID, nil
   326  }
   327  
   328  // ResetRoot resets the root directory to the absolute root and clears
   329  // the DirCache
   330  func (dc *DirCache) ResetRoot() {
   331  	dc.mu.Lock()
   332  	defer dc.mu.Unlock()
   333  	dc.foundRoot = false
   334  	dc.Flush()
   335  
   336  	// Put the true root in
   337  	dc.rootID = dc.trueRootID
   338  
   339  	// Put the root directory in
   340  	dc.Put("", dc.rootID)
   341  }