github.com/xhghs/rclone@v1.51.1-0.20200430155106-e186a28cced8/lib/dircache/dircache.go (about) 1 // Package dircache provides a simple cache for caching directory to path lookups 2 package dircache 3 4 // _methods are called without the lock 5 6 import ( 7 "bytes" 8 "context" 9 "fmt" 10 "log" 11 "strings" 12 "sync" 13 14 "github.com/pkg/errors" 15 "github.com/rclone/rclone/fs" 16 ) 17 18 // DirCache caches paths to directory IDs and vice versa 19 type DirCache struct { 20 cacheMu sync.RWMutex 21 cache map[string]string 22 invCache map[string]string 23 mu sync.Mutex 24 fs DirCacher // Interface to find and make stuff 25 trueRootID string // ID of the absolute root 26 root string // the path we are working on 27 rootID string // ID of the root directory 28 rootParentID string // ID of the root's parent directory 29 foundRoot bool // Whether we have found the root or not 30 } 31 32 // DirCacher describes an interface for doing the low level directory work 33 type DirCacher interface { 34 FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) 35 CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) 36 } 37 38 // New makes a DirCache 39 // 40 // The cache is safe for concurrent use 41 func New(root string, trueRootID string, fs DirCacher) *DirCache { 42 d := &DirCache{ 43 trueRootID: trueRootID, 44 root: root, 45 fs: fs, 46 } 47 d.Flush() 48 d.ResetRoot() 49 return d 50 } 51 52 // String returns the directory cache in string form for debugging 53 func (dc *DirCache) String() string { 54 dc.cacheMu.RLock() 55 defer dc.cacheMu.RUnlock() 56 var buf bytes.Buffer 57 _, _ = buf.WriteString("DirCache{\n") 58 _, _ = fmt.Fprintf(&buf, "\ttrueRootID: %q,\n", dc.trueRootID) 59 _, _ = fmt.Fprintf(&buf, "\troot: %q,\n", dc.root) 60 _, _ = fmt.Fprintf(&buf, "\trootID: %q,\n", dc.rootID) 61 _, _ = fmt.Fprintf(&buf, "\trootParentID: %q,\n", dc.rootParentID) 62 _, _ = fmt.Fprintf(&buf, "\tfoundRoot: %v,\n", dc.foundRoot) 63 _, _ = buf.WriteString("\tcache: {\n") 64 for k, v := range dc.cache { 65 _, _ = fmt.Fprintf(&buf, "\t\t%q: %q,\n", k, v) 66 } 67 _, _ = buf.WriteString("\t},\n") 68 _, _ = buf.WriteString("\tinvCache: {\n") 69 for k, v := range dc.invCache { 70 _, _ = fmt.Fprintf(&buf, "\t\t%q: %q,\n", k, v) 71 } 72 _, _ = buf.WriteString("\t},\n") 73 _, _ = buf.WriteString("}\n") 74 return buf.String() 75 } 76 77 // Get an ID given a path 78 func (dc *DirCache) Get(path string) (id string, ok bool) { 79 dc.cacheMu.RLock() 80 id, ok = dc.cache[path] 81 dc.cacheMu.RUnlock() 82 return 83 } 84 85 // GetInv gets a path given an ID 86 func (dc *DirCache) GetInv(id string) (path string, ok bool) { 87 dc.cacheMu.RLock() 88 path, ok = dc.invCache[id] 89 dc.cacheMu.RUnlock() 90 return 91 } 92 93 // Put a path, id into the map 94 func (dc *DirCache) Put(path, id string) { 95 dc.cacheMu.Lock() 96 dc.cache[path] = id 97 dc.invCache[id] = path 98 dc.cacheMu.Unlock() 99 } 100 101 // Flush the map of all data 102 func (dc *DirCache) Flush() { 103 dc.cacheMu.Lock() 104 dc.cache = make(map[string]string) 105 dc.invCache = make(map[string]string) 106 dc.cacheMu.Unlock() 107 } 108 109 // FlushDir flushes the map of all data starting with dir 110 // 111 // If dir is empty then this is equivalent to calling ResetRoot 112 func (dc *DirCache) FlushDir(dir string) { 113 if dir == "" { 114 dc.ResetRoot() 115 return 116 } 117 dc.cacheMu.Lock() 118 119 // Delete the root dir 120 ID, ok := dc.cache[dir] 121 if ok { 122 delete(dc.cache, dir) 123 delete(dc.invCache, ID) 124 } 125 126 // And any sub directories 127 dir += "/" 128 for key, ID := range dc.cache { 129 if strings.HasPrefix(key, dir) { 130 delete(dc.cache, key) 131 delete(dc.invCache, ID) 132 } 133 } 134 135 dc.cacheMu.Unlock() 136 } 137 138 // SplitPath splits a path into directory, leaf 139 // 140 // Path shouldn't start or end with a / 141 // 142 // If there are no slashes then directory will be "" and leaf = path 143 func SplitPath(path string) (directory, leaf string) { 144 lastSlash := strings.LastIndex(path, "/") 145 if lastSlash >= 0 { 146 directory = path[:lastSlash] 147 leaf = path[lastSlash+1:] 148 } else { 149 directory = "" 150 leaf = path 151 } 152 return 153 } 154 155 // FindDir finds the directory passed in returning the directory ID 156 // starting from pathID 157 // 158 // Path shouldn't start or end with a / 159 // 160 // If create is set it will make the directory if not found 161 // 162 // Algorithm: 163 // Look in the cache for the path, if found return the pathID 164 // If not found strip the last path off the path and recurse 165 // Now have a parent directory id, so look in the parent for self and return it 166 func (dc *DirCache) FindDir(ctx context.Context, path string, create bool) (pathID string, err error) { 167 dc.mu.Lock() 168 defer dc.mu.Unlock() 169 return dc._findDir(ctx, path, create) 170 } 171 172 // Look for the root and in the cache - safe to call without the mu 173 func (dc *DirCache) _findDirInCache(path string) string { 174 // fmt.Println("Finding",path,"create",create,"cache",cache) 175 // If it is the root, then return it 176 if path == "" { 177 // fmt.Println("Root") 178 return dc.rootID 179 } 180 181 // If it is in the cache then return it 182 pathID, ok := dc.Get(path) 183 if ok { 184 // fmt.Println("Cache hit on", path) 185 return pathID 186 } 187 188 return "" 189 } 190 191 // Unlocked findDir - must have mu 192 func (dc *DirCache) _findDir(ctx context.Context, path string, create bool) (pathID string, err error) { 193 pathID = dc._findDirInCache(path) 194 if pathID != "" { 195 return pathID, nil 196 } 197 198 // Split the path into directory, leaf 199 directory, leaf := SplitPath(path) 200 201 // Recurse and find pathID for parent directory 202 parentPathID, err := dc._findDir(ctx, directory, create) 203 if err != nil { 204 return "", err 205 206 } 207 208 // Find the leaf in parentPathID 209 pathID, found, err := dc.fs.FindLeaf(ctx, parentPathID, leaf) 210 if err != nil { 211 return "", err 212 } 213 214 // If not found create the directory if required or return an error 215 if !found { 216 if create { 217 pathID, err = dc.fs.CreateDir(ctx, parentPathID, leaf) 218 if err != nil { 219 return "", errors.Wrap(err, "failed to make directory") 220 } 221 } else { 222 return "", fs.ErrorDirNotFound 223 } 224 } 225 226 // Store the leaf directory in the cache 227 dc.Put(path, pathID) 228 229 // fmt.Println("Dir", path, "is", pathID) 230 return pathID, nil 231 } 232 233 // FindPath finds the leaf and directoryID from a path 234 // 235 // Do not call FindPath with the root directory - it will return an error 236 // 237 // If create is set parent directories will be created if they don't exist 238 func (dc *DirCache) FindPath(ctx context.Context, path string, create bool) (leaf, directoryID string, err error) { 239 if path == "" { 240 err = errors.New("internal error: can't call FindPath with root directory") 241 return 242 } 243 dc.mu.Lock() 244 defer dc.mu.Unlock() 245 directory, leaf := SplitPath(path) 246 directoryID, err = dc._findDir(ctx, directory, create) 247 return 248 } 249 250 // FindRoot finds the root directory if not already found 251 // 252 // Resets the root directory 253 // 254 // If create is set it will make the directory if not found 255 func (dc *DirCache) FindRoot(ctx context.Context, create bool) error { 256 dc.mu.Lock() 257 defer dc.mu.Unlock() 258 if dc.foundRoot { 259 return nil 260 } 261 rootID, err := dc._findDir(ctx, dc.root, create) 262 if err != nil { 263 return err 264 } 265 dc.foundRoot = true 266 dc.rootID = rootID 267 268 // Find the parent of the root while we still have the root 269 // directory tree cached 270 rootParentPath, _ := SplitPath(dc.root) 271 dc.rootParentID, _ = dc.Get(rootParentPath) 272 273 // Reset the tree based on dc.root 274 dc.Flush() 275 // Put the root directory in 276 dc.Put("", dc.rootID) 277 return nil 278 } 279 280 // FindRootAndPath finds the root first if not found then finds leaf and directoryID from a path 281 // 282 // If create is set parent directories will be created if they don't exist 283 func (dc *DirCache) FindRootAndPath(ctx context.Context, path string, create bool) (leaf, directoryID string, err error) { 284 err = dc.FindRoot(ctx, create) 285 if err != nil { 286 return 287 } 288 return dc.FindPath(ctx, path, create) 289 } 290 291 // FoundRoot returns whether the root directory has been found yet 292 // 293 // Call this from FindLeaf or CreateDir only 294 func (dc *DirCache) FoundRoot() bool { 295 return dc.foundRoot 296 } 297 298 // RootID returns the ID of the root directory 299 // 300 // This should be called after FindRoot 301 func (dc *DirCache) RootID() string { 302 dc.mu.Lock() 303 defer dc.mu.Unlock() 304 if !dc.foundRoot { 305 log.Fatalf("Internal Error: RootID() called before FindRoot") 306 } 307 return dc.rootID 308 } 309 310 // RootParentID returns the ID of the parent of the root directory 311 // 312 // This should be called after FindRoot 313 func (dc *DirCache) RootParentID() (string, error) { 314 dc.mu.Lock() 315 defer dc.mu.Unlock() 316 if !dc.foundRoot { 317 return "", errors.New("internal error: RootID() called before FindRoot") 318 } 319 if dc.rootParentID == "" { 320 return "", errors.New("internal error: didn't find rootParentID") 321 } 322 if dc.rootID == dc.trueRootID { 323 return "", errors.New("is root directory") 324 } 325 return dc.rootParentID, nil 326 } 327 328 // ResetRoot resets the root directory to the absolute root and clears 329 // the DirCache 330 func (dc *DirCache) ResetRoot() { 331 dc.mu.Lock() 332 defer dc.mu.Unlock() 333 dc.foundRoot = false 334 dc.Flush() 335 336 // Put the true root in 337 dc.rootID = dc.trueRootID 338 339 // Put the root directory in 340 dc.Put("", dc.rootID) 341 }