github.com/gernest/nezuko@v0.1.2/internal/cache/cache.go (about) 1 // Copyright 2017 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Package cache implements a build artifact cache. 6 package cache 7 8 import ( 9 "bytes" 10 "crypto/sha256" 11 "encoding/hex" 12 "errors" 13 "fmt" 14 "io" 15 "io/ioutil" 16 "os" 17 "path/filepath" 18 "strconv" 19 "strings" 20 "time" 21 22 "github.com/gernest/nezuko/internal/renameio" 23 ) 24 25 // An ActionID is a cache action key, the hash of a complete description of a 26 // repeatable computation (command line, environment variables, 27 // input file contents, executable contents). 28 type ActionID [HashSize]byte 29 30 // An OutputID is a cache output key, the hash of an output of a computation. 31 type OutputID [HashSize]byte 32 33 // A Cache is a package cache, backed by a file system directory tree. 34 type Cache struct { 35 dir string 36 log *os.File 37 now func() time.Time 38 } 39 40 // Open opens and returns the cache in the given directory. 41 // 42 // It is safe for multiple processes on a single machine to use the 43 // same cache directory in a local file system simultaneously. 44 // They will coordinate using operating system file locks and may 45 // duplicate effort but will not corrupt the cache. 46 // 47 // However, it is NOT safe for multiple processes on different machines 48 // to share a cache directory (for example, if the directory were stored 49 // in a network file system). File locking is notoriously unreliable in 50 // network file systems and may not suffice to protect the cache. 51 // 52 func Open(dir string) (*Cache, error) { 53 info, err := os.Stat(dir) 54 if err != nil { 55 return nil, err 56 } 57 if !info.IsDir() { 58 return nil, &os.PathError{Op: "open", Path: dir, Err: fmt.Errorf("not a directory")} 59 } 60 for i := 0; i < 256; i++ { 61 name := filepath.Join(dir, fmt.Sprintf("%02x", i)) 62 if err := os.MkdirAll(name, 0777); err != nil { 63 return nil, err 64 } 65 } 66 f, err := os.OpenFile(filepath.Join(dir, "log.txt"), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666) 67 if err != nil { 68 return nil, err 69 } 70 c := &Cache{ 71 dir: dir, 72 log: f, 73 now: time.Now, 74 } 75 return c, nil 76 } 77 78 // fileName returns the name of the file corresponding to the given id. 79 func (c *Cache) fileName(id [HashSize]byte, key string) string { 80 return filepath.Join(c.dir, fmt.Sprintf("%02x", id[0]), fmt.Sprintf("%x", id)+"-"+key) 81 } 82 83 var errMissing = errors.New("cache entry not found") 84 85 const ( 86 // action entry file is "v1 <hex id> <hex out> <decimal size space-padded to 20 bytes> <unixnano space-padded to 20 bytes>\n" 87 hexSize = HashSize * 2 88 entrySize = 2 + 1 + hexSize + 1 + hexSize + 1 + 20 + 1 + 20 + 1 89 ) 90 91 // verify controls whether to run the cache in verify mode. 92 // In verify mode, the cache always returns errMissing from Get 93 // but then double-checks in Put that the data being written 94 // exactly matches any existing entry. This provides an easy 95 // way to detect program behavior that would have been different 96 // had the cache entry been returned from Get. 97 // 98 // verify is enabled by setting the environment variable 99 // GODEBUG=gocacheverify=1. 100 var verify = false 101 102 // DebugTest is set when GODEBUG=gocachetest=1 is in the environment. 103 var DebugTest = false 104 105 func init() { initEnv() } 106 107 func initEnv() { 108 verify = false 109 debugHash = false 110 debug := strings.Split(os.Getenv("GODEBUG"), ",") 111 for _, f := range debug { 112 if f == "gocacheverify=1" { 113 verify = true 114 } 115 if f == "gocachehash=1" { 116 debugHash = true 117 } 118 if f == "gocachetest=1" { 119 DebugTest = true 120 } 121 } 122 } 123 124 // Get looks up the action ID in the cache, 125 // returning the corresponding output ID and file size, if any. 126 // Note that finding an output ID does not guarantee that the 127 // saved file for that output ID is still available. 128 func (c *Cache) Get(id ActionID) (Entry, error) { 129 if verify { 130 return Entry{}, errMissing 131 } 132 return c.get(id) 133 } 134 135 type Entry struct { 136 OutputID OutputID 137 Size int64 138 Time time.Time 139 } 140 141 // get is Get but does not respect verify mode, so that Put can use it. 142 func (c *Cache) get(id ActionID) (Entry, error) { 143 missing := func() (Entry, error) { 144 fmt.Fprintf(c.log, "%d miss %x\n", c.now().Unix(), id) 145 return Entry{}, errMissing 146 } 147 f, err := os.Open(c.fileName(id, "a")) 148 if err != nil { 149 return missing() 150 } 151 defer f.Close() 152 entry := make([]byte, entrySize+1) // +1 to detect whether f is too long 153 if n, err := io.ReadFull(f, entry); n != entrySize || err != io.ErrUnexpectedEOF { 154 return missing() 155 } 156 if entry[0] != 'v' || entry[1] != '1' || entry[2] != ' ' || entry[3+hexSize] != ' ' || entry[3+hexSize+1+hexSize] != ' ' || entry[3+hexSize+1+hexSize+1+20] != ' ' || entry[entrySize-1] != '\n' { 157 return missing() 158 } 159 eid, entry := entry[3:3+hexSize], entry[3+hexSize:] 160 eout, entry := entry[1:1+hexSize], entry[1+hexSize:] 161 esize, entry := entry[1:1+20], entry[1+20:] 162 etime, entry := entry[1:1+20], entry[1+20:] 163 var buf [HashSize]byte 164 if _, err := hex.Decode(buf[:], eid); err != nil || buf != id { 165 return missing() 166 } 167 if _, err := hex.Decode(buf[:], eout); err != nil { 168 return missing() 169 } 170 i := 0 171 for i < len(esize) && esize[i] == ' ' { 172 i++ 173 } 174 size, err := strconv.ParseInt(string(esize[i:]), 10, 64) 175 if err != nil || size < 0 { 176 return missing() 177 } 178 i = 0 179 for i < len(etime) && etime[i] == ' ' { 180 i++ 181 } 182 tm, err := strconv.ParseInt(string(etime[i:]), 10, 64) 183 if err != nil || size < 0 { 184 return missing() 185 } 186 187 fmt.Fprintf(c.log, "%d get %x\n", c.now().Unix(), id) 188 189 c.used(c.fileName(id, "a")) 190 191 return Entry{buf, size, time.Unix(0, tm)}, nil 192 } 193 194 // GetFile looks up the action ID in the cache and returns 195 // the name of the corresponding data file. 196 func (c *Cache) GetFile(id ActionID) (file string, entry Entry, err error) { 197 entry, err = c.Get(id) 198 if err != nil { 199 return "", Entry{}, err 200 } 201 file = c.OutputFile(entry.OutputID) 202 info, err := os.Stat(file) 203 if err != nil || info.Size() != entry.Size { 204 return "", Entry{}, errMissing 205 } 206 return file, entry, nil 207 } 208 209 // GetBytes looks up the action ID in the cache and returns 210 // the corresponding output bytes. 211 // GetBytes should only be used for data that can be expected to fit in memory. 212 func (c *Cache) GetBytes(id ActionID) ([]byte, Entry, error) { 213 entry, err := c.Get(id) 214 if err != nil { 215 return nil, entry, err 216 } 217 data, _ := ioutil.ReadFile(c.OutputFile(entry.OutputID)) 218 if sha256.Sum256(data) != entry.OutputID { 219 return nil, entry, errMissing 220 } 221 return data, entry, nil 222 } 223 224 // OutputFile returns the name of the cache file storing output with the given OutputID. 225 func (c *Cache) OutputFile(out OutputID) string { 226 file := c.fileName(out, "d") 227 c.used(file) 228 return file 229 } 230 231 // Time constants for cache expiration. 232 // 233 // We set the mtime on a cache file on each use, but at most one per mtimeInterval (1 hour), 234 // to avoid causing many unnecessary inode updates. The mtimes therefore 235 // roughly reflect "time of last use" but may in fact be older by at most an hour. 236 // 237 // We scan the cache for entries to delete at most once per trimInterval (1 day). 238 // 239 // When we do scan the cache, we delete entries that have not been used for 240 // at least trimLimit (5 days). Statistics gathered from a month of usage by 241 // Go developers found that essentially all reuse of cached entries happened 242 // within 5 days of the previous reuse. See golang.org/issue/22990. 243 const ( 244 mtimeInterval = 1 * time.Hour 245 trimInterval = 24 * time.Hour 246 trimLimit = 5 * 24 * time.Hour 247 ) 248 249 // used makes a best-effort attempt to update mtime on file, 250 // so that mtime reflects cache access time. 251 // 252 // Because the reflection only needs to be approximate, 253 // and to reduce the amount of disk activity caused by using 254 // cache entries, used only updates the mtime if the current 255 // mtime is more than an hour old. This heuristic eliminates 256 // nearly all of the mtime updates that would otherwise happen, 257 // while still keeping the mtimes useful for cache trimming. 258 func (c *Cache) used(file string) { 259 info, err := os.Stat(file) 260 if err == nil && c.now().Sub(info.ModTime()) < mtimeInterval { 261 return 262 } 263 os.Chtimes(file, c.now(), c.now()) 264 } 265 266 // Trim removes old cache entries that are likely not to be reused. 267 func (c *Cache) Trim() { 268 now := c.now() 269 270 // We maintain in dir/trim.txt the time of the last completed cache trim. 271 // If the cache has been trimmed recently enough, do nothing. 272 // This is the common case. 273 data, _ := ioutil.ReadFile(filepath.Join(c.dir, "trim.txt")) 274 t, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64) 275 if err == nil && now.Sub(time.Unix(t, 0)) < trimInterval { 276 return 277 } 278 279 // Trim each of the 256 subdirectories. 280 // We subtract an additional mtimeInterval 281 // to account for the imprecision of our "last used" mtimes. 282 cutoff := now.Add(-trimLimit - mtimeInterval) 283 for i := 0; i < 256; i++ { 284 subdir := filepath.Join(c.dir, fmt.Sprintf("%02x", i)) 285 c.trimSubdir(subdir, cutoff) 286 } 287 288 // Ignore errors from here: if we don't write the complete timestamp, the 289 // cache will appear older than it is, and we'll trim it again next time. 290 renameio.WriteFile(filepath.Join(c.dir, "trim.txt"), []byte(fmt.Sprintf("%d", now.Unix()))) 291 } 292 293 // trimSubdir trims a single cache subdirectory. 294 func (c *Cache) trimSubdir(subdir string, cutoff time.Time) { 295 // Read all directory entries from subdir before removing 296 // any files, in case removing files invalidates the file offset 297 // in the directory scan. Also, ignore error from f.Readdirnames, 298 // because we don't care about reporting the error and we still 299 // want to process any entries found before the error. 300 f, err := os.Open(subdir) 301 if err != nil { 302 return 303 } 304 names, _ := f.Readdirnames(-1) 305 f.Close() 306 307 for _, name := range names { 308 // Remove only cache entries (xxxx-a and xxxx-d). 309 if !strings.HasSuffix(name, "-a") && !strings.HasSuffix(name, "-d") { 310 continue 311 } 312 entry := filepath.Join(subdir, name) 313 info, err := os.Stat(entry) 314 if err == nil && info.ModTime().Before(cutoff) { 315 os.Remove(entry) 316 } 317 } 318 } 319 320 // putIndexEntry adds an entry to the cache recording that executing the action 321 // with the given id produces an output with the given output id (hash) and size. 322 func (c *Cache) putIndexEntry(id ActionID, out OutputID, size int64, allowVerify bool) error { 323 // Note: We expect that for one reason or another it may happen 324 // that repeating an action produces a different output hash 325 // (for example, if the output contains a time stamp or temp dir name). 326 // While not ideal, this is also not a correctness problem, so we 327 // don't make a big deal about it. In particular, we leave the action 328 // cache entries writable specifically so that they can be overwritten. 329 // 330 // Setting GODEBUG=gocacheverify=1 does make a big deal: 331 // in verify mode we are double-checking that the cache entries 332 // are entirely reproducible. As just noted, this may be unrealistic 333 // in some cases but the check is also useful for shaking out real bugs. 334 entry := []byte(fmt.Sprintf("v1 %x %x %20d %20d\n", id, out, size, time.Now().UnixNano())) 335 if verify && allowVerify { 336 old, err := c.get(id) 337 if err == nil && (old.OutputID != out || old.Size != size) { 338 // panic to show stack trace, so we can see what code is generating this cache entry. 339 msg := fmt.Sprintf("z: internal cache error: cache verify failed: id=%x changed:<<<\n%s\n>>>\nold: %x %d\nnew: %x %d", id, reverseHash(id), out, size, old.OutputID, old.Size) 340 panic(msg) 341 } 342 } 343 file := c.fileName(id, "a") 344 if err := ioutil.WriteFile(file, entry, 0666); err != nil { 345 // TODO(bcmills): This Remove potentially races with another go command writing to file. 346 // Can we eliminate it? 347 os.Remove(file) 348 return err 349 } 350 os.Chtimes(file, c.now(), c.now()) // mainly for tests 351 352 fmt.Fprintf(c.log, "%d put %x %x %d\n", c.now().Unix(), id, out, size) 353 return nil 354 } 355 356 // Put stores the given output in the cache as the output for the action ID. 357 // It may read file twice. The content of file must not change between the two passes. 358 func (c *Cache) Put(id ActionID, file io.ReadSeeker) (OutputID, int64, error) { 359 return c.put(id, file, true) 360 } 361 362 // PutNoVerify is like Put but disables the verify check 363 // when GODEBUG=goverifycache=1 is set. 364 // It is meant for data that is OK to cache but that we expect to vary slightly from run to run, 365 // like test output containing times and the like. 366 func (c *Cache) PutNoVerify(id ActionID, file io.ReadSeeker) (OutputID, int64, error) { 367 return c.put(id, file, false) 368 } 369 370 func (c *Cache) put(id ActionID, file io.ReadSeeker, allowVerify bool) (OutputID, int64, error) { 371 // Compute output ID. 372 h := sha256.New() 373 if _, err := file.Seek(0, 0); err != nil { 374 return OutputID{}, 0, err 375 } 376 size, err := io.Copy(h, file) 377 if err != nil { 378 return OutputID{}, 0, err 379 } 380 var out OutputID 381 h.Sum(out[:0]) 382 383 // Copy to cached output file (if not already present). 384 if err := c.copyFile(file, out, size); err != nil { 385 return out, size, err 386 } 387 388 // Add to cache index. 389 return out, size, c.putIndexEntry(id, out, size, allowVerify) 390 } 391 392 // PutBytes stores the given bytes in the cache as the output for the action ID. 393 func (c *Cache) PutBytes(id ActionID, data []byte) error { 394 _, _, err := c.Put(id, bytes.NewReader(data)) 395 return err 396 } 397 398 // copyFile copies file into the cache, expecting it to have the given 399 // output ID and size, if that file is not present already. 400 func (c *Cache) copyFile(file io.ReadSeeker, out OutputID, size int64) error { 401 name := c.fileName(out, "d") 402 info, err := os.Stat(name) 403 if err == nil && info.Size() == size { 404 // Check hash. 405 if f, err := os.Open(name); err == nil { 406 h := sha256.New() 407 io.Copy(h, f) 408 f.Close() 409 var out2 OutputID 410 h.Sum(out2[:0]) 411 if out == out2 { 412 return nil 413 } 414 } 415 // Hash did not match. Fall through and rewrite file. 416 } 417 418 // Copy file to cache directory. 419 mode := os.O_RDWR | os.O_CREATE 420 if err == nil && info.Size() > size { // shouldn't happen but fix in case 421 mode |= os.O_TRUNC 422 } 423 f, err := os.OpenFile(name, mode, 0666) 424 if err != nil { 425 return err 426 } 427 defer f.Close() 428 if size == 0 { 429 // File now exists with correct size. 430 // Only one possible zero-length file, so contents are OK too. 431 // Early return here makes sure there's a "last byte" for code below. 432 return nil 433 } 434 435 // From here on, if any of the I/O writing the file fails, 436 // we make a best-effort attempt to truncate the file f 437 // before returning, to avoid leaving bad bytes in the file. 438 439 // Copy file to f, but also into h to double-check hash. 440 if _, err := file.Seek(0, 0); err != nil { 441 f.Truncate(0) 442 return err 443 } 444 h := sha256.New() 445 w := io.MultiWriter(f, h) 446 if _, err := io.CopyN(w, file, size-1); err != nil { 447 f.Truncate(0) 448 return err 449 } 450 // Check last byte before writing it; writing it will make the size match 451 // what other processes expect to find and might cause them to start 452 // using the file. 453 buf := make([]byte, 1) 454 if _, err := file.Read(buf); err != nil { 455 f.Truncate(0) 456 return err 457 } 458 h.Write(buf) 459 sum := h.Sum(nil) 460 if !bytes.Equal(sum, out[:]) { 461 f.Truncate(0) 462 return fmt.Errorf("file content changed underfoot") 463 } 464 465 // Commit cache file entry. 466 if _, err := f.Write(buf); err != nil { 467 f.Truncate(0) 468 return err 469 } 470 if err := f.Close(); err != nil { 471 // Data might not have been written, 472 // but file may look like it is the right size. 473 // To be extra careful, remove cached file. 474 os.Remove(name) 475 return err 476 } 477 os.Chtimes(name, c.now(), c.now()) // mainly for tests 478 479 return nil 480 }