github.com/cockroachdb/pebble@v1.1.2/internal/cache/clockpro.go (about) 1 // Copyright 2018. All rights reserved. Use of this source code is governed by 2 // an MIT-style license that can be found in the LICENSE file. 3 4 // Package cache implements the CLOCK-Pro caching algorithm. 5 // 6 // CLOCK-Pro is a patent-free alternative to the Adaptive Replacement Cache, 7 // https://en.wikipedia.org/wiki/Adaptive_replacement_cache. 8 // It is an approximation of LIRS ( https://en.wikipedia.org/wiki/LIRS_caching_algorithm ), 9 // much like the CLOCK page replacement algorithm is an approximation of LRU. 10 // 11 // This implementation is based on the python code from https://bitbucket.org/SamiLehtinen/pyclockpro . 12 // 13 // Slides describing the algorithm: http://fr.slideshare.net/huliang64/clockpro 14 // 15 // The original paper: http://static.usenix.org/event/usenix05/tech/general/full_papers/jiang/jiang_html/html.html 16 // 17 // It is MIT licensed, like the original. 18 package cache // import "github.com/cockroachdb/pebble/internal/cache" 19 20 import ( 21 "fmt" 22 "os" 23 "runtime" 24 "runtime/debug" 25 "strings" 26 "sync" 27 "sync/atomic" 28 29 "github.com/cockroachdb/pebble/internal/base" 30 "github.com/cockroachdb/pebble/internal/invariants" 31 ) 32 33 type fileKey struct { 34 // id is the namespace for fileNums. 35 id uint64 36 fileNum base.DiskFileNum 37 } 38 39 type key struct { 40 fileKey 41 offset uint64 42 } 43 44 // file returns the "file key" for the receiver. This is the key used for the 45 // shard.files map. 46 func (k key) file() key { 47 k.offset = 0 48 return k 49 } 50 51 func (k key) String() string { 52 return fmt.Sprintf("%d/%d/%d", k.id, k.fileNum, k.offset) 53 } 54 55 // Handle provides a strong reference to a value in the cache. The reference 56 // does not pin the value in the cache, but it does prevent the underlying byte 57 // slice from being reused. 58 type Handle struct { 59 value *Value 60 } 61 62 // Get returns the value stored in handle. 63 func (h Handle) Get() []byte { 64 if h.value != nil { 65 // NB: We don't increment shard.hits in this code path because we only want 66 // to record a hit when the handle is retrieved from the cache. 67 return h.value.buf 68 } 69 return nil 70 } 71 72 // Release releases the reference to the cache entry. 73 func (h Handle) Release() { 74 h.value.release() 75 } 76 77 type shard struct { 78 hits atomic.Int64 79 misses atomic.Int64 80 81 mu sync.RWMutex 82 83 reservedSize int64 84 maxSize int64 85 coldTarget int64 86 blocks robinHoodMap // fileNum+offset -> block 87 files robinHoodMap // fileNum -> list of blocks 88 89 // The blocks and files maps store values in manually managed memory that is 90 // invisible to the Go GC. This is fine for Value and entry objects that are 91 // stored in manually managed memory, but when the "invariants" build tag is 92 // set, all Value and entry objects are Go allocated and the entries map will 93 // contain a reference to every entry. 94 entries map[*entry]struct{} 95 96 handHot *entry 97 handCold *entry 98 handTest *entry 99 100 sizeHot int64 101 sizeCold int64 102 sizeTest int64 103 104 // The count fields are used exclusively for asserting expectations. 105 // We've seen infinite looping (cockroachdb/cockroach#70154) that 106 // could be explained by a corrupted sizeCold. Through asserting on 107 // these fields, we hope to gain more insight from any future 108 // reproductions. 109 countHot int64 110 countCold int64 111 countTest int64 112 } 113 114 func (c *shard) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle { 115 c.mu.RLock() 116 var value *Value 117 if e := c.blocks.Get(key{fileKey{id, fileNum}, offset}); e != nil { 118 value = e.acquireValue() 119 if value != nil { 120 e.referenced.Store(true) 121 } 122 } 123 c.mu.RUnlock() 124 if value == nil { 125 c.misses.Add(1) 126 return Handle{} 127 } 128 c.hits.Add(1) 129 return Handle{value: value} 130 } 131 132 func (c *shard) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *Value) Handle { 133 if n := value.refs(); n != 1 { 134 panic(fmt.Sprintf("pebble: Value has already been added to the cache: refs=%d", n)) 135 } 136 137 c.mu.Lock() 138 defer c.mu.Unlock() 139 140 k := key{fileKey{id, fileNum}, offset} 141 e := c.blocks.Get(k) 142 143 switch { 144 case e == nil: 145 // no cache entry? add it 146 e = newEntry(c, k, int64(len(value.buf))) 147 e.setValue(value) 148 if c.metaAdd(k, e) { 149 value.ref.trace("add-cold") 150 c.sizeCold += e.size 151 c.countCold++ 152 } else { 153 value.ref.trace("skip-cold") 154 e.free() 155 e = nil 156 } 157 158 case e.peekValue() != nil: 159 // cache entry was a hot or cold page 160 e.setValue(value) 161 e.referenced.Store(true) 162 delta := int64(len(value.buf)) - e.size 163 e.size = int64(len(value.buf)) 164 if e.ptype == etHot { 165 value.ref.trace("add-hot") 166 c.sizeHot += delta 167 } else { 168 value.ref.trace("add-cold") 169 c.sizeCold += delta 170 } 171 c.evict() 172 173 default: 174 // cache entry was a test page 175 c.sizeTest -= e.size 176 c.countTest-- 177 c.metaDel(e).release() 178 c.metaCheck(e) 179 180 e.size = int64(len(value.buf)) 181 c.coldTarget += e.size 182 if c.coldTarget > c.targetSize() { 183 c.coldTarget = c.targetSize() 184 } 185 186 e.referenced.Store(false) 187 e.setValue(value) 188 e.ptype = etHot 189 if c.metaAdd(k, e) { 190 value.ref.trace("add-hot") 191 c.sizeHot += e.size 192 c.countHot++ 193 } else { 194 value.ref.trace("skip-hot") 195 e.free() 196 e = nil 197 } 198 } 199 200 c.checkConsistency() 201 202 // Values are initialized with a reference count of 1. That reference count 203 // is being transferred to the returned Handle. 204 return Handle{value: value} 205 } 206 207 func (c *shard) checkConsistency() { 208 // See the comment above the count{Hot,Cold,Test} fields. 209 switch { 210 case c.sizeHot < 0 || c.sizeCold < 0 || c.sizeTest < 0 || c.countHot < 0 || c.countCold < 0 || c.countTest < 0: 211 panic(fmt.Sprintf("pebble: unexpected negative: %d (%d bytes) hot, %d (%d bytes) cold, %d (%d bytes) test", 212 c.countHot, c.sizeHot, c.countCold, c.sizeCold, c.countTest, c.sizeTest)) 213 case c.sizeHot > 0 && c.countHot == 0: 214 panic(fmt.Sprintf("pebble: mismatch %d hot size, %d hot count", c.sizeHot, c.countHot)) 215 case c.sizeCold > 0 && c.countCold == 0: 216 panic(fmt.Sprintf("pebble: mismatch %d cold size, %d cold count", c.sizeCold, c.countCold)) 217 case c.sizeTest > 0 && c.countTest == 0: 218 panic(fmt.Sprintf("pebble: mismatch %d test size, %d test count", c.sizeTest, c.countTest)) 219 } 220 } 221 222 // Delete deletes the cached value for the specified file and offset. 223 func (c *shard) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) { 224 // The common case is there is nothing to delete, so do a quick check with 225 // shared lock. 226 k := key{fileKey{id, fileNum}, offset} 227 c.mu.RLock() 228 exists := c.blocks.Get(k) != nil 229 c.mu.RUnlock() 230 if !exists { 231 return 232 } 233 234 var deletedValue *Value 235 func() { 236 c.mu.Lock() 237 defer c.mu.Unlock() 238 239 e := c.blocks.Get(k) 240 if e == nil { 241 return 242 } 243 deletedValue = c.metaEvict(e) 244 c.checkConsistency() 245 }() 246 // Now that the mutex has been dropped, release the reference which will 247 // potentially free the memory associated with the previous cached value. 248 deletedValue.release() 249 } 250 251 // EvictFile evicts all of the cache values for the specified file. 252 func (c *shard) EvictFile(id uint64, fileNum base.DiskFileNum) { 253 fkey := key{fileKey{id, fileNum}, 0} 254 for c.evictFileRun(fkey) { 255 // Sched switch to give another goroutine an opportunity to acquire the 256 // shard mutex. 257 runtime.Gosched() 258 } 259 } 260 261 func (c *shard) evictFileRun(fkey key) (moreRemaining bool) { 262 // If most of the file's blocks are held in the block cache, evicting all 263 // the blocks may take a while. We don't want to block the entire cache 264 // shard, forcing concurrent readers to wait until we're finished. We drop 265 // the mutex every [blocksPerMutexAcquisition] blocks to give other 266 // goroutines an opportunity to make progress. 267 const blocksPerMutexAcquisition = 5 268 c.mu.Lock() 269 270 // Releasing a value may result in free-ing it back to the memory allocator. 271 // This can have a nontrivial cost that we'd prefer to not pay while holding 272 // the shard mutex, so we collect the evicted values in a local slice and 273 // only release them in a defer after dropping the cache mutex. 274 var obsoleteValuesAlloc [blocksPerMutexAcquisition]*Value 275 obsoleteValues := obsoleteValuesAlloc[:0] 276 defer func() { 277 c.mu.Unlock() 278 for _, v := range obsoleteValues { 279 v.release() 280 } 281 }() 282 283 blocks := c.files.Get(fkey) 284 if blocks == nil { 285 // No blocks for this file. 286 return false 287 } 288 289 // b is the current head of the doubly linked list, and n is the entry after b. 290 for b, n := blocks, (*entry)(nil); len(obsoleteValues) < cap(obsoleteValues); b = n { 291 n = b.fileLink.next 292 obsoleteValues = append(obsoleteValues, c.metaEvict(b)) 293 if b == n { 294 // b == n represents the case where b was the last entry remaining 295 // in the doubly linked list, which is why it pointed at itself. So 296 // no more entries left. 297 c.checkConsistency() 298 return false 299 } 300 } 301 // Exhausted blocksPerMutexAcquisition. 302 return true 303 } 304 305 func (c *shard) Free() { 306 c.mu.Lock() 307 defer c.mu.Unlock() 308 309 // NB: we use metaDel rather than metaEvict in order to avoid the expensive 310 // metaCheck call when the "invariants" build tag is specified. 311 for c.handHot != nil { 312 e := c.handHot 313 c.metaDel(c.handHot).release() 314 e.free() 315 } 316 317 c.blocks.free() 318 c.files.free() 319 } 320 321 func (c *shard) Reserve(n int) { 322 c.mu.Lock() 323 defer c.mu.Unlock() 324 c.reservedSize += int64(n) 325 326 // Changing c.reservedSize will either increase or decrease 327 // the targetSize. But we want coldTarget to be in the range 328 // [0, targetSize]. So, if c.targetSize decreases, make sure 329 // that the coldTarget fits within the limits. 330 targetSize := c.targetSize() 331 if c.coldTarget > targetSize { 332 c.coldTarget = targetSize 333 } 334 335 c.evict() 336 c.checkConsistency() 337 } 338 339 // Size returns the current space used by the cache. 340 func (c *shard) Size() int64 { 341 c.mu.RLock() 342 size := c.sizeHot + c.sizeCold 343 c.mu.RUnlock() 344 return size 345 } 346 347 func (c *shard) targetSize() int64 { 348 target := c.maxSize - c.reservedSize 349 // Always return a positive integer for targetSize. This is so that we don't 350 // end up in an infinite loop in evict(), in cases where reservedSize is 351 // greater than or equal to maxSize. 352 if target < 1 { 353 return 1 354 } 355 return target 356 } 357 358 // Add the entry to the cache, returning true if the entry was added and false 359 // if it would not fit in the cache. 360 func (c *shard) metaAdd(key key, e *entry) bool { 361 c.evict() 362 if e.size > c.targetSize() { 363 // The entry is larger than the target cache size. 364 return false 365 } 366 367 c.blocks.Put(key, e) 368 if entriesGoAllocated { 369 // Go allocated entries need to be referenced from Go memory. The entries 370 // map provides that reference. 371 c.entries[e] = struct{}{} 372 } 373 374 if c.handHot == nil { 375 // first element 376 c.handHot = e 377 c.handCold = e 378 c.handTest = e 379 } else { 380 c.handHot.link(e) 381 } 382 383 if c.handCold == c.handHot { 384 c.handCold = c.handCold.prev() 385 } 386 387 fkey := key.file() 388 if fileBlocks := c.files.Get(fkey); fileBlocks == nil { 389 c.files.Put(fkey, e) 390 } else { 391 fileBlocks.linkFile(e) 392 } 393 return true 394 } 395 396 // Remove the entry from the cache. This removes the entry from the blocks map, 397 // the files map, and ensures that hand{Hot,Cold,Test} are not pointing at the 398 // entry. Returns the deleted value that must be released, if any. 399 func (c *shard) metaDel(e *entry) (deletedValue *Value) { 400 if value := e.peekValue(); value != nil { 401 value.ref.trace("metaDel") 402 } 403 // Remove the pointer to the value. 404 deletedValue = e.val 405 e.val = nil 406 407 c.blocks.Delete(e.key) 408 if entriesGoAllocated { 409 // Go allocated entries need to be referenced from Go memory. The entries 410 // map provides that reference. 411 delete(c.entries, e) 412 } 413 414 if e == c.handHot { 415 c.handHot = c.handHot.prev() 416 } 417 if e == c.handCold { 418 c.handCold = c.handCold.prev() 419 } 420 if e == c.handTest { 421 c.handTest = c.handTest.prev() 422 } 423 424 if e.unlink() == e { 425 // This was the last entry in the cache. 426 c.handHot = nil 427 c.handCold = nil 428 c.handTest = nil 429 } 430 431 fkey := e.key.file() 432 if next := e.unlinkFile(); e == next { 433 c.files.Delete(fkey) 434 } else { 435 c.files.Put(fkey, next) 436 } 437 return deletedValue 438 } 439 440 // Check that the specified entry is not referenced by the cache. 441 func (c *shard) metaCheck(e *entry) { 442 if invariants.Enabled { 443 if _, ok := c.entries[e]; ok { 444 fmt.Fprintf(os.Stderr, "%p: %s unexpectedly found in entries map\n%s", 445 e, e.key, debug.Stack()) 446 os.Exit(1) 447 } 448 if c.blocks.findByValue(e) != nil { 449 fmt.Fprintf(os.Stderr, "%p: %s unexpectedly found in blocks map\n%s\n%s", 450 e, e.key, &c.blocks, debug.Stack()) 451 os.Exit(1) 452 } 453 if c.files.findByValue(e) != nil { 454 fmt.Fprintf(os.Stderr, "%p: %s unexpectedly found in files map\n%s\n%s", 455 e, e.key, &c.files, debug.Stack()) 456 os.Exit(1) 457 } 458 // NB: c.hand{Hot,Cold,Test} are pointers into a single linked list. We 459 // only have to traverse one of them to check all of them. 460 var countHot, countCold, countTest int64 461 var sizeHot, sizeCold, sizeTest int64 462 for t := c.handHot.next(); t != nil; t = t.next() { 463 // Recompute count{Hot,Cold,Test} and size{Hot,Cold,Test}. 464 switch t.ptype { 465 case etHot: 466 countHot++ 467 sizeHot += t.size 468 case etCold: 469 countCold++ 470 sizeCold += t.size 471 case etTest: 472 countTest++ 473 sizeTest += t.size 474 } 475 if e == t { 476 fmt.Fprintf(os.Stderr, "%p: %s unexpectedly found in blocks list\n%s", 477 e, e.key, debug.Stack()) 478 os.Exit(1) 479 } 480 if t == c.handHot { 481 break 482 } 483 } 484 if countHot != c.countHot || countCold != c.countCold || countTest != c.countTest || 485 sizeHot != c.sizeHot || sizeCold != c.sizeCold || sizeTest != c.sizeTest { 486 fmt.Fprintf(os.Stderr, `divergence of Hot,Cold,Test statistics 487 cache's statistics: hot %d, %d, cold %d, %d, test %d, %d 488 recalculated statistics: hot %d, %d, cold %d, %d, test %d, %d\n%s`, 489 c.countHot, c.sizeHot, c.countCold, c.sizeCold, c.countTest, c.sizeTest, 490 countHot, sizeHot, countCold, sizeCold, countTest, sizeTest, 491 debug.Stack()) 492 os.Exit(1) 493 } 494 } 495 } 496 497 func (c *shard) metaEvict(e *entry) (evictedValue *Value) { 498 switch e.ptype { 499 case etHot: 500 c.sizeHot -= e.size 501 c.countHot-- 502 case etCold: 503 c.sizeCold -= e.size 504 c.countCold-- 505 case etTest: 506 c.sizeTest -= e.size 507 c.countTest-- 508 } 509 evictedValue = c.metaDel(e) 510 c.metaCheck(e) 511 e.free() 512 return evictedValue 513 } 514 515 func (c *shard) evict() { 516 for c.targetSize() <= c.sizeHot+c.sizeCold && c.handCold != nil { 517 c.runHandCold(c.countCold, c.sizeCold) 518 } 519 } 520 521 func (c *shard) runHandCold(countColdDebug, sizeColdDebug int64) { 522 // countColdDebug and sizeColdDebug should equal c.countCold and 523 // c.sizeCold. They're parameters only to aid in debugging of 524 // cockroachdb/cockroach#70154. Since they're parameters, their 525 // arguments will appear within stack traces should we encounter 526 // a reproduction. 527 if c.countCold != countColdDebug || c.sizeCold != sizeColdDebug { 528 panic(fmt.Sprintf("runHandCold: cold count and size are %d, %d, arguments are %d and %d", 529 c.countCold, c.sizeCold, countColdDebug, sizeColdDebug)) 530 } 531 532 e := c.handCold 533 if e.ptype == etCold { 534 if e.referenced.Load() { 535 e.referenced.Store(false) 536 e.ptype = etHot 537 c.sizeCold -= e.size 538 c.countCold-- 539 c.sizeHot += e.size 540 c.countHot++ 541 } else { 542 e.setValue(nil) 543 e.ptype = etTest 544 c.sizeCold -= e.size 545 c.countCold-- 546 c.sizeTest += e.size 547 c.countTest++ 548 for c.targetSize() < c.sizeTest && c.handTest != nil { 549 c.runHandTest() 550 } 551 } 552 } 553 554 c.handCold = c.handCold.next() 555 556 for c.targetSize()-c.coldTarget <= c.sizeHot && c.handHot != nil { 557 c.runHandHot() 558 } 559 } 560 561 func (c *shard) runHandHot() { 562 if c.handHot == c.handTest && c.handTest != nil { 563 c.runHandTest() 564 if c.handHot == nil { 565 return 566 } 567 } 568 569 e := c.handHot 570 if e.ptype == etHot { 571 if e.referenced.Load() { 572 e.referenced.Store(false) 573 } else { 574 e.ptype = etCold 575 c.sizeHot -= e.size 576 c.countHot-- 577 c.sizeCold += e.size 578 c.countCold++ 579 } 580 } 581 582 c.handHot = c.handHot.next() 583 } 584 585 func (c *shard) runHandTest() { 586 if c.sizeCold > 0 && c.handTest == c.handCold && c.handCold != nil { 587 // sizeCold is > 0, so assert that countCold == 0. See the 588 // comment above count{Hot,Cold,Test}. 589 if c.countCold == 0 { 590 panic(fmt.Sprintf("pebble: mismatch %d cold size, %d cold count", c.sizeCold, c.countCold)) 591 } 592 593 c.runHandCold(c.countCold, c.sizeCold) 594 if c.handTest == nil { 595 return 596 } 597 } 598 599 e := c.handTest 600 if e.ptype == etTest { 601 c.sizeTest -= e.size 602 c.countTest-- 603 c.coldTarget -= e.size 604 if c.coldTarget < 0 { 605 c.coldTarget = 0 606 } 607 c.metaDel(e).release() 608 c.metaCheck(e) 609 e.free() 610 } 611 612 c.handTest = c.handTest.next() 613 } 614 615 // Metrics holds metrics for the cache. 616 type Metrics struct { 617 // The number of bytes inuse by the cache. 618 Size int64 619 // The count of objects (blocks or tables) in the cache. 620 Count int64 621 // The number of cache hits. 622 Hits int64 623 // The number of cache misses. 624 Misses int64 625 } 626 627 // Cache implements Pebble's sharded block cache. The Clock-PRO algorithm is 628 // used for page replacement 629 // (http://static.usenix.org/event/usenix05/tech/general/full_papers/jiang/jiang_html/html.html). In 630 // order to provide better concurrency, 4 x NumCPUs shards are created, with 631 // each shard being given 1/n of the target cache size. The Clock-PRO algorithm 632 // is run independently on each shard. 633 // 634 // Blocks are keyed by an (id, fileNum, offset) triple. The ID is a namespace 635 // for file numbers and allows a single Cache to be shared between multiple 636 // Pebble instances. The fileNum and offset refer to an sstable file number and 637 // the offset of the block within the file. Because sstables are immutable and 638 // file numbers are never reused, (fileNum,offset) are unique for the lifetime 639 // of a Pebble instance. 640 // 641 // In addition to maintaining a map from (fileNum,offset) to data, each shard 642 // maintains a map of the cached blocks for a particular fileNum. This allows 643 // efficient eviction of all of the blocks for a file which is used when an 644 // sstable is deleted from disk. 645 // 646 // # Memory Management 647 // 648 // In order to reduce pressure on the Go GC, manual memory management is 649 // performed for the data stored in the cache. Manual memory management is 650 // performed by calling into C.{malloc,free} to allocate memory. Cache.Values 651 // are reference counted and the memory backing a manual value is freed when 652 // the reference count drops to 0. 653 // 654 // Manual memory management brings the possibility of memory leaks. It is 655 // imperative that every Handle returned by Cache.{Get,Set} is eventually 656 // released. The "invariants" build tag enables a leak detection facility that 657 // places a GC finalizer on cache.Value. When the cache.Value finalizer is run, 658 // if the underlying buffer is still present a leak has occurred. The "tracing" 659 // build tag enables tracing of cache.Value reference count manipulation and 660 // eases finding where a leak has occurred. These two facilities are usually 661 // used in combination by specifying `-tags invariants,tracing`. Note that 662 // "tracing" produces a significant slowdown, while "invariants" does not. 663 type Cache struct { 664 refs atomic.Int64 665 maxSize int64 666 idAlloc atomic.Uint64 667 shards []shard 668 669 // Traces recorded by Cache.trace. Used for debugging. 670 tr struct { 671 sync.Mutex 672 msgs []string 673 } 674 } 675 676 // New creates a new cache of the specified size. Memory for the cache is 677 // allocated on demand, not during initialization. The cache is created with a 678 // reference count of 1. Each DB it is associated with adds a reference, so the 679 // creator of the cache should usually release their reference after the DB is 680 // created. 681 // 682 // c := cache.New(...) 683 // defer c.Unref() 684 // d, err := pebble.Open(pebble.Options{Cache: c}) 685 func New(size int64) *Cache { 686 // How many cache shards should we create? 687 // 688 // Note that the probability two processors will try to access the same 689 // shard at the same time increases superlinearly with the number of 690 // processors (Eg, consider the brithday problem where each CPU is a person, 691 // and each shard is a possible birthday). 692 // 693 // We could consider growing the number of shards superlinearly, but 694 // increasing the shard count may reduce the effectiveness of the caching 695 // algorithm if frequently-accessed blocks are insufficiently distributed 696 // across shards. If a shard's size is smaller than a single frequently 697 // scanned sstable, then the shard will be unable to hold the entire 698 // frequently-scanned table in memory despite other shards still holding 699 // infrequently accessed blocks. 700 // 701 // Experimentally, we've observed contention contributing to tail latencies 702 // at 2 shards per processor. For now we use 4 shards per processor, 703 // recognizing this may not be final word. 704 m := 4 * runtime.GOMAXPROCS(0) 705 706 // In tests we can use large CPU machines with small cache sizes and have 707 // many caches in existence at a time. If sharding into m shards would 708 // produce too small shards, constrain the number of shards to 4. 709 const minimumShardSize = 4 << 20 // 4 MiB 710 if m > 4 && int(size)/m < minimumShardSize { 711 m = 4 712 } 713 return newShards(size, m) 714 } 715 716 func newShards(size int64, shards int) *Cache { 717 c := &Cache{ 718 maxSize: size, 719 shards: make([]shard, shards), 720 } 721 c.refs.Store(1) 722 c.idAlloc.Store(1) 723 c.trace("alloc", c.refs.Load()) 724 for i := range c.shards { 725 c.shards[i] = shard{ 726 maxSize: size / int64(len(c.shards)), 727 coldTarget: size / int64(len(c.shards)), 728 } 729 if entriesGoAllocated { 730 c.shards[i].entries = make(map[*entry]struct{}) 731 } 732 c.shards[i].blocks.init(16) 733 c.shards[i].files.init(16) 734 } 735 736 // Note: this is a no-op if invariants are disabled or race is enabled. 737 invariants.SetFinalizer(c, func(obj interface{}) { 738 c := obj.(*Cache) 739 if v := c.refs.Load(); v != 0 { 740 c.tr.Lock() 741 fmt.Fprintf(os.Stderr, 742 "pebble: cache (%p) has non-zero reference count: %d\n", c, v) 743 if len(c.tr.msgs) > 0 { 744 fmt.Fprintf(os.Stderr, "%s\n", strings.Join(c.tr.msgs, "\n")) 745 } 746 c.tr.Unlock() 747 os.Exit(1) 748 } 749 }) 750 return c 751 } 752 753 func (c *Cache) getShard(id uint64, fileNum base.DiskFileNum, offset uint64) *shard { 754 if id == 0 { 755 panic("pebble: 0 cache ID is invalid") 756 } 757 758 // Inlined version of fnv.New64 + Write. 759 const offset64 = 14695981039346656037 760 const prime64 = 1099511628211 761 762 h := uint64(offset64) 763 for i := 0; i < 8; i++ { 764 h *= prime64 765 h ^= uint64(id & 0xff) 766 id >>= 8 767 } 768 fileNumVal := uint64(fileNum.FileNum()) 769 for i := 0; i < 8; i++ { 770 h *= prime64 771 h ^= uint64(fileNumVal) & 0xff 772 fileNumVal >>= 8 773 } 774 for i := 0; i < 8; i++ { 775 h *= prime64 776 h ^= uint64(offset & 0xff) 777 offset >>= 8 778 } 779 780 return &c.shards[h%uint64(len(c.shards))] 781 } 782 783 // Ref adds a reference to the cache. The cache only remains valid as long a 784 // reference is maintained to it. 785 func (c *Cache) Ref() { 786 v := c.refs.Add(1) 787 if v <= 1 { 788 panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v)) 789 } 790 c.trace("ref", v) 791 } 792 793 // Unref releases a reference on the cache. 794 func (c *Cache) Unref() { 795 v := c.refs.Add(-1) 796 c.trace("unref", v) 797 switch { 798 case v < 0: 799 panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v)) 800 case v == 0: 801 for i := range c.shards { 802 c.shards[i].Free() 803 } 804 } 805 } 806 807 // Get retrieves the cache value for the specified file and offset, returning 808 // nil if no value is present. 809 func (c *Cache) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle { 810 return c.getShard(id, fileNum, offset).Get(id, fileNum, offset) 811 } 812 813 // Set sets the cache value for the specified file and offset, overwriting an 814 // existing value if present. A Handle is returned which provides faster 815 // retrieval of the cached value than Get (lock-free and avoidance of the map 816 // lookup). The value must have been allocated by Cache.Alloc. 817 func (c *Cache) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *Value) Handle { 818 return c.getShard(id, fileNum, offset).Set(id, fileNum, offset, value) 819 } 820 821 // Delete deletes the cached value for the specified file and offset. 822 func (c *Cache) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) { 823 c.getShard(id, fileNum, offset).Delete(id, fileNum, offset) 824 } 825 826 // EvictFile evicts all of the cache values for the specified file. 827 func (c *Cache) EvictFile(id uint64, fileNum base.DiskFileNum) { 828 if id == 0 { 829 panic("pebble: 0 cache ID is invalid") 830 } 831 for i := range c.shards { 832 c.shards[i].EvictFile(id, fileNum) 833 } 834 } 835 836 // MaxSize returns the max size of the cache. 837 func (c *Cache) MaxSize() int64 { 838 return c.maxSize 839 } 840 841 // Size returns the current space used by the cache. 842 func (c *Cache) Size() int64 { 843 var size int64 844 for i := range c.shards { 845 size += c.shards[i].Size() 846 } 847 return size 848 } 849 850 // Alloc allocates a byte slice of the specified size, possibly reusing 851 // previously allocated but unused memory. The memory backing the value is 852 // manually managed. The caller MUST either add the value to the cache (via 853 // Cache.Set), or release the value (via Cache.Free). Failure to do so will 854 // result in a memory leak. 855 func Alloc(n int) *Value { 856 return newValue(n) 857 } 858 859 // Free frees the specified value. The buffer associated with the value will 860 // possibly be reused, making it invalid to use the buffer after calling 861 // Free. Do not call Free on a value that has been added to the cache. 862 func Free(v *Value) { 863 if n := v.refs(); n > 1 { 864 panic(fmt.Sprintf("pebble: Value has been added to the cache: refs=%d", n)) 865 } 866 v.release() 867 } 868 869 // Reserve N bytes in the cache. This effectively shrinks the size of the cache 870 // by N bytes, without actually consuming any memory. The returned closure 871 // should be invoked to release the reservation. 872 func (c *Cache) Reserve(n int) func() { 873 // Round-up the per-shard reservation. Most reservations should be large, so 874 // this probably doesn't matter in practice. 875 shardN := (n + len(c.shards) - 1) / len(c.shards) 876 for i := range c.shards { 877 c.shards[i].Reserve(shardN) 878 } 879 return func() { 880 if shardN == -1 { 881 panic("pebble: cache reservation already released") 882 } 883 for i := range c.shards { 884 c.shards[i].Reserve(-shardN) 885 } 886 shardN = -1 887 } 888 } 889 890 // Metrics returns the metrics for the cache. 891 func (c *Cache) Metrics() Metrics { 892 var m Metrics 893 for i := range c.shards { 894 s := &c.shards[i] 895 s.mu.RLock() 896 m.Count += int64(s.blocks.Count()) 897 m.Size += s.sizeHot + s.sizeCold 898 s.mu.RUnlock() 899 m.Hits += s.hits.Load() 900 m.Misses += s.misses.Load() 901 } 902 return m 903 } 904 905 // NewID returns a new ID to be used as a namespace for cached file 906 // blocks. 907 func (c *Cache) NewID() uint64 { 908 return c.idAlloc.Add(1) 909 }