github.com/koko1123/flow-go-1@v0.29.6/module/mempool/herocache/backdata/cache.go (about) 1 package herocache 2 3 import ( 4 "encoding/binary" 5 "time" 6 _ "unsafe" // for linking runtimeNano 7 8 "github.com/rs/zerolog" 9 "go.uber.org/atomic" 10 11 "github.com/koko1123/flow-go-1/model/flow" 12 "github.com/koko1123/flow-go-1/module" 13 "github.com/koko1123/flow-go-1/module/mempool/herocache/backdata/heropool" 14 "github.com/koko1123/flow-go-1/utils/logging" 15 ) 16 17 //go:linkname runtimeNano runtime.nanotime 18 func runtimeNano() int64 19 20 const ( 21 slotsPerBucket = uint64(16) 22 23 // slotAgeUnallocated defines an unallocated slot with zero age. 24 slotAgeUnallocated = uint64(0) 25 26 // telemetryCounterInterval is the number of required interactions with 27 // this back data prior to printing any log. This is done as a slow-down mechanism 28 // to avoid spamming logs upon read/write heavy operations. An interaction can be 29 // a read or write. 30 telemetryCounterInterval = uint64(10_000) 31 32 // telemetryDurationInterval is the required elapsed duration interval 33 // prior to printing any log. This is done as a slow-down mechanism 34 // to avoid spamming logs upon read/write heavy operations. 35 telemetryDurationInterval = 10 * time.Second 36 ) 37 38 // bucketIndex is data type representing a bucket index. 39 type bucketIndex uint64 40 41 // slotIndex is data type representing a slot index in a bucket. 42 type slotIndex uint64 43 44 // sha32of256 is a 32-bits prefix flow.Identifier used to determine the bucketIndex of the entity 45 // it represents. 46 type sha32of256 uint32 47 48 // slot is an internal notion corresponding to the identifier of an entity that is 49 // meant to be stored in this Cache. 50 type slot struct { 51 slotAge uint64 // age of this slot. 52 entityIndex heropool.EIndex // link to actual entity. 53 entityId32of256 sha32of256 // the 32-bits prefix of entity identifier. 54 } 55 56 // slotBucket represents a bucket of slots. 57 type slotBucket struct { 58 slots [slotsPerBucket]slot 59 } 60 61 // Cache implements an array-based generic memory pool backed by a fixed total array. 62 type Cache struct { 63 logger zerolog.Logger 64 collector module.HeroCacheMetrics 65 // NOTE: as a BackData implementation, Cache must be non-blocking. 66 // Concurrency management is done by overlay Backend. 67 sizeLimit uint32 68 slotCount uint64 // total number of non-expired key-values 69 bucketNum uint64 // total number of buckets (i.e., total of buckets) 70 ejectionMode heropool.EjectionMode 71 // buckets keeps the slots (i.e., entityId) of the (entityId, entity) pairs that are maintained in this BackData. 72 buckets []slotBucket 73 // entities keeps the values (i.e., entity) of the (entityId, entity) pairs that are maintained in this BackData. 74 entities *heropool.Pool 75 // telemetry 76 // 77 // availableSlotHistogram[i] represents number of buckets with i 78 // available (i.e., empty) slots to take. 79 availableSlotHistogram []uint64 80 // interactionCounter keeps track of interactions made with 81 // Cache. Invoking any methods of this BackData is considered 82 // towards an interaction. The interaction counter is set to zero whenever 83 // it reaches a predefined limit. Its purpose is to manage the speed at which 84 // telemetry logs are printed. 85 interactionCounter *atomic.Uint64 86 // lastTelemetryDump keeps track of the last time telemetry logs dumped. 87 // Its purpose is to manage the speed at which telemetry logs are printed. 88 lastTelemetryDump *atomic.Int64 89 } 90 91 // DefaultOversizeFactor determines the default oversizing factor of HeroCache. 92 // What is oversize factor? 93 // Imagine adding n keys, rounds times to a hash table with a fixed number slots per bucket. 94 // The number of buckets can be chosen upon initialization and then never changes. 95 // If a bucket is full then the oldest key is ejected, and if that key is too new, this is a bucket overflow. 96 // How many buckets are needed to avoid a bucket overflow assuming cryptographic key hashing is used? 97 // The overSizeFactor is used to determine the number of buckets. 98 // Assume n 16, rounds 3, & slotsPerBucket 3 for the tiny example below showing overSizeFactor 1 thru 6. 99 // As overSizeFactor is increased the chance of overflowing a bucket is decreased. 100 // With overSizeFactor 1: 8 from 48 keys can be added before bucket overflow. 101 // With overSizeFactor 2: 10 from 48 keys can be added before bucket overflow. 102 // With overSizeFactor 3: 13 from 48 keys can be added before bucket overflow. 103 // With overSizeFactor 4: 15 from 48 keys can be added before bucket overflow. 104 // With overSizeFactor 5: 27 from 48 keys can be added before bucket overflow. 105 // With overSizeFactor 6: 48 from 48 keys can be added. 106 // The default overSizeFactor factor is different in the package code because slotsPerBucket is > 3. 107 const DefaultOversizeFactor = uint32(8) 108 109 func NewCache(sizeLimit uint32, 110 oversizeFactor uint32, 111 ejectionMode heropool.EjectionMode, 112 logger zerolog.Logger, 113 collector module.HeroCacheMetrics) *Cache { 114 115 // total buckets. 116 capacity := uint64(sizeLimit * oversizeFactor) 117 bucketNum := capacity / slotsPerBucket 118 if capacity%slotsPerBucket != 0 { 119 // accounting for remainder. 120 bucketNum++ 121 } 122 123 bd := &Cache{ 124 logger: logger, 125 collector: collector, 126 bucketNum: bucketNum, 127 sizeLimit: sizeLimit, 128 buckets: make([]slotBucket, bucketNum), 129 ejectionMode: ejectionMode, 130 entities: heropool.NewHeroPool(sizeLimit, ejectionMode), 131 availableSlotHistogram: make([]uint64, slotsPerBucket+1), // +1 is to account for empty buckets as well. 132 interactionCounter: atomic.NewUint64(0), 133 lastTelemetryDump: atomic.NewInt64(0), 134 } 135 136 return bd 137 } 138 139 // Has checks if backdata already contains the entity with the given identifier. 140 func (c *Cache) Has(entityID flow.Identifier) bool { 141 defer c.logTelemetry() 142 143 _, _, _, ok := c.get(entityID) 144 return ok 145 } 146 147 // Add adds the given entity to the backdata. 148 func (c *Cache) Add(entityID flow.Identifier, entity flow.Entity) bool { 149 defer c.logTelemetry() 150 151 return c.put(entityID, entity) 152 } 153 154 // Remove removes the entity with the given identifier. 155 func (c *Cache) Remove(entityID flow.Identifier) (flow.Entity, bool) { 156 defer c.logTelemetry() 157 158 entity, bucketIndex, sliceIndex, exists := c.get(entityID) 159 if !exists { 160 return nil, false 161 } 162 // removes value from underlying entities list. 163 c.invalidateEntity(bucketIndex, sliceIndex) 164 165 // frees up slot 166 c.unuseSlot(bucketIndex, sliceIndex) 167 168 c.collector.OnKeyRemoved(c.entities.Size()) 169 return entity, true 170 } 171 172 // Adjust adjusts the entity using the given function if the given identifier can be found. 173 // Returns a bool which indicates whether the entity was updated as well as the updated entity. 174 func (c *Cache) Adjust(entityID flow.Identifier, f func(flow.Entity) flow.Entity) (flow.Entity, bool) { 175 defer c.logTelemetry() 176 177 entity, removed := c.Remove(entityID) 178 if !removed { 179 return nil, false 180 } 181 182 newEntity := f(entity) 183 newEntityID := newEntity.ID() 184 185 c.put(newEntityID, newEntity) 186 187 return newEntity, true 188 } 189 190 // ByID returns the given entity from the backdata. 191 func (c *Cache) ByID(entityID flow.Identifier) (flow.Entity, bool) { 192 defer c.logTelemetry() 193 194 entity, _, _, ok := c.get(entityID) 195 return entity, ok 196 } 197 198 // Size returns the size of the backdata, i.e., total number of stored (entityId, entity) pairs. 199 func (c Cache) Size() uint { 200 defer c.logTelemetry() 201 202 return uint(c.entities.Size()) 203 } 204 205 // Head returns the head of queue. 206 // Boolean return value determines whether there is a head available. 207 func (c Cache) Head() (flow.Entity, bool) { 208 return c.entities.Head() 209 } 210 211 // All returns all entities stored in the backdata. 212 func (c Cache) All() map[flow.Identifier]flow.Entity { 213 defer c.logTelemetry() 214 215 entitiesList := c.entities.All() 216 all := make(map[flow.Identifier]flow.Entity, len(c.entities.All())) 217 218 total := len(entitiesList) 219 for i := 0; i < total; i++ { 220 p := entitiesList[i] 221 all[p.Id()] = p.Entity() 222 } 223 224 return all 225 } 226 227 // Identifiers returns the list of identifiers of entities stored in the backdata. 228 func (c Cache) Identifiers() flow.IdentifierList { 229 defer c.logTelemetry() 230 231 ids := make(flow.IdentifierList, c.entities.Size()) 232 for i, p := range c.entities.All() { 233 ids[i] = p.Id() 234 } 235 236 return ids 237 } 238 239 // Entities returns the list of entities stored in the backdata. 240 func (c Cache) Entities() []flow.Entity { 241 defer c.logTelemetry() 242 243 entities := make([]flow.Entity, c.entities.Size()) 244 for i, p := range c.entities.All() { 245 entities[i] = p.Entity() 246 } 247 248 return entities 249 } 250 251 // Clear removes all entities from the backdata. 252 func (c *Cache) Clear() { 253 defer c.logTelemetry() 254 255 c.buckets = make([]slotBucket, c.bucketNum) 256 c.entities = heropool.NewHeroPool(c.sizeLimit, c.ejectionMode) 257 c.availableSlotHistogram = make([]uint64, slotsPerBucket+1) 258 c.interactionCounter = atomic.NewUint64(0) 259 c.lastTelemetryDump = atomic.NewInt64(0) 260 c.slotCount = 0 261 } 262 263 // put writes the (entityId, entity) pair into this BackData. Boolean return value 264 // determines whether the write operation was successful. A write operation fails when there is already 265 // a duplicate entityId exists in the BackData, and that entityId is linked to a valid entity. 266 func (c *Cache) put(entityId flow.Identifier, entity flow.Entity) bool { 267 c.collector.OnKeyPutAttempt(c.entities.Size()) 268 269 entityId32of256, b := c.entityId32of256AndBucketIndex(entityId) 270 slotToUse, unique := c.slotIndexInBucket(b, entityId32of256, entityId) 271 if !unique { 272 // entityId already exists 273 c.collector.OnKeyPutDeduplicated() 274 return false 275 } 276 277 if linkedId, _, ok := c.linkedEntityOf(b, slotToUse); ok { 278 // bucket is full, and we are replacing an already linked (but old) slot that has a valid value, hence 279 // we should remove its value from underlying entities list. 280 c.invalidateEntity(b, slotToUse) 281 c.collector.OnEntityEjectionDueToEmergency() 282 c.logger.Warn(). 283 Hex("replaced_entity_id", logging.ID(linkedId)). 284 Hex("added_entity_id", logging.ID(entityId)). 285 Msg("emergency ejection, adding entity to cache resulted in replacing a valid key, potential collision") 286 } 287 288 c.slotCount++ 289 entityIndex, slotAvailable, ejectionHappened := c.entities.Add(entityId, entity, c.ownerIndexOf(b, slotToUse)) 290 if !slotAvailable { 291 c.collector.OnKeyPutDrop() 292 return false 293 } 294 295 if ejectionHappened { 296 // cache is at its full size and ejection happened to make room for this new entity. 297 c.collector.OnEntityEjectionDueToFullCapacity() 298 } 299 300 c.buckets[b].slots[slotToUse].slotAge = c.slotCount 301 c.buckets[b].slots[slotToUse].entityIndex = entityIndex 302 c.buckets[b].slots[slotToUse].entityId32of256 = entityId32of256 303 c.collector.OnKeyPutSuccess(c.entities.Size()) 304 return true 305 } 306 307 // get retrieves the entity corresponding to given identifier from underlying entities list. 308 // The boolean return value determines whether an entity with given id exists in the BackData. 309 func (c *Cache) get(entityID flow.Identifier) (flow.Entity, bucketIndex, slotIndex, bool) { 310 entityId32of256, b := c.entityId32of256AndBucketIndex(entityID) 311 for s := slotIndex(0); s < slotIndex(slotsPerBucket); s++ { 312 if c.buckets[b].slots[s].entityId32of256 != entityId32of256 { 313 continue 314 } 315 316 id, entity, linked := c.linkedEntityOf(b, s) 317 if !linked { 318 // no linked entity for this (bucketIndex, slotIndex) pair. 319 c.collector.OnKeyGetFailure() 320 return nil, 0, 0, false 321 } 322 323 if id != entityID { 324 // checking identifiers fully. 325 continue 326 } 327 328 c.collector.OnKeyGetSuccess() 329 return entity, b, s, true 330 } 331 332 c.collector.OnKeyGetFailure() 333 return nil, 0, 0, false 334 } 335 336 // entityId32of256AndBucketIndex determines the id prefix as well as the bucket index corresponding to the 337 // given identifier. 338 func (c Cache) entityId32of256AndBucketIndex(id flow.Identifier) (sha32of256, bucketIndex) { 339 // uint64(id[0:8]) used to compute bucket index for which this identifier belongs to 340 b := binary.LittleEndian.Uint64(id[0:8]) % c.bucketNum 341 342 // uint32(id[8:12]) used to compute a shorter identifier for this id to represent in memory. 343 entityId32of256 := binary.LittleEndian.Uint32(id[8:12]) 344 345 return sha32of256(entityId32of256), bucketIndex(b) 346 } 347 348 // expiryThreshold returns the threshold for which all slots with index below threshold are considered old enough for eviction. 349 func (c Cache) expiryThreshold() uint64 { 350 var expiryThreshold uint64 = 0 351 if c.slotCount > uint64(c.sizeLimit) { 352 // total number of slots written are above the predefined limit 353 expiryThreshold = c.slotCount - uint64(c.sizeLimit) 354 } 355 356 return expiryThreshold 357 } 358 359 // slotIndexInBucket returns a free slot for this entityId in the bucket. In case the bucket is full, it invalidates the oldest valid slot, 360 // and returns its index as free slot. It returns false if the entityId already exists in this bucket. 361 func (c *Cache) slotIndexInBucket(b bucketIndex, slotId sha32of256, entityId flow.Identifier) (slotIndex, bool) { 362 slotToUse := slotIndex(0) 363 expiryThreshold := c.expiryThreshold() 364 availableSlotCount := uint64(0) // for telemetry logs. 365 366 oldestSlotInBucket := c.slotCount + 1 // initializes the oldest slot to current max. 367 368 for s := slotIndex(0); s < slotIndex(slotsPerBucket); s++ { 369 if c.buckets[b].slots[s].slotAge < oldestSlotInBucket { 370 // record slot s as oldest slot 371 oldestSlotInBucket = c.buckets[b].slots[s].slotAge 372 slotToUse = s 373 } 374 375 if c.buckets[b].slots[s].slotAge <= expiryThreshold { 376 // slot technically expired or never assigned 377 availableSlotCount++ 378 continue 379 } 380 381 if c.buckets[b].slots[s].entityId32of256 != slotId { 382 // slot id is distinct and fresh, and hence move to next slot. 383 continue 384 } 385 386 id, _, linked := c.linkedEntityOf(b, s) 387 if !linked { 388 // slot is not linked to a valid entity, hence, can be used 389 // as an available slot. 390 availableSlotCount++ 391 slotToUse = s 392 continue 393 } 394 395 if id != entityId { 396 // slot is fresh, fully distinct, and linked. Hence, 397 // moving to next slot. 398 continue 399 } 400 401 // entity ID already exists in the bucket 402 return 0, false 403 } 404 405 c.availableSlotHistogram[availableSlotCount]++ 406 c.collector.BucketAvailableSlots(availableSlotCount, slotsPerBucket) 407 return slotToUse, true 408 } 409 410 // ownerIndexOf maps the (bucketIndex, slotIndex) pair to a canonical unique (scalar) index. 411 // This scalar index is used to represent this (bucketIndex, slotIndex) pair in the underlying 412 // entities list. 413 func (c Cache) ownerIndexOf(b bucketIndex, s slotIndex) uint64 { 414 return (uint64(b) * slotsPerBucket) + uint64(s) 415 } 416 417 // linkedEntityOf returns the entity linked to this (bucketIndex, slotIndex) pair from the underlying entities list. 418 // By a linked entity, we mean if the entity has an owner index matching to (bucketIndex, slotIndex). 419 // The bool return value corresponds to whether there is a linked entity to this (bucketIndex, slotIndex) or not. 420 func (c *Cache) linkedEntityOf(b bucketIndex, s slotIndex) (flow.Identifier, flow.Entity, bool) { 421 if c.buckets[b].slots[s].slotAge == slotAgeUnallocated { 422 // slotIndex never used, or recently invalidated, hence 423 // does not have any linked entity 424 return flow.Identifier{}, nil, false 425 } 426 427 // retrieving entity index in the underlying entities linked-list 428 valueIndex := c.buckets[b].slots[s].entityIndex 429 id, entity, owner := c.entities.Get(valueIndex) 430 if c.ownerIndexOf(b, s) != owner { 431 // entity is not linked to this (bucketIndex, slotIndex) 432 c.buckets[b].slots[s].slotAge = slotAgeUnallocated 433 return flow.Identifier{}, nil, false 434 } 435 436 return id, entity, true 437 } 438 439 // logTelemetry prints telemetry logs depending on number of interactions and last time telemetry has been logged. 440 func (c *Cache) logTelemetry() { 441 counter := c.interactionCounter.Inc() 442 if counter < telemetryCounterInterval { 443 // not enough interactions to log. 444 return 445 } 446 if time.Duration(runtimeNano()-c.lastTelemetryDump.Load()) < telemetryDurationInterval { 447 // not long elapsed since last log. 448 return 449 } 450 if !c.interactionCounter.CompareAndSwap(counter, 0) { 451 // raced on CAS, hence, not logging. 452 return 453 } 454 455 lg := c.logger.With(). 456 Uint64("total_slots_written", c.slotCount). 457 Uint64("total_interactions_since_last_log", counter).Logger() 458 459 for i := range c.availableSlotHistogram { 460 lg = lg.With(). 461 Int("available_slots", i). 462 Uint64("total_buckets", c.availableSlotHistogram[i]). 463 Logger() 464 } 465 466 lg.Info().Msg("logging telemetry") 467 c.lastTelemetryDump.Store(runtimeNano()) 468 } 469 470 // unuseSlot marks slot as free so that it is ready to be re-used. 471 func (c *Cache) unuseSlot(b bucketIndex, s slotIndex) { 472 c.buckets[b].slots[s].slotAge = slotAgeUnallocated 473 } 474 475 // invalidateEntity removes the entity linked to the specified slot from the underlying entities 476 // list. So that entity slot is made available to take if needed. 477 func (c *Cache) invalidateEntity(b bucketIndex, s slotIndex) { 478 c.entities.Remove(c.buckets[b].slots[s].entityIndex) 479 }