github.com/badrootd/nibiru-cometbft@v0.37.5-0.20240307173500-2a75559eee9b/store/store.go (about) 1 package store 2 3 import ( 4 "fmt" 5 "strconv" 6 7 dbm "github.com/badrootd/nibiru-db" 8 "github.com/cosmos/gogoproto/proto" 9 10 cmtsync "github.com/badrootd/nibiru-cometbft/libs/sync" 11 cmtstore "github.com/badrootd/nibiru-cometbft/proto/tendermint/store" 12 cmtproto "github.com/badrootd/nibiru-cometbft/proto/tendermint/types" 13 "github.com/badrootd/nibiru-cometbft/types" 14 ) 15 16 /* 17 BlockStore is a simple low level store for blocks. 18 19 There are three types of information stored: 20 - BlockMeta: Meta information about each block 21 - Block part: Parts of each block, aggregated w/ PartSet 22 - Commit: The commit part of each block, for gossiping precommit votes 23 24 Currently the precommit signatures are duplicated in the Block parts as 25 well as the Commit. In the future this may change, perhaps by moving 26 the Commit data outside the Block. (TODO) 27 28 The store can be assumed to contain all contiguous blocks between base and height (inclusive). 29 30 // NOTE: BlockStore methods will panic if they encounter errors 31 // deserializing loaded data, indicating probable corruption on disk. 32 */ 33 type BlockStore struct { 34 db dbm.DB 35 36 // mtx guards access to the struct fields listed below it. We rely on the database to enforce 37 // fine-grained concurrency control for its data, and thus this mutex does not apply to 38 // database contents. The only reason for keeping these fields in the struct is that the data 39 // can't efficiently be queried from the database since the key encoding we use is not 40 // lexicographically ordered (see https://github.com/tendermint/tendermint/issues/4567). 41 mtx cmtsync.RWMutex 42 base int64 43 height int64 44 } 45 46 // NewBlockStore returns a new BlockStore with the given DB, 47 // initialized to the last height that was committed to the DB. 48 func NewBlockStore(db dbm.DB) *BlockStore { 49 bs := LoadBlockStoreState(db) 50 return &BlockStore{ 51 base: bs.Base, 52 height: bs.Height, 53 db: db, 54 } 55 } 56 57 func (bs *BlockStore) IsEmpty() bool { 58 bs.mtx.RLock() 59 defer bs.mtx.RUnlock() 60 return bs.base == bs.height && bs.base == 0 61 } 62 63 // Base returns the first known contiguous block height, or 0 for empty block stores. 64 func (bs *BlockStore) Base() int64 { 65 bs.mtx.RLock() 66 defer bs.mtx.RUnlock() 67 return bs.base 68 } 69 70 // Height returns the last known contiguous block height, or 0 for empty block stores. 71 func (bs *BlockStore) Height() int64 { 72 bs.mtx.RLock() 73 defer bs.mtx.RUnlock() 74 return bs.height 75 } 76 77 // Size returns the number of blocks in the block store. 78 func (bs *BlockStore) Size() int64 { 79 bs.mtx.RLock() 80 defer bs.mtx.RUnlock() 81 if bs.height == 0 { 82 return 0 83 } 84 return bs.height - bs.base + 1 85 } 86 87 // LoadBase atomically loads the base block meta, or returns nil if no base is found. 88 func (bs *BlockStore) LoadBaseMeta() *types.BlockMeta { 89 bs.mtx.RLock() 90 defer bs.mtx.RUnlock() 91 if bs.base == 0 { 92 return nil 93 } 94 return bs.LoadBlockMeta(bs.base) 95 } 96 97 // LoadBlock returns the block with the given height. 98 // If no block is found for that height, it returns nil. 99 func (bs *BlockStore) LoadBlock(height int64) *types.Block { 100 blockMeta := bs.LoadBlockMeta(height) 101 if blockMeta == nil { 102 return nil 103 } 104 105 pbb := new(cmtproto.Block) 106 buf := []byte{} 107 for i := 0; i < int(blockMeta.BlockID.PartSetHeader.Total); i++ { 108 part := bs.LoadBlockPart(height, i) 109 // If the part is missing (e.g. since it has been deleted after we 110 // loaded the block meta) we consider the whole block to be missing. 111 if part == nil { 112 return nil 113 } 114 buf = append(buf, part.Bytes...) 115 } 116 err := proto.Unmarshal(buf, pbb) 117 if err != nil { 118 // NOTE: The existence of meta should imply the existence of the 119 // block. So, make sure meta is only saved after blocks are saved. 120 panic(fmt.Sprintf("Error reading block: %v", err)) 121 } 122 123 block, err := types.BlockFromProto(pbb) 124 if err != nil { 125 panic(fmt.Errorf("error from proto block: %w", err)) 126 } 127 128 return block 129 } 130 131 // LoadBlockByHash returns the block with the given hash. 132 // If no block is found for that hash, it returns nil. 133 // Panics if it fails to parse height associated with the given hash. 134 func (bs *BlockStore) LoadBlockByHash(hash []byte) *types.Block { 135 bz, err := bs.db.Get(calcBlockHashKey(hash)) 136 if err != nil { 137 panic(err) 138 } 139 if len(bz) == 0 { 140 return nil 141 } 142 143 s := string(bz) 144 height, err := strconv.ParseInt(s, 10, 64) 145 if err != nil { 146 panic(fmt.Sprintf("failed to extract height from %s: %v", s, err)) 147 } 148 return bs.LoadBlock(height) 149 } 150 151 // LoadBlockPart returns the Part at the given index 152 // from the block at the given height. 153 // If no part is found for the given height and index, it returns nil. 154 func (bs *BlockStore) LoadBlockPart(height int64, index int) *types.Part { 155 pbpart := new(cmtproto.Part) 156 157 bz, err := bs.db.Get(calcBlockPartKey(height, index)) 158 if err != nil { 159 panic(err) 160 } 161 if len(bz) == 0 { 162 return nil 163 } 164 165 err = proto.Unmarshal(bz, pbpart) 166 if err != nil { 167 panic(fmt.Errorf("unmarshal to cmtproto.Part failed: %w", err)) 168 } 169 part, err := types.PartFromProto(pbpart) 170 if err != nil { 171 panic(fmt.Sprintf("Error reading block part: %v", err)) 172 } 173 174 return part 175 } 176 177 // LoadBlockMeta returns the BlockMeta for the given height. 178 // If no block is found for the given height, it returns nil. 179 func (bs *BlockStore) LoadBlockMeta(height int64) *types.BlockMeta { 180 pbbm := new(cmtproto.BlockMeta) 181 bz, err := bs.db.Get(calcBlockMetaKey(height)) 182 if err != nil { 183 panic(err) 184 } 185 186 if len(bz) == 0 { 187 return nil 188 } 189 190 err = proto.Unmarshal(bz, pbbm) 191 if err != nil { 192 panic(fmt.Errorf("unmarshal to cmtproto.BlockMeta: %w", err)) 193 } 194 195 blockMeta, err := types.BlockMetaFromProto(pbbm) 196 if err != nil { 197 panic(fmt.Errorf("error from proto blockMeta: %w", err)) 198 } 199 200 return blockMeta 201 } 202 203 // LoadBlockMetaByHash returns the blockmeta who's header corresponds to the given 204 // hash. If none is found, returns nil. 205 func (bs *BlockStore) LoadBlockMetaByHash(hash []byte) *types.BlockMeta { 206 bz, err := bs.db.Get(calcBlockHashKey(hash)) 207 if err != nil { 208 panic(err) 209 } 210 if len(bz) == 0 { 211 return nil 212 } 213 214 s := string(bz) 215 height, err := strconv.ParseInt(s, 10, 64) 216 if err != nil { 217 panic(fmt.Sprintf("failed to extract height from %s: %v", s, err)) 218 } 219 return bs.LoadBlockMeta(height) 220 } 221 222 // LoadBlockCommit returns the Commit for the given height. 223 // This commit consists of the +2/3 and other Precommit-votes for block at `height`, 224 // and it comes from the block.LastCommit for `height+1`. 225 // If no commit is found for the given height, it returns nil. 226 func (bs *BlockStore) LoadBlockCommit(height int64) *types.Commit { 227 pbc := new(cmtproto.Commit) 228 bz, err := bs.db.Get(calcBlockCommitKey(height)) 229 if err != nil { 230 panic(err) 231 } 232 if len(bz) == 0 { 233 return nil 234 } 235 err = proto.Unmarshal(bz, pbc) 236 if err != nil { 237 panic(fmt.Errorf("error reading block commit: %w", err)) 238 } 239 commit, err := types.CommitFromProto(pbc) 240 if err != nil { 241 panic(fmt.Sprintf("Error reading block commit: %v", err)) 242 } 243 return commit 244 } 245 246 // LoadSeenCommit returns the locally seen Commit for the given height. 247 // This is useful when we've seen a commit, but there has not yet been 248 // a new block at `height + 1` that includes this commit in its block.LastCommit. 249 func (bs *BlockStore) LoadSeenCommit(height int64) *types.Commit { 250 pbc := new(cmtproto.Commit) 251 bz, err := bs.db.Get(calcSeenCommitKey(height)) 252 if err != nil { 253 panic(err) 254 } 255 if len(bz) == 0 { 256 return nil 257 } 258 err = proto.Unmarshal(bz, pbc) 259 if err != nil { 260 panic(fmt.Sprintf("error reading block seen commit: %v", err)) 261 } 262 263 commit, err := types.CommitFromProto(pbc) 264 if err != nil { 265 panic(fmt.Errorf("error from proto commit: %w", err)) 266 } 267 return commit 268 } 269 270 // PruneBlocks removes block up to (but not including) a height. It returns number of blocks pruned. 271 func (bs *BlockStore) PruneBlocks(height int64) (uint64, error) { 272 if height <= 0 { 273 return 0, fmt.Errorf("height must be greater than 0") 274 } 275 bs.mtx.RLock() 276 if height > bs.height { 277 bs.mtx.RUnlock() 278 return 0, fmt.Errorf("cannot prune beyond the latest height %v", bs.height) 279 } 280 base := bs.base 281 bs.mtx.RUnlock() 282 if height < base { 283 return 0, fmt.Errorf("cannot prune to height %v, it is lower than base height %v", 284 height, base) 285 } 286 287 pruned := uint64(0) 288 batch := bs.db.NewBatch() 289 defer batch.Close() 290 flush := func(batch dbm.Batch, base int64) error { 291 // We can't trust batches to be atomic, so update base first to make sure noone 292 // tries to access missing blocks. 293 bs.mtx.Lock() 294 bs.base = base 295 bs.mtx.Unlock() 296 bs.saveState() 297 298 err := batch.WriteSync() 299 if err != nil { 300 return fmt.Errorf("failed to prune up to height %v: %w", base, err) 301 } 302 batch.Close() 303 return nil 304 } 305 306 for h := base; h < height; h++ { 307 meta := bs.LoadBlockMeta(h) 308 if meta == nil { // assume already deleted 309 continue 310 } 311 if err := batch.Delete(calcBlockMetaKey(h)); err != nil { 312 return 0, err 313 } 314 if err := batch.Delete(calcBlockHashKey(meta.BlockID.Hash)); err != nil { 315 return 0, err 316 } 317 if err := batch.Delete(calcBlockCommitKey(h)); err != nil { 318 return 0, err 319 } 320 if err := batch.Delete(calcSeenCommitKey(h)); err != nil { 321 return 0, err 322 } 323 for p := 0; p < int(meta.BlockID.PartSetHeader.Total); p++ { 324 if err := batch.Delete(calcBlockPartKey(h, p)); err != nil { 325 return 0, err 326 } 327 } 328 pruned++ 329 330 // flush every 1000 blocks to avoid batches becoming too large 331 if pruned%1000 == 0 && pruned > 0 { 332 err := flush(batch, h) 333 if err != nil { 334 return 0, err 335 } 336 batch = bs.db.NewBatch() 337 defer batch.Close() 338 } 339 } 340 341 err := flush(batch, height) 342 if err != nil { 343 return 0, err 344 } 345 return pruned, nil 346 } 347 348 // SaveBlock persists the given block, blockParts, and seenCommit to the underlying db. 349 // blockParts: Must be parts of the block 350 // seenCommit: The +2/3 precommits that were seen which committed at height. 351 // 352 // If all the nodes restart after committing a block, 353 // we need this to reload the precommits to catch-up nodes to the 354 // most recent height. Otherwise they'd stall at H-1. 355 func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) { 356 if block == nil { 357 panic("BlockStore can only save a non-nil block") 358 } 359 360 height := block.Height 361 hash := block.Hash() 362 363 if g, w := height, bs.Height()+1; bs.Base() > 0 && g != w { 364 panic(fmt.Sprintf("BlockStore can only save contiguous blocks. Wanted %v, got %v", w, g)) 365 } 366 if !blockParts.IsComplete() { 367 panic("BlockStore can only save complete block part sets") 368 } 369 370 // Save block parts. This must be done before the block meta, since callers 371 // typically load the block meta first as an indication that the block exists 372 // and then go on to load block parts - we must make sure the block is 373 // complete as soon as the block meta is written. 374 for i := 0; i < int(blockParts.Total()); i++ { 375 part := blockParts.GetPart(i) 376 bs.saveBlockPart(height, i, part) 377 } 378 379 // Save block meta 380 blockMeta := types.NewBlockMeta(block, blockParts) 381 pbm := blockMeta.ToProto() 382 if pbm == nil { 383 panic("nil blockmeta") 384 } 385 metaBytes := mustEncode(pbm) 386 if err := bs.db.Set(calcBlockMetaKey(height), metaBytes); err != nil { 387 panic(err) 388 } 389 if err := bs.db.Set(calcBlockHashKey(hash), []byte(fmt.Sprintf("%d", height))); err != nil { 390 panic(err) 391 } 392 393 // Save block commit (duplicate and separate from the Block) 394 pbc := block.LastCommit.ToProto() 395 blockCommitBytes := mustEncode(pbc) 396 if err := bs.db.Set(calcBlockCommitKey(height-1), blockCommitBytes); err != nil { 397 panic(err) 398 } 399 400 // Save seen commit (seen +2/3 precommits for block) 401 // NOTE: we can delete this at a later height 402 pbsc := seenCommit.ToProto() 403 seenCommitBytes := mustEncode(pbsc) 404 if err := bs.db.Set(calcSeenCommitKey(height), seenCommitBytes); err != nil { 405 panic(err) 406 } 407 408 // Done! 409 bs.mtx.Lock() 410 bs.height = height 411 if bs.base == 0 { 412 bs.base = height 413 } 414 bs.mtx.Unlock() 415 416 // Save new BlockStoreState descriptor. This also flushes the database. 417 bs.saveState() 418 } 419 420 func (bs *BlockStore) saveBlockPart(height int64, index int, part *types.Part) { 421 pbp, err := part.ToProto() 422 if err != nil { 423 panic(fmt.Errorf("unable to make part into proto: %w", err)) 424 } 425 partBytes := mustEncode(pbp) 426 if err := bs.db.Set(calcBlockPartKey(height, index), partBytes); err != nil { 427 panic(err) 428 } 429 } 430 431 func (bs *BlockStore) saveState() { 432 bs.mtx.RLock() 433 bss := cmtstore.BlockStoreState{ 434 Base: bs.base, 435 Height: bs.height, 436 } 437 bs.mtx.RUnlock() 438 SaveBlockStoreState(&bss, bs.db) 439 } 440 441 // SaveSeenCommit saves a seen commit, used by e.g. the state sync reactor when bootstrapping node. 442 func (bs *BlockStore) SaveSeenCommit(height int64, seenCommit *types.Commit) error { 443 pbc := seenCommit.ToProto() 444 seenCommitBytes, err := proto.Marshal(pbc) 445 if err != nil { 446 return fmt.Errorf("unable to marshal commit: %w", err) 447 } 448 return bs.db.Set(calcSeenCommitKey(height), seenCommitBytes) 449 } 450 451 func (bs *BlockStore) Close() error { 452 return bs.db.Close() 453 } 454 455 //----------------------------------------------------------------------------- 456 457 func calcBlockMetaKey(height int64) []byte { 458 return []byte(fmt.Sprintf("H:%v", height)) 459 } 460 461 func calcBlockPartKey(height int64, partIndex int) []byte { 462 return []byte(fmt.Sprintf("P:%v:%v", height, partIndex)) 463 } 464 465 func calcBlockCommitKey(height int64) []byte { 466 return []byte(fmt.Sprintf("C:%v", height)) 467 } 468 469 func calcSeenCommitKey(height int64) []byte { 470 return []byte(fmt.Sprintf("SC:%v", height)) 471 } 472 473 func calcBlockHashKey(hash []byte) []byte { 474 return []byte(fmt.Sprintf("BH:%x", hash)) 475 } 476 477 //----------------------------------------------------------------------------- 478 479 var blockStoreKey = []byte("blockStore") 480 481 // SaveBlockStoreState persists the blockStore state to the database. 482 func SaveBlockStoreState(bsj *cmtstore.BlockStoreState, db dbm.DB) { 483 bytes, err := proto.Marshal(bsj) 484 if err != nil { 485 panic(fmt.Sprintf("Could not marshal state bytes: %v", err)) 486 } 487 if err := db.SetSync(blockStoreKey, bytes); err != nil { 488 panic(err) 489 } 490 } 491 492 // LoadBlockStoreState returns the BlockStoreState as loaded from disk. 493 // If no BlockStoreState was previously persisted, it returns the zero value. 494 func LoadBlockStoreState(db dbm.DB) cmtstore.BlockStoreState { 495 bytes, err := db.Get(blockStoreKey) 496 if err != nil { 497 panic(err) 498 } 499 500 if len(bytes) == 0 { 501 return cmtstore.BlockStoreState{ 502 Base: 0, 503 Height: 0, 504 } 505 } 506 507 var bsj cmtstore.BlockStoreState 508 if err := proto.Unmarshal(bytes, &bsj); err != nil { 509 panic(fmt.Sprintf("Could not unmarshal bytes: %X", bytes)) 510 } 511 512 // Backwards compatibility with persisted data from before Base existed. 513 if bsj.Height > 0 && bsj.Base == 0 { 514 bsj.Base = 1 515 } 516 return bsj 517 } 518 519 // mustEncode proto encodes a proto.message and panics if fails 520 func mustEncode(pb proto.Message) []byte { 521 bz, err := proto.Marshal(pb) 522 if err != nil { 523 panic(fmt.Errorf("unable to marshal: %w", err)) 524 } 525 return bz 526 } 527 528 //----------------------------------------------------------------------------- 529 530 // DeleteLatestBlock removes the block pointed to by height, 531 // lowering height by one. 532 func (bs *BlockStore) DeleteLatestBlock() error { 533 bs.mtx.RLock() 534 targetHeight := bs.height 535 bs.mtx.RUnlock() 536 537 batch := bs.db.NewBatch() 538 defer batch.Close() 539 540 // delete what we can, skipping what's already missing, to ensure partial 541 // blocks get deleted fully. 542 if meta := bs.LoadBlockMeta(targetHeight); meta != nil { 543 if err := batch.Delete(calcBlockHashKey(meta.BlockID.Hash)); err != nil { 544 return err 545 } 546 for p := 0; p < int(meta.BlockID.PartSetHeader.Total); p++ { 547 if err := batch.Delete(calcBlockPartKey(targetHeight, p)); err != nil { 548 return err 549 } 550 } 551 } 552 if err := batch.Delete(calcBlockCommitKey(targetHeight)); err != nil { 553 return err 554 } 555 if err := batch.Delete(calcSeenCommitKey(targetHeight)); err != nil { 556 return err 557 } 558 // delete last, so as to not leave keys built on meta.BlockID dangling 559 if err := batch.Delete(calcBlockMetaKey(targetHeight)); err != nil { 560 return err 561 } 562 563 bs.mtx.Lock() 564 bs.height = targetHeight - 1 565 bs.mtx.Unlock() 566 bs.saveState() 567 568 err := batch.WriteSync() 569 if err != nil { 570 return fmt.Errorf("failed to delete height %v: %w", targetHeight, err) 571 } 572 return nil 573 }