github.com/noirx94/tendermintmp@v0.0.1/evidence/pool.go (about) 1 package evidence 2 3 import ( 4 "bytes" 5 "errors" 6 "fmt" 7 "sync" 8 "sync/atomic" 9 "time" 10 11 "github.com/gogo/protobuf/proto" 12 gogotypes "github.com/gogo/protobuf/types" 13 dbm "github.com/tendermint/tm-db" 14 15 clist "github.com/tendermint/tendermint/libs/clist" 16 "github.com/tendermint/tendermint/libs/log" 17 tmproto "github.com/tendermint/tendermint/proto/tendermint/types" 18 sm "github.com/tendermint/tendermint/state" 19 "github.com/tendermint/tendermint/types" 20 ) 21 22 const ( 23 baseKeyCommitted = byte(0x00) 24 baseKeyPending = byte(0x01) 25 ) 26 27 // Pool maintains a pool of valid evidence to be broadcasted and committed 28 type Pool struct { 29 logger log.Logger 30 31 evidenceStore dbm.DB 32 evidenceList *clist.CList // concurrent linked-list of evidence 33 evidenceSize uint32 // amount of pending evidence 34 35 // needed to load validators to verify evidence 36 stateDB sm.Store 37 // needed to load headers and commits to verify evidence 38 blockStore BlockStore 39 40 mtx sync.Mutex 41 // latest state 42 state sm.State 43 // evidence from consensus is buffered to this slice, awaiting until the next height 44 // before being flushed to the pool. This prevents broadcasting and proposing of 45 // evidence before the height with which the evidence happened is finished. 46 consensusBuffer []duplicateVoteSet 47 48 pruningHeight int64 49 pruningTime time.Time 50 } 51 52 // NewPool creates an evidence pool. If using an existing evidence store, 53 // it will add all pending evidence to the concurrent list. 54 func NewPool(evidenceDB dbm.DB, stateDB sm.Store, blockStore BlockStore) (*Pool, error) { 55 56 state, err := stateDB.Load() 57 if err != nil { 58 return nil, fmt.Errorf("cannot load state: %w", err) 59 } 60 61 pool := &Pool{ 62 stateDB: stateDB, 63 blockStore: blockStore, 64 state: state, 65 logger: log.NewNopLogger(), 66 evidenceStore: evidenceDB, 67 evidenceList: clist.New(), 68 consensusBuffer: make([]duplicateVoteSet, 0), 69 } 70 71 // if pending evidence already in db, in event of prior failure, then check for expiration, 72 // update the size and load it back to the evidenceList 73 pool.pruningHeight, pool.pruningTime = pool.removeExpiredPendingEvidence() 74 evList, _, err := pool.listEvidence(baseKeyPending, -1) 75 if err != nil { 76 return nil, err 77 } 78 atomic.StoreUint32(&pool.evidenceSize, uint32(len(evList))) 79 for _, ev := range evList { 80 pool.evidenceList.PushBack(ev) 81 } 82 83 return pool, nil 84 } 85 86 // PendingEvidence is used primarily as part of block proposal and returns up to maxNum of uncommitted evidence. 87 func (evpool *Pool) PendingEvidence(maxBytes int64) ([]types.Evidence, int64) { 88 if evpool.Size() == 0 { 89 return []types.Evidence{}, 0 90 } 91 evidence, size, err := evpool.listEvidence(baseKeyPending, maxBytes) 92 if err != nil { 93 evpool.logger.Error("Unable to retrieve pending evidence", "err", err) 94 } 95 return evidence, size 96 } 97 98 // Update takes both the new state and the evidence committed at that height and performs 99 // the following operations: 100 // 1. Take any conflicting votes from consensus and use the state's LastBlockTime to form 101 // DuplicateVoteEvidence and add it to the pool. 102 // 2. Update the pool's state which contains evidence params relating to expiry. 103 // 3. Moves pending evidence that has now been committed into the committed pool. 104 // 4. Removes any expired evidence based on both height and time. 105 func (evpool *Pool) Update(state sm.State, ev types.EvidenceList) { 106 // sanity check 107 if state.LastBlockHeight <= evpool.state.LastBlockHeight { 108 panic(fmt.Sprintf( 109 "failed EvidencePool.Update new state height is less than or equal to previous state height: %d <= %d", 110 state.LastBlockHeight, 111 evpool.state.LastBlockHeight, 112 )) 113 } 114 evpool.logger.Debug("Updating evidence pool", "last_block_height", state.LastBlockHeight, 115 "last_block_time", state.LastBlockTime) 116 117 // flush conflicting vote pairs from the buffer, producing DuplicateVoteEvidence and 118 // adding it to the pool 119 evpool.processConsensusBuffer(state) 120 // update state 121 evpool.updateState(state) 122 123 // move committed evidence out from the pending pool and into the committed pool 124 evpool.markEvidenceAsCommitted(ev) 125 126 // prune pending evidence when it has expired. This also updates when the next evidence will expire 127 if evpool.Size() > 0 && state.LastBlockHeight > evpool.pruningHeight && 128 state.LastBlockTime.After(evpool.pruningTime) { 129 evpool.pruningHeight, evpool.pruningTime = evpool.removeExpiredPendingEvidence() 130 } 131 } 132 133 // AddEvidence checks the evidence is valid and adds it to the pool. 134 func (evpool *Pool) AddEvidence(ev types.Evidence) error { 135 evpool.logger.Debug("Attempting to add evidence", "ev", ev) 136 137 // We have already verified this piece of evidence - no need to do it again 138 if evpool.isPending(ev) { 139 evpool.logger.Debug("Evidence already pending, ignoring this one", "ev", ev) 140 return nil 141 } 142 143 // check that the evidence isn't already committed 144 if evpool.isCommitted(ev) { 145 // this can happen if the peer that sent us the evidence is behind so we shouldn't 146 // punish the peer. 147 evpool.logger.Debug("Evidence was already committed, ignoring this one", "ev", ev) 148 return nil 149 } 150 151 // 1) Verify against state. 152 err := evpool.verify(ev) 153 if err != nil { 154 return types.NewErrInvalidEvidence(ev, err) 155 } 156 157 // 2) Save to store. 158 if err := evpool.addPendingEvidence(ev); err != nil { 159 return fmt.Errorf("can't add evidence to pending list: %w", err) 160 } 161 162 // 3) Add evidence to clist. 163 evpool.evidenceList.PushBack(ev) 164 165 evpool.logger.Info("Verified new evidence of byzantine behavior", "evidence", ev) 166 167 return nil 168 } 169 170 // ReportConflictingVotes takes two conflicting votes and forms duplicate vote evidence, 171 // adding it eventually to the evidence pool. 172 // 173 // Duplicate vote attacks happen before the block is committed and the timestamp is 174 // finalized, thus the evidence pool holds these votes in a buffer, forming the 175 // evidence from them once consensus at that height has been reached and `Update()` with 176 // the new state called. 177 // 178 // Votes are not verified. 179 func (evpool *Pool) ReportConflictingVotes(voteA, voteB *types.Vote) { 180 evpool.mtx.Lock() 181 defer evpool.mtx.Unlock() 182 evpool.consensusBuffer = append(evpool.consensusBuffer, duplicateVoteSet{ 183 VoteA: voteA, 184 VoteB: voteB, 185 }) 186 } 187 188 // CheckEvidence takes an array of evidence from a block and verifies all the evidence there. 189 // If it has already verified the evidence then it jumps to the next one. It ensures that no 190 // evidence has already been committed or is being proposed twice. It also adds any 191 // evidence that it doesn't currently have so that it can quickly form ABCI Evidence later. 192 func (evpool *Pool) CheckEvidence(evList types.EvidenceList) error { 193 hashes := make([][]byte, len(evList)) 194 for idx, ev := range evList { 195 196 _, isLightEv := ev.(*types.LightClientAttackEvidence) 197 198 // We must verify light client attack evidence regardless because there could be a 199 // different conflicting block with the same hash. 200 if isLightEv || !evpool.isPending(ev) { 201 // check that the evidence isn't already committed 202 if evpool.isCommitted(ev) { 203 return &types.ErrInvalidEvidence{Evidence: ev, Reason: errors.New("evidence was already committed")} 204 } 205 206 err := evpool.verify(ev) 207 if err != nil { 208 return err 209 } 210 211 if err := evpool.addPendingEvidence(ev); err != nil { 212 // Something went wrong with adding the evidence but we already know it is valid 213 // hence we log an error and continue 214 evpool.logger.Error("Can't add evidence to pending list", "err", err, "ev", ev) 215 } 216 217 evpool.logger.Info("Check evidence: verified evidence of byzantine behavior", "evidence", ev) 218 } 219 220 // check for duplicate evidence. We cache hashes so we don't have to work them out again. 221 hashes[idx] = ev.Hash() 222 for i := idx - 1; i >= 0; i-- { 223 if bytes.Equal(hashes[i], hashes[idx]) { 224 return &types.ErrInvalidEvidence{Evidence: ev, Reason: errors.New("duplicate evidence")} 225 } 226 } 227 } 228 229 return nil 230 } 231 232 // EvidenceFront goes to the first evidence in the clist 233 func (evpool *Pool) EvidenceFront() *clist.CElement { 234 return evpool.evidenceList.Front() 235 } 236 237 // EvidenceWaitChan is a channel that closes once the first evidence in the list is there. i.e Front is not nil 238 func (evpool *Pool) EvidenceWaitChan() <-chan struct{} { 239 return evpool.evidenceList.WaitChan() 240 } 241 242 // SetLogger sets the Logger. 243 func (evpool *Pool) SetLogger(l log.Logger) { 244 evpool.logger = l 245 } 246 247 // Size returns the number of evidence in the pool. 248 func (evpool *Pool) Size() uint32 { 249 return atomic.LoadUint32(&evpool.evidenceSize) 250 } 251 252 // State returns the current state of the evpool. 253 func (evpool *Pool) State() sm.State { 254 evpool.mtx.Lock() 255 defer evpool.mtx.Unlock() 256 return evpool.state 257 } 258 259 // IsExpired checks whether evidence or a polc is expired by checking whether a height and time is older 260 // than set by the evidence consensus parameters 261 func (evpool *Pool) isExpired(height int64, time time.Time) bool { 262 var ( 263 params = evpool.State().ConsensusParams.Evidence 264 ageDuration = evpool.State().LastBlockTime.Sub(time) 265 ageNumBlocks = evpool.State().LastBlockHeight - height 266 ) 267 return ageNumBlocks > params.MaxAgeNumBlocks && 268 ageDuration > params.MaxAgeDuration 269 } 270 271 // IsCommitted returns true if we have already seen this exact evidence and it is already marked as committed. 272 func (evpool *Pool) isCommitted(evidence types.Evidence) bool { 273 key := keyCommitted(evidence) 274 ok, err := evpool.evidenceStore.Has(key) 275 if err != nil { 276 evpool.logger.Error("Unable to find committed evidence", "err", err) 277 } 278 return ok 279 } 280 281 // IsPending checks whether the evidence is already pending. DB errors are passed to the logger. 282 func (evpool *Pool) isPending(evidence types.Evidence) bool { 283 key := keyPending(evidence) 284 ok, err := evpool.evidenceStore.Has(key) 285 if err != nil { 286 evpool.logger.Error("Unable to find pending evidence", "err", err) 287 } 288 return ok 289 } 290 291 func (evpool *Pool) addPendingEvidence(ev types.Evidence) error { 292 evpb, err := types.EvidenceToProto(ev) 293 if err != nil { 294 return fmt.Errorf("unable to convert to proto, err: %w", err) 295 } 296 297 evBytes, err := evpb.Marshal() 298 if err != nil { 299 return fmt.Errorf("unable to marshal evidence: %w", err) 300 } 301 302 key := keyPending(ev) 303 304 err = evpool.evidenceStore.Set(key, evBytes) 305 if err != nil { 306 return fmt.Errorf("can't persist evidence: %w", err) 307 } 308 atomic.AddUint32(&evpool.evidenceSize, 1) 309 return nil 310 } 311 312 func (evpool *Pool) removePendingEvidence(evidence types.Evidence) { 313 key := keyPending(evidence) 314 if err := evpool.evidenceStore.Delete(key); err != nil { 315 evpool.logger.Error("Unable to delete pending evidence", "err", err) 316 } else { 317 atomic.AddUint32(&evpool.evidenceSize, ^uint32(0)) 318 evpool.logger.Debug("Deleted pending evidence", "evidence", evidence) 319 } 320 } 321 322 // markEvidenceAsCommitted processes all the evidence in the block, marking it as 323 // committed and removing it from the pending database. 324 func (evpool *Pool) markEvidenceAsCommitted(evidence types.EvidenceList) { 325 blockEvidenceMap := make(map[string]struct{}, len(evidence)) 326 for _, ev := range evidence { 327 if evpool.isPending(ev) { 328 evpool.removePendingEvidence(ev) 329 blockEvidenceMap[evMapKey(ev)] = struct{}{} 330 } 331 332 // Add evidence to the committed list. As the evidence is stored in the block store 333 // we only need to record the height that it was saved at. 334 key := keyCommitted(ev) 335 336 h := gogotypes.Int64Value{Value: ev.Height()} 337 evBytes, err := proto.Marshal(&h) 338 if err != nil { 339 evpool.logger.Error("failed to marshal committed evidence", "err", err, "key(height/hash)", key) 340 continue 341 } 342 343 if err := evpool.evidenceStore.Set(key, evBytes); err != nil { 344 evpool.logger.Error("Unable to save committed evidence", "err", err, "key(height/hash)", key) 345 } 346 } 347 348 // remove committed evidence from the clist 349 if len(blockEvidenceMap) != 0 { 350 evpool.removeEvidenceFromList(blockEvidenceMap) 351 } 352 } 353 354 // listEvidence retrieves lists evidence from oldest to newest within maxBytes. 355 // If maxBytes is -1, there's no cap on the size of returned evidence. 356 func (evpool *Pool) listEvidence(prefixKey byte, maxBytes int64) ([]types.Evidence, int64, error) { 357 var ( 358 evSize int64 359 totalSize int64 360 evidence []types.Evidence 361 evList tmproto.EvidenceList // used for calculating the bytes size 362 ) 363 364 iter, err := dbm.IteratePrefix(evpool.evidenceStore, []byte{prefixKey}) 365 if err != nil { 366 return nil, totalSize, fmt.Errorf("database error: %v", err) 367 } 368 defer iter.Close() 369 for ; iter.Valid(); iter.Next() { 370 var evpb tmproto.Evidence 371 err := evpb.Unmarshal(iter.Value()) 372 if err != nil { 373 return evidence, totalSize, err 374 } 375 evList.Evidence = append(evList.Evidence, evpb) 376 evSize = int64(evList.Size()) 377 if maxBytes != -1 && evSize > maxBytes { 378 if err := iter.Error(); err != nil { 379 return evidence, totalSize, err 380 } 381 return evidence, totalSize, nil 382 } 383 384 ev, err := types.EvidenceFromProto(&evpb) 385 if err != nil { 386 return nil, totalSize, err 387 } 388 389 totalSize = evSize 390 evidence = append(evidence, ev) 391 } 392 393 if err := iter.Error(); err != nil { 394 return evidence, totalSize, err 395 } 396 return evidence, totalSize, nil 397 } 398 399 func (evpool *Pool) removeExpiredPendingEvidence() (int64, time.Time) { 400 iter, err := dbm.IteratePrefix(evpool.evidenceStore, []byte{baseKeyPending}) 401 if err != nil { 402 evpool.logger.Error("Unable to iterate over pending evidence", "err", err) 403 return evpool.State().LastBlockHeight, evpool.State().LastBlockTime 404 } 405 defer iter.Close() 406 blockEvidenceMap := make(map[string]struct{}) 407 for ; iter.Valid(); iter.Next() { 408 ev, err := bytesToEv(iter.Value()) 409 if err != nil { 410 evpool.logger.Error("Error in transition evidence from protobuf", "err", err) 411 continue 412 } 413 if !evpool.isExpired(ev.Height(), ev.Time()) { 414 if len(blockEvidenceMap) != 0 { 415 evpool.removeEvidenceFromList(blockEvidenceMap) 416 } 417 418 // return the height and time with which this evidence will have expired so we know when to prune next 419 return ev.Height() + evpool.State().ConsensusParams.Evidence.MaxAgeNumBlocks + 1, 420 ev.Time().Add(evpool.State().ConsensusParams.Evidence.MaxAgeDuration).Add(time.Second) 421 } 422 evpool.removePendingEvidence(ev) 423 blockEvidenceMap[evMapKey(ev)] = struct{}{} 424 } 425 // We either have no pending evidence or all evidence has expired 426 if len(blockEvidenceMap) != 0 { 427 evpool.removeEvidenceFromList(blockEvidenceMap) 428 } 429 return evpool.State().LastBlockHeight, evpool.State().LastBlockTime 430 } 431 432 func (evpool *Pool) removeEvidenceFromList( 433 blockEvidenceMap map[string]struct{}) { 434 435 for e := evpool.evidenceList.Front(); e != nil; e = e.Next() { 436 // Remove from clist 437 ev := e.Value.(types.Evidence) 438 if _, ok := blockEvidenceMap[evMapKey(ev)]; ok { 439 evpool.evidenceList.Remove(e) 440 e.DetachPrev() 441 } 442 } 443 } 444 445 func (evpool *Pool) updateState(state sm.State) { 446 evpool.mtx.Lock() 447 defer evpool.mtx.Unlock() 448 evpool.state = state 449 } 450 451 // processConsensusBuffer converts all the duplicate votes witnessed from consensus 452 // into DuplicateVoteEvidence. It sets the evidence timestamp to the block height 453 // from the most recently committed block. 454 // Evidence is then added to the pool so as to be ready to be broadcasted and proposed. 455 func (evpool *Pool) processConsensusBuffer(state sm.State) { 456 evpool.mtx.Lock() 457 defer evpool.mtx.Unlock() 458 for _, voteSet := range evpool.consensusBuffer { 459 460 // Check the height of the conflicting votes and fetch the corresponding time and validator set 461 // to produce the valid evidence 462 var dve *types.DuplicateVoteEvidence 463 switch { 464 case voteSet.VoteA.Height == state.LastBlockHeight: 465 dve = types.NewDuplicateVoteEvidence( 466 voteSet.VoteA, 467 voteSet.VoteB, 468 state.LastBlockTime, 469 state.LastValidators, 470 ) 471 472 case voteSet.VoteA.Height < state.LastBlockHeight: 473 valSet, err := evpool.stateDB.LoadValidators(voteSet.VoteA.Height) 474 if err != nil { 475 evpool.logger.Error("failed to load validator set for conflicting votes", "height", 476 voteSet.VoteA.Height, "err", err, 477 ) 478 continue 479 } 480 blockMeta := evpool.blockStore.LoadBlockMeta(voteSet.VoteA.Height) 481 if blockMeta == nil { 482 evpool.logger.Error("failed to load block time for conflicting votes", "height", voteSet.VoteA.Height) 483 continue 484 } 485 dve = types.NewDuplicateVoteEvidence( 486 voteSet.VoteA, 487 voteSet.VoteB, 488 blockMeta.Header.Time, 489 valSet, 490 ) 491 492 default: 493 // evidence pool shouldn't expect to get votes from consensus of a height that is above the current 494 // state. If this error is seen then perhaps consider keeping the votes in the buffer and retry 495 // in following heights 496 evpool.logger.Error("inbound duplicate votes from consensus are of a greater height than current state", 497 "duplicate vote height", voteSet.VoteA.Height, 498 "state.LastBlockHeight", state.LastBlockHeight) 499 continue 500 } 501 502 // check if we already have this evidence 503 if evpool.isPending(dve) { 504 evpool.logger.Debug("evidence already pending; ignoring", "evidence", dve) 505 continue 506 } 507 508 // check that the evidence is not already committed on chain 509 if evpool.isCommitted(dve) { 510 evpool.logger.Debug("evidence already committed; ignoring", "evidence", dve) 511 continue 512 } 513 514 if err := evpool.addPendingEvidence(dve); err != nil { 515 evpool.logger.Error("failed to flush evidence from consensus buffer to pending list: %w", err) 516 continue 517 } 518 519 evpool.evidenceList.PushBack(dve) 520 521 evpool.logger.Info("verified new evidence of byzantine behavior", "evidence", dve) 522 } 523 // reset consensus buffer 524 evpool.consensusBuffer = make([]duplicateVoteSet, 0) 525 } 526 527 type duplicateVoteSet struct { 528 VoteA *types.Vote 529 VoteB *types.Vote 530 } 531 532 func bytesToEv(evBytes []byte) (types.Evidence, error) { 533 var evpb tmproto.Evidence 534 err := evpb.Unmarshal(evBytes) 535 if err != nil { 536 return &types.DuplicateVoteEvidence{}, err 537 } 538 539 return types.EvidenceFromProto(&evpb) 540 } 541 542 func evMapKey(ev types.Evidence) string { 543 return string(ev.Hash()) 544 } 545 546 // big endian padded hex 547 func bE(h int64) string { 548 return fmt.Sprintf("%0.16X", h) 549 } 550 551 func keyCommitted(evidence types.Evidence) []byte { 552 return append([]byte{baseKeyCommitted}, keySuffix(evidence)...) 553 } 554 555 func keyPending(evidence types.Evidence) []byte { 556 return append([]byte{baseKeyPending}, keySuffix(evidence)...) 557 } 558 559 func keySuffix(evidence types.Evidence) []byte { 560 return []byte(fmt.Sprintf("%s/%X", bE(evidence.Height()), evidence.Hash())) 561 }