github.com/vipernet-xyz/tm@v0.34.24/test/maverick/consensus/replay.go (about) 1 package consensus 2 3 import ( 4 "bytes" 5 "fmt" 6 "hash/crc32" 7 "io" 8 "reflect" 9 "time" 10 11 abci "github.com/vipernet-xyz/tm/abci/types" 12 tmcon "github.com/vipernet-xyz/tm/consensus" 13 "github.com/vipernet-xyz/tm/crypto/merkle" 14 "github.com/vipernet-xyz/tm/libs/log" 15 "github.com/vipernet-xyz/tm/proxy" 16 sm "github.com/vipernet-xyz/tm/state" 17 "github.com/vipernet-xyz/tm/types" 18 ) 19 20 var crc32c = crc32.MakeTable(crc32.Castagnoli) 21 22 // Functionality to replay blocks and messages on recovery from a crash. 23 // There are two general failure scenarios: 24 // 25 // 1. failure during consensus 26 // 2. failure while applying the block 27 // 28 // The former is handled by the WAL, the latter by the proxyApp Handshake on 29 // restart, which ultimately hands off the work to the WAL. 30 31 //----------------------------------------- 32 // 1. Recover from failure during consensus 33 // (by replaying messages from the WAL) 34 //----------------------------------------- 35 36 // Unmarshal and apply a single message to the consensus state as if it were 37 // received in receiveRoutine. Lines that start with "#" are ignored. 38 // NOTE: receiveRoutine should not be running. 39 func (cs *State) readReplayMessage(msg *tmcon.TimedWALMessage, newStepSub types.Subscription) error { 40 // Skip meta messages which exist for demarcating boundaries. 41 if _, ok := msg.Msg.(tmcon.EndHeightMessage); ok { 42 return nil 43 } 44 45 // for logging 46 switch m := msg.Msg.(type) { 47 case types.EventDataRoundState: 48 cs.Logger.Info("Replay: New Step", "height", m.Height, "round", m.Round, "step", m.Step) 49 // these are playback checks 50 ticker := time.After(time.Second * 2) 51 if newStepSub != nil { 52 select { 53 case stepMsg := <-newStepSub.Out(): 54 m2 := stepMsg.Data().(types.EventDataRoundState) 55 if m.Height != m2.Height || m.Round != m2.Round || m.Step != m2.Step { 56 return fmt.Errorf("roundState mismatch. Got %v; Expected %v", m2, m) 57 } 58 case <-newStepSub.Cancelled(): 59 return fmt.Errorf("failed to read off newStepSub.Out(). newStepSub was cancelled") 60 case <-ticker: 61 return fmt.Errorf("failed to read off newStepSub.Out()") 62 } 63 } 64 case msgInfo: 65 peerID := m.PeerID 66 if peerID == "" { 67 peerID = "local" 68 } 69 switch msg := m.Msg.(type) { 70 case *tmcon.ProposalMessage: 71 p := msg.Proposal 72 cs.Logger.Info("Replay: Proposal", "height", p.Height, "round", p.Round, "header", 73 p.BlockID.PartSetHeader, "pol", p.POLRound, "peer", peerID) 74 case *tmcon.BlockPartMessage: 75 cs.Logger.Info("Replay: BlockPart", "height", msg.Height, "round", msg.Round, "peer", peerID) 76 case *tmcon.VoteMessage: 77 v := msg.Vote 78 cs.Logger.Info("Replay: Vote", "height", v.Height, "round", v.Round, "type", v.Type, 79 "blockID", v.BlockID, "peer", peerID) 80 } 81 82 cs.handleMsg(m) 83 case timeoutInfo: 84 cs.Logger.Info("Replay: Timeout", "height", m.Height, "round", m.Round, "step", m.Step, "dur", m.Duration) 85 cs.handleTimeout(m, cs.RoundState) 86 default: 87 return fmt.Errorf("replay: Unknown TimedWALMessage type: %v", reflect.TypeOf(msg.Msg)) 88 } 89 return nil 90 } 91 92 // Replay only those messages since the last block. `timeoutRoutine` should 93 // run concurrently to read off tickChan. 94 func (cs *State) catchupReplay(csHeight int64) error { 95 96 // Set replayMode to true so we don't log signing errors. 97 cs.replayMode = true 98 defer func() { cs.replayMode = false }() 99 100 // Ensure that #ENDHEIGHT for this height doesn't exist. 101 // NOTE: This is just a sanity check. As far as we know things work fine 102 // without it, and Handshake could reuse State if it weren't for 103 // this check (since we can crash after writing #ENDHEIGHT). 104 // 105 // Ignore data corruption errors since this is a sanity check. 106 gr, found, err := cs.wal.SearchForEndHeight(csHeight, &tmcon.WALSearchOptions{IgnoreDataCorruptionErrors: true}) 107 if err != nil { 108 return err 109 } 110 if gr != nil { 111 if err := gr.Close(); err != nil { 112 return err 113 } 114 } 115 if found { 116 return fmt.Errorf("wal should not contain #ENDHEIGHT %d", csHeight) 117 } 118 119 // Search for last height marker. 120 // 121 // Ignore data corruption errors in previous heights because we only care about last height 122 if csHeight < cs.state.InitialHeight { 123 return fmt.Errorf("cannot replay height %v, below initial height %v", csHeight, cs.state.InitialHeight) 124 } 125 endHeight := csHeight - 1 126 if csHeight == cs.state.InitialHeight { 127 endHeight = 0 128 } 129 gr, found, err = cs.wal.SearchForEndHeight(endHeight, &tmcon.WALSearchOptions{IgnoreDataCorruptionErrors: true}) 130 if err == io.EOF { 131 cs.Logger.Error("Replay: wal.group.Search returned EOF", "#ENDHEIGHT", endHeight) 132 } else if err != nil { 133 return err 134 } 135 if !found { 136 return fmt.Errorf("cannot replay height %d. WAL does not contain #ENDHEIGHT for %d", csHeight, endHeight) 137 } 138 defer gr.Close() 139 140 cs.Logger.Info("Catchup by replaying consensus messages", "height", csHeight) 141 142 var msg *tmcon.TimedWALMessage 143 dec := WALDecoder{gr} 144 145 LOOP: 146 for { 147 msg, err = dec.Decode() 148 switch { 149 case err == io.EOF: 150 break LOOP 151 case IsDataCorruptionError(err): 152 cs.Logger.Error("data has been corrupted in last height of consensus WAL", "err", err, "height", csHeight) 153 return err 154 case err != nil: 155 return err 156 } 157 158 // NOTE: since the priv key is set when the msgs are received 159 // it will attempt to eg double sign but we can just ignore it 160 // since the votes will be replayed and we'll get to the next step 161 if err := cs.readReplayMessage(msg, nil); err != nil { 162 return err 163 } 164 } 165 cs.Logger.Info("Replay: Done") 166 return nil 167 } 168 169 //-------------------------------------------------------------------------------- 170 171 // Parses marker lines of the form: 172 // #ENDHEIGHT: 12345 173 /* 174 func makeHeightSearchFunc(height int64) auto.SearchFunc { 175 return func(line string) (int, error) { 176 line = strings.TrimRight(line, "\n") 177 parts := strings.Split(line, " ") 178 if len(parts) != 2 { 179 return -1, errors.New("line did not have 2 parts") 180 } 181 i, err := strconv.Atoi(parts[1]) 182 if err != nil { 183 return -1, errors.New("failed to parse INFO: " + err.Error()) 184 } 185 if height < i { 186 return 1, nil 187 } else if height == i { 188 return 0, nil 189 } else { 190 return -1, nil 191 } 192 } 193 }*/ 194 195 //--------------------------------------------------- 196 // 2. Recover from failure while applying the block. 197 // (by handshaking with the app to figure out where 198 // we were last, and using the WAL to recover there.) 199 //--------------------------------------------------- 200 201 type Handshaker struct { 202 stateStore sm.Store 203 initialState sm.State 204 store sm.BlockStore 205 eventBus types.BlockEventPublisher 206 genDoc *types.GenesisDoc 207 logger log.Logger 208 209 nBlocks int // number of blocks applied to the state 210 } 211 212 func NewHandshaker(stateStore sm.Store, state sm.State, 213 store sm.BlockStore, genDoc *types.GenesisDoc) *Handshaker { 214 215 return &Handshaker{ 216 stateStore: stateStore, 217 initialState: state, 218 store: store, 219 eventBus: types.NopEventBus{}, 220 genDoc: genDoc, 221 logger: log.NewNopLogger(), 222 nBlocks: 0, 223 } 224 } 225 226 func (h *Handshaker) SetLogger(l log.Logger) { 227 h.logger = l 228 } 229 230 // SetEventBus - sets the event bus for publishing block related events. 231 // If not called, it defaults to types.NopEventBus. 232 func (h *Handshaker) SetEventBus(eventBus types.BlockEventPublisher) { 233 h.eventBus = eventBus 234 } 235 236 // NBlocks returns the number of blocks applied to the state. 237 func (h *Handshaker) NBlocks() int { 238 return h.nBlocks 239 } 240 241 // TODO: retry the handshake/replay if it fails ? 242 func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error { 243 244 // Handshake is done via ABCI Info on the query conn. 245 res, err := proxyApp.Query().InfoSync(proxy.RequestInfo) 246 if err != nil { 247 return fmt.Errorf("error calling Info: %v", err) 248 } 249 250 blockHeight := res.LastBlockHeight 251 if blockHeight < 0 { 252 return fmt.Errorf("got a negative last block height (%d) from the app", blockHeight) 253 } 254 appHash := res.LastBlockAppHash 255 256 h.logger.Info("ABCI Handshake App Info", 257 "height", blockHeight, 258 "hash", appHash, 259 "software-version", res.Version, 260 "protocol-version", res.AppVersion, 261 ) 262 263 // Only set the version if there is no existing state. 264 if h.initialState.LastBlockHeight == 0 { 265 h.initialState.Version.Consensus.App = res.AppVersion 266 } 267 268 // Replay blocks up to the latest in the blockstore. 269 _, err = h.ReplayBlocks(h.initialState, appHash, blockHeight, proxyApp) 270 if err != nil { 271 return fmt.Errorf("error on replay: %v", err) 272 } 273 274 h.logger.Info("Completed ABCI Handshake - Tendermint and App are synced", 275 "appHeight", blockHeight, "appHash", appHash) 276 277 // TODO: (on restart) replay mempool 278 279 return nil 280 } 281 282 // ReplayBlocks replays all blocks since appBlockHeight and ensures the result 283 // matches the current state. 284 // Returns the final AppHash or an error. 285 func (h *Handshaker) ReplayBlocks( 286 state sm.State, 287 appHash []byte, 288 appBlockHeight int64, 289 proxyApp proxy.AppConns, 290 ) ([]byte, error) { 291 storeBlockBase := h.store.Base() 292 storeBlockHeight := h.store.Height() 293 stateBlockHeight := state.LastBlockHeight 294 h.logger.Info( 295 "ABCI Replay Blocks", 296 "appHeight", 297 appBlockHeight, 298 "storeHeight", 299 storeBlockHeight, 300 "stateHeight", 301 stateBlockHeight) 302 303 // If appBlockHeight == 0 it means that we are at genesis and hence should send InitChain. 304 if appBlockHeight == 0 { 305 validators := make([]*types.Validator, len(h.genDoc.Validators)) 306 for i, val := range h.genDoc.Validators { 307 validators[i] = types.NewValidator(val.PubKey, val.Power) 308 } 309 validatorSet := types.NewValidatorSet(validators) 310 nextVals := types.TM2PB.ValidatorUpdates(validatorSet) 311 csParams := types.TM2PB.ConsensusParams(h.genDoc.ConsensusParams) 312 req := abci.RequestInitChain{ 313 Time: h.genDoc.GenesisTime, 314 ChainId: h.genDoc.ChainID, 315 InitialHeight: h.genDoc.InitialHeight, 316 ConsensusParams: csParams, 317 Validators: nextVals, 318 AppStateBytes: h.genDoc.AppState, 319 } 320 res, err := proxyApp.Consensus().InitChainSync(req) 321 if err != nil { 322 return nil, err 323 } 324 325 appHash = res.AppHash 326 327 if stateBlockHeight == 0 { // we only update state when we are in initial state 328 // If the app did not return an app hash, we keep the one set from the genesis doc in 329 // the state. We don't set appHash since we don't want the genesis doc app hash 330 // recorded in the genesis block. We should probably just remove GenesisDoc.AppHash. 331 if len(res.AppHash) > 0 { 332 state.AppHash = res.AppHash 333 } 334 // If the app returned validators or consensus params, update the state. 335 if len(res.Validators) > 0 { 336 vals, err := types.PB2TM.ValidatorUpdates(res.Validators) 337 if err != nil { 338 return nil, err 339 } 340 state.Validators = types.NewValidatorSet(vals) 341 state.NextValidators = types.NewValidatorSet(vals).CopyIncrementProposerPriority(1) 342 } else if len(h.genDoc.Validators) == 0 { 343 // If validator set is not set in genesis and still empty after InitChain, exit. 344 return nil, fmt.Errorf("validator set is nil in genesis and still empty after InitChain") 345 } 346 347 if res.ConsensusParams != nil { 348 state.ConsensusParams = types.UpdateConsensusParams(state.ConsensusParams, res.ConsensusParams) 349 state.Version.Consensus.App = state.ConsensusParams.Version.AppVersion 350 } 351 // We update the last results hash with the empty hash, to conform with RFC-6962. 352 state.LastResultsHash = merkle.HashFromByteSlices(nil) 353 if err := h.stateStore.Save(state); err != nil { 354 return nil, err 355 } 356 } 357 } 358 359 // First handle edge cases and constraints on the storeBlockHeight and storeBlockBase. 360 switch { 361 case storeBlockHeight == 0: 362 assertAppHashEqualsOneFromState(appHash, state) 363 return appHash, nil 364 365 case appBlockHeight == 0 && state.InitialHeight < storeBlockBase: 366 // the app has no state, and the block store is truncated above the initial height 367 return appHash, sm.ErrAppBlockHeightTooLow{AppHeight: appBlockHeight, StoreBase: storeBlockBase} 368 369 case appBlockHeight > 0 && appBlockHeight < storeBlockBase-1: 370 // the app is too far behind truncated store (can be 1 behind since we replay the next) 371 return appHash, sm.ErrAppBlockHeightTooLow{AppHeight: appBlockHeight, StoreBase: storeBlockBase} 372 373 case storeBlockHeight < appBlockHeight: 374 // the app should never be ahead of the store (but this is under app's control) 375 return appHash, sm.ErrAppBlockHeightTooHigh{CoreHeight: storeBlockHeight, AppHeight: appBlockHeight} 376 377 case storeBlockHeight < stateBlockHeight: 378 // the state should never be ahead of the store (this is under tendermint's control) 379 panic(fmt.Sprintf("StateBlockHeight (%d) > StoreBlockHeight (%d)", stateBlockHeight, storeBlockHeight)) 380 381 case storeBlockHeight > stateBlockHeight+1: 382 // store should be at most one ahead of the state (this is under tendermint's control) 383 panic(fmt.Sprintf("StoreBlockHeight (%d) > StateBlockHeight + 1 (%d)", storeBlockHeight, stateBlockHeight+1)) 384 } 385 386 var err error 387 // Now either store is equal to state, or one ahead. 388 // For each, consider all cases of where the app could be, given app <= store 389 if storeBlockHeight == stateBlockHeight { 390 // Tendermint ran Commit and saved the state. 391 // Either the app is asking for replay, or we're all synced up. 392 if appBlockHeight < storeBlockHeight { 393 // the app is behind, so replay blocks, but no need to go through WAL (state is already synced to store) 394 return h.replayBlocks(state, proxyApp, appBlockHeight, storeBlockHeight, false) 395 396 } else if appBlockHeight == storeBlockHeight { 397 // We're good! 398 assertAppHashEqualsOneFromState(appHash, state) 399 return appHash, nil 400 } 401 402 } else if storeBlockHeight == stateBlockHeight+1 { 403 // We saved the block in the store but haven't updated the state, 404 // so we'll need to replay a block using the WAL. 405 switch { 406 case appBlockHeight < stateBlockHeight: 407 // the app is further behind than it should be, so replay blocks 408 // but leave the last block to go through the WAL 409 return h.replayBlocks(state, proxyApp, appBlockHeight, storeBlockHeight, true) 410 411 case appBlockHeight == stateBlockHeight: 412 // We haven't run Commit (both the state and app are one block behind), 413 // so replayBlock with the real app. 414 // NOTE: We could instead use the cs.WAL on cs.Start, 415 // but we'd have to allow the WAL to replay a block that wrote it's #ENDHEIGHT 416 h.logger.Info("Replay last block using real app") 417 state, err = h.replayBlock(state, storeBlockHeight, proxyApp.Consensus()) 418 return state.AppHash, err 419 420 case appBlockHeight == storeBlockHeight: 421 // We ran Commit, but didn't save the state, so replayBlock with mock app. 422 abciResponses, err := h.stateStore.LoadABCIResponses(storeBlockHeight) 423 if err != nil { 424 return nil, err 425 } 426 mockApp := newMockProxyApp(appHash, abciResponses) 427 h.logger.Info("Replay last block using mock app") 428 state, err = h.replayBlock(state, storeBlockHeight, mockApp) 429 return state.AppHash, err 430 } 431 432 } 433 434 panic(fmt.Sprintf("uncovered case! appHeight: %d, storeHeight: %d, stateHeight: %d", 435 appBlockHeight, storeBlockHeight, stateBlockHeight)) 436 } 437 438 func (h *Handshaker) replayBlocks( 439 state sm.State, 440 proxyApp proxy.AppConns, 441 appBlockHeight, 442 storeBlockHeight int64, 443 mutateState bool) ([]byte, error) { 444 // App is further behind than it should be, so we need to replay blocks. 445 // We replay all blocks from appBlockHeight+1. 446 // 447 // Note that we don't have an old version of the state, 448 // so we by-pass state validation/mutation using sm.ExecCommitBlock. 449 // This also means we won't be saving validator sets if they change during this period. 450 // TODO: Load the historical information to fix this and just use state.ApplyBlock 451 // 452 // If mutateState == true, the final block is replayed with h.replayBlock() 453 454 var appHash []byte 455 var err error 456 finalBlock := storeBlockHeight 457 if mutateState { 458 finalBlock-- 459 } 460 firstBlock := appBlockHeight + 1 461 if firstBlock == 1 { 462 firstBlock = state.InitialHeight 463 } 464 for i := firstBlock; i <= finalBlock; i++ { 465 h.logger.Info("Applying block", "height", i) 466 block := h.store.LoadBlock(i) 467 // Extra check to ensure the app was not changed in a way it shouldn't have. 468 if len(appHash) > 0 { 469 assertAppHashEqualsOneFromBlock(appHash, block) 470 } 471 472 appHash, err = sm.ExecCommitBlock(proxyApp.Consensus(), block, h.logger, h.stateStore, h.genDoc.InitialHeight) 473 if err != nil { 474 return nil, err 475 } 476 477 h.nBlocks++ 478 } 479 480 if mutateState { 481 // sync the final block 482 state, err = h.replayBlock(state, storeBlockHeight, proxyApp.Consensus()) 483 if err != nil { 484 return nil, err 485 } 486 appHash = state.AppHash 487 } 488 489 assertAppHashEqualsOneFromState(appHash, state) 490 return appHash, nil 491 } 492 493 // ApplyBlock on the proxyApp with the last block. 494 func (h *Handshaker) replayBlock(state sm.State, height int64, proxyApp proxy.AppConnConsensus) (sm.State, error) { 495 block := h.store.LoadBlock(height) 496 meta := h.store.LoadBlockMeta(height) 497 498 // Use stubs for both mempool and evidence pool since no transactions nor 499 // evidence are needed here - block already exists. 500 blockExec := sm.NewBlockExecutor(h.stateStore, h.logger, proxyApp, emptyMempool{}, sm.EmptyEvidencePool{}) 501 blockExec.SetEventBus(h.eventBus) 502 503 var err error 504 state, _, err = blockExec.ApplyBlock(state, meta.BlockID, block) 505 if err != nil { 506 return sm.State{}, err 507 } 508 509 h.nBlocks++ 510 511 return state, nil 512 } 513 514 func assertAppHashEqualsOneFromBlock(appHash []byte, block *types.Block) { 515 if !bytes.Equal(appHash, block.AppHash) { 516 panic(fmt.Sprintf(`block.AppHash does not match AppHash after replay. Got %X, expected %X. 517 518 Block: %v 519 `, 520 appHash, block.AppHash, block)) 521 } 522 } 523 524 func assertAppHashEqualsOneFromState(appHash []byte, state sm.State) { 525 if !bytes.Equal(appHash, state.AppHash) { 526 panic(fmt.Sprintf(`state.AppHash does not match AppHash after replay. Got 527 %X, expected %X. 528 529 State: %v 530 531 Did you reset Tendermint without resetting your application's data?`, 532 appHash, state.AppHash, state)) 533 } 534 }