github.com/noirx94/tendermintmp@v0.0.1/consensus/replay.go (about) 1 package consensus 2 3 import ( 4 "bytes" 5 "fmt" 6 "hash/crc32" 7 "io" 8 "reflect" 9 "time" 10 11 abci "github.com/tendermint/tendermint/abci/types" 12 "github.com/tendermint/tendermint/crypto/merkle" 13 "github.com/tendermint/tendermint/libs/log" 14 "github.com/tendermint/tendermint/proxy" 15 sm "github.com/tendermint/tendermint/state" 16 "github.com/tendermint/tendermint/types" 17 ) 18 19 var crc32c = crc32.MakeTable(crc32.Castagnoli) 20 21 // Functionality to replay blocks and messages on recovery from a crash. 22 // There are two general failure scenarios: 23 // 24 // 1. failure during consensus 25 // 2. failure while applying the block 26 // 27 // The former is handled by the WAL, the latter by the proxyApp Handshake on 28 // restart, which ultimately hands off the work to the WAL. 29 30 //----------------------------------------- 31 // 1. Recover from failure during consensus 32 // (by replaying messages from the WAL) 33 //----------------------------------------- 34 35 // Unmarshal and apply a single message to the consensus state as if it were 36 // received in receiveRoutine. Lines that start with "#" are ignored. 37 // NOTE: receiveRoutine should not be running. 38 func (cs *State) readReplayMessage(msg *TimedWALMessage, newStepSub types.Subscription) error { 39 // Skip meta messages which exist for demarcating boundaries. 40 if _, ok := msg.Msg.(EndHeightMessage); ok { 41 return nil 42 } 43 44 // for logging 45 switch m := msg.Msg.(type) { 46 case types.EventDataRoundState: 47 cs.Logger.Info("Replay: New Step", "height", m.Height, "round", m.Round, "step", m.Step) 48 // these are playback checks 49 ticker := time.After(time.Second * 2) 50 if newStepSub != nil { 51 select { 52 case stepMsg := <-newStepSub.Out(): 53 m2 := stepMsg.Data().(types.EventDataRoundState) 54 if m.Height != m2.Height || m.Round != m2.Round || m.Step != m2.Step { 55 return fmt.Errorf("roundState mismatch. Got %v; Expected %v", m2, m) 56 } 57 case <-newStepSub.Cancelled(): 58 return fmt.Errorf("failed to read off newStepSub.Out(). newStepSub was cancelled") 59 case <-ticker: 60 return fmt.Errorf("failed to read off newStepSub.Out()") 61 } 62 } 63 case msgInfo: 64 peerID := m.PeerID 65 if peerID == "" { 66 peerID = "local" 67 } 68 switch msg := m.Msg.(type) { 69 case *ProposalMessage: 70 p := msg.Proposal 71 cs.Logger.Info("Replay: Proposal", "height", p.Height, "round", p.Round, "header", 72 p.BlockID.PartSetHeader, "pol", p.POLRound, "peer", peerID) 73 case *BlockPartMessage: 74 cs.Logger.Info("Replay: BlockPart", "height", msg.Height, "round", msg.Round, "peer", peerID) 75 case *VoteMessage: 76 v := msg.Vote 77 cs.Logger.Info("Replay: Vote", "height", v.Height, "round", v.Round, "type", v.Type, 78 "blockID", v.BlockID, "peer", peerID) 79 } 80 81 cs.handleMsg(m) 82 case timeoutInfo: 83 cs.Logger.Info("Replay: Timeout", "height", m.Height, "round", m.Round, "step", m.Step, "dur", m.Duration) 84 cs.handleTimeout(m, cs.RoundState) 85 default: 86 return fmt.Errorf("replay: Unknown TimedWALMessage type: %v", reflect.TypeOf(msg.Msg)) 87 } 88 return nil 89 } 90 91 // Replay only those messages since the last block. `timeoutRoutine` should 92 // run concurrently to read off tickChan. 93 func (cs *State) catchupReplay(csHeight int64) error { 94 95 // Set replayMode to true so we don't log signing errors. 96 cs.replayMode = true 97 defer func() { cs.replayMode = false }() 98 99 // Ensure that #ENDHEIGHT for this height doesn't exist. 100 // NOTE: This is just a sanity check. As far as we know things work fine 101 // without it, and Handshake could reuse State if it weren't for 102 // this check (since we can crash after writing #ENDHEIGHT). 103 // 104 // Ignore data corruption errors since this is a sanity check. 105 gr, found, err := cs.wal.SearchForEndHeight(csHeight, &WALSearchOptions{IgnoreDataCorruptionErrors: true}) 106 if err != nil { 107 return err 108 } 109 if gr != nil { 110 if err := gr.Close(); err != nil { 111 return err 112 } 113 } 114 if found { 115 return fmt.Errorf("wal should not contain #ENDHEIGHT %d", csHeight) 116 } 117 118 // Search for last height marker. 119 // 120 // Ignore data corruption errors in previous heights because we only care about last height 121 if csHeight < cs.state.InitialHeight { 122 return fmt.Errorf("cannot replay height %v, below initial height %v", csHeight, cs.state.InitialHeight) 123 } 124 endHeight := csHeight - 1 125 if csHeight == cs.state.InitialHeight { 126 endHeight = 0 127 } 128 gr, found, err = cs.wal.SearchForEndHeight(endHeight, &WALSearchOptions{IgnoreDataCorruptionErrors: true}) 129 if err == io.EOF { 130 cs.Logger.Error("Replay: wal.group.Search returned EOF", "#ENDHEIGHT", endHeight) 131 } else if err != nil { 132 return err 133 } 134 if !found { 135 return fmt.Errorf("cannot replay height %d. WAL does not contain #ENDHEIGHT for %d", csHeight, endHeight) 136 } 137 defer gr.Close() 138 139 cs.Logger.Info("Catchup by replaying consensus messages", "height", csHeight) 140 141 var msg *TimedWALMessage 142 dec := WALDecoder{gr} 143 144 LOOP: 145 for { 146 msg, err = dec.Decode() 147 switch { 148 case err == io.EOF: 149 break LOOP 150 case IsDataCorruptionError(err): 151 cs.Logger.Error("data has been corrupted in last height of consensus WAL", "err", err, "height", csHeight) 152 return err 153 case err != nil: 154 return err 155 } 156 157 // NOTE: since the priv key is set when the msgs are received 158 // it will attempt to eg double sign but we can just ignore it 159 // since the votes will be replayed and we'll get to the next step 160 if err := cs.readReplayMessage(msg, nil); err != nil { 161 return err 162 } 163 } 164 cs.Logger.Info("Replay: Done") 165 return nil 166 } 167 168 //-------------------------------------------------------------------------------- 169 170 // Parses marker lines of the form: 171 // #ENDHEIGHT: 12345 172 /* 173 func makeHeightSearchFunc(height int64) auto.SearchFunc { 174 return func(line string) (int, error) { 175 line = strings.TrimRight(line, "\n") 176 parts := strings.Split(line, " ") 177 if len(parts) != 2 { 178 return -1, errors.New("line did not have 2 parts") 179 } 180 i, err := strconv.Atoi(parts[1]) 181 if err != nil { 182 return -1, errors.New("failed to parse INFO: " + err.Error()) 183 } 184 if height < i { 185 return 1, nil 186 } else if height == i { 187 return 0, nil 188 } else { 189 return -1, nil 190 } 191 } 192 }*/ 193 194 //--------------------------------------------------- 195 // 2. Recover from failure while applying the block. 196 // (by handshaking with the app to figure out where 197 // we were last, and using the WAL to recover there.) 198 //--------------------------------------------------- 199 200 type Handshaker struct { 201 stateStore sm.Store 202 initialState sm.State 203 store sm.BlockStore 204 eventBus types.BlockEventPublisher 205 genDoc *types.GenesisDoc 206 logger log.Logger 207 208 nBlocks int // number of blocks applied to the state 209 } 210 211 func NewHandshaker(stateStore sm.Store, state sm.State, 212 store sm.BlockStore, genDoc *types.GenesisDoc) *Handshaker { 213 214 return &Handshaker{ 215 stateStore: stateStore, 216 initialState: state, 217 store: store, 218 eventBus: types.NopEventBus{}, 219 genDoc: genDoc, 220 logger: log.NewNopLogger(), 221 nBlocks: 0, 222 } 223 } 224 225 func (h *Handshaker) SetLogger(l log.Logger) { 226 h.logger = l 227 } 228 229 // SetEventBus - sets the event bus for publishing block related events. 230 // If not called, it defaults to types.NopEventBus. 231 func (h *Handshaker) SetEventBus(eventBus types.BlockEventPublisher) { 232 h.eventBus = eventBus 233 } 234 235 // NBlocks returns the number of blocks applied to the state. 236 func (h *Handshaker) NBlocks() int { 237 return h.nBlocks 238 } 239 240 // TODO: retry the handshake/replay if it fails ? 241 func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error { 242 243 // Handshake is done via ABCI Info on the query conn. 244 res, err := proxyApp.Query().InfoSync(proxy.RequestInfo) 245 if err != nil { 246 return fmt.Errorf("error calling Info: %v", err) 247 } 248 249 blockHeight := res.LastBlockHeight 250 if blockHeight < 0 { 251 return fmt.Errorf("got a negative last block height (%d) from the app", blockHeight) 252 } 253 appHash := res.LastBlockAppHash 254 255 h.logger.Info("ABCI Handshake App Info", 256 "height", blockHeight, 257 "hash", appHash, 258 "software-version", res.Version, 259 "protocol-version", res.AppVersion, 260 ) 261 262 // Only set the version if there is no existing state. 263 if h.initialState.LastBlockHeight == 0 { 264 h.initialState.Version.Consensus.App = res.AppVersion 265 } 266 267 // Replay blocks up to the latest in the blockstore. 268 _, err = h.ReplayBlocks(h.initialState, appHash, blockHeight, proxyApp) 269 if err != nil { 270 return fmt.Errorf("error on replay: %v", err) 271 } 272 273 h.logger.Info("Completed ABCI Handshake - Tendermint and App are synced", 274 "appHeight", blockHeight, "appHash", appHash) 275 276 // TODO: (on restart) replay mempool 277 278 return nil 279 } 280 281 // ReplayBlocks replays all blocks since appBlockHeight and ensures the result 282 // matches the current state. 283 // Returns the final AppHash or an error. 284 func (h *Handshaker) ReplayBlocks( 285 state sm.State, 286 appHash []byte, 287 appBlockHeight int64, 288 proxyApp proxy.AppConns, 289 ) ([]byte, error) { 290 storeBlockBase := h.store.Base() 291 storeBlockHeight := h.store.Height() 292 stateBlockHeight := state.LastBlockHeight 293 h.logger.Info( 294 "ABCI Replay Blocks", 295 "appHeight", 296 appBlockHeight, 297 "storeHeight", 298 storeBlockHeight, 299 "stateHeight", 300 stateBlockHeight) 301 302 // If appBlockHeight == 0 it means that we are at genesis and hence should send InitChain. 303 if appBlockHeight == 0 { 304 validators := make([]*types.Validator, len(h.genDoc.Validators)) 305 for i, val := range h.genDoc.Validators { 306 validators[i] = types.NewValidator(val.PubKey, val.Power) 307 } 308 validatorSet := types.NewValidatorSet(validators) 309 nextVals := types.TM2PB.ValidatorUpdates(validatorSet) 310 csParams := types.TM2PB.ConsensusParams(h.genDoc.ConsensusParams) 311 req := abci.RequestInitChain{ 312 Time: h.genDoc.GenesisTime, 313 ChainId: h.genDoc.ChainID, 314 InitialHeight: h.genDoc.InitialHeight, 315 ConsensusParams: csParams, 316 Validators: nextVals, 317 AppStateBytes: h.genDoc.AppState, 318 } 319 res, err := proxyApp.Consensus().InitChainSync(req) 320 if err != nil { 321 return nil, err 322 } 323 324 appHash = res.AppHash 325 326 if stateBlockHeight == 0 { // we only update state when we are in initial state 327 // If the app did not return an app hash, we keep the one set from the genesis doc in 328 // the state. We don't set appHash since we don't want the genesis doc app hash 329 // recorded in the genesis block. We should probably just remove GenesisDoc.AppHash. 330 if len(res.AppHash) > 0 { 331 state.AppHash = res.AppHash 332 } 333 // If the app returned validators or consensus params, update the state. 334 if len(res.Validators) > 0 { 335 vals, err := types.PB2TM.ValidatorUpdates(res.Validators) 336 if err != nil { 337 return nil, err 338 } 339 state.Validators = types.NewValidatorSet(vals) 340 state.NextValidators = types.NewValidatorSet(vals).CopyIncrementProposerPriority(1) 341 } else if len(h.genDoc.Validators) == 0 { 342 // If validator set is not set in genesis and still empty after InitChain, exit. 343 return nil, fmt.Errorf("validator set is nil in genesis and still empty after InitChain") 344 } 345 346 if res.ConsensusParams != nil { 347 state.ConsensusParams = types.UpdateConsensusParams(state.ConsensusParams, res.ConsensusParams) 348 state.Version.Consensus.App = state.ConsensusParams.Version.AppVersion 349 } 350 // We update the last results hash with the empty hash, to conform with RFC-6962. 351 state.LastResultsHash = merkle.HashFromByteSlices(nil) 352 if err := h.stateStore.Save(state); err != nil { 353 return nil, err 354 } 355 } 356 } 357 358 // First handle edge cases and constraints on the storeBlockHeight and storeBlockBase. 359 switch { 360 case storeBlockHeight == 0: 361 assertAppHashEqualsOneFromState(appHash, state) 362 return appHash, nil 363 364 case appBlockHeight == 0 && state.InitialHeight < storeBlockBase: 365 // the app has no state, and the block store is truncated above the initial height 366 return appHash, sm.ErrAppBlockHeightTooLow{AppHeight: appBlockHeight, StoreBase: storeBlockBase} 367 368 case appBlockHeight > 0 && appBlockHeight < storeBlockBase-1: 369 // the app is too far behind truncated store (can be 1 behind since we replay the next) 370 return appHash, sm.ErrAppBlockHeightTooLow{AppHeight: appBlockHeight, StoreBase: storeBlockBase} 371 372 case storeBlockHeight < appBlockHeight: 373 // the app should never be ahead of the store (but this is under app's control) 374 return appHash, sm.ErrAppBlockHeightTooHigh{CoreHeight: storeBlockHeight, AppHeight: appBlockHeight} 375 376 case storeBlockHeight < stateBlockHeight: 377 // the state should never be ahead of the store (this is under tendermint's control) 378 panic(fmt.Sprintf("StateBlockHeight (%d) > StoreBlockHeight (%d)", stateBlockHeight, storeBlockHeight)) 379 380 case storeBlockHeight > stateBlockHeight+1: 381 // store should be at most one ahead of the state (this is under tendermint's control) 382 panic(fmt.Sprintf("StoreBlockHeight (%d) > StateBlockHeight + 1 (%d)", storeBlockHeight, stateBlockHeight+1)) 383 } 384 385 var err error 386 // Now either store is equal to state, or one ahead. 387 // For each, consider all cases of where the app could be, given app <= store 388 if storeBlockHeight == stateBlockHeight { 389 // Tendermint ran Commit and saved the state. 390 // Either the app is asking for replay, or we're all synced up. 391 if appBlockHeight < storeBlockHeight { 392 // the app is behind, so replay blocks, but no need to go through WAL (state is already synced to store) 393 return h.replayBlocks(state, proxyApp, appBlockHeight, storeBlockHeight, false) 394 395 } else if appBlockHeight == storeBlockHeight { 396 // We're good! 397 assertAppHashEqualsOneFromState(appHash, state) 398 return appHash, nil 399 } 400 401 } else if storeBlockHeight == stateBlockHeight+1 { 402 // We saved the block in the store but haven't updated the state, 403 // so we'll need to replay a block using the WAL. 404 switch { 405 case appBlockHeight < stateBlockHeight: 406 // the app is further behind than it should be, so replay blocks 407 // but leave the last block to go through the WAL 408 return h.replayBlocks(state, proxyApp, appBlockHeight, storeBlockHeight, true) 409 410 case appBlockHeight == stateBlockHeight: 411 // We haven't run Commit (both the state and app are one block behind), 412 // so replayBlock with the real app. 413 // NOTE: We could instead use the cs.WAL on cs.Start, 414 // but we'd have to allow the WAL to replay a block that wrote it's #ENDHEIGHT 415 h.logger.Info("Replay last block using real app") 416 state, err = h.replayBlock(state, storeBlockHeight, proxyApp.Consensus()) 417 return state.AppHash, err 418 419 case appBlockHeight == storeBlockHeight: 420 // We ran Commit, but didn't save the state, so replayBlock with mock app. 421 abciResponses, err := h.stateStore.LoadABCIResponses(storeBlockHeight) 422 if err != nil { 423 return nil, err 424 } 425 mockApp := newMockProxyApp(appHash, abciResponses) 426 h.logger.Info("Replay last block using mock app") 427 state, err = h.replayBlock(state, storeBlockHeight, mockApp) 428 return state.AppHash, err 429 } 430 431 } 432 433 panic(fmt.Sprintf("uncovered case! appHeight: %d, storeHeight: %d, stateHeight: %d", 434 appBlockHeight, storeBlockHeight, stateBlockHeight)) 435 } 436 437 func (h *Handshaker) replayBlocks( 438 state sm.State, 439 proxyApp proxy.AppConns, 440 appBlockHeight, 441 storeBlockHeight int64, 442 mutateState bool) ([]byte, error) { 443 // App is further behind than it should be, so we need to replay blocks. 444 // We replay all blocks from appBlockHeight+1. 445 // 446 // Note that we don't have an old version of the state, 447 // so we by-pass state validation/mutation using sm.ExecCommitBlock. 448 // This also means we won't be saving validator sets if they change during this period. 449 // TODO: Load the historical information to fix this and just use state.ApplyBlock 450 // 451 // If mutateState == true, the final block is replayed with h.replayBlock() 452 453 var appHash []byte 454 var err error 455 finalBlock := storeBlockHeight 456 if mutateState { 457 finalBlock-- 458 } 459 firstBlock := appBlockHeight + 1 460 if firstBlock == 1 { 461 firstBlock = state.InitialHeight 462 } 463 for i := firstBlock; i <= finalBlock; i++ { 464 h.logger.Info("Applying block", "height", i) 465 block := h.store.LoadBlock(i) 466 // Extra check to ensure the app was not changed in a way it shouldn't have. 467 if len(appHash) > 0 { 468 assertAppHashEqualsOneFromBlock(appHash, block) 469 } 470 471 appHash, err = sm.ExecCommitBlock(proxyApp.Consensus(), block, h.logger, h.stateStore, h.genDoc.InitialHeight) 472 if err != nil { 473 return nil, err 474 } 475 476 h.nBlocks++ 477 } 478 479 if mutateState { 480 // sync the final block 481 state, err = h.replayBlock(state, storeBlockHeight, proxyApp.Consensus()) 482 if err != nil { 483 return nil, err 484 } 485 appHash = state.AppHash 486 } 487 488 assertAppHashEqualsOneFromState(appHash, state) 489 return appHash, nil 490 } 491 492 // ApplyBlock on the proxyApp with the last block. 493 func (h *Handshaker) replayBlock(state sm.State, height int64, proxyApp proxy.AppConnConsensus) (sm.State, error) { 494 block := h.store.LoadBlock(height) 495 meta := h.store.LoadBlockMeta(height) 496 497 // Use stubs for both mempool and evidence pool since no transactions nor 498 // evidence are needed here - block already exists. 499 blockExec := sm.NewBlockExecutor(h.stateStore, h.logger, proxyApp, emptyMempool{}, sm.EmptyEvidencePool{}) 500 blockExec.SetEventBus(h.eventBus) 501 502 var err error 503 state, _, err = blockExec.ApplyBlock(state, meta.BlockID, block) 504 if err != nil { 505 return sm.State{}, err 506 } 507 508 h.nBlocks++ 509 510 return state, nil 511 } 512 513 func assertAppHashEqualsOneFromBlock(appHash []byte, block *types.Block) { 514 if !bytes.Equal(appHash, block.AppHash) { 515 panic(fmt.Sprintf(`block.AppHash does not match AppHash after replay. Got %X, expected %X. 516 517 Block: %v 518 `, 519 appHash, block.AppHash, block)) 520 } 521 } 522 523 func assertAppHashEqualsOneFromState(appHash []byte, state sm.State) { 524 if !bytes.Equal(appHash, state.AppHash) { 525 panic(fmt.Sprintf(`state.AppHash does not match AppHash after replay. Got 526 %X, expected %X. 527 528 State: %v 529 530 Did you reset Tendermint without resetting your application's data?`, 531 appHash, state.AppHash, state)) 532 } 533 }