github.com/franono/tendermint@v0.32.2-0.20200527150959-749313264ce9/consensus/replay_test.go (about) 1 package consensus 2 3 import ( 4 "bytes" 5 "context" 6 "fmt" 7 "io" 8 "io/ioutil" 9 "os" 10 "path/filepath" 11 "runtime" 12 "testing" 13 "time" 14 15 "github.com/stretchr/testify/assert" 16 "github.com/stretchr/testify/require" 17 18 "sort" 19 20 dbm "github.com/tendermint/tm-db" 21 22 "github.com/franono/tendermint/abci/example/kvstore" 23 abci "github.com/franono/tendermint/abci/types" 24 cfg "github.com/franono/tendermint/config" 25 "github.com/franono/tendermint/crypto" 26 "github.com/franono/tendermint/libs/log" 27 tmrand "github.com/franono/tendermint/libs/rand" 28 mempl "github.com/franono/tendermint/mempool" 29 "github.com/franono/tendermint/privval" 30 "github.com/franono/tendermint/proxy" 31 sm "github.com/franono/tendermint/state" 32 "github.com/franono/tendermint/types" 33 "github.com/franono/tendermint/version" 34 ) 35 36 func TestMain(m *testing.M) { 37 config = ResetConfig("consensus_reactor_test") 38 consensusReplayConfig = ResetConfig("consensus_replay_test") 39 configStateTest := ResetConfig("consensus_state_test") 40 configMempoolTest := ResetConfig("consensus_mempool_test") 41 configByzantineTest := ResetConfig("consensus_byzantine_test") 42 code := m.Run() 43 os.RemoveAll(config.RootDir) 44 os.RemoveAll(consensusReplayConfig.RootDir) 45 os.RemoveAll(configStateTest.RootDir) 46 os.RemoveAll(configMempoolTest.RootDir) 47 os.RemoveAll(configByzantineTest.RootDir) 48 os.Exit(code) 49 } 50 51 // These tests ensure we can always recover from failure at any part of the consensus process. 52 // There are two general failure scenarios: failure during consensus, and failure while applying the block. 53 // Only the latter interacts with the app and store, 54 // but the former has to deal with restrictions on re-use of priv_validator keys. 55 // The `WAL Tests` are for failures during the consensus; 56 // the `Handshake Tests` are for failures in applying the block. 57 // With the help of the WAL, we can recover from it all! 58 59 //------------------------------------------------------------------------------------------ 60 // WAL Tests 61 62 // TODO: It would be better to verify explicitly which states we can recover from without the wal 63 // and which ones we need the wal for - then we'd also be able to only flush the 64 // wal writer when we need to, instead of with every message. 65 66 func startNewStateAndWaitForBlock(t *testing.T, consensusReplayConfig *cfg.Config, 67 lastBlockHeight int64, blockDB dbm.DB, stateDB dbm.DB) { 68 logger := log.TestingLogger() 69 state, _ := sm.LoadStateFromDBOrGenesisFile(stateDB, consensusReplayConfig.GenesisFile()) 70 privValidator := loadPrivValidator(consensusReplayConfig) 71 cs := newStateWithConfigAndBlockStore( 72 consensusReplayConfig, 73 state, 74 privValidator, 75 kvstore.NewApplication(), 76 blockDB, 77 ) 78 cs.SetLogger(logger) 79 80 bytes, _ := ioutil.ReadFile(cs.config.WalFile()) 81 t.Logf("====== WAL: \n\r%X\n", bytes) 82 83 err := cs.Start() 84 require.NoError(t, err) 85 defer cs.Stop() 86 87 // This is just a signal that we haven't halted; its not something contained 88 // in the WAL itself. Assuming the consensus state is running, replay of any 89 // WAL, including the empty one, should eventually be followed by a new 90 // block, or else something is wrong. 91 newBlockSub, err := cs.eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock) 92 require.NoError(t, err) 93 select { 94 case <-newBlockSub.Out(): 95 case <-newBlockSub.Cancelled(): 96 t.Fatal("newBlockSub was cancelled") 97 case <-time.After(120 * time.Second): 98 t.Fatal("Timed out waiting for new block (see trace above)") 99 } 100 } 101 102 func sendTxs(ctx context.Context, cs *State) { 103 for i := 0; i < 256; i++ { 104 select { 105 case <-ctx.Done(): 106 return 107 default: 108 tx := []byte{byte(i)} 109 assertMempool(cs.txNotifier).CheckTx(tx, nil, mempl.TxInfo{}) 110 i++ 111 } 112 } 113 } 114 115 // TestWALCrash uses crashing WAL to test we can recover from any WAL failure. 116 func TestWALCrash(t *testing.T) { 117 testCases := []struct { 118 name string 119 initFn func(dbm.DB, *State, context.Context) 120 heightToStop int64 121 }{ 122 {"empty block", 123 func(stateDB dbm.DB, cs *State, ctx context.Context) {}, 124 1}, 125 {"many non-empty blocks", 126 func(stateDB dbm.DB, cs *State, ctx context.Context) { 127 go sendTxs(ctx, cs) 128 }, 129 3}, 130 } 131 132 for i, tc := range testCases { 133 tc := tc 134 consensusReplayConfig := ResetConfig(fmt.Sprintf("%s_%d", t.Name(), i)) 135 t.Run(tc.name, func(t *testing.T) { 136 crashWALandCheckLiveness(t, consensusReplayConfig, tc.initFn, tc.heightToStop) 137 }) 138 } 139 } 140 141 func crashWALandCheckLiveness(t *testing.T, consensusReplayConfig *cfg.Config, 142 initFn func(dbm.DB, *State, context.Context), heightToStop int64) { 143 walPanicked := make(chan error) 144 crashingWal := &crashingWAL{panicCh: walPanicked, heightToStop: heightToStop} 145 146 i := 1 147 LOOP: 148 for { 149 t.Logf("====== LOOP %d\n", i) 150 151 // create consensus state from a clean slate 152 logger := log.NewNopLogger() 153 blockDB := dbm.NewMemDB() 154 stateDB := blockDB 155 state, _ := sm.MakeGenesisStateFromFile(consensusReplayConfig.GenesisFile()) 156 privValidator := loadPrivValidator(consensusReplayConfig) 157 cs := newStateWithConfigAndBlockStore( 158 consensusReplayConfig, 159 state, 160 privValidator, 161 kvstore.NewApplication(), 162 blockDB, 163 ) 164 cs.SetLogger(logger) 165 166 // start sending transactions 167 ctx, cancel := context.WithCancel(context.Background()) 168 initFn(stateDB, cs, ctx) 169 170 // clean up WAL file from the previous iteration 171 walFile := cs.config.WalFile() 172 os.Remove(walFile) 173 174 // set crashing WAL 175 csWal, err := cs.OpenWAL(walFile) 176 require.NoError(t, err) 177 crashingWal.next = csWal 178 // reset the message counter 179 crashingWal.msgIndex = 1 180 cs.wal = crashingWal 181 182 // start consensus state 183 err = cs.Start() 184 require.NoError(t, err) 185 186 i++ 187 188 select { 189 case err := <-walPanicked: 190 t.Logf("WAL panicked: %v", err) 191 192 // make sure we can make blocks after a crash 193 startNewStateAndWaitForBlock(t, consensusReplayConfig, cs.Height, blockDB, stateDB) 194 195 // stop consensus state and transactions sender (initFn) 196 cs.Stop() 197 cancel() 198 199 // if we reached the required height, exit 200 if _, ok := err.(ReachedHeightToStopError); ok { 201 break LOOP 202 } 203 case <-time.After(10 * time.Second): 204 t.Fatal("WAL did not panic for 10 seconds (check the log)") 205 } 206 } 207 } 208 209 // crashingWAL is a WAL which crashes or rather simulates a crash during Save 210 // (before and after). It remembers a message for which we last panicked 211 // (lastPanickedForMsgIndex), so we don't panic for it in subsequent iterations. 212 type crashingWAL struct { 213 next WAL 214 panicCh chan error 215 heightToStop int64 216 217 msgIndex int // current message index 218 lastPanickedForMsgIndex int // last message for which we panicked 219 } 220 221 var _ WAL = &crashingWAL{} 222 223 // WALWriteError indicates a WAL crash. 224 type WALWriteError struct { 225 msg string 226 } 227 228 func (e WALWriteError) Error() string { 229 return e.msg 230 } 231 232 // ReachedHeightToStopError indicates we've reached the required consensus 233 // height and may exit. 234 type ReachedHeightToStopError struct { 235 height int64 236 } 237 238 func (e ReachedHeightToStopError) Error() string { 239 return fmt.Sprintf("reached height to stop %d", e.height) 240 } 241 242 // Write simulate WAL's crashing by sending an error to the panicCh and then 243 // exiting the cs.receiveRoutine. 244 func (w *crashingWAL) Write(m WALMessage) error { 245 if endMsg, ok := m.(EndHeightMessage); ok { 246 if endMsg.Height == w.heightToStop { 247 w.panicCh <- ReachedHeightToStopError{endMsg.Height} 248 runtime.Goexit() 249 return nil 250 } 251 252 return w.next.Write(m) 253 } 254 255 if w.msgIndex > w.lastPanickedForMsgIndex { 256 w.lastPanickedForMsgIndex = w.msgIndex 257 _, file, line, _ := runtime.Caller(1) 258 w.panicCh <- WALWriteError{fmt.Sprintf("failed to write %T to WAL (fileline: %s:%d)", m, file, line)} 259 runtime.Goexit() 260 return nil 261 } 262 263 w.msgIndex++ 264 return w.next.Write(m) 265 } 266 267 func (w *crashingWAL) WriteSync(m WALMessage) error { 268 return w.Write(m) 269 } 270 271 func (w *crashingWAL) FlushAndSync() error { return w.next.FlushAndSync() } 272 273 func (w *crashingWAL) SearchForEndHeight( 274 height int64, 275 options *WALSearchOptions) (rd io.ReadCloser, found bool, err error) { 276 return w.next.SearchForEndHeight(height, options) 277 } 278 279 func (w *crashingWAL) Start() error { return w.next.Start() } 280 func (w *crashingWAL) Stop() error { return w.next.Stop() } 281 func (w *crashingWAL) Wait() { w.next.Wait() } 282 283 //------------------------------------------------------------------------------------------ 284 type testSim struct { 285 GenesisState sm.State 286 Config *cfg.Config 287 Chain []*types.Block 288 Commits []*types.Commit 289 CleanupFunc cleanupFunc 290 } 291 292 const ( 293 numBlocks = 6 294 ) 295 296 var ( 297 mempool = emptyMempool{} 298 evpool = emptyEvidencePool{} 299 300 sim testSim 301 ) 302 303 //--------------------------------------- 304 // Test handshake/replay 305 306 // 0 - all synced up 307 // 1 - saved block but app and state are behind 308 // 2 - save block and committed but state is behind 309 // 3 - save block and committed with truncated block store and state behind 310 var modes = []uint{0, 1, 2, 3} 311 312 // This is actually not a test, it's for storing validator change tx data for testHandshakeReplay 313 func TestSimulateValidatorsChange(t *testing.T) { 314 nPeers := 7 315 nVals := 4 316 css, genDoc, config, cleanup := randConsensusNetWithPeers( 317 nVals, 318 nPeers, 319 "replay_test", 320 newMockTickerFunc(true), 321 newPersistentKVStoreWithPath) 322 sim.Config = config 323 sim.GenesisState, _ = sm.MakeGenesisState(genDoc) 324 sim.CleanupFunc = cleanup 325 326 partSize := types.BlockPartSizeBytes 327 328 newRoundCh := subscribe(css[0].eventBus, types.EventQueryNewRound) 329 proposalCh := subscribe(css[0].eventBus, types.EventQueryCompleteProposal) 330 331 vss := make([]*validatorStub, nPeers) 332 for i := 0; i < nPeers; i++ { 333 vss[i] = newValidatorStub(css[i].privValidator, i) 334 } 335 height, round := css[0].Height, css[0].Round 336 337 // start the machine 338 startTestRound(css[0], height, round) 339 incrementHeight(vss...) 340 ensureNewRound(newRoundCh, height, 0) 341 ensureNewProposal(proposalCh, height, round) 342 rs := css[0].GetRoundState() 343 signAddVotes(css[0], types.PrecommitType, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), vss[1:nVals]...) 344 ensureNewRound(newRoundCh, height+1, 0) 345 346 ///////////////////////////////////////////////////////////////////////////// 347 // HEIGHT 2 348 ///////////////////////////////////////////////////////////////////////////// 349 height++ 350 incrementHeight(vss...) 351 newValidatorPubKey1, err := css[nVals].privValidator.GetPubKey() 352 require.NoError(t, err) 353 valPubKey1ABCI := types.TM2PB.PubKey(newValidatorPubKey1) 354 newValidatorTx1 := kvstore.MakeValSetChangeTx(valPubKey1ABCI, testMinPower) 355 err = assertMempool(css[0].txNotifier).CheckTx(newValidatorTx1, nil, mempl.TxInfo{}) 356 assert.Nil(t, err) 357 propBlock, _ := css[0].createProposalBlock() //changeProposer(t, cs1, vs2) 358 propBlockParts := propBlock.MakePartSet(partSize) 359 blockID := types.BlockID{Hash: propBlock.Hash(), PartsHeader: propBlockParts.Header()} 360 proposal := types.NewProposal(vss[1].Height, round, -1, blockID) 361 if err := vss[1].SignProposal(config.ChainID(), proposal); err != nil { 362 t.Fatal("failed to sign bad proposal", err) 363 } 364 365 // set the proposal block 366 if err := css[0].SetProposalAndBlock(proposal, propBlock, propBlockParts, "some peer"); err != nil { 367 t.Fatal(err) 368 } 369 ensureNewProposal(proposalCh, height, round) 370 rs = css[0].GetRoundState() 371 signAddVotes(css[0], types.PrecommitType, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), vss[1:nVals]...) 372 ensureNewRound(newRoundCh, height+1, 0) 373 374 ///////////////////////////////////////////////////////////////////////////// 375 // HEIGHT 3 376 ///////////////////////////////////////////////////////////////////////////// 377 height++ 378 incrementHeight(vss...) 379 updateValidatorPubKey1, err := css[nVals].privValidator.GetPubKey() 380 require.NoError(t, err) 381 updatePubKey1ABCI := types.TM2PB.PubKey(updateValidatorPubKey1) 382 updateValidatorTx1 := kvstore.MakeValSetChangeTx(updatePubKey1ABCI, 25) 383 err = assertMempool(css[0].txNotifier).CheckTx(updateValidatorTx1, nil, mempl.TxInfo{}) 384 assert.Nil(t, err) 385 propBlock, _ = css[0].createProposalBlock() //changeProposer(t, cs1, vs2) 386 propBlockParts = propBlock.MakePartSet(partSize) 387 blockID = types.BlockID{Hash: propBlock.Hash(), PartsHeader: propBlockParts.Header()} 388 proposal = types.NewProposal(vss[2].Height, round, -1, blockID) 389 if err := vss[2].SignProposal(config.ChainID(), proposal); err != nil { 390 t.Fatal("failed to sign bad proposal", err) 391 } 392 393 // set the proposal block 394 if err := css[0].SetProposalAndBlock(proposal, propBlock, propBlockParts, "some peer"); err != nil { 395 t.Fatal(err) 396 } 397 ensureNewProposal(proposalCh, height, round) 398 rs = css[0].GetRoundState() 399 signAddVotes(css[0], types.PrecommitType, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), vss[1:nVals]...) 400 ensureNewRound(newRoundCh, height+1, 0) 401 402 ///////////////////////////////////////////////////////////////////////////// 403 // HEIGHT 4 404 ///////////////////////////////////////////////////////////////////////////// 405 height++ 406 incrementHeight(vss...) 407 newValidatorPubKey2, err := css[nVals+1].privValidator.GetPubKey() 408 require.NoError(t, err) 409 newVal2ABCI := types.TM2PB.PubKey(newValidatorPubKey2) 410 newValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, testMinPower) 411 err = assertMempool(css[0].txNotifier).CheckTx(newValidatorTx2, nil, mempl.TxInfo{}) 412 assert.Nil(t, err) 413 newValidatorPubKey3, err := css[nVals+2].privValidator.GetPubKey() 414 require.NoError(t, err) 415 newVal3ABCI := types.TM2PB.PubKey(newValidatorPubKey3) 416 newValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, testMinPower) 417 err = assertMempool(css[0].txNotifier).CheckTx(newValidatorTx3, nil, mempl.TxInfo{}) 418 assert.Nil(t, err) 419 propBlock, _ = css[0].createProposalBlock() //changeProposer(t, cs1, vs2) 420 propBlockParts = propBlock.MakePartSet(partSize) 421 blockID = types.BlockID{Hash: propBlock.Hash(), PartsHeader: propBlockParts.Header()} 422 newVss := make([]*validatorStub, nVals+1) 423 copy(newVss, vss[:nVals+1]) 424 sort.Sort(ValidatorStubsByPower(newVss)) 425 426 valIndexFn := func(cssIdx int) int { 427 for i, vs := range newVss { 428 vsPubKey, err := vs.GetPubKey() 429 require.NoError(t, err) 430 431 cssPubKey, err := css[cssIdx].privValidator.GetPubKey() 432 require.NoError(t, err) 433 434 if vsPubKey.Equals(cssPubKey) { 435 return i 436 } 437 } 438 panic(fmt.Sprintf("validator css[%d] not found in newVss", cssIdx)) 439 } 440 441 selfIndex := valIndexFn(0) 442 443 proposal = types.NewProposal(vss[3].Height, round, -1, blockID) 444 if err := vss[3].SignProposal(config.ChainID(), proposal); err != nil { 445 t.Fatal("failed to sign bad proposal", err) 446 } 447 448 // set the proposal block 449 if err := css[0].SetProposalAndBlock(proposal, propBlock, propBlockParts, "some peer"); err != nil { 450 t.Fatal(err) 451 } 452 ensureNewProposal(proposalCh, height, round) 453 454 removeValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, 0) 455 err = assertMempool(css[0].txNotifier).CheckTx(removeValidatorTx2, nil, mempl.TxInfo{}) 456 assert.Nil(t, err) 457 458 rs = css[0].GetRoundState() 459 for i := 0; i < nVals+1; i++ { 460 if i == selfIndex { 461 continue 462 } 463 signAddVotes(css[0], types.PrecommitType, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), newVss[i]) 464 } 465 466 ensureNewRound(newRoundCh, height+1, 0) 467 468 ///////////////////////////////////////////////////////////////////////////// 469 // HEIGHT 5 470 ///////////////////////////////////////////////////////////////////////////// 471 height++ 472 incrementHeight(vss...) 473 // Reflect the changes to vss[nVals] at height 3 and resort newVss. 474 newVssIdx := valIndexFn(nVals) 475 newVss[newVssIdx].VotingPower = 25 476 sort.Sort(ValidatorStubsByPower(newVss)) 477 selfIndex = valIndexFn(0) 478 ensureNewProposal(proposalCh, height, round) 479 rs = css[0].GetRoundState() 480 for i := 0; i < nVals+1; i++ { 481 if i == selfIndex { 482 continue 483 } 484 signAddVotes(css[0], types.PrecommitType, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), newVss[i]) 485 } 486 ensureNewRound(newRoundCh, height+1, 0) 487 488 ///////////////////////////////////////////////////////////////////////////// 489 // HEIGHT 6 490 ///////////////////////////////////////////////////////////////////////////// 491 height++ 492 incrementHeight(vss...) 493 removeValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, 0) 494 err = assertMempool(css[0].txNotifier).CheckTx(removeValidatorTx3, nil, mempl.TxInfo{}) 495 assert.Nil(t, err) 496 propBlock, _ = css[0].createProposalBlock() //changeProposer(t, cs1, vs2) 497 propBlockParts = propBlock.MakePartSet(partSize) 498 blockID = types.BlockID{Hash: propBlock.Hash(), PartsHeader: propBlockParts.Header()} 499 newVss = make([]*validatorStub, nVals+3) 500 copy(newVss, vss[:nVals+3]) 501 sort.Sort(ValidatorStubsByPower(newVss)) 502 503 selfIndex = valIndexFn(0) 504 proposal = types.NewProposal(vss[1].Height, round, -1, blockID) 505 if err := vss[1].SignProposal(config.ChainID(), proposal); err != nil { 506 t.Fatal("failed to sign bad proposal", err) 507 } 508 509 // set the proposal block 510 if err := css[0].SetProposalAndBlock(proposal, propBlock, propBlockParts, "some peer"); err != nil { 511 t.Fatal(err) 512 } 513 ensureNewProposal(proposalCh, height, round) 514 rs = css[0].GetRoundState() 515 for i := 0; i < nVals+3; i++ { 516 if i == selfIndex { 517 continue 518 } 519 signAddVotes(css[0], types.PrecommitType, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), newVss[i]) 520 } 521 ensureNewRound(newRoundCh, height+1, 0) 522 523 sim.Chain = make([]*types.Block, 0) 524 sim.Commits = make([]*types.Commit, 0) 525 for i := 1; i <= numBlocks; i++ { 526 sim.Chain = append(sim.Chain, css[0].blockStore.LoadBlock(int64(i))) 527 sim.Commits = append(sim.Commits, css[0].blockStore.LoadBlockCommit(int64(i))) 528 } 529 } 530 531 // Sync from scratch 532 func TestHandshakeReplayAll(t *testing.T) { 533 for _, m := range modes { 534 testHandshakeReplay(t, config, 0, m, false) 535 } 536 for _, m := range modes { 537 testHandshakeReplay(t, config, 0, m, true) 538 } 539 } 540 541 // Sync many, not from scratch 542 func TestHandshakeReplaySome(t *testing.T) { 543 for _, m := range modes { 544 testHandshakeReplay(t, config, 2, m, false) 545 } 546 for _, m := range modes { 547 testHandshakeReplay(t, config, 2, m, true) 548 } 549 } 550 551 // Sync from lagging by one 552 func TestHandshakeReplayOne(t *testing.T) { 553 for _, m := range modes { 554 testHandshakeReplay(t, config, numBlocks-1, m, false) 555 } 556 for _, m := range modes { 557 testHandshakeReplay(t, config, numBlocks-1, m, true) 558 } 559 } 560 561 // Sync from caught up 562 func TestHandshakeReplayNone(t *testing.T) { 563 for _, m := range modes { 564 testHandshakeReplay(t, config, numBlocks, m, false) 565 } 566 for _, m := range modes { 567 testHandshakeReplay(t, config, numBlocks, m, true) 568 } 569 } 570 571 // Test mockProxyApp should not panic when app return ABCIResponses with some empty ResponseDeliverTx 572 func TestMockProxyApp(t *testing.T) { 573 sim.CleanupFunc() //clean the test env created in TestSimulateValidatorsChange 574 logger := log.TestingLogger() 575 var validTxs, invalidTxs = 0, 0 576 txIndex := 0 577 578 assert.NotPanics(t, func() { 579 abciResWithEmptyDeliverTx := new(sm.ABCIResponses) 580 abciResWithEmptyDeliverTx.DeliverTxs = make([]*abci.ResponseDeliverTx, 0) 581 abciResWithEmptyDeliverTx.DeliverTxs = append(abciResWithEmptyDeliverTx.DeliverTxs, &abci.ResponseDeliverTx{}) 582 583 // called when saveABCIResponses: 584 bytes := cdc.MustMarshalBinaryBare(abciResWithEmptyDeliverTx) 585 loadedAbciRes := new(sm.ABCIResponses) 586 587 // this also happens sm.LoadABCIResponses 588 err := cdc.UnmarshalBinaryBare(bytes, loadedAbciRes) 589 require.NoError(t, err) 590 591 mock := newMockProxyApp([]byte("mock_hash"), loadedAbciRes) 592 593 abciRes := new(sm.ABCIResponses) 594 abciRes.DeliverTxs = make([]*abci.ResponseDeliverTx, len(loadedAbciRes.DeliverTxs)) 595 // Execute transactions and get hash. 596 proxyCb := func(req *abci.Request, res *abci.Response) { 597 if r, ok := res.Value.(*abci.Response_DeliverTx); ok { 598 // TODO: make use of res.Log 599 // TODO: make use of this info 600 // Blocks may include invalid txs. 601 txRes := r.DeliverTx 602 if txRes.Code == abci.CodeTypeOK { 603 validTxs++ 604 } else { 605 logger.Debug("Invalid tx", "code", txRes.Code, "log", txRes.Log) 606 invalidTxs++ 607 } 608 abciRes.DeliverTxs[txIndex] = txRes 609 txIndex++ 610 } 611 } 612 mock.SetResponseCallback(proxyCb) 613 614 someTx := []byte("tx") 615 mock.DeliverTxAsync(abci.RequestDeliverTx{Tx: someTx}) 616 }) 617 assert.True(t, validTxs == 1) 618 assert.True(t, invalidTxs == 0) 619 } 620 621 func tempWALWithData(data []byte) string { 622 walFile, err := ioutil.TempFile("", "wal") 623 if err != nil { 624 panic(fmt.Sprintf("failed to create temp WAL file: %v", err)) 625 } 626 _, err = walFile.Write(data) 627 if err != nil { 628 panic(fmt.Sprintf("failed to write to temp WAL file: %v", err)) 629 } 630 if err := walFile.Close(); err != nil { 631 panic(fmt.Sprintf("failed to close temp WAL file: %v", err)) 632 } 633 return walFile.Name() 634 } 635 636 // Make some blocks. Start a fresh app and apply nBlocks blocks. 637 // Then restart the app and sync it up with the remaining blocks 638 func testHandshakeReplay(t *testing.T, config *cfg.Config, nBlocks int, mode uint, testValidatorsChange bool) { 639 var chain []*types.Block 640 var commits []*types.Commit 641 var store *mockBlockStore 642 var stateDB dbm.DB 643 var genisisState sm.State 644 if testValidatorsChange { 645 testConfig := ResetConfig(fmt.Sprintf("%s_%v_m", t.Name(), mode)) 646 defer os.RemoveAll(testConfig.RootDir) 647 stateDB = dbm.NewMemDB() 648 genisisState = sim.GenesisState 649 config = sim.Config 650 chain = append([]*types.Block{}, sim.Chain...) // copy chain 651 commits = sim.Commits 652 store = newMockBlockStore(config, genisisState.ConsensusParams) 653 } else { //test single node 654 testConfig := ResetConfig(fmt.Sprintf("%s_%v_s", t.Name(), mode)) 655 defer os.RemoveAll(testConfig.RootDir) 656 walBody, err := WALWithNBlocks(t, numBlocks) 657 require.NoError(t, err) 658 walFile := tempWALWithData(walBody) 659 config.Consensus.SetWalFile(walFile) 660 661 privVal := privval.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) 662 663 wal, err := NewWAL(walFile) 664 require.NoError(t, err) 665 wal.SetLogger(log.TestingLogger()) 666 err = wal.Start() 667 require.NoError(t, err) 668 defer wal.Stop() 669 670 chain, commits, err = makeBlockchainFromWAL(wal) 671 require.NoError(t, err) 672 pubKey, err := privVal.GetPubKey() 673 require.NoError(t, err) 674 stateDB, genisisState, store = stateAndStore(config, pubKey, kvstore.ProtocolVersion) 675 } 676 store.chain = chain 677 store.commits = commits 678 679 state := genisisState.Copy() 680 // run the chain through state.ApplyBlock to build up the tendermint state 681 state = buildTMStateFromChain(config, stateDB, state, chain, nBlocks, mode) 682 latestAppHash := state.AppHash 683 684 // make a new client creator 685 kvstoreApp := kvstore.NewPersistentKVStoreApplication( 686 filepath.Join(config.DBDir(), fmt.Sprintf("replay_test_%d_%d_a", nBlocks, mode))) 687 688 clientCreator2 := proxy.NewLocalClientCreator(kvstoreApp) 689 if nBlocks > 0 { 690 // run nBlocks against a new client to build up the app state. 691 // use a throwaway tendermint state 692 proxyApp := proxy.NewAppConns(clientCreator2) 693 stateDB1 := dbm.NewMemDB() 694 sm.SaveState(stateDB1, genisisState) 695 buildAppStateFromChain(proxyApp, stateDB1, genisisState, chain, nBlocks, mode) 696 } 697 698 // Prune block store if requested 699 expectError := false 700 if mode == 3 { 701 pruned, err := store.PruneBlocks(2) 702 require.NoError(t, err) 703 require.EqualValues(t, 1, pruned) 704 expectError = int64(nBlocks) < 2 705 } 706 707 // now start the app using the handshake - it should sync 708 genDoc, _ := sm.MakeGenesisDocFromFile(config.GenesisFile()) 709 handshaker := NewHandshaker(stateDB, state, store, genDoc) 710 proxyApp := proxy.NewAppConns(clientCreator2) 711 if err := proxyApp.Start(); err != nil { 712 t.Fatalf("Error starting proxy app connections: %v", err) 713 } 714 defer proxyApp.Stop() 715 err := handshaker.Handshake(proxyApp) 716 if expectError { 717 require.Error(t, err) 718 return 719 } else if err != nil { 720 t.Fatalf("Error on abci handshake: %v", err) 721 } 722 723 // get the latest app hash from the app 724 res, err := proxyApp.Query().InfoSync(abci.RequestInfo{Version: ""}) 725 if err != nil { 726 t.Fatal(err) 727 } 728 729 // the app hash should be synced up 730 if !bytes.Equal(latestAppHash, res.LastBlockAppHash) { 731 t.Fatalf( 732 "Expected app hashes to match after handshake/replay. got %X, expected %X", 733 res.LastBlockAppHash, 734 latestAppHash) 735 } 736 737 expectedBlocksToSync := numBlocks - nBlocks 738 if nBlocks == numBlocks && mode > 0 { 739 expectedBlocksToSync++ 740 } else if nBlocks > 0 && mode == 1 { 741 expectedBlocksToSync++ 742 } 743 744 if handshaker.NBlocks() != expectedBlocksToSync { 745 t.Fatalf("Expected handshake to sync %d blocks, got %d", expectedBlocksToSync, handshaker.NBlocks()) 746 } 747 } 748 749 func applyBlock(stateDB dbm.DB, st sm.State, blk *types.Block, proxyApp proxy.AppConns) sm.State { 750 testPartSize := types.BlockPartSizeBytes 751 blockExec := sm.NewBlockExecutor(stateDB, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool) 752 753 blkID := types.BlockID{Hash: blk.Hash(), PartsHeader: blk.MakePartSet(testPartSize).Header()} 754 newState, _, err := blockExec.ApplyBlock(st, blkID, blk) 755 if err != nil { 756 panic(err) 757 } 758 return newState 759 } 760 761 func buildAppStateFromChain(proxyApp proxy.AppConns, stateDB dbm.DB, 762 state sm.State, chain []*types.Block, nBlocks int, mode uint) { 763 // start a new app without handshake, play nBlocks blocks 764 if err := proxyApp.Start(); err != nil { 765 panic(err) 766 } 767 defer proxyApp.Stop() 768 769 state.Version.Consensus.App = kvstore.ProtocolVersion //simulate handshake, receive app version 770 validators := types.TM2PB.ValidatorUpdates(state.Validators) 771 if _, err := proxyApp.Consensus().InitChainSync(abci.RequestInitChain{ 772 Validators: validators, 773 }); err != nil { 774 panic(err) 775 } 776 sm.SaveState(stateDB, state) //save height 1's validatorsInfo 777 778 switch mode { 779 case 0: 780 for i := 0; i < nBlocks; i++ { 781 block := chain[i] 782 state = applyBlock(stateDB, state, block, proxyApp) 783 } 784 case 1, 2, 3: 785 for i := 0; i < nBlocks-1; i++ { 786 block := chain[i] 787 state = applyBlock(stateDB, state, block, proxyApp) 788 } 789 790 if mode == 2 || mode == 3 { 791 // update the kvstore height and apphash 792 // as if we ran commit but not 793 state = applyBlock(stateDB, state, chain[nBlocks-1], proxyApp) 794 } 795 default: 796 panic(fmt.Sprintf("unknown mode %v", mode)) 797 } 798 799 } 800 801 func buildTMStateFromChain( 802 config *cfg.Config, 803 stateDB dbm.DB, 804 state sm.State, 805 chain []*types.Block, 806 nBlocks int, 807 mode uint) sm.State { 808 // run the whole chain against this client to build up the tendermint state 809 clientCreator := proxy.NewLocalClientCreator( 810 kvstore.NewPersistentKVStoreApplication( 811 filepath.Join(config.DBDir(), fmt.Sprintf("replay_test_%d_%d_t", nBlocks, mode)))) 812 proxyApp := proxy.NewAppConns(clientCreator) 813 if err := proxyApp.Start(); err != nil { 814 panic(err) 815 } 816 defer proxyApp.Stop() 817 818 state.Version.Consensus.App = kvstore.ProtocolVersion //simulate handshake, receive app version 819 validators := types.TM2PB.ValidatorUpdates(state.Validators) 820 if _, err := proxyApp.Consensus().InitChainSync(abci.RequestInitChain{ 821 Validators: validators, 822 }); err != nil { 823 panic(err) 824 } 825 sm.SaveState(stateDB, state) //save height 1's validatorsInfo 826 827 switch mode { 828 case 0: 829 // sync right up 830 for _, block := range chain { 831 state = applyBlock(stateDB, state, block, proxyApp) 832 } 833 834 case 1, 2, 3: 835 // sync up to the penultimate as if we stored the block. 836 // whether we commit or not depends on the appHash 837 for _, block := range chain[:len(chain)-1] { 838 state = applyBlock(stateDB, state, block, proxyApp) 839 } 840 841 // apply the final block to a state copy so we can 842 // get the right next appHash but keep the state back 843 applyBlock(stateDB, state, chain[len(chain)-1], proxyApp) 844 default: 845 panic(fmt.Sprintf("unknown mode %v", mode)) 846 } 847 848 return state 849 } 850 851 func TestHandshakePanicsIfAppReturnsWrongAppHash(t *testing.T) { 852 // 1. Initialize tendermint and commit 3 blocks with the following app hashes: 853 // - 0x01 854 // - 0x02 855 // - 0x03 856 config := ResetConfig("handshake_test_") 857 defer os.RemoveAll(config.RootDir) 858 privVal := privval.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) 859 const appVersion = 0x0 860 pubKey, err := privVal.GetPubKey() 861 require.NoError(t, err) 862 stateDB, state, store := stateAndStore(config, pubKey, appVersion) 863 genDoc, _ := sm.MakeGenesisDocFromFile(config.GenesisFile()) 864 state.LastValidators = state.Validators.Copy() 865 // mode = 0 for committing all the blocks 866 blocks := makeBlocks(3, &state, privVal) 867 store.chain = blocks 868 869 // 2. Tendermint must panic if app returns wrong hash for the first block 870 // - RANDOM HASH 871 // - 0x02 872 // - 0x03 873 { 874 app := &badApp{numBlocks: 3, allHashesAreWrong: true} 875 clientCreator := proxy.NewLocalClientCreator(app) 876 proxyApp := proxy.NewAppConns(clientCreator) 877 err := proxyApp.Start() 878 require.NoError(t, err) 879 defer proxyApp.Stop() 880 881 assert.Panics(t, func() { 882 h := NewHandshaker(stateDB, state, store, genDoc) 883 h.Handshake(proxyApp) 884 }) 885 } 886 887 // 3. Tendermint must panic if app returns wrong hash for the last block 888 // - 0x01 889 // - 0x02 890 // - RANDOM HASH 891 { 892 app := &badApp{numBlocks: 3, onlyLastHashIsWrong: true} 893 clientCreator := proxy.NewLocalClientCreator(app) 894 proxyApp := proxy.NewAppConns(clientCreator) 895 err := proxyApp.Start() 896 require.NoError(t, err) 897 defer proxyApp.Stop() 898 899 assert.Panics(t, func() { 900 h := NewHandshaker(stateDB, state, store, genDoc) 901 h.Handshake(proxyApp) 902 }) 903 } 904 } 905 906 func makeBlocks(n int, state *sm.State, privVal types.PrivValidator) []*types.Block { 907 blocks := make([]*types.Block, 0) 908 909 var ( 910 prevBlock *types.Block 911 prevBlockMeta *types.BlockMeta 912 ) 913 914 appHeight := byte(0x01) 915 for i := 0; i < n; i++ { 916 height := int64(i + 1) 917 918 block, parts := makeBlock(*state, prevBlock, prevBlockMeta, privVal, height) 919 blocks = append(blocks, block) 920 921 prevBlock = block 922 prevBlockMeta = types.NewBlockMeta(block, parts) 923 924 // update state 925 state.AppHash = []byte{appHeight} 926 appHeight++ 927 state.LastBlockHeight = height 928 } 929 930 return blocks 931 } 932 933 func makeBlock(state sm.State, lastBlock *types.Block, lastBlockMeta *types.BlockMeta, 934 privVal types.PrivValidator, height int64) (*types.Block, *types.PartSet) { 935 936 lastCommit := types.NewCommit(height-1, 0, types.BlockID{}, nil) 937 if height > 1 { 938 vote, _ := types.MakeVote( 939 lastBlock.Header.Height, 940 lastBlockMeta.BlockID, 941 state.Validators, 942 privVal, 943 lastBlock.Header.ChainID, 944 time.Now()) 945 lastCommit = types.NewCommit(vote.Height, vote.Round, 946 lastBlockMeta.BlockID, []types.CommitSig{vote.CommitSig()}) 947 } 948 949 return state.MakeBlock(height, []types.Tx{}, lastCommit, nil, state.Validators.GetProposer().Address) 950 } 951 952 type badApp struct { 953 abci.BaseApplication 954 numBlocks byte 955 height byte 956 allHashesAreWrong bool 957 onlyLastHashIsWrong bool 958 } 959 960 func (app *badApp) Commit() abci.ResponseCommit { 961 app.height++ 962 if app.onlyLastHashIsWrong { 963 if app.height == app.numBlocks { 964 return abci.ResponseCommit{Data: tmrand.Bytes(8)} 965 } 966 return abci.ResponseCommit{Data: []byte{app.height}} 967 } else if app.allHashesAreWrong { 968 return abci.ResponseCommit{Data: tmrand.Bytes(8)} 969 } 970 971 panic("either allHashesAreWrong or onlyLastHashIsWrong must be set") 972 } 973 974 //-------------------------- 975 // utils for making blocks 976 977 func makeBlockchainFromWAL(wal WAL) ([]*types.Block, []*types.Commit, error) { 978 var height int64 979 980 // Search for height marker 981 gr, found, err := wal.SearchForEndHeight(height, &WALSearchOptions{}) 982 if err != nil { 983 return nil, nil, err 984 } 985 if !found { 986 return nil, nil, fmt.Errorf("wal does not contain height %d", height) 987 } 988 defer gr.Close() // nolint: errcheck 989 990 // log.Notice("Build a blockchain by reading from the WAL") 991 992 var ( 993 blocks []*types.Block 994 commits []*types.Commit 995 thisBlockParts *types.PartSet 996 thisBlockCommit *types.Commit 997 ) 998 999 dec := NewWALDecoder(gr) 1000 for { 1001 msg, err := dec.Decode() 1002 if err == io.EOF { 1003 break 1004 } else if err != nil { 1005 return nil, nil, err 1006 } 1007 1008 piece := readPieceFromWAL(msg) 1009 if piece == nil { 1010 continue 1011 } 1012 1013 switch p := piece.(type) { 1014 case EndHeightMessage: 1015 // if its not the first one, we have a full block 1016 if thisBlockParts != nil { 1017 var block = new(types.Block) 1018 _, err = cdc.UnmarshalBinaryLengthPrefixedReader(thisBlockParts.GetReader(), block, 0) 1019 if err != nil { 1020 panic(err) 1021 } 1022 if block.Height != height+1 { 1023 panic(fmt.Sprintf("read bad block from wal. got height %d, expected %d", block.Height, height+1)) 1024 } 1025 commitHeight := thisBlockCommit.Height 1026 if commitHeight != height+1 { 1027 panic(fmt.Sprintf("commit doesnt match. got height %d, expected %d", commitHeight, height+1)) 1028 } 1029 blocks = append(blocks, block) 1030 commits = append(commits, thisBlockCommit) 1031 height++ 1032 } 1033 case *types.PartSetHeader: 1034 thisBlockParts = types.NewPartSetFromHeader(*p) 1035 case *types.Part: 1036 _, err := thisBlockParts.AddPart(p) 1037 if err != nil { 1038 return nil, nil, err 1039 } 1040 case *types.Vote: 1041 if p.Type == types.PrecommitType { 1042 thisBlockCommit = types.NewCommit(p.Height, p.Round, 1043 p.BlockID, []types.CommitSig{p.CommitSig()}) 1044 } 1045 } 1046 } 1047 // grab the last block too 1048 var block = new(types.Block) 1049 _, err = cdc.UnmarshalBinaryLengthPrefixedReader(thisBlockParts.GetReader(), block, 0) 1050 if err != nil { 1051 panic(err) 1052 } 1053 if block.Height != height+1 { 1054 panic(fmt.Sprintf("read bad block from wal. got height %d, expected %d", block.Height, height+1)) 1055 } 1056 commitHeight := thisBlockCommit.Height 1057 if commitHeight != height+1 { 1058 panic(fmt.Sprintf("commit doesnt match. got height %d, expected %d", commitHeight, height+1)) 1059 } 1060 blocks = append(blocks, block) 1061 commits = append(commits, thisBlockCommit) 1062 return blocks, commits, nil 1063 } 1064 1065 func readPieceFromWAL(msg *TimedWALMessage) interface{} { 1066 // for logging 1067 switch m := msg.Msg.(type) { 1068 case msgInfo: 1069 switch msg := m.Msg.(type) { 1070 case *ProposalMessage: 1071 return &msg.Proposal.BlockID.PartsHeader 1072 case *BlockPartMessage: 1073 return msg.Part 1074 case *VoteMessage: 1075 return msg.Vote 1076 } 1077 case EndHeightMessage: 1078 return m 1079 } 1080 1081 return nil 1082 } 1083 1084 // fresh state and mock store 1085 func stateAndStore( 1086 config *cfg.Config, 1087 pubKey crypto.PubKey, 1088 appVersion version.Protocol) (dbm.DB, sm.State, *mockBlockStore) { 1089 stateDB := dbm.NewMemDB() 1090 state, _ := sm.MakeGenesisStateFromFile(config.GenesisFile()) 1091 state.Version.Consensus.App = appVersion 1092 store := newMockBlockStore(config, state.ConsensusParams) 1093 sm.SaveState(stateDB, state) 1094 return stateDB, state, store 1095 } 1096 1097 //---------------------------------- 1098 // mock block store 1099 1100 type mockBlockStore struct { 1101 config *cfg.Config 1102 params types.ConsensusParams 1103 chain []*types.Block 1104 commits []*types.Commit 1105 base int64 1106 } 1107 1108 // TODO: NewBlockStore(db.NewMemDB) ... 1109 func newMockBlockStore(config *cfg.Config, params types.ConsensusParams) *mockBlockStore { 1110 return &mockBlockStore{config, params, nil, nil, 0} 1111 } 1112 1113 func (bs *mockBlockStore) Height() int64 { return int64(len(bs.chain)) } 1114 func (bs *mockBlockStore) Base() int64 { return bs.base } 1115 func (bs *mockBlockStore) Size() int64 { return bs.Height() - bs.Base() + 1 } 1116 func (bs *mockBlockStore) LoadBlock(height int64) *types.Block { return bs.chain[height-1] } 1117 func (bs *mockBlockStore) LoadBlockByHash(hash []byte) *types.Block { 1118 return bs.chain[int64(len(bs.chain))-1] 1119 } 1120 func (bs *mockBlockStore) LoadBlockMeta(height int64) *types.BlockMeta { 1121 block := bs.chain[height-1] 1122 return &types.BlockMeta{ 1123 BlockID: types.BlockID{Hash: block.Hash(), PartsHeader: block.MakePartSet(types.BlockPartSizeBytes).Header()}, 1124 Header: block.Header, 1125 } 1126 } 1127 func (bs *mockBlockStore) LoadBlockPart(height int64, index int) *types.Part { return nil } 1128 func (bs *mockBlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) { 1129 } 1130 func (bs *mockBlockStore) LoadBlockCommit(height int64) *types.Commit { 1131 return bs.commits[height-1] 1132 } 1133 func (bs *mockBlockStore) LoadSeenCommit(height int64) *types.Commit { 1134 return bs.commits[height-1] 1135 } 1136 1137 func (bs *mockBlockStore) PruneBlocks(height int64) (uint64, error) { 1138 pruned := uint64(0) 1139 for i := int64(0); i < height-1; i++ { 1140 bs.chain[i] = nil 1141 bs.commits[i] = nil 1142 pruned++ 1143 } 1144 bs.base = height 1145 return pruned, nil 1146 } 1147 1148 //--------------------------------------- 1149 // Test handshake/init chain 1150 1151 func TestHandshakeUpdatesValidators(t *testing.T) { 1152 val, _ := types.RandValidator(true, 10) 1153 vals := types.NewValidatorSet([]*types.Validator{val}) 1154 app := &initChainApp{vals: types.TM2PB.ValidatorUpdates(vals)} 1155 clientCreator := proxy.NewLocalClientCreator(app) 1156 1157 config := ResetConfig("handshake_test_") 1158 defer os.RemoveAll(config.RootDir) 1159 privVal := privval.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) 1160 pubKey, err := privVal.GetPubKey() 1161 require.NoError(t, err) 1162 stateDB, state, store := stateAndStore(config, pubKey, 0x0) 1163 1164 oldValAddr := state.Validators.Validators[0].Address 1165 1166 // now start the app using the handshake - it should sync 1167 genDoc, _ := sm.MakeGenesisDocFromFile(config.GenesisFile()) 1168 handshaker := NewHandshaker(stateDB, state, store, genDoc) 1169 proxyApp := proxy.NewAppConns(clientCreator) 1170 if err := proxyApp.Start(); err != nil { 1171 t.Fatalf("Error starting proxy app connections: %v", err) 1172 } 1173 defer proxyApp.Stop() 1174 if err := handshaker.Handshake(proxyApp); err != nil { 1175 t.Fatalf("Error on abci handshake: %v", err) 1176 } 1177 1178 // reload the state, check the validator set was updated 1179 state = sm.LoadState(stateDB) 1180 1181 newValAddr := state.Validators.Validators[0].Address 1182 expectValAddr := val.Address 1183 assert.NotEqual(t, oldValAddr, newValAddr) 1184 assert.Equal(t, newValAddr, expectValAddr) 1185 } 1186 1187 // returns the vals on InitChain 1188 type initChainApp struct { 1189 abci.BaseApplication 1190 vals []abci.ValidatorUpdate 1191 } 1192 1193 func (ica *initChainApp) InitChain(req abci.RequestInitChain) abci.ResponseInitChain { 1194 return abci.ResponseInitChain{ 1195 Validators: ica.vals, 1196 } 1197 }