github.com/noirx94/tendermintmp@v0.0.1/consensus/replay_test.go (about) 1 package consensus 2 3 import ( 4 "bytes" 5 "context" 6 "fmt" 7 "io" 8 "io/ioutil" 9 "os" 10 "path/filepath" 11 "runtime" 12 "sort" 13 "testing" 14 "time" 15 16 "github.com/gogo/protobuf/proto" 17 "github.com/stretchr/testify/assert" 18 "github.com/stretchr/testify/require" 19 dbm "github.com/tendermint/tm-db" 20 21 "github.com/tendermint/tendermint/abci/example/kvstore" 22 abci "github.com/tendermint/tendermint/abci/types" 23 cfg "github.com/tendermint/tendermint/config" 24 "github.com/tendermint/tendermint/crypto" 25 cryptoenc "github.com/tendermint/tendermint/crypto/encoding" 26 "github.com/tendermint/tendermint/libs/log" 27 tmrand "github.com/tendermint/tendermint/libs/rand" 28 mempl "github.com/tendermint/tendermint/mempool" 29 "github.com/tendermint/tendermint/privval" 30 tmstate "github.com/tendermint/tendermint/proto/tendermint/state" 31 tmproto "github.com/tendermint/tendermint/proto/tendermint/types" 32 "github.com/tendermint/tendermint/proxy" 33 sm "github.com/tendermint/tendermint/state" 34 "github.com/tendermint/tendermint/types" 35 ) 36 37 func TestMain(m *testing.M) { 38 config = ResetConfig("consensus_reactor_test") 39 consensusReplayConfig = ResetConfig("consensus_replay_test") 40 configStateTest := ResetConfig("consensus_state_test") 41 configMempoolTest := ResetConfig("consensus_mempool_test") 42 configByzantineTest := ResetConfig("consensus_byzantine_test") 43 code := m.Run() 44 os.RemoveAll(config.RootDir) 45 os.RemoveAll(consensusReplayConfig.RootDir) 46 os.RemoveAll(configStateTest.RootDir) 47 os.RemoveAll(configMempoolTest.RootDir) 48 os.RemoveAll(configByzantineTest.RootDir) 49 os.Exit(code) 50 } 51 52 // These tests ensure we can always recover from failure at any part of the consensus process. 53 // There are two general failure scenarios: failure during consensus, and failure while applying the block. 54 // Only the latter interacts with the app and store, 55 // but the former has to deal with restrictions on re-use of priv_validator keys. 56 // The `WAL Tests` are for failures during the consensus; 57 // the `Handshake Tests` are for failures in applying the block. 58 // With the help of the WAL, we can recover from it all! 59 60 //------------------------------------------------------------------------------------------ 61 // WAL Tests 62 63 // TODO: It would be better to verify explicitly which states we can recover from without the wal 64 // and which ones we need the wal for - then we'd also be able to only flush the 65 // wal writer when we need to, instead of with every message. 66 67 func startNewStateAndWaitForBlock(t *testing.T, consensusReplayConfig *cfg.Config, 68 lastBlockHeight int64, blockDB dbm.DB, stateStore sm.Store) { 69 logger := log.TestingLogger() 70 state, _ := stateStore.LoadFromDBOrGenesisFile(consensusReplayConfig.GenesisFile()) 71 privValidator := loadPrivValidator(consensusReplayConfig) 72 cs := newStateWithConfigAndBlockStore( 73 consensusReplayConfig, 74 state, 75 privValidator, 76 kvstore.NewApplication(), 77 blockDB, 78 ) 79 cs.SetLogger(logger) 80 81 bytes, _ := ioutil.ReadFile(cs.config.WalFile()) 82 t.Logf("====== WAL: \n\r%X\n", bytes) 83 84 err := cs.Start() 85 require.NoError(t, err) 86 defer func() { 87 if err := cs.Stop(); err != nil { 88 t.Error(err) 89 } 90 }() 91 92 // This is just a signal that we haven't halted; its not something contained 93 // in the WAL itself. Assuming the consensus state is running, replay of any 94 // WAL, including the empty one, should eventually be followed by a new 95 // block, or else something is wrong. 96 newBlockSub, err := cs.eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock) 97 require.NoError(t, err) 98 select { 99 case <-newBlockSub.Out(): 100 case <-newBlockSub.Cancelled(): 101 t.Fatal("newBlockSub was cancelled") 102 case <-time.After(120 * time.Second): 103 t.Fatal("Timed out waiting for new block (see trace above)") 104 } 105 } 106 107 func sendTxs(ctx context.Context, cs *State) { 108 for i := 0; i < 256; i++ { 109 select { 110 case <-ctx.Done(): 111 return 112 default: 113 tx := []byte{byte(i)} 114 if err := assertMempool(cs.txNotifier).CheckTx(tx, nil, mempl.TxInfo{}); err != nil { 115 panic(err) 116 } 117 i++ 118 } 119 } 120 } 121 122 // TestWALCrash uses crashing WAL to test we can recover from any WAL failure. 123 func TestWALCrash(t *testing.T) { 124 testCases := []struct { 125 name string 126 initFn func(dbm.DB, *State, context.Context) 127 heightToStop int64 128 }{ 129 {"empty block", 130 func(stateDB dbm.DB, cs *State, ctx context.Context) {}, 131 1}, 132 {"many non-empty blocks", 133 func(stateDB dbm.DB, cs *State, ctx context.Context) { 134 go sendTxs(ctx, cs) 135 }, 136 3}, 137 } 138 139 for i, tc := range testCases { 140 tc := tc 141 consensusReplayConfig := ResetConfig(fmt.Sprintf("%s_%d", t.Name(), i)) 142 t.Run(tc.name, func(t *testing.T) { 143 crashWALandCheckLiveness(t, consensusReplayConfig, tc.initFn, tc.heightToStop) 144 }) 145 } 146 } 147 148 func crashWALandCheckLiveness(t *testing.T, consensusReplayConfig *cfg.Config, 149 initFn func(dbm.DB, *State, context.Context), heightToStop int64) { 150 walPanicked := make(chan error) 151 crashingWal := &crashingWAL{panicCh: walPanicked, heightToStop: heightToStop} 152 153 i := 1 154 LOOP: 155 for { 156 t.Logf("====== LOOP %d\n", i) 157 158 // create consensus state from a clean slate 159 logger := log.NewNopLogger() 160 blockDB := dbm.NewMemDB() 161 stateDB := blockDB 162 stateStore := sm.NewStore(stateDB) 163 state, err := sm.MakeGenesisStateFromFile(consensusReplayConfig.GenesisFile()) 164 require.NoError(t, err) 165 privValidator := loadPrivValidator(consensusReplayConfig) 166 cs := newStateWithConfigAndBlockStore( 167 consensusReplayConfig, 168 state, 169 privValidator, 170 kvstore.NewApplication(), 171 blockDB, 172 ) 173 cs.SetLogger(logger) 174 175 // start sending transactions 176 ctx, cancel := context.WithCancel(context.Background()) 177 initFn(stateDB, cs, ctx) 178 179 // clean up WAL file from the previous iteration 180 walFile := cs.config.WalFile() 181 os.Remove(walFile) 182 183 // set crashing WAL 184 csWal, err := cs.OpenWAL(walFile) 185 require.NoError(t, err) 186 crashingWal.next = csWal 187 188 // reset the message counter 189 crashingWal.msgIndex = 1 190 cs.wal = crashingWal 191 192 // start consensus state 193 err = cs.Start() 194 require.NoError(t, err) 195 196 i++ 197 198 select { 199 case err := <-walPanicked: 200 t.Logf("WAL panicked: %v", err) 201 202 // make sure we can make blocks after a crash 203 startNewStateAndWaitForBlock(t, consensusReplayConfig, cs.Height, blockDB, stateStore) 204 205 // stop consensus state and transactions sender (initFn) 206 cs.Stop() //nolint:errcheck // Logging this error causes failure 207 cancel() 208 209 // if we reached the required height, exit 210 if _, ok := err.(ReachedHeightToStopError); ok { 211 break LOOP 212 } 213 case <-time.After(10 * time.Second): 214 t.Fatal("WAL did not panic for 10 seconds (check the log)") 215 } 216 } 217 } 218 219 // crashingWAL is a WAL which crashes or rather simulates a crash during Save 220 // (before and after). It remembers a message for which we last panicked 221 // (lastPanickedForMsgIndex), so we don't panic for it in subsequent iterations. 222 type crashingWAL struct { 223 next WAL 224 panicCh chan error 225 heightToStop int64 226 227 msgIndex int // current message index 228 lastPanickedForMsgIndex int // last message for which we panicked 229 } 230 231 var _ WAL = &crashingWAL{} 232 233 // WALWriteError indicates a WAL crash. 234 type WALWriteError struct { 235 msg string 236 } 237 238 func (e WALWriteError) Error() string { 239 return e.msg 240 } 241 242 // ReachedHeightToStopError indicates we've reached the required consensus 243 // height and may exit. 244 type ReachedHeightToStopError struct { 245 height int64 246 } 247 248 func (e ReachedHeightToStopError) Error() string { 249 return fmt.Sprintf("reached height to stop %d", e.height) 250 } 251 252 // Write simulate WAL's crashing by sending an error to the panicCh and then 253 // exiting the cs.receiveRoutine. 254 func (w *crashingWAL) Write(m WALMessage) error { 255 if endMsg, ok := m.(EndHeightMessage); ok { 256 if endMsg.Height == w.heightToStop { 257 w.panicCh <- ReachedHeightToStopError{endMsg.Height} 258 runtime.Goexit() 259 return nil 260 } 261 262 return w.next.Write(m) 263 } 264 265 if w.msgIndex > w.lastPanickedForMsgIndex { 266 w.lastPanickedForMsgIndex = w.msgIndex 267 _, file, line, _ := runtime.Caller(1) 268 w.panicCh <- WALWriteError{fmt.Sprintf("failed to write %T to WAL (fileline: %s:%d)", m, file, line)} 269 runtime.Goexit() 270 return nil 271 } 272 273 w.msgIndex++ 274 return w.next.Write(m) 275 } 276 277 func (w *crashingWAL) WriteSync(m WALMessage) error { 278 return w.Write(m) 279 } 280 281 func (w *crashingWAL) FlushAndSync() error { return w.next.FlushAndSync() } 282 283 func (w *crashingWAL) SearchForEndHeight( 284 height int64, 285 options *WALSearchOptions) (rd io.ReadCloser, found bool, err error) { 286 return w.next.SearchForEndHeight(height, options) 287 } 288 289 func (w *crashingWAL) Start() error { return w.next.Start() } 290 func (w *crashingWAL) Stop() error { return w.next.Stop() } 291 func (w *crashingWAL) Wait() { w.next.Wait() } 292 293 //------------------------------------------------------------------------------------------ 294 type testSim struct { 295 GenesisState sm.State 296 Config *cfg.Config 297 Chain []*types.Block 298 Commits []*types.Commit 299 CleanupFunc cleanupFunc 300 } 301 302 const ( 303 numBlocks = 6 304 ) 305 306 var ( 307 mempool = emptyMempool{} 308 evpool = sm.EmptyEvidencePool{} 309 310 sim testSim 311 ) 312 313 //--------------------------------------- 314 // Test handshake/replay 315 316 // 0 - all synced up 317 // 1 - saved block but app and state are behind 318 // 2 - save block and committed but state is behind 319 // 3 - save block and committed with truncated block store and state behind 320 var modes = []uint{0, 1, 2, 3} 321 322 // This is actually not a test, it's for storing validator change tx data for testHandshakeReplay 323 func TestSimulateValidatorsChange(t *testing.T) { 324 nPeers := 7 325 nVals := 4 326 css, genDoc, config, cleanup := randConsensusNetWithPeers( 327 nVals, 328 nPeers, 329 "replay_test", 330 newMockTickerFunc(true), 331 newPersistentKVStoreWithPath) 332 sim.Config = config 333 sim.GenesisState, _ = sm.MakeGenesisState(genDoc) 334 sim.CleanupFunc = cleanup 335 336 partSize := types.BlockPartSizeBytes 337 338 newRoundCh := subscribe(css[0].eventBus, types.EventQueryNewRound) 339 proposalCh := subscribe(css[0].eventBus, types.EventQueryCompleteProposal) 340 341 vss := make([]*validatorStub, nPeers) 342 for i := 0; i < nPeers; i++ { 343 vss[i] = newValidatorStub(css[i].privValidator, int32(i)) 344 } 345 height, round := css[0].Height, css[0].Round 346 347 // start the machine 348 startTestRound(css[0], height, round) 349 incrementHeight(vss...) 350 ensureNewRound(newRoundCh, height, 0) 351 ensureNewProposal(proposalCh, height, round) 352 rs := css[0].GetRoundState() 353 signAddVotes(css[0], tmproto.PrecommitType, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), vss[1:nVals]...) 354 ensureNewRound(newRoundCh, height+1, 0) 355 356 // HEIGHT 2 357 height++ 358 incrementHeight(vss...) 359 newValidatorPubKey1, err := css[nVals].privValidator.GetPubKey() 360 require.NoError(t, err) 361 valPubKey1ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey1) 362 require.NoError(t, err) 363 newValidatorTx1 := kvstore.MakeValSetChangeTx(valPubKey1ABCI, testMinPower) 364 err = assertMempool(css[0].txNotifier).CheckTx(newValidatorTx1, nil, mempl.TxInfo{}) 365 assert.Nil(t, err) 366 propBlock, _ := css[0].createProposalBlock() // changeProposer(t, cs1, vs2) 367 propBlockParts := propBlock.MakePartSet(partSize) 368 blockID := types.BlockID{Hash: propBlock.Hash(), PartSetHeader: propBlockParts.Header()} 369 370 proposal := types.NewProposal(vss[1].Height, round, -1, blockID) 371 p := proposal.ToProto() 372 if err := vss[1].SignProposal(config.ChainID(), p); err != nil { 373 t.Fatal("failed to sign bad proposal", err) 374 } 375 proposal.Signature = p.Signature 376 377 // set the proposal block 378 if err := css[0].SetProposalAndBlock(proposal, propBlock, propBlockParts, "some peer"); err != nil { 379 t.Fatal(err) 380 } 381 ensureNewProposal(proposalCh, height, round) 382 rs = css[0].GetRoundState() 383 signAddVotes(css[0], tmproto.PrecommitType, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), vss[1:nVals]...) 384 ensureNewRound(newRoundCh, height+1, 0) 385 386 // HEIGHT 3 387 height++ 388 incrementHeight(vss...) 389 updateValidatorPubKey1, err := css[nVals].privValidator.GetPubKey() 390 require.NoError(t, err) 391 updatePubKey1ABCI, err := cryptoenc.PubKeyToProto(updateValidatorPubKey1) 392 require.NoError(t, err) 393 updateValidatorTx1 := kvstore.MakeValSetChangeTx(updatePubKey1ABCI, 25) 394 err = assertMempool(css[0].txNotifier).CheckTx(updateValidatorTx1, nil, mempl.TxInfo{}) 395 assert.Nil(t, err) 396 propBlock, _ = css[0].createProposalBlock() // changeProposer(t, cs1, vs2) 397 propBlockParts = propBlock.MakePartSet(partSize) 398 blockID = types.BlockID{Hash: propBlock.Hash(), PartSetHeader: propBlockParts.Header()} 399 400 proposal = types.NewProposal(vss[2].Height, round, -1, blockID) 401 p = proposal.ToProto() 402 if err := vss[2].SignProposal(config.ChainID(), p); err != nil { 403 t.Fatal("failed to sign bad proposal", err) 404 } 405 proposal.Signature = p.Signature 406 407 // set the proposal block 408 if err := css[0].SetProposalAndBlock(proposal, propBlock, propBlockParts, "some peer"); err != nil { 409 t.Fatal(err) 410 } 411 ensureNewProposal(proposalCh, height, round) 412 rs = css[0].GetRoundState() 413 signAddVotes(css[0], tmproto.PrecommitType, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), vss[1:nVals]...) 414 ensureNewRound(newRoundCh, height+1, 0) 415 416 // HEIGHT 4 417 height++ 418 incrementHeight(vss...) 419 newValidatorPubKey2, err := css[nVals+1].privValidator.GetPubKey() 420 require.NoError(t, err) 421 newVal2ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey2) 422 require.NoError(t, err) 423 newValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, testMinPower) 424 err = assertMempool(css[0].txNotifier).CheckTx(newValidatorTx2, nil, mempl.TxInfo{}) 425 assert.Nil(t, err) 426 newValidatorPubKey3, err := css[nVals+2].privValidator.GetPubKey() 427 require.NoError(t, err) 428 newVal3ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey3) 429 require.NoError(t, err) 430 newValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, testMinPower) 431 err = assertMempool(css[0].txNotifier).CheckTx(newValidatorTx3, nil, mempl.TxInfo{}) 432 assert.Nil(t, err) 433 propBlock, _ = css[0].createProposalBlock() // changeProposer(t, cs1, vs2) 434 propBlockParts = propBlock.MakePartSet(partSize) 435 blockID = types.BlockID{Hash: propBlock.Hash(), PartSetHeader: propBlockParts.Header()} 436 newVss := make([]*validatorStub, nVals+1) 437 copy(newVss, vss[:nVals+1]) 438 sort.Sort(ValidatorStubsByPower(newVss)) 439 440 valIndexFn := func(cssIdx int) int { 441 for i, vs := range newVss { 442 vsPubKey, err := vs.GetPubKey() 443 require.NoError(t, err) 444 445 cssPubKey, err := css[cssIdx].privValidator.GetPubKey() 446 require.NoError(t, err) 447 448 if vsPubKey.Equals(cssPubKey) { 449 return i 450 } 451 } 452 panic(fmt.Sprintf("validator css[%d] not found in newVss", cssIdx)) 453 } 454 455 selfIndex := valIndexFn(0) 456 457 proposal = types.NewProposal(vss[3].Height, round, -1, blockID) 458 p = proposal.ToProto() 459 if err := vss[3].SignProposal(config.ChainID(), p); err != nil { 460 t.Fatal("failed to sign bad proposal", err) 461 } 462 proposal.Signature = p.Signature 463 464 // set the proposal block 465 if err := css[0].SetProposalAndBlock(proposal, propBlock, propBlockParts, "some peer"); err != nil { 466 t.Fatal(err) 467 } 468 ensureNewProposal(proposalCh, height, round) 469 470 removeValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, 0) 471 err = assertMempool(css[0].txNotifier).CheckTx(removeValidatorTx2, nil, mempl.TxInfo{}) 472 assert.Nil(t, err) 473 474 rs = css[0].GetRoundState() 475 for i := 0; i < nVals+1; i++ { 476 if i == selfIndex { 477 continue 478 } 479 signAddVotes(css[0], tmproto.PrecommitType, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), newVss[i]) 480 } 481 482 ensureNewRound(newRoundCh, height+1, 0) 483 484 // HEIGHT 5 485 height++ 486 incrementHeight(vss...) 487 // Reflect the changes to vss[nVals] at height 3 and resort newVss. 488 newVssIdx := valIndexFn(nVals) 489 newVss[newVssIdx].VotingPower = 25 490 sort.Sort(ValidatorStubsByPower(newVss)) 491 selfIndex = valIndexFn(0) 492 ensureNewProposal(proposalCh, height, round) 493 rs = css[0].GetRoundState() 494 for i := 0; i < nVals+1; i++ { 495 if i == selfIndex { 496 continue 497 } 498 signAddVotes(css[0], tmproto.PrecommitType, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), newVss[i]) 499 } 500 ensureNewRound(newRoundCh, height+1, 0) 501 502 // HEIGHT 6 503 height++ 504 incrementHeight(vss...) 505 removeValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, 0) 506 err = assertMempool(css[0].txNotifier).CheckTx(removeValidatorTx3, nil, mempl.TxInfo{}) 507 assert.Nil(t, err) 508 propBlock, _ = css[0].createProposalBlock() // changeProposer(t, cs1, vs2) 509 propBlockParts = propBlock.MakePartSet(partSize) 510 blockID = types.BlockID{Hash: propBlock.Hash(), PartSetHeader: propBlockParts.Header()} 511 newVss = make([]*validatorStub, nVals+3) 512 copy(newVss, vss[:nVals+3]) 513 sort.Sort(ValidatorStubsByPower(newVss)) 514 515 selfIndex = valIndexFn(0) 516 proposal = types.NewProposal(vss[1].Height, round, -1, blockID) 517 p = proposal.ToProto() 518 if err := vss[1].SignProposal(config.ChainID(), p); err != nil { 519 t.Fatal("failed to sign bad proposal", err) 520 } 521 proposal.Signature = p.Signature 522 523 // set the proposal block 524 if err := css[0].SetProposalAndBlock(proposal, propBlock, propBlockParts, "some peer"); err != nil { 525 t.Fatal(err) 526 } 527 ensureNewProposal(proposalCh, height, round) 528 rs = css[0].GetRoundState() 529 for i := 0; i < nVals+3; i++ { 530 if i == selfIndex { 531 continue 532 } 533 signAddVotes(css[0], tmproto.PrecommitType, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), newVss[i]) 534 } 535 ensureNewRound(newRoundCh, height+1, 0) 536 537 sim.Chain = make([]*types.Block, 0) 538 sim.Commits = make([]*types.Commit, 0) 539 for i := 1; i <= numBlocks; i++ { 540 sim.Chain = append(sim.Chain, css[0].blockStore.LoadBlock(int64(i))) 541 sim.Commits = append(sim.Commits, css[0].blockStore.LoadBlockCommit(int64(i))) 542 } 543 } 544 545 // Sync from scratch 546 func TestHandshakeReplayAll(t *testing.T) { 547 for _, m := range modes { 548 testHandshakeReplay(t, config, 0, m, false) 549 } 550 for _, m := range modes { 551 testHandshakeReplay(t, config, 0, m, true) 552 } 553 } 554 555 // Sync many, not from scratch 556 func TestHandshakeReplaySome(t *testing.T) { 557 for _, m := range modes { 558 testHandshakeReplay(t, config, 2, m, false) 559 } 560 for _, m := range modes { 561 testHandshakeReplay(t, config, 2, m, true) 562 } 563 } 564 565 // Sync from lagging by one 566 func TestHandshakeReplayOne(t *testing.T) { 567 for _, m := range modes { 568 testHandshakeReplay(t, config, numBlocks-1, m, false) 569 } 570 for _, m := range modes { 571 testHandshakeReplay(t, config, numBlocks-1, m, true) 572 } 573 } 574 575 // Sync from caught up 576 func TestHandshakeReplayNone(t *testing.T) { 577 for _, m := range modes { 578 testHandshakeReplay(t, config, numBlocks, m, false) 579 } 580 for _, m := range modes { 581 testHandshakeReplay(t, config, numBlocks, m, true) 582 } 583 } 584 585 // Test mockProxyApp should not panic when app return ABCIResponses with some empty ResponseDeliverTx 586 func TestMockProxyApp(t *testing.T) { 587 sim.CleanupFunc() // clean the test env created in TestSimulateValidatorsChange 588 logger := log.TestingLogger() 589 var validTxs, invalidTxs = 0, 0 590 txIndex := 0 591 592 assert.NotPanics(t, func() { 593 abciResWithEmptyDeliverTx := new(tmstate.ABCIResponses) 594 abciResWithEmptyDeliverTx.DeliverTxs = make([]*abci.ResponseDeliverTx, 0) 595 abciResWithEmptyDeliverTx.DeliverTxs = append(abciResWithEmptyDeliverTx.DeliverTxs, &abci.ResponseDeliverTx{}) 596 597 // called when saveABCIResponses: 598 bytes, err := proto.Marshal(abciResWithEmptyDeliverTx) 599 require.NoError(t, err) 600 loadedAbciRes := new(tmstate.ABCIResponses) 601 602 // this also happens sm.LoadABCIResponses 603 err = proto.Unmarshal(bytes, loadedAbciRes) 604 require.NoError(t, err) 605 606 mock := newMockProxyApp([]byte("mock_hash"), loadedAbciRes) 607 608 abciRes := new(tmstate.ABCIResponses) 609 abciRes.DeliverTxs = make([]*abci.ResponseDeliverTx, len(loadedAbciRes.DeliverTxs)) 610 // Execute transactions and get hash. 611 proxyCb := func(req *abci.Request, res *abci.Response) { 612 if r, ok := res.Value.(*abci.Response_DeliverTx); ok { 613 // TODO: make use of res.Log 614 // TODO: make use of this info 615 // Blocks may include invalid txs. 616 txRes := r.DeliverTx 617 if txRes.Code == abci.CodeTypeOK { 618 validTxs++ 619 } else { 620 logger.Debug("Invalid tx", "code", txRes.Code, "log", txRes.Log) 621 invalidTxs++ 622 } 623 abciRes.DeliverTxs[txIndex] = txRes 624 txIndex++ 625 } 626 } 627 mock.SetResponseCallback(proxyCb) 628 629 someTx := []byte("tx") 630 mock.DeliverTxAsync(abci.RequestDeliverTx{Tx: someTx}) 631 }) 632 assert.True(t, validTxs == 1) 633 assert.True(t, invalidTxs == 0) 634 } 635 636 func tempWALWithData(data []byte) string { 637 walFile, err := ioutil.TempFile("", "wal") 638 if err != nil { 639 panic(fmt.Sprintf("failed to create temp WAL file: %v", err)) 640 } 641 _, err = walFile.Write(data) 642 if err != nil { 643 panic(fmt.Sprintf("failed to write to temp WAL file: %v", err)) 644 } 645 if err := walFile.Close(); err != nil { 646 panic(fmt.Sprintf("failed to close temp WAL file: %v", err)) 647 } 648 return walFile.Name() 649 } 650 651 // Make some blocks. Start a fresh app and apply nBlocks blocks. 652 // Then restart the app and sync it up with the remaining blocks 653 func testHandshakeReplay(t *testing.T, config *cfg.Config, nBlocks int, mode uint, testValidatorsChange bool) { 654 var chain []*types.Block 655 var commits []*types.Commit 656 var store *mockBlockStore 657 var stateDB dbm.DB 658 var genesisState sm.State 659 if testValidatorsChange { 660 testConfig := ResetConfig(fmt.Sprintf("%s_%v_m", t.Name(), mode)) 661 defer os.RemoveAll(testConfig.RootDir) 662 stateDB = dbm.NewMemDB() 663 664 genesisState = sim.GenesisState 665 config = sim.Config 666 chain = append([]*types.Block{}, sim.Chain...) // copy chain 667 commits = sim.Commits 668 store = newMockBlockStore(config, genesisState.ConsensusParams) 669 } else { // test single node 670 testConfig := ResetConfig(fmt.Sprintf("%s_%v_s", t.Name(), mode)) 671 defer os.RemoveAll(testConfig.RootDir) 672 walBody, err := WALWithNBlocks(t, numBlocks) 673 require.NoError(t, err) 674 walFile := tempWALWithData(walBody) 675 config.Consensus.SetWalFile(walFile) 676 677 privVal := privval.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) 678 679 wal, err := NewWAL(walFile) 680 require.NoError(t, err) 681 wal.SetLogger(log.TestingLogger()) 682 err = wal.Start() 683 require.NoError(t, err) 684 t.Cleanup(func() { 685 if err := wal.Stop(); err != nil { 686 t.Error(err) 687 } 688 }) 689 chain, commits, err = makeBlockchainFromWAL(wal) 690 require.NoError(t, err) 691 pubKey, err := privVal.GetPubKey() 692 require.NoError(t, err) 693 stateDB, genesisState, store = stateAndStore(config, pubKey, kvstore.ProtocolVersion) 694 695 } 696 stateStore := sm.NewStore(stateDB) 697 store.chain = chain 698 store.commits = commits 699 700 state := genesisState.Copy() 701 // run the chain through state.ApplyBlock to build up the tendermint state 702 state = buildTMStateFromChain(config, stateStore, state, chain, nBlocks, mode) 703 latestAppHash := state.AppHash 704 705 // make a new client creator 706 kvstoreApp := kvstore.NewPersistentKVStoreApplication( 707 filepath.Join(config.DBDir(), fmt.Sprintf("replay_test_%d_%d_a", nBlocks, mode))) 708 709 clientCreator2 := proxy.NewLocalClientCreator(kvstoreApp) 710 if nBlocks > 0 { 711 // run nBlocks against a new client to build up the app state. 712 // use a throwaway tendermint state 713 proxyApp := proxy.NewAppConns(clientCreator2) 714 stateDB1 := dbm.NewMemDB() 715 stateStore := sm.NewStore(stateDB1) 716 err := stateStore.Save(genesisState) 717 require.NoError(t, err) 718 buildAppStateFromChain(proxyApp, stateStore, genesisState, chain, nBlocks, mode) 719 } 720 721 // Prune block store if requested 722 expectError := false 723 if mode == 3 { 724 pruned, err := store.PruneBlocks(2) 725 require.NoError(t, err) 726 require.EqualValues(t, 1, pruned) 727 expectError = int64(nBlocks) < 2 728 } 729 730 // now start the app using the handshake - it should sync 731 genDoc, _ := sm.MakeGenesisDocFromFile(config.GenesisFile()) 732 handshaker := NewHandshaker(stateStore, state, store, genDoc) 733 proxyApp := proxy.NewAppConns(clientCreator2) 734 if err := proxyApp.Start(); err != nil { 735 t.Fatalf("Error starting proxy app connections: %v", err) 736 } 737 738 t.Cleanup(func() { 739 if err := proxyApp.Stop(); err != nil { 740 t.Error(err) 741 } 742 }) 743 744 err := handshaker.Handshake(proxyApp) 745 if expectError { 746 require.Error(t, err) 747 return 748 } else if err != nil { 749 t.Fatalf("Error on abci handshake: %v", err) 750 } 751 752 // get the latest app hash from the app 753 res, err := proxyApp.Query().InfoSync(abci.RequestInfo{Version: ""}) 754 if err != nil { 755 t.Fatal(err) 756 } 757 758 // the app hash should be synced up 759 if !bytes.Equal(latestAppHash, res.LastBlockAppHash) { 760 t.Fatalf( 761 "Expected app hashes to match after handshake/replay. got %X, expected %X", 762 res.LastBlockAppHash, 763 latestAppHash) 764 } 765 766 expectedBlocksToSync := numBlocks - nBlocks 767 if nBlocks == numBlocks && mode > 0 { 768 expectedBlocksToSync++ 769 } else if nBlocks > 0 && mode == 1 { 770 expectedBlocksToSync++ 771 } 772 773 if handshaker.NBlocks() != expectedBlocksToSync { 774 t.Fatalf("Expected handshake to sync %d blocks, got %d", expectedBlocksToSync, handshaker.NBlocks()) 775 } 776 } 777 778 func applyBlock(stateStore sm.Store, st sm.State, blk *types.Block, proxyApp proxy.AppConns) sm.State { 779 testPartSize := types.BlockPartSizeBytes 780 blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool) 781 782 blkID := types.BlockID{Hash: blk.Hash(), PartSetHeader: blk.MakePartSet(testPartSize).Header()} 783 newState, _, err := blockExec.ApplyBlock(st, blkID, blk) 784 if err != nil { 785 panic(err) 786 } 787 return newState 788 } 789 790 func buildAppStateFromChain(proxyApp proxy.AppConns, stateStore sm.Store, 791 state sm.State, chain []*types.Block, nBlocks int, mode uint) { 792 // start a new app without handshake, play nBlocks blocks 793 if err := proxyApp.Start(); err != nil { 794 panic(err) 795 } 796 defer proxyApp.Stop() //nolint:errcheck // ignore 797 798 state.Version.Consensus.App = kvstore.ProtocolVersion // simulate handshake, receive app version 799 validators := types.TM2PB.ValidatorUpdates(state.Validators) 800 if _, err := proxyApp.Consensus().InitChainSync(abci.RequestInitChain{ 801 Validators: validators, 802 }); err != nil { 803 panic(err) 804 } 805 if err := stateStore.Save(state); err != nil { // save height 1's validatorsInfo 806 panic(err) 807 } 808 switch mode { 809 case 0: 810 for i := 0; i < nBlocks; i++ { 811 block := chain[i] 812 state = applyBlock(stateStore, state, block, proxyApp) 813 } 814 case 1, 2, 3: 815 for i := 0; i < nBlocks-1; i++ { 816 block := chain[i] 817 state = applyBlock(stateStore, state, block, proxyApp) 818 } 819 820 if mode == 2 || mode == 3 { 821 // update the kvstore height and apphash 822 // as if we ran commit but not 823 state = applyBlock(stateStore, state, chain[nBlocks-1], proxyApp) 824 } 825 default: 826 panic(fmt.Sprintf("unknown mode %v", mode)) 827 } 828 829 } 830 831 func buildTMStateFromChain( 832 config *cfg.Config, 833 stateStore sm.Store, 834 state sm.State, 835 chain []*types.Block, 836 nBlocks int, 837 mode uint) sm.State { 838 // run the whole chain against this client to build up the tendermint state 839 clientCreator := proxy.NewLocalClientCreator( 840 kvstore.NewPersistentKVStoreApplication( 841 filepath.Join(config.DBDir(), fmt.Sprintf("replay_test_%d_%d_t", nBlocks, mode)))) 842 proxyApp := proxy.NewAppConns(clientCreator) 843 if err := proxyApp.Start(); err != nil { 844 panic(err) 845 } 846 defer proxyApp.Stop() //nolint:errcheck 847 848 state.Version.Consensus.App = kvstore.ProtocolVersion // simulate handshake, receive app version 849 validators := types.TM2PB.ValidatorUpdates(state.Validators) 850 if _, err := proxyApp.Consensus().InitChainSync(abci.RequestInitChain{ 851 Validators: validators, 852 }); err != nil { 853 panic(err) 854 } 855 if err := stateStore.Save(state); err != nil { // save height 1's validatorsInfo 856 panic(err) 857 } 858 switch mode { 859 case 0: 860 // sync right up 861 for _, block := range chain { 862 state = applyBlock(stateStore, state, block, proxyApp) 863 } 864 865 case 1, 2, 3: 866 // sync up to the penultimate as if we stored the block. 867 // whether we commit or not depends on the appHash 868 for _, block := range chain[:len(chain)-1] { 869 state = applyBlock(stateStore, state, block, proxyApp) 870 } 871 872 // apply the final block to a state copy so we can 873 // get the right next appHash but keep the state back 874 applyBlock(stateStore, state, chain[len(chain)-1], proxyApp) 875 default: 876 panic(fmt.Sprintf("unknown mode %v", mode)) 877 } 878 879 return state 880 } 881 882 func TestHandshakePanicsIfAppReturnsWrongAppHash(t *testing.T) { 883 // 1. Initialize tendermint and commit 3 blocks with the following app hashes: 884 // - 0x01 885 // - 0x02 886 // - 0x03 887 config := ResetConfig("handshake_test_") 888 defer os.RemoveAll(config.RootDir) 889 privVal := privval.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) 890 const appVersion = 0x0 891 pubKey, err := privVal.GetPubKey() 892 require.NoError(t, err) 893 stateDB, state, store := stateAndStore(config, pubKey, appVersion) 894 stateStore := sm.NewStore(stateDB) 895 genDoc, _ := sm.MakeGenesisDocFromFile(config.GenesisFile()) 896 state.LastValidators = state.Validators.Copy() 897 // mode = 0 for committing all the blocks 898 blocks := makeBlocks(3, &state, privVal) 899 store.chain = blocks 900 901 // 2. Tendermint must panic if app returns wrong hash for the first block 902 // - RANDOM HASH 903 // - 0x02 904 // - 0x03 905 { 906 app := &badApp{numBlocks: 3, allHashesAreWrong: true} 907 clientCreator := proxy.NewLocalClientCreator(app) 908 proxyApp := proxy.NewAppConns(clientCreator) 909 err := proxyApp.Start() 910 require.NoError(t, err) 911 t.Cleanup(func() { 912 if err := proxyApp.Stop(); err != nil { 913 t.Error(err) 914 } 915 }) 916 917 assert.Panics(t, func() { 918 h := NewHandshaker(stateStore, state, store, genDoc) 919 if err = h.Handshake(proxyApp); err != nil { 920 t.Log(err) 921 } 922 }) 923 } 924 925 // 3. Tendermint must panic if app returns wrong hash for the last block 926 // - 0x01 927 // - 0x02 928 // - RANDOM HASH 929 { 930 app := &badApp{numBlocks: 3, onlyLastHashIsWrong: true} 931 clientCreator := proxy.NewLocalClientCreator(app) 932 proxyApp := proxy.NewAppConns(clientCreator) 933 err := proxyApp.Start() 934 require.NoError(t, err) 935 t.Cleanup(func() { 936 if err := proxyApp.Stop(); err != nil { 937 t.Error(err) 938 } 939 }) 940 941 assert.Panics(t, func() { 942 h := NewHandshaker(stateStore, state, store, genDoc) 943 if err = h.Handshake(proxyApp); err != nil { 944 t.Log(err) 945 } 946 }) 947 } 948 } 949 950 func makeBlocks(n int, state *sm.State, privVal types.PrivValidator) []*types.Block { 951 blocks := make([]*types.Block, 0) 952 953 var ( 954 prevBlock *types.Block 955 prevBlockMeta *types.BlockMeta 956 ) 957 958 appHeight := byte(0x01) 959 for i := 0; i < n; i++ { 960 height := int64(i + 1) 961 962 block, parts := makeBlock(*state, prevBlock, prevBlockMeta, privVal, height) 963 blocks = append(blocks, block) 964 965 prevBlock = block 966 prevBlockMeta = types.NewBlockMeta(block, parts) 967 968 // update state 969 state.AppHash = []byte{appHeight} 970 appHeight++ 971 state.LastBlockHeight = height 972 } 973 974 return blocks 975 } 976 977 func makeBlock(state sm.State, lastBlock *types.Block, lastBlockMeta *types.BlockMeta, 978 privVal types.PrivValidator, height int64) (*types.Block, *types.PartSet) { 979 980 lastCommit := types.NewCommit(height-1, 0, types.BlockID{}, nil) 981 if height > 1 { 982 vote, _ := types.MakeVote( 983 lastBlock.Header.Height, 984 lastBlockMeta.BlockID, 985 state.Validators, 986 privVal, 987 lastBlock.Header.ChainID, 988 time.Now()) 989 lastCommit = types.NewCommit(vote.Height, vote.Round, 990 lastBlockMeta.BlockID, []types.CommitSig{vote.CommitSig()}) 991 } 992 993 return state.MakeBlock(height, []types.Tx{}, lastCommit, nil, state.Validators.GetProposer().Address) 994 } 995 996 type badApp struct { 997 abci.BaseApplication 998 numBlocks byte 999 height byte 1000 allHashesAreWrong bool 1001 onlyLastHashIsWrong bool 1002 } 1003 1004 func (app *badApp) Commit() abci.ResponseCommit { 1005 app.height++ 1006 if app.onlyLastHashIsWrong { 1007 if app.height == app.numBlocks { 1008 return abci.ResponseCommit{Data: tmrand.Bytes(8)} 1009 } 1010 return abci.ResponseCommit{Data: []byte{app.height}} 1011 } else if app.allHashesAreWrong { 1012 return abci.ResponseCommit{Data: tmrand.Bytes(8)} 1013 } 1014 1015 panic("either allHashesAreWrong or onlyLastHashIsWrong must be set") 1016 } 1017 1018 //-------------------------- 1019 // utils for making blocks 1020 1021 func makeBlockchainFromWAL(wal WAL) ([]*types.Block, []*types.Commit, error) { 1022 var height int64 1023 1024 // Search for height marker 1025 gr, found, err := wal.SearchForEndHeight(height, &WALSearchOptions{}) 1026 if err != nil { 1027 return nil, nil, err 1028 } 1029 if !found { 1030 return nil, nil, fmt.Errorf("wal does not contain height %d", height) 1031 } 1032 defer gr.Close() 1033 1034 // log.Notice("Build a blockchain by reading from the WAL") 1035 1036 var ( 1037 blocks []*types.Block 1038 commits []*types.Commit 1039 thisBlockParts *types.PartSet 1040 thisBlockCommit *types.Commit 1041 ) 1042 1043 dec := NewWALDecoder(gr) 1044 for { 1045 msg, err := dec.Decode() 1046 if err == io.EOF { 1047 break 1048 } else if err != nil { 1049 return nil, nil, err 1050 } 1051 1052 piece := readPieceFromWAL(msg) 1053 if piece == nil { 1054 continue 1055 } 1056 1057 switch p := piece.(type) { 1058 case EndHeightMessage: 1059 // if its not the first one, we have a full block 1060 if thisBlockParts != nil { 1061 var pbb = new(tmproto.Block) 1062 bz, err := ioutil.ReadAll(thisBlockParts.GetReader()) 1063 if err != nil { 1064 panic(err) 1065 } 1066 err = proto.Unmarshal(bz, pbb) 1067 if err != nil { 1068 panic(err) 1069 } 1070 block, err := types.BlockFromProto(pbb) 1071 if err != nil { 1072 panic(err) 1073 } 1074 1075 if block.Height != height+1 { 1076 panic(fmt.Sprintf("read bad block from wal. got height %d, expected %d", block.Height, height+1)) 1077 } 1078 commitHeight := thisBlockCommit.Height 1079 if commitHeight != height+1 { 1080 panic(fmt.Sprintf("commit doesnt match. got height %d, expected %d", commitHeight, height+1)) 1081 } 1082 blocks = append(blocks, block) 1083 commits = append(commits, thisBlockCommit) 1084 height++ 1085 } 1086 case *types.PartSetHeader: 1087 thisBlockParts = types.NewPartSetFromHeader(*p) 1088 case *types.Part: 1089 _, err := thisBlockParts.AddPart(p) 1090 if err != nil { 1091 return nil, nil, err 1092 } 1093 case *types.Vote: 1094 if p.Type == tmproto.PrecommitType { 1095 thisBlockCommit = types.NewCommit(p.Height, p.Round, 1096 p.BlockID, []types.CommitSig{p.CommitSig()}) 1097 } 1098 } 1099 } 1100 // grab the last block too 1101 bz, err := ioutil.ReadAll(thisBlockParts.GetReader()) 1102 if err != nil { 1103 panic(err) 1104 } 1105 var pbb = new(tmproto.Block) 1106 err = proto.Unmarshal(bz, pbb) 1107 if err != nil { 1108 panic(err) 1109 } 1110 block, err := types.BlockFromProto(pbb) 1111 if err != nil { 1112 panic(err) 1113 } 1114 if block.Height != height+1 { 1115 panic(fmt.Sprintf("read bad block from wal. got height %d, expected %d", block.Height, height+1)) 1116 } 1117 commitHeight := thisBlockCommit.Height 1118 if commitHeight != height+1 { 1119 panic(fmt.Sprintf("commit doesnt match. got height %d, expected %d", commitHeight, height+1)) 1120 } 1121 blocks = append(blocks, block) 1122 commits = append(commits, thisBlockCommit) 1123 return blocks, commits, nil 1124 } 1125 1126 func readPieceFromWAL(msg *TimedWALMessage) interface{} { 1127 // for logging 1128 switch m := msg.Msg.(type) { 1129 case msgInfo: 1130 switch msg := m.Msg.(type) { 1131 case *ProposalMessage: 1132 return &msg.Proposal.BlockID.PartSetHeader 1133 case *BlockPartMessage: 1134 return msg.Part 1135 case *VoteMessage: 1136 return msg.Vote 1137 } 1138 case EndHeightMessage: 1139 return m 1140 } 1141 1142 return nil 1143 } 1144 1145 // fresh state and mock store 1146 func stateAndStore( 1147 config *cfg.Config, 1148 pubKey crypto.PubKey, 1149 appVersion uint64) (dbm.DB, sm.State, *mockBlockStore) { 1150 stateDB := dbm.NewMemDB() 1151 stateStore := sm.NewStore(stateDB) 1152 state, _ := sm.MakeGenesisStateFromFile(config.GenesisFile()) 1153 state.Version.Consensus.App = appVersion 1154 store := newMockBlockStore(config, state.ConsensusParams) 1155 if err := stateStore.Save(state); err != nil { 1156 panic(err) 1157 } 1158 return stateDB, state, store 1159 } 1160 1161 //---------------------------------- 1162 // mock block store 1163 1164 type mockBlockStore struct { 1165 config *cfg.Config 1166 params tmproto.ConsensusParams 1167 chain []*types.Block 1168 commits []*types.Commit 1169 base int64 1170 } 1171 1172 // TODO: NewBlockStore(db.NewMemDB) ... 1173 func newMockBlockStore(config *cfg.Config, params tmproto.ConsensusParams) *mockBlockStore { 1174 return &mockBlockStore{config, params, nil, nil, 0} 1175 } 1176 1177 func (bs *mockBlockStore) Height() int64 { return int64(len(bs.chain)) } 1178 func (bs *mockBlockStore) Base() int64 { return bs.base } 1179 func (bs *mockBlockStore) Size() int64 { return bs.Height() - bs.Base() + 1 } 1180 func (bs *mockBlockStore) LoadBaseMeta() *types.BlockMeta { return bs.LoadBlockMeta(bs.base) } 1181 func (bs *mockBlockStore) LoadBlock(height int64) *types.Block { return bs.chain[height-1] } 1182 func (bs *mockBlockStore) LoadBlockByHash(hash []byte) *types.Block { 1183 return bs.chain[int64(len(bs.chain))-1] 1184 } 1185 func (bs *mockBlockStore) LoadBlockMeta(height int64) *types.BlockMeta { 1186 block := bs.chain[height-1] 1187 return &types.BlockMeta{ 1188 BlockID: types.BlockID{Hash: block.Hash(), PartSetHeader: block.MakePartSet(types.BlockPartSizeBytes).Header()}, 1189 Header: block.Header, 1190 } 1191 } 1192 func (bs *mockBlockStore) LoadBlockPart(height int64, index int) *types.Part { return nil } 1193 func (bs *mockBlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) { 1194 } 1195 func (bs *mockBlockStore) LoadBlockCommit(height int64) *types.Commit { 1196 return bs.commits[height-1] 1197 } 1198 func (bs *mockBlockStore) LoadSeenCommit(height int64) *types.Commit { 1199 return bs.commits[height-1] 1200 } 1201 1202 func (bs *mockBlockStore) PruneBlocks(height int64) (uint64, error) { 1203 pruned := uint64(0) 1204 for i := int64(0); i < height-1; i++ { 1205 bs.chain[i] = nil 1206 bs.commits[i] = nil 1207 pruned++ 1208 } 1209 bs.base = height 1210 return pruned, nil 1211 } 1212 1213 //--------------------------------------- 1214 // Test handshake/init chain 1215 1216 func TestHandshakeUpdatesValidators(t *testing.T) { 1217 val, _ := types.RandValidator(true, 10) 1218 vals := types.NewValidatorSet([]*types.Validator{val}) 1219 app := &initChainApp{vals: types.TM2PB.ValidatorUpdates(vals)} 1220 clientCreator := proxy.NewLocalClientCreator(app) 1221 1222 config := ResetConfig("handshake_test_") 1223 defer os.RemoveAll(config.RootDir) 1224 privVal := privval.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) 1225 pubKey, err := privVal.GetPubKey() 1226 require.NoError(t, err) 1227 stateDB, state, store := stateAndStore(config, pubKey, 0x0) 1228 stateStore := sm.NewStore(stateDB) 1229 1230 oldValAddr := state.Validators.Validators[0].Address 1231 1232 // now start the app using the handshake - it should sync 1233 genDoc, _ := sm.MakeGenesisDocFromFile(config.GenesisFile()) 1234 handshaker := NewHandshaker(stateStore, state, store, genDoc) 1235 proxyApp := proxy.NewAppConns(clientCreator) 1236 if err := proxyApp.Start(); err != nil { 1237 t.Fatalf("Error starting proxy app connections: %v", err) 1238 } 1239 t.Cleanup(func() { 1240 if err := proxyApp.Stop(); err != nil { 1241 t.Error(err) 1242 } 1243 }) 1244 if err := handshaker.Handshake(proxyApp); err != nil { 1245 t.Fatalf("Error on abci handshake: %v", err) 1246 } 1247 // reload the state, check the validator set was updated 1248 state, err = stateStore.Load() 1249 require.NoError(t, err) 1250 1251 newValAddr := state.Validators.Validators[0].Address 1252 expectValAddr := val.Address 1253 assert.NotEqual(t, oldValAddr, newValAddr) 1254 assert.Equal(t, newValAddr, expectValAddr) 1255 } 1256 1257 // returns the vals on InitChain 1258 type initChainApp struct { 1259 abci.BaseApplication 1260 vals []abci.ValidatorUpdate 1261 } 1262 1263 func (ica *initChainApp) InitChain(req abci.RequestInitChain) abci.ResponseInitChain { 1264 return abci.ResponseInitChain{ 1265 Validators: ica.vals, 1266 } 1267 }