bitbucket.org/number571/tendermint@v0.8.14/internal/consensus/replay_test.go (about) 1 package consensus 2 3 import ( 4 "bytes" 5 "context" 6 "fmt" 7 "io" 8 "io/ioutil" 9 "math/rand" 10 "os" 11 "path/filepath" 12 "runtime" 13 "sort" 14 "testing" 15 "time" 16 17 "github.com/gogo/protobuf/proto" 18 "github.com/stretchr/testify/assert" 19 "github.com/stretchr/testify/require" 20 dbm "github.com/tendermint/tm-db" 21 22 "bitbucket.org/number571/tendermint/abci/example/kvstore" 23 abci "bitbucket.org/number571/tendermint/abci/types" 24 cfg "bitbucket.org/number571/tendermint/config" 25 "bitbucket.org/number571/tendermint/crypto" 26 cryptoenc "bitbucket.org/number571/tendermint/crypto/encoding" 27 mempl "bitbucket.org/number571/tendermint/internal/mempool" 28 "bitbucket.org/number571/tendermint/internal/test/factory" 29 "bitbucket.org/number571/tendermint/libs/log" 30 tmrand "bitbucket.org/number571/tendermint/libs/rand" 31 "bitbucket.org/number571/tendermint/privval" 32 tmstate "bitbucket.org/number571/tendermint/proto/tendermint/state" 33 tmproto "bitbucket.org/number571/tendermint/proto/tendermint/types" 34 "bitbucket.org/number571/tendermint/proxy" 35 sm "bitbucket.org/number571/tendermint/state" 36 sf "bitbucket.org/number571/tendermint/state/test/factory" 37 "bitbucket.org/number571/tendermint/store" 38 "bitbucket.org/number571/tendermint/types" 39 ) 40 41 // These tests ensure we can always recover from failure at any part of the consensus process. 42 // There are two general failure scenarios: failure during consensus, and failure while applying the block. 43 // Only the latter interacts with the app and store, 44 // but the former has to deal with restrictions on re-use of priv_validator keys. 45 // The `WAL Tests` are for failures during the consensus; 46 // the `Handshake Tests` are for failures in applying the block. 47 // With the help of the WAL, we can recover from it all! 48 49 //------------------------------------------------------------------------------------------ 50 // WAL Tests 51 52 // TODO: It would be better to verify explicitly which states we can recover from without the wal 53 // and which ones we need the wal for - then we'd also be able to only flush the 54 // wal writer when we need to, instead of with every message. 55 56 func startNewStateAndWaitForBlock(t *testing.T, consensusReplayConfig *cfg.Config, 57 lastBlockHeight int64, blockDB dbm.DB, stateStore sm.Store) { 58 logger := log.TestingLogger() 59 state, err := sm.MakeGenesisStateFromFile(consensusReplayConfig.GenesisFile()) 60 require.NoError(t, err) 61 privValidator := loadPrivValidator(consensusReplayConfig) 62 blockStore := store.NewBlockStore(dbm.NewMemDB()) 63 cs := newStateWithConfigAndBlockStore( 64 consensusReplayConfig, 65 state, 66 privValidator, 67 kvstore.NewApplication(), 68 blockStore, 69 ) 70 cs.SetLogger(logger) 71 72 bytes, _ := ioutil.ReadFile(cs.config.WalFile()) 73 t.Logf("====== WAL: \n\r%X\n", bytes) 74 75 err = cs.Start() 76 require.NoError(t, err) 77 defer func() { 78 if err := cs.Stop(); err != nil { 79 t.Error(err) 80 } 81 }() 82 83 // This is just a signal that we haven't halted; its not something contained 84 // in the WAL itself. Assuming the consensus state is running, replay of any 85 // WAL, including the empty one, should eventually be followed by a new 86 // block, or else something is wrong. 87 newBlockSub, err := cs.eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock) 88 require.NoError(t, err) 89 select { 90 case <-newBlockSub.Out(): 91 case <-newBlockSub.Canceled(): 92 t.Fatal("newBlockSub was canceled") 93 case <-time.After(120 * time.Second): 94 t.Fatal("Timed out waiting for new block (see trace above)") 95 } 96 } 97 98 func sendTxs(ctx context.Context, cs *State) { 99 for i := 0; i < 256; i++ { 100 select { 101 case <-ctx.Done(): 102 return 103 default: 104 tx := []byte{byte(i)} 105 if err := assertMempool(cs.txNotifier).CheckTx(context.Background(), tx, nil, mempl.TxInfo{}); err != nil { 106 panic(err) 107 } 108 i++ 109 } 110 } 111 } 112 113 // TestWALCrash uses crashing WAL to test we can recover from any WAL failure. 114 func TestWALCrash(t *testing.T) { 115 testCases := []struct { 116 name string 117 initFn func(dbm.DB, *State, context.Context) 118 heightToStop int64 119 }{ 120 {"empty block", 121 func(stateDB dbm.DB, cs *State, ctx context.Context) {}, 122 1}, 123 {"many non-empty blocks", 124 func(stateDB dbm.DB, cs *State, ctx context.Context) { 125 go sendTxs(ctx, cs) 126 }, 127 3}, 128 } 129 130 for _, tc := range testCases { 131 tc := tc 132 t.Run(tc.name, func(t *testing.T) { 133 consensusReplayConfig := ResetConfig(tc.name) 134 crashWALandCheckLiveness(t, consensusReplayConfig, tc.initFn, tc.heightToStop) 135 }) 136 } 137 } 138 139 func crashWALandCheckLiveness(t *testing.T, consensusReplayConfig *cfg.Config, 140 initFn func(dbm.DB, *State, context.Context), heightToStop int64) { 141 walPanicked := make(chan error) 142 crashingWal := &crashingWAL{panicCh: walPanicked, heightToStop: heightToStop} 143 144 i := 1 145 LOOP: 146 for { 147 t.Logf("====== LOOP %d\n", i) 148 149 // create consensus state from a clean slate 150 logger := log.NewNopLogger() 151 blockDB := dbm.NewMemDB() 152 stateDB := dbm.NewMemDB() 153 stateStore := sm.NewStore(stateDB) 154 blockStore := store.NewBlockStore(blockDB) 155 state, err := sm.MakeGenesisStateFromFile(consensusReplayConfig.GenesisFile()) 156 require.NoError(t, err) 157 privValidator := loadPrivValidator(consensusReplayConfig) 158 cs := newStateWithConfigAndBlockStore( 159 consensusReplayConfig, 160 state, 161 privValidator, 162 kvstore.NewApplication(), 163 blockStore, 164 ) 165 cs.SetLogger(logger) 166 167 // start sending transactions 168 ctx, cancel := context.WithCancel(context.Background()) 169 initFn(stateDB, cs, ctx) 170 171 // clean up WAL file from the previous iteration 172 walFile := cs.config.WalFile() 173 os.Remove(walFile) 174 175 // set crashing WAL 176 csWal, err := cs.OpenWAL(walFile) 177 require.NoError(t, err) 178 crashingWal.next = csWal 179 180 // reset the message counter 181 crashingWal.msgIndex = 1 182 cs.wal = crashingWal 183 184 // start consensus state 185 err = cs.Start() 186 require.NoError(t, err) 187 188 i++ 189 190 select { 191 case err := <-walPanicked: 192 t.Logf("WAL panicked: %v", err) 193 194 // make sure we can make blocks after a crash 195 startNewStateAndWaitForBlock(t, consensusReplayConfig, cs.Height, blockDB, stateStore) 196 197 // stop consensus state and transactions sender (initFn) 198 cs.Stop() //nolint:errcheck // Logging this error causes failure 199 cancel() 200 201 // if we reached the required height, exit 202 if _, ok := err.(ReachedHeightToStopError); ok { 203 break LOOP 204 } 205 case <-time.After(10 * time.Second): 206 t.Fatal("WAL did not panic for 10 seconds (check the log)") 207 } 208 } 209 } 210 211 // crashingWAL is a WAL which crashes or rather simulates a crash during Save 212 // (before and after). It remembers a message for which we last panicked 213 // (lastPanickedForMsgIndex), so we don't panic for it in subsequent iterations. 214 type crashingWAL struct { 215 next WAL 216 panicCh chan error 217 heightToStop int64 218 219 msgIndex int // current message index 220 lastPanickedForMsgIndex int // last message for which we panicked 221 } 222 223 var _ WAL = &crashingWAL{} 224 225 // WALWriteError indicates a WAL crash. 226 type WALWriteError struct { 227 msg string 228 } 229 230 func (e WALWriteError) Error() string { 231 return e.msg 232 } 233 234 // ReachedHeightToStopError indicates we've reached the required consensus 235 // height and may exit. 236 type ReachedHeightToStopError struct { 237 height int64 238 } 239 240 func (e ReachedHeightToStopError) Error() string { 241 return fmt.Sprintf("reached height to stop %d", e.height) 242 } 243 244 // Write simulate WAL's crashing by sending an error to the panicCh and then 245 // exiting the cs.receiveRoutine. 246 func (w *crashingWAL) Write(m WALMessage) error { 247 if endMsg, ok := m.(EndHeightMessage); ok { 248 if endMsg.Height == w.heightToStop { 249 w.panicCh <- ReachedHeightToStopError{endMsg.Height} 250 runtime.Goexit() 251 return nil 252 } 253 254 return w.next.Write(m) 255 } 256 257 if w.msgIndex > w.lastPanickedForMsgIndex { 258 w.lastPanickedForMsgIndex = w.msgIndex 259 _, file, line, _ := runtime.Caller(1) 260 w.panicCh <- WALWriteError{fmt.Sprintf("failed to write %T to WAL (fileline: %s:%d)", m, file, line)} 261 runtime.Goexit() 262 return nil 263 } 264 265 w.msgIndex++ 266 return w.next.Write(m) 267 } 268 269 func (w *crashingWAL) WriteSync(m WALMessage) error { 270 return w.Write(m) 271 } 272 273 func (w *crashingWAL) FlushAndSync() error { return w.next.FlushAndSync() } 274 275 func (w *crashingWAL) SearchForEndHeight( 276 height int64, 277 options *WALSearchOptions) (rd io.ReadCloser, found bool, err error) { 278 return w.next.SearchForEndHeight(height, options) 279 } 280 281 func (w *crashingWAL) Start() error { return w.next.Start() } 282 func (w *crashingWAL) Stop() error { return w.next.Stop() } 283 func (w *crashingWAL) Wait() { w.next.Wait() } 284 285 //------------------------------------------------------------------------------------------ 286 type simulatorTestSuite struct { 287 GenesisState sm.State 288 Config *cfg.Config 289 Chain []*types.Block 290 Commits []*types.Commit 291 CleanupFunc cleanupFunc 292 293 Mempool mempl.Mempool 294 Evpool sm.EvidencePool 295 } 296 297 const ( 298 numBlocks = 6 299 ) 300 301 //--------------------------------------- 302 // Test handshake/replay 303 304 // 0 - all synced up 305 // 1 - saved block but app and state are behind 306 // 2 - save block and committed but state is behind 307 // 3 - save block and committed with truncated block store and state behind 308 var modes = []uint{0, 1, 2, 3} 309 310 // This is actually not a test, it's for storing validator change tx data for testHandshakeReplay 311 func setupSimulator(t *testing.T) *simulatorTestSuite { 312 t.Helper() 313 config := configSetup(t) 314 315 sim := &simulatorTestSuite{ 316 Mempool: emptyMempool{}, 317 Evpool: sm.EmptyEvidencePool{}, 318 } 319 320 nPeers := 7 321 nVals := 4 322 323 css, genDoc, config, cleanup := randConsensusNetWithPeers( 324 config, 325 nVals, 326 nPeers, 327 "replay_test", 328 newMockTickerFunc(true), 329 newPersistentKVStoreWithPath) 330 sim.Config = config 331 sim.GenesisState, _ = sm.MakeGenesisState(genDoc) 332 sim.CleanupFunc = cleanup 333 334 partSize := types.BlockPartSizeBytes 335 336 newRoundCh := subscribe(css[0].eventBus, types.EventQueryNewRound) 337 proposalCh := subscribe(css[0].eventBus, types.EventQueryCompleteProposal) 338 339 vss := make([]*validatorStub, nPeers) 340 for i := 0; i < nPeers; i++ { 341 vss[i] = newValidatorStub(css[i].privValidator, int32(i)) 342 } 343 height, round := css[0].Height, css[0].Round 344 345 // start the machine 346 startTestRound(css[0], height, round) 347 incrementHeight(vss...) 348 ensureNewRound(newRoundCh, height, 0) 349 ensureNewProposal(proposalCh, height, round) 350 rs := css[0].GetRoundState() 351 352 signAddVotes(sim.Config, css[0], tmproto.PrecommitType, 353 rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), 354 vss[1:nVals]...) 355 356 ensureNewRound(newRoundCh, height+1, 0) 357 358 // HEIGHT 2 359 height++ 360 incrementHeight(vss...) 361 newValidatorPubKey1, err := css[nVals].privValidator.GetPubKey(context.Background()) 362 require.NoError(t, err) 363 valPubKey1ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey1) 364 require.NoError(t, err) 365 newValidatorTx1 := kvstore.MakeValSetChangeTx(valPubKey1ABCI, testMinPower) 366 err = assertMempool(css[0].txNotifier).CheckTx(context.Background(), newValidatorTx1, nil, mempl.TxInfo{}) 367 assert.Nil(t, err) 368 propBlock, _ := css[0].createProposalBlock() // changeProposer(t, cs1, vs2) 369 propBlockParts := propBlock.MakePartSet(partSize) 370 blockID := types.BlockID{Hash: propBlock.Hash(), PartSetHeader: propBlockParts.Header()} 371 372 proposal := types.NewProposal(vss[1].Height, round, -1, blockID) 373 p := proposal.ToProto() 374 if err := vss[1].SignProposal(context.Background(), config.ChainID(), p); err != nil { 375 t.Fatal("failed to sign bad proposal", err) 376 } 377 proposal.Signature = p.Signature 378 379 // set the proposal block 380 if err := css[0].SetProposalAndBlock(proposal, propBlock, propBlockParts, "some peer"); err != nil { 381 t.Fatal(err) 382 } 383 ensureNewProposal(proposalCh, height, round) 384 rs = css[0].GetRoundState() 385 signAddVotes(sim.Config, css[0], tmproto.PrecommitType, 386 rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), 387 vss[1:nVals]...) 388 ensureNewRound(newRoundCh, height+1, 0) 389 390 // HEIGHT 3 391 height++ 392 incrementHeight(vss...) 393 updateValidatorPubKey1, err := css[nVals].privValidator.GetPubKey(context.Background()) 394 require.NoError(t, err) 395 updatePubKey1ABCI, err := cryptoenc.PubKeyToProto(updateValidatorPubKey1) 396 require.NoError(t, err) 397 updateValidatorTx1 := kvstore.MakeValSetChangeTx(updatePubKey1ABCI, 25) 398 err = assertMempool(css[0].txNotifier).CheckTx(context.Background(), updateValidatorTx1, nil, mempl.TxInfo{}) 399 assert.Nil(t, err) 400 propBlock, _ = css[0].createProposalBlock() // changeProposer(t, cs1, vs2) 401 propBlockParts = propBlock.MakePartSet(partSize) 402 blockID = types.BlockID{Hash: propBlock.Hash(), PartSetHeader: propBlockParts.Header()} 403 404 proposal = types.NewProposal(vss[2].Height, round, -1, blockID) 405 p = proposal.ToProto() 406 if err := vss[2].SignProposal(context.Background(), config.ChainID(), p); err != nil { 407 t.Fatal("failed to sign bad proposal", err) 408 } 409 proposal.Signature = p.Signature 410 411 // set the proposal block 412 if err := css[0].SetProposalAndBlock(proposal, propBlock, propBlockParts, "some peer"); err != nil { 413 t.Fatal(err) 414 } 415 ensureNewProposal(proposalCh, height, round) 416 rs = css[0].GetRoundState() 417 signAddVotes(sim.Config, css[0], tmproto.PrecommitType, 418 rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), 419 vss[1:nVals]...) 420 ensureNewRound(newRoundCh, height+1, 0) 421 422 // HEIGHT 4 423 height++ 424 incrementHeight(vss...) 425 newValidatorPubKey2, err := css[nVals+1].privValidator.GetPubKey(context.Background()) 426 require.NoError(t, err) 427 newVal2ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey2) 428 require.NoError(t, err) 429 newValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, testMinPower) 430 err = assertMempool(css[0].txNotifier).CheckTx(context.Background(), newValidatorTx2, nil, mempl.TxInfo{}) 431 assert.Nil(t, err) 432 newValidatorPubKey3, err := css[nVals+2].privValidator.GetPubKey(context.Background()) 433 require.NoError(t, err) 434 newVal3ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey3) 435 require.NoError(t, err) 436 newValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, testMinPower) 437 err = assertMempool(css[0].txNotifier).CheckTx(context.Background(), newValidatorTx3, nil, mempl.TxInfo{}) 438 assert.Nil(t, err) 439 propBlock, _ = css[0].createProposalBlock() // changeProposer(t, cs1, vs2) 440 propBlockParts = propBlock.MakePartSet(partSize) 441 blockID = types.BlockID{Hash: propBlock.Hash(), PartSetHeader: propBlockParts.Header()} 442 newVss := make([]*validatorStub, nVals+1) 443 copy(newVss, vss[:nVals+1]) 444 sort.Sort(ValidatorStubsByPower(newVss)) 445 446 valIndexFn := func(cssIdx int) int { 447 for i, vs := range newVss { 448 vsPubKey, err := vs.GetPubKey(context.Background()) 449 require.NoError(t, err) 450 451 cssPubKey, err := css[cssIdx].privValidator.GetPubKey(context.Background()) 452 require.NoError(t, err) 453 454 if vsPubKey.Equals(cssPubKey) { 455 return i 456 } 457 } 458 panic(fmt.Sprintf("validator css[%d] not found in newVss", cssIdx)) 459 } 460 461 selfIndex := valIndexFn(0) 462 463 proposal = types.NewProposal(vss[3].Height, round, -1, blockID) 464 p = proposal.ToProto() 465 if err := vss[3].SignProposal(context.Background(), config.ChainID(), p); err != nil { 466 t.Fatal("failed to sign bad proposal", err) 467 } 468 proposal.Signature = p.Signature 469 470 // set the proposal block 471 if err := css[0].SetProposalAndBlock(proposal, propBlock, propBlockParts, "some peer"); err != nil { 472 t.Fatal(err) 473 } 474 ensureNewProposal(proposalCh, height, round) 475 476 removeValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, 0) 477 err = assertMempool(css[0].txNotifier).CheckTx(context.Background(), removeValidatorTx2, nil, mempl.TxInfo{}) 478 assert.Nil(t, err) 479 480 rs = css[0].GetRoundState() 481 for i := 0; i < nVals+1; i++ { 482 if i == selfIndex { 483 continue 484 } 485 signAddVotes(sim.Config, css[0], 486 tmproto.PrecommitType, rs.ProposalBlock.Hash(), 487 rs.ProposalBlockParts.Header(), newVss[i]) 488 } 489 490 ensureNewRound(newRoundCh, height+1, 0) 491 492 // HEIGHT 5 493 height++ 494 incrementHeight(vss...) 495 // Reflect the changes to vss[nVals] at height 3 and resort newVss. 496 newVssIdx := valIndexFn(nVals) 497 newVss[newVssIdx].VotingPower = 25 498 sort.Sort(ValidatorStubsByPower(newVss)) 499 selfIndex = valIndexFn(0) 500 ensureNewProposal(proposalCh, height, round) 501 rs = css[0].GetRoundState() 502 for i := 0; i < nVals+1; i++ { 503 if i == selfIndex { 504 continue 505 } 506 signAddVotes(sim.Config, css[0], 507 tmproto.PrecommitType, rs.ProposalBlock.Hash(), 508 rs.ProposalBlockParts.Header(), newVss[i]) 509 } 510 ensureNewRound(newRoundCh, height+1, 0) 511 512 // HEIGHT 6 513 height++ 514 incrementHeight(vss...) 515 removeValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, 0) 516 err = assertMempool(css[0].txNotifier).CheckTx(context.Background(), removeValidatorTx3, nil, mempl.TxInfo{}) 517 assert.Nil(t, err) 518 propBlock, _ = css[0].createProposalBlock() // changeProposer(t, cs1, vs2) 519 propBlockParts = propBlock.MakePartSet(partSize) 520 blockID = types.BlockID{Hash: propBlock.Hash(), PartSetHeader: propBlockParts.Header()} 521 newVss = make([]*validatorStub, nVals+3) 522 copy(newVss, vss[:nVals+3]) 523 sort.Sort(ValidatorStubsByPower(newVss)) 524 525 selfIndex = valIndexFn(0) 526 proposal = types.NewProposal(vss[1].Height, round, -1, blockID) 527 p = proposal.ToProto() 528 if err := vss[1].SignProposal(context.Background(), config.ChainID(), p); err != nil { 529 t.Fatal("failed to sign bad proposal", err) 530 } 531 proposal.Signature = p.Signature 532 533 // set the proposal block 534 if err := css[0].SetProposalAndBlock(proposal, propBlock, propBlockParts, "some peer"); err != nil { 535 t.Fatal(err) 536 } 537 ensureNewProposal(proposalCh, height, round) 538 rs = css[0].GetRoundState() 539 for i := 0; i < nVals+3; i++ { 540 if i == selfIndex { 541 continue 542 } 543 signAddVotes(sim.Config, css[0], 544 tmproto.PrecommitType, rs.ProposalBlock.Hash(), 545 rs.ProposalBlockParts.Header(), newVss[i]) 546 } 547 ensureNewRound(newRoundCh, height+1, 0) 548 549 sim.Chain = make([]*types.Block, 0) 550 sim.Commits = make([]*types.Commit, 0) 551 for i := 1; i <= numBlocks; i++ { 552 sim.Chain = append(sim.Chain, css[0].blockStore.LoadBlock(int64(i))) 553 sim.Commits = append(sim.Commits, css[0].blockStore.LoadBlockCommit(int64(i))) 554 } 555 if sim.CleanupFunc != nil { 556 t.Cleanup(sim.CleanupFunc) 557 } 558 559 return sim 560 } 561 562 // Sync from scratch 563 func TestHandshakeReplayAll(t *testing.T) { 564 sim := setupSimulator(t) 565 566 for _, m := range modes { 567 testHandshakeReplay(t, sim, 0, m, false) 568 } 569 for _, m := range modes { 570 testHandshakeReplay(t, sim, 0, m, true) 571 } 572 } 573 574 // Sync many, not from scratch 575 func TestHandshakeReplaySome(t *testing.T) { 576 sim := setupSimulator(t) 577 578 for _, m := range modes { 579 testHandshakeReplay(t, sim, 2, m, false) 580 } 581 for _, m := range modes { 582 testHandshakeReplay(t, sim, 2, m, true) 583 } 584 } 585 586 // Sync from lagging by one 587 func TestHandshakeReplayOne(t *testing.T) { 588 sim := setupSimulator(t) 589 590 for _, m := range modes { 591 testHandshakeReplay(t, sim, numBlocks-1, m, false) 592 } 593 for _, m := range modes { 594 testHandshakeReplay(t, sim, numBlocks-1, m, true) 595 } 596 } 597 598 // Sync from caught up 599 func TestHandshakeReplayNone(t *testing.T) { 600 sim := setupSimulator(t) 601 602 for _, m := range modes { 603 testHandshakeReplay(t, sim, numBlocks, m, false) 604 } 605 for _, m := range modes { 606 testHandshakeReplay(t, sim, numBlocks, m, true) 607 } 608 } 609 610 // Test mockProxyApp should not panic when app return ABCIResponses with some empty ResponseDeliverTx 611 func TestMockProxyApp(t *testing.T) { 612 sim := setupSimulator(t) // setup config and simulator 613 config := sim.Config 614 assert.NotNil(t, config) 615 616 logger := log.TestingLogger() 617 var validTxs, invalidTxs = 0, 0 618 txIndex := 0 619 620 assert.NotPanics(t, func() { 621 abciResWithEmptyDeliverTx := new(tmstate.ABCIResponses) 622 abciResWithEmptyDeliverTx.DeliverTxs = make([]*abci.ResponseDeliverTx, 0) 623 abciResWithEmptyDeliverTx.DeliverTxs = append(abciResWithEmptyDeliverTx.DeliverTxs, &abci.ResponseDeliverTx{}) 624 625 // called when saveABCIResponses: 626 bytes, err := proto.Marshal(abciResWithEmptyDeliverTx) 627 require.NoError(t, err) 628 loadedAbciRes := new(tmstate.ABCIResponses) 629 630 // this also happens sm.LoadABCIResponses 631 err = proto.Unmarshal(bytes, loadedAbciRes) 632 require.NoError(t, err) 633 634 mock := newMockProxyApp([]byte("mock_hash"), loadedAbciRes) 635 636 abciRes := new(tmstate.ABCIResponses) 637 abciRes.DeliverTxs = make([]*abci.ResponseDeliverTx, len(loadedAbciRes.DeliverTxs)) 638 // Execute transactions and get hash. 639 proxyCb := func(req *abci.Request, res *abci.Response) { 640 if r, ok := res.Value.(*abci.Response_DeliverTx); ok { 641 // TODO: make use of res.Log 642 // TODO: make use of this info 643 // Blocks may include invalid txs. 644 txRes := r.DeliverTx 645 if txRes.Code == abci.CodeTypeOK { 646 validTxs++ 647 } else { 648 logger.Debug("Invalid tx", "code", txRes.Code, "log", txRes.Log) 649 invalidTxs++ 650 } 651 abciRes.DeliverTxs[txIndex] = txRes 652 txIndex++ 653 } 654 } 655 mock.SetResponseCallback(proxyCb) 656 657 someTx := []byte("tx") 658 _, err = mock.DeliverTxAsync(context.Background(), abci.RequestDeliverTx{Tx: someTx}) 659 assert.NoError(t, err) 660 }) 661 assert.True(t, validTxs == 1) 662 assert.True(t, invalidTxs == 0) 663 } 664 665 func tempWALWithData(data []byte) string { 666 walFile, err := ioutil.TempFile("", "wal") 667 if err != nil { 668 panic(fmt.Sprintf("failed to create temp WAL file: %v", err)) 669 } 670 _, err = walFile.Write(data) 671 if err != nil { 672 panic(fmt.Sprintf("failed to write to temp WAL file: %v", err)) 673 } 674 if err := walFile.Close(); err != nil { 675 panic(fmt.Sprintf("failed to close temp WAL file: %v", err)) 676 } 677 return walFile.Name() 678 } 679 680 // Make some blocks. Start a fresh app and apply nBlocks blocks. 681 // Then restart the app and sync it up with the remaining blocks 682 func testHandshakeReplay(t *testing.T, sim *simulatorTestSuite, nBlocks int, mode uint, testValidatorsChange bool) { 683 var chain []*types.Block 684 var commits []*types.Commit 685 var store *mockBlockStore 686 var stateDB dbm.DB 687 var genesisState sm.State 688 689 config := sim.Config 690 691 if testValidatorsChange { 692 testConfig := ResetConfig(fmt.Sprintf("%s_%v_m", t.Name(), mode)) 693 defer func() { _ = os.RemoveAll(testConfig.RootDir) }() 694 stateDB = dbm.NewMemDB() 695 696 genesisState = sim.GenesisState 697 config = sim.Config 698 chain = append([]*types.Block{}, sim.Chain...) // copy chain 699 commits = sim.Commits 700 store = newMockBlockStore(config, genesisState.ConsensusParams) 701 } else { // test single node 702 testConfig := ResetConfig(fmt.Sprintf("%s_%v_s", t.Name(), mode)) 703 defer func() { _ = os.RemoveAll(testConfig.RootDir) }() 704 walBody, err := WALWithNBlocks(t, numBlocks) 705 require.NoError(t, err) 706 walFile := tempWALWithData(walBody) 707 config.Consensus.SetWalFile(walFile) 708 709 privVal, err := privval.LoadFilePV(config.PrivValidator.KeyFile(), config.PrivValidator.StateFile()) 710 require.NoError(t, err) 711 712 wal, err := NewWAL(walFile) 713 require.NoError(t, err) 714 wal.SetLogger(log.TestingLogger()) 715 err = wal.Start() 716 require.NoError(t, err) 717 t.Cleanup(func() { 718 if err := wal.Stop(); err != nil { 719 t.Error(err) 720 } 721 }) 722 chain, commits, err = makeBlockchainFromWAL(wal) 723 require.NoError(t, err) 724 pubKey, err := privVal.GetPubKey(context.Background()) 725 require.NoError(t, err) 726 stateDB, genesisState, store = stateAndStore(config, pubKey, kvstore.ProtocolVersion) 727 728 } 729 stateStore := sm.NewStore(stateDB) 730 store.chain = chain 731 store.commits = commits 732 733 state := genesisState.Copy() 734 // run the chain through state.ApplyBlock to build up the tendermint state 735 state = buildTMStateFromChain(config, sim.Mempool, sim.Evpool, stateStore, state, chain, nBlocks, mode, store) 736 latestAppHash := state.AppHash 737 738 // make a new client creator 739 kvstoreApp := kvstore.NewPersistentKVStoreApplication( 740 filepath.Join(config.DBDir(), fmt.Sprintf("replay_test_%d_%d_a_r%d", nBlocks, mode, rand.Int()))) 741 t.Cleanup(func() { require.NoError(t, kvstoreApp.Close()) }) 742 743 clientCreator2 := proxy.NewLocalClientCreator(kvstoreApp) 744 if nBlocks > 0 { 745 // run nBlocks against a new client to build up the app state. 746 // use a throwaway tendermint state 747 proxyApp := proxy.NewAppConns(clientCreator2) 748 stateDB1 := dbm.NewMemDB() 749 stateStore := sm.NewStore(stateDB1) 750 err := stateStore.Save(genesisState) 751 require.NoError(t, err) 752 buildAppStateFromChain(proxyApp, stateStore, sim.Mempool, sim.Evpool, genesisState, chain, nBlocks, mode, store) 753 } 754 755 // Prune block store if requested 756 expectError := false 757 if mode == 3 { 758 pruned, err := store.PruneBlocks(2) 759 require.NoError(t, err) 760 require.EqualValues(t, 1, pruned) 761 expectError = int64(nBlocks) < 2 762 } 763 764 // now start the app using the handshake - it should sync 765 genDoc, _ := sm.MakeGenesisDocFromFile(config.GenesisFile()) 766 handshaker := NewHandshaker(stateStore, state, store, genDoc) 767 proxyApp := proxy.NewAppConns(clientCreator2) 768 if err := proxyApp.Start(); err != nil { 769 t.Fatalf("Error starting proxy app connections: %v", err) 770 } 771 772 t.Cleanup(func() { 773 if err := proxyApp.Stop(); err != nil { 774 t.Error(err) 775 } 776 }) 777 778 err := handshaker.Handshake(proxyApp) 779 if expectError { 780 require.Error(t, err) 781 return 782 } else if err != nil { 783 t.Fatalf("Error on abci handshake: %v", err) 784 } 785 786 // get the latest app hash from the app 787 res, err := proxyApp.Query().InfoSync(context.Background(), abci.RequestInfo{Version: ""}) 788 if err != nil { 789 t.Fatal(err) 790 } 791 792 // the app hash should be synced up 793 if !bytes.Equal(latestAppHash, res.LastBlockAppHash) { 794 t.Fatalf( 795 "Expected app hashes to match after handshake/replay. got %X, expected %X", 796 res.LastBlockAppHash, 797 latestAppHash) 798 } 799 800 expectedBlocksToSync := numBlocks - nBlocks 801 if nBlocks == numBlocks && mode > 0 { 802 expectedBlocksToSync++ 803 } else if nBlocks > 0 && mode == 1 { 804 expectedBlocksToSync++ 805 } 806 807 if handshaker.NBlocks() != expectedBlocksToSync { 808 t.Fatalf("Expected handshake to sync %d blocks, got %d", expectedBlocksToSync, handshaker.NBlocks()) 809 } 810 } 811 812 func applyBlock(stateStore sm.Store, 813 mempool mempl.Mempool, 814 evpool sm.EvidencePool, 815 st sm.State, 816 blk *types.Block, 817 proxyApp proxy.AppConns, 818 blockStore *mockBlockStore) sm.State { 819 testPartSize := types.BlockPartSizeBytes 820 blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool, blockStore) 821 822 blkID := types.BlockID{Hash: blk.Hash(), PartSetHeader: blk.MakePartSet(testPartSize).Header()} 823 newState, err := blockExec.ApplyBlock(st, blkID, blk) 824 if err != nil { 825 panic(err) 826 } 827 return newState 828 } 829 830 func buildAppStateFromChain( 831 proxyApp proxy.AppConns, 832 stateStore sm.Store, 833 mempool mempl.Mempool, 834 evpool sm.EvidencePool, 835 state sm.State, 836 chain []*types.Block, 837 nBlocks int, 838 mode uint, 839 blockStore *mockBlockStore) { 840 // start a new app without handshake, play nBlocks blocks 841 if err := proxyApp.Start(); err != nil { 842 panic(err) 843 } 844 defer proxyApp.Stop() //nolint:errcheck // ignore 845 846 state.Version.Consensus.App = kvstore.ProtocolVersion // simulate handshake, receive app version 847 validators := types.TM2PB.ValidatorUpdates(state.Validators) 848 if _, err := proxyApp.Consensus().InitChainSync(context.Background(), abci.RequestInitChain{ 849 Validators: validators, 850 }); err != nil { 851 panic(err) 852 } 853 if err := stateStore.Save(state); err != nil { // save height 1's validatorsInfo 854 panic(err) 855 } 856 switch mode { 857 case 0: 858 for i := 0; i < nBlocks; i++ { 859 block := chain[i] 860 state = applyBlock(stateStore, mempool, evpool, state, block, proxyApp, blockStore) 861 } 862 case 1, 2, 3: 863 for i := 0; i < nBlocks-1; i++ { 864 block := chain[i] 865 state = applyBlock(stateStore, mempool, evpool, state, block, proxyApp, blockStore) 866 } 867 868 if mode == 2 || mode == 3 { 869 // update the kvstore height and apphash 870 // as if we ran commit but not 871 state = applyBlock(stateStore, mempool, evpool, state, chain[nBlocks-1], proxyApp, blockStore) 872 } 873 default: 874 panic(fmt.Sprintf("unknown mode %v", mode)) 875 } 876 877 } 878 879 func buildTMStateFromChain( 880 config *cfg.Config, 881 mempool mempl.Mempool, 882 evpool sm.EvidencePool, 883 stateStore sm.Store, 884 state sm.State, 885 chain []*types.Block, 886 nBlocks int, 887 mode uint, 888 blockStore *mockBlockStore) sm.State { 889 // run the whole chain against this client to build up the tendermint state 890 kvstoreApp := kvstore.NewPersistentKVStoreApplication( 891 filepath.Join(config.DBDir(), fmt.Sprintf("replay_test_%d_%d_t", nBlocks, mode))) 892 defer kvstoreApp.Close() 893 clientCreator := proxy.NewLocalClientCreator(kvstoreApp) 894 895 proxyApp := proxy.NewAppConns(clientCreator) 896 if err := proxyApp.Start(); err != nil { 897 panic(err) 898 } 899 defer proxyApp.Stop() //nolint:errcheck 900 901 state.Version.Consensus.App = kvstore.ProtocolVersion // simulate handshake, receive app version 902 validators := types.TM2PB.ValidatorUpdates(state.Validators) 903 if _, err := proxyApp.Consensus().InitChainSync(context.Background(), abci.RequestInitChain{ 904 Validators: validators, 905 }); err != nil { 906 panic(err) 907 } 908 if err := stateStore.Save(state); err != nil { // save height 1's validatorsInfo 909 panic(err) 910 } 911 switch mode { 912 case 0: 913 // sync right up 914 for _, block := range chain { 915 state = applyBlock(stateStore, mempool, evpool, state, block, proxyApp, blockStore) 916 } 917 918 case 1, 2, 3: 919 // sync up to the penultimate as if we stored the block. 920 // whether we commit or not depends on the appHash 921 for _, block := range chain[:len(chain)-1] { 922 state = applyBlock(stateStore, mempool, evpool, state, block, proxyApp, blockStore) 923 } 924 925 // apply the final block to a state copy so we can 926 // get the right next appHash but keep the state back 927 applyBlock(stateStore, mempool, evpool, state, chain[len(chain)-1], proxyApp, blockStore) 928 default: 929 panic(fmt.Sprintf("unknown mode %v", mode)) 930 } 931 932 return state 933 } 934 935 func TestHandshakePanicsIfAppReturnsWrongAppHash(t *testing.T) { 936 // 1. Initialize tendermint and commit 3 blocks with the following app hashes: 937 // - 0x01 938 // - 0x02 939 // - 0x03 940 config := ResetConfig("handshake_test_") 941 t.Cleanup(func() { os.RemoveAll(config.RootDir) }) 942 privVal, err := privval.LoadFilePV(config.PrivValidator.KeyFile(), config.PrivValidator.StateFile()) 943 require.NoError(t, err) 944 const appVersion = 0x0 945 pubKey, err := privVal.GetPubKey(context.Background()) 946 require.NoError(t, err) 947 stateDB, state, store := stateAndStore(config, pubKey, appVersion) 948 stateStore := sm.NewStore(stateDB) 949 genDoc, _ := sm.MakeGenesisDocFromFile(config.GenesisFile()) 950 state.LastValidators = state.Validators.Copy() 951 // mode = 0 for committing all the blocks 952 blocks := sf.MakeBlocks(3, &state, privVal) 953 store.chain = blocks 954 955 // 2. Tendermint must panic if app returns wrong hash for the first block 956 // - RANDOM HASH 957 // - 0x02 958 // - 0x03 959 { 960 app := &badApp{numBlocks: 3, allHashesAreWrong: true} 961 clientCreator := proxy.NewLocalClientCreator(app) 962 proxyApp := proxy.NewAppConns(clientCreator) 963 err := proxyApp.Start() 964 require.NoError(t, err) 965 t.Cleanup(func() { 966 if err := proxyApp.Stop(); err != nil { 967 t.Error(err) 968 } 969 }) 970 971 assert.Panics(t, func() { 972 h := NewHandshaker(stateStore, state, store, genDoc) 973 if err = h.Handshake(proxyApp); err != nil { 974 t.Log(err) 975 } 976 }) 977 } 978 979 // 3. Tendermint must panic if app returns wrong hash for the last block 980 // - 0x01 981 // - 0x02 982 // - RANDOM HASH 983 { 984 app := &badApp{numBlocks: 3, onlyLastHashIsWrong: true} 985 clientCreator := proxy.NewLocalClientCreator(app) 986 proxyApp := proxy.NewAppConns(clientCreator) 987 err := proxyApp.Start() 988 require.NoError(t, err) 989 t.Cleanup(func() { 990 if err := proxyApp.Stop(); err != nil { 991 t.Error(err) 992 } 993 }) 994 995 assert.Panics(t, func() { 996 h := NewHandshaker(stateStore, state, store, genDoc) 997 if err = h.Handshake(proxyApp); err != nil { 998 t.Log(err) 999 } 1000 }) 1001 } 1002 } 1003 1004 type badApp struct { 1005 abci.BaseApplication 1006 numBlocks byte 1007 height byte 1008 allHashesAreWrong bool 1009 onlyLastHashIsWrong bool 1010 } 1011 1012 func (app *badApp) Commit() abci.ResponseCommit { 1013 app.height++ 1014 if app.onlyLastHashIsWrong { 1015 if app.height == app.numBlocks { 1016 return abci.ResponseCommit{Data: tmrand.Bytes(8)} 1017 } 1018 return abci.ResponseCommit{Data: []byte{app.height}} 1019 } else if app.allHashesAreWrong { 1020 return abci.ResponseCommit{Data: tmrand.Bytes(8)} 1021 } 1022 1023 panic("either allHashesAreWrong or onlyLastHashIsWrong must be set") 1024 } 1025 1026 //-------------------------- 1027 // utils for making blocks 1028 1029 func makeBlockchainFromWAL(wal WAL) ([]*types.Block, []*types.Commit, error) { 1030 var height int64 1031 1032 // Search for height marker 1033 gr, found, err := wal.SearchForEndHeight(height, &WALSearchOptions{}) 1034 if err != nil { 1035 return nil, nil, err 1036 } 1037 if !found { 1038 return nil, nil, fmt.Errorf("wal does not contain height %d", height) 1039 } 1040 defer gr.Close() 1041 1042 // log.Notice("Build a blockchain by reading from the WAL") 1043 1044 var ( 1045 blocks []*types.Block 1046 commits []*types.Commit 1047 thisBlockParts *types.PartSet 1048 thisBlockCommit *types.Commit 1049 ) 1050 1051 dec := NewWALDecoder(gr) 1052 for { 1053 msg, err := dec.Decode() 1054 if err == io.EOF { 1055 break 1056 } else if err != nil { 1057 return nil, nil, err 1058 } 1059 1060 piece := readPieceFromWAL(msg) 1061 if piece == nil { 1062 continue 1063 } 1064 1065 switch p := piece.(type) { 1066 case EndHeightMessage: 1067 // if its not the first one, we have a full block 1068 if thisBlockParts != nil { 1069 var pbb = new(tmproto.Block) 1070 bz, err := ioutil.ReadAll(thisBlockParts.GetReader()) 1071 if err != nil { 1072 panic(err) 1073 } 1074 err = proto.Unmarshal(bz, pbb) 1075 if err != nil { 1076 panic(err) 1077 } 1078 block, err := types.BlockFromProto(pbb) 1079 if err != nil { 1080 panic(err) 1081 } 1082 1083 if block.Height != height+1 { 1084 panic(fmt.Sprintf("read bad block from wal. got height %d, expected %d", block.Height, height+1)) 1085 } 1086 commitHeight := thisBlockCommit.Height 1087 if commitHeight != height+1 { 1088 panic(fmt.Sprintf("commit doesnt match. got height %d, expected %d", commitHeight, height+1)) 1089 } 1090 blocks = append(blocks, block) 1091 commits = append(commits, thisBlockCommit) 1092 height++ 1093 } 1094 case *types.PartSetHeader: 1095 thisBlockParts = types.NewPartSetFromHeader(*p) 1096 case *types.Part: 1097 _, err := thisBlockParts.AddPart(p) 1098 if err != nil { 1099 return nil, nil, err 1100 } 1101 case *types.Vote: 1102 if p.Type == tmproto.PrecommitType { 1103 thisBlockCommit = types.NewCommit(p.Height, p.Round, 1104 p.BlockID, []types.CommitSig{p.CommitSig()}) 1105 } 1106 } 1107 } 1108 // grab the last block too 1109 bz, err := ioutil.ReadAll(thisBlockParts.GetReader()) 1110 if err != nil { 1111 panic(err) 1112 } 1113 var pbb = new(tmproto.Block) 1114 err = proto.Unmarshal(bz, pbb) 1115 if err != nil { 1116 panic(err) 1117 } 1118 block, err := types.BlockFromProto(pbb) 1119 if err != nil { 1120 panic(err) 1121 } 1122 if block.Height != height+1 { 1123 panic(fmt.Sprintf("read bad block from wal. got height %d, expected %d", block.Height, height+1)) 1124 } 1125 commitHeight := thisBlockCommit.Height 1126 if commitHeight != height+1 { 1127 panic(fmt.Sprintf("commit doesnt match. got height %d, expected %d", commitHeight, height+1)) 1128 } 1129 blocks = append(blocks, block) 1130 commits = append(commits, thisBlockCommit) 1131 return blocks, commits, nil 1132 } 1133 1134 func readPieceFromWAL(msg *TimedWALMessage) interface{} { 1135 // for logging 1136 switch m := msg.Msg.(type) { 1137 case msgInfo: 1138 switch msg := m.Msg.(type) { 1139 case *ProposalMessage: 1140 return &msg.Proposal.BlockID.PartSetHeader 1141 case *BlockPartMessage: 1142 return msg.Part 1143 case *VoteMessage: 1144 return msg.Vote 1145 } 1146 case EndHeightMessage: 1147 return m 1148 } 1149 1150 return nil 1151 } 1152 1153 // fresh state and mock store 1154 func stateAndStore( 1155 config *cfg.Config, 1156 pubKey crypto.PubKey, 1157 appVersion uint64) (dbm.DB, sm.State, *mockBlockStore) { 1158 stateDB := dbm.NewMemDB() 1159 stateStore := sm.NewStore(stateDB) 1160 state, _ := sm.MakeGenesisStateFromFile(config.GenesisFile()) 1161 state.Version.Consensus.App = appVersion 1162 store := newMockBlockStore(config, state.ConsensusParams) 1163 if err := stateStore.Save(state); err != nil { 1164 panic(err) 1165 } 1166 return stateDB, state, store 1167 } 1168 1169 //---------------------------------- 1170 // mock block store 1171 1172 type mockBlockStore struct { 1173 config *cfg.Config 1174 params types.ConsensusParams 1175 chain []*types.Block 1176 commits []*types.Commit 1177 base int64 1178 } 1179 1180 // TODO: NewBlockStore(db.NewMemDB) ... 1181 func newMockBlockStore(config *cfg.Config, params types.ConsensusParams) *mockBlockStore { 1182 return &mockBlockStore{config, params, nil, nil, 0} 1183 } 1184 1185 func (bs *mockBlockStore) Height() int64 { return int64(len(bs.chain)) } 1186 func (bs *mockBlockStore) Base() int64 { return bs.base } 1187 func (bs *mockBlockStore) Size() int64 { return bs.Height() - bs.Base() + 1 } 1188 func (bs *mockBlockStore) LoadBaseMeta() *types.BlockMeta { return bs.LoadBlockMeta(bs.base) } 1189 func (bs *mockBlockStore) LoadBlock(height int64) *types.Block { return bs.chain[height-1] } 1190 func (bs *mockBlockStore) LoadBlockByHash(hash []byte) *types.Block { 1191 return bs.chain[int64(len(bs.chain))-1] 1192 } 1193 func (bs *mockBlockStore) LoadBlockMeta(height int64) *types.BlockMeta { 1194 block := bs.chain[height-1] 1195 return &types.BlockMeta{ 1196 BlockID: types.BlockID{Hash: block.Hash(), PartSetHeader: block.MakePartSet(types.BlockPartSizeBytes).Header()}, 1197 Header: block.Header, 1198 } 1199 } 1200 func (bs *mockBlockStore) LoadBlockPart(height int64, index int) *types.Part { return nil } 1201 func (bs *mockBlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) { 1202 } 1203 func (bs *mockBlockStore) LoadBlockCommit(height int64) *types.Commit { 1204 return bs.commits[height-1] 1205 } 1206 func (bs *mockBlockStore) LoadSeenCommit(height int64) *types.Commit { 1207 return bs.commits[height-1] 1208 } 1209 1210 func (bs *mockBlockStore) PruneBlocks(height int64) (uint64, error) { 1211 pruned := uint64(0) 1212 for i := int64(0); i < height-1; i++ { 1213 bs.chain[i] = nil 1214 bs.commits[i] = nil 1215 pruned++ 1216 } 1217 bs.base = height 1218 return pruned, nil 1219 } 1220 1221 //--------------------------------------- 1222 // Test handshake/init chain 1223 1224 func TestHandshakeUpdatesValidators(t *testing.T) { 1225 val, _ := factory.RandValidator(true, 10) 1226 vals := types.NewValidatorSet([]*types.Validator{val}) 1227 app := &initChainApp{vals: types.TM2PB.ValidatorUpdates(vals)} 1228 clientCreator := proxy.NewLocalClientCreator(app) 1229 1230 config := ResetConfig("handshake_test_") 1231 t.Cleanup(func() { _ = os.RemoveAll(config.RootDir) }) 1232 1233 privVal, err := privval.LoadFilePV(config.PrivValidator.KeyFile(), config.PrivValidator.StateFile()) 1234 require.NoError(t, err) 1235 pubKey, err := privVal.GetPubKey(context.Background()) 1236 require.NoError(t, err) 1237 stateDB, state, store := stateAndStore(config, pubKey, 0x0) 1238 stateStore := sm.NewStore(stateDB) 1239 1240 oldValAddr := state.Validators.Validators[0].Address 1241 1242 // now start the app using the handshake - it should sync 1243 genDoc, _ := sm.MakeGenesisDocFromFile(config.GenesisFile()) 1244 handshaker := NewHandshaker(stateStore, state, store, genDoc) 1245 proxyApp := proxy.NewAppConns(clientCreator) 1246 if err := proxyApp.Start(); err != nil { 1247 t.Fatalf("Error starting proxy app connections: %v", err) 1248 } 1249 t.Cleanup(func() { 1250 if err := proxyApp.Stop(); err != nil { 1251 t.Error(err) 1252 } 1253 }) 1254 if err := handshaker.Handshake(proxyApp); err != nil { 1255 t.Fatalf("Error on abci handshake: %v", err) 1256 } 1257 // reload the state, check the validator set was updated 1258 state, err = stateStore.Load() 1259 require.NoError(t, err) 1260 1261 newValAddr := state.Validators.Validators[0].Address 1262 expectValAddr := val.Address 1263 assert.NotEqual(t, oldValAddr, newValAddr) 1264 assert.Equal(t, newValAddr, expectValAddr) 1265 } 1266 1267 // returns the vals on InitChain 1268 type initChainApp struct { 1269 abci.BaseApplication 1270 vals []abci.ValidatorUpdate 1271 } 1272 1273 func (ica *initChainApp) InitChain(req abci.RequestInitChain) abci.ResponseInitChain { 1274 return abci.ResponseInitChain{ 1275 Validators: ica.vals, 1276 } 1277 }