github.com/okex/exchain@v1.8.0/libs/tendermint/consensus/common_test.go (about)

     1  package consensus
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"fmt"
     7  	"io/ioutil"
     8  	"os"
     9  	"path"
    10  	"path/filepath"
    11  	"sort"
    12  	"sync"
    13  	"testing"
    14  	"time"
    15  
    16  	"github.com/go-kit/kit/log/term"
    17  	"github.com/pkg/errors"
    18  	"github.com/stretchr/testify/require"
    19  
    20  	dbm "github.com/okex/exchain/libs/tm-db"
    21  
    22  	abcicli "github.com/okex/exchain/libs/tendermint/abci/client"
    23  	"github.com/okex/exchain/libs/tendermint/abci/example/counter"
    24  	"github.com/okex/exchain/libs/tendermint/abci/example/kvstore"
    25  	abci "github.com/okex/exchain/libs/tendermint/abci/types"
    26  	cfg "github.com/okex/exchain/libs/tendermint/config"
    27  	cstypes "github.com/okex/exchain/libs/tendermint/consensus/types"
    28  	tmbytes "github.com/okex/exchain/libs/tendermint/libs/bytes"
    29  	"github.com/okex/exchain/libs/tendermint/libs/log"
    30  	tmos "github.com/okex/exchain/libs/tendermint/libs/os"
    31  	tmpubsub "github.com/okex/exchain/libs/tendermint/libs/pubsub"
    32  	mempl "github.com/okex/exchain/libs/tendermint/mempool"
    33  	"github.com/okex/exchain/libs/tendermint/p2p"
    34  	"github.com/okex/exchain/libs/tendermint/privval"
    35  	sm "github.com/okex/exchain/libs/tendermint/state"
    36  	"github.com/okex/exchain/libs/tendermint/store"
    37  	"github.com/okex/exchain/libs/tendermint/types"
    38  	tmtime "github.com/okex/exchain/libs/tendermint/types/time"
    39  )
    40  
    41  const (
    42  	testSubscriber = "test-client"
    43  )
    44  
    45  // A cleanupFunc cleans up any config / test files created for a particular
    46  // test.
    47  type cleanupFunc func()
    48  
    49  // genesis, chain_id, priv_val
    50  var config *cfg.Config = configSetup() // NOTE: must be reset for each _test.go file
    51  var consensusReplayConfig *cfg.Config
    52  var ensureTimeout = time.Millisecond * 100
    53  
    54  // timeout=3s always above 3 second so use timeout 3100 to check,can also use 3010
    55  var ensureRoundTimeout = 4000 * time.Millisecond
    56  
    57  func configSetup() *cfg.Config {
    58  	t := &testing.T{}
    59  	t.Helper()
    60  
    61  	cfg := ResetConfig("consensus_reactor_test")
    62  
    63  	t.Cleanup(func() { os.RemoveAll(cfg.RootDir) })
    64  
    65  	consensusReplayConfig := ResetConfig("consensus_replay_test")
    66  
    67  	t.Cleanup(func() { os.RemoveAll(consensusReplayConfig.RootDir) })
    68  
    69  	configStateTest := ResetConfig("consensus_state_test")
    70  
    71  	t.Cleanup(func() { os.RemoveAll(configStateTest.RootDir) })
    72  
    73  	configMempoolTest := ResetConfig("consensus_mempool_test")
    74  
    75  	t.Cleanup(func() { os.RemoveAll(configMempoolTest.RootDir) })
    76  
    77  	configByzantineTest := ResetConfig("consensus_byzantine_test")
    78  
    79  	t.Cleanup(func() { os.RemoveAll(configByzantineTest.RootDir) })
    80  
    81  	return cfg
    82  }
    83  
    84  func ensureDir(dir string, mode os.FileMode) {
    85  	if err := tmos.EnsureDir(dir, mode); err != nil {
    86  		panic(err)
    87  	}
    88  }
    89  
    90  func ResetConfig(name string) *cfg.Config {
    91  	return cfg.ResetTestRoot(name)
    92  }
    93  
    94  //-------------------------------------------------------------------------------
    95  // validator stub (a kvstore consensus peer we control)
    96  
    97  type validatorStub struct {
    98  	Index  int // Validator index. NOTE: we don't assume validator set changes.
    99  	Height int64
   100  	Round  int
   101  	types.PrivValidator
   102  }
   103  
   104  var testMinPower int64 = 10
   105  
   106  func newValidatorStub(privValidator types.PrivValidator, valIndex int) *validatorStub {
   107  	return &validatorStub{
   108  		Index:         valIndex,
   109  		PrivValidator: privValidator,
   110  	}
   111  }
   112  
   113  func (vs *validatorStub) signVote(
   114  	voteType types.SignedMsgType,
   115  	hash []byte,
   116  	header types.PartSetHeader) (*types.Vote, error) {
   117  
   118  	pubKey, err := vs.PrivValidator.GetPubKey()
   119  	if err != nil {
   120  		return nil, errors.Wrap(err, "can't get pubkey")
   121  	}
   122  
   123  	vote := &types.Vote{
   124  		ValidatorIndex:   vs.Index,
   125  		ValidatorAddress: pubKey.Address(),
   126  		Height:           vs.Height,
   127  		Round:            vs.Round,
   128  		Timestamp:        tmtime.Now(),
   129  		Type:             voteType,
   130  		BlockID:          types.BlockID{Hash: hash, PartsHeader: header},
   131  	}
   132  
   133  	err = vs.PrivValidator.SignVote(config.ChainID(), vote)
   134  	return vote, err
   135  }
   136  
   137  // Sign vote for type/hash/header
   138  func signVote(vs *validatorStub, voteType types.SignedMsgType, hash []byte, header types.PartSetHeader) *types.Vote {
   139  	v, err := vs.signVote(voteType, hash, header)
   140  	if err != nil {
   141  		panic(fmt.Errorf("failed to sign vote: %v", err))
   142  	}
   143  	return v
   144  }
   145  
   146  func signVotes(
   147  	voteType types.SignedMsgType,
   148  	hash []byte,
   149  	header types.PartSetHeader,
   150  	vss ...*validatorStub) []*types.Vote {
   151  	votes := make([]*types.Vote, len(vss))
   152  	for i, vs := range vss {
   153  		votes[i] = signVote(vs, voteType, hash, header)
   154  	}
   155  	return votes
   156  }
   157  
   158  func incrementHeight(vss ...*validatorStub) {
   159  	for _, vs := range vss {
   160  		vs.Height++
   161  	}
   162  }
   163  
   164  func incrementRound(vss ...*validatorStub) {
   165  	for _, vs := range vss {
   166  		vs.Round++
   167  	}
   168  }
   169  
   170  type ValidatorStubsByAddress []*validatorStub
   171  
   172  func (vss ValidatorStubsByAddress) Len() int {
   173  	return len(vss)
   174  }
   175  
   176  func (vss ValidatorStubsByAddress) Less(i, j int) bool {
   177  	vssi, err := vss[i].GetPubKey()
   178  	if err != nil {
   179  		panic(err)
   180  	}
   181  	vssj, err := vss[j].GetPubKey()
   182  	if err != nil {
   183  		panic(err)
   184  	}
   185  	return bytes.Compare(vssi.Address(), vssj.Address()) == -1
   186  }
   187  
   188  func (vss ValidatorStubsByAddress) Swap(i, j int) {
   189  	it := vss[i]
   190  	vss[i] = vss[j]
   191  	vss[i].Index = i
   192  	vss[j] = it
   193  	vss[j].Index = j
   194  }
   195  
   196  //-------------------------------------------------------------------------------
   197  // Functions for transitioning the consensus state
   198  // timeoutRoutine: receive requests for timeouts on tickChan and fire timeouts on tockChan
   199  // receiveRoutine: serializes processing of proposoals, block parts, votes; coordinates state transitions
   200  func (cs *State) startTestRoutines(maxSteps int) {
   201  	err := cs.timeoutTicker.Start()
   202  	if err != nil {
   203  		cs.Logger.Error("Error starting timeout ticker", "err", err)
   204  		return
   205  	}
   206  	go cs.receiveRoutine(maxSteps) // for test
   207  }
   208  
   209  func startTestRound(cs *State, height int64, round int) {
   210  	cs.enterNewRound(height, round)
   211  	cs.startTestRoutines(0)
   212  }
   213  
   214  // Create proposal block from cs1 but sign it with vs.
   215  func decideProposal(
   216  	cs1 *State,
   217  	vs *validatorStub,
   218  	height int64,
   219  	round int,
   220  ) (proposal *types.Proposal, block *types.Block) {
   221  	cs1.mtx.Lock()
   222  	block, blockParts := cs1.createProposalBlock()
   223  	validRound := cs1.ValidRound
   224  	chainID := cs1.state.ChainID
   225  	cs1.mtx.Unlock()
   226  	if block == nil {
   227  		panic("Failed to createProposalBlock. Did you forget to add commit for previous block?")
   228  	}
   229  
   230  	// Make proposal
   231  	polRound, propBlockID := validRound, types.BlockID{Hash: block.Hash(), PartsHeader: blockParts.Header()}
   232  	proposal = types.NewProposal(height, round, polRound, propBlockID)
   233  	if err := vs.SignProposal(chainID, proposal); err != nil {
   234  		panic(err)
   235  	}
   236  	return
   237  }
   238  
   239  func addVotes(to *State, votes ...*types.Vote) {
   240  	for _, vote := range votes {
   241  		to.peerMsgQueue <- msgInfo{Msg: &VoteMessage{vote}}
   242  	}
   243  }
   244  
   245  func signAddVotes(
   246  	to *State,
   247  	voteType types.SignedMsgType,
   248  	hash []byte,
   249  	header types.PartSetHeader,
   250  	vss ...*validatorStub,
   251  ) {
   252  	votes := signVotes(voteType, hash, header, vss...)
   253  	addVotes(to, votes...)
   254  }
   255  
   256  func validatePrevote(t *testing.T, cs *State, round int, privVal *validatorStub, blockHash []byte) {
   257  	prevotes := cs.Votes.Prevotes(round)
   258  	pubKey, err := privVal.GetPubKey()
   259  	require.NoError(t, err)
   260  	address := pubKey.Address()
   261  	var vote *types.Vote
   262  	if vote = prevotes.GetByAddress(address); vote == nil {
   263  		panic("Failed to find prevote from validator")
   264  	}
   265  	if blockHash == nil {
   266  		if vote.BlockID.Hash != nil {
   267  			panic(fmt.Sprintf("Expected prevote to be for nil, got %X", vote.BlockID.Hash))
   268  		}
   269  	} else {
   270  		if !bytes.Equal(vote.BlockID.Hash, blockHash) {
   271  			panic(fmt.Sprintf("Expected prevote to be for %X, got %X", blockHash, vote.BlockID.Hash))
   272  		}
   273  	}
   274  }
   275  
   276  func validateLastPrecommit(t *testing.T, cs *State, privVal *validatorStub, blockHash []byte) {
   277  	votes := cs.LastCommit
   278  	pv, err := privVal.GetPubKey()
   279  	require.NoError(t, err)
   280  	address := pv.Address()
   281  	var vote *types.Vote
   282  	if vote = votes.GetByAddress(address); vote == nil {
   283  		panic("Failed to find precommit from validator")
   284  	}
   285  	if !bytes.Equal(vote.BlockID.Hash, blockHash) {
   286  		panic(fmt.Sprintf("Expected precommit to be for %X, got %X", blockHash, vote.BlockID.Hash))
   287  	}
   288  }
   289  
   290  func validatePrecommit(
   291  	t *testing.T,
   292  	cs *State,
   293  	thisRound,
   294  	lockRound int,
   295  	privVal *validatorStub,
   296  	votedBlockHash,
   297  	lockedBlockHash []byte,
   298  ) {
   299  	precommits := cs.Votes.Precommits(thisRound)
   300  	pv, err := privVal.GetPubKey()
   301  	require.NoError(t, err)
   302  	address := pv.Address()
   303  	var vote *types.Vote
   304  	if vote = precommits.GetByAddress(address); vote == nil {
   305  		panic("Failed to find precommit from validator")
   306  	}
   307  
   308  	if votedBlockHash == nil {
   309  		if vote.BlockID.Hash != nil {
   310  			panic("Expected precommit to be for nil")
   311  		}
   312  	} else {
   313  		if !bytes.Equal(vote.BlockID.Hash, votedBlockHash) {
   314  			panic("Expected precommit to be for proposal block")
   315  		}
   316  	}
   317  
   318  	if lockedBlockHash == nil {
   319  		if cs.LockedRound != lockRound || cs.LockedBlock != nil {
   320  			panic(fmt.Sprintf(
   321  				"Expected to be locked on nil at round %d. Got locked at round %d with block %v",
   322  				lockRound,
   323  				cs.LockedRound,
   324  				cs.LockedBlock))
   325  		}
   326  	} else {
   327  		if cs.LockedRound != lockRound || !bytes.Equal(cs.LockedBlock.Hash(), lockedBlockHash) {
   328  			panic(fmt.Sprintf(
   329  				"Expected block to be locked on round %d, got %d. Got locked block %X, expected %X",
   330  				lockRound,
   331  				cs.LockedRound,
   332  				cs.LockedBlock.Hash(),
   333  				lockedBlockHash))
   334  		}
   335  	}
   336  
   337  }
   338  
   339  func validatePrevoteAndPrecommit(
   340  	t *testing.T,
   341  	cs *State,
   342  	thisRound,
   343  	lockRound int,
   344  	privVal *validatorStub,
   345  	votedBlockHash,
   346  	lockedBlockHash []byte,
   347  ) {
   348  	// verify the prevote
   349  	validatePrevote(t, cs, thisRound, privVal, votedBlockHash)
   350  	// verify precommit
   351  	cs.mtx.Lock()
   352  	validatePrecommit(t, cs, thisRound, lockRound, privVal, votedBlockHash, lockedBlockHash)
   353  	cs.mtx.Unlock()
   354  }
   355  
   356  func subscribeToVoter(cs *State, addr []byte) <-chan tmpubsub.Message {
   357  	votesSub, err := cs.eventBus.SubscribeUnbuffered(context.Background(), testSubscriber, types.EventQueryVote)
   358  	if err != nil {
   359  		panic(fmt.Sprintf("failed to subscribe %s to %v", testSubscriber, types.EventQueryVote))
   360  	}
   361  	ch := make(chan tmpubsub.Message)
   362  	go func() {
   363  		for msg := range votesSub.Out() {
   364  			vote := msg.Data().(types.EventDataVote)
   365  			// we only fire for our own votes
   366  			if bytes.Equal(addr, vote.Vote.ValidatorAddress) {
   367  				ch <- msg
   368  			}
   369  		}
   370  	}()
   371  	return ch
   372  }
   373  
   374  //-------------------------------------------------------------------------------
   375  // consensus states
   376  
   377  func newState(state sm.State, pv types.PrivValidator, app abci.Application) *State {
   378  	config := cfg.ResetTestRoot("consensus_state_test")
   379  	return newStateWithConfig(config, state, pv, app)
   380  }
   381  
   382  func newStateWithConfig(
   383  	thisConfig *cfg.Config,
   384  	state sm.State,
   385  	pv types.PrivValidator,
   386  	app abci.Application,
   387  ) *State {
   388  	blockDB := dbm.NewMemDB()
   389  	return newStateWithConfigAndBlockStore(thisConfig, state, pv, app, blockDB)
   390  }
   391  
   392  func newStateWithConfigAndBlockStore(
   393  	thisConfig *cfg.Config,
   394  	state sm.State,
   395  	pv types.PrivValidator,
   396  	app abci.Application,
   397  	blockDB dbm.DB,
   398  ) *State {
   399  	// Get BlockStore
   400  	blockStore := store.NewBlockStore(blockDB)
   401  
   402  	// one for mempool, one for consensus
   403  	mtx := new(sync.Mutex)
   404  	proxyAppConnMem := abcicli.NewLocalClient(mtx, app)
   405  	proxyAppConnCon := abcicli.NewLocalClient(mtx, app)
   406  
   407  	// Make Mempool
   408  	mempool := mempl.NewCListMempool(thisConfig.Mempool, proxyAppConnMem, 0)
   409  	mempool.SetLogger(log.TestingLogger().With("module", "mempool"))
   410  	if thisConfig.Consensus.WaitForTxs() {
   411  		mempool.EnableTxsAvailable()
   412  	}
   413  
   414  	// mock the evidence pool
   415  	evpool := sm.MockEvidencePool{}
   416  
   417  	// Make State
   418  	stateDB := blockDB
   419  	sm.SaveState(stateDB, state) //for save height 1's validators info
   420  	blockExec := sm.NewBlockExecutor(stateDB, log.TestingLogger(), proxyAppConnCon, mempool, evpool)
   421  	cs := NewState(thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool)
   422  	cs.SetLogger(log.TestingLogger().With("module", "consensus"))
   423  	cs.SetPrivValidator(pv)
   424  
   425  	eventBus := types.NewEventBus()
   426  	eventBus.SetLogger(log.TestingLogger().With("module", "events"))
   427  	eventBus.Start()
   428  	cs.SetEventBus(eventBus)
   429  	return cs
   430  }
   431  
   432  func loadPrivValidator(config *cfg.Config) *privval.FilePV {
   433  	privValidatorKeyFile := config.PrivValidatorKeyFile()
   434  	ensureDir(filepath.Dir(privValidatorKeyFile), 0700)
   435  	privValidatorStateFile := config.PrivValidatorStateFile()
   436  	privValidator := privval.LoadOrGenFilePV(privValidatorKeyFile, privValidatorStateFile)
   437  	privValidator.Reset()
   438  	return privValidator
   439  }
   440  
   441  func randState(nValidators int) (*State, []*validatorStub) {
   442  	// Get State
   443  	state, privVals := randGenesisState(nValidators, false, 10)
   444  
   445  	vss := make([]*validatorStub, nValidators)
   446  
   447  	cs := newState(state, privVals[0], counter.NewApplication(true))
   448  
   449  	for i := 0; i < nValidators; i++ {
   450  		vss[i] = newValidatorStub(privVals[i], i)
   451  	}
   452  	// since cs1 starts at 1
   453  	incrementHeight(vss[1:]...)
   454  
   455  	return cs, vss
   456  }
   457  
   458  //-------------------------------------------------------------------------------
   459  
   460  func ensureNoNewEvent(ch <-chan tmpubsub.Message, timeout time.Duration,
   461  	errorMessage string) {
   462  	select {
   463  	case <-time.After(timeout):
   464  		break
   465  	case <-ch:
   466  		panic(errorMessage)
   467  	}
   468  }
   469  
   470  func ensureNoNewEventOnChannel(ch <-chan tmpubsub.Message) {
   471  	ensureNoNewEvent(
   472  		ch,
   473  		ensureTimeout,
   474  		"We should be stuck waiting, not receiving new event on the channel")
   475  }
   476  
   477  func ensureNoNewRoundStep(stepCh <-chan tmpubsub.Message) {
   478  	ensureNoNewEvent(
   479  		stepCh,
   480  		ensureTimeout,
   481  		"We should be stuck waiting, not receiving NewRoundStep event")
   482  }
   483  
   484  func ensureNoNewUnlock(unlockCh <-chan tmpubsub.Message) {
   485  	ensureNoNewEvent(
   486  		unlockCh,
   487  		ensureTimeout,
   488  		"We should be stuck waiting, not receiving Unlock event")
   489  }
   490  
   491  func ensureNoNewTimeout(stepCh <-chan tmpubsub.Message, timeout int64) {
   492  	timeoutDuration := time.Duration(timeout*10) * time.Nanosecond
   493  	ensureNoNewEvent(
   494  		stepCh,
   495  		timeoutDuration,
   496  		"We should be stuck waiting, not receiving NewTimeout event")
   497  }
   498  
   499  func ensureNewEvent(ch <-chan tmpubsub.Message, height int64, round int, timeout time.Duration, errorMessage string) {
   500  	select {
   501  	case <-time.After(timeout):
   502  		panic(errorMessage)
   503  	case msg := <-ch:
   504  		roundStateEvent, ok := msg.Data().(types.EventDataRoundState)
   505  		if !ok {
   506  			panic(fmt.Sprintf("expected a EventDataRoundState, got %T. Wrong subscription channel?",
   507  				msg.Data()))
   508  		}
   509  		if roundStateEvent.Height != height {
   510  			panic(fmt.Sprintf("expected height %v, got %v", height, roundStateEvent.Height))
   511  		}
   512  		if roundStateEvent.Round != round {
   513  			panic(fmt.Sprintf("expected round %v, got %v", round, roundStateEvent.Round))
   514  		}
   515  		// TODO: We could check also for a step at this point!
   516  	}
   517  }
   518  
   519  func ensureNewRound(roundCh <-chan tmpubsub.Message, height int64, round int) {
   520  	select {
   521  	case <-time.After(ensureRoundTimeout):
   522  		panic("Timeout expired while waiting for NewRound event")
   523  	case msg := <-roundCh:
   524  		newRoundEvent, ok := msg.Data().(types.EventDataNewRound)
   525  		if !ok {
   526  			panic(fmt.Sprintf("expected a EventDataNewRound, got %T. Wrong subscription channel?",
   527  				msg.Data()))
   528  		}
   529  		if newRoundEvent.Height != height {
   530  			panic(fmt.Sprintf("expected height %v, got %v", height, newRoundEvent.Height))
   531  		}
   532  		if newRoundEvent.Round != round {
   533  			panic(fmt.Sprintf("expected round %v, got %v", round, newRoundEvent.Round))
   534  		}
   535  	}
   536  }
   537  
   538  func ensureNewTimeout(timeoutCh <-chan tmpubsub.Message, height int64, round int, timeout int64) {
   539  	timeoutDuration := time.Duration(timeout*100) * time.Nanosecond
   540  	ensureNewEvent(timeoutCh, height, round, timeoutDuration,
   541  		"Timeout expired while waiting for NewTimeout event")
   542  }
   543  
   544  func ensureNewProposal(proposalCh <-chan tmpubsub.Message, height int64, round int) {
   545  	select {
   546  	case <-time.After(ensureTimeout):
   547  		panic("Timeout expired while waiting for NewProposal event")
   548  	case msg := <-proposalCh:
   549  		proposalEvent, ok := msg.Data().(types.EventDataCompleteProposal)
   550  		if !ok {
   551  			panic(fmt.Sprintf("expected a EventDataCompleteProposal, got %T. Wrong subscription channel?",
   552  				msg.Data()))
   553  		}
   554  		if proposalEvent.Height != height {
   555  			panic(fmt.Sprintf("expected height %v, got %v", height, proposalEvent.Height))
   556  		}
   557  		if proposalEvent.Round != round {
   558  			panic(fmt.Sprintf("expected round %v, got %v", round, proposalEvent.Round))
   559  		}
   560  	}
   561  }
   562  
   563  func ensureNewValidBlock(validBlockCh <-chan tmpubsub.Message, height int64, round int) {
   564  	ensureNewEvent(validBlockCh, height, round, ensureTimeout,
   565  		"Timeout expired while waiting for NewValidBlock event")
   566  }
   567  
   568  func ensureNewBlock(blockCh <-chan tmpubsub.Message, height int64) {
   569  	select {
   570  	case <-time.After(ensureTimeout):
   571  		panic("Timeout expired while waiting for NewBlock event")
   572  	case msg := <-blockCh:
   573  		blockEvent, ok := msg.Data().(types.EventDataNewBlock)
   574  		if !ok {
   575  			panic(fmt.Sprintf("expected a EventDataNewBlock, got %T. Wrong subscription channel?",
   576  				msg.Data()))
   577  		}
   578  		if blockEvent.Block.Height != height {
   579  			panic(fmt.Sprintf("expected height %v, got %v", height, blockEvent.Block.Height))
   580  		}
   581  	}
   582  }
   583  
   584  func ensureNewBlockHeader(blockCh <-chan tmpubsub.Message, height int64, blockHash tmbytes.HexBytes) {
   585  	select {
   586  	case <-time.After(ensureTimeout):
   587  		panic("Timeout expired while waiting for NewBlockHeader event")
   588  	case msg := <-blockCh:
   589  		blockHeaderEvent, ok := msg.Data().(types.EventDataNewBlockHeader)
   590  		if !ok {
   591  			panic(fmt.Sprintf("expected a EventDataNewBlockHeader, got %T. Wrong subscription channel?",
   592  				msg.Data()))
   593  		}
   594  		if blockHeaderEvent.Header.Height != height {
   595  			panic(fmt.Sprintf("expected height %v, got %v", height, blockHeaderEvent.Header.Height))
   596  		}
   597  		if !bytes.Equal(blockHeaderEvent.Header.Hash(), blockHash) {
   598  			panic(fmt.Sprintf("expected header %X, got %X", blockHash, blockHeaderEvent.Header.Hash()))
   599  		}
   600  	}
   601  }
   602  
   603  func ensureNewUnlock(unlockCh <-chan tmpubsub.Message, height int64, round int) {
   604  	ensureNewEvent(unlockCh, height, round, ensureTimeout,
   605  		"Timeout expired while waiting for NewUnlock event")
   606  }
   607  
   608  func ensureProposal(proposalCh <-chan tmpubsub.Message, height int64, round int, propID types.BlockID) {
   609  	select {
   610  	case <-time.After(ensureTimeout):
   611  		panic("Timeout expired while waiting for NewProposal event")
   612  	case msg := <-proposalCh:
   613  		proposalEvent, ok := msg.Data().(types.EventDataCompleteProposal)
   614  		if !ok {
   615  			panic(fmt.Sprintf("expected a EventDataCompleteProposal, got %T. Wrong subscription channel?",
   616  				msg.Data()))
   617  		}
   618  		if proposalEvent.Height != height {
   619  			panic(fmt.Sprintf("expected height %v, got %v", height, proposalEvent.Height))
   620  		}
   621  		if proposalEvent.Round != round {
   622  			panic(fmt.Sprintf("expected round %v, got %v", round, proposalEvent.Round))
   623  		}
   624  		if !proposalEvent.BlockID.Equals(propID) {
   625  			panic("Proposed block does not match expected block")
   626  		}
   627  	}
   628  }
   629  
   630  func ensurePrecommit(voteCh <-chan tmpubsub.Message, height int64, round int) {
   631  	ensureVote(voteCh, height, round, types.PrecommitType)
   632  }
   633  
   634  func ensurePrevote(voteCh <-chan tmpubsub.Message, height int64, round int) {
   635  	ensureVote(voteCh, height, round, types.PrevoteType)
   636  }
   637  
   638  func ensureVote(voteCh <-chan tmpubsub.Message, height int64, round int,
   639  	voteType types.SignedMsgType) {
   640  	select {
   641  	case <-time.After(ensureRoundTimeout):
   642  		panic("Timeout expired while waiting for NewVote event")
   643  	case msg := <-voteCh:
   644  		voteEvent, ok := msg.Data().(types.EventDataVote)
   645  		if !ok {
   646  			panic(fmt.Sprintf("expected a EventDataVote, got %T. Wrong subscription channel?",
   647  				msg.Data()))
   648  		}
   649  		vote := voteEvent.Vote
   650  		if vote.Height != height {
   651  			panic(fmt.Sprintf("expected height %v, got %v", height, vote.Height))
   652  		}
   653  		if vote.Round != round {
   654  			panic(fmt.Sprintf("expected round %v, got %v", round, vote.Round))
   655  		}
   656  		if vote.Type != voteType {
   657  			panic(fmt.Sprintf("expected type %v, got %v", voteType, vote.Type))
   658  		}
   659  	}
   660  }
   661  
   662  func ensureNewEventOnChannel(ch <-chan tmpubsub.Message) {
   663  	select {
   664  	case <-time.After(ensureTimeout):
   665  		panic("Timeout expired while waiting for new activity on the channel")
   666  	case <-ch:
   667  	}
   668  }
   669  
   670  //-------------------------------------------------------------------------------
   671  // consensus nets
   672  
   673  // consensusLogger is a TestingLogger which uses a different
   674  // color for each validator ("validator" key must exist).
   675  func consensusLogger() log.Logger {
   676  	return log.TestingLoggerWithColorFn(func(keyvals ...interface{}) term.FgBgColor {
   677  		for i := 0; i < len(keyvals)-1; i += 2 {
   678  			if keyvals[i] == "validator" {
   679  				return term.FgBgColor{Fg: term.Color(uint8(keyvals[i+1].(int) + 1))}
   680  			}
   681  		}
   682  		return term.FgBgColor{}
   683  	}).With("module", "consensus")
   684  }
   685  
   686  func randConsensusNet(nValidators int, testName string, tickerFunc func() TimeoutTicker,
   687  	appFunc func() abci.Application, configOpts ...func(*cfg.Config)) ([]*State, cleanupFunc) {
   688  	genDoc, privVals := randGenesisDoc(nValidators, false, 30)
   689  	css := make([]*State, nValidators)
   690  	logger := consensusLogger()
   691  	configRootDirs := make([]string, 0, nValidators)
   692  	for i := 0; i < nValidators; i++ {
   693  		stateDB := dbm.NewMemDB() // each state needs its own db
   694  		state, _ := sm.LoadStateFromDBOrGenesisDoc(stateDB, genDoc)
   695  		thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i))
   696  		configRootDirs = append(configRootDirs, thisConfig.RootDir)
   697  		for _, opt := range configOpts {
   698  			opt(thisConfig)
   699  		}
   700  		ensureDir(filepath.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
   701  		app := appFunc()
   702  		vals := types.TM2PB.ValidatorUpdates(state.Validators)
   703  		app.InitChain(abci.RequestInitChain{Validators: vals})
   704  
   705  		css[i] = newStateWithConfigAndBlockStore(thisConfig, state, privVals[i], app, stateDB)
   706  		css[i].SetTimeoutTicker(tickerFunc())
   707  		css[i].SetLogger(logger.With("validator", i, "module", "consensus"))
   708  	}
   709  	return css, func() {
   710  		for _, dir := range configRootDirs {
   711  			os.RemoveAll(dir)
   712  		}
   713  	}
   714  }
   715  
   716  // nPeers = nValidators + nNotValidator
   717  func randConsensusNetWithPeers(
   718  	nValidators,
   719  	nPeers int,
   720  	testName string,
   721  	tickerFunc func() TimeoutTicker,
   722  	appFunc func(string) abci.Application,
   723  ) ([]*State, *types.GenesisDoc, *cfg.Config, cleanupFunc) {
   724  	genDoc, privVals := randGenesisDoc(nValidators, false, testMinPower)
   725  	css := make([]*State, nPeers)
   726  	logger := consensusLogger()
   727  	var peer0Config *cfg.Config
   728  	configRootDirs := make([]string, 0, nPeers)
   729  	for i := 0; i < nPeers; i++ {
   730  		stateDB := dbm.NewMemDB() // each state needs its own db
   731  		state, _ := sm.LoadStateFromDBOrGenesisDoc(stateDB, genDoc)
   732  		thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i))
   733  		configRootDirs = append(configRootDirs, thisConfig.RootDir)
   734  		ensureDir(filepath.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
   735  		if i == 0 {
   736  			peer0Config = thisConfig
   737  		}
   738  		var privVal types.PrivValidator
   739  		if i < nValidators {
   740  			privVal = privVals[i]
   741  		} else {
   742  			tempKeyFile, err := ioutil.TempFile("", "priv_validator_key_")
   743  			if err != nil {
   744  				panic(err)
   745  			}
   746  			tempStateFile, err := ioutil.TempFile("", "priv_validator_state_")
   747  			if err != nil {
   748  				panic(err)
   749  			}
   750  
   751  			privVal = privval.GenFilePV(tempKeyFile.Name(), tempStateFile.Name())
   752  		}
   753  
   754  		app := appFunc(path.Join(config.DBDir(), fmt.Sprintf("%s_%d", testName, i)))
   755  		vals := types.TM2PB.ValidatorUpdates(state.Validators)
   756  		if _, ok := app.(*kvstore.PersistentKVStoreApplication); ok {
   757  			// simulate handshake, receive app version. If don't do this, replay test will fail
   758  			state.Version.Consensus.App = kvstore.ProtocolVersion
   759  		}
   760  		app.InitChain(abci.RequestInitChain{Validators: vals})
   761  		//sm.SaveState(stateDB,state)	//height 1's validatorsInfo already saved in LoadStateFromDBOrGenesisDoc above
   762  
   763  		css[i] = newStateWithConfig(thisConfig, state, privVal, app)
   764  		css[i].SetTimeoutTicker(tickerFunc())
   765  		css[i].SetLogger(logger.With("validator", i, "module", "consensus"))
   766  	}
   767  	return css, genDoc, peer0Config, func() {
   768  		for _, dir := range configRootDirs {
   769  			os.RemoveAll(dir)
   770  		}
   771  	}
   772  }
   773  
   774  func getSwitchIndex(switches []*p2p.Switch, peer p2p.Peer) int {
   775  	for i, s := range switches {
   776  		if peer.NodeInfo().ID() == s.NodeInfo().ID() {
   777  			return i
   778  		}
   779  	}
   780  	panic("didnt find peer in switches")
   781  }
   782  
   783  //-------------------------------------------------------------------------------
   784  // genesis
   785  
   786  func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.GenesisDoc, []types.PrivValidator) {
   787  	validators := make([]types.GenesisValidator, numValidators)
   788  	privValidators := make([]types.PrivValidator, numValidators)
   789  	for i := 0; i < numValidators; i++ {
   790  		val, privVal := types.RandValidator(randPower, minPower)
   791  		validators[i] = types.GenesisValidator{
   792  			PubKey: val.PubKey,
   793  			Power:  val.VotingPower,
   794  		}
   795  		privValidators[i] = privVal
   796  	}
   797  	sort.Sort(types.PrivValidatorsByAddress(privValidators))
   798  
   799  	return &types.GenesisDoc{
   800  		GenesisTime: tmtime.Now(),
   801  		ChainID:     config.ChainID(),
   802  		Validators:  validators,
   803  	}, privValidators
   804  }
   805  
   806  func randGenesisState(numValidators int, randPower bool, minPower int64) (sm.State, []types.PrivValidator) {
   807  	genDoc, privValidators := randGenesisDoc(numValidators, randPower, minPower)
   808  	s0, _ := sm.MakeGenesisState(genDoc)
   809  	return s0, privValidators
   810  }
   811  
   812  //------------------------------------
   813  // mock ticker
   814  
   815  func newMockTickerFunc(onlyOnce bool) func() TimeoutTicker {
   816  	return func() TimeoutTicker {
   817  		return &mockTicker{
   818  			c:        make(chan timeoutInfo, 10),
   819  			onlyOnce: onlyOnce,
   820  		}
   821  	}
   822  }
   823  
   824  // mock ticker only fires on RoundStepNewHeight
   825  // and only once if onlyOnce=true
   826  type mockTicker struct {
   827  	c chan timeoutInfo
   828  
   829  	mtx      sync.Mutex
   830  	onlyOnce bool
   831  	fired    bool
   832  }
   833  
   834  func (m *mockTicker) Start() error {
   835  	return nil
   836  }
   837  
   838  func (m *mockTicker) Stop() error {
   839  	return nil
   840  }
   841  
   842  //add noop Reset function for TimeoutTicker interface
   843  //need to implement when used
   844  func (m *mockTicker) Reset() error {
   845  	return nil
   846  }
   847  
   848  func (m *mockTicker) ScheduleTimeout(ti timeoutInfo) {
   849  	m.mtx.Lock()
   850  	defer m.mtx.Unlock()
   851  	if m.onlyOnce && m.fired {
   852  		return
   853  	}
   854  	if ti.Step == cstypes.RoundStepNewHeight {
   855  		m.c <- ti
   856  		m.fired = true
   857  	}
   858  }
   859  
   860  func (m *mockTicker) Chan() <-chan timeoutInfo {
   861  	return m.c
   862  }
   863  
   864  func (*mockTicker) SetLogger(log.Logger) {}
   865  
   866  //------------------------------------
   867  
   868  func newCounter() abci.Application {
   869  	return counter.NewApplication(true)
   870  }
   871  
   872  func newPersistentKVStore() abci.Application {
   873  	dir, err := ioutil.TempDir("", "persistent-kvstore")
   874  	if err != nil {
   875  		panic(err)
   876  	}
   877  	return kvstore.NewPersistentKVStoreApplication(dir)
   878  }
   879  
   880  func newPersistentKVStoreWithPath(dbDir string) abci.Application {
   881  	return kvstore.NewPersistentKVStoreApplication(dbDir)
   882  }