github.com/franono/tendermint@v0.32.2-0.20200527150959-749313264ce9/consensus/state.go (about)

     1  package consensus
     2  
     3  import (
     4  	"bytes"
     5  	"errors"
     6  	"fmt"
     7  	"reflect"
     8  	"runtime/debug"
     9  	"sync"
    10  	"time"
    11  
    12  	"github.com/franono/tendermint/libs/fail"
    13  	"github.com/franono/tendermint/libs/log"
    14  	tmos "github.com/franono/tendermint/libs/os"
    15  	"github.com/franono/tendermint/libs/service"
    16  	tmtime "github.com/franono/tendermint/types/time"
    17  
    18  	cfg "github.com/franono/tendermint/config"
    19  	cstypes "github.com/franono/tendermint/consensus/types"
    20  	tmevents "github.com/franono/tendermint/libs/events"
    21  	"github.com/franono/tendermint/p2p"
    22  	sm "github.com/franono/tendermint/state"
    23  	"github.com/franono/tendermint/types"
    24  )
    25  
    26  //-----------------------------------------------------------------------------
    27  // Errors
    28  
    29  var (
    30  	ErrInvalidProposalSignature = errors.New("error invalid proposal signature")
    31  	ErrInvalidProposalPOLRound  = errors.New("error invalid proposal POL round")
    32  	ErrAddingVote               = errors.New("error adding vote")
    33  	ErrVoteHeightMismatch       = errors.New("error vote height mismatch")
    34  )
    35  
    36  //-----------------------------------------------------------------------------
    37  
    38  var (
    39  	msgQueueSize = 1000
    40  )
    41  
    42  // msgs from the reactor which may update the state
    43  type msgInfo struct {
    44  	Msg    Message `json:"msg"`
    45  	PeerID p2p.ID  `json:"peer_key"`
    46  }
    47  
    48  // internally generated messages which may update the state
    49  type timeoutInfo struct {
    50  	Duration time.Duration         `json:"duration"`
    51  	Height   int64                 `json:"height"`
    52  	Round    int                   `json:"round"`
    53  	Step     cstypes.RoundStepType `json:"step"`
    54  }
    55  
    56  func (ti *timeoutInfo) String() string {
    57  	return fmt.Sprintf("%v ; %d/%d %v", ti.Duration, ti.Height, ti.Round, ti.Step)
    58  }
    59  
    60  // interface to the mempool
    61  type txNotifier interface {
    62  	TxsAvailable() <-chan struct{}
    63  }
    64  
    65  // interface to the evidence pool
    66  type evidencePool interface {
    67  	AddEvidence(types.Evidence) error
    68  	AddPOLC(types.ProofOfLockChange) error
    69  }
    70  
    71  // State handles execution of the consensus algorithm.
    72  // It processes votes and proposals, and upon reaching agreement,
    73  // commits blocks to the chain and executes them against the application.
    74  // The internal state machine receives input from peers, the internal validator, and from a timer.
    75  type State struct {
    76  	service.BaseService
    77  
    78  	// config details
    79  	config        *cfg.ConsensusConfig
    80  	privValidator types.PrivValidator // for signing votes
    81  
    82  	// store blocks and commits
    83  	blockStore sm.BlockStore
    84  
    85  	// create and execute blocks
    86  	blockExec *sm.BlockExecutor
    87  
    88  	// notify us if txs are available
    89  	txNotifier txNotifier
    90  
    91  	// add evidence to the pool
    92  	// when it's detected
    93  	evpool evidencePool
    94  
    95  	// internal state
    96  	mtx sync.RWMutex
    97  	cstypes.RoundState
    98  	state sm.State // State until height-1.
    99  
   100  	// state changes may be triggered by: msgs from peers,
   101  	// msgs from ourself, or by timeouts
   102  	peerMsgQueue     chan msgInfo
   103  	internalMsgQueue chan msgInfo
   104  	timeoutTicker    TimeoutTicker
   105  
   106  	// information about about added votes and block parts are written on this channel
   107  	// so statistics can be computed by reactor
   108  	statsMsgQueue chan msgInfo
   109  
   110  	// we use eventBus to trigger msg broadcasts in the reactor,
   111  	// and to notify external subscribers, eg. through a websocket
   112  	eventBus *types.EventBus
   113  
   114  	// a Write-Ahead Log ensures we can recover from any kind of crash
   115  	// and helps us avoid signing conflicting votes
   116  	wal          WAL
   117  	replayMode   bool // so we don't log signing errors during replay
   118  	doWALCatchup bool // determines if we even try to do the catchup
   119  
   120  	// for tests where we want to limit the number of transitions the state makes
   121  	nSteps int
   122  
   123  	// some functions can be overwritten for testing
   124  	decideProposal func(height int64, round int)
   125  	doPrevote      func(height int64, round int)
   126  	setProposal    func(proposal *types.Proposal) error
   127  
   128  	// closed when we finish shutting down
   129  	done chan struct{}
   130  
   131  	// synchronous pubsub between consensus state and reactor.
   132  	// state only emits EventNewRoundStep and EventVote
   133  	evsw tmevents.EventSwitch
   134  
   135  	// for reporting metrics
   136  	metrics *Metrics
   137  }
   138  
   139  // StateOption sets an optional parameter on the State.
   140  type StateOption func(*State)
   141  
   142  // NewState returns a new State.
   143  func NewState(
   144  	config *cfg.ConsensusConfig,
   145  	state sm.State,
   146  	blockExec *sm.BlockExecutor,
   147  	blockStore sm.BlockStore,
   148  	txNotifier txNotifier,
   149  	evpool evidencePool,
   150  	options ...StateOption,
   151  ) *State {
   152  	cs := &State{
   153  		config:           config,
   154  		blockExec:        blockExec,
   155  		blockStore:       blockStore,
   156  		txNotifier:       txNotifier,
   157  		peerMsgQueue:     make(chan msgInfo, msgQueueSize),
   158  		internalMsgQueue: make(chan msgInfo, msgQueueSize),
   159  		timeoutTicker:    NewTimeoutTicker(),
   160  		statsMsgQueue:    make(chan msgInfo, msgQueueSize),
   161  		done:             make(chan struct{}),
   162  		doWALCatchup:     true,
   163  		wal:              nilWAL{},
   164  		evpool:           evpool,
   165  		evsw:             tmevents.NewEventSwitch(),
   166  		metrics:          NopMetrics(),
   167  	}
   168  	// set function defaults (may be overwritten before calling Start)
   169  	cs.decideProposal = cs.defaultDecideProposal
   170  	cs.doPrevote = cs.defaultDoPrevote
   171  	cs.setProposal = cs.defaultSetProposal
   172  
   173  	cs.updateToState(state)
   174  
   175  	// Don't call scheduleRound0 yet.
   176  	// We do that upon Start().
   177  	cs.reconstructLastCommit(state)
   178  	cs.BaseService = *service.NewBaseService(nil, "State", cs)
   179  	for _, option := range options {
   180  		option(cs)
   181  	}
   182  	return cs
   183  }
   184  
   185  //----------------------------------------
   186  // Public interface
   187  
   188  // SetLogger implements Service.
   189  func (cs *State) SetLogger(l log.Logger) {
   190  	cs.BaseService.Logger = l
   191  	cs.timeoutTicker.SetLogger(l)
   192  }
   193  
   194  // SetEventBus sets event bus.
   195  func (cs *State) SetEventBus(b *types.EventBus) {
   196  	cs.eventBus = b
   197  	cs.blockExec.SetEventBus(b)
   198  }
   199  
   200  // StateMetrics sets the metrics.
   201  func StateMetrics(metrics *Metrics) StateOption {
   202  	return func(cs *State) { cs.metrics = metrics }
   203  }
   204  
   205  // String returns a string.
   206  func (cs *State) String() string {
   207  	// better not to access shared variables
   208  	return "ConsensusState"
   209  }
   210  
   211  // GetState returns a copy of the chain state.
   212  func (cs *State) GetState() sm.State {
   213  	cs.mtx.RLock()
   214  	defer cs.mtx.RUnlock()
   215  	return cs.state.Copy()
   216  }
   217  
   218  // GetLastHeight returns the last height committed.
   219  // If there were no blocks, returns 0.
   220  func (cs *State) GetLastHeight() int64 {
   221  	cs.mtx.RLock()
   222  	defer cs.mtx.RUnlock()
   223  	return cs.RoundState.Height - 1
   224  }
   225  
   226  // GetRoundState returns a shallow copy of the internal consensus state.
   227  func (cs *State) GetRoundState() *cstypes.RoundState {
   228  	cs.mtx.RLock()
   229  	rs := cs.RoundState // copy
   230  	cs.mtx.RUnlock()
   231  	return &rs
   232  }
   233  
   234  // GetRoundStateJSON returns a json of RoundState, marshalled using go-amino.
   235  func (cs *State) GetRoundStateJSON() ([]byte, error) {
   236  	cs.mtx.RLock()
   237  	defer cs.mtx.RUnlock()
   238  	return cdc.MarshalJSON(cs.RoundState)
   239  }
   240  
   241  // GetRoundStateSimpleJSON returns a json of RoundStateSimple, marshalled using go-amino.
   242  func (cs *State) GetRoundStateSimpleJSON() ([]byte, error) {
   243  	cs.mtx.RLock()
   244  	defer cs.mtx.RUnlock()
   245  	return cdc.MarshalJSON(cs.RoundState.RoundStateSimple())
   246  }
   247  
   248  // GetValidators returns a copy of the current validators.
   249  func (cs *State) GetValidators() (int64, []*types.Validator) {
   250  	cs.mtx.RLock()
   251  	defer cs.mtx.RUnlock()
   252  	return cs.state.LastBlockHeight, cs.state.Validators.Copy().Validators
   253  }
   254  
   255  // SetPrivValidator sets the private validator account for signing votes.
   256  func (cs *State) SetPrivValidator(priv types.PrivValidator) {
   257  	cs.mtx.Lock()
   258  	cs.privValidator = priv
   259  	cs.mtx.Unlock()
   260  }
   261  
   262  // SetTimeoutTicker sets the local timer. It may be useful to overwrite for testing.
   263  func (cs *State) SetTimeoutTicker(timeoutTicker TimeoutTicker) {
   264  	cs.mtx.Lock()
   265  	cs.timeoutTicker = timeoutTicker
   266  	cs.mtx.Unlock()
   267  }
   268  
   269  // LoadCommit loads the commit for a given height.
   270  func (cs *State) LoadCommit(height int64) *types.Commit {
   271  	cs.mtx.RLock()
   272  	defer cs.mtx.RUnlock()
   273  	if height == cs.blockStore.Height() {
   274  		return cs.blockStore.LoadSeenCommit(height)
   275  	}
   276  	return cs.blockStore.LoadBlockCommit(height)
   277  }
   278  
   279  // OnStart implements service.Service.
   280  // It loads the latest state via the WAL, and starts the timeout and receive routines.
   281  func (cs *State) OnStart() error {
   282  	if err := cs.evsw.Start(); err != nil {
   283  		return err
   284  	}
   285  
   286  	// we may set the WAL in testing before calling Start,
   287  	// so only OpenWAL if its still the nilWAL
   288  	if _, ok := cs.wal.(nilWAL); ok {
   289  		walFile := cs.config.WalFile()
   290  		wal, err := cs.OpenWAL(walFile)
   291  		if err != nil {
   292  			cs.Logger.Error("Error loading State wal", "err", err.Error())
   293  			return err
   294  		}
   295  		cs.wal = wal
   296  	}
   297  
   298  	// we need the timeoutRoutine for replay so
   299  	// we don't block on the tick chan.
   300  	// NOTE: we will get a build up of garbage go routines
   301  	// firing on the tockChan until the receiveRoutine is started
   302  	// to deal with them (by that point, at most one will be valid)
   303  	if err := cs.timeoutTicker.Start(); err != nil {
   304  		return err
   305  	}
   306  
   307  	// we may have lost some votes if the process crashed
   308  	// reload from consensus log to catchup
   309  	if cs.doWALCatchup {
   310  		if err := cs.catchupReplay(cs.Height); err != nil {
   311  			// don't try to recover from data corruption error
   312  			if IsDataCorruptionError(err) {
   313  				cs.Logger.Error("Encountered corrupt WAL file", "err", err.Error())
   314  				cs.Logger.Error("Please repair the WAL file before restarting")
   315  				fmt.Println(`You can attempt to repair the WAL as follows:
   316  
   317  ----
   318  WALFILE=~/.tendermint/data/cs.wal/wal
   319  cp $WALFILE ${WALFILE}.bak # backup the file
   320  go run scripts/wal2json/main.go $WALFILE > wal.json # this will panic, but can be ignored
   321  rm $WALFILE # remove the corrupt file
   322  go run scripts/json2wal/main.go wal.json $WALFILE # rebuild the file without corruption
   323  ----`)
   324  
   325  				return err
   326  			}
   327  
   328  			cs.Logger.Error("Error on catchup replay. Proceeding to start State anyway", "err", err.Error())
   329  			// NOTE: if we ever do return an error here,
   330  			// make sure to stop the timeoutTicker
   331  		}
   332  	}
   333  
   334  	// now start the receiveRoutine
   335  	go cs.receiveRoutine(0)
   336  
   337  	// schedule the first round!
   338  	// use GetRoundState so we don't race the receiveRoutine for access
   339  	cs.scheduleRound0(cs.GetRoundState())
   340  
   341  	return nil
   342  }
   343  
   344  // timeoutRoutine: receive requests for timeouts on tickChan and fire timeouts on tockChan
   345  // receiveRoutine: serializes processing of proposoals, block parts, votes; coordinates state transitions
   346  func (cs *State) startRoutines(maxSteps int) {
   347  	err := cs.timeoutTicker.Start()
   348  	if err != nil {
   349  		cs.Logger.Error("Error starting timeout ticker", "err", err)
   350  		return
   351  	}
   352  	go cs.receiveRoutine(maxSteps)
   353  }
   354  
   355  // OnStop implements service.Service.
   356  func (cs *State) OnStop() {
   357  	cs.evsw.Stop()
   358  	cs.timeoutTicker.Stop()
   359  	// WAL is stopped in receiveRoutine.
   360  }
   361  
   362  // Wait waits for the the main routine to return.
   363  // NOTE: be sure to Stop() the event switch and drain
   364  // any event channels or this may deadlock
   365  func (cs *State) Wait() {
   366  	<-cs.done
   367  }
   368  
   369  // OpenWAL opens a file to log all consensus messages and timeouts for deterministic accountability
   370  func (cs *State) OpenWAL(walFile string) (WAL, error) {
   371  	wal, err := NewWAL(walFile)
   372  	if err != nil {
   373  		cs.Logger.Error("Failed to open WAL for consensus state", "wal", walFile, "err", err)
   374  		return nil, err
   375  	}
   376  	wal.SetLogger(cs.Logger.With("wal", walFile))
   377  	if err := wal.Start(); err != nil {
   378  		return nil, err
   379  	}
   380  	return wal, nil
   381  }
   382  
   383  //------------------------------------------------------------
   384  // Public interface for passing messages into the consensus state, possibly causing a state transition.
   385  // If peerID == "", the msg is considered internal.
   386  // Messages are added to the appropriate queue (peer or internal).
   387  // If the queue is full, the function may block.
   388  // TODO: should these return anything or let callers just use events?
   389  
   390  // AddVote inputs a vote.
   391  func (cs *State) AddVote(vote *types.Vote, peerID p2p.ID) (added bool, err error) {
   392  	if peerID == "" {
   393  		cs.internalMsgQueue <- msgInfo{&VoteMessage{vote}, ""}
   394  	} else {
   395  		cs.peerMsgQueue <- msgInfo{&VoteMessage{vote}, peerID}
   396  	}
   397  
   398  	// TODO: wait for event?!
   399  	return false, nil
   400  }
   401  
   402  // SetProposal inputs a proposal.
   403  func (cs *State) SetProposal(proposal *types.Proposal, peerID p2p.ID) error {
   404  
   405  	if peerID == "" {
   406  		cs.internalMsgQueue <- msgInfo{&ProposalMessage{proposal}, ""}
   407  	} else {
   408  		cs.peerMsgQueue <- msgInfo{&ProposalMessage{proposal}, peerID}
   409  	}
   410  
   411  	// TODO: wait for event?!
   412  	return nil
   413  }
   414  
   415  // AddProposalBlockPart inputs a part of the proposal block.
   416  func (cs *State) AddProposalBlockPart(height int64, round int, part *types.Part, peerID p2p.ID) error {
   417  
   418  	if peerID == "" {
   419  		cs.internalMsgQueue <- msgInfo{&BlockPartMessage{height, round, part}, ""}
   420  	} else {
   421  		cs.peerMsgQueue <- msgInfo{&BlockPartMessage{height, round, part}, peerID}
   422  	}
   423  
   424  	// TODO: wait for event?!
   425  	return nil
   426  }
   427  
   428  // SetProposalAndBlock inputs the proposal and all block parts.
   429  func (cs *State) SetProposalAndBlock(
   430  	proposal *types.Proposal,
   431  	block *types.Block,
   432  	parts *types.PartSet,
   433  	peerID p2p.ID,
   434  ) error {
   435  	if err := cs.SetProposal(proposal, peerID); err != nil {
   436  		return err
   437  	}
   438  	for i := 0; i < parts.Total(); i++ {
   439  		part := parts.GetPart(i)
   440  		if err := cs.AddProposalBlockPart(proposal.Height, proposal.Round, part, peerID); err != nil {
   441  			return err
   442  		}
   443  	}
   444  	return nil
   445  }
   446  
   447  //------------------------------------------------------------
   448  // internal functions for managing the state
   449  
   450  func (cs *State) updateHeight(height int64) {
   451  	cs.metrics.Height.Set(float64(height))
   452  	cs.Height = height
   453  }
   454  
   455  func (cs *State) updateRoundStep(round int, step cstypes.RoundStepType) {
   456  	cs.Round = round
   457  	cs.Step = step
   458  }
   459  
   460  // enterNewRound(height, 0) at cs.StartTime.
   461  func (cs *State) scheduleRound0(rs *cstypes.RoundState) {
   462  	//cs.Logger.Info("scheduleRound0", "now", tmtime.Now(), "startTime", cs.StartTime)
   463  	sleepDuration := rs.StartTime.Sub(tmtime.Now())
   464  	cs.scheduleTimeout(sleepDuration, rs.Height, 0, cstypes.RoundStepNewHeight)
   465  }
   466  
   467  // Attempt to schedule a timeout (by sending timeoutInfo on the tickChan)
   468  func (cs *State) scheduleTimeout(duration time.Duration, height int64, round int, step cstypes.RoundStepType) {
   469  	cs.timeoutTicker.ScheduleTimeout(timeoutInfo{duration, height, round, step})
   470  }
   471  
   472  // send a msg into the receiveRoutine regarding our own proposal, block part, or vote
   473  func (cs *State) sendInternalMessage(mi msgInfo) {
   474  	select {
   475  	case cs.internalMsgQueue <- mi:
   476  	default:
   477  		// NOTE: using the go-routine means our votes can
   478  		// be processed out of order.
   479  		// TODO: use CList here for strict determinism and
   480  		// attempt push to internalMsgQueue in receiveRoutine
   481  		cs.Logger.Info("Internal msg queue is full. Using a go-routine")
   482  		go func() { cs.internalMsgQueue <- mi }()
   483  	}
   484  }
   485  
   486  // Reconstruct LastCommit from SeenCommit, which we saved along with the block,
   487  // (which happens even before saving the state)
   488  func (cs *State) reconstructLastCommit(state sm.State) {
   489  	if state.LastBlockHeight == 0 {
   490  		return
   491  	}
   492  	seenCommit := cs.blockStore.LoadSeenCommit(state.LastBlockHeight)
   493  	if seenCommit == nil {
   494  		panic(fmt.Sprintf("Failed to reconstruct LastCommit: seen commit for height %v not found",
   495  			state.LastBlockHeight))
   496  	}
   497  	lastPrecommits := types.CommitToVoteSet(state.ChainID, seenCommit, state.LastValidators)
   498  	if !lastPrecommits.HasTwoThirdsMajority() {
   499  		panic("Failed to reconstruct LastCommit: Does not have +2/3 maj")
   500  	}
   501  	cs.LastCommit = lastPrecommits
   502  }
   503  
   504  // Updates State and increments height to match that of state.
   505  // The round becomes 0 and cs.Step becomes cstypes.RoundStepNewHeight.
   506  func (cs *State) updateToState(state sm.State) {
   507  	if cs.CommitRound > -1 && 0 < cs.Height && cs.Height != state.LastBlockHeight {
   508  		panic(fmt.Sprintf("updateToState() expected state height of %v but found %v",
   509  			cs.Height, state.LastBlockHeight))
   510  	}
   511  	if !cs.state.IsEmpty() && cs.state.LastBlockHeight+1 != cs.Height {
   512  		// This might happen when someone else is mutating cs.state.
   513  		// Someone forgot to pass in state.Copy() somewhere?!
   514  		panic(fmt.Sprintf("Inconsistent cs.state.LastBlockHeight+1 %v vs cs.Height %v",
   515  			cs.state.LastBlockHeight+1, cs.Height))
   516  	}
   517  
   518  	// If state isn't further out than cs.state, just ignore.
   519  	// This happens when SwitchToConsensus() is called in the reactor.
   520  	// We don't want to reset e.g. the Votes, but we still want to
   521  	// signal the new round step, because other services (eg. txNotifier)
   522  	// depend on having an up-to-date peer state!
   523  	if !cs.state.IsEmpty() && (state.LastBlockHeight <= cs.state.LastBlockHeight) {
   524  		cs.Logger.Info(
   525  			"Ignoring updateToState()",
   526  			"newHeight",
   527  			state.LastBlockHeight+1,
   528  			"oldHeight",
   529  			cs.state.LastBlockHeight+1)
   530  		cs.newStep()
   531  		return
   532  	}
   533  
   534  	// Reset fields based on state.
   535  	validators := state.Validators
   536  	lastPrecommits := (*types.VoteSet)(nil)
   537  	if cs.CommitRound > -1 && cs.Votes != nil {
   538  		if !cs.Votes.Precommits(cs.CommitRound).HasTwoThirdsMajority() {
   539  			panic("updateToState(state) called but last Precommit round didn't have +2/3")
   540  		}
   541  		lastPrecommits = cs.Votes.Precommits(cs.CommitRound)
   542  	}
   543  
   544  	// Next desired block height
   545  	height := state.LastBlockHeight + 1
   546  
   547  	// RoundState fields
   548  	cs.updateHeight(height)
   549  	cs.updateRoundStep(0, cstypes.RoundStepNewHeight)
   550  	if cs.CommitTime.IsZero() {
   551  		// "Now" makes it easier to sync up dev nodes.
   552  		// We add timeoutCommit to allow transactions
   553  		// to be gathered for the first block.
   554  		// And alternative solution that relies on clocks:
   555  		// cs.StartTime = state.LastBlockTime.Add(timeoutCommit)
   556  		cs.StartTime = cs.config.Commit(tmtime.Now())
   557  	} else {
   558  		cs.StartTime = cs.config.Commit(cs.CommitTime)
   559  	}
   560  
   561  	cs.Validators = validators
   562  	cs.Proposal = nil
   563  	cs.ProposalBlock = nil
   564  	cs.ProposalBlockParts = nil
   565  	cs.LockedRound = -1
   566  	cs.LockedBlock = nil
   567  	cs.LockedBlockParts = nil
   568  	cs.ValidRound = -1
   569  	cs.ValidBlock = nil
   570  	cs.ValidBlockParts = nil
   571  	cs.Votes = cstypes.NewHeightVoteSet(state.ChainID, height, validators)
   572  	cs.CommitRound = -1
   573  	cs.LastCommit = lastPrecommits
   574  	cs.LastValidators = state.LastValidators
   575  	cs.TriggeredTimeoutPrecommit = false
   576  
   577  	cs.state = state
   578  
   579  	// Finally, broadcast RoundState
   580  	cs.newStep()
   581  }
   582  
   583  func (cs *State) newStep() {
   584  	rs := cs.RoundStateEvent()
   585  	cs.wal.Write(rs)
   586  	cs.nSteps++
   587  	// newStep is called by updateToState in NewState before the eventBus is set!
   588  	if cs.eventBus != nil {
   589  		cs.eventBus.PublishEventNewRoundStep(rs)
   590  		cs.evsw.FireEvent(types.EventNewRoundStep, &cs.RoundState)
   591  	}
   592  }
   593  
   594  //-----------------------------------------
   595  // the main go routines
   596  
   597  // receiveRoutine handles messages which may cause state transitions.
   598  // it's argument (n) is the number of messages to process before exiting - use 0 to run forever
   599  // It keeps the RoundState and is the only thing that updates it.
   600  // Updates (state transitions) happen on timeouts, complete proposals, and 2/3 majorities.
   601  // State must be locked before any internal state is updated.
   602  func (cs *State) receiveRoutine(maxSteps int) {
   603  	onExit := func(cs *State) {
   604  		// NOTE: the internalMsgQueue may have signed messages from our
   605  		// priv_val that haven't hit the WAL, but its ok because
   606  		// priv_val tracks LastSig
   607  
   608  		// close wal now that we're done writing to it
   609  		cs.wal.Stop()
   610  		cs.wal.Wait()
   611  
   612  		close(cs.done)
   613  	}
   614  
   615  	defer func() {
   616  		if r := recover(); r != nil {
   617  			cs.Logger.Error("CONSENSUS FAILURE!!!", "err", r, "stack", string(debug.Stack()))
   618  			// stop gracefully
   619  			//
   620  			// NOTE: We most probably shouldn't be running any further when there is
   621  			// some unexpected panic. Some unknown error happened, and so we don't
   622  			// know if that will result in the validator signing an invalid thing. It
   623  			// might be worthwhile to explore a mechanism for manual resuming via
   624  			// some console or secure RPC system, but for now, halting the chain upon
   625  			// unexpected consensus bugs sounds like the better option.
   626  			onExit(cs)
   627  		}
   628  	}()
   629  
   630  	for {
   631  		if maxSteps > 0 {
   632  			if cs.nSteps >= maxSteps {
   633  				cs.Logger.Info("reached max steps. exiting receive routine")
   634  				cs.nSteps = 0
   635  				return
   636  			}
   637  		}
   638  		rs := cs.RoundState
   639  		var mi msgInfo
   640  
   641  		select {
   642  		case <-cs.txNotifier.TxsAvailable():
   643  			cs.handleTxsAvailable()
   644  		case mi = <-cs.peerMsgQueue:
   645  			cs.wal.Write(mi)
   646  			// handles proposals, block parts, votes
   647  			// may generate internal events (votes, complete proposals, 2/3 majorities)
   648  			cs.handleMsg(mi)
   649  		case mi = <-cs.internalMsgQueue:
   650  			err := cs.wal.WriteSync(mi) // NOTE: fsync
   651  			if err != nil {
   652  				panic(fmt.Sprintf("Failed to write %v msg to consensus wal due to %v. Check your FS and restart the node", mi, err))
   653  			}
   654  
   655  			if _, ok := mi.Msg.(*VoteMessage); ok {
   656  				// we actually want to simulate failing during
   657  				// the previous WriteSync, but this isn't easy to do.
   658  				// Equivalent would be to fail here and manually remove
   659  				// some bytes from the end of the wal.
   660  				fail.Fail() // XXX
   661  			}
   662  
   663  			// handles proposals, block parts, votes
   664  			cs.handleMsg(mi)
   665  		case ti := <-cs.timeoutTicker.Chan(): // tockChan:
   666  			cs.wal.Write(ti)
   667  			// if the timeout is relevant to the rs
   668  			// go to the next step
   669  			cs.handleTimeout(ti, rs)
   670  		case <-cs.Quit():
   671  			onExit(cs)
   672  			return
   673  		}
   674  	}
   675  }
   676  
   677  // state transitions on complete-proposal, 2/3-any, 2/3-one
   678  func (cs *State) handleMsg(mi msgInfo) {
   679  	cs.mtx.Lock()
   680  	defer cs.mtx.Unlock()
   681  
   682  	var (
   683  		added bool
   684  		err   error
   685  	)
   686  	msg, peerID := mi.Msg, mi.PeerID
   687  	switch msg := msg.(type) {
   688  	case *ProposalMessage:
   689  		// will not cause transition.
   690  		// once proposal is set, we can receive block parts
   691  		err = cs.setProposal(msg.Proposal)
   692  	case *BlockPartMessage:
   693  		// if the proposal is complete, we'll enterPrevote or tryFinalizeCommit
   694  		added, err = cs.addProposalBlockPart(msg, peerID)
   695  		if added {
   696  			cs.statsMsgQueue <- mi
   697  		}
   698  
   699  		if err != nil && msg.Round != cs.Round {
   700  			cs.Logger.Debug(
   701  				"Received block part from wrong round",
   702  				"height",
   703  				cs.Height,
   704  				"csRound",
   705  				cs.Round,
   706  				"blockRound",
   707  				msg.Round)
   708  			err = nil
   709  		}
   710  	case *VoteMessage:
   711  		// attempt to add the vote and dupeout the validator if its a duplicate signature
   712  		// if the vote gives us a 2/3-any or 2/3-one, we transition
   713  		added, err = cs.tryAddVote(msg.Vote, peerID)
   714  		if added {
   715  			cs.statsMsgQueue <- mi
   716  		}
   717  
   718  		// if err == ErrAddingVote {
   719  		// TODO: punish peer
   720  		// We probably don't want to stop the peer here. The vote does not
   721  		// necessarily comes from a malicious peer but can be just broadcasted by
   722  		// a typical peer.
   723  		// https://github.com/franono/tendermint/issues/1281
   724  		// }
   725  
   726  		// NOTE: the vote is broadcast to peers by the reactor listening
   727  		// for vote events
   728  
   729  		// TODO: If rs.Height == vote.Height && rs.Round < vote.Round,
   730  		// the peer is sending us CatchupCommit precommits.
   731  		// We could make note of this and help filter in broadcastHasVoteMessage().
   732  	default:
   733  		cs.Logger.Error("Unknown msg type", "type", reflect.TypeOf(msg))
   734  		return
   735  	}
   736  
   737  	if err != nil {
   738  		cs.Logger.Error("Error with msg", "height", cs.Height, "round", cs.Round,
   739  			"peer", peerID, "err", err, "msg", msg)
   740  	}
   741  }
   742  
   743  func (cs *State) handleTimeout(ti timeoutInfo, rs cstypes.RoundState) {
   744  	cs.Logger.Debug("Received tock", "timeout", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step)
   745  
   746  	// timeouts must be for current height, round, step
   747  	if ti.Height != rs.Height || ti.Round < rs.Round || (ti.Round == rs.Round && ti.Step < rs.Step) {
   748  		cs.Logger.Debug("Ignoring tock because we're ahead", "height", rs.Height, "round", rs.Round, "step", rs.Step)
   749  		return
   750  	}
   751  
   752  	// the timeout will now cause a state transition
   753  	cs.mtx.Lock()
   754  	defer cs.mtx.Unlock()
   755  
   756  	switch ti.Step {
   757  	case cstypes.RoundStepNewHeight:
   758  		// NewRound event fired from enterNewRound.
   759  		// XXX: should we fire timeout here (for timeout commit)?
   760  		cs.enterNewRound(ti.Height, 0)
   761  	case cstypes.RoundStepNewRound:
   762  		cs.enterPropose(ti.Height, 0)
   763  	case cstypes.RoundStepPropose:
   764  		cs.eventBus.PublishEventTimeoutPropose(cs.RoundStateEvent())
   765  		cs.enterPrevote(ti.Height, ti.Round)
   766  	case cstypes.RoundStepPrevoteWait:
   767  		cs.eventBus.PublishEventTimeoutWait(cs.RoundStateEvent())
   768  		cs.enterPrecommit(ti.Height, ti.Round)
   769  	case cstypes.RoundStepPrecommitWait:
   770  		cs.eventBus.PublishEventTimeoutWait(cs.RoundStateEvent())
   771  		cs.enterPrecommit(ti.Height, ti.Round)
   772  		cs.enterNewRound(ti.Height, ti.Round+1)
   773  	default:
   774  		panic(fmt.Sprintf("Invalid timeout step: %v", ti.Step))
   775  	}
   776  
   777  }
   778  
   779  func (cs *State) handleTxsAvailable() {
   780  	cs.mtx.Lock()
   781  	defer cs.mtx.Unlock()
   782  
   783  	// We only need to do this for round 0.
   784  	if cs.Round != 0 {
   785  		return
   786  	}
   787  
   788  	switch cs.Step {
   789  	case cstypes.RoundStepNewHeight: // timeoutCommit phase
   790  		if cs.needProofBlock(cs.Height) {
   791  			// enterPropose will be called by enterNewRound
   792  			return
   793  		}
   794  
   795  		// +1ms to ensure RoundStepNewRound timeout always happens after RoundStepNewHeight
   796  		timeoutCommit := cs.StartTime.Sub(tmtime.Now()) + 1*time.Millisecond
   797  		cs.scheduleTimeout(timeoutCommit, cs.Height, 0, cstypes.RoundStepNewRound)
   798  	case cstypes.RoundStepNewRound: // after timeoutCommit
   799  		cs.enterPropose(cs.Height, 0)
   800  	}
   801  }
   802  
   803  //-----------------------------------------------------------------------------
   804  // State functions
   805  // Used internally by handleTimeout and handleMsg to make state transitions
   806  
   807  // Enter: `timeoutNewHeight` by startTime (commitTime+timeoutCommit),
   808  // 	or, if SkipTimeoutCommit==true, after receiving all precommits from (height,round-1)
   809  // Enter: `timeoutPrecommits` after any +2/3 precommits from (height,round-1)
   810  // Enter: +2/3 precommits for nil at (height,round-1)
   811  // Enter: +2/3 prevotes any or +2/3 precommits for block or any from (height, round)
   812  // NOTE: cs.StartTime was already set for height.
   813  func (cs *State) enterNewRound(height int64, round int) {
   814  	logger := cs.Logger.With("height", height, "round", round)
   815  
   816  	if cs.Height != height || round < cs.Round || (cs.Round == round && cs.Step != cstypes.RoundStepNewHeight) {
   817  		logger.Debug(fmt.Sprintf(
   818  			"enterNewRound(%v/%v): Invalid args. Current step: %v/%v/%v",
   819  			height,
   820  			round,
   821  			cs.Height,
   822  			cs.Round,
   823  			cs.Step))
   824  		return
   825  	}
   826  
   827  	if now := tmtime.Now(); cs.StartTime.After(now) {
   828  		logger.Info("Need to set a buffer and log message here for sanity.", "startTime", cs.StartTime, "now", now)
   829  	}
   830  
   831  	logger.Info(fmt.Sprintf("enterNewRound(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
   832  
   833  	// Increment validators if necessary
   834  	validators := cs.Validators
   835  	if cs.Round < round {
   836  		validators = validators.Copy()
   837  		validators.IncrementProposerPriority(round - cs.Round)
   838  	}
   839  
   840  	// Setup new round
   841  	// we don't fire newStep for this step,
   842  	// but we fire an event, so update the round step first
   843  	cs.updateRoundStep(round, cstypes.RoundStepNewRound)
   844  	cs.Validators = validators
   845  	if round == 0 {
   846  		// We've already reset these upon new height,
   847  		// and meanwhile we might have received a proposal
   848  		// for round 0.
   849  	} else {
   850  		logger.Info("Resetting Proposal info")
   851  		cs.Proposal = nil
   852  		cs.ProposalBlock = nil
   853  		cs.ProposalBlockParts = nil
   854  	}
   855  	cs.Votes.SetRound(round + 1) // also track next round (round+1) to allow round-skipping
   856  	cs.TriggeredTimeoutPrecommit = false
   857  
   858  	cs.eventBus.PublishEventNewRound(cs.NewRoundEvent())
   859  	cs.metrics.Rounds.Set(float64(round))
   860  
   861  	// Wait for txs to be available in the mempool
   862  	// before we enterPropose in round 0. If the last block changed the app hash,
   863  	// we may need an empty "proof" block, and enterPropose immediately.
   864  	waitForTxs := cs.config.WaitForTxs() && round == 0 && !cs.needProofBlock(height)
   865  	if waitForTxs {
   866  		if cs.config.CreateEmptyBlocksInterval > 0 {
   867  			cs.scheduleTimeout(cs.config.CreateEmptyBlocksInterval, height, round,
   868  				cstypes.RoundStepNewRound)
   869  		}
   870  	} else {
   871  		cs.enterPropose(height, round)
   872  	}
   873  }
   874  
   875  // needProofBlock returns true on the first height (so the genesis app hash is signed right away)
   876  // and where the last block (height-1) caused the app hash to change
   877  func (cs *State) needProofBlock(height int64) bool {
   878  	if height == 1 {
   879  		return true
   880  	}
   881  
   882  	lastBlockMeta := cs.blockStore.LoadBlockMeta(height - 1)
   883  	if lastBlockMeta == nil {
   884  		panic(fmt.Sprintf("needProofBlock: last block meta for height %d not found", height-1))
   885  	}
   886  	return !bytes.Equal(cs.state.AppHash, lastBlockMeta.Header.AppHash)
   887  }
   888  
   889  // Enter (CreateEmptyBlocks): from enterNewRound(height,round)
   890  // Enter (CreateEmptyBlocks, CreateEmptyBlocksInterval > 0 ):
   891  // 		after enterNewRound(height,round), after timeout of CreateEmptyBlocksInterval
   892  // Enter (!CreateEmptyBlocks) : after enterNewRound(height,round), once txs are in the mempool
   893  func (cs *State) enterPropose(height int64, round int) {
   894  	logger := cs.Logger.With("height", height, "round", round)
   895  
   896  	if cs.Height != height || round < cs.Round || (cs.Round == round && cstypes.RoundStepPropose <= cs.Step) {
   897  		logger.Debug(fmt.Sprintf(
   898  			"enterPropose(%v/%v): Invalid args. Current step: %v/%v/%v",
   899  			height,
   900  			round,
   901  			cs.Height,
   902  			cs.Round,
   903  			cs.Step))
   904  		return
   905  	}
   906  	logger.Info(fmt.Sprintf("enterPropose(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
   907  
   908  	defer func() {
   909  		// Done enterPropose:
   910  		cs.updateRoundStep(round, cstypes.RoundStepPropose)
   911  		cs.newStep()
   912  
   913  		// If we have the whole proposal + POL, then goto Prevote now.
   914  		// else, we'll enterPrevote when the rest of the proposal is received (in AddProposalBlockPart),
   915  		// or else after timeoutPropose
   916  		if cs.isProposalComplete() {
   917  			cs.enterPrevote(height, cs.Round)
   918  		}
   919  	}()
   920  
   921  	// If we don't get the proposal and all block parts quick enough, enterPrevote
   922  	cs.scheduleTimeout(cs.config.Propose(round), height, round, cstypes.RoundStepPropose)
   923  
   924  	// Nothing more to do if we're not a validator
   925  	if cs.privValidator == nil {
   926  		logger.Debug("This node is not a validator")
   927  		return
   928  	}
   929  	logger.Debug("This node is a validator")
   930  
   931  	pubKey, err := cs.privValidator.GetPubKey()
   932  	if err != nil {
   933  		// If this node is a validator & proposer in the current round, it will
   934  		// miss the opportunity to create a block.
   935  		logger.Error("Error on retrival of pubkey", "err", err)
   936  		return
   937  	}
   938  	address := pubKey.Address()
   939  
   940  	// if not a validator, we're done
   941  	if !cs.Validators.HasAddress(address) {
   942  		logger.Debug("This node is not a validator", "addr", address, "vals", cs.Validators)
   943  		return
   944  	}
   945  
   946  	if cs.isProposer(address) {
   947  		logger.Info("enterPropose: Our turn to propose",
   948  			"proposer",
   949  			address,
   950  			"privValidator",
   951  			cs.privValidator)
   952  		cs.decideProposal(height, round)
   953  	} else {
   954  		logger.Info("enterPropose: Not our turn to propose",
   955  			"proposer",
   956  			cs.Validators.GetProposer().Address,
   957  			"privValidator",
   958  			cs.privValidator)
   959  	}
   960  }
   961  
   962  func (cs *State) isProposer(address []byte) bool {
   963  	return bytes.Equal(cs.Validators.GetProposer().Address, address)
   964  }
   965  
   966  func (cs *State) defaultDecideProposal(height int64, round int) {
   967  	var block *types.Block
   968  	var blockParts *types.PartSet
   969  
   970  	// Decide on block
   971  	if cs.ValidBlock != nil {
   972  		// If there is valid block, choose that.
   973  		block, blockParts = cs.ValidBlock, cs.ValidBlockParts
   974  	} else {
   975  		// Create a new proposal block from state/txs from the mempool.
   976  		block, blockParts = cs.createProposalBlock()
   977  		if block == nil {
   978  			return
   979  		}
   980  	}
   981  
   982  	// Flush the WAL. Otherwise, we may not recompute the same proposal to sign,
   983  	// and the privValidator will refuse to sign anything.
   984  	cs.wal.FlushAndSync()
   985  
   986  	// Make proposal
   987  	propBlockID := types.BlockID{Hash: block.Hash(), PartsHeader: blockParts.Header()}
   988  	proposal := types.NewProposal(height, round, cs.ValidRound, propBlockID)
   989  	if err := cs.privValidator.SignProposal(cs.state.ChainID, proposal); err == nil {
   990  
   991  		// send proposal and block parts on internal msg queue
   992  		cs.sendInternalMessage(msgInfo{&ProposalMessage{proposal}, ""})
   993  		for i := 0; i < blockParts.Total(); i++ {
   994  			part := blockParts.GetPart(i)
   995  			cs.sendInternalMessage(msgInfo{&BlockPartMessage{cs.Height, cs.Round, part}, ""})
   996  		}
   997  		cs.Logger.Info("Signed proposal", "height", height, "round", round, "proposal", proposal)
   998  		cs.Logger.Debug(fmt.Sprintf("Signed proposal block: %v", block))
   999  	} else if !cs.replayMode {
  1000  		cs.Logger.Error("enterPropose: Error signing proposal", "height", height, "round", round, "err", err)
  1001  	}
  1002  }
  1003  
  1004  // Returns true if the proposal block is complete &&
  1005  // (if POLRound was proposed, we have +2/3 prevotes from there).
  1006  func (cs *State) isProposalComplete() bool {
  1007  	if cs.Proposal == nil || cs.ProposalBlock == nil {
  1008  		return false
  1009  	}
  1010  	// we have the proposal. if there's a POLRound,
  1011  	// make sure we have the prevotes from it too
  1012  	if cs.Proposal.POLRound < 0 {
  1013  		return true
  1014  	}
  1015  	// if this is false the proposer is lying or we haven't received the POL yet
  1016  	return cs.Votes.Prevotes(cs.Proposal.POLRound).HasTwoThirdsMajority()
  1017  
  1018  }
  1019  
  1020  // Create the next block to propose and return it. Returns nil block upon error.
  1021  //
  1022  // We really only need to return the parts, but the block is returned for
  1023  // convenience so we can log the proposal block.
  1024  //
  1025  // NOTE: keep it side-effect free for clarity.
  1026  // CONTRACT: cs.privValidator is not nil.
  1027  func (cs *State) createProposalBlock() (block *types.Block, blockParts *types.PartSet) {
  1028  	var commit *types.Commit
  1029  	switch {
  1030  	case cs.Height == 1:
  1031  		// We're creating a proposal for the first block.
  1032  		// The commit is empty, but not nil.
  1033  		commit = types.NewCommit(0, 0, types.BlockID{}, nil)
  1034  	case cs.LastCommit.HasTwoThirdsMajority():
  1035  		// Make the commit from LastCommit
  1036  		commit = cs.LastCommit.MakeCommit()
  1037  	default: // This shouldn't happen.
  1038  		cs.Logger.Error("enterPropose: Cannot propose anything: No commit for the previous block")
  1039  		return
  1040  	}
  1041  
  1042  	if cs.privValidator == nil {
  1043  		panic("entered createProposalBlock with privValidator being nil")
  1044  	}
  1045  	pubKey, err := cs.privValidator.GetPubKey()
  1046  	if err != nil {
  1047  		// If this node is a validator & proposer in the current round, it will
  1048  		// miss the opportunity to create a block.
  1049  		cs.Logger.Error("Error on retrival of pubkey", "err", err)
  1050  		return
  1051  	}
  1052  	proposerAddr := pubKey.Address()
  1053  
  1054  	return cs.blockExec.CreateProposalBlock(cs.Height, cs.state, commit, proposerAddr)
  1055  }
  1056  
  1057  // Enter: `timeoutPropose` after entering Propose.
  1058  // Enter: proposal block and POL is ready.
  1059  // Prevote for LockedBlock if we're locked, or ProposalBlock if valid.
  1060  // Otherwise vote nil.
  1061  func (cs *State) enterPrevote(height int64, round int) {
  1062  	if cs.Height != height || round < cs.Round || (cs.Round == round && cstypes.RoundStepPrevote <= cs.Step) {
  1063  		cs.Logger.Debug(fmt.Sprintf(
  1064  			"enterPrevote(%v/%v): Invalid args. Current step: %v/%v/%v",
  1065  			height,
  1066  			round,
  1067  			cs.Height,
  1068  			cs.Round,
  1069  			cs.Step))
  1070  		return
  1071  	}
  1072  
  1073  	defer func() {
  1074  		// Done enterPrevote:
  1075  		cs.updateRoundStep(round, cstypes.RoundStepPrevote)
  1076  		cs.newStep()
  1077  	}()
  1078  
  1079  	cs.Logger.Info(fmt.Sprintf("enterPrevote(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  1080  
  1081  	// Sign and broadcast vote as necessary
  1082  	cs.doPrevote(height, round)
  1083  
  1084  	// Once `addVote` hits any +2/3 prevotes, we will go to PrevoteWait
  1085  	// (so we have more time to try and collect +2/3 prevotes for a single block)
  1086  }
  1087  
  1088  func (cs *State) defaultDoPrevote(height int64, round int) {
  1089  	logger := cs.Logger.With("height", height, "round", round)
  1090  
  1091  	// If a block is locked, prevote that.
  1092  	if cs.LockedBlock != nil {
  1093  		logger.Info("enterPrevote: Already locked on a block, prevoting locked block")
  1094  		cs.signAddVote(types.PrevoteType, cs.LockedBlock.Hash(), cs.LockedBlockParts.Header())
  1095  		return
  1096  	}
  1097  
  1098  	// If ProposalBlock is nil, prevote nil.
  1099  	if cs.ProposalBlock == nil {
  1100  		logger.Info("enterPrevote: ProposalBlock is nil")
  1101  		cs.signAddVote(types.PrevoteType, nil, types.PartSetHeader{})
  1102  		return
  1103  	}
  1104  
  1105  	// Validate proposal block
  1106  	err := cs.blockExec.ValidateBlock(cs.state, cs.ProposalBlock)
  1107  	if err != nil {
  1108  		// ProposalBlock is invalid, prevote nil.
  1109  		logger.Error("enterPrevote: ProposalBlock is invalid", "err", err)
  1110  		cs.signAddVote(types.PrevoteType, nil, types.PartSetHeader{})
  1111  		return
  1112  	}
  1113  
  1114  	// Prevote cs.ProposalBlock
  1115  	// NOTE: the proposal signature is validated when it is received,
  1116  	// and the proposal block parts are validated as they are received (against the merkle hash in the proposal)
  1117  	logger.Info("enterPrevote: ProposalBlock is valid")
  1118  	cs.signAddVote(types.PrevoteType, cs.ProposalBlock.Hash(), cs.ProposalBlockParts.Header())
  1119  }
  1120  
  1121  // Enter: any +2/3 prevotes at next round.
  1122  func (cs *State) enterPrevoteWait(height int64, round int) {
  1123  	logger := cs.Logger.With("height", height, "round", round)
  1124  
  1125  	if cs.Height != height || round < cs.Round || (cs.Round == round && cstypes.RoundStepPrevoteWait <= cs.Step) {
  1126  		logger.Debug(fmt.Sprintf(
  1127  			"enterPrevoteWait(%v/%v): Invalid args. Current step: %v/%v/%v",
  1128  			height,
  1129  			round,
  1130  			cs.Height,
  1131  			cs.Round,
  1132  			cs.Step))
  1133  		return
  1134  	}
  1135  	if !cs.Votes.Prevotes(round).HasTwoThirdsAny() {
  1136  		panic(fmt.Sprintf("enterPrevoteWait(%v/%v), but Prevotes does not have any +2/3 votes", height, round))
  1137  	}
  1138  	logger.Info(fmt.Sprintf("enterPrevoteWait(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  1139  
  1140  	defer func() {
  1141  		// Done enterPrevoteWait:
  1142  		cs.updateRoundStep(round, cstypes.RoundStepPrevoteWait)
  1143  		cs.newStep()
  1144  	}()
  1145  
  1146  	// Wait for some more prevotes; enterPrecommit
  1147  	cs.scheduleTimeout(cs.config.Prevote(round), height, round, cstypes.RoundStepPrevoteWait)
  1148  }
  1149  
  1150  // Enter: `timeoutPrevote` after any +2/3 prevotes.
  1151  // Enter: `timeoutPrecommit` after any +2/3 precommits.
  1152  // Enter: +2/3 precomits for block or nil.
  1153  // Lock & precommit the ProposalBlock if we have enough prevotes for it (a POL in this round)
  1154  // else, unlock an existing lock and precommit nil if +2/3 of prevotes were nil,
  1155  // else, precommit nil otherwise.
  1156  func (cs *State) enterPrecommit(height int64, round int) {
  1157  	logger := cs.Logger.With("height", height, "round", round)
  1158  
  1159  	if cs.Height != height || round < cs.Round || (cs.Round == round && cstypes.RoundStepPrecommit <= cs.Step) {
  1160  		logger.Debug(fmt.Sprintf(
  1161  			"enterPrecommit(%v/%v): Invalid args. Current step: %v/%v/%v",
  1162  			height,
  1163  			round,
  1164  			cs.Height,
  1165  			cs.Round,
  1166  			cs.Step))
  1167  		return
  1168  	}
  1169  
  1170  	logger.Info(fmt.Sprintf("enterPrecommit(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  1171  
  1172  	defer func() {
  1173  		// Done enterPrecommit:
  1174  		cs.updateRoundStep(round, cstypes.RoundStepPrecommit)
  1175  		cs.newStep()
  1176  	}()
  1177  
  1178  	// check for a polka
  1179  	blockID, ok := cs.Votes.Prevotes(round).TwoThirdsMajority()
  1180  
  1181  	// If we don't have a polka, we must precommit nil.
  1182  	if !ok {
  1183  		if cs.LockedBlock != nil {
  1184  			logger.Info("enterPrecommit: No +2/3 prevotes during enterPrecommit while we're locked. Precommitting nil")
  1185  		} else {
  1186  			logger.Info("enterPrecommit: No +2/3 prevotes during enterPrecommit. Precommitting nil.")
  1187  		}
  1188  		cs.signAddVote(types.PrecommitType, nil, types.PartSetHeader{})
  1189  		return
  1190  	}
  1191  
  1192  	// At this point +2/3 prevoted for a particular block or nil.
  1193  	cs.eventBus.PublishEventPolka(cs.RoundStateEvent())
  1194  
  1195  	// the latest POLRound should be this round.
  1196  	polRound, _ := cs.Votes.POLInfo()
  1197  	if polRound < round {
  1198  		panic(fmt.Sprintf("This POLRound should be %v but got %v", round, polRound))
  1199  	}
  1200  
  1201  	// +2/3 prevoted nil. Unlock and precommit nil.
  1202  	if len(blockID.Hash) == 0 {
  1203  		if cs.LockedBlock == nil {
  1204  			logger.Info("enterPrecommit: +2/3 prevoted for nil.")
  1205  		} else {
  1206  			logger.Info("enterPrecommit: +2/3 prevoted for nil. Unlocking")
  1207  			cs.LockedRound = -1
  1208  			cs.LockedBlock = nil
  1209  			cs.LockedBlockParts = nil
  1210  			cs.eventBus.PublishEventUnlock(cs.RoundStateEvent())
  1211  		}
  1212  		cs.signAddVote(types.PrecommitType, nil, types.PartSetHeader{})
  1213  		return
  1214  	}
  1215  
  1216  	// At this point, +2/3 prevoted for a particular block.
  1217  
  1218  	// If we're already locked on that block, precommit it, and update the LockedRound
  1219  	if cs.LockedBlock.HashesTo(blockID.Hash) {
  1220  		logger.Info("enterPrecommit: +2/3 prevoted locked block. Relocking")
  1221  		cs.LockedRound = round
  1222  		cs.eventBus.PublishEventRelock(cs.RoundStateEvent())
  1223  		cs.signAddVote(types.PrecommitType, blockID.Hash, blockID.PartsHeader)
  1224  		return
  1225  	}
  1226  
  1227  	// If +2/3 prevoted for proposal block, stage and precommit it
  1228  	if cs.ProposalBlock.HashesTo(blockID.Hash) {
  1229  		logger.Info("enterPrecommit: +2/3 prevoted proposal block. Locking", "hash", blockID.Hash)
  1230  		// Validate the block.
  1231  		if err := cs.blockExec.ValidateBlock(cs.state, cs.ProposalBlock); err != nil {
  1232  			panic(fmt.Sprintf("enterPrecommit: +2/3 prevoted for an invalid block: %v", err))
  1233  		}
  1234  		cs.LockedRound = round
  1235  		cs.LockedBlock = cs.ProposalBlock
  1236  		cs.LockedBlockParts = cs.ProposalBlockParts
  1237  		cs.eventBus.PublishEventLock(cs.RoundStateEvent())
  1238  		cs.signAddVote(types.PrecommitType, blockID.Hash, blockID.PartsHeader)
  1239  		return
  1240  	}
  1241  
  1242  	// There was a polka in this round for a block we don't have.
  1243  	// Fetch that block, unlock, and precommit nil.
  1244  	// The +2/3 prevotes for this round is the POL for our unlock.
  1245  	logger.Info("enterPrecommit: +2/3 prevotes for a block we don't have. Voting nil", "blockID", blockID)
  1246  	cs.LockedRound = -1
  1247  	cs.LockedBlock = nil
  1248  	cs.LockedBlockParts = nil
  1249  	if !cs.ProposalBlockParts.HasHeader(blockID.PartsHeader) {
  1250  		cs.ProposalBlock = nil
  1251  		cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartsHeader)
  1252  	}
  1253  	cs.eventBus.PublishEventUnlock(cs.RoundStateEvent())
  1254  	cs.signAddVote(types.PrecommitType, nil, types.PartSetHeader{})
  1255  }
  1256  
  1257  func (cs *State) savePOLC(round int, blockID types.BlockID) {
  1258  	// polc must be for rounds greater than 0
  1259  	if round == 0 {
  1260  		return
  1261  	}
  1262  	pubKey, err := cs.privValidator.GetPubKey()
  1263  	if err != nil {
  1264  		cs.Logger.Error("Error on retrieval of pubkey", "err", err)
  1265  		return
  1266  	}
  1267  	polc, err := types.MakePOLCFromVoteSet(cs.Votes.Prevotes(round), pubKey, blockID)
  1268  	if err != nil {
  1269  		cs.Logger.Error("Error on forming POLC", "err", err)
  1270  		return
  1271  	}
  1272  	err = cs.evpool.AddPOLC(polc)
  1273  	if err != nil {
  1274  		cs.Logger.Error("Error on saving POLC", "err", err)
  1275  		return
  1276  	}
  1277  	cs.Logger.Info("Saved POLC to evidence pool", "round", round, "height", polc.Height())
  1278  }
  1279  
  1280  // Enter: any +2/3 precommits for next round.
  1281  func (cs *State) enterPrecommitWait(height int64, round int) {
  1282  	logger := cs.Logger.With("height", height, "round", round)
  1283  
  1284  	if cs.Height != height || round < cs.Round || (cs.Round == round && cs.TriggeredTimeoutPrecommit) {
  1285  		logger.Debug(
  1286  			fmt.Sprintf(
  1287  				"enterPrecommitWait(%v/%v): Invalid args. "+
  1288  					"Current state is Height/Round: %v/%v/, TriggeredTimeoutPrecommit:%v",
  1289  				height, round, cs.Height, cs.Round, cs.TriggeredTimeoutPrecommit))
  1290  		return
  1291  	}
  1292  	if !cs.Votes.Precommits(round).HasTwoThirdsAny() {
  1293  		panic(fmt.Sprintf("enterPrecommitWait(%v/%v), but Precommits does not have any +2/3 votes", height, round))
  1294  	}
  1295  	logger.Info(fmt.Sprintf("enterPrecommitWait(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
  1296  
  1297  	defer func() {
  1298  		// Done enterPrecommitWait:
  1299  		cs.TriggeredTimeoutPrecommit = true
  1300  		cs.newStep()
  1301  	}()
  1302  
  1303  	// Wait for some more precommits; enterNewRound
  1304  	cs.scheduleTimeout(cs.config.Precommit(round), height, round, cstypes.RoundStepPrecommitWait)
  1305  }
  1306  
  1307  // Enter: +2/3 precommits for block
  1308  func (cs *State) enterCommit(height int64, commitRound int) {
  1309  	logger := cs.Logger.With("height", height, "commitRound", commitRound)
  1310  
  1311  	if cs.Height != height || cstypes.RoundStepCommit <= cs.Step {
  1312  		logger.Debug(fmt.Sprintf(
  1313  			"enterCommit(%v/%v): Invalid args. Current step: %v/%v/%v",
  1314  			height,
  1315  			commitRound,
  1316  			cs.Height,
  1317  			cs.Round,
  1318  			cs.Step))
  1319  		return
  1320  	}
  1321  	logger.Info(fmt.Sprintf("enterCommit(%v/%v). Current: %v/%v/%v", height, commitRound, cs.Height, cs.Round, cs.Step))
  1322  
  1323  	defer func() {
  1324  		// Done enterCommit:
  1325  		// keep cs.Round the same, commitRound points to the right Precommits set.
  1326  		cs.updateRoundStep(cs.Round, cstypes.RoundStepCommit)
  1327  		cs.CommitRound = commitRound
  1328  		cs.CommitTime = tmtime.Now()
  1329  		cs.newStep()
  1330  
  1331  		// Maybe finalize immediately.
  1332  		cs.tryFinalizeCommit(height)
  1333  	}()
  1334  
  1335  	blockID, ok := cs.Votes.Precommits(commitRound).TwoThirdsMajority()
  1336  	if !ok {
  1337  		panic("RunActionCommit() expects +2/3 precommits")
  1338  	}
  1339  
  1340  	// The Locked* fields no longer matter.
  1341  	// Move them over to ProposalBlock if they match the commit hash,
  1342  	// otherwise they'll be cleared in updateToState.
  1343  	if cs.LockedBlock.HashesTo(blockID.Hash) {
  1344  		logger.Info("Commit is for locked block. Set ProposalBlock=LockedBlock", "blockHash", blockID.Hash)
  1345  		cs.ProposalBlock = cs.LockedBlock
  1346  		cs.ProposalBlockParts = cs.LockedBlockParts
  1347  	}
  1348  
  1349  	// If we don't have the block being committed, set up to get it.
  1350  	if !cs.ProposalBlock.HashesTo(blockID.Hash) {
  1351  		if !cs.ProposalBlockParts.HasHeader(blockID.PartsHeader) {
  1352  			logger.Info(
  1353  				"Commit is for a block we don't know about. Set ProposalBlock=nil",
  1354  				"proposal",
  1355  				cs.ProposalBlock.Hash(),
  1356  				"commit",
  1357  				blockID.Hash)
  1358  			// We're getting the wrong block.
  1359  			// Set up ProposalBlockParts and keep waiting.
  1360  			cs.ProposalBlock = nil
  1361  			cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartsHeader)
  1362  			cs.eventBus.PublishEventValidBlock(cs.RoundStateEvent())
  1363  			cs.evsw.FireEvent(types.EventValidBlock, &cs.RoundState)
  1364  		}
  1365  		// else {
  1366  		// We just need to keep waiting.
  1367  		// }
  1368  	}
  1369  }
  1370  
  1371  // If we have the block AND +2/3 commits for it, finalize.
  1372  func (cs *State) tryFinalizeCommit(height int64) {
  1373  	logger := cs.Logger.With("height", height)
  1374  
  1375  	if cs.Height != height {
  1376  		panic(fmt.Sprintf("tryFinalizeCommit() cs.Height: %v vs height: %v", cs.Height, height))
  1377  	}
  1378  
  1379  	blockID, ok := cs.Votes.Precommits(cs.CommitRound).TwoThirdsMajority()
  1380  	if !ok || len(blockID.Hash) == 0 {
  1381  		logger.Error("Attempt to finalize failed. There was no +2/3 majority, or +2/3 was for <nil>.")
  1382  		return
  1383  	}
  1384  	if !cs.ProposalBlock.HashesTo(blockID.Hash) {
  1385  		// TODO: this happens every time if we're not a validator (ugly logs)
  1386  		// TODO: ^^ wait, why does it matter that we're a validator?
  1387  		logger.Info(
  1388  			"Attempt to finalize failed. We don't have the commit block.",
  1389  			"proposal-block",
  1390  			cs.ProposalBlock.Hash(),
  1391  			"commit-block",
  1392  			blockID.Hash)
  1393  		return
  1394  	}
  1395  
  1396  	//	go
  1397  	cs.finalizeCommit(height)
  1398  }
  1399  
  1400  // Increment height and goto cstypes.RoundStepNewHeight
  1401  func (cs *State) finalizeCommit(height int64) {
  1402  	if cs.Height != height || cs.Step != cstypes.RoundStepCommit {
  1403  		cs.Logger.Debug(fmt.Sprintf(
  1404  			"finalizeCommit(%v): Invalid args. Current step: %v/%v/%v",
  1405  			height,
  1406  			cs.Height,
  1407  			cs.Round,
  1408  			cs.Step))
  1409  		return
  1410  	}
  1411  
  1412  	blockID, ok := cs.Votes.Precommits(cs.CommitRound).TwoThirdsMajority()
  1413  	block, blockParts := cs.ProposalBlock, cs.ProposalBlockParts
  1414  
  1415  	if !ok {
  1416  		panic("Cannot finalizeCommit, commit does not have two thirds majority")
  1417  	}
  1418  	if !blockParts.HasHeader(blockID.PartsHeader) {
  1419  		panic("Expected ProposalBlockParts header to be commit header")
  1420  	}
  1421  	if !block.HashesTo(blockID.Hash) {
  1422  		panic("Cannot finalizeCommit, ProposalBlock does not hash to commit hash")
  1423  	}
  1424  	if err := cs.blockExec.ValidateBlock(cs.state, block); err != nil {
  1425  		panic(fmt.Errorf("+2/3 committed an invalid block: %w", err))
  1426  	}
  1427  
  1428  	cs.Logger.Info("Finalizing commit of block with N txs",
  1429  		"height", block.Height,
  1430  		"hash", block.Hash(),
  1431  		"root", block.AppHash,
  1432  		"N", len(block.Txs))
  1433  	cs.Logger.Info(fmt.Sprintf("%v", block))
  1434  
  1435  	fail.Fail() // XXX
  1436  
  1437  	// Save to blockStore.
  1438  	if cs.blockStore.Height() < block.Height {
  1439  		// NOTE: the seenCommit is local justification to commit this block,
  1440  		// but may differ from the LastCommit included in the next block
  1441  		precommits := cs.Votes.Precommits(cs.CommitRound)
  1442  		seenCommit := precommits.MakeCommit()
  1443  		cs.blockStore.SaveBlock(block, blockParts, seenCommit)
  1444  	} else {
  1445  		// Happens during replay if we already saved the block but didn't commit
  1446  		cs.Logger.Info("Calling finalizeCommit on already stored block", "height", block.Height)
  1447  	}
  1448  
  1449  	fail.Fail() // XXX
  1450  
  1451  	// Write EndHeightMessage{} for this height, implying that the blockstore
  1452  	// has saved the block.
  1453  	//
  1454  	// If we crash before writing this EndHeightMessage{}, we will recover by
  1455  	// running ApplyBlock during the ABCI handshake when we restart.  If we
  1456  	// didn't save the block to the blockstore before writing
  1457  	// EndHeightMessage{}, we'd have to change WAL replay -- currently it
  1458  	// complains about replaying for heights where an #ENDHEIGHT entry already
  1459  	// exists.
  1460  	//
  1461  	// Either way, the State should not be resumed until we
  1462  	// successfully call ApplyBlock (ie. later here, or in Handshake after
  1463  	// restart).
  1464  	endMsg := EndHeightMessage{height}
  1465  	if err := cs.wal.WriteSync(endMsg); err != nil { // NOTE: fsync
  1466  		panic(fmt.Sprintf("Failed to write %v msg to consensus wal due to %v. Check your FS and restart the node",
  1467  			endMsg, err))
  1468  	}
  1469  
  1470  	fail.Fail() // XXX
  1471  
  1472  	// Create a copy of the state for staging and an event cache for txs.
  1473  	stateCopy := cs.state.Copy()
  1474  
  1475  	// Execute and commit the block, update and save the state, and update the mempool.
  1476  	// NOTE The block.AppHash wont reflect these txs until the next block.
  1477  	var err error
  1478  	var retainHeight int64
  1479  	stateCopy, retainHeight, err = cs.blockExec.ApplyBlock(
  1480  		stateCopy,
  1481  		types.BlockID{Hash: block.Hash(), PartsHeader: blockParts.Header()},
  1482  		block)
  1483  	if err != nil {
  1484  		cs.Logger.Error("Error on ApplyBlock. Did the application crash? Please restart tendermint", "err", err)
  1485  		err := tmos.Kill()
  1486  		if err != nil {
  1487  			cs.Logger.Error("Failed to kill this process - please do so manually", "err", err)
  1488  		}
  1489  		return
  1490  	}
  1491  
  1492  	fail.Fail() // XXX
  1493  
  1494  	// Prune old heights, if requested by ABCI app.
  1495  	if retainHeight > 0 {
  1496  		pruned, err := cs.pruneBlocks(retainHeight)
  1497  		if err != nil {
  1498  			cs.Logger.Error("Failed to prune blocks", "retainHeight", retainHeight, "err", err)
  1499  		} else {
  1500  			cs.Logger.Info("Pruned blocks", "pruned", pruned, "retainHeight", retainHeight)
  1501  		}
  1502  	}
  1503  
  1504  	// must be called before we update state
  1505  	cs.recordMetrics(height, block)
  1506  
  1507  	// NewHeightStep!
  1508  	cs.updateToState(stateCopy)
  1509  
  1510  	fail.Fail() // XXX
  1511  
  1512  	// cs.StartTime is already set.
  1513  	// Schedule Round0 to start soon.
  1514  	cs.scheduleRound0(&cs.RoundState)
  1515  
  1516  	// By here,
  1517  	// * cs.Height has been increment to height+1
  1518  	// * cs.Step is now cstypes.RoundStepNewHeight
  1519  	// * cs.StartTime is set to when we will start round0.
  1520  }
  1521  
  1522  func (cs *State) pruneBlocks(retainHeight int64) (uint64, error) {
  1523  	base := cs.blockStore.Base()
  1524  	if retainHeight <= base {
  1525  		return 0, nil
  1526  	}
  1527  	pruned, err := cs.blockStore.PruneBlocks(retainHeight)
  1528  	if err != nil {
  1529  		return 0, fmt.Errorf("failed to prune block store: %w", err)
  1530  	}
  1531  	err = sm.PruneStates(cs.blockExec.DB(), base, retainHeight)
  1532  	if err != nil {
  1533  		return 0, fmt.Errorf("failed to prune state database: %w", err)
  1534  	}
  1535  	return pruned, nil
  1536  }
  1537  
  1538  func (cs *State) recordMetrics(height int64, block *types.Block) {
  1539  	cs.metrics.Validators.Set(float64(cs.Validators.Size()))
  1540  	cs.metrics.ValidatorsPower.Set(float64(cs.Validators.TotalVotingPower()))
  1541  
  1542  	var (
  1543  		missingValidators      int
  1544  		missingValidatorsPower int64
  1545  	)
  1546  	// height=0 -> MissingValidators and MissingValidatorsPower are both 0.
  1547  	// Remember that the first LastCommit is intentionally empty, so it's not
  1548  	// fair to increment missing validators number.
  1549  	if height > 1 {
  1550  		// Sanity check that commit size matches validator set size - only applies
  1551  		// after first block.
  1552  		var (
  1553  			commitSize = block.LastCommit.Size()
  1554  			valSetLen  = len(cs.LastValidators.Validators)
  1555  		)
  1556  		if commitSize != valSetLen {
  1557  			panic(fmt.Sprintf("commit size (%d) doesn't match valset length (%d) at height %d\n\n%v\n\n%v",
  1558  				commitSize, valSetLen, block.Height, block.LastCommit.Signatures, cs.LastValidators.Validators))
  1559  		}
  1560  
  1561  		for i, val := range cs.LastValidators.Validators {
  1562  			commitSig := block.LastCommit.Signatures[i]
  1563  			if commitSig.Absent() {
  1564  				missingValidators++
  1565  				missingValidatorsPower += val.VotingPower
  1566  			}
  1567  
  1568  			if cs.privValidator != nil {
  1569  				pubKey, err := cs.privValidator.GetPubKey()
  1570  				if err != nil {
  1571  					// Metrics won't be updated, but it's not critical.
  1572  					cs.Logger.Error("Error on retrival of pubkey", "err", err)
  1573  					continue
  1574  				}
  1575  
  1576  				if bytes.Equal(val.Address, pubKey.Address()) {
  1577  					label := []string{
  1578  						"validator_address", val.Address.String(),
  1579  					}
  1580  					cs.metrics.ValidatorPower.With(label...).Set(float64(val.VotingPower))
  1581  					if commitSig.ForBlock() {
  1582  						cs.metrics.ValidatorLastSignedHeight.With(label...).Set(float64(height))
  1583  					} else {
  1584  						cs.metrics.ValidatorMissedBlocks.With(label...).Add(float64(1))
  1585  					}
  1586  				}
  1587  			}
  1588  		}
  1589  	}
  1590  	cs.metrics.MissingValidators.Set(float64(missingValidators))
  1591  	cs.metrics.MissingValidatorsPower.Set(float64(missingValidatorsPower))
  1592  
  1593  	cs.metrics.ByzantineValidators.Set(float64(len(block.Evidence.Evidence)))
  1594  	byzantineValidatorsPower := int64(0)
  1595  	for _, ev := range block.Evidence.Evidence {
  1596  		if _, val := cs.Validators.GetByAddress(ev.Address()); val != nil {
  1597  			byzantineValidatorsPower += val.VotingPower
  1598  		}
  1599  	}
  1600  	cs.metrics.ByzantineValidatorsPower.Set(float64(byzantineValidatorsPower))
  1601  
  1602  	if height > 1 {
  1603  		lastBlockMeta := cs.blockStore.LoadBlockMeta(height - 1)
  1604  		if lastBlockMeta != nil {
  1605  			cs.metrics.BlockIntervalSeconds.Set(
  1606  				block.Time.Sub(lastBlockMeta.Header.Time).Seconds(),
  1607  			)
  1608  		}
  1609  	}
  1610  
  1611  	cs.metrics.NumTxs.Set(float64(len(block.Data.Txs)))
  1612  	cs.metrics.TotalTxs.Add(float64(len(block.Data.Txs)))
  1613  	cs.metrics.BlockSizeBytes.Set(float64(block.Size()))
  1614  	cs.metrics.CommittedHeight.Set(float64(block.Height))
  1615  }
  1616  
  1617  //-----------------------------------------------------------------------------
  1618  
  1619  func (cs *State) defaultSetProposal(proposal *types.Proposal) error {
  1620  	// Already have one
  1621  	// TODO: possibly catch double proposals
  1622  	if cs.Proposal != nil {
  1623  		return nil
  1624  	}
  1625  
  1626  	// Does not apply
  1627  	if proposal.Height != cs.Height || proposal.Round != cs.Round {
  1628  		return nil
  1629  	}
  1630  
  1631  	// Verify POLRound, which must be -1 or in range [0, proposal.Round).
  1632  	if proposal.POLRound < -1 ||
  1633  		(proposal.POLRound >= 0 && proposal.POLRound >= proposal.Round) {
  1634  		return ErrInvalidProposalPOLRound
  1635  	}
  1636  
  1637  	// Verify signature
  1638  	if !cs.Validators.GetProposer().PubKey.VerifyBytes(proposal.SignBytes(cs.state.ChainID), proposal.Signature) {
  1639  		return ErrInvalidProposalSignature
  1640  	}
  1641  
  1642  	cs.Proposal = proposal
  1643  	// We don't update cs.ProposalBlockParts if it is already set.
  1644  	// This happens if we're already in cstypes.RoundStepCommit or if there is a valid block in the current round.
  1645  	// TODO: We can check if Proposal is for a different block as this is a sign of misbehavior!
  1646  	if cs.ProposalBlockParts == nil {
  1647  		cs.ProposalBlockParts = types.NewPartSetFromHeader(proposal.BlockID.PartsHeader)
  1648  	}
  1649  	cs.Logger.Info("Received proposal", "proposal", proposal)
  1650  	return nil
  1651  }
  1652  
  1653  // NOTE: block is not necessarily valid.
  1654  // Asynchronously triggers either enterPrevote (before we timeout of propose) or tryFinalizeCommit,
  1655  // once we have the full block.
  1656  func (cs *State) addProposalBlockPart(msg *BlockPartMessage, peerID p2p.ID) (added bool, err error) {
  1657  	height, round, part := msg.Height, msg.Round, msg.Part
  1658  
  1659  	// Blocks might be reused, so round mismatch is OK
  1660  	if cs.Height != height {
  1661  		cs.Logger.Debug("Received block part from wrong height", "height", height, "round", round)
  1662  		return false, nil
  1663  	}
  1664  
  1665  	// We're not expecting a block part.
  1666  	if cs.ProposalBlockParts == nil {
  1667  		// NOTE: this can happen when we've gone to a higher round and
  1668  		// then receive parts from the previous round - not necessarily a bad peer.
  1669  		cs.Logger.Info("Received a block part when we're not expecting any",
  1670  			"height", height, "round", round, "index", part.Index, "peer", peerID)
  1671  		return false, nil
  1672  	}
  1673  
  1674  	added, err = cs.ProposalBlockParts.AddPart(part)
  1675  	if err != nil {
  1676  		return added, err
  1677  	}
  1678  	if added && cs.ProposalBlockParts.IsComplete() {
  1679  		// Added and completed!
  1680  		_, err = cdc.UnmarshalBinaryLengthPrefixedReader(
  1681  			cs.ProposalBlockParts.GetReader(),
  1682  			&cs.ProposalBlock,
  1683  			cs.state.ConsensusParams.Block.MaxBytes,
  1684  		)
  1685  		if err != nil {
  1686  			return added, err
  1687  		}
  1688  		// NOTE: it's possible to receive complete proposal blocks for future rounds without having the proposal
  1689  		cs.Logger.Info("Received complete proposal block", "height", cs.ProposalBlock.Height, "hash", cs.ProposalBlock.Hash())
  1690  		cs.eventBus.PublishEventCompleteProposal(cs.CompleteProposalEvent())
  1691  
  1692  		// Update Valid* if we can.
  1693  		prevotes := cs.Votes.Prevotes(cs.Round)
  1694  		blockID, hasTwoThirds := prevotes.TwoThirdsMajority()
  1695  		if hasTwoThirds && !blockID.IsZero() && (cs.ValidRound < cs.Round) {
  1696  			if cs.ProposalBlock.HashesTo(blockID.Hash) {
  1697  				cs.Logger.Info("Updating valid block to new proposal block",
  1698  					"valid-round", cs.Round, "valid-block-hash", cs.ProposalBlock.Hash())
  1699  				cs.ValidRound = cs.Round
  1700  				cs.ValidBlock = cs.ProposalBlock
  1701  				cs.ValidBlockParts = cs.ProposalBlockParts
  1702  			}
  1703  			// TODO: In case there is +2/3 majority in Prevotes set for some
  1704  			// block and cs.ProposalBlock contains different block, either
  1705  			// proposer is faulty or voting power of faulty processes is more
  1706  			// than 1/3. We should trigger in the future accountability
  1707  			// procedure at this point.
  1708  		}
  1709  
  1710  		if cs.Step <= cstypes.RoundStepPropose && cs.isProposalComplete() {
  1711  			// Move onto the next step
  1712  			cs.enterPrevote(height, cs.Round)
  1713  			if hasTwoThirds { // this is optimisation as this will be triggered when prevote is added
  1714  				cs.enterPrecommit(height, cs.Round)
  1715  			}
  1716  		} else if cs.Step == cstypes.RoundStepCommit {
  1717  			// If we're waiting on the proposal block...
  1718  			cs.tryFinalizeCommit(height)
  1719  		}
  1720  		return added, nil
  1721  	}
  1722  	return added, nil
  1723  }
  1724  
  1725  // Attempt to add the vote. if its a duplicate signature, dupeout the validator
  1726  func (cs *State) tryAddVote(vote *types.Vote, peerID p2p.ID) (bool, error) {
  1727  	added, err := cs.addVote(vote, peerID)
  1728  	if err != nil {
  1729  		// If the vote height is off, we'll just ignore it,
  1730  		// But if it's a conflicting sig, add it to the cs.evpool.
  1731  		// If it's otherwise invalid, punish peer.
  1732  		// nolint: gocritic
  1733  		if err == ErrVoteHeightMismatch {
  1734  			return added, err
  1735  		} else if voteErr, ok := err.(*types.ErrVoteConflictingVotes); ok {
  1736  			pubKey, err := cs.privValidator.GetPubKey()
  1737  			if err != nil {
  1738  				return false, fmt.Errorf("can't get pubkey: %w", err)
  1739  			}
  1740  
  1741  			if bytes.Equal(vote.ValidatorAddress, pubKey.Address()) {
  1742  				cs.Logger.Error(
  1743  					"Found conflicting vote from ourselves. Did you unsafe_reset a validator?",
  1744  					"height",
  1745  					vote.Height,
  1746  					"round",
  1747  					vote.Round,
  1748  					"type",
  1749  					vote.Type)
  1750  				return added, err
  1751  			}
  1752  			cs.evpool.AddEvidence(voteErr.DuplicateVoteEvidence)
  1753  			return added, err
  1754  		} else if err == types.ErrVoteNonDeterministicSignature {
  1755  			cs.Logger.Debug("Vote has non-deterministic signature", "err", err)
  1756  		} else {
  1757  			// Either
  1758  			// 1) bad peer OR
  1759  			// 2) not a bad peer? this can also err sometimes with "Unexpected step" OR
  1760  			// 3) tmkms use with multiple validators connecting to a single tmkms instance
  1761  			// 		(https://github.com/franono/tendermint/issues/3839).
  1762  			cs.Logger.Info("Error attempting to add vote", "err", err)
  1763  			return added, ErrAddingVote
  1764  		}
  1765  	}
  1766  	return added, nil
  1767  }
  1768  
  1769  //-----------------------------------------------------------------------------
  1770  
  1771  func (cs *State) addVote(
  1772  	vote *types.Vote,
  1773  	peerID p2p.ID) (added bool, err error) {
  1774  	cs.Logger.Debug(
  1775  		"addVote",
  1776  		"voteHeight",
  1777  		vote.Height,
  1778  		"voteType",
  1779  		vote.Type,
  1780  		"valIndex",
  1781  		vote.ValidatorIndex,
  1782  		"csHeight",
  1783  		cs.Height,
  1784  	)
  1785  
  1786  	// A precommit for the previous height?
  1787  	// These come in while we wait timeoutCommit
  1788  	if vote.Height+1 == cs.Height {
  1789  		if !(cs.Step == cstypes.RoundStepNewHeight && vote.Type == types.PrecommitType) {
  1790  			// TODO: give the reason ..
  1791  			// fmt.Errorf("tryAddVote: Wrong height, not a LastCommit straggler commit.")
  1792  			return added, ErrVoteHeightMismatch
  1793  		}
  1794  		added, err = cs.LastCommit.AddVote(vote)
  1795  		if !added {
  1796  			return added, err
  1797  		}
  1798  
  1799  		cs.Logger.Info(fmt.Sprintf("Added to lastPrecommits: %v", cs.LastCommit.StringShort()))
  1800  		cs.eventBus.PublishEventVote(types.EventDataVote{Vote: vote})
  1801  		cs.evsw.FireEvent(types.EventVote, vote)
  1802  
  1803  		// if we can skip timeoutCommit and have all the votes now,
  1804  		if cs.config.SkipTimeoutCommit && cs.LastCommit.HasAll() {
  1805  			// go straight to new round (skip timeout commit)
  1806  			// cs.scheduleTimeout(time.Duration(0), cs.Height, 0, cstypes.RoundStepNewHeight)
  1807  			cs.enterNewRound(cs.Height, 0)
  1808  		}
  1809  
  1810  		return
  1811  	}
  1812  
  1813  	// Height mismatch is ignored.
  1814  	// Not necessarily a bad peer, but not favourable behaviour.
  1815  	if vote.Height != cs.Height {
  1816  		err = ErrVoteHeightMismatch
  1817  		cs.Logger.Info("Vote ignored and not added", "voteHeight", vote.Height, "csHeight", cs.Height, "peerID", peerID)
  1818  		return
  1819  	}
  1820  
  1821  	height := cs.Height
  1822  	added, err = cs.Votes.AddVote(vote, peerID)
  1823  	if !added {
  1824  		// Either duplicate, or error upon cs.Votes.AddByIndex()
  1825  		return
  1826  	}
  1827  
  1828  	cs.eventBus.PublishEventVote(types.EventDataVote{Vote: vote})
  1829  	cs.evsw.FireEvent(types.EventVote, vote)
  1830  
  1831  	switch vote.Type {
  1832  	case types.PrevoteType:
  1833  		prevotes := cs.Votes.Prevotes(vote.Round)
  1834  		cs.Logger.Info("Added to prevote", "vote", vote, "prevotes", prevotes.StringShort())
  1835  
  1836  		// If +2/3 prevotes for a block or nil for *any* round:
  1837  		if blockID, ok := prevotes.TwoThirdsMajority(); ok {
  1838  
  1839  			// There was a polka!
  1840  			// If we're locked but this is a recent polka, unlock.
  1841  			// If it matches our ProposalBlock, update the ValidBlock
  1842  
  1843  			// Unlock if `cs.LockedRound < vote.Round <= cs.Round`
  1844  			// NOTE: If vote.Round > cs.Round, we'll deal with it when we get to vote.Round
  1845  			if (cs.LockedBlock != nil) &&
  1846  				(cs.LockedRound < vote.Round) &&
  1847  				(vote.Round <= cs.Round) &&
  1848  				!cs.LockedBlock.HashesTo(blockID.Hash) {
  1849  
  1850  				cs.Logger.Info("Unlocking because of POL.", "lockedRound", cs.LockedRound, "POLRound", vote.Round)
  1851  				cs.LockedRound = -1
  1852  				cs.LockedBlock = nil
  1853  				cs.LockedBlockParts = nil
  1854  				// If this is not the first round and we have already locked onto something then we are
  1855  				// changing the locked block so save POLC prevotes in evidence db in case of future justification
  1856  				cs.savePOLC(vote.Round, blockID)
  1857  				cs.eventBus.PublishEventUnlock(cs.RoundStateEvent())
  1858  			}
  1859  
  1860  			// Update Valid* if we can.
  1861  			// NOTE: our proposal block may be nil or not what received a polka..
  1862  			if len(blockID.Hash) != 0 && (cs.ValidRound < vote.Round) && (vote.Round == cs.Round) {
  1863  
  1864  				if cs.ProposalBlock.HashesTo(blockID.Hash) {
  1865  					cs.Logger.Info(
  1866  						"Updating ValidBlock because of POL.", "validRound", cs.ValidRound, "POLRound", vote.Round)
  1867  					cs.ValidRound = vote.Round
  1868  					cs.ValidBlock = cs.ProposalBlock
  1869  					cs.ValidBlockParts = cs.ProposalBlockParts
  1870  				} else {
  1871  					cs.Logger.Info(
  1872  						"Valid block we don't know about. Set ProposalBlock=nil",
  1873  						"proposal", cs.ProposalBlock.Hash(), "blockID", blockID.Hash)
  1874  					// We're getting the wrong block.
  1875  					cs.ProposalBlock = nil
  1876  				}
  1877  				if !cs.ProposalBlockParts.HasHeader(blockID.PartsHeader) {
  1878  					cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartsHeader)
  1879  				}
  1880  				cs.evsw.FireEvent(types.EventValidBlock, &cs.RoundState)
  1881  				cs.eventBus.PublishEventValidBlock(cs.RoundStateEvent())
  1882  			}
  1883  		}
  1884  
  1885  		// If +2/3 prevotes for *anything* for future round:
  1886  		switch {
  1887  		case cs.Round < vote.Round && prevotes.HasTwoThirdsAny():
  1888  			// Round-skip if there is any 2/3+ of votes ahead of us
  1889  			cs.enterNewRound(height, vote.Round)
  1890  		case cs.Round == vote.Round && cstypes.RoundStepPrevote <= cs.Step: // current round
  1891  			blockID, ok := prevotes.TwoThirdsMajority()
  1892  			if ok && (cs.isProposalComplete() || len(blockID.Hash) == 0) {
  1893  				cs.enterPrecommit(height, vote.Round)
  1894  			} else if prevotes.HasTwoThirdsAny() {
  1895  				cs.enterPrevoteWait(height, vote.Round)
  1896  			}
  1897  		case cs.Proposal != nil && 0 <= cs.Proposal.POLRound && cs.Proposal.POLRound == vote.Round:
  1898  			// If the proposal is now complete, enter prevote of cs.Round.
  1899  			if cs.isProposalComplete() {
  1900  				cs.enterPrevote(height, cs.Round)
  1901  			}
  1902  		}
  1903  
  1904  	case types.PrecommitType:
  1905  		precommits := cs.Votes.Precommits(vote.Round)
  1906  		cs.Logger.Info("Added to precommit", "vote", vote, "precommits", precommits.StringShort())
  1907  
  1908  		blockID, ok := precommits.TwoThirdsMajority()
  1909  		if ok {
  1910  			// Executed as TwoThirdsMajority could be from a higher round
  1911  			cs.enterNewRound(height, vote.Round)
  1912  			cs.enterPrecommit(height, vote.Round)
  1913  			if len(blockID.Hash) != 0 {
  1914  				cs.enterCommit(height, vote.Round)
  1915  				if cs.config.SkipTimeoutCommit && precommits.HasAll() {
  1916  					cs.enterNewRound(cs.Height, 0)
  1917  				}
  1918  			} else {
  1919  				cs.enterPrecommitWait(height, vote.Round)
  1920  			}
  1921  		} else if cs.Round <= vote.Round && precommits.HasTwoThirdsAny() {
  1922  			cs.enterNewRound(height, vote.Round)
  1923  			cs.enterPrecommitWait(height, vote.Round)
  1924  		}
  1925  
  1926  	default:
  1927  		panic(fmt.Sprintf("Unexpected vote type %X", vote.Type)) // go-amino should prevent this.
  1928  	}
  1929  
  1930  	return added, err
  1931  }
  1932  
  1933  // CONTRACT: cs.privValidator is not nil.
  1934  func (cs *State) signVote(
  1935  	msgType types.SignedMsgType,
  1936  	hash []byte,
  1937  	header types.PartSetHeader,
  1938  ) (*types.Vote, error) {
  1939  	// Flush the WAL. Otherwise, we may not recompute the same vote to sign,
  1940  	// and the privValidator will refuse to sign anything.
  1941  	cs.wal.FlushAndSync()
  1942  
  1943  	pubKey, err := cs.privValidator.GetPubKey()
  1944  	if err != nil {
  1945  		return nil, fmt.Errorf("can't get pubkey: %w", err)
  1946  	}
  1947  	addr := pubKey.Address()
  1948  	valIdx, _ := cs.Validators.GetByAddress(addr)
  1949  
  1950  	vote := &types.Vote{
  1951  		ValidatorAddress: addr,
  1952  		ValidatorIndex:   valIdx,
  1953  		Height:           cs.Height,
  1954  		Round:            cs.Round,
  1955  		Timestamp:        cs.voteTime(),
  1956  		Type:             msgType,
  1957  		BlockID:          types.BlockID{Hash: hash, PartsHeader: header},
  1958  	}
  1959  
  1960  	err = cs.privValidator.SignVote(cs.state.ChainID, vote)
  1961  	return vote, err
  1962  }
  1963  
  1964  func (cs *State) voteTime() time.Time {
  1965  	now := tmtime.Now()
  1966  	minVoteTime := now
  1967  	// TODO: We should remove next line in case we don't vote for v in case cs.ProposalBlock == nil,
  1968  	// even if cs.LockedBlock != nil. See https://docs.tendermint.com/master/spec/.
  1969  	timeIota := time.Duration(cs.state.ConsensusParams.Block.TimeIotaMs) * time.Millisecond
  1970  	if cs.LockedBlock != nil {
  1971  		// See the BFT time spec https://docs.tendermint.com/master/spec/consensus/bft-time.html
  1972  		minVoteTime = cs.LockedBlock.Time.Add(timeIota)
  1973  	} else if cs.ProposalBlock != nil {
  1974  		minVoteTime = cs.ProposalBlock.Time.Add(timeIota)
  1975  	}
  1976  
  1977  	if now.After(minVoteTime) {
  1978  		return now
  1979  	}
  1980  	return minVoteTime
  1981  }
  1982  
  1983  // sign the vote and publish on internalMsgQueue
  1984  func (cs *State) signAddVote(msgType types.SignedMsgType, hash []byte, header types.PartSetHeader) *types.Vote {
  1985  	if cs.privValidator == nil { // the node does not have a key
  1986  		return nil
  1987  	}
  1988  
  1989  	pubKey, err := cs.privValidator.GetPubKey()
  1990  	if err != nil {
  1991  		// Vote won't be signed, but it's not critical.
  1992  		cs.Logger.Error("Error on retrival of pubkey", "err", err)
  1993  		return nil
  1994  	}
  1995  
  1996  	// If the node not in the validator set, do nothing.
  1997  	if !cs.Validators.HasAddress(pubKey.Address()) {
  1998  		return nil
  1999  	}
  2000  
  2001  	// TODO: pass pubKey to signVote
  2002  	vote, err := cs.signVote(msgType, hash, header)
  2003  	if err == nil {
  2004  		cs.sendInternalMessage(msgInfo{&VoteMessage{vote}, ""})
  2005  		cs.Logger.Info("Signed and pushed vote", "height", cs.Height, "round", cs.Round, "vote", vote, "err", err)
  2006  		return vote
  2007  	}
  2008  	//if !cs.replayMode {
  2009  	cs.Logger.Error("Error signing vote", "height", cs.Height, "round", cs.Round, "vote", vote, "err", err)
  2010  	//}
  2011  	return nil
  2012  }
  2013  
  2014  //---------------------------------------------------------
  2015  
  2016  func CompareHRS(h1 int64, r1 int, s1 cstypes.RoundStepType, h2 int64, r2 int, s2 cstypes.RoundStepType) int {
  2017  	if h1 < h2 {
  2018  		return -1
  2019  	} else if h1 > h2 {
  2020  		return 1
  2021  	}
  2022  	if r1 < r2 {
  2023  		return -1
  2024  	} else if r1 > r2 {
  2025  		return 1
  2026  	}
  2027  	if s1 < s2 {
  2028  		return -1
  2029  	} else if s1 > s2 {
  2030  		return 1
  2031  	}
  2032  	return 0
  2033  }