github.com/devwanda/aphelion-staking@v0.33.9/consensus/state.go (about) 1 package consensus 2 3 import ( 4 "bytes" 5 "fmt" 6 "reflect" 7 "runtime/debug" 8 "sync" 9 "time" 10 11 "github.com/pkg/errors" 12 13 "github.com/devwanda/aphelion-staking/crypto" 14 "github.com/devwanda/aphelion-staking/libs/fail" 15 "github.com/devwanda/aphelion-staking/libs/log" 16 tmos "github.com/devwanda/aphelion-staking/libs/os" 17 "github.com/devwanda/aphelion-staking/libs/service" 18 tmtime "github.com/devwanda/aphelion-staking/types/time" 19 20 cfg "github.com/devwanda/aphelion-staking/config" 21 cstypes "github.com/devwanda/aphelion-staking/consensus/types" 22 tmevents "github.com/devwanda/aphelion-staking/libs/events" 23 "github.com/devwanda/aphelion-staking/p2p" 24 sm "github.com/devwanda/aphelion-staking/state" 25 "github.com/devwanda/aphelion-staking/types" 26 ) 27 28 //----------------------------------------------------------------------------- 29 // Errors 30 31 var ( 32 ErrInvalidProposalSignature = errors.New("error invalid proposal signature") 33 ErrInvalidProposalPOLRound = errors.New("error invalid proposal POL round") 34 ErrAddingVote = errors.New("error adding vote") 35 ErrVoteHeightMismatch = errors.New("error vote height mismatch") 36 37 errPubKeyIsNotSet = errors.New("pubkey is not set. Look for \"Can't get private validator pubkey\" errors") 38 ) 39 40 //----------------------------------------------------------------------------- 41 42 var ( 43 msgQueueSize = 1000 44 ) 45 46 // msgs from the reactor which may update the state 47 type msgInfo struct { 48 Msg Message `json:"msg"` 49 PeerID p2p.ID `json:"peer_key"` 50 } 51 52 // internally generated messages which may update the state 53 type timeoutInfo struct { 54 Duration time.Duration `json:"duration"` 55 Height int64 `json:"height"` 56 Round int `json:"round"` 57 Step cstypes.RoundStepType `json:"step"` 58 } 59 60 func (ti *timeoutInfo) String() string { 61 return fmt.Sprintf("%v ; %d/%d %v", ti.Duration, ti.Height, ti.Round, ti.Step) 62 } 63 64 // interface to the mempool 65 type txNotifier interface { 66 TxsAvailable() <-chan struct{} 67 } 68 69 // interface to the evidence pool 70 type evidencePool interface { 71 AddEvidence(types.Evidence) error 72 } 73 74 // State handles execution of the consensus algorithm. 75 // It processes votes and proposals, and upon reaching agreement, 76 // commits blocks to the chain and executes them against the application. 77 // The internal state machine receives input from peers, the internal validator, and from a timer. 78 type State struct { 79 service.BaseService 80 81 // config details 82 config *cfg.ConsensusConfig 83 privValidator types.PrivValidator // for signing votes 84 85 // store blocks and commits 86 blockStore sm.BlockStore 87 88 // create and execute blocks 89 blockExec *sm.BlockExecutor 90 91 // notify us if txs are available 92 txNotifier txNotifier 93 94 // add evidence to the pool 95 // when it's detected 96 evpool evidencePool 97 98 // internal state 99 mtx sync.RWMutex 100 cstypes.RoundState 101 state sm.State // State until height-1. 102 // privValidator pubkey, memoized for the duration of one block 103 // to avoid extra requests to HSM 104 privValidatorPubKey crypto.PubKey 105 106 // state changes may be triggered by: msgs from peers, 107 // msgs from ourself, or by timeouts 108 peerMsgQueue chan msgInfo 109 internalMsgQueue chan msgInfo 110 timeoutTicker TimeoutTicker 111 112 // information about about added votes and block parts are written on this channel 113 // so statistics can be computed by reactor 114 statsMsgQueue chan msgInfo 115 116 // we use eventBus to trigger msg broadcasts in the reactor, 117 // and to notify external subscribers, eg. through a websocket 118 eventBus *types.EventBus 119 120 // a Write-Ahead Log ensures we can recover from any kind of crash 121 // and helps us avoid signing conflicting votes 122 wal WAL 123 replayMode bool // so we don't log signing errors during replay 124 doWALCatchup bool // determines if we even try to do the catchup 125 126 // for tests where we want to limit the number of transitions the state makes 127 nSteps int 128 129 // some functions can be overwritten for testing 130 decideProposal func(height int64, round int) 131 doPrevote func(height int64, round int) 132 setProposal func(proposal *types.Proposal) error 133 134 // closed when we finish shutting down 135 done chan struct{} 136 137 // synchronous pubsub between consensus state and reactor. 138 // state only emits EventNewRoundStep and EventVote 139 evsw tmevents.EventSwitch 140 141 // for reporting metrics 142 metrics *Metrics 143 } 144 145 // StateOption sets an optional parameter on the State. 146 type StateOption func(*State) 147 148 // NewState returns a new State. 149 func NewState( 150 config *cfg.ConsensusConfig, 151 state sm.State, 152 blockExec *sm.BlockExecutor, 153 blockStore sm.BlockStore, 154 txNotifier txNotifier, 155 evpool evidencePool, 156 options ...StateOption, 157 ) *State { 158 cs := &State{ 159 config: config, 160 blockExec: blockExec, 161 blockStore: blockStore, 162 txNotifier: txNotifier, 163 peerMsgQueue: make(chan msgInfo, msgQueueSize), 164 internalMsgQueue: make(chan msgInfo, msgQueueSize), 165 timeoutTicker: NewTimeoutTicker(), 166 statsMsgQueue: make(chan msgInfo, msgQueueSize), 167 done: make(chan struct{}), 168 doWALCatchup: true, 169 wal: nilWAL{}, 170 evpool: evpool, 171 evsw: tmevents.NewEventSwitch(), 172 metrics: NopMetrics(), 173 } 174 // set function defaults (may be overwritten before calling Start) 175 cs.decideProposal = cs.defaultDecideProposal 176 cs.doPrevote = cs.defaultDoPrevote 177 cs.setProposal = cs.defaultSetProposal 178 179 cs.updateToState(state) 180 181 // Don't call scheduleRound0 yet. 182 // We do that upon Start(). 183 cs.reconstructLastCommit(state) 184 cs.BaseService = *service.NewBaseService(nil, "State", cs) 185 for _, option := range options { 186 option(cs) 187 } 188 return cs 189 } 190 191 //---------------------------------------- 192 // Public interface 193 194 // SetLogger implements Service. 195 func (cs *State) SetLogger(l log.Logger) { 196 cs.BaseService.Logger = l 197 cs.timeoutTicker.SetLogger(l) 198 } 199 200 // SetEventBus sets event bus. 201 func (cs *State) SetEventBus(b *types.EventBus) { 202 cs.eventBus = b 203 cs.blockExec.SetEventBus(b) 204 } 205 206 // StateMetrics sets the metrics. 207 func StateMetrics(metrics *Metrics) StateOption { 208 return func(cs *State) { cs.metrics = metrics } 209 } 210 211 // String returns a string. 212 func (cs *State) String() string { 213 // better not to access shared variables 214 return fmt.Sprintf("ConsensusState") //(H:%v R:%v S:%v", cs.Height, cs.Round, cs.Step) 215 } 216 217 // GetState returns a copy of the chain state. 218 func (cs *State) GetState() sm.State { 219 cs.mtx.RLock() 220 defer cs.mtx.RUnlock() 221 return cs.state.Copy() 222 } 223 224 // GetLastHeight returns the last height committed. 225 // If there were no blocks, returns 0. 226 func (cs *State) GetLastHeight() int64 { 227 cs.mtx.RLock() 228 defer cs.mtx.RUnlock() 229 return cs.RoundState.Height - 1 230 } 231 232 // GetRoundState returns a shallow copy of the internal consensus state. 233 func (cs *State) GetRoundState() *cstypes.RoundState { 234 cs.mtx.RLock() 235 rs := cs.RoundState // copy 236 cs.mtx.RUnlock() 237 return &rs 238 } 239 240 // GetRoundStateJSON returns a json of RoundState, marshalled using go-amino. 241 func (cs *State) GetRoundStateJSON() ([]byte, error) { 242 cs.mtx.RLock() 243 defer cs.mtx.RUnlock() 244 return cdc.MarshalJSON(cs.RoundState) 245 } 246 247 // GetRoundStateSimpleJSON returns a json of RoundStateSimple, marshalled using go-amino. 248 func (cs *State) GetRoundStateSimpleJSON() ([]byte, error) { 249 cs.mtx.RLock() 250 defer cs.mtx.RUnlock() 251 return cdc.MarshalJSON(cs.RoundState.RoundStateSimple()) 252 } 253 254 // GetValidators returns a copy of the current validators. 255 func (cs *State) GetValidators() (int64, []*types.Validator) { 256 cs.mtx.RLock() 257 defer cs.mtx.RUnlock() 258 return cs.state.LastBlockHeight, cs.state.Validators.Copy().Validators 259 } 260 261 // SetPrivValidator sets the private validator account for signing votes. It 262 // immediately requests pubkey and caches it. 263 func (cs *State) SetPrivValidator(priv types.PrivValidator) { 264 cs.mtx.Lock() 265 defer cs.mtx.Unlock() 266 267 cs.privValidator = priv 268 269 if err := cs.updatePrivValidatorPubKey(); err != nil { 270 cs.Logger.Error("Can't get private validator pubkey", "err", err) 271 } 272 } 273 274 // SetTimeoutTicker sets the local timer. It may be useful to overwrite for testing. 275 func (cs *State) SetTimeoutTicker(timeoutTicker TimeoutTicker) { 276 cs.mtx.Lock() 277 cs.timeoutTicker = timeoutTicker 278 cs.mtx.Unlock() 279 } 280 281 // LoadCommit loads the commit for a given height. 282 func (cs *State) LoadCommit(height int64) *types.Commit { 283 cs.mtx.RLock() 284 defer cs.mtx.RUnlock() 285 if height == cs.blockStore.Height() { 286 return cs.blockStore.LoadSeenCommit(height) 287 } 288 return cs.blockStore.LoadBlockCommit(height) 289 } 290 291 // OnStart implements service.Service. 292 // It loads the latest state via the WAL, and starts the timeout and receive routines. 293 func (cs *State) OnStart() error { 294 if err := cs.evsw.Start(); err != nil { 295 return err 296 } 297 298 // we may set the WAL in testing before calling Start, 299 // so only OpenWAL if its still the nilWAL 300 if _, ok := cs.wal.(nilWAL); ok { 301 walFile := cs.config.WalFile() 302 wal, err := cs.OpenWAL(walFile) 303 if err != nil { 304 cs.Logger.Error("Error loading State wal", "err", err.Error()) 305 return err 306 } 307 cs.wal = wal 308 } 309 310 // we need the timeoutRoutine for replay so 311 // we don't block on the tick chan. 312 // NOTE: we will get a build up of garbage go routines 313 // firing on the tockChan until the receiveRoutine is started 314 // to deal with them (by that point, at most one will be valid) 315 if err := cs.timeoutTicker.Start(); err != nil { 316 return err 317 } 318 319 // we may have lost some votes if the process crashed 320 // reload from consensus log to catchup 321 if cs.doWALCatchup { 322 if err := cs.catchupReplay(cs.Height); err != nil { 323 // don't try to recover from data corruption error 324 if IsDataCorruptionError(err) { 325 cs.Logger.Error("Encountered corrupt WAL file", "err", err.Error()) 326 cs.Logger.Error("Please repair the WAL file before restarting") 327 fmt.Println(`You can attempt to repair the WAL as follows: 328 329 ---- 330 WALFILE=~/.tendermint/data/cs.wal/wal 331 cp $WALFILE ${WALFILE}.bak # backup the file 332 go run scripts/wal2json/main.go $WALFILE > wal.json # this will panic, but can be ignored 333 rm $WALFILE # remove the corrupt file 334 go run scripts/json2wal/main.go wal.json $WALFILE # rebuild the file without corruption 335 ----`) 336 337 return err 338 } 339 340 cs.Logger.Error("Error on catchup replay. Proceeding to start State anyway", "err", err.Error()) 341 // NOTE: if we ever do return an error here, 342 // make sure to stop the timeoutTicker 343 } 344 } 345 346 // now start the receiveRoutine 347 go cs.receiveRoutine(0) 348 349 // schedule the first round! 350 // use GetRoundState so we don't race the receiveRoutine for access 351 cs.scheduleRound0(cs.GetRoundState()) 352 353 return nil 354 } 355 356 // timeoutRoutine: receive requests for timeouts on tickChan and fire timeouts on tockChan 357 // receiveRoutine: serializes processing of proposoals, block parts, votes; coordinates state transitions 358 func (cs *State) startRoutines(maxSteps int) { 359 err := cs.timeoutTicker.Start() 360 if err != nil { 361 cs.Logger.Error("Error starting timeout ticker", "err", err) 362 return 363 } 364 go cs.receiveRoutine(maxSteps) 365 } 366 367 // OnStop implements service.Service. 368 func (cs *State) OnStop() { 369 cs.evsw.Stop() 370 cs.timeoutTicker.Stop() 371 // WAL is stopped in receiveRoutine. 372 } 373 374 // Wait waits for the the main routine to return. 375 // NOTE: be sure to Stop() the event switch and drain 376 // any event channels or this may deadlock 377 func (cs *State) Wait() { 378 <-cs.done 379 } 380 381 // OpenWAL opens a file to log all consensus messages and timeouts for deterministic accountability 382 func (cs *State) OpenWAL(walFile string) (WAL, error) { 383 wal, err := NewWAL(walFile) 384 if err != nil { 385 cs.Logger.Error("Failed to open WAL for consensus state", "wal", walFile, "err", err) 386 return nil, err 387 } 388 wal.SetLogger(cs.Logger.With("wal", walFile)) 389 if err := wal.Start(); err != nil { 390 return nil, err 391 } 392 return wal, nil 393 } 394 395 //------------------------------------------------------------ 396 // Public interface for passing messages into the consensus state, possibly causing a state transition. 397 // If peerID == "", the msg is considered internal. 398 // Messages are added to the appropriate queue (peer or internal). 399 // If the queue is full, the function may block. 400 // TODO: should these return anything or let callers just use events? 401 402 // AddVote inputs a vote. 403 func (cs *State) AddVote(vote *types.Vote, peerID p2p.ID) (added bool, err error) { 404 if peerID == "" { 405 cs.internalMsgQueue <- msgInfo{&VoteMessage{vote}, ""} 406 } else { 407 cs.peerMsgQueue <- msgInfo{&VoteMessage{vote}, peerID} 408 } 409 410 // TODO: wait for event?! 411 return false, nil 412 } 413 414 // SetProposal inputs a proposal. 415 func (cs *State) SetProposal(proposal *types.Proposal, peerID p2p.ID) error { 416 417 if peerID == "" { 418 cs.internalMsgQueue <- msgInfo{&ProposalMessage{proposal}, ""} 419 } else { 420 cs.peerMsgQueue <- msgInfo{&ProposalMessage{proposal}, peerID} 421 } 422 423 // TODO: wait for event?! 424 return nil 425 } 426 427 // AddProposalBlockPart inputs a part of the proposal block. 428 func (cs *State) AddProposalBlockPart(height int64, round int, part *types.Part, peerID p2p.ID) error { 429 430 if peerID == "" { 431 cs.internalMsgQueue <- msgInfo{&BlockPartMessage{height, round, part}, ""} 432 } else { 433 cs.peerMsgQueue <- msgInfo{&BlockPartMessage{height, round, part}, peerID} 434 } 435 436 // TODO: wait for event?! 437 return nil 438 } 439 440 // SetProposalAndBlock inputs the proposal and all block parts. 441 func (cs *State) SetProposalAndBlock( 442 proposal *types.Proposal, 443 block *types.Block, 444 parts *types.PartSet, 445 peerID p2p.ID, 446 ) error { 447 if err := cs.SetProposal(proposal, peerID); err != nil { 448 return err 449 } 450 for i := 0; i < parts.Total(); i++ { 451 part := parts.GetPart(i) 452 if err := cs.AddProposalBlockPart(proposal.Height, proposal.Round, part, peerID); err != nil { 453 return err 454 } 455 } 456 return nil 457 } 458 459 //------------------------------------------------------------ 460 // internal functions for managing the state 461 462 func (cs *State) updateHeight(height int64) { 463 cs.metrics.Height.Set(float64(height)) 464 cs.Height = height 465 } 466 467 func (cs *State) updateRoundStep(round int, step cstypes.RoundStepType) { 468 cs.Round = round 469 cs.Step = step 470 } 471 472 // enterNewRound(height, 0) at cs.StartTime. 473 func (cs *State) scheduleRound0(rs *cstypes.RoundState) { 474 //cs.Logger.Info("scheduleRound0", "now", tmtime.Now(), "startTime", cs.StartTime) 475 sleepDuration := rs.StartTime.Sub(tmtime.Now()) 476 cs.scheduleTimeout(sleepDuration, rs.Height, 0, cstypes.RoundStepNewHeight) 477 } 478 479 // Attempt to schedule a timeout (by sending timeoutInfo on the tickChan) 480 func (cs *State) scheduleTimeout(duration time.Duration, height int64, round int, step cstypes.RoundStepType) { 481 cs.timeoutTicker.ScheduleTimeout(timeoutInfo{duration, height, round, step}) 482 } 483 484 // send a msg into the receiveRoutine regarding our own proposal, block part, or vote 485 func (cs *State) sendInternalMessage(mi msgInfo) { 486 select { 487 case cs.internalMsgQueue <- mi: 488 default: 489 // NOTE: using the go-routine means our votes can 490 // be processed out of order. 491 // TODO: use CList here for strict determinism and 492 // attempt push to internalMsgQueue in receiveRoutine 493 cs.Logger.Info("Internal msg queue is full. Using a go-routine") 494 go func() { cs.internalMsgQueue <- mi }() 495 } 496 } 497 498 // Reconstruct LastCommit from SeenCommit, which we saved along with the block, 499 // (which happens even before saving the state) 500 func (cs *State) reconstructLastCommit(state sm.State) { 501 if state.LastBlockHeight == 0 { 502 return 503 } 504 seenCommit := cs.blockStore.LoadSeenCommit(state.LastBlockHeight) 505 if seenCommit == nil { 506 panic(fmt.Sprintf("Failed to reconstruct LastCommit: seen commit for height %v not found", 507 state.LastBlockHeight)) 508 } 509 lastPrecommits := types.CommitToVoteSet(state.ChainID, seenCommit, state.LastValidators) 510 if !lastPrecommits.HasTwoThirdsMajority() { 511 panic("Failed to reconstruct LastCommit: Does not have +2/3 maj") 512 } 513 cs.LastCommit = lastPrecommits 514 } 515 516 // Updates State and increments height to match that of state. 517 // The round becomes 0 and cs.Step becomes cstypes.RoundStepNewHeight. 518 func (cs *State) updateToState(state sm.State) { 519 if cs.CommitRound > -1 && 0 < cs.Height && cs.Height != state.LastBlockHeight { 520 panic(fmt.Sprintf("updateToState() expected state height of %v but found %v", 521 cs.Height, state.LastBlockHeight)) 522 } 523 if !cs.state.IsEmpty() && cs.state.LastBlockHeight+1 != cs.Height { 524 // This might happen when someone else is mutating cs.state. 525 // Someone forgot to pass in state.Copy() somewhere?! 526 panic(fmt.Sprintf("Inconsistent cs.state.LastBlockHeight+1 %v vs cs.Height %v", 527 cs.state.LastBlockHeight+1, cs.Height)) 528 } 529 530 // If state isn't further out than cs.state, just ignore. 531 // This happens when SwitchToConsensus() is called in the reactor. 532 // We don't want to reset e.g. the Votes, but we still want to 533 // signal the new round step, because other services (eg. txNotifier) 534 // depend on having an up-to-date peer state! 535 if !cs.state.IsEmpty() && (state.LastBlockHeight <= cs.state.LastBlockHeight) { 536 cs.Logger.Info( 537 "Ignoring updateToState()", 538 "newHeight", 539 state.LastBlockHeight+1, 540 "oldHeight", 541 cs.state.LastBlockHeight+1) 542 cs.newStep() 543 return 544 } 545 546 // Reset fields based on state. 547 validators := state.Validators 548 lastPrecommits := (*types.VoteSet)(nil) 549 if cs.CommitRound > -1 && cs.Votes != nil { 550 if !cs.Votes.Precommits(cs.CommitRound).HasTwoThirdsMajority() { 551 panic("updateToState(state) called but last Precommit round didn't have +2/3") 552 } 553 lastPrecommits = cs.Votes.Precommits(cs.CommitRound) 554 } 555 556 // Next desired block height 557 height := state.LastBlockHeight + 1 558 559 // RoundState fields 560 cs.updateHeight(height) 561 cs.updateRoundStep(0, cstypes.RoundStepNewHeight) 562 if cs.CommitTime.IsZero() { 563 // "Now" makes it easier to sync up dev nodes. 564 // We add timeoutCommit to allow transactions 565 // to be gathered for the first block. 566 // And alternative solution that relies on clocks: 567 // cs.StartTime = state.LastBlockTime.Add(timeoutCommit) 568 cs.StartTime = cs.config.Commit(tmtime.Now()) 569 } else { 570 cs.StartTime = cs.config.Commit(cs.CommitTime) 571 } 572 573 cs.Validators = validators 574 cs.Proposal = nil 575 cs.ProposalBlock = nil 576 cs.ProposalBlockParts = nil 577 cs.LockedRound = -1 578 cs.LockedBlock = nil 579 cs.LockedBlockParts = nil 580 cs.ValidRound = -1 581 cs.ValidBlock = nil 582 cs.ValidBlockParts = nil 583 cs.Votes = cstypes.NewHeightVoteSet(state.ChainID, height, validators) 584 cs.CommitRound = -1 585 cs.LastCommit = lastPrecommits 586 cs.LastValidators = state.LastValidators 587 cs.TriggeredTimeoutPrecommit = false 588 589 cs.state = state 590 591 // Finally, broadcast RoundState 592 cs.newStep() 593 } 594 595 func (cs *State) newStep() { 596 rs := cs.RoundStateEvent() 597 cs.wal.Write(rs) 598 cs.nSteps++ 599 // newStep is called by updateToState in NewState before the eventBus is set! 600 if cs.eventBus != nil { 601 cs.eventBus.PublishEventNewRoundStep(rs) 602 cs.evsw.FireEvent(types.EventNewRoundStep, &cs.RoundState) 603 } 604 } 605 606 //----------------------------------------- 607 // the main go routines 608 609 // receiveRoutine handles messages which may cause state transitions. 610 // it's argument (n) is the number of messages to process before exiting - use 0 to run forever 611 // It keeps the RoundState and is the only thing that updates it. 612 // Updates (state transitions) happen on timeouts, complete proposals, and 2/3 majorities. 613 // State must be locked before any internal state is updated. 614 func (cs *State) receiveRoutine(maxSteps int) { 615 onExit := func(cs *State) { 616 // NOTE: the internalMsgQueue may have signed messages from our 617 // priv_val that haven't hit the WAL, but its ok because 618 // priv_val tracks LastSig 619 620 // close wal now that we're done writing to it 621 cs.wal.Stop() 622 cs.wal.Wait() 623 624 close(cs.done) 625 } 626 627 defer func() { 628 if r := recover(); r != nil { 629 cs.Logger.Error("CONSENSUS FAILURE!!!", "err", r, "stack", string(debug.Stack())) 630 // stop gracefully 631 // 632 // NOTE: We most probably shouldn't be running any further when there is 633 // some unexpected panic. Some unknown error happened, and so we don't 634 // know if that will result in the validator signing an invalid thing. It 635 // might be worthwhile to explore a mechanism for manual resuming via 636 // some console or secure RPC system, but for now, halting the chain upon 637 // unexpected consensus bugs sounds like the better option. 638 onExit(cs) 639 } 640 }() 641 642 for { 643 if maxSteps > 0 { 644 if cs.nSteps >= maxSteps { 645 cs.Logger.Info("reached max steps. exiting receive routine") 646 cs.nSteps = 0 647 return 648 } 649 } 650 rs := cs.RoundState 651 var mi msgInfo 652 653 select { 654 case <-cs.txNotifier.TxsAvailable(): 655 cs.handleTxsAvailable() 656 case mi = <-cs.peerMsgQueue: 657 cs.wal.Write(mi) 658 // handles proposals, block parts, votes 659 // may generate internal events (votes, complete proposals, 2/3 majorities) 660 cs.handleMsg(mi) 661 case mi = <-cs.internalMsgQueue: 662 err := cs.wal.WriteSync(mi) // NOTE: fsync 663 if err != nil { 664 panic(fmt.Sprintf("Failed to write %v msg to consensus wal due to %v. Check your FS and restart the node", mi, err)) 665 } 666 667 if _, ok := mi.Msg.(*VoteMessage); ok { 668 // we actually want to simulate failing during 669 // the previous WriteSync, but this isn't easy to do. 670 // Equivalent would be to fail here and manually remove 671 // some bytes from the end of the wal. 672 fail.Fail() // XXX 673 } 674 675 // handles proposals, block parts, votes 676 cs.handleMsg(mi) 677 case ti := <-cs.timeoutTicker.Chan(): // tockChan: 678 cs.wal.Write(ti) 679 // if the timeout is relevant to the rs 680 // go to the next step 681 cs.handleTimeout(ti, rs) 682 case <-cs.Quit(): 683 onExit(cs) 684 return 685 } 686 } 687 } 688 689 // state transitions on complete-proposal, 2/3-any, 2/3-one 690 func (cs *State) handleMsg(mi msgInfo) { 691 cs.mtx.Lock() 692 defer cs.mtx.Unlock() 693 694 var ( 695 added bool 696 err error 697 ) 698 msg, peerID := mi.Msg, mi.PeerID 699 switch msg := msg.(type) { 700 case *ProposalMessage: 701 // will not cause transition. 702 // once proposal is set, we can receive block parts 703 err = cs.setProposal(msg.Proposal) 704 case *BlockPartMessage: 705 // if the proposal is complete, we'll enterPrevote or tryFinalizeCommit 706 added, err = cs.addProposalBlockPart(msg, peerID) 707 if added { 708 cs.statsMsgQueue <- mi 709 } 710 711 if err != nil && msg.Round != cs.Round { 712 cs.Logger.Debug( 713 "Received block part from wrong round", 714 "height", 715 cs.Height, 716 "csRound", 717 cs.Round, 718 "blockRound", 719 msg.Round) 720 err = nil 721 } 722 case *VoteMessage: 723 // attempt to add the vote and dupeout the validator if its a duplicate signature 724 // if the vote gives us a 2/3-any or 2/3-one, we transition 725 added, err = cs.tryAddVote(msg.Vote, peerID) 726 if added { 727 cs.statsMsgQueue <- mi 728 } 729 730 // if err == ErrAddingVote { 731 // TODO: punish peer 732 // We probably don't want to stop the peer here. The vote does not 733 // necessarily comes from a malicious peer but can be just broadcasted by 734 // a typical peer. 735 // https://github.com/tendermint/tendermint/issues/1281 736 // } 737 738 // NOTE: the vote is broadcast to peers by the reactor listening 739 // for vote events 740 741 // TODO: If rs.Height == vote.Height && rs.Round < vote.Round, 742 // the peer is sending us CatchupCommit precommits. 743 // We could make note of this and help filter in broadcastHasVoteMessage(). 744 default: 745 cs.Logger.Error("Unknown msg type", "type", reflect.TypeOf(msg)) 746 return 747 } 748 749 if err != nil { // nolint:staticcheck 750 // Causes TestReactorValidatorSetChanges to timeout 751 // https://github.com/tendermint/tendermint/issues/3406 752 // cs.Logger.Error("Error with msg", "height", cs.Height, "round", cs.Round, 753 // "peer", peerID, "err", err, "msg", msg) 754 } 755 } 756 757 func (cs *State) handleTimeout(ti timeoutInfo, rs cstypes.RoundState) { 758 cs.Logger.Debug("Received tock", "timeout", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step) 759 760 // timeouts must be for current height, round, step 761 if ti.Height != rs.Height || ti.Round < rs.Round || (ti.Round == rs.Round && ti.Step < rs.Step) { 762 cs.Logger.Debug("Ignoring tock because we're ahead", "height", rs.Height, "round", rs.Round, "step", rs.Step) 763 return 764 } 765 766 // the timeout will now cause a state transition 767 cs.mtx.Lock() 768 defer cs.mtx.Unlock() 769 770 switch ti.Step { 771 case cstypes.RoundStepNewHeight: 772 // NewRound event fired from enterNewRound. 773 // XXX: should we fire timeout here (for timeout commit)? 774 cs.enterNewRound(ti.Height, 0) 775 case cstypes.RoundStepNewRound: 776 cs.enterPropose(ti.Height, 0) 777 case cstypes.RoundStepPropose: 778 cs.eventBus.PublishEventTimeoutPropose(cs.RoundStateEvent()) 779 cs.enterPrevote(ti.Height, ti.Round) 780 case cstypes.RoundStepPrevoteWait: 781 cs.eventBus.PublishEventTimeoutWait(cs.RoundStateEvent()) 782 cs.enterPrecommit(ti.Height, ti.Round) 783 case cstypes.RoundStepPrecommitWait: 784 cs.eventBus.PublishEventTimeoutWait(cs.RoundStateEvent()) 785 cs.enterPrecommit(ti.Height, ti.Round) 786 cs.enterNewRound(ti.Height, ti.Round+1) 787 default: 788 panic(fmt.Sprintf("Invalid timeout step: %v", ti.Step)) 789 } 790 791 } 792 793 func (cs *State) handleTxsAvailable() { 794 cs.mtx.Lock() 795 defer cs.mtx.Unlock() 796 797 // We only need to do this for round 0. 798 if cs.Round != 0 { 799 return 800 } 801 802 switch cs.Step { 803 case cstypes.RoundStepNewHeight: // timeoutCommit phase 804 if cs.needProofBlock(cs.Height) { 805 // enterPropose will be called by enterNewRound 806 return 807 } 808 809 // +1ms to ensure RoundStepNewRound timeout always happens after RoundStepNewHeight 810 timeoutCommit := cs.StartTime.Sub(tmtime.Now()) + 1*time.Millisecond 811 cs.scheduleTimeout(timeoutCommit, cs.Height, 0, cstypes.RoundStepNewRound) 812 case cstypes.RoundStepNewRound: // after timeoutCommit 813 cs.enterPropose(cs.Height, 0) 814 } 815 } 816 817 //----------------------------------------------------------------------------- 818 // State functions 819 // Used internally by handleTimeout and handleMsg to make state transitions 820 821 // Enter: `timeoutNewHeight` by startTime (commitTime+timeoutCommit), 822 // or, if SkipTimeoutCommit==true, after receiving all precommits from (height,round-1) 823 // Enter: `timeoutPrecommits` after any +2/3 precommits from (height,round-1) 824 // Enter: +2/3 precommits for nil at (height,round-1) 825 // Enter: +2/3 prevotes any or +2/3 precommits for block or any from (height, round) 826 // NOTE: cs.StartTime was already set for height. 827 func (cs *State) enterNewRound(height int64, round int) { 828 logger := cs.Logger.With("height", height, "round", round) 829 830 if cs.Height != height || round < cs.Round || (cs.Round == round && cs.Step != cstypes.RoundStepNewHeight) { 831 logger.Debug(fmt.Sprintf( 832 "enterNewRound(%v/%v): Invalid args. Current step: %v/%v/%v", 833 height, 834 round, 835 cs.Height, 836 cs.Round, 837 cs.Step)) 838 return 839 } 840 841 if now := tmtime.Now(); cs.StartTime.After(now) { 842 logger.Info("Need to set a buffer and log message here for sanity.", "startTime", cs.StartTime, "now", now) 843 } 844 845 logger.Info(fmt.Sprintf("enterNewRound(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step)) 846 847 // Increment validators if necessary 848 validators := cs.Validators 849 if cs.Round < round { 850 validators = validators.Copy() 851 validators.IncrementProposerPriority(round - cs.Round) 852 } 853 854 // Setup new round 855 // we don't fire newStep for this step, 856 // but we fire an event, so update the round step first 857 cs.updateRoundStep(round, cstypes.RoundStepNewRound) 858 cs.Validators = validators 859 if round == 0 { 860 // We've already reset these upon new height, 861 // and meanwhile we might have received a proposal 862 // for round 0. 863 } else { 864 logger.Info("Resetting Proposal info") 865 cs.Proposal = nil 866 cs.ProposalBlock = nil 867 cs.ProposalBlockParts = nil 868 } 869 cs.Votes.SetRound(round + 1) // also track next round (round+1) to allow round-skipping 870 cs.TriggeredTimeoutPrecommit = false 871 872 cs.eventBus.PublishEventNewRound(cs.NewRoundEvent()) 873 cs.metrics.Rounds.Set(float64(round)) 874 875 // Wait for txs to be available in the mempool 876 // before we enterPropose in round 0. If the last block changed the app hash, 877 // we may need an empty "proof" block, and enterPropose immediately. 878 waitForTxs := cs.config.WaitForTxs() && round == 0 && !cs.needProofBlock(height) 879 if waitForTxs { 880 if cs.config.CreateEmptyBlocksInterval > 0 { 881 cs.scheduleTimeout(cs.config.CreateEmptyBlocksInterval, height, round, 882 cstypes.RoundStepNewRound) 883 } 884 } else { 885 cs.enterPropose(height, round) 886 } 887 } 888 889 // needProofBlock returns true on the first height (so the genesis app hash is signed right away) 890 // and where the last block (height-1) caused the app hash to change 891 func (cs *State) needProofBlock(height int64) bool { 892 if height == 1 { 893 return true 894 } 895 896 lastBlockMeta := cs.blockStore.LoadBlockMeta(height - 1) 897 if lastBlockMeta == nil { 898 panic(fmt.Sprintf("needProofBlock: last block meta for height %d not found", height-1)) 899 } 900 return !bytes.Equal(cs.state.AppHash, lastBlockMeta.Header.AppHash) 901 } 902 903 // Enter (CreateEmptyBlocks): from enterNewRound(height,round) 904 // Enter (CreateEmptyBlocks, CreateEmptyBlocksInterval > 0 ): 905 // after enterNewRound(height,round), after timeout of CreateEmptyBlocksInterval 906 // Enter (!CreateEmptyBlocks) : after enterNewRound(height,round), once txs are in the mempool 907 func (cs *State) enterPropose(height int64, round int) { 908 logger := cs.Logger.With("height", height, "round", round) 909 910 if cs.Height != height || round < cs.Round || (cs.Round == round && cstypes.RoundStepPropose <= cs.Step) { 911 logger.Debug(fmt.Sprintf( 912 "enterPropose(%v/%v): Invalid args. Current step: %v/%v/%v", 913 height, 914 round, 915 cs.Height, 916 cs.Round, 917 cs.Step)) 918 return 919 } 920 logger.Info(fmt.Sprintf("enterPropose(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step)) 921 922 defer func() { 923 // Done enterPropose: 924 cs.updateRoundStep(round, cstypes.RoundStepPropose) 925 cs.newStep() 926 927 // If we have the whole proposal + POL, then goto Prevote now. 928 // else, we'll enterPrevote when the rest of the proposal is received (in AddProposalBlockPart), 929 // or else after timeoutPropose 930 if cs.isProposalComplete() { 931 cs.enterPrevote(height, cs.Round) 932 } 933 }() 934 935 // If we don't get the proposal and all block parts quick enough, enterPrevote 936 cs.scheduleTimeout(cs.config.Propose(round), height, round, cstypes.RoundStepPropose) 937 938 // Nothing more to do if we're not a validator 939 if cs.privValidator == nil { 940 logger.Debug("This node is not a validator") 941 return 942 } 943 logger.Debug("This node is a validator") 944 945 if cs.privValidatorPubKey == nil { 946 // If this node is a validator & proposer in the current round, it will 947 // miss the opportunity to create a block. 948 logger.Error(fmt.Sprintf("enterPropose: %v", errPubKeyIsNotSet)) 949 return 950 } 951 address := cs.privValidatorPubKey.Address() 952 953 // if not a validator, we're done 954 if !cs.Validators.HasAddress(address) { 955 logger.Debug("This node is not a validator", "addr", address, "vals", cs.Validators) 956 return 957 } 958 959 if cs.isProposer(address) { 960 logger.Info("enterPropose: Our turn to propose", 961 "proposer", 962 address, 963 "privValidator", 964 cs.privValidator) 965 cs.decideProposal(height, round) 966 } else { 967 logger.Info("enterPropose: Not our turn to propose", 968 "proposer", 969 cs.Validators.GetProposer().Address, 970 "privValidator", 971 cs.privValidator) 972 } 973 } 974 975 func (cs *State) isProposer(address []byte) bool { 976 return bytes.Equal(cs.Validators.GetProposer().Address, address) 977 } 978 979 func (cs *State) defaultDecideProposal(height int64, round int) { 980 var block *types.Block 981 var blockParts *types.PartSet 982 983 // Decide on block 984 if cs.ValidBlock != nil { 985 // If there is valid block, choose that. 986 block, blockParts = cs.ValidBlock, cs.ValidBlockParts 987 } else { 988 // Create a new proposal block from state/txs from the mempool. 989 block, blockParts = cs.createProposalBlock() 990 if block == nil { 991 return 992 } 993 } 994 995 // Flush the WAL. Otherwise, we may not recompute the same proposal to sign, 996 // and the privValidator will refuse to sign anything. 997 cs.wal.FlushAndSync() 998 999 // Make proposal 1000 propBlockID := types.BlockID{Hash: block.Hash(), PartsHeader: blockParts.Header()} 1001 proposal := types.NewProposal(height, round, cs.ValidRound, propBlockID) 1002 if err := cs.privValidator.SignProposal(cs.state.ChainID, proposal); err == nil { 1003 1004 // send proposal and block parts on internal msg queue 1005 cs.sendInternalMessage(msgInfo{&ProposalMessage{proposal}, ""}) 1006 for i := 0; i < blockParts.Total(); i++ { 1007 part := blockParts.GetPart(i) 1008 cs.sendInternalMessage(msgInfo{&BlockPartMessage{cs.Height, cs.Round, part}, ""}) 1009 } 1010 cs.Logger.Info("Signed proposal", "height", height, "round", round, "proposal", proposal) 1011 cs.Logger.Debug(fmt.Sprintf("Signed proposal block: %v", block)) 1012 } else if !cs.replayMode { 1013 cs.Logger.Error("enterPropose: Error signing proposal", "height", height, "round", round, "err", err) 1014 } 1015 } 1016 1017 // Returns true if the proposal block is complete && 1018 // (if POLRound was proposed, we have +2/3 prevotes from there). 1019 func (cs *State) isProposalComplete() bool { 1020 if cs.Proposal == nil || cs.ProposalBlock == nil { 1021 return false 1022 } 1023 // we have the proposal. if there's a POLRound, 1024 // make sure we have the prevotes from it too 1025 if cs.Proposal.POLRound < 0 { 1026 return true 1027 } 1028 // if this is false the proposer is lying or we haven't received the POL yet 1029 return cs.Votes.Prevotes(cs.Proposal.POLRound).HasTwoThirdsMajority() 1030 1031 } 1032 1033 // Create the next block to propose and return it. Returns nil block upon error. 1034 // 1035 // We really only need to return the parts, but the block is returned for 1036 // convenience so we can log the proposal block. 1037 // 1038 // NOTE: keep it side-effect free for clarity. 1039 // CONTRACT: cs.privValidator is not nil. 1040 func (cs *State) createProposalBlock() (block *types.Block, blockParts *types.PartSet) { 1041 if cs.privValidator == nil { 1042 panic("entered createProposalBlock with privValidator being nil") 1043 } 1044 1045 var commit *types.Commit 1046 switch { 1047 case cs.Height == 1: 1048 // We're creating a proposal for the first block. 1049 // The commit is empty, but not nil. 1050 commit = types.NewCommit(0, 0, types.BlockID{}, nil) 1051 case cs.LastCommit.HasTwoThirdsMajority(): 1052 // Make the commit from LastCommit 1053 commit = cs.LastCommit.MakeCommit() 1054 default: // This shouldn't happen. 1055 cs.Logger.Error("enterPropose: Cannot propose anything: No commit for the previous block") 1056 return 1057 } 1058 1059 if cs.privValidatorPubKey == nil { 1060 // If this node is a validator & proposer in the current round, it will 1061 // miss the opportunity to create a block. 1062 cs.Logger.Error(fmt.Sprintf("enterPropose: %v", errPubKeyIsNotSet)) 1063 return 1064 } 1065 proposerAddr := cs.privValidatorPubKey.Address() 1066 1067 return cs.blockExec.CreateProposalBlock(cs.Height, cs.state, commit, proposerAddr) 1068 } 1069 1070 // Enter: `timeoutPropose` after entering Propose. 1071 // Enter: proposal block and POL is ready. 1072 // Prevote for LockedBlock if we're locked, or ProposalBlock if valid. 1073 // Otherwise vote nil. 1074 func (cs *State) enterPrevote(height int64, round int) { 1075 if cs.Height != height || round < cs.Round || (cs.Round == round && cstypes.RoundStepPrevote <= cs.Step) { 1076 cs.Logger.Debug(fmt.Sprintf( 1077 "enterPrevote(%v/%v): Invalid args. Current step: %v/%v/%v", 1078 height, 1079 round, 1080 cs.Height, 1081 cs.Round, 1082 cs.Step)) 1083 return 1084 } 1085 1086 defer func() { 1087 // Done enterPrevote: 1088 cs.updateRoundStep(round, cstypes.RoundStepPrevote) 1089 cs.newStep() 1090 }() 1091 1092 cs.Logger.Info(fmt.Sprintf("enterPrevote(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step)) 1093 1094 // Sign and broadcast vote as necessary 1095 cs.doPrevote(height, round) 1096 1097 // Once `addVote` hits any +2/3 prevotes, we will go to PrevoteWait 1098 // (so we have more time to try and collect +2/3 prevotes for a single block) 1099 } 1100 1101 func (cs *State) defaultDoPrevote(height int64, round int) { 1102 logger := cs.Logger.With("height", height, "round", round) 1103 1104 // If a block is locked, prevote that. 1105 if cs.LockedBlock != nil { 1106 logger.Info("enterPrevote: Block was locked") 1107 cs.signAddVote(types.PrevoteType, cs.LockedBlock.Hash(), cs.LockedBlockParts.Header()) 1108 return 1109 } 1110 1111 // If ProposalBlock is nil, prevote nil. 1112 if cs.ProposalBlock == nil { 1113 logger.Info("enterPrevote: ProposalBlock is nil") 1114 cs.signAddVote(types.PrevoteType, nil, types.PartSetHeader{}) 1115 return 1116 } 1117 1118 // Validate proposal block 1119 err := cs.blockExec.ValidateBlock(cs.state, cs.ProposalBlock) 1120 if err != nil { 1121 // ProposalBlock is invalid, prevote nil. 1122 logger.Error("enterPrevote: ProposalBlock is invalid", "err", err) 1123 cs.signAddVote(types.PrevoteType, nil, types.PartSetHeader{}) 1124 return 1125 } 1126 1127 // Prevote cs.ProposalBlock 1128 // NOTE: the proposal signature is validated when it is received, 1129 // and the proposal block parts are validated as they are received (against the merkle hash in the proposal) 1130 logger.Info("enterPrevote: ProposalBlock is valid") 1131 cs.signAddVote(types.PrevoteType, cs.ProposalBlock.Hash(), cs.ProposalBlockParts.Header()) 1132 } 1133 1134 // Enter: any +2/3 prevotes at next round. 1135 func (cs *State) enterPrevoteWait(height int64, round int) { 1136 logger := cs.Logger.With("height", height, "round", round) 1137 1138 if cs.Height != height || round < cs.Round || (cs.Round == round && cstypes.RoundStepPrevoteWait <= cs.Step) { 1139 logger.Debug(fmt.Sprintf( 1140 "enterPrevoteWait(%v/%v): Invalid args. Current step: %v/%v/%v", 1141 height, 1142 round, 1143 cs.Height, 1144 cs.Round, 1145 cs.Step)) 1146 return 1147 } 1148 if !cs.Votes.Prevotes(round).HasTwoThirdsAny() { 1149 panic(fmt.Sprintf("enterPrevoteWait(%v/%v), but Prevotes does not have any +2/3 votes", height, round)) 1150 } 1151 logger.Info(fmt.Sprintf("enterPrevoteWait(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step)) 1152 1153 defer func() { 1154 // Done enterPrevoteWait: 1155 cs.updateRoundStep(round, cstypes.RoundStepPrevoteWait) 1156 cs.newStep() 1157 }() 1158 1159 // Wait for some more prevotes; enterPrecommit 1160 cs.scheduleTimeout(cs.config.Prevote(round), height, round, cstypes.RoundStepPrevoteWait) 1161 } 1162 1163 // Enter: `timeoutPrevote` after any +2/3 prevotes. 1164 // Enter: `timeoutPrecommit` after any +2/3 precommits. 1165 // Enter: +2/3 precomits for block or nil. 1166 // Lock & precommit the ProposalBlock if we have enough prevotes for it (a POL in this round) 1167 // else, unlock an existing lock and precommit nil if +2/3 of prevotes were nil, 1168 // else, precommit nil otherwise. 1169 func (cs *State) enterPrecommit(height int64, round int) { 1170 logger := cs.Logger.With("height", height, "round", round) 1171 1172 if cs.Height != height || round < cs.Round || (cs.Round == round && cstypes.RoundStepPrecommit <= cs.Step) { 1173 logger.Debug(fmt.Sprintf( 1174 "enterPrecommit(%v/%v): Invalid args. Current step: %v/%v/%v", 1175 height, 1176 round, 1177 cs.Height, 1178 cs.Round, 1179 cs.Step)) 1180 return 1181 } 1182 1183 logger.Info(fmt.Sprintf("enterPrecommit(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step)) 1184 1185 defer func() { 1186 // Done enterPrecommit: 1187 cs.updateRoundStep(round, cstypes.RoundStepPrecommit) 1188 cs.newStep() 1189 }() 1190 1191 // check for a polka 1192 blockID, ok := cs.Votes.Prevotes(round).TwoThirdsMajority() 1193 1194 // If we don't have a polka, we must precommit nil. 1195 if !ok { 1196 if cs.LockedBlock != nil { 1197 logger.Info("enterPrecommit: No +2/3 prevotes during enterPrecommit while we're locked. Precommitting nil") 1198 } else { 1199 logger.Info("enterPrecommit: No +2/3 prevotes during enterPrecommit. Precommitting nil.") 1200 } 1201 cs.signAddVote(types.PrecommitType, nil, types.PartSetHeader{}) 1202 return 1203 } 1204 1205 // At this point +2/3 prevoted for a particular block or nil. 1206 cs.eventBus.PublishEventPolka(cs.RoundStateEvent()) 1207 1208 // the latest POLRound should be this round. 1209 polRound, _ := cs.Votes.POLInfo() 1210 if polRound < round { 1211 panic(fmt.Sprintf("This POLRound should be %v but got %v", round, polRound)) 1212 } 1213 1214 // +2/3 prevoted nil. Unlock and precommit nil. 1215 if len(blockID.Hash) == 0 { 1216 if cs.LockedBlock == nil { 1217 logger.Info("enterPrecommit: +2/3 prevoted for nil.") 1218 } else { 1219 logger.Info("enterPrecommit: +2/3 prevoted for nil. Unlocking") 1220 cs.LockedRound = -1 1221 cs.LockedBlock = nil 1222 cs.LockedBlockParts = nil 1223 cs.eventBus.PublishEventUnlock(cs.RoundStateEvent()) 1224 } 1225 cs.signAddVote(types.PrecommitType, nil, types.PartSetHeader{}) 1226 return 1227 } 1228 1229 // At this point, +2/3 prevoted for a particular block. 1230 1231 // If we're already locked on that block, precommit it, and update the LockedRound 1232 if cs.LockedBlock.HashesTo(blockID.Hash) { 1233 logger.Info("enterPrecommit: +2/3 prevoted locked block. Relocking") 1234 cs.LockedRound = round 1235 cs.eventBus.PublishEventRelock(cs.RoundStateEvent()) 1236 cs.signAddVote(types.PrecommitType, blockID.Hash, blockID.PartsHeader) 1237 return 1238 } 1239 1240 // If +2/3 prevoted for proposal block, stage and precommit it 1241 if cs.ProposalBlock.HashesTo(blockID.Hash) { 1242 logger.Info("enterPrecommit: +2/3 prevoted proposal block. Locking", "hash", blockID.Hash) 1243 // Validate the block. 1244 if err := cs.blockExec.ValidateBlock(cs.state, cs.ProposalBlock); err != nil { 1245 panic(fmt.Sprintf("enterPrecommit: +2/3 prevoted for an invalid block: %v", err)) 1246 } 1247 cs.LockedRound = round 1248 cs.LockedBlock = cs.ProposalBlock 1249 cs.LockedBlockParts = cs.ProposalBlockParts 1250 cs.eventBus.PublishEventLock(cs.RoundStateEvent()) 1251 cs.signAddVote(types.PrecommitType, blockID.Hash, blockID.PartsHeader) 1252 return 1253 } 1254 1255 // There was a polka in this round for a block we don't have. 1256 // Fetch that block, unlock, and precommit nil. 1257 // The +2/3 prevotes for this round is the POL for our unlock. 1258 // TODO: In the future save the POL prevotes for justification. 1259 cs.LockedRound = -1 1260 cs.LockedBlock = nil 1261 cs.LockedBlockParts = nil 1262 if !cs.ProposalBlockParts.HasHeader(blockID.PartsHeader) { 1263 cs.ProposalBlock = nil 1264 cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartsHeader) 1265 } 1266 cs.eventBus.PublishEventUnlock(cs.RoundStateEvent()) 1267 cs.signAddVote(types.PrecommitType, nil, types.PartSetHeader{}) 1268 } 1269 1270 // Enter: any +2/3 precommits for next round. 1271 func (cs *State) enterPrecommitWait(height int64, round int) { 1272 logger := cs.Logger.With("height", height, "round", round) 1273 1274 if cs.Height != height || round < cs.Round || (cs.Round == round && cs.TriggeredTimeoutPrecommit) { 1275 logger.Debug( 1276 fmt.Sprintf( 1277 "enterPrecommitWait(%v/%v): Invalid args. "+ 1278 "Current state is Height/Round: %v/%v/, TriggeredTimeoutPrecommit:%v", 1279 height, round, cs.Height, cs.Round, cs.TriggeredTimeoutPrecommit)) 1280 return 1281 } 1282 if !cs.Votes.Precommits(round).HasTwoThirdsAny() { 1283 panic(fmt.Sprintf("enterPrecommitWait(%v/%v), but Precommits does not have any +2/3 votes", height, round)) 1284 } 1285 logger.Info(fmt.Sprintf("enterPrecommitWait(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step)) 1286 1287 defer func() { 1288 // Done enterPrecommitWait: 1289 cs.TriggeredTimeoutPrecommit = true 1290 cs.newStep() 1291 }() 1292 1293 // Wait for some more precommits; enterNewRound 1294 cs.scheduleTimeout(cs.config.Precommit(round), height, round, cstypes.RoundStepPrecommitWait) 1295 1296 } 1297 1298 // Enter: +2/3 precommits for block 1299 func (cs *State) enterCommit(height int64, commitRound int) { 1300 logger := cs.Logger.With("height", height, "commitRound", commitRound) 1301 1302 if cs.Height != height || cstypes.RoundStepCommit <= cs.Step { 1303 logger.Debug(fmt.Sprintf( 1304 "enterCommit(%v/%v): Invalid args. Current step: %v/%v/%v", 1305 height, 1306 commitRound, 1307 cs.Height, 1308 cs.Round, 1309 cs.Step)) 1310 return 1311 } 1312 logger.Info(fmt.Sprintf("enterCommit(%v/%v). Current: %v/%v/%v", height, commitRound, cs.Height, cs.Round, cs.Step)) 1313 1314 defer func() { 1315 // Done enterCommit: 1316 // keep cs.Round the same, commitRound points to the right Precommits set. 1317 cs.updateRoundStep(cs.Round, cstypes.RoundStepCommit) 1318 cs.CommitRound = commitRound 1319 cs.CommitTime = tmtime.Now() 1320 cs.newStep() 1321 1322 // Maybe finalize immediately. 1323 cs.tryFinalizeCommit(height) 1324 }() 1325 1326 blockID, ok := cs.Votes.Precommits(commitRound).TwoThirdsMajority() 1327 if !ok { 1328 panic("RunActionCommit() expects +2/3 precommits") 1329 } 1330 1331 // The Locked* fields no longer matter. 1332 // Move them over to ProposalBlock if they match the commit hash, 1333 // otherwise they'll be cleared in updateToState. 1334 if cs.LockedBlock.HashesTo(blockID.Hash) { 1335 logger.Info("Commit is for locked block. Set ProposalBlock=LockedBlock", "blockHash", blockID.Hash) 1336 cs.ProposalBlock = cs.LockedBlock 1337 cs.ProposalBlockParts = cs.LockedBlockParts 1338 } 1339 1340 // If we don't have the block being committed, set up to get it. 1341 if !cs.ProposalBlock.HashesTo(blockID.Hash) { 1342 if !cs.ProposalBlockParts.HasHeader(blockID.PartsHeader) { 1343 logger.Info( 1344 "Commit is for a block we don't know about. Set ProposalBlock=nil", 1345 "proposal", 1346 cs.ProposalBlock.Hash(), 1347 "commit", 1348 blockID.Hash) 1349 // We're getting the wrong block. 1350 // Set up ProposalBlockParts and keep waiting. 1351 cs.ProposalBlock = nil 1352 cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartsHeader) 1353 cs.eventBus.PublishEventValidBlock(cs.RoundStateEvent()) 1354 cs.evsw.FireEvent(types.EventValidBlock, &cs.RoundState) 1355 } 1356 // else { 1357 // We just need to keep waiting. 1358 // } 1359 } 1360 } 1361 1362 // If we have the block AND +2/3 commits for it, finalize. 1363 func (cs *State) tryFinalizeCommit(height int64) { 1364 logger := cs.Logger.With("height", height) 1365 1366 if cs.Height != height { 1367 panic(fmt.Sprintf("tryFinalizeCommit() cs.Height: %v vs height: %v", cs.Height, height)) 1368 } 1369 1370 blockID, ok := cs.Votes.Precommits(cs.CommitRound).TwoThirdsMajority() 1371 if !ok || len(blockID.Hash) == 0 { 1372 logger.Error("Attempt to finalize failed. There was no +2/3 majority, or +2/3 was for <nil>.") 1373 return 1374 } 1375 if !cs.ProposalBlock.HashesTo(blockID.Hash) { 1376 // TODO: this happens every time if we're not a validator (ugly logs) 1377 // TODO: ^^ wait, why does it matter that we're a validator? 1378 logger.Info( 1379 "Attempt to finalize failed. We don't have the commit block.", 1380 "proposal-block", 1381 cs.ProposalBlock.Hash(), 1382 "commit-block", 1383 blockID.Hash) 1384 return 1385 } 1386 1387 // go 1388 cs.finalizeCommit(height) 1389 } 1390 1391 // Increment height and goto cstypes.RoundStepNewHeight 1392 func (cs *State) finalizeCommit(height int64) { 1393 if cs.Height != height || cs.Step != cstypes.RoundStepCommit { 1394 cs.Logger.Debug(fmt.Sprintf( 1395 "finalizeCommit(%v): Invalid args. Current step: %v/%v/%v", 1396 height, 1397 cs.Height, 1398 cs.Round, 1399 cs.Step)) 1400 return 1401 } 1402 1403 blockID, ok := cs.Votes.Precommits(cs.CommitRound).TwoThirdsMajority() 1404 block, blockParts := cs.ProposalBlock, cs.ProposalBlockParts 1405 1406 if !ok { 1407 panic(fmt.Sprintf("Cannot finalizeCommit, commit does not have two thirds majority")) 1408 } 1409 if !blockParts.HasHeader(blockID.PartsHeader) { 1410 panic(fmt.Sprintf("Expected ProposalBlockParts header to be commit header")) 1411 } 1412 if !block.HashesTo(blockID.Hash) { 1413 panic(fmt.Sprintf("Cannot finalizeCommit, ProposalBlock does not hash to commit hash")) 1414 } 1415 if err := cs.blockExec.ValidateBlock(cs.state, block); err != nil { 1416 panic(fmt.Sprintf("+2/3 committed an invalid block: %v", err)) 1417 } 1418 1419 cs.Logger.Info("Finalizing commit of block with N txs", 1420 "height", block.Height, 1421 "hash", block.Hash(), 1422 "root", block.AppHash, 1423 "N", len(block.Txs)) 1424 cs.Logger.Info(fmt.Sprintf("%v", block)) 1425 1426 fail.Fail() // XXX 1427 1428 // Save to blockStore. 1429 if cs.blockStore.Height() < block.Height { 1430 // NOTE: the seenCommit is local justification to commit this block, 1431 // but may differ from the LastCommit included in the next block 1432 precommits := cs.Votes.Precommits(cs.CommitRound) 1433 seenCommit := precommits.MakeCommit() 1434 cs.blockStore.SaveBlock(block, blockParts, seenCommit) 1435 } else { 1436 // Happens during replay if we already saved the block but didn't commit 1437 cs.Logger.Info("Calling finalizeCommit on already stored block", "height", block.Height) 1438 } 1439 1440 fail.Fail() // XXX 1441 1442 // Write EndHeightMessage{} for this height, implying that the blockstore 1443 // has saved the block. 1444 // 1445 // If we crash before writing this EndHeightMessage{}, we will recover by 1446 // running ApplyBlock during the ABCI handshake when we restart. If we 1447 // didn't save the block to the blockstore before writing 1448 // EndHeightMessage{}, we'd have to change WAL replay -- currently it 1449 // complains about replaying for heights where an #ENDHEIGHT entry already 1450 // exists. 1451 // 1452 // Either way, the State should not be resumed until we 1453 // successfully call ApplyBlock (ie. later here, or in Handshake after 1454 // restart). 1455 endMsg := EndHeightMessage{height} 1456 if err := cs.wal.WriteSync(endMsg); err != nil { // NOTE: fsync 1457 panic(fmt.Sprintf("Failed to write %v msg to consensus wal due to %v. Check your FS and restart the node", 1458 endMsg, err)) 1459 } 1460 1461 fail.Fail() // XXX 1462 1463 // Create a copy of the state for staging and an event cache for txs. 1464 stateCopy := cs.state.Copy() 1465 1466 // Execute and commit the block, update and save the state, and update the mempool. 1467 // NOTE The block.AppHash wont reflect these txs until the next block. 1468 var err error 1469 var retainHeight int64 1470 stateCopy, retainHeight, err = cs.blockExec.ApplyBlock( 1471 stateCopy, 1472 types.BlockID{Hash: block.Hash(), PartsHeader: blockParts.Header()}, 1473 block) 1474 if err != nil { 1475 cs.Logger.Error("Error on ApplyBlock. Did the application crash? Please restart tendermint", "err", err) 1476 err := tmos.Kill() 1477 if err != nil { 1478 cs.Logger.Error("Failed to kill this process - please do so manually", "err", err) 1479 } 1480 return 1481 } 1482 1483 fail.Fail() // XXX 1484 1485 // Prune old heights, if requested by ABCI app. 1486 if retainHeight > 0 { 1487 pruned, err := cs.pruneBlocks(retainHeight) 1488 if err != nil { 1489 cs.Logger.Error("Failed to prune blocks", "retainHeight", retainHeight, "err", err) 1490 } else { 1491 cs.Logger.Info("Pruned blocks", "pruned", pruned, "retainHeight", retainHeight) 1492 } 1493 } 1494 1495 // must be called before we update state 1496 cs.recordMetrics(height, block) 1497 1498 // NewHeightStep! 1499 cs.updateToState(stateCopy) 1500 1501 fail.Fail() // XXX 1502 1503 // Private validator might have changed it's key pair => refetch pubkey. 1504 if err := cs.updatePrivValidatorPubKey(); err != nil { 1505 cs.Logger.Error("Can't get private validator pubkey", "err", err) 1506 } 1507 1508 // cs.StartTime is already set. 1509 // Schedule Round0 to start soon. 1510 cs.scheduleRound0(&cs.RoundState) 1511 1512 // By here, 1513 // * cs.Height has been increment to height+1 1514 // * cs.Step is now cstypes.RoundStepNewHeight 1515 // * cs.StartTime is set to when we will start round0. 1516 } 1517 1518 func (cs *State) pruneBlocks(retainHeight int64) (uint64, error) { 1519 base := cs.blockStore.Base() 1520 if retainHeight <= base { 1521 return 0, nil 1522 } 1523 pruned, err := cs.blockStore.PruneBlocks(retainHeight) 1524 if err != nil { 1525 return 0, fmt.Errorf("failed to prune block store: %w", err) 1526 } 1527 err = sm.PruneStates(cs.blockExec.DB(), base, retainHeight) 1528 if err != nil { 1529 return 0, fmt.Errorf("failed to prune state database: %w", err) 1530 } 1531 return pruned, nil 1532 } 1533 1534 func (cs *State) recordMetrics(height int64, block *types.Block) { 1535 cs.metrics.Validators.Set(float64(cs.Validators.Size())) 1536 cs.metrics.ValidatorsPower.Set(float64(cs.Validators.TotalVotingPower())) 1537 1538 var ( 1539 missingValidators int 1540 missingValidatorsPower int64 1541 ) 1542 // height=0 -> MissingValidators and MissingValidatorsPower are both 0. 1543 // Remember that the first LastCommit is intentionally empty, so it's not 1544 // fair to increment missing validators number. 1545 if height > 1 { 1546 // Sanity check that commit size matches validator set size - only applies 1547 // after first block. 1548 var ( 1549 commitSize = block.LastCommit.Size() 1550 valSetLen = len(cs.LastValidators.Validators) 1551 address types.Address 1552 ) 1553 if commitSize != valSetLen { 1554 panic(fmt.Sprintf("commit size (%d) doesn't match valset length (%d) at height %d\n\n%v\n\n%v", 1555 commitSize, valSetLen, block.Height, block.LastCommit.Signatures, cs.LastValidators.Validators)) 1556 } 1557 1558 if cs.privValidator != nil { 1559 if cs.privValidatorPubKey == nil { 1560 // Metrics won't be updated, but it's not critical. 1561 cs.Logger.Error(fmt.Sprintf("recordMetrics: %v", errPubKeyIsNotSet)) 1562 } else { 1563 address = cs.privValidatorPubKey.Address() 1564 } 1565 } 1566 1567 for i, val := range cs.LastValidators.Validators { 1568 commitSig := block.LastCommit.Signatures[i] 1569 if commitSig.Absent() { 1570 missingValidators++ 1571 missingValidatorsPower += val.VotingPower 1572 } 1573 1574 if bytes.Equal(val.Address, address) { 1575 label := []string{ 1576 "validator_address", val.Address.String(), 1577 } 1578 cs.metrics.ValidatorPower.With(label...).Set(float64(val.VotingPower)) 1579 if commitSig.ForBlock() { 1580 cs.metrics.ValidatorLastSignedHeight.With(label...).Set(float64(height)) 1581 } else { 1582 cs.metrics.ValidatorMissedBlocks.With(label...).Add(float64(1)) 1583 } 1584 } 1585 1586 } 1587 } 1588 cs.metrics.MissingValidators.Set(float64(missingValidators)) 1589 cs.metrics.MissingValidatorsPower.Set(float64(missingValidatorsPower)) 1590 1591 cs.metrics.ByzantineValidators.Set(float64(len(block.Evidence.Evidence))) 1592 byzantineValidatorsPower := int64(0) 1593 for _, ev := range block.Evidence.Evidence { 1594 if _, val := cs.Validators.GetByAddress(ev.Address()); val != nil { 1595 byzantineValidatorsPower += val.VotingPower 1596 } 1597 } 1598 cs.metrics.ByzantineValidatorsPower.Set(float64(byzantineValidatorsPower)) 1599 1600 if height > 1 { 1601 lastBlockMeta := cs.blockStore.LoadBlockMeta(height - 1) 1602 if lastBlockMeta != nil { 1603 cs.metrics.BlockIntervalSeconds.Set( 1604 block.Time.Sub(lastBlockMeta.Header.Time).Seconds(), 1605 ) 1606 } 1607 } 1608 1609 cs.metrics.NumTxs.Set(float64(len(block.Data.Txs))) 1610 cs.metrics.TotalTxs.Add(float64(len(block.Data.Txs))) 1611 cs.metrics.BlockSizeBytes.Set(float64(block.Size())) 1612 cs.metrics.CommittedHeight.Set(float64(block.Height)) 1613 } 1614 1615 //----------------------------------------------------------------------------- 1616 1617 func (cs *State) defaultSetProposal(proposal *types.Proposal) error { 1618 // Already have one 1619 // TODO: possibly catch double proposals 1620 if cs.Proposal != nil { 1621 return nil 1622 } 1623 1624 // Does not apply 1625 if proposal.Height != cs.Height || proposal.Round != cs.Round { 1626 return nil 1627 } 1628 1629 // Verify POLRound, which must be -1 or in range [0, proposal.Round). 1630 if proposal.POLRound < -1 || 1631 (proposal.POLRound >= 0 && proposal.POLRound >= proposal.Round) { 1632 return ErrInvalidProposalPOLRound 1633 } 1634 1635 // Verify signature 1636 if !cs.Validators.GetProposer().PubKey.VerifyBytes(proposal.SignBytes(cs.state.ChainID), proposal.Signature) { 1637 return ErrInvalidProposalSignature 1638 } 1639 1640 cs.Proposal = proposal 1641 // We don't update cs.ProposalBlockParts if it is already set. 1642 // This happens if we're already in cstypes.RoundStepCommit or if there is a valid block in the current round. 1643 // TODO: We can check if Proposal is for a different block as this is a sign of misbehavior! 1644 if cs.ProposalBlockParts == nil { 1645 cs.ProposalBlockParts = types.NewPartSetFromHeader(proposal.BlockID.PartsHeader) 1646 } 1647 cs.Logger.Info("Received proposal", "proposal", proposal) 1648 return nil 1649 } 1650 1651 // NOTE: block is not necessarily valid. 1652 // Asynchronously triggers either enterPrevote (before we timeout of propose) or tryFinalizeCommit, 1653 // once we have the full block. 1654 func (cs *State) addProposalBlockPart(msg *BlockPartMessage, peerID p2p.ID) (added bool, err error) { 1655 height, round, part := msg.Height, msg.Round, msg.Part 1656 1657 // Blocks might be reused, so round mismatch is OK 1658 if cs.Height != height { 1659 cs.Logger.Debug("Received block part from wrong height", "height", height, "round", round) 1660 return false, nil 1661 } 1662 1663 // We're not expecting a block part. 1664 if cs.ProposalBlockParts == nil { 1665 // NOTE: this can happen when we've gone to a higher round and 1666 // then receive parts from the previous round - not necessarily a bad peer. 1667 cs.Logger.Info("Received a block part when we're not expecting any", 1668 "height", height, "round", round, "index", part.Index, "peer", peerID) 1669 return false, nil 1670 } 1671 1672 added, err = cs.ProposalBlockParts.AddPart(part) 1673 if err != nil { 1674 return added, err 1675 } 1676 if added && cs.ProposalBlockParts.IsComplete() { 1677 // Added and completed! 1678 _, err = cdc.UnmarshalBinaryLengthPrefixedReader( 1679 cs.ProposalBlockParts.GetReader(), 1680 &cs.ProposalBlock, 1681 cs.state.ConsensusParams.Block.MaxBytes, 1682 ) 1683 if err != nil { 1684 return added, err 1685 } 1686 // NOTE: it's possible to receive complete proposal blocks for future rounds without having the proposal 1687 cs.Logger.Info("Received complete proposal block", "height", cs.ProposalBlock.Height, "hash", cs.ProposalBlock.Hash()) 1688 cs.eventBus.PublishEventCompleteProposal(cs.CompleteProposalEvent()) 1689 1690 // Update Valid* if we can. 1691 prevotes := cs.Votes.Prevotes(cs.Round) 1692 blockID, hasTwoThirds := prevotes.TwoThirdsMajority() 1693 if hasTwoThirds && !blockID.IsZero() && (cs.ValidRound < cs.Round) { 1694 if cs.ProposalBlock.HashesTo(blockID.Hash) { 1695 cs.Logger.Info("Updating valid block to new proposal block", 1696 "valid-round", cs.Round, "valid-block-hash", cs.ProposalBlock.Hash()) 1697 cs.ValidRound = cs.Round 1698 cs.ValidBlock = cs.ProposalBlock 1699 cs.ValidBlockParts = cs.ProposalBlockParts 1700 } 1701 // TODO: In case there is +2/3 majority in Prevotes set for some 1702 // block and cs.ProposalBlock contains different block, either 1703 // proposer is faulty or voting power of faulty processes is more 1704 // than 1/3. We should trigger in the future accountability 1705 // procedure at this point. 1706 } 1707 1708 if cs.Step <= cstypes.RoundStepPropose && cs.isProposalComplete() { 1709 // Move onto the next step 1710 cs.enterPrevote(height, cs.Round) 1711 if hasTwoThirds { // this is optimisation as this will be triggered when prevote is added 1712 cs.enterPrecommit(height, cs.Round) 1713 } 1714 } else if cs.Step == cstypes.RoundStepCommit { 1715 // If we're waiting on the proposal block... 1716 cs.tryFinalizeCommit(height) 1717 } 1718 return added, nil 1719 } 1720 return added, nil 1721 } 1722 1723 // Attempt to add the vote. if its a duplicate signature, dupeout the validator 1724 func (cs *State) tryAddVote(vote *types.Vote, peerID p2p.ID) (bool, error) { 1725 added, err := cs.addVote(vote, peerID) 1726 if err != nil { 1727 // If the vote height is off, we'll just ignore it, 1728 // But if it's a conflicting sig, add it to the cs.evpool. 1729 // If it's otherwise invalid, punish peer. 1730 // nolint: gocritic 1731 if err == ErrVoteHeightMismatch { 1732 return added, err 1733 } else if voteErr, ok := err.(*types.ErrVoteConflictingVotes); ok { 1734 if cs.privValidatorPubKey == nil { 1735 return false, errPubKeyIsNotSet 1736 } 1737 1738 if bytes.Equal(vote.ValidatorAddress, cs.privValidatorPubKey.Address()) { 1739 cs.Logger.Error( 1740 "Found conflicting vote from ourselves. Did you unsafe_reset a validator?", 1741 "height", 1742 vote.Height, 1743 "round", 1744 vote.Round, 1745 "type", 1746 vote.Type) 1747 return added, err 1748 } 1749 cs.evpool.AddEvidence(voteErr.DuplicateVoteEvidence) 1750 return added, err 1751 } else if err == types.ErrVoteNonDeterministicSignature { 1752 cs.Logger.Debug("Vote has non-deterministic signature", "err", err) 1753 } else { 1754 // Either 1755 // 1) bad peer OR 1756 // 2) not a bad peer? this can also err sometimes with "Unexpected step" OR 1757 // 3) tmkms use with multiple validators connecting to a single tmkms instance 1758 // (https://github.com/tendermint/tendermint/issues/3839). 1759 cs.Logger.Info("Error attempting to add vote", "err", err) 1760 return added, ErrAddingVote 1761 } 1762 } 1763 return added, nil 1764 } 1765 1766 //----------------------------------------------------------------------------- 1767 1768 func (cs *State) addVote( 1769 vote *types.Vote, 1770 peerID p2p.ID) (added bool, err error) { 1771 cs.Logger.Debug( 1772 "addVote", 1773 "voteHeight", 1774 vote.Height, 1775 "voteType", 1776 vote.Type, 1777 "valIndex", 1778 vote.ValidatorIndex, 1779 "csHeight", 1780 cs.Height, 1781 ) 1782 1783 // A precommit for the previous height? 1784 // These come in while we wait timeoutCommit 1785 if vote.Height+1 == cs.Height { 1786 if !(cs.Step == cstypes.RoundStepNewHeight && vote.Type == types.PrecommitType) { 1787 // TODO: give the reason .. 1788 // fmt.Errorf("tryAddVote: Wrong height, not a LastCommit straggler commit.") 1789 return added, ErrVoteHeightMismatch 1790 } 1791 added, err = cs.LastCommit.AddVote(vote) 1792 if !added { 1793 return added, err 1794 } 1795 1796 cs.Logger.Info(fmt.Sprintf("Added to lastPrecommits: %v", cs.LastCommit.StringShort())) 1797 cs.eventBus.PublishEventVote(types.EventDataVote{Vote: vote}) 1798 cs.evsw.FireEvent(types.EventVote, vote) 1799 1800 // if we can skip timeoutCommit and have all the votes now, 1801 if cs.config.SkipTimeoutCommit && cs.LastCommit.HasAll() { 1802 // go straight to new round (skip timeout commit) 1803 // cs.scheduleTimeout(time.Duration(0), cs.Height, 0, cstypes.RoundStepNewHeight) 1804 cs.enterNewRound(cs.Height, 0) 1805 } 1806 1807 return 1808 } 1809 1810 // Height mismatch is ignored. 1811 // Not necessarily a bad peer, but not favourable behaviour. 1812 if vote.Height != cs.Height { 1813 err = ErrVoteHeightMismatch 1814 cs.Logger.Info("Vote ignored and not added", "voteHeight", vote.Height, "csHeight", cs.Height, "peerID", peerID) 1815 return 1816 } 1817 1818 height := cs.Height 1819 added, err = cs.Votes.AddVote(vote, peerID) 1820 if !added { 1821 // Either duplicate, or error upon cs.Votes.AddByIndex() 1822 return 1823 } 1824 1825 cs.eventBus.PublishEventVote(types.EventDataVote{Vote: vote}) 1826 cs.evsw.FireEvent(types.EventVote, vote) 1827 1828 switch vote.Type { 1829 case types.PrevoteType: 1830 prevotes := cs.Votes.Prevotes(vote.Round) 1831 cs.Logger.Info("Added to prevote", "vote", vote, "prevotes", prevotes.StringShort()) 1832 1833 // If +2/3 prevotes for a block or nil for *any* round: 1834 if blockID, ok := prevotes.TwoThirdsMajority(); ok { 1835 1836 // There was a polka! 1837 // If we're locked but this is a recent polka, unlock. 1838 // If it matches our ProposalBlock, update the ValidBlock 1839 1840 // Unlock if `cs.LockedRound < vote.Round <= cs.Round` 1841 // NOTE: If vote.Round > cs.Round, we'll deal with it when we get to vote.Round 1842 if (cs.LockedBlock != nil) && 1843 (cs.LockedRound < vote.Round) && 1844 (vote.Round <= cs.Round) && 1845 !cs.LockedBlock.HashesTo(blockID.Hash) { 1846 1847 cs.Logger.Info("Unlocking because of POL.", "lockedRound", cs.LockedRound, "POLRound", vote.Round) 1848 cs.LockedRound = -1 1849 cs.LockedBlock = nil 1850 cs.LockedBlockParts = nil 1851 cs.eventBus.PublishEventUnlock(cs.RoundStateEvent()) 1852 } 1853 1854 // Update Valid* if we can. 1855 // NOTE: our proposal block may be nil or not what received a polka.. 1856 if len(blockID.Hash) != 0 && (cs.ValidRound < vote.Round) && (vote.Round == cs.Round) { 1857 1858 if cs.ProposalBlock.HashesTo(blockID.Hash) { 1859 cs.Logger.Info( 1860 "Updating ValidBlock because of POL.", "validRound", cs.ValidRound, "POLRound", vote.Round) 1861 cs.ValidRound = vote.Round 1862 cs.ValidBlock = cs.ProposalBlock 1863 cs.ValidBlockParts = cs.ProposalBlockParts 1864 } else { 1865 cs.Logger.Info( 1866 "Valid block we don't know about. Set ProposalBlock=nil", 1867 "proposal", cs.ProposalBlock.Hash(), "blockID", blockID.Hash) 1868 // We're getting the wrong block. 1869 cs.ProposalBlock = nil 1870 } 1871 if !cs.ProposalBlockParts.HasHeader(blockID.PartsHeader) { 1872 cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartsHeader) 1873 } 1874 cs.evsw.FireEvent(types.EventValidBlock, &cs.RoundState) 1875 cs.eventBus.PublishEventValidBlock(cs.RoundStateEvent()) 1876 } 1877 } 1878 1879 // If +2/3 prevotes for *anything* for future round: 1880 switch { 1881 case cs.Round < vote.Round && prevotes.HasTwoThirdsAny(): 1882 // Round-skip if there is any 2/3+ of votes ahead of us 1883 cs.enterNewRound(height, vote.Round) 1884 case cs.Round == vote.Round && cstypes.RoundStepPrevote <= cs.Step: // current round 1885 blockID, ok := prevotes.TwoThirdsMajority() 1886 if ok && (cs.isProposalComplete() || len(blockID.Hash) == 0) { 1887 cs.enterPrecommit(height, vote.Round) 1888 } else if prevotes.HasTwoThirdsAny() { 1889 cs.enterPrevoteWait(height, vote.Round) 1890 } 1891 case cs.Proposal != nil && 0 <= cs.Proposal.POLRound && cs.Proposal.POLRound == vote.Round: 1892 // If the proposal is now complete, enter prevote of cs.Round. 1893 if cs.isProposalComplete() { 1894 cs.enterPrevote(height, cs.Round) 1895 } 1896 } 1897 1898 case types.PrecommitType: 1899 precommits := cs.Votes.Precommits(vote.Round) 1900 cs.Logger.Info("Added to precommit", "vote", vote, "precommits", precommits.StringShort()) 1901 1902 blockID, ok := precommits.TwoThirdsMajority() 1903 if ok { 1904 // Executed as TwoThirdsMajority could be from a higher round 1905 cs.enterNewRound(height, vote.Round) 1906 cs.enterPrecommit(height, vote.Round) 1907 if len(blockID.Hash) != 0 { 1908 cs.enterCommit(height, vote.Round) 1909 if cs.config.SkipTimeoutCommit && precommits.HasAll() { 1910 cs.enterNewRound(cs.Height, 0) 1911 } 1912 } else { 1913 cs.enterPrecommitWait(height, vote.Round) 1914 } 1915 } else if cs.Round <= vote.Round && precommits.HasTwoThirdsAny() { 1916 cs.enterNewRound(height, vote.Round) 1917 cs.enterPrecommitWait(height, vote.Round) 1918 } 1919 1920 default: 1921 panic(fmt.Sprintf("Unexpected vote type %X", vote.Type)) // go-amino should prevent this. 1922 } 1923 1924 return added, err 1925 } 1926 1927 // CONTRACT: cs.privValidator is not nil. 1928 func (cs *State) signVote( 1929 msgType types.SignedMsgType, 1930 hash []byte, 1931 header types.PartSetHeader, 1932 ) (*types.Vote, error) { 1933 // Flush the WAL. Otherwise, we may not recompute the same vote to sign, 1934 // and the privValidator will refuse to sign anything. 1935 cs.wal.FlushAndSync() 1936 1937 if cs.privValidatorPubKey == nil { 1938 return nil, errPubKeyIsNotSet 1939 } 1940 addr := cs.privValidatorPubKey.Address() 1941 valIdx, _ := cs.Validators.GetByAddress(addr) 1942 1943 vote := &types.Vote{ 1944 ValidatorAddress: addr, 1945 ValidatorIndex: valIdx, 1946 Height: cs.Height, 1947 Round: cs.Round, 1948 Timestamp: cs.voteTime(), 1949 Type: msgType, 1950 BlockID: types.BlockID{Hash: hash, PartsHeader: header}, 1951 } 1952 1953 err := cs.privValidator.SignVote(cs.state.ChainID, vote) 1954 return vote, err 1955 } 1956 1957 func (cs *State) voteTime() time.Time { 1958 now := tmtime.Now() 1959 minVoteTime := now 1960 // TODO: We should remove next line in case we don't vote for v in case cs.ProposalBlock == nil, 1961 // even if cs.LockedBlock != nil. See https://docs.tendermint.com/master/spec/. 1962 timeIotaMs := time.Duration(cs.state.ConsensusParams.Block.TimeIotaMs) * time.Millisecond 1963 if cs.LockedBlock != nil { 1964 // See the BFT time spec https://docs.tendermint.com/master/spec/consensus/bft-time.html 1965 minVoteTime = cs.LockedBlock.Time.Add(timeIotaMs) 1966 } else if cs.ProposalBlock != nil { 1967 minVoteTime = cs.ProposalBlock.Time.Add(timeIotaMs) 1968 } 1969 1970 if now.After(minVoteTime) { 1971 return now 1972 } 1973 return minVoteTime 1974 } 1975 1976 // sign the vote and publish on internalMsgQueue 1977 func (cs *State) signAddVote(msgType types.SignedMsgType, hash []byte, header types.PartSetHeader) *types.Vote { 1978 if cs.privValidator == nil { // the node does not have a key 1979 return nil 1980 } 1981 1982 if cs.privValidatorPubKey == nil { 1983 // Vote won't be signed, but it's not critical. 1984 cs.Logger.Error(fmt.Sprintf("signAddVote: %v", errPubKeyIsNotSet)) 1985 return nil 1986 } 1987 1988 // If the node not in the validator set, do nothing. 1989 if !cs.Validators.HasAddress(cs.privValidatorPubKey.Address()) { 1990 return nil 1991 } 1992 1993 // TODO: pass pubKey to signVote 1994 vote, err := cs.signVote(msgType, hash, header) 1995 if err == nil { 1996 cs.sendInternalMessage(msgInfo{&VoteMessage{vote}, ""}) 1997 cs.Logger.Info("Signed and pushed vote", "height", cs.Height, "round", cs.Round, "vote", vote, "err", err) 1998 return vote 1999 } 2000 //if !cs.replayMode { 2001 cs.Logger.Error("Error signing vote", "height", cs.Height, "round", cs.Round, "vote", vote, "err", err) 2002 //} 2003 return nil 2004 } 2005 2006 // updatePrivValidatorPubKey get's the private validator public key and 2007 // memoizes it. This func returns an error if the private validator is not 2008 // responding or responds with an error. 2009 func (cs *State) updatePrivValidatorPubKey() error { 2010 if cs.privValidator == nil { 2011 return nil 2012 } 2013 2014 pubKey, err := cs.privValidator.GetPubKey() 2015 if err != nil { 2016 return err 2017 } 2018 cs.privValidatorPubKey = pubKey 2019 return nil 2020 } 2021 2022 //--------------------------------------------------------- 2023 2024 func CompareHRS(h1 int64, r1 int, s1 cstypes.RoundStepType, h2 int64, r2 int, s2 cstypes.RoundStepType) int { 2025 if h1 < h2 { 2026 return -1 2027 } else if h1 > h2 { 2028 return 1 2029 } 2030 if r1 < r2 { 2031 return -1 2032 } else if r1 > r2 { 2033 return 1 2034 } 2035 if s1 < s2 { 2036 return -1 2037 } else if s1 > s2 { 2038 return 1 2039 } 2040 return 0 2041 }