github.com/Team-Kujira/tendermint@v0.34.24-indexer/consensus/reactor.go (about) 1 package consensus 2 3 import ( 4 "errors" 5 "fmt" 6 "reflect" 7 "sync" 8 "time" 9 10 "github.com/gogo/protobuf/proto" 11 cstypes "github.com/tendermint/tendermint/consensus/types" 12 "github.com/tendermint/tendermint/libs/bits" 13 tmevents "github.com/tendermint/tendermint/libs/events" 14 tmjson "github.com/tendermint/tendermint/libs/json" 15 "github.com/tendermint/tendermint/libs/log" 16 tmsync "github.com/tendermint/tendermint/libs/sync" 17 "github.com/tendermint/tendermint/p2p" 18 tmcons "github.com/tendermint/tendermint/proto/tendermint/consensus" 19 tmproto "github.com/tendermint/tendermint/proto/tendermint/types" 20 sm "github.com/tendermint/tendermint/state" 21 "github.com/tendermint/tendermint/types" 22 tmtime "github.com/tendermint/tendermint/types/time" 23 ) 24 25 const ( 26 StateChannel = byte(0x20) 27 DataChannel = byte(0x21) 28 VoteChannel = byte(0x22) 29 VoteSetBitsChannel = byte(0x23) 30 31 maxMsgSize = 1048576 // 1MB; NOTE/TODO: keep in sync with types.PartSet sizes. 32 33 blocksToContributeToBecomeGoodPeer = 10000 34 votesToContributeToBecomeGoodPeer = 10000 35 ) 36 37 //----------------------------------------------------------------------------- 38 39 // Reactor defines a reactor for the consensus service. 40 type Reactor struct { 41 p2p.BaseReactor // BaseService + p2p.Switch 42 43 conS *State 44 45 mtx tmsync.RWMutex 46 waitSync bool 47 eventBus *types.EventBus 48 rs *cstypes.RoundState 49 50 Metrics *Metrics 51 } 52 53 type ReactorOption func(*Reactor) 54 55 // NewReactor returns a new Reactor with the given 56 // consensusState. 57 func NewReactor(consensusState *State, waitSync bool, options ...ReactorOption) *Reactor { 58 conR := &Reactor{ 59 conS: consensusState, 60 waitSync: waitSync, 61 rs: consensusState.GetRoundState(), 62 Metrics: NopMetrics(), 63 } 64 conR.BaseReactor = *p2p.NewBaseReactor("Consensus", conR) 65 66 for _, option := range options { 67 option(conR) 68 } 69 70 return conR 71 } 72 73 // OnStart implements BaseService by subscribing to events, which later will be 74 // broadcasted to other peers and starting state if we're not in fast sync. 75 func (conR *Reactor) OnStart() error { 76 conR.Logger.Info("Reactor ", "waitSync", conR.WaitSync()) 77 78 // start routine that computes peer statistics for evaluating peer quality 79 go conR.peerStatsRoutine() 80 81 conR.subscribeToBroadcastEvents() 82 go conR.updateRoundStateRoutine() 83 84 if !conR.WaitSync() { 85 err := conR.conS.Start() 86 if err != nil { 87 return err 88 } 89 } 90 91 return nil 92 } 93 94 // OnStop implements BaseService by unsubscribing from events and stopping 95 // state. 96 func (conR *Reactor) OnStop() { 97 conR.unsubscribeFromBroadcastEvents() 98 if err := conR.conS.Stop(); err != nil { 99 conR.Logger.Error("Error stopping consensus state", "err", err) 100 } 101 if !conR.WaitSync() { 102 conR.conS.Wait() 103 } 104 } 105 106 // SwitchToConsensus switches from fast_sync mode to consensus mode. 107 // It resets the state, turns off fast_sync, and starts the consensus state-machine 108 func (conR *Reactor) SwitchToConsensus(state sm.State, skipWAL bool) { 109 conR.Logger.Info("SwitchToConsensus") 110 111 // We have no votes, so reconstruct LastCommit from SeenCommit. 112 if state.LastBlockHeight > 0 { 113 conR.conS.reconstructLastCommit(state) 114 } 115 116 // NOTE: The line below causes broadcastNewRoundStepRoutine() to broadcast a 117 // NewRoundStepMessage. 118 conR.conS.updateToState(state) 119 120 conR.mtx.Lock() 121 conR.waitSync = false 122 conR.mtx.Unlock() 123 conR.Metrics.FastSyncing.Set(0) 124 conR.Metrics.StateSyncing.Set(0) 125 126 if skipWAL { 127 conR.conS.doWALCatchup = false 128 } 129 err := conR.conS.Start() 130 if err != nil { 131 panic(fmt.Sprintf(`Failed to start consensus state: %v 132 133 conS: 134 %+v 135 136 conR: 137 %+v`, err, conR.conS, conR)) 138 } 139 } 140 141 // GetChannels implements Reactor 142 func (conR *Reactor) GetChannels() []*p2p.ChannelDescriptor { 143 // TODO optimize 144 return []*p2p.ChannelDescriptor{ 145 { 146 ID: StateChannel, 147 Priority: 6, 148 SendQueueCapacity: 100, 149 RecvMessageCapacity: maxMsgSize, 150 MessageType: &tmcons.Message{}, 151 }, 152 { 153 ID: DataChannel, // maybe split between gossiping current block and catchup stuff 154 // once we gossip the whole block there's nothing left to send until next height or round 155 Priority: 10, 156 SendQueueCapacity: 100, 157 RecvBufferCapacity: 50 * 4096, 158 RecvMessageCapacity: maxMsgSize, 159 MessageType: &tmcons.Message{}, 160 }, 161 { 162 ID: VoteChannel, 163 Priority: 7, 164 SendQueueCapacity: 100, 165 RecvBufferCapacity: 100 * 100, 166 RecvMessageCapacity: maxMsgSize, 167 MessageType: &tmcons.Message{}, 168 }, 169 { 170 ID: VoteSetBitsChannel, 171 Priority: 1, 172 SendQueueCapacity: 2, 173 RecvBufferCapacity: 1024, 174 RecvMessageCapacity: maxMsgSize, 175 MessageType: &tmcons.Message{}, 176 }, 177 } 178 } 179 180 // InitPeer implements Reactor by creating a state for the peer. 181 func (conR *Reactor) InitPeer(peer p2p.Peer) p2p.Peer { 182 peerState := NewPeerState(peer).SetLogger(conR.Logger) 183 peer.Set(types.PeerStateKey, peerState) 184 return peer 185 } 186 187 // AddPeer implements Reactor by spawning multiple gossiping goroutines for the 188 // peer. 189 func (conR *Reactor) AddPeer(peer p2p.Peer) { 190 if !conR.IsRunning() { 191 return 192 } 193 194 peerState, ok := peer.Get(types.PeerStateKey).(*PeerState) 195 if !ok { 196 panic(fmt.Sprintf("peer %v has no state", peer)) 197 } 198 // Begin routines for this peer. 199 go conR.gossipDataRoutine(peer, peerState) 200 go conR.gossipVotesRoutine(peer, peerState) 201 go conR.queryMaj23Routine(peer, peerState) 202 203 // Send our state to peer. 204 // If we're fast_syncing, broadcast a RoundStepMessage later upon SwitchToConsensus(). 205 if !conR.WaitSync() { 206 conR.sendNewRoundStepMessage(peer) 207 } 208 } 209 210 // RemovePeer is a noop. 211 func (conR *Reactor) RemovePeer(peer p2p.Peer, reason interface{}) { 212 if !conR.IsRunning() { 213 return 214 } 215 // TODO 216 // ps, ok := peer.Get(PeerStateKey).(*PeerState) 217 // if !ok { 218 // panic(fmt.Sprintf("Peer %v has no state", peer)) 219 // } 220 // ps.Disconnect() 221 } 222 223 // Receive implements Reactor 224 // NOTE: We process these messages even when we're fast_syncing. 225 // Messages affect either a peer state or the consensus state. 226 // Peer state updates can happen in parallel, but processing of 227 // proposals, block parts, and votes are ordered by the receiveRoutine 228 // NOTE: blocks on consensus state for proposals, block parts, and votes 229 func (conR *Reactor) ReceiveEnvelope(e p2p.Envelope) { 230 if !conR.IsRunning() { 231 conR.Logger.Debug("Receive", "src", e.Src, "chId", e.ChannelID) 232 return 233 } 234 m := e.Message 235 if wm, ok := m.(p2p.Wrapper); ok { 236 m = wm.Wrap() 237 } 238 msg, err := MsgFromProto(m.(*tmcons.Message)) 239 if err != nil { 240 conR.Logger.Error("Error decoding message", "src", e.Src, "chId", e.ChannelID, "err", err) 241 conR.Switch.StopPeerForError(e.Src, err) 242 return 243 } 244 245 if err = msg.ValidateBasic(); err != nil { 246 conR.Logger.Error("Peer sent us invalid msg", "peer", e.Src, "msg", e.Message, "err", err) 247 conR.Switch.StopPeerForError(e.Src, err) 248 return 249 } 250 251 conR.Logger.Debug("Receive", "src", e.Src, "chId", e.ChannelID, "msg", msg) 252 253 // Get peer states 254 ps, ok := e.Src.Get(types.PeerStateKey).(*PeerState) 255 if !ok { 256 panic(fmt.Sprintf("Peer %v has no state", e.Src)) 257 } 258 259 switch e.ChannelID { 260 case StateChannel: 261 switch msg := msg.(type) { 262 case *NewRoundStepMessage: 263 conR.conS.mtx.Lock() 264 initialHeight := conR.conS.state.InitialHeight 265 conR.conS.mtx.Unlock() 266 if err = msg.ValidateHeight(initialHeight); err != nil { 267 conR.Logger.Error("Peer sent us invalid msg", "peer", e.Src, "msg", msg, "err", err) 268 conR.Switch.StopPeerForError(e.Src, err) 269 return 270 } 271 ps.ApplyNewRoundStepMessage(msg) 272 case *NewValidBlockMessage: 273 ps.ApplyNewValidBlockMessage(msg) 274 case *HasVoteMessage: 275 ps.ApplyHasVoteMessage(msg) 276 case *VoteSetMaj23Message: 277 cs := conR.conS 278 cs.mtx.Lock() 279 height, votes := cs.Height, cs.Votes 280 cs.mtx.Unlock() 281 if height != msg.Height { 282 return 283 } 284 // Peer claims to have a maj23 for some BlockID at H,R,S, 285 err := votes.SetPeerMaj23(msg.Round, msg.Type, ps.peer.ID(), msg.BlockID) 286 if err != nil { 287 conR.Switch.StopPeerForError(e.Src, err) 288 return 289 } 290 // Respond with a VoteSetBitsMessage showing which votes we have. 291 // (and consequently shows which we don't have) 292 var ourVotes *bits.BitArray 293 switch msg.Type { 294 case tmproto.PrevoteType: 295 ourVotes = votes.Prevotes(msg.Round).BitArrayByBlockID(msg.BlockID) 296 case tmproto.PrecommitType: 297 ourVotes = votes.Precommits(msg.Round).BitArrayByBlockID(msg.BlockID) 298 default: 299 panic("Bad VoteSetBitsMessage field Type. Forgot to add a check in ValidateBasic?") 300 } 301 eMsg := &tmcons.VoteSetBits{ 302 Height: msg.Height, 303 Round: msg.Round, 304 Type: msg.Type, 305 BlockID: msg.BlockID.ToProto(), 306 } 307 if votes := ourVotes.ToProto(); votes != nil { 308 eMsg.Votes = *votes 309 } 310 p2p.TrySendEnvelopeShim(e.Src, p2p.Envelope{ //nolint: staticcheck 311 ChannelID: VoteSetBitsChannel, 312 Message: eMsg, 313 }, conR.Logger) 314 default: 315 conR.Logger.Error(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg))) 316 } 317 318 case DataChannel: 319 if conR.WaitSync() { 320 conR.Logger.Info("Ignoring message received during sync", "msg", msg) 321 return 322 } 323 switch msg := msg.(type) { 324 case *ProposalMessage: 325 ps.SetHasProposal(msg.Proposal) 326 conR.conS.peerMsgQueue <- msgInfo{msg, e.Src.ID()} 327 case *ProposalPOLMessage: 328 ps.ApplyProposalPOLMessage(msg) 329 case *BlockPartMessage: 330 ps.SetHasProposalBlockPart(msg.Height, msg.Round, int(msg.Part.Index)) 331 conR.Metrics.BlockParts.With("peer_id", string(e.Src.ID())).Add(1) 332 conR.conS.peerMsgQueue <- msgInfo{msg, e.Src.ID()} 333 default: 334 conR.Logger.Error(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg))) 335 } 336 337 case VoteChannel: 338 if conR.WaitSync() { 339 conR.Logger.Info("Ignoring message received during sync", "msg", msg) 340 return 341 } 342 switch msg := msg.(type) { 343 case *VoteMessage: 344 cs := conR.conS 345 cs.mtx.RLock() 346 height, valSize, lastCommitSize := cs.Height, cs.Validators.Size(), cs.LastCommit.Size() 347 cs.mtx.RUnlock() 348 ps.EnsureVoteBitArrays(height, valSize) 349 ps.EnsureVoteBitArrays(height-1, lastCommitSize) 350 ps.SetHasVote(msg.Vote) 351 352 cs.peerMsgQueue <- msgInfo{msg, e.Src.ID()} 353 354 default: 355 // don't punish (leave room for soft upgrades) 356 conR.Logger.Error(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg))) 357 } 358 359 case VoteSetBitsChannel: 360 if conR.WaitSync() { 361 conR.Logger.Info("Ignoring message received during sync", "msg", msg) 362 return 363 } 364 switch msg := msg.(type) { 365 case *VoteSetBitsMessage: 366 cs := conR.conS 367 cs.mtx.Lock() 368 height, votes := cs.Height, cs.Votes 369 cs.mtx.Unlock() 370 371 if height == msg.Height { 372 var ourVotes *bits.BitArray 373 switch msg.Type { 374 case tmproto.PrevoteType: 375 ourVotes = votes.Prevotes(msg.Round).BitArrayByBlockID(msg.BlockID) 376 case tmproto.PrecommitType: 377 ourVotes = votes.Precommits(msg.Round).BitArrayByBlockID(msg.BlockID) 378 default: 379 panic("Bad VoteSetBitsMessage field Type. Forgot to add a check in ValidateBasic?") 380 } 381 ps.ApplyVoteSetBitsMessage(msg, ourVotes) 382 } else { 383 ps.ApplyVoteSetBitsMessage(msg, nil) 384 } 385 default: 386 // don't punish (leave room for soft upgrades) 387 conR.Logger.Error(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg))) 388 } 389 390 default: 391 conR.Logger.Error(fmt.Sprintf("Unknown chId %X", e.ChannelID)) 392 } 393 } 394 395 func (conR *Reactor) Receive(chID byte, peer p2p.Peer, msgBytes []byte) { 396 msg := &tmcons.Message{} 397 err := proto.Unmarshal(msgBytes, msg) 398 if err != nil { 399 panic(err) 400 } 401 uw, err := msg.Unwrap() 402 if err != nil { 403 panic(err) 404 } 405 conR.ReceiveEnvelope(p2p.Envelope{ 406 ChannelID: chID, 407 Src: peer, 408 Message: uw, 409 }) 410 } 411 412 // SetEventBus sets event bus. 413 func (conR *Reactor) SetEventBus(b *types.EventBus) { 414 conR.eventBus = b 415 conR.conS.SetEventBus(b) 416 } 417 418 // WaitSync returns whether the consensus reactor is waiting for state/fast sync. 419 func (conR *Reactor) WaitSync() bool { 420 conR.mtx.RLock() 421 defer conR.mtx.RUnlock() 422 return conR.waitSync 423 } 424 425 //-------------------------------------- 426 427 // subscribeToBroadcastEvents subscribes for new round steps and votes 428 // using internal pubsub defined on state to broadcast 429 // them to peers upon receiving. 430 func (conR *Reactor) subscribeToBroadcastEvents() { 431 const subscriber = "consensus-reactor" 432 if err := conR.conS.evsw.AddListenerForEvent(subscriber, types.EventNewRoundStep, 433 func(data tmevents.EventData) { 434 conR.broadcastNewRoundStepMessage(data.(*cstypes.RoundState)) 435 }); err != nil { 436 conR.Logger.Error("Error adding listener for events", "err", err) 437 } 438 439 if err := conR.conS.evsw.AddListenerForEvent(subscriber, types.EventValidBlock, 440 func(data tmevents.EventData) { 441 conR.broadcastNewValidBlockMessage(data.(*cstypes.RoundState)) 442 }); err != nil { 443 conR.Logger.Error("Error adding listener for events", "err", err) 444 } 445 446 if err := conR.conS.evsw.AddListenerForEvent(subscriber, types.EventVote, 447 func(data tmevents.EventData) { 448 conR.broadcastHasVoteMessage(data.(*types.Vote)) 449 }); err != nil { 450 conR.Logger.Error("Error adding listener for events", "err", err) 451 } 452 453 } 454 455 func (conR *Reactor) unsubscribeFromBroadcastEvents() { 456 const subscriber = "consensus-reactor" 457 conR.conS.evsw.RemoveListener(subscriber) 458 } 459 460 func (conR *Reactor) broadcastNewRoundStepMessage(rs *cstypes.RoundState) { 461 nrsMsg := makeRoundStepMessage(rs) 462 conR.Switch.BroadcastEnvelope(p2p.Envelope{ 463 ChannelID: StateChannel, 464 Message: nrsMsg, 465 }) 466 } 467 468 func (conR *Reactor) broadcastNewValidBlockMessage(rs *cstypes.RoundState) { 469 psh := rs.ProposalBlockParts.Header() 470 csMsg := &tmcons.NewValidBlock{ 471 Height: rs.Height, 472 Round: rs.Round, 473 BlockPartSetHeader: psh.ToProto(), 474 BlockParts: rs.ProposalBlockParts.BitArray().ToProto(), 475 IsCommit: rs.Step == cstypes.RoundStepCommit, 476 } 477 conR.Switch.BroadcastEnvelope(p2p.Envelope{ 478 ChannelID: StateChannel, 479 Message: csMsg, 480 }) 481 } 482 483 // Broadcasts HasVoteMessage to peers that care. 484 func (conR *Reactor) broadcastHasVoteMessage(vote *types.Vote) { 485 msg := &tmcons.HasVote{ 486 Height: vote.Height, 487 Round: vote.Round, 488 Type: vote.Type, 489 Index: vote.ValidatorIndex, 490 } 491 conR.Switch.BroadcastEnvelope(p2p.Envelope{ 492 ChannelID: StateChannel, 493 Message: msg, 494 }) 495 /* 496 // TODO: Make this broadcast more selective. 497 for _, peer := range conR.Switch.Peers().List() { 498 ps, ok := peer.Get(PeerStateKey).(*PeerState) 499 if !ok { 500 panic(fmt.Sprintf("Peer %v has no state", peer)) 501 } 502 prs := ps.GetRoundState() 503 if prs.Height == vote.Height { 504 // TODO: Also filter on round? 505 e := p2p.Envelope{ 506 ChannelID: StateChannel, struct{ ConsensusMessage }{msg}, 507 Message: p, 508 } 509 p2p.TrySendEnvelopeShim(peer, e) //nolint: staticcheck 510 } else { 511 // Height doesn't match 512 // TODO: check a field, maybe CatchupCommitRound? 513 // TODO: But that requires changing the struct field comment. 514 } 515 } 516 */ 517 } 518 519 func makeRoundStepMessage(rs *cstypes.RoundState) (nrsMsg *tmcons.NewRoundStep) { 520 nrsMsg = &tmcons.NewRoundStep{ 521 Height: rs.Height, 522 Round: rs.Round, 523 Step: uint32(rs.Step), 524 SecondsSinceStartTime: int64(time.Since(rs.StartTime).Seconds()), 525 LastCommitRound: rs.LastCommit.GetRound(), 526 } 527 return 528 } 529 530 func (conR *Reactor) sendNewRoundStepMessage(peer p2p.Peer) { 531 rs := conR.getRoundState() 532 nrsMsg := makeRoundStepMessage(rs) 533 p2p.SendEnvelopeShim(peer, p2p.Envelope{ //nolint: staticcheck 534 ChannelID: StateChannel, 535 Message: nrsMsg, 536 }, conR.Logger) 537 } 538 539 func (conR *Reactor) updateRoundStateRoutine() { 540 t := time.NewTicker(100 * time.Microsecond) 541 defer t.Stop() 542 for range t.C { 543 if !conR.IsRunning() { 544 return 545 } 546 rs := conR.conS.GetRoundState() 547 conR.mtx.Lock() 548 conR.rs = rs 549 conR.mtx.Unlock() 550 } 551 } 552 553 func (conR *Reactor) getRoundState() *cstypes.RoundState { 554 conR.mtx.RLock() 555 defer conR.mtx.RUnlock() 556 return conR.rs 557 } 558 559 func (conR *Reactor) gossipDataRoutine(peer p2p.Peer, ps *PeerState) { 560 logger := conR.Logger.With("peer", peer) 561 562 OUTER_LOOP: 563 for { 564 // Manage disconnects from self or peer. 565 if !peer.IsRunning() || !conR.IsRunning() { 566 return 567 } 568 rs := conR.getRoundState() 569 prs := ps.GetRoundState() 570 571 // Send proposal Block parts? 572 if rs.ProposalBlockParts.HasHeader(prs.ProposalBlockPartSetHeader) { 573 if index, ok := rs.ProposalBlockParts.BitArray().Sub(prs.ProposalBlockParts.Copy()).PickRandom(); ok { 574 part := rs.ProposalBlockParts.GetPart(index) 575 parts, err := part.ToProto() 576 if err != nil { 577 panic(err) 578 } 579 logger.Debug("Sending block part", "height", prs.Height, "round", prs.Round) 580 if p2p.SendEnvelopeShim(peer, p2p.Envelope{ //nolint: staticcheck 581 ChannelID: DataChannel, 582 Message: &tmcons.BlockPart{ 583 Height: rs.Height, // This tells peer that this part applies to us. 584 Round: rs.Round, // This tells peer that this part applies to us. 585 Part: *parts, 586 }, 587 }, logger) { 588 ps.SetHasProposalBlockPart(prs.Height, prs.Round, index) 589 } 590 continue OUTER_LOOP 591 } 592 } 593 594 // If the peer is on a previous height that we have, help catch up. 595 blockStoreBase := conR.conS.blockStore.Base() 596 if blockStoreBase > 0 && 0 < prs.Height && prs.Height < rs.Height && prs.Height >= blockStoreBase { 597 heightLogger := logger.With("height", prs.Height) 598 599 // if we never received the commit message from the peer, the block parts wont be initialized 600 if prs.ProposalBlockParts == nil { 601 blockMeta := conR.conS.blockStore.LoadBlockMeta(prs.Height) 602 if blockMeta == nil { 603 heightLogger.Error("Failed to load block meta", 604 "blockstoreBase", blockStoreBase, "blockstoreHeight", conR.conS.blockStore.Height()) 605 time.Sleep(conR.conS.config.PeerGossipSleepDuration) 606 } else { 607 ps.InitProposalBlockParts(blockMeta.BlockID.PartSetHeader) 608 } 609 // continue the loop since prs is a copy and not effected by this initialization 610 continue OUTER_LOOP 611 } 612 conR.gossipDataForCatchup(heightLogger, rs, prs, ps, peer) 613 continue OUTER_LOOP 614 } 615 616 // If height and round don't match, sleep. 617 if (rs.Height != prs.Height) || (rs.Round != prs.Round) { 618 // logger.Info("Peer Height|Round mismatch, sleeping", 619 // "peerHeight", prs.Height, "peerRound", prs.Round, "peer", peer) 620 time.Sleep(conR.conS.config.PeerGossipSleepDuration) 621 continue OUTER_LOOP 622 } 623 624 // By here, height and round match. 625 // Proposal block parts were already matched and sent if any were wanted. 626 // (These can match on hash so the round doesn't matter) 627 // Now consider sending other things, like the Proposal itself. 628 629 // Send Proposal && ProposalPOL BitArray? 630 if rs.Proposal != nil && !prs.Proposal { 631 // Proposal: share the proposal metadata with peer. 632 { 633 logger.Debug("Sending proposal", "height", prs.Height, "round", prs.Round) 634 if p2p.SendEnvelopeShim(peer, p2p.Envelope{ //nolint: staticcheck 635 ChannelID: DataChannel, 636 Message: &tmcons.Proposal{Proposal: *rs.Proposal.ToProto()}, 637 }, logger) { 638 // NOTE[ZM]: A peer might have received different proposal msg so this Proposal msg will be rejected! 639 ps.SetHasProposal(rs.Proposal) 640 } 641 } 642 // ProposalPOL: lets peer know which POL votes we have so far. 643 // Peer must receive ProposalMessage first. 644 // rs.Proposal was validated, so rs.Proposal.POLRound <= rs.Round, 645 // so we definitely have rs.Votes.Prevotes(rs.Proposal.POLRound). 646 if 0 <= rs.Proposal.POLRound { 647 logger.Debug("Sending POL", "height", prs.Height, "round", prs.Round) 648 p2p.SendEnvelopeShim(peer, p2p.Envelope{ //nolint: staticcheck 649 ChannelID: DataChannel, 650 Message: &tmcons.ProposalPOL{ 651 Height: rs.Height, 652 ProposalPolRound: rs.Proposal.POLRound, 653 ProposalPol: *rs.Votes.Prevotes(rs.Proposal.POLRound).BitArray().ToProto(), 654 }, 655 }, logger) 656 } 657 continue OUTER_LOOP 658 } 659 660 // Nothing to do. Sleep. 661 time.Sleep(conR.conS.config.PeerGossipSleepDuration) 662 continue OUTER_LOOP 663 } 664 } 665 666 func (conR *Reactor) gossipDataForCatchup(logger log.Logger, rs *cstypes.RoundState, 667 prs *cstypes.PeerRoundState, ps *PeerState, peer p2p.Peer) { 668 669 if index, ok := prs.ProposalBlockParts.Not().PickRandom(); ok { 670 // Ensure that the peer's PartSetHeader is correct 671 blockMeta := conR.conS.blockStore.LoadBlockMeta(prs.Height) 672 if blockMeta == nil { 673 logger.Error("Failed to load block meta", "ourHeight", rs.Height, 674 "blockstoreBase", conR.conS.blockStore.Base(), "blockstoreHeight", conR.conS.blockStore.Height()) 675 time.Sleep(conR.conS.config.PeerGossipSleepDuration) 676 return 677 } else if !blockMeta.BlockID.PartSetHeader.Equals(prs.ProposalBlockPartSetHeader) { 678 logger.Info("Peer ProposalBlockPartSetHeader mismatch, sleeping", 679 "blockPartSetHeader", blockMeta.BlockID.PartSetHeader, "peerBlockPartSetHeader", prs.ProposalBlockPartSetHeader) 680 time.Sleep(conR.conS.config.PeerGossipSleepDuration) 681 return 682 } 683 // Load the part 684 part := conR.conS.blockStore.LoadBlockPart(prs.Height, index) 685 if part == nil { 686 logger.Error("Could not load part", "index", index, 687 "blockPartSetHeader", blockMeta.BlockID.PartSetHeader, "peerBlockPartSetHeader", prs.ProposalBlockPartSetHeader) 688 time.Sleep(conR.conS.config.PeerGossipSleepDuration) 689 return 690 } 691 // Send the part 692 logger.Debug("Sending block part for catchup", "round", prs.Round, "index", index) 693 pp, err := part.ToProto() 694 if err != nil { 695 logger.Error("Could not convert part to proto", "index", index, "error", err) 696 return 697 } 698 if p2p.SendEnvelopeShim(peer, p2p.Envelope{ //nolint: staticcheck 699 ChannelID: DataChannel, 700 Message: &tmcons.BlockPart{ 701 Height: prs.Height, // Not our height, so it doesn't matter. 702 Round: prs.Round, // Not our height, so it doesn't matter. 703 Part: *pp, 704 }, 705 }, logger) { 706 ps.SetHasProposalBlockPart(prs.Height, prs.Round, index) 707 } else { 708 logger.Debug("Sending block part for catchup failed") 709 } 710 return 711 } 712 // logger.Info("No parts to send in catch-up, sleeping") 713 time.Sleep(conR.conS.config.PeerGossipSleepDuration) 714 } 715 716 func (conR *Reactor) gossipVotesRoutine(peer p2p.Peer, ps *PeerState) { 717 logger := conR.Logger.With("peer", peer) 718 719 // Simple hack to throttle logs upon sleep. 720 var sleeping = 0 721 722 OUTER_LOOP: 723 for { 724 // Manage disconnects from self or peer. 725 if !peer.IsRunning() || !conR.IsRunning() { 726 return 727 } 728 rs := conR.getRoundState() 729 prs := ps.GetRoundState() 730 731 switch sleeping { 732 case 1: // First sleep 733 sleeping = 2 734 case 2: // No more sleep 735 sleeping = 0 736 } 737 738 // logger.Debug("gossipVotesRoutine", "rsHeight", rs.Height, "rsRound", rs.Round, 739 // "prsHeight", prs.Height, "prsRound", prs.Round, "prsStep", prs.Step) 740 741 // If height matches, then send LastCommit, Prevotes, Precommits. 742 if rs.Height == prs.Height { 743 heightLogger := logger.With("height", prs.Height) 744 if conR.gossipVotesForHeight(heightLogger, rs, prs, ps) { 745 continue OUTER_LOOP 746 } 747 } 748 749 // Special catchup logic. 750 // If peer is lagging by height 1, send LastCommit. 751 if prs.Height != 0 && rs.Height == prs.Height+1 { 752 if ps.PickSendVote(rs.LastCommit) { 753 logger.Debug("Picked rs.LastCommit to send", "height", prs.Height) 754 continue OUTER_LOOP 755 } 756 } 757 758 // Catchup logic 759 // If peer is lagging by more than 1, send Commit. 760 blockStoreBase := conR.conS.blockStore.Base() 761 if blockStoreBase > 0 && prs.Height != 0 && rs.Height >= prs.Height+2 && prs.Height >= blockStoreBase { 762 // Load the block commit for prs.Height, 763 // which contains precommit signatures for prs.Height. 764 if commit := conR.conS.blockStore.LoadBlockCommit(prs.Height); commit != nil { 765 if ps.PickSendVote(commit) { 766 logger.Debug("Picked Catchup commit to send", "height", prs.Height) 767 continue OUTER_LOOP 768 } 769 } 770 } 771 772 if sleeping == 0 { 773 // We sent nothing. Sleep... 774 sleeping = 1 775 logger.Debug("No votes to send, sleeping", "rs.Height", rs.Height, "prs.Height", prs.Height, 776 "localPV", rs.Votes.Prevotes(rs.Round).BitArray(), "peerPV", prs.Prevotes, 777 "localPC", rs.Votes.Precommits(rs.Round).BitArray(), "peerPC", prs.Precommits) 778 } else if sleeping == 2 { 779 // Continued sleep... 780 sleeping = 1 781 } 782 783 time.Sleep(conR.conS.config.PeerGossipSleepDuration) 784 continue OUTER_LOOP 785 } 786 } 787 788 func (conR *Reactor) gossipVotesForHeight( 789 logger log.Logger, 790 rs *cstypes.RoundState, 791 prs *cstypes.PeerRoundState, 792 ps *PeerState, 793 ) bool { 794 795 // If there are lastCommits to send... 796 if prs.Step == cstypes.RoundStepNewHeight { 797 if ps.PickSendVote(rs.LastCommit) { 798 logger.Debug("Picked rs.LastCommit to send") 799 return true 800 } 801 } 802 // If there are POL prevotes to send... 803 if prs.Step <= cstypes.RoundStepPropose && prs.Round != -1 && prs.Round <= rs.Round && prs.ProposalPOLRound != -1 { 804 if polPrevotes := rs.Votes.Prevotes(prs.ProposalPOLRound); polPrevotes != nil { 805 if ps.PickSendVote(polPrevotes) { 806 logger.Debug("Picked rs.Prevotes(prs.ProposalPOLRound) to send", 807 "round", prs.ProposalPOLRound) 808 return true 809 } 810 } 811 } 812 // If there are prevotes to send... 813 if prs.Step <= cstypes.RoundStepPrevoteWait && prs.Round != -1 && prs.Round <= rs.Round { 814 if ps.PickSendVote(rs.Votes.Prevotes(prs.Round)) { 815 logger.Debug("Picked rs.Prevotes(prs.Round) to send", "round", prs.Round) 816 return true 817 } 818 } 819 // If there are precommits to send... 820 if prs.Step <= cstypes.RoundStepPrecommitWait && prs.Round != -1 && prs.Round <= rs.Round { 821 if ps.PickSendVote(rs.Votes.Precommits(prs.Round)) { 822 logger.Debug("Picked rs.Precommits(prs.Round) to send", "round", prs.Round) 823 return true 824 } 825 } 826 // If there are prevotes to send...Needed because of validBlock mechanism 827 if prs.Round != -1 && prs.Round <= rs.Round { 828 if ps.PickSendVote(rs.Votes.Prevotes(prs.Round)) { 829 logger.Debug("Picked rs.Prevotes(prs.Round) to send", "round", prs.Round) 830 return true 831 } 832 } 833 // If there are POLPrevotes to send... 834 if prs.ProposalPOLRound != -1 { 835 if polPrevotes := rs.Votes.Prevotes(prs.ProposalPOLRound); polPrevotes != nil { 836 if ps.PickSendVote(polPrevotes) { 837 logger.Debug("Picked rs.Prevotes(prs.ProposalPOLRound) to send", 838 "round", prs.ProposalPOLRound) 839 return true 840 } 841 } 842 } 843 844 return false 845 } 846 847 // NOTE: `queryMaj23Routine` has a simple crude design since it only comes 848 // into play for liveness when there's a signature DDoS attack happening. 849 func (conR *Reactor) queryMaj23Routine(peer p2p.Peer, ps *PeerState) { 850 851 OUTER_LOOP: 852 for { 853 // Manage disconnects from self or peer. 854 if !peer.IsRunning() || !conR.IsRunning() { 855 return 856 } 857 858 // Maybe send Height/Round/Prevotes 859 { 860 rs := conR.getRoundState() 861 prs := ps.GetRoundState() 862 if rs.Height == prs.Height { 863 if maj23, ok := rs.Votes.Prevotes(prs.Round).TwoThirdsMajority(); ok { 864 865 p2p.TrySendEnvelopeShim(peer, p2p.Envelope{ //nolint: staticcheck 866 ChannelID: StateChannel, 867 Message: &tmcons.VoteSetMaj23{ 868 Height: prs.Height, 869 Round: prs.Round, 870 Type: tmproto.PrevoteType, 871 BlockID: maj23.ToProto(), 872 }, 873 }, ps.logger) 874 time.Sleep(conR.conS.config.PeerQueryMaj23SleepDuration) 875 } 876 } 877 } 878 879 // Maybe send Height/Round/Precommits 880 { 881 rs := conR.getRoundState() 882 prs := ps.GetRoundState() 883 if rs.Height == prs.Height { 884 if maj23, ok := rs.Votes.Precommits(prs.Round).TwoThirdsMajority(); ok { 885 p2p.TrySendEnvelopeShim(peer, p2p.Envelope{ //nolint: staticcheck 886 ChannelID: StateChannel, 887 Message: &tmcons.VoteSetMaj23{ 888 Height: prs.Height, 889 Round: prs.Round, 890 Type: tmproto.PrecommitType, 891 BlockID: maj23.ToProto(), 892 }, 893 }, ps.logger) 894 time.Sleep(conR.conS.config.PeerQueryMaj23SleepDuration) 895 } 896 } 897 } 898 899 // Maybe send Height/Round/ProposalPOL 900 { 901 rs := conR.getRoundState() 902 prs := ps.GetRoundState() 903 if rs.Height == prs.Height && prs.ProposalPOLRound >= 0 { 904 if maj23, ok := rs.Votes.Prevotes(prs.ProposalPOLRound).TwoThirdsMajority(); ok { 905 906 p2p.TrySendEnvelopeShim(peer, p2p.Envelope{ //nolint: staticcheck 907 ChannelID: StateChannel, 908 Message: &tmcons.VoteSetMaj23{ 909 Height: prs.Height, 910 Round: prs.ProposalPOLRound, 911 Type: tmproto.PrevoteType, 912 BlockID: maj23.ToProto(), 913 }, 914 }, ps.logger) 915 time.Sleep(conR.conS.config.PeerQueryMaj23SleepDuration) 916 } 917 } 918 } 919 920 // Little point sending LastCommitRound/LastCommit, 921 // These are fleeting and non-blocking. 922 923 // Maybe send Height/CatchupCommitRound/CatchupCommit. 924 { 925 prs := ps.GetRoundState() 926 if prs.CatchupCommitRound != -1 && prs.Height > 0 && prs.Height <= conR.conS.blockStore.Height() && 927 prs.Height >= conR.conS.blockStore.Base() { 928 if commit := conR.conS.LoadCommit(prs.Height); commit != nil { 929 p2p.TrySendEnvelopeShim(peer, p2p.Envelope{ //nolint: staticcheck 930 ChannelID: StateChannel, 931 Message: &tmcons.VoteSetMaj23{ 932 Height: prs.Height, 933 Round: commit.Round, 934 Type: tmproto.PrecommitType, 935 BlockID: commit.BlockID.ToProto(), 936 }, 937 }, ps.logger) 938 time.Sleep(conR.conS.config.PeerQueryMaj23SleepDuration) 939 } 940 } 941 } 942 943 time.Sleep(conR.conS.config.PeerQueryMaj23SleepDuration) 944 945 continue OUTER_LOOP 946 } 947 } 948 949 func (conR *Reactor) peerStatsRoutine() { 950 for { 951 if !conR.IsRunning() { 952 conR.Logger.Info("Stopping peerStatsRoutine") 953 return 954 } 955 956 select { 957 case msg := <-conR.conS.statsMsgQueue: 958 // Get peer 959 peer := conR.Switch.Peers().Get(msg.PeerID) 960 if peer == nil { 961 conR.Logger.Debug("Attempt to update stats for non-existent peer", 962 "peer", msg.PeerID) 963 continue 964 } 965 // Get peer state 966 ps, ok := peer.Get(types.PeerStateKey).(*PeerState) 967 if !ok { 968 panic(fmt.Sprintf("Peer %v has no state", peer)) 969 } 970 switch msg.Msg.(type) { 971 case *VoteMessage: 972 if numVotes := ps.RecordVote(); numVotes%votesToContributeToBecomeGoodPeer == 0 { 973 conR.Switch.MarkPeerAsGood(peer) 974 } 975 case *BlockPartMessage: 976 if numParts := ps.RecordBlockPart(); numParts%blocksToContributeToBecomeGoodPeer == 0 { 977 conR.Switch.MarkPeerAsGood(peer) 978 } 979 } 980 case <-conR.conS.Quit(): 981 return 982 983 case <-conR.Quit(): 984 return 985 } 986 } 987 } 988 989 // String returns a string representation of the Reactor. 990 // NOTE: For now, it is just a hard-coded string to avoid accessing unprotected shared variables. 991 // TODO: improve! 992 func (conR *Reactor) String() string { 993 // better not to access shared variables 994 return "ConsensusReactor" // conR.StringIndented("") 995 } 996 997 // StringIndented returns an indented string representation of the Reactor 998 func (conR *Reactor) StringIndented(indent string) string { 999 s := "ConsensusReactor{\n" 1000 s += indent + " " + conR.conS.StringIndented(indent+" ") + "\n" 1001 for _, peer := range conR.Switch.Peers().List() { 1002 ps, ok := peer.Get(types.PeerStateKey).(*PeerState) 1003 if !ok { 1004 panic(fmt.Sprintf("Peer %v has no state", peer)) 1005 } 1006 s += indent + " " + ps.StringIndented(indent+" ") + "\n" 1007 } 1008 s += indent + "}" 1009 return s 1010 } 1011 1012 // ReactorMetrics sets the metrics 1013 func ReactorMetrics(metrics *Metrics) ReactorOption { 1014 return func(conR *Reactor) { conR.Metrics = metrics } 1015 } 1016 1017 //----------------------------------------------------------------------------- 1018 1019 var ( 1020 ErrPeerStateHeightRegression = errors.New("error peer state height regression") 1021 ErrPeerStateInvalidStartTime = errors.New("error peer state invalid startTime") 1022 ) 1023 1024 // PeerState contains the known state of a peer, including its connection and 1025 // threadsafe access to its PeerRoundState. 1026 // NOTE: THIS GETS DUMPED WITH rpc/core/consensus.go. 1027 // Be mindful of what you Expose. 1028 type PeerState struct { 1029 peer p2p.Peer 1030 logger log.Logger 1031 1032 mtx sync.Mutex // NOTE: Modify below using setters, never directly. 1033 PRS cstypes.PeerRoundState `json:"round_state"` // Exposed. 1034 Stats *peerStateStats `json:"stats"` // Exposed. 1035 } 1036 1037 // peerStateStats holds internal statistics for a peer. 1038 type peerStateStats struct { 1039 Votes int `json:"votes"` 1040 BlockParts int `json:"block_parts"` 1041 } 1042 1043 func (pss peerStateStats) String() string { 1044 return fmt.Sprintf("peerStateStats{votes: %d, blockParts: %d}", 1045 pss.Votes, pss.BlockParts) 1046 } 1047 1048 // NewPeerState returns a new PeerState for the given Peer 1049 func NewPeerState(peer p2p.Peer) *PeerState { 1050 return &PeerState{ 1051 peer: peer, 1052 logger: log.NewNopLogger(), 1053 PRS: cstypes.PeerRoundState{ 1054 Round: -1, 1055 ProposalPOLRound: -1, 1056 LastCommitRound: -1, 1057 CatchupCommitRound: -1, 1058 }, 1059 Stats: &peerStateStats{}, 1060 } 1061 } 1062 1063 // SetLogger allows to set a logger on the peer state. Returns the peer state 1064 // itself. 1065 func (ps *PeerState) SetLogger(logger log.Logger) *PeerState { 1066 ps.logger = logger 1067 return ps 1068 } 1069 1070 // GetRoundState returns an shallow copy of the PeerRoundState. 1071 // There's no point in mutating it since it won't change PeerState. 1072 func (ps *PeerState) GetRoundState() *cstypes.PeerRoundState { 1073 ps.mtx.Lock() 1074 defer ps.mtx.Unlock() 1075 1076 prs := ps.PRS // copy 1077 return &prs 1078 } 1079 1080 // ToJSON returns a json of PeerState. 1081 func (ps *PeerState) ToJSON() ([]byte, error) { 1082 ps.mtx.Lock() 1083 defer ps.mtx.Unlock() 1084 1085 return tmjson.Marshal(ps) 1086 } 1087 1088 // GetHeight returns an atomic snapshot of the PeerRoundState's height 1089 // used by the mempool to ensure peers are caught up before broadcasting new txs 1090 func (ps *PeerState) GetHeight() int64 { 1091 ps.mtx.Lock() 1092 defer ps.mtx.Unlock() 1093 return ps.PRS.Height 1094 } 1095 1096 // SetHasProposal sets the given proposal as known for the peer. 1097 func (ps *PeerState) SetHasProposal(proposal *types.Proposal) { 1098 ps.mtx.Lock() 1099 defer ps.mtx.Unlock() 1100 1101 if ps.PRS.Height != proposal.Height || ps.PRS.Round != proposal.Round { 1102 return 1103 } 1104 1105 if ps.PRS.Proposal { 1106 return 1107 } 1108 1109 ps.PRS.Proposal = true 1110 1111 // ps.PRS.ProposalBlockParts is set due to NewValidBlockMessage 1112 if ps.PRS.ProposalBlockParts != nil { 1113 return 1114 } 1115 1116 ps.PRS.ProposalBlockPartSetHeader = proposal.BlockID.PartSetHeader 1117 ps.PRS.ProposalBlockParts = bits.NewBitArray(int(proposal.BlockID.PartSetHeader.Total)) 1118 ps.PRS.ProposalPOLRound = proposal.POLRound 1119 ps.PRS.ProposalPOL = nil // Nil until ProposalPOLMessage received. 1120 } 1121 1122 // InitProposalBlockParts initializes the peer's proposal block parts header and bit array. 1123 func (ps *PeerState) InitProposalBlockParts(partSetHeader types.PartSetHeader) { 1124 ps.mtx.Lock() 1125 defer ps.mtx.Unlock() 1126 1127 if ps.PRS.ProposalBlockParts != nil { 1128 return 1129 } 1130 1131 ps.PRS.ProposalBlockPartSetHeader = partSetHeader 1132 ps.PRS.ProposalBlockParts = bits.NewBitArray(int(partSetHeader.Total)) 1133 } 1134 1135 // SetHasProposalBlockPart sets the given block part index as known for the peer. 1136 func (ps *PeerState) SetHasProposalBlockPart(height int64, round int32, index int) { 1137 ps.mtx.Lock() 1138 defer ps.mtx.Unlock() 1139 1140 if ps.PRS.Height != height || ps.PRS.Round != round { 1141 return 1142 } 1143 1144 ps.PRS.ProposalBlockParts.SetIndex(index, true) 1145 } 1146 1147 // PickSendVote picks a vote and sends it to the peer. 1148 // Returns true if vote was sent. 1149 func (ps *PeerState) PickSendVote(votes types.VoteSetReader) bool { 1150 if vote, ok := ps.PickVoteToSend(votes); ok { 1151 ps.logger.Debug("Sending vote message", "ps", ps, "vote", vote) 1152 if p2p.SendEnvelopeShim(ps.peer, p2p.Envelope{ //nolint: staticcheck 1153 ChannelID: VoteChannel, 1154 Message: &tmcons.Vote{ 1155 Vote: vote.ToProto(), 1156 }, 1157 }, ps.logger) { 1158 ps.SetHasVote(vote) 1159 return true 1160 } 1161 return false 1162 } 1163 return false 1164 } 1165 1166 // PickVoteToSend picks a vote to send to the peer. 1167 // Returns true if a vote was picked. 1168 // NOTE: `votes` must be the correct Size() for the Height(). 1169 func (ps *PeerState) PickVoteToSend(votes types.VoteSetReader) (vote *types.Vote, ok bool) { 1170 ps.mtx.Lock() 1171 defer ps.mtx.Unlock() 1172 1173 if votes.Size() == 0 { 1174 return nil, false 1175 } 1176 1177 height, round, votesType, size := 1178 votes.GetHeight(), votes.GetRound(), tmproto.SignedMsgType(votes.Type()), votes.Size() 1179 1180 // Lazily set data using 'votes'. 1181 if votes.IsCommit() { 1182 ps.ensureCatchupCommitRound(height, round, size) 1183 } 1184 ps.ensureVoteBitArrays(height, size) 1185 1186 psVotes := ps.getVoteBitArray(height, round, votesType) 1187 if psVotes == nil { 1188 return nil, false // Not something worth sending 1189 } 1190 if index, ok := votes.BitArray().Sub(psVotes).PickRandom(); ok { 1191 return votes.GetByIndex(int32(index)), true 1192 } 1193 return nil, false 1194 } 1195 1196 func (ps *PeerState) getVoteBitArray(height int64, round int32, votesType tmproto.SignedMsgType) *bits.BitArray { 1197 if !types.IsVoteTypeValid(votesType) { 1198 return nil 1199 } 1200 1201 if ps.PRS.Height == height { 1202 if ps.PRS.Round == round { 1203 switch votesType { 1204 case tmproto.PrevoteType: 1205 return ps.PRS.Prevotes 1206 case tmproto.PrecommitType: 1207 return ps.PRS.Precommits 1208 } 1209 } 1210 if ps.PRS.CatchupCommitRound == round { 1211 switch votesType { 1212 case tmproto.PrevoteType: 1213 return nil 1214 case tmproto.PrecommitType: 1215 return ps.PRS.CatchupCommit 1216 } 1217 } 1218 if ps.PRS.ProposalPOLRound == round { 1219 switch votesType { 1220 case tmproto.PrevoteType: 1221 return ps.PRS.ProposalPOL 1222 case tmproto.PrecommitType: 1223 return nil 1224 } 1225 } 1226 return nil 1227 } 1228 if ps.PRS.Height == height+1 { 1229 if ps.PRS.LastCommitRound == round { 1230 switch votesType { 1231 case tmproto.PrevoteType: 1232 return nil 1233 case tmproto.PrecommitType: 1234 return ps.PRS.LastCommit 1235 } 1236 } 1237 return nil 1238 } 1239 return nil 1240 } 1241 1242 // 'round': A round for which we have a +2/3 commit. 1243 func (ps *PeerState) ensureCatchupCommitRound(height int64, round int32, numValidators int) { 1244 if ps.PRS.Height != height { 1245 return 1246 } 1247 /* 1248 NOTE: This is wrong, 'round' could change. 1249 e.g. if orig round is not the same as block LastCommit round. 1250 if ps.CatchupCommitRound != -1 && ps.CatchupCommitRound != round { 1251 panic(fmt.Sprintf( 1252 "Conflicting CatchupCommitRound. Height: %v, 1253 Orig: %v, 1254 New: %v", 1255 height, 1256 ps.CatchupCommitRound, 1257 round)) 1258 } 1259 */ 1260 if ps.PRS.CatchupCommitRound == round { 1261 return // Nothing to do! 1262 } 1263 ps.PRS.CatchupCommitRound = round 1264 if round == ps.PRS.Round { 1265 ps.PRS.CatchupCommit = ps.PRS.Precommits 1266 } else { 1267 ps.PRS.CatchupCommit = bits.NewBitArray(numValidators) 1268 } 1269 } 1270 1271 // EnsureVoteBitArrays ensures the bit-arrays have been allocated for tracking 1272 // what votes this peer has received. 1273 // NOTE: It's important to make sure that numValidators actually matches 1274 // what the node sees as the number of validators for height. 1275 func (ps *PeerState) EnsureVoteBitArrays(height int64, numValidators int) { 1276 ps.mtx.Lock() 1277 defer ps.mtx.Unlock() 1278 ps.ensureVoteBitArrays(height, numValidators) 1279 } 1280 1281 func (ps *PeerState) ensureVoteBitArrays(height int64, numValidators int) { 1282 if ps.PRS.Height == height { 1283 if ps.PRS.Prevotes == nil { 1284 ps.PRS.Prevotes = bits.NewBitArray(numValidators) 1285 } 1286 if ps.PRS.Precommits == nil { 1287 ps.PRS.Precommits = bits.NewBitArray(numValidators) 1288 } 1289 if ps.PRS.CatchupCommit == nil { 1290 ps.PRS.CatchupCommit = bits.NewBitArray(numValidators) 1291 } 1292 if ps.PRS.ProposalPOL == nil { 1293 ps.PRS.ProposalPOL = bits.NewBitArray(numValidators) 1294 } 1295 } else if ps.PRS.Height == height+1 { 1296 if ps.PRS.LastCommit == nil { 1297 ps.PRS.LastCommit = bits.NewBitArray(numValidators) 1298 } 1299 } 1300 } 1301 1302 // RecordVote increments internal votes related statistics for this peer. 1303 // It returns the total number of added votes. 1304 func (ps *PeerState) RecordVote() int { 1305 ps.mtx.Lock() 1306 defer ps.mtx.Unlock() 1307 1308 ps.Stats.Votes++ 1309 1310 return ps.Stats.Votes 1311 } 1312 1313 // VotesSent returns the number of blocks for which peer has been sending us 1314 // votes. 1315 func (ps *PeerState) VotesSent() int { 1316 ps.mtx.Lock() 1317 defer ps.mtx.Unlock() 1318 1319 return ps.Stats.Votes 1320 } 1321 1322 // RecordBlockPart increments internal block part related statistics for this peer. 1323 // It returns the total number of added block parts. 1324 func (ps *PeerState) RecordBlockPart() int { 1325 ps.mtx.Lock() 1326 defer ps.mtx.Unlock() 1327 1328 ps.Stats.BlockParts++ 1329 return ps.Stats.BlockParts 1330 } 1331 1332 // BlockPartsSent returns the number of useful block parts the peer has sent us. 1333 func (ps *PeerState) BlockPartsSent() int { 1334 ps.mtx.Lock() 1335 defer ps.mtx.Unlock() 1336 1337 return ps.Stats.BlockParts 1338 } 1339 1340 // SetHasVote sets the given vote as known by the peer 1341 func (ps *PeerState) SetHasVote(vote *types.Vote) { 1342 ps.mtx.Lock() 1343 defer ps.mtx.Unlock() 1344 1345 ps.setHasVote(vote.Height, vote.Round, vote.Type, vote.ValidatorIndex) 1346 } 1347 1348 func (ps *PeerState) setHasVote(height int64, round int32, voteType tmproto.SignedMsgType, index int32) { 1349 ps.logger.Debug("setHasVote", 1350 "peerH/R", 1351 log.NewLazySprintf("%d/%d", ps.PRS.Height, ps.PRS.Round), 1352 "H/R", 1353 log.NewLazySprintf("%d/%d", height, round), 1354 "type", voteType, "index", index) 1355 1356 // NOTE: some may be nil BitArrays -> no side effects. 1357 psVotes := ps.getVoteBitArray(height, round, voteType) 1358 if psVotes != nil { 1359 psVotes.SetIndex(int(index), true) 1360 } 1361 } 1362 1363 // ApplyNewRoundStepMessage updates the peer state for the new round. 1364 func (ps *PeerState) ApplyNewRoundStepMessage(msg *NewRoundStepMessage) { 1365 ps.mtx.Lock() 1366 defer ps.mtx.Unlock() 1367 1368 // Ignore duplicates or decreases 1369 if CompareHRS(msg.Height, msg.Round, msg.Step, ps.PRS.Height, ps.PRS.Round, ps.PRS.Step) <= 0 { 1370 return 1371 } 1372 1373 // Just remember these values. 1374 psHeight := ps.PRS.Height 1375 psRound := ps.PRS.Round 1376 psCatchupCommitRound := ps.PRS.CatchupCommitRound 1377 psCatchupCommit := ps.PRS.CatchupCommit 1378 1379 startTime := tmtime.Now().Add(-1 * time.Duration(msg.SecondsSinceStartTime) * time.Second) 1380 ps.PRS.Height = msg.Height 1381 ps.PRS.Round = msg.Round 1382 ps.PRS.Step = msg.Step 1383 ps.PRS.StartTime = startTime 1384 if psHeight != msg.Height || psRound != msg.Round { 1385 ps.PRS.Proposal = false 1386 ps.PRS.ProposalBlockPartSetHeader = types.PartSetHeader{} 1387 ps.PRS.ProposalBlockParts = nil 1388 ps.PRS.ProposalPOLRound = -1 1389 ps.PRS.ProposalPOL = nil 1390 // We'll update the BitArray capacity later. 1391 ps.PRS.Prevotes = nil 1392 ps.PRS.Precommits = nil 1393 } 1394 if psHeight == msg.Height && psRound != msg.Round && msg.Round == psCatchupCommitRound { 1395 // Peer caught up to CatchupCommitRound. 1396 // Preserve psCatchupCommit! 1397 // NOTE: We prefer to use prs.Precommits if 1398 // pr.Round matches pr.CatchupCommitRound. 1399 ps.PRS.Precommits = psCatchupCommit 1400 } 1401 if psHeight != msg.Height { 1402 // Shift Precommits to LastCommit. 1403 if psHeight+1 == msg.Height && psRound == msg.LastCommitRound { 1404 ps.PRS.LastCommitRound = msg.LastCommitRound 1405 ps.PRS.LastCommit = ps.PRS.Precommits 1406 } else { 1407 ps.PRS.LastCommitRound = msg.LastCommitRound 1408 ps.PRS.LastCommit = nil 1409 } 1410 // We'll update the BitArray capacity later. 1411 ps.PRS.CatchupCommitRound = -1 1412 ps.PRS.CatchupCommit = nil 1413 } 1414 } 1415 1416 // ApplyNewValidBlockMessage updates the peer state for the new valid block. 1417 func (ps *PeerState) ApplyNewValidBlockMessage(msg *NewValidBlockMessage) { 1418 ps.mtx.Lock() 1419 defer ps.mtx.Unlock() 1420 1421 if ps.PRS.Height != msg.Height { 1422 return 1423 } 1424 1425 if ps.PRS.Round != msg.Round && !msg.IsCommit { 1426 return 1427 } 1428 1429 ps.PRS.ProposalBlockPartSetHeader = msg.BlockPartSetHeader 1430 ps.PRS.ProposalBlockParts = msg.BlockParts 1431 } 1432 1433 // ApplyProposalPOLMessage updates the peer state for the new proposal POL. 1434 func (ps *PeerState) ApplyProposalPOLMessage(msg *ProposalPOLMessage) { 1435 ps.mtx.Lock() 1436 defer ps.mtx.Unlock() 1437 1438 if ps.PRS.Height != msg.Height { 1439 return 1440 } 1441 if ps.PRS.ProposalPOLRound != msg.ProposalPOLRound { 1442 return 1443 } 1444 1445 // TODO: Merge onto existing ps.PRS.ProposalPOL? 1446 // We might have sent some prevotes in the meantime. 1447 ps.PRS.ProposalPOL = msg.ProposalPOL 1448 } 1449 1450 // ApplyHasVoteMessage updates the peer state for the new vote. 1451 func (ps *PeerState) ApplyHasVoteMessage(msg *HasVoteMessage) { 1452 ps.mtx.Lock() 1453 defer ps.mtx.Unlock() 1454 1455 if ps.PRS.Height != msg.Height { 1456 return 1457 } 1458 1459 ps.setHasVote(msg.Height, msg.Round, msg.Type, msg.Index) 1460 } 1461 1462 // ApplyVoteSetBitsMessage updates the peer state for the bit-array of votes 1463 // it claims to have for the corresponding BlockID. 1464 // `ourVotes` is a BitArray of votes we have for msg.BlockID 1465 // NOTE: if ourVotes is nil (e.g. msg.Height < rs.Height), 1466 // we conservatively overwrite ps's votes w/ msg.Votes. 1467 func (ps *PeerState) ApplyVoteSetBitsMessage(msg *VoteSetBitsMessage, ourVotes *bits.BitArray) { 1468 ps.mtx.Lock() 1469 defer ps.mtx.Unlock() 1470 1471 votes := ps.getVoteBitArray(msg.Height, msg.Round, msg.Type) 1472 if votes != nil { 1473 if ourVotes == nil { 1474 votes.Update(msg.Votes) 1475 } else { 1476 otherVotes := votes.Sub(ourVotes) 1477 hasVotes := otherVotes.Or(msg.Votes) 1478 votes.Update(hasVotes) 1479 } 1480 } 1481 } 1482 1483 // String returns a string representation of the PeerState 1484 func (ps *PeerState) String() string { 1485 return ps.StringIndented("") 1486 } 1487 1488 // StringIndented returns a string representation of the PeerState 1489 func (ps *PeerState) StringIndented(indent string) string { 1490 ps.mtx.Lock() 1491 defer ps.mtx.Unlock() 1492 return fmt.Sprintf(`PeerState{ 1493 %s Key %v 1494 %s RoundState %v 1495 %s Stats %v 1496 %s}`, 1497 indent, ps.peer.ID(), 1498 indent, ps.PRS.StringIndented(indent+" "), 1499 indent, ps.Stats, 1500 indent) 1501 } 1502 1503 //----------------------------------------------------------------------------- 1504 // Messages 1505 1506 // Message is a message that can be sent and received on the Reactor 1507 type Message interface { 1508 ValidateBasic() error 1509 } 1510 1511 func init() { 1512 tmjson.RegisterType(&NewRoundStepMessage{}, "tendermint/NewRoundStepMessage") 1513 tmjson.RegisterType(&NewValidBlockMessage{}, "tendermint/NewValidBlockMessage") 1514 tmjson.RegisterType(&ProposalMessage{}, "tendermint/Proposal") 1515 tmjson.RegisterType(&ProposalPOLMessage{}, "tendermint/ProposalPOL") 1516 tmjson.RegisterType(&BlockPartMessage{}, "tendermint/BlockPart") 1517 tmjson.RegisterType(&VoteMessage{}, "tendermint/Vote") 1518 tmjson.RegisterType(&HasVoteMessage{}, "tendermint/HasVote") 1519 tmjson.RegisterType(&VoteSetMaj23Message{}, "tendermint/VoteSetMaj23") 1520 tmjson.RegisterType(&VoteSetBitsMessage{}, "tendermint/VoteSetBits") 1521 } 1522 1523 //------------------------------------- 1524 1525 // NewRoundStepMessage is sent for every step taken in the ConsensusState. 1526 // For every height/round/step transition 1527 type NewRoundStepMessage struct { 1528 Height int64 1529 Round int32 1530 Step cstypes.RoundStepType 1531 SecondsSinceStartTime int64 1532 LastCommitRound int32 1533 } 1534 1535 // ValidateBasic performs basic validation. 1536 func (m *NewRoundStepMessage) ValidateBasic() error { 1537 if m.Height < 0 { 1538 return errors.New("negative Height") 1539 } 1540 if m.Round < 0 { 1541 return errors.New("negative Round") 1542 } 1543 if !m.Step.IsValid() { 1544 return errors.New("invalid Step") 1545 } 1546 1547 // NOTE: SecondsSinceStartTime may be negative 1548 1549 // LastCommitRound will be -1 for the initial height, but we don't know what height this is 1550 // since it can be specified in genesis. The reactor will have to validate this via 1551 // ValidateHeight(). 1552 if m.LastCommitRound < -1 { 1553 return errors.New("invalid LastCommitRound (cannot be < -1)") 1554 } 1555 1556 return nil 1557 } 1558 1559 // ValidateHeight validates the height given the chain's initial height. 1560 func (m *NewRoundStepMessage) ValidateHeight(initialHeight int64) error { 1561 if m.Height < initialHeight { 1562 return fmt.Errorf("invalid Height %v (lower than initial height %v)", 1563 m.Height, initialHeight) 1564 } 1565 if m.Height == initialHeight && m.LastCommitRound != -1 { 1566 return fmt.Errorf("invalid LastCommitRound %v (must be -1 for initial height %v)", 1567 m.LastCommitRound, initialHeight) 1568 } 1569 if m.Height > initialHeight && m.LastCommitRound < 0 { 1570 return fmt.Errorf("LastCommitRound can only be negative for initial height %v", 1571 initialHeight) 1572 } 1573 return nil 1574 } 1575 1576 // String returns a string representation. 1577 func (m *NewRoundStepMessage) String() string { 1578 return fmt.Sprintf("[NewRoundStep H:%v R:%v S:%v LCR:%v]", 1579 m.Height, m.Round, m.Step, m.LastCommitRound) 1580 } 1581 1582 //------------------------------------- 1583 1584 // NewValidBlockMessage is sent when a validator observes a valid block B in some round r, 1585 // i.e., there is a Proposal for block B and 2/3+ prevotes for the block B in the round r. 1586 // In case the block is also committed, then IsCommit flag is set to true. 1587 type NewValidBlockMessage struct { 1588 Height int64 1589 Round int32 1590 BlockPartSetHeader types.PartSetHeader 1591 BlockParts *bits.BitArray 1592 IsCommit bool 1593 } 1594 1595 // ValidateBasic performs basic validation. 1596 func (m *NewValidBlockMessage) ValidateBasic() error { 1597 if m.Height < 0 { 1598 return errors.New("negative Height") 1599 } 1600 if m.Round < 0 { 1601 return errors.New("negative Round") 1602 } 1603 if err := m.BlockPartSetHeader.ValidateBasic(); err != nil { 1604 return fmt.Errorf("wrong BlockPartSetHeader: %v", err) 1605 } 1606 if m.BlockParts.Size() == 0 { 1607 return errors.New("empty blockParts") 1608 } 1609 if m.BlockParts.Size() != int(m.BlockPartSetHeader.Total) { 1610 return fmt.Errorf("blockParts bit array size %d not equal to BlockPartSetHeader.Total %d", 1611 m.BlockParts.Size(), 1612 m.BlockPartSetHeader.Total) 1613 } 1614 if m.BlockParts.Size() > int(types.MaxBlockPartsCount) { 1615 return fmt.Errorf("blockParts bit array is too big: %d, max: %d", m.BlockParts.Size(), types.MaxBlockPartsCount) 1616 } 1617 return nil 1618 } 1619 1620 // String returns a string representation. 1621 func (m *NewValidBlockMessage) String() string { 1622 return fmt.Sprintf("[ValidBlockMessage H:%v R:%v BP:%v BA:%v IsCommit:%v]", 1623 m.Height, m.Round, m.BlockPartSetHeader, m.BlockParts, m.IsCommit) 1624 } 1625 1626 //------------------------------------- 1627 1628 // ProposalMessage is sent when a new block is proposed. 1629 type ProposalMessage struct { 1630 Proposal *types.Proposal 1631 } 1632 1633 // ValidateBasic performs basic validation. 1634 func (m *ProposalMessage) ValidateBasic() error { 1635 return m.Proposal.ValidateBasic() 1636 } 1637 1638 // String returns a string representation. 1639 func (m *ProposalMessage) String() string { 1640 return fmt.Sprintf("[Proposal %v]", m.Proposal) 1641 } 1642 1643 //------------------------------------- 1644 1645 // ProposalPOLMessage is sent when a previous proposal is re-proposed. 1646 type ProposalPOLMessage struct { 1647 Height int64 1648 ProposalPOLRound int32 1649 ProposalPOL *bits.BitArray 1650 } 1651 1652 // ValidateBasic performs basic validation. 1653 func (m *ProposalPOLMessage) ValidateBasic() error { 1654 if m.Height < 0 { 1655 return errors.New("negative Height") 1656 } 1657 if m.ProposalPOLRound < 0 { 1658 return errors.New("negative ProposalPOLRound") 1659 } 1660 if m.ProposalPOL.Size() == 0 { 1661 return errors.New("empty ProposalPOL bit array") 1662 } 1663 if m.ProposalPOL.Size() > types.MaxVotesCount { 1664 return fmt.Errorf("proposalPOL bit array is too big: %d, max: %d", m.ProposalPOL.Size(), types.MaxVotesCount) 1665 } 1666 return nil 1667 } 1668 1669 // String returns a string representation. 1670 func (m *ProposalPOLMessage) String() string { 1671 return fmt.Sprintf("[ProposalPOL H:%v POLR:%v POL:%v]", m.Height, m.ProposalPOLRound, m.ProposalPOL) 1672 } 1673 1674 //------------------------------------- 1675 1676 // BlockPartMessage is sent when gossipping a piece of the proposed block. 1677 type BlockPartMessage struct { 1678 Height int64 1679 Round int32 1680 Part *types.Part 1681 } 1682 1683 // ValidateBasic performs basic validation. 1684 func (m *BlockPartMessage) ValidateBasic() error { 1685 if m.Height < 0 { 1686 return errors.New("negative Height") 1687 } 1688 if m.Round < 0 { 1689 return errors.New("negative Round") 1690 } 1691 if err := m.Part.ValidateBasic(); err != nil { 1692 return fmt.Errorf("wrong Part: %v", err) 1693 } 1694 return nil 1695 } 1696 1697 // String returns a string representation. 1698 func (m *BlockPartMessage) String() string { 1699 return fmt.Sprintf("[BlockPart H:%v R:%v P:%v]", m.Height, m.Round, m.Part) 1700 } 1701 1702 //------------------------------------- 1703 1704 // VoteMessage is sent when voting for a proposal (or lack thereof). 1705 type VoteMessage struct { 1706 Vote *types.Vote 1707 } 1708 1709 // ValidateBasic performs basic validation. 1710 func (m *VoteMessage) ValidateBasic() error { 1711 return m.Vote.ValidateBasic() 1712 } 1713 1714 // String returns a string representation. 1715 func (m *VoteMessage) String() string { 1716 return fmt.Sprintf("[Vote %v]", m.Vote) 1717 } 1718 1719 //------------------------------------- 1720 1721 // HasVoteMessage is sent to indicate that a particular vote has been received. 1722 type HasVoteMessage struct { 1723 Height int64 1724 Round int32 1725 Type tmproto.SignedMsgType 1726 Index int32 1727 } 1728 1729 // ValidateBasic performs basic validation. 1730 func (m *HasVoteMessage) ValidateBasic() error { 1731 if m.Height < 0 { 1732 return errors.New("negative Height") 1733 } 1734 if m.Round < 0 { 1735 return errors.New("negative Round") 1736 } 1737 if !types.IsVoteTypeValid(m.Type) { 1738 return errors.New("invalid Type") 1739 } 1740 if m.Index < 0 { 1741 return errors.New("negative Index") 1742 } 1743 return nil 1744 } 1745 1746 // String returns a string representation. 1747 func (m *HasVoteMessage) String() string { 1748 return fmt.Sprintf("[HasVote VI:%v V:{%v/%02d/%v}]", m.Index, m.Height, m.Round, m.Type) 1749 } 1750 1751 //------------------------------------- 1752 1753 // VoteSetMaj23Message is sent to indicate that a given BlockID has seen +2/3 votes. 1754 type VoteSetMaj23Message struct { 1755 Height int64 1756 Round int32 1757 Type tmproto.SignedMsgType 1758 BlockID types.BlockID 1759 } 1760 1761 // ValidateBasic performs basic validation. 1762 func (m *VoteSetMaj23Message) ValidateBasic() error { 1763 if m.Height < 0 { 1764 return errors.New("negative Height") 1765 } 1766 if m.Round < 0 { 1767 return errors.New("negative Round") 1768 } 1769 if !types.IsVoteTypeValid(m.Type) { 1770 return errors.New("invalid Type") 1771 } 1772 if err := m.BlockID.ValidateBasic(); err != nil { 1773 return fmt.Errorf("wrong BlockID: %v", err) 1774 } 1775 return nil 1776 } 1777 1778 // String returns a string representation. 1779 func (m *VoteSetMaj23Message) String() string { 1780 return fmt.Sprintf("[VSM23 %v/%02d/%v %v]", m.Height, m.Round, m.Type, m.BlockID) 1781 } 1782 1783 //------------------------------------- 1784 1785 // VoteSetBitsMessage is sent to communicate the bit-array of votes seen for the BlockID. 1786 type VoteSetBitsMessage struct { 1787 Height int64 1788 Round int32 1789 Type tmproto.SignedMsgType 1790 BlockID types.BlockID 1791 Votes *bits.BitArray 1792 } 1793 1794 // ValidateBasic performs basic validation. 1795 func (m *VoteSetBitsMessage) ValidateBasic() error { 1796 if m.Height < 0 { 1797 return errors.New("negative Height") 1798 } 1799 if !types.IsVoteTypeValid(m.Type) { 1800 return errors.New("invalid Type") 1801 } 1802 if err := m.BlockID.ValidateBasic(); err != nil { 1803 return fmt.Errorf("wrong BlockID: %v", err) 1804 } 1805 // NOTE: Votes.Size() can be zero if the node does not have any 1806 if m.Votes.Size() > types.MaxVotesCount { 1807 return fmt.Errorf("votes bit array is too big: %d, max: %d", m.Votes.Size(), types.MaxVotesCount) 1808 } 1809 return nil 1810 } 1811 1812 // String returns a string representation. 1813 func (m *VoteSetBitsMessage) String() string { 1814 return fmt.Sprintf("[VSB %v/%02d/%v %v %v]", m.Height, m.Round, m.Type, m.BlockID, m.Votes) 1815 } 1816 1817 //-------------------------------------