gitlab.com/gpdionisio/tendermint@v0.34.19-dev2/consensus/reactor.go (about) 1 package consensus 2 3 import ( 4 "errors" 5 "fmt" 6 "reflect" 7 "sync" 8 "time" 9 10 "github.com/gogo/protobuf/proto" 11 12 cstypes "github.com/tendermint/tendermint/consensus/types" 13 "github.com/tendermint/tendermint/libs/bits" 14 tmevents "github.com/tendermint/tendermint/libs/events" 15 tmjson "github.com/tendermint/tendermint/libs/json" 16 "github.com/tendermint/tendermint/libs/log" 17 tmsync "github.com/tendermint/tendermint/libs/sync" 18 "github.com/tendermint/tendermint/p2p" 19 tmcons "github.com/tendermint/tendermint/proto/tendermint/consensus" 20 tmproto "github.com/tendermint/tendermint/proto/tendermint/types" 21 sm "github.com/tendermint/tendermint/state" 22 "github.com/tendermint/tendermint/types" 23 tmtime "github.com/tendermint/tendermint/types/time" 24 ) 25 26 const ( 27 StateChannel = byte(0x20) 28 DataChannel = byte(0x21) 29 VoteChannel = byte(0x22) 30 VoteSetBitsChannel = byte(0x23) 31 32 maxMsgSize = 1048576 // 1MB; NOTE/TODO: keep in sync with types.PartSet sizes. 33 34 blocksToContributeToBecomeGoodPeer = 10000 35 votesToContributeToBecomeGoodPeer = 10000 36 ) 37 38 //----------------------------------------------------------------------------- 39 40 // Reactor defines a reactor for the consensus service. 41 type Reactor struct { 42 p2p.BaseReactor // BaseService + p2p.Switch 43 44 conS *State 45 46 mtx tmsync.RWMutex 47 waitSync bool 48 eventBus *types.EventBus 49 rs *cstypes.RoundState 50 51 Metrics *Metrics 52 } 53 54 type ReactorOption func(*Reactor) 55 56 // NewReactor returns a new Reactor with the given 57 // consensusState. 58 func NewReactor(consensusState *State, waitSync bool, options ...ReactorOption) *Reactor { 59 conR := &Reactor{ 60 conS: consensusState, 61 waitSync: waitSync, 62 rs: consensusState.GetRoundState(), 63 Metrics: NopMetrics(), 64 } 65 conR.BaseReactor = *p2p.NewBaseReactor("Consensus", conR) 66 67 for _, option := range options { 68 option(conR) 69 } 70 71 return conR 72 } 73 74 // OnStart implements BaseService by subscribing to events, which later will be 75 // broadcasted to other peers and starting state if we're not in fast sync. 76 func (conR *Reactor) OnStart() error { 77 conR.Logger.Info("Reactor ", "waitSync", conR.WaitSync()) 78 79 // start routine that computes peer statistics for evaluating peer quality 80 go conR.peerStatsRoutine() 81 82 conR.subscribeToBroadcastEvents() 83 go conR.updateRoundStateRoutine() 84 85 if !conR.WaitSync() { 86 err := conR.conS.Start() 87 if err != nil { 88 return err 89 } 90 } 91 92 return nil 93 } 94 95 // OnStop implements BaseService by unsubscribing from events and stopping 96 // state. 97 func (conR *Reactor) OnStop() { 98 conR.unsubscribeFromBroadcastEvents() 99 if err := conR.conS.Stop(); err != nil { 100 conR.Logger.Error("Error stopping consensus state", "err", err) 101 } 102 if !conR.WaitSync() { 103 conR.conS.Wait() 104 } 105 } 106 107 // SwitchToConsensus switches from fast_sync mode to consensus mode. 108 // It resets the state, turns off fast_sync, and starts the consensus state-machine 109 func (conR *Reactor) SwitchToConsensus(state sm.State, skipWAL bool) { 110 conR.Logger.Info("SwitchToConsensus") 111 112 // We have no votes, so reconstruct LastCommit from SeenCommit. 113 if state.LastBlockHeight > 0 { 114 conR.conS.reconstructLastCommit(state) 115 } 116 117 // NOTE: The line below causes broadcastNewRoundStepRoutine() to broadcast a 118 // NewRoundStepMessage. 119 conR.conS.updateToState(state) 120 121 conR.mtx.Lock() 122 conR.waitSync = false 123 conR.mtx.Unlock() 124 conR.Metrics.FastSyncing.Set(0) 125 conR.Metrics.StateSyncing.Set(0) 126 127 if skipWAL { 128 conR.conS.doWALCatchup = false 129 } 130 err := conR.conS.Start() 131 if err != nil { 132 panic(fmt.Sprintf(`Failed to start consensus state: %v 133 134 conS: 135 %+v 136 137 conR: 138 %+v`, err, conR.conS, conR)) 139 } 140 } 141 142 // GetChannels implements Reactor 143 func (conR *Reactor) GetChannels() []*p2p.ChannelDescriptor { 144 // TODO optimize 145 return []*p2p.ChannelDescriptor{ 146 { 147 ID: StateChannel, 148 Priority: 6, 149 SendQueueCapacity: 100, 150 RecvMessageCapacity: maxMsgSize, 151 }, 152 { 153 ID: DataChannel, // maybe split between gossiping current block and catchup stuff 154 // once we gossip the whole block there's nothing left to send until next height or round 155 Priority: 10, 156 SendQueueCapacity: 100, 157 RecvBufferCapacity: 50 * 4096, 158 RecvMessageCapacity: maxMsgSize, 159 }, 160 { 161 ID: VoteChannel, 162 Priority: 7, 163 SendQueueCapacity: 100, 164 RecvBufferCapacity: 100 * 100, 165 RecvMessageCapacity: maxMsgSize, 166 }, 167 { 168 ID: VoteSetBitsChannel, 169 Priority: 1, 170 SendQueueCapacity: 2, 171 RecvBufferCapacity: 1024, 172 RecvMessageCapacity: maxMsgSize, 173 }, 174 } 175 } 176 177 // InitPeer implements Reactor by creating a state for the peer. 178 func (conR *Reactor) InitPeer(peer p2p.Peer) p2p.Peer { 179 peerState := NewPeerState(peer).SetLogger(conR.Logger) 180 peer.Set(types.PeerStateKey, peerState) 181 return peer 182 } 183 184 // AddPeer implements Reactor by spawning multiple gossiping goroutines for the 185 // peer. 186 func (conR *Reactor) AddPeer(peer p2p.Peer) { 187 if !conR.IsRunning() { 188 return 189 } 190 191 peerState, ok := peer.Get(types.PeerStateKey).(*PeerState) 192 if !ok { 193 panic(fmt.Sprintf("peer %v has no state", peer)) 194 } 195 // Begin routines for this peer. 196 go conR.gossipDataRoutine(peer, peerState) 197 go conR.gossipVotesRoutine(peer, peerState) 198 go conR.queryMaj23Routine(peer, peerState) 199 200 // Send our state to peer. 201 // If we're fast_syncing, broadcast a RoundStepMessage later upon SwitchToConsensus(). 202 if !conR.WaitSync() { 203 conR.sendNewRoundStepMessage(peer) 204 } 205 } 206 207 // RemovePeer is a noop. 208 func (conR *Reactor) RemovePeer(peer p2p.Peer, reason interface{}) { 209 if !conR.IsRunning() { 210 return 211 } 212 // TODO 213 // ps, ok := peer.Get(PeerStateKey).(*PeerState) 214 // if !ok { 215 // panic(fmt.Sprintf("Peer %v has no state", peer)) 216 // } 217 // ps.Disconnect() 218 } 219 220 // Receive implements Reactor 221 // NOTE: We process these messages even when we're fast_syncing. 222 // Messages affect either a peer state or the consensus state. 223 // Peer state updates can happen in parallel, but processing of 224 // proposals, block parts, and votes are ordered by the receiveRoutine 225 // NOTE: blocks on consensus state for proposals, block parts, and votes 226 func (conR *Reactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { 227 if !conR.IsRunning() { 228 conR.Logger.Debug("Receive", "src", src, "chId", chID, "bytes", msgBytes) 229 return 230 } 231 232 msg, err := decodeMsg(msgBytes) 233 if err != nil { 234 conR.Logger.Error("Error decoding message", "src", src, "chId", chID, "err", err) 235 conR.Switch.StopPeerForError(src, err) 236 return 237 } 238 239 if err = msg.ValidateBasic(); err != nil { 240 conR.Logger.Error("Peer sent us invalid msg", "peer", src, "msg", msg, "err", err) 241 conR.Switch.StopPeerForError(src, err) 242 return 243 } 244 245 conR.Logger.Debug("Receive", "src", src, "chId", chID, "msg", msg) 246 247 // Get peer states 248 ps, ok := src.Get(types.PeerStateKey).(*PeerState) 249 if !ok { 250 panic(fmt.Sprintf("Peer %v has no state", src)) 251 } 252 253 switch chID { 254 case StateChannel: 255 switch msg := msg.(type) { 256 case *NewRoundStepMessage: 257 conR.conS.mtx.Lock() 258 initialHeight := conR.conS.state.InitialHeight 259 conR.conS.mtx.Unlock() 260 if err = msg.ValidateHeight(initialHeight); err != nil { 261 conR.Logger.Error("Peer sent us invalid msg", "peer", src, "msg", msg, "err", err) 262 conR.Switch.StopPeerForError(src, err) 263 return 264 } 265 ps.ApplyNewRoundStepMessage(msg) 266 case *NewValidBlockMessage: 267 ps.ApplyNewValidBlockMessage(msg) 268 case *HasVoteMessage: 269 ps.ApplyHasVoteMessage(msg) 270 case *VoteSetMaj23Message: 271 cs := conR.conS 272 cs.mtx.Lock() 273 height, votes := cs.Height, cs.Votes 274 cs.mtx.Unlock() 275 if height != msg.Height { 276 return 277 } 278 // Peer claims to have a maj23 for some BlockID at H,R,S, 279 err := votes.SetPeerMaj23(msg.Round, msg.Type, ps.peer.ID(), msg.BlockID) 280 if err != nil { 281 conR.Switch.StopPeerForError(src, err) 282 return 283 } 284 // Respond with a VoteSetBitsMessage showing which votes we have. 285 // (and consequently shows which we don't have) 286 var ourVotes *bits.BitArray 287 switch msg.Type { 288 case tmproto.PrevoteType: 289 ourVotes = votes.Prevotes(msg.Round).BitArrayByBlockID(msg.BlockID) 290 case tmproto.PrecommitType: 291 ourVotes = votes.Precommits(msg.Round).BitArrayByBlockID(msg.BlockID) 292 default: 293 panic("Bad VoteSetBitsMessage field Type. Forgot to add a check in ValidateBasic?") 294 } 295 src.TrySend(VoteSetBitsChannel, MustEncode(&VoteSetBitsMessage{ 296 Height: msg.Height, 297 Round: msg.Round, 298 Type: msg.Type, 299 BlockID: msg.BlockID, 300 Votes: ourVotes, 301 })) 302 default: 303 conR.Logger.Error(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg))) 304 } 305 306 case DataChannel: 307 if conR.WaitSync() { 308 conR.Logger.Info("Ignoring message received during sync", "msg", msg) 309 return 310 } 311 switch msg := msg.(type) { 312 case *ProposalMessage: 313 ps.SetHasProposal(msg.Proposal) 314 conR.conS.peerMsgQueue <- msgInfo{msg, src.ID()} 315 case *ProposalPOLMessage: 316 ps.ApplyProposalPOLMessage(msg) 317 case *BlockPartMessage: 318 ps.SetHasProposalBlockPart(msg.Height, msg.Round, int(msg.Part.Index)) 319 conR.Metrics.BlockParts.With("peer_id", string(src.ID())).Add(1) 320 conR.conS.peerMsgQueue <- msgInfo{msg, src.ID()} 321 default: 322 conR.Logger.Error(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg))) 323 } 324 325 case VoteChannel: 326 if conR.WaitSync() { 327 conR.Logger.Info("Ignoring message received during sync", "msg", msg) 328 return 329 } 330 switch msg := msg.(type) { 331 case *VoteMessage: 332 cs := conR.conS 333 cs.mtx.RLock() 334 height, valSize, lastCommitSize := cs.Height, cs.Validators.Size(), cs.LastCommit.Size() 335 cs.mtx.RUnlock() 336 ps.EnsureVoteBitArrays(height, valSize) 337 ps.EnsureVoteBitArrays(height-1, lastCommitSize) 338 ps.SetHasVote(msg.Vote) 339 340 cs.peerMsgQueue <- msgInfo{msg, src.ID()} 341 342 default: 343 // don't punish (leave room for soft upgrades) 344 conR.Logger.Error(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg))) 345 } 346 347 case VoteSetBitsChannel: 348 if conR.WaitSync() { 349 conR.Logger.Info("Ignoring message received during sync", "msg", msg) 350 return 351 } 352 switch msg := msg.(type) { 353 case *VoteSetBitsMessage: 354 cs := conR.conS 355 cs.mtx.Lock() 356 height, votes := cs.Height, cs.Votes 357 cs.mtx.Unlock() 358 359 if height == msg.Height { 360 var ourVotes *bits.BitArray 361 switch msg.Type { 362 case tmproto.PrevoteType: 363 ourVotes = votes.Prevotes(msg.Round).BitArrayByBlockID(msg.BlockID) 364 case tmproto.PrecommitType: 365 ourVotes = votes.Precommits(msg.Round).BitArrayByBlockID(msg.BlockID) 366 default: 367 panic("Bad VoteSetBitsMessage field Type. Forgot to add a check in ValidateBasic?") 368 } 369 ps.ApplyVoteSetBitsMessage(msg, ourVotes) 370 } else { 371 ps.ApplyVoteSetBitsMessage(msg, nil) 372 } 373 default: 374 // don't punish (leave room for soft upgrades) 375 conR.Logger.Error(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg))) 376 } 377 378 default: 379 conR.Logger.Error(fmt.Sprintf("Unknown chId %X", chID)) 380 } 381 } 382 383 // SetEventBus sets event bus. 384 func (conR *Reactor) SetEventBus(b *types.EventBus) { 385 conR.eventBus = b 386 conR.conS.SetEventBus(b) 387 } 388 389 // WaitSync returns whether the consensus reactor is waiting for state/fast sync. 390 func (conR *Reactor) WaitSync() bool { 391 conR.mtx.RLock() 392 defer conR.mtx.RUnlock() 393 return conR.waitSync 394 } 395 396 //-------------------------------------- 397 398 // subscribeToBroadcastEvents subscribes for new round steps and votes 399 // using internal pubsub defined on state to broadcast 400 // them to peers upon receiving. 401 func (conR *Reactor) subscribeToBroadcastEvents() { 402 const subscriber = "consensus-reactor" 403 if err := conR.conS.evsw.AddListenerForEvent(subscriber, types.EventNewRoundStep, 404 func(data tmevents.EventData) { 405 conR.broadcastNewRoundStepMessage(data.(*cstypes.RoundState)) 406 }); err != nil { 407 conR.Logger.Error("Error adding listener for events", "err", err) 408 } 409 410 if err := conR.conS.evsw.AddListenerForEvent(subscriber, types.EventValidBlock, 411 func(data tmevents.EventData) { 412 conR.broadcastNewValidBlockMessage(data.(*cstypes.RoundState)) 413 }); err != nil { 414 conR.Logger.Error("Error adding listener for events", "err", err) 415 } 416 417 if err := conR.conS.evsw.AddListenerForEvent(subscriber, types.EventVote, 418 func(data tmevents.EventData) { 419 conR.broadcastHasVoteMessage(data.(*types.Vote)) 420 }); err != nil { 421 conR.Logger.Error("Error adding listener for events", "err", err) 422 } 423 424 } 425 426 func (conR *Reactor) unsubscribeFromBroadcastEvents() { 427 const subscriber = "consensus-reactor" 428 conR.conS.evsw.RemoveListener(subscriber) 429 } 430 431 func (conR *Reactor) broadcastNewRoundStepMessage(rs *cstypes.RoundState) { 432 nrsMsg := makeRoundStepMessage(rs) 433 conR.Switch.Broadcast(StateChannel, MustEncode(nrsMsg)) 434 } 435 436 func (conR *Reactor) broadcastNewValidBlockMessage(rs *cstypes.RoundState) { 437 csMsg := &NewValidBlockMessage{ 438 Height: rs.Height, 439 Round: rs.Round, 440 BlockPartSetHeader: rs.ProposalBlockParts.Header(), 441 BlockParts: rs.ProposalBlockParts.BitArray(), 442 IsCommit: rs.Step == cstypes.RoundStepCommit, 443 } 444 conR.Switch.Broadcast(StateChannel, MustEncode(csMsg)) 445 } 446 447 // Broadcasts HasVoteMessage to peers that care. 448 func (conR *Reactor) broadcastHasVoteMessage(vote *types.Vote) { 449 msg := &HasVoteMessage{ 450 Height: vote.Height, 451 Round: vote.Round, 452 Type: vote.Type, 453 Index: vote.ValidatorIndex, 454 } 455 conR.Switch.Broadcast(StateChannel, MustEncode(msg)) 456 /* 457 // TODO: Make this broadcast more selective. 458 for _, peer := range conR.Switch.Peers().List() { 459 ps, ok := peer.Get(PeerStateKey).(*PeerState) 460 if !ok { 461 panic(fmt.Sprintf("Peer %v has no state", peer)) 462 } 463 prs := ps.GetRoundState() 464 if prs.Height == vote.Height { 465 // TODO: Also filter on round? 466 peer.TrySend(StateChannel, struct{ ConsensusMessage }{msg}) 467 } else { 468 // Height doesn't match 469 // TODO: check a field, maybe CatchupCommitRound? 470 // TODO: But that requires changing the struct field comment. 471 } 472 } 473 */ 474 } 475 476 func makeRoundStepMessage(rs *cstypes.RoundState) (nrsMsg *NewRoundStepMessage) { 477 nrsMsg = &NewRoundStepMessage{ 478 Height: rs.Height, 479 Round: rs.Round, 480 Step: rs.Step, 481 SecondsSinceStartTime: int64(time.Since(rs.StartTime).Seconds()), 482 LastCommitRound: rs.LastCommit.GetRound(), 483 } 484 return 485 } 486 487 func (conR *Reactor) sendNewRoundStepMessage(peer p2p.Peer) { 488 rs := conR.getRoundState() 489 nrsMsg := makeRoundStepMessage(rs) 490 peer.Send(StateChannel, MustEncode(nrsMsg)) 491 } 492 493 func (conR *Reactor) updateRoundStateRoutine() { 494 t := time.NewTicker(100 * time.Microsecond) 495 defer t.Stop() 496 for range t.C { 497 if !conR.IsRunning() { 498 return 499 } 500 rs := conR.conS.GetRoundState() 501 conR.mtx.Lock() 502 conR.rs = rs 503 conR.mtx.Unlock() 504 } 505 } 506 507 func (conR *Reactor) getRoundState() *cstypes.RoundState { 508 conR.mtx.RLock() 509 defer conR.mtx.RUnlock() 510 return conR.rs 511 } 512 513 func (conR *Reactor) gossipDataRoutine(peer p2p.Peer, ps *PeerState) { 514 logger := conR.Logger.With("peer", peer) 515 516 OUTER_LOOP: 517 for { 518 // Manage disconnects from self or peer. 519 if !peer.IsRunning() || !conR.IsRunning() { 520 return 521 } 522 rs := conR.getRoundState() 523 prs := ps.GetRoundState() 524 525 // Send proposal Block parts? 526 if rs.ProposalBlockParts.HasHeader(prs.ProposalBlockPartSetHeader) { 527 if index, ok := rs.ProposalBlockParts.BitArray().Sub(prs.ProposalBlockParts.Copy()).PickRandom(); ok { 528 part := rs.ProposalBlockParts.GetPart(index) 529 msg := &BlockPartMessage{ 530 Height: rs.Height, // This tells peer that this part applies to us. 531 Round: rs.Round, // This tells peer that this part applies to us. 532 Part: part, 533 } 534 logger.Debug("Sending block part", "height", prs.Height, "round", prs.Round) 535 if peer.Send(DataChannel, MustEncode(msg)) { 536 ps.SetHasProposalBlockPart(prs.Height, prs.Round, index) 537 } 538 continue OUTER_LOOP 539 } 540 } 541 542 // If the peer is on a previous height that we have, help catch up. 543 blockStoreBase := conR.conS.blockStore.Base() 544 if blockStoreBase > 0 && 0 < prs.Height && prs.Height < rs.Height && prs.Height >= blockStoreBase { 545 heightLogger := logger.With("height", prs.Height) 546 547 // if we never received the commit message from the peer, the block parts wont be initialized 548 if prs.ProposalBlockParts == nil { 549 blockMeta := conR.conS.blockStore.LoadBlockMeta(prs.Height) 550 if blockMeta == nil { 551 heightLogger.Error("Failed to load block meta", 552 "blockstoreBase", blockStoreBase, "blockstoreHeight", conR.conS.blockStore.Height()) 553 time.Sleep(conR.conS.config.PeerGossipSleepDuration) 554 } else { 555 ps.InitProposalBlockParts(blockMeta.BlockID.PartSetHeader) 556 } 557 // continue the loop since prs is a copy and not effected by this initialization 558 continue OUTER_LOOP 559 } 560 conR.gossipDataForCatchup(heightLogger, rs, prs, ps, peer) 561 continue OUTER_LOOP 562 } 563 564 // If height and round don't match, sleep. 565 if (rs.Height != prs.Height) || (rs.Round != prs.Round) { 566 // logger.Info("Peer Height|Round mismatch, sleeping", 567 // "peerHeight", prs.Height, "peerRound", prs.Round, "peer", peer) 568 time.Sleep(conR.conS.config.PeerGossipSleepDuration) 569 continue OUTER_LOOP 570 } 571 572 // By here, height and round match. 573 // Proposal block parts were already matched and sent if any were wanted. 574 // (These can match on hash so the round doesn't matter) 575 // Now consider sending other things, like the Proposal itself. 576 577 // Send Proposal && ProposalPOL BitArray? 578 if rs.Proposal != nil && !prs.Proposal { 579 // Proposal: share the proposal metadata with peer. 580 { 581 msg := &ProposalMessage{Proposal: rs.Proposal} 582 logger.Debug("Sending proposal", "height", prs.Height, "round", prs.Round) 583 if peer.Send(DataChannel, MustEncode(msg)) { 584 // NOTE[ZM]: A peer might have received different proposal msg so this Proposal msg will be rejected! 585 ps.SetHasProposal(rs.Proposal) 586 } 587 } 588 // ProposalPOL: lets peer know which POL votes we have so far. 589 // Peer must receive ProposalMessage first. 590 // rs.Proposal was validated, so rs.Proposal.POLRound <= rs.Round, 591 // so we definitely have rs.Votes.Prevotes(rs.Proposal.POLRound). 592 if 0 <= rs.Proposal.POLRound { 593 msg := &ProposalPOLMessage{ 594 Height: rs.Height, 595 ProposalPOLRound: rs.Proposal.POLRound, 596 ProposalPOL: rs.Votes.Prevotes(rs.Proposal.POLRound).BitArray(), 597 } 598 logger.Debug("Sending POL", "height", prs.Height, "round", prs.Round) 599 peer.Send(DataChannel, MustEncode(msg)) 600 } 601 continue OUTER_LOOP 602 } 603 604 // Nothing to do. Sleep. 605 time.Sleep(conR.conS.config.PeerGossipSleepDuration) 606 continue OUTER_LOOP 607 } 608 } 609 610 func (conR *Reactor) gossipDataForCatchup(logger log.Logger, rs *cstypes.RoundState, 611 prs *cstypes.PeerRoundState, ps *PeerState, peer p2p.Peer) { 612 613 if index, ok := prs.ProposalBlockParts.Not().PickRandom(); ok { 614 // Ensure that the peer's PartSetHeader is correct 615 blockMeta := conR.conS.blockStore.LoadBlockMeta(prs.Height) 616 if blockMeta == nil { 617 logger.Error("Failed to load block meta", "ourHeight", rs.Height, 618 "blockstoreBase", conR.conS.blockStore.Base(), "blockstoreHeight", conR.conS.blockStore.Height()) 619 time.Sleep(conR.conS.config.PeerGossipSleepDuration) 620 return 621 } else if !blockMeta.BlockID.PartSetHeader.Equals(prs.ProposalBlockPartSetHeader) { 622 logger.Info("Peer ProposalBlockPartSetHeader mismatch, sleeping", 623 "blockPartSetHeader", blockMeta.BlockID.PartSetHeader, "peerBlockPartSetHeader", prs.ProposalBlockPartSetHeader) 624 time.Sleep(conR.conS.config.PeerGossipSleepDuration) 625 return 626 } 627 // Load the part 628 part := conR.conS.blockStore.LoadBlockPart(prs.Height, index) 629 if part == nil { 630 logger.Error("Could not load part", "index", index, 631 "blockPartSetHeader", blockMeta.BlockID.PartSetHeader, "peerBlockPartSetHeader", prs.ProposalBlockPartSetHeader) 632 time.Sleep(conR.conS.config.PeerGossipSleepDuration) 633 return 634 } 635 // Send the part 636 msg := &BlockPartMessage{ 637 Height: prs.Height, // Not our height, so it doesn't matter. 638 Round: prs.Round, // Not our height, so it doesn't matter. 639 Part: part, 640 } 641 logger.Debug("Sending block part for catchup", "round", prs.Round, "index", index) 642 if peer.Send(DataChannel, MustEncode(msg)) { 643 ps.SetHasProposalBlockPart(prs.Height, prs.Round, index) 644 } else { 645 logger.Debug("Sending block part for catchup failed") 646 } 647 return 648 } 649 // logger.Info("No parts to send in catch-up, sleeping") 650 time.Sleep(conR.conS.config.PeerGossipSleepDuration) 651 } 652 653 func (conR *Reactor) gossipVotesRoutine(peer p2p.Peer, ps *PeerState) { 654 logger := conR.Logger.With("peer", peer) 655 656 // Simple hack to throttle logs upon sleep. 657 var sleeping = 0 658 659 OUTER_LOOP: 660 for { 661 // Manage disconnects from self or peer. 662 if !peer.IsRunning() || !conR.IsRunning() { 663 return 664 } 665 rs := conR.getRoundState() 666 prs := ps.GetRoundState() 667 668 switch sleeping { 669 case 1: // First sleep 670 sleeping = 2 671 case 2: // No more sleep 672 sleeping = 0 673 } 674 675 // logger.Debug("gossipVotesRoutine", "rsHeight", rs.Height, "rsRound", rs.Round, 676 // "prsHeight", prs.Height, "prsRound", prs.Round, "prsStep", prs.Step) 677 678 // If height matches, then send LastCommit, Prevotes, Precommits. 679 if rs.Height == prs.Height { 680 heightLogger := logger.With("height", prs.Height) 681 if conR.gossipVotesForHeight(heightLogger, rs, prs, ps) { 682 continue OUTER_LOOP 683 } 684 } 685 686 // Special catchup logic. 687 // If peer is lagging by height 1, send LastCommit. 688 if prs.Height != 0 && rs.Height == prs.Height+1 { 689 if ps.PickSendVote(rs.LastCommit) { 690 logger.Debug("Picked rs.LastCommit to send", "height", prs.Height) 691 continue OUTER_LOOP 692 } 693 } 694 695 // Catchup logic 696 // If peer is lagging by more than 1, send Commit. 697 blockStoreBase := conR.conS.blockStore.Base() 698 if blockStoreBase > 0 && prs.Height != 0 && rs.Height >= prs.Height+2 && prs.Height >= blockStoreBase { 699 // Load the block commit for prs.Height, 700 // which contains precommit signatures for prs.Height. 701 if commit := conR.conS.blockStore.LoadBlockCommit(prs.Height); commit != nil { 702 if ps.PickSendVote(commit) { 703 logger.Debug("Picked Catchup commit to send", "height", prs.Height) 704 continue OUTER_LOOP 705 } 706 } 707 } 708 709 if sleeping == 0 { 710 // We sent nothing. Sleep... 711 sleeping = 1 712 logger.Debug("No votes to send, sleeping", "rs.Height", rs.Height, "prs.Height", prs.Height, 713 "localPV", rs.Votes.Prevotes(rs.Round).BitArray(), "peerPV", prs.Prevotes, 714 "localPC", rs.Votes.Precommits(rs.Round).BitArray(), "peerPC", prs.Precommits) 715 } else if sleeping == 2 { 716 // Continued sleep... 717 sleeping = 1 718 } 719 720 time.Sleep(conR.conS.config.PeerGossipSleepDuration) 721 continue OUTER_LOOP 722 } 723 } 724 725 func (conR *Reactor) gossipVotesForHeight( 726 logger log.Logger, 727 rs *cstypes.RoundState, 728 prs *cstypes.PeerRoundState, 729 ps *PeerState, 730 ) bool { 731 732 // If there are lastCommits to send... 733 if prs.Step == cstypes.RoundStepNewHeight { 734 if ps.PickSendVote(rs.LastCommit) { 735 logger.Debug("Picked rs.LastCommit to send") 736 return true 737 } 738 } 739 // If there are POL prevotes to send... 740 if prs.Step <= cstypes.RoundStepPropose && prs.Round != -1 && prs.Round <= rs.Round && prs.ProposalPOLRound != -1 { 741 if polPrevotes := rs.Votes.Prevotes(prs.ProposalPOLRound); polPrevotes != nil { 742 if ps.PickSendVote(polPrevotes) { 743 logger.Debug("Picked rs.Prevotes(prs.ProposalPOLRound) to send", 744 "round", prs.ProposalPOLRound) 745 return true 746 } 747 } 748 } 749 // If there are prevotes to send... 750 if prs.Step <= cstypes.RoundStepPrevoteWait && prs.Round != -1 && prs.Round <= rs.Round { 751 if ps.PickSendVote(rs.Votes.Prevotes(prs.Round)) { 752 logger.Debug("Picked rs.Prevotes(prs.Round) to send", "round", prs.Round) 753 return true 754 } 755 } 756 // If there are precommits to send... 757 if prs.Step <= cstypes.RoundStepPrecommitWait && prs.Round != -1 && prs.Round <= rs.Round { 758 if ps.PickSendVote(rs.Votes.Precommits(prs.Round)) { 759 logger.Debug("Picked rs.Precommits(prs.Round) to send", "round", prs.Round) 760 return true 761 } 762 } 763 // If there are prevotes to send...Needed because of validBlock mechanism 764 if prs.Round != -1 && prs.Round <= rs.Round { 765 if ps.PickSendVote(rs.Votes.Prevotes(prs.Round)) { 766 logger.Debug("Picked rs.Prevotes(prs.Round) to send", "round", prs.Round) 767 return true 768 } 769 } 770 // If there are POLPrevotes to send... 771 if prs.ProposalPOLRound != -1 { 772 if polPrevotes := rs.Votes.Prevotes(prs.ProposalPOLRound); polPrevotes != nil { 773 if ps.PickSendVote(polPrevotes) { 774 logger.Debug("Picked rs.Prevotes(prs.ProposalPOLRound) to send", 775 "round", prs.ProposalPOLRound) 776 return true 777 } 778 } 779 } 780 781 return false 782 } 783 784 // NOTE: `queryMaj23Routine` has a simple crude design since it only comes 785 // into play for liveness when there's a signature DDoS attack happening. 786 func (conR *Reactor) queryMaj23Routine(peer p2p.Peer, ps *PeerState) { 787 788 OUTER_LOOP: 789 for { 790 // Manage disconnects from self or peer. 791 if !peer.IsRunning() || !conR.IsRunning() { 792 return 793 } 794 795 // Maybe send Height/Round/Prevotes 796 { 797 rs := conR.getRoundState() 798 prs := ps.GetRoundState() 799 if rs.Height == prs.Height { 800 if maj23, ok := rs.Votes.Prevotes(prs.Round).TwoThirdsMajority(); ok { 801 peer.TrySend(StateChannel, MustEncode(&VoteSetMaj23Message{ 802 Height: prs.Height, 803 Round: prs.Round, 804 Type: tmproto.PrevoteType, 805 BlockID: maj23, 806 })) 807 time.Sleep(conR.conS.config.PeerQueryMaj23SleepDuration) 808 } 809 } 810 } 811 812 // Maybe send Height/Round/Precommits 813 { 814 rs := conR.getRoundState() 815 prs := ps.GetRoundState() 816 if rs.Height == prs.Height { 817 if maj23, ok := rs.Votes.Precommits(prs.Round).TwoThirdsMajority(); ok { 818 peer.TrySend(StateChannel, MustEncode(&VoteSetMaj23Message{ 819 Height: prs.Height, 820 Round: prs.Round, 821 Type: tmproto.PrecommitType, 822 BlockID: maj23, 823 })) 824 time.Sleep(conR.conS.config.PeerQueryMaj23SleepDuration) 825 } 826 } 827 } 828 829 // Maybe send Height/Round/ProposalPOL 830 { 831 rs := conR.getRoundState() 832 prs := ps.GetRoundState() 833 if rs.Height == prs.Height && prs.ProposalPOLRound >= 0 { 834 if maj23, ok := rs.Votes.Prevotes(prs.ProposalPOLRound).TwoThirdsMajority(); ok { 835 peer.TrySend(StateChannel, MustEncode(&VoteSetMaj23Message{ 836 Height: prs.Height, 837 Round: prs.ProposalPOLRound, 838 Type: tmproto.PrevoteType, 839 BlockID: maj23, 840 })) 841 time.Sleep(conR.conS.config.PeerQueryMaj23SleepDuration) 842 } 843 } 844 } 845 846 // Little point sending LastCommitRound/LastCommit, 847 // These are fleeting and non-blocking. 848 849 // Maybe send Height/CatchupCommitRound/CatchupCommit. 850 { 851 prs := ps.GetRoundState() 852 if prs.CatchupCommitRound != -1 && prs.Height > 0 && prs.Height <= conR.conS.blockStore.Height() && 853 prs.Height >= conR.conS.blockStore.Base() { 854 if commit := conR.conS.LoadCommit(prs.Height); commit != nil { 855 peer.TrySend(StateChannel, MustEncode(&VoteSetMaj23Message{ 856 Height: prs.Height, 857 Round: commit.Round, 858 Type: tmproto.PrecommitType, 859 BlockID: commit.BlockID, 860 })) 861 time.Sleep(conR.conS.config.PeerQueryMaj23SleepDuration) 862 } 863 } 864 } 865 866 time.Sleep(conR.conS.config.PeerQueryMaj23SleepDuration) 867 868 continue OUTER_LOOP 869 } 870 } 871 872 func (conR *Reactor) peerStatsRoutine() { 873 for { 874 if !conR.IsRunning() { 875 conR.Logger.Info("Stopping peerStatsRoutine") 876 return 877 } 878 879 select { 880 case msg := <-conR.conS.statsMsgQueue: 881 // Get peer 882 peer := conR.Switch.Peers().Get(msg.PeerID) 883 if peer == nil { 884 conR.Logger.Debug("Attempt to update stats for non-existent peer", 885 "peer", msg.PeerID) 886 continue 887 } 888 // Get peer state 889 ps, ok := peer.Get(types.PeerStateKey).(*PeerState) 890 if !ok { 891 panic(fmt.Sprintf("Peer %v has no state", peer)) 892 } 893 switch msg.Msg.(type) { 894 case *VoteMessage: 895 if numVotes := ps.RecordVote(); numVotes%votesToContributeToBecomeGoodPeer == 0 { 896 conR.Switch.MarkPeerAsGood(peer) 897 } 898 case *BlockPartMessage: 899 if numParts := ps.RecordBlockPart(); numParts%blocksToContributeToBecomeGoodPeer == 0 { 900 conR.Switch.MarkPeerAsGood(peer) 901 } 902 } 903 case <-conR.conS.Quit(): 904 return 905 906 case <-conR.Quit(): 907 return 908 } 909 } 910 } 911 912 // String returns a string representation of the Reactor. 913 // NOTE: For now, it is just a hard-coded string to avoid accessing unprotected shared variables. 914 // TODO: improve! 915 func (conR *Reactor) String() string { 916 // better not to access shared variables 917 return "ConsensusReactor" // conR.StringIndented("") 918 } 919 920 // StringIndented returns an indented string representation of the Reactor 921 func (conR *Reactor) StringIndented(indent string) string { 922 s := "ConsensusReactor{\n" 923 s += indent + " " + conR.conS.StringIndented(indent+" ") + "\n" 924 for _, peer := range conR.Switch.Peers().List() { 925 ps, ok := peer.Get(types.PeerStateKey).(*PeerState) 926 if !ok { 927 panic(fmt.Sprintf("Peer %v has no state", peer)) 928 } 929 s += indent + " " + ps.StringIndented(indent+" ") + "\n" 930 } 931 s += indent + "}" 932 return s 933 } 934 935 // ReactorMetrics sets the metrics 936 func ReactorMetrics(metrics *Metrics) ReactorOption { 937 return func(conR *Reactor) { conR.Metrics = metrics } 938 } 939 940 //----------------------------------------------------------------------------- 941 942 var ( 943 ErrPeerStateHeightRegression = errors.New("error peer state height regression") 944 ErrPeerStateInvalidStartTime = errors.New("error peer state invalid startTime") 945 ) 946 947 // PeerState contains the known state of a peer, including its connection and 948 // threadsafe access to its PeerRoundState. 949 // NOTE: THIS GETS DUMPED WITH rpc/core/consensus.go. 950 // Be mindful of what you Expose. 951 type PeerState struct { 952 peer p2p.Peer 953 logger log.Logger 954 955 mtx sync.Mutex // NOTE: Modify below using setters, never directly. 956 PRS cstypes.PeerRoundState `json:"round_state"` // Exposed. 957 Stats *peerStateStats `json:"stats"` // Exposed. 958 } 959 960 // peerStateStats holds internal statistics for a peer. 961 type peerStateStats struct { 962 Votes int `json:"votes"` 963 BlockParts int `json:"block_parts"` 964 } 965 966 func (pss peerStateStats) String() string { 967 return fmt.Sprintf("peerStateStats{votes: %d, blockParts: %d}", 968 pss.Votes, pss.BlockParts) 969 } 970 971 // NewPeerState returns a new PeerState for the given Peer 972 func NewPeerState(peer p2p.Peer) *PeerState { 973 return &PeerState{ 974 peer: peer, 975 logger: log.NewNopLogger(), 976 PRS: cstypes.PeerRoundState{ 977 Round: -1, 978 ProposalPOLRound: -1, 979 LastCommitRound: -1, 980 CatchupCommitRound: -1, 981 }, 982 Stats: &peerStateStats{}, 983 } 984 } 985 986 // SetLogger allows to set a logger on the peer state. Returns the peer state 987 // itself. 988 func (ps *PeerState) SetLogger(logger log.Logger) *PeerState { 989 ps.logger = logger 990 return ps 991 } 992 993 // GetRoundState returns an shallow copy of the PeerRoundState. 994 // There's no point in mutating it since it won't change PeerState. 995 func (ps *PeerState) GetRoundState() *cstypes.PeerRoundState { 996 ps.mtx.Lock() 997 defer ps.mtx.Unlock() 998 999 prs := ps.PRS // copy 1000 return &prs 1001 } 1002 1003 // ToJSON returns a json of PeerState. 1004 func (ps *PeerState) ToJSON() ([]byte, error) { 1005 ps.mtx.Lock() 1006 defer ps.mtx.Unlock() 1007 1008 return tmjson.Marshal(ps) 1009 } 1010 1011 // GetHeight returns an atomic snapshot of the PeerRoundState's height 1012 // used by the mempool to ensure peers are caught up before broadcasting new txs 1013 func (ps *PeerState) GetHeight() int64 { 1014 ps.mtx.Lock() 1015 defer ps.mtx.Unlock() 1016 return ps.PRS.Height 1017 } 1018 1019 // SetHasProposal sets the given proposal as known for the peer. 1020 func (ps *PeerState) SetHasProposal(proposal *types.Proposal) { 1021 ps.mtx.Lock() 1022 defer ps.mtx.Unlock() 1023 1024 if ps.PRS.Height != proposal.Height || ps.PRS.Round != proposal.Round { 1025 return 1026 } 1027 1028 if ps.PRS.Proposal { 1029 return 1030 } 1031 1032 ps.PRS.Proposal = true 1033 1034 // ps.PRS.ProposalBlockParts is set due to NewValidBlockMessage 1035 if ps.PRS.ProposalBlockParts != nil { 1036 return 1037 } 1038 1039 ps.PRS.ProposalBlockPartSetHeader = proposal.BlockID.PartSetHeader 1040 ps.PRS.ProposalBlockParts = bits.NewBitArray(int(proposal.BlockID.PartSetHeader.Total)) 1041 ps.PRS.ProposalPOLRound = proposal.POLRound 1042 ps.PRS.ProposalPOL = nil // Nil until ProposalPOLMessage received. 1043 } 1044 1045 // InitProposalBlockParts initializes the peer's proposal block parts header and bit array. 1046 func (ps *PeerState) InitProposalBlockParts(partSetHeader types.PartSetHeader) { 1047 ps.mtx.Lock() 1048 defer ps.mtx.Unlock() 1049 1050 if ps.PRS.ProposalBlockParts != nil { 1051 return 1052 } 1053 1054 ps.PRS.ProposalBlockPartSetHeader = partSetHeader 1055 ps.PRS.ProposalBlockParts = bits.NewBitArray(int(partSetHeader.Total)) 1056 } 1057 1058 // SetHasProposalBlockPart sets the given block part index as known for the peer. 1059 func (ps *PeerState) SetHasProposalBlockPart(height int64, round int32, index int) { 1060 ps.mtx.Lock() 1061 defer ps.mtx.Unlock() 1062 1063 if ps.PRS.Height != height || ps.PRS.Round != round { 1064 return 1065 } 1066 1067 ps.PRS.ProposalBlockParts.SetIndex(index, true) 1068 } 1069 1070 // PickSendVote picks a vote and sends it to the peer. 1071 // Returns true if vote was sent. 1072 func (ps *PeerState) PickSendVote(votes types.VoteSetReader) bool { 1073 if vote, ok := ps.PickVoteToSend(votes); ok { 1074 msg := &VoteMessage{vote} 1075 ps.logger.Debug("Sending vote message", "ps", ps, "vote", vote) 1076 if ps.peer.Send(VoteChannel, MustEncode(msg)) { 1077 ps.SetHasVote(vote) 1078 return true 1079 } 1080 return false 1081 } 1082 return false 1083 } 1084 1085 // PickVoteToSend picks a vote to send to the peer. 1086 // Returns true if a vote was picked. 1087 // NOTE: `votes` must be the correct Size() for the Height(). 1088 func (ps *PeerState) PickVoteToSend(votes types.VoteSetReader) (vote *types.Vote, ok bool) { 1089 ps.mtx.Lock() 1090 defer ps.mtx.Unlock() 1091 1092 if votes.Size() == 0 { 1093 return nil, false 1094 } 1095 1096 height, round, votesType, size := 1097 votes.GetHeight(), votes.GetRound(), tmproto.SignedMsgType(votes.Type()), votes.Size() 1098 1099 // Lazily set data using 'votes'. 1100 if votes.IsCommit() { 1101 ps.ensureCatchupCommitRound(height, round, size) 1102 } 1103 ps.ensureVoteBitArrays(height, size) 1104 1105 psVotes := ps.getVoteBitArray(height, round, votesType) 1106 if psVotes == nil { 1107 return nil, false // Not something worth sending 1108 } 1109 if index, ok := votes.BitArray().Sub(psVotes).PickRandom(); ok { 1110 return votes.GetByIndex(int32(index)), true 1111 } 1112 return nil, false 1113 } 1114 1115 func (ps *PeerState) getVoteBitArray(height int64, round int32, votesType tmproto.SignedMsgType) *bits.BitArray { 1116 if !types.IsVoteTypeValid(votesType) { 1117 return nil 1118 } 1119 1120 if ps.PRS.Height == height { 1121 if ps.PRS.Round == round { 1122 switch votesType { 1123 case tmproto.PrevoteType: 1124 return ps.PRS.Prevotes 1125 case tmproto.PrecommitType: 1126 return ps.PRS.Precommits 1127 } 1128 } 1129 if ps.PRS.CatchupCommitRound == round { 1130 switch votesType { 1131 case tmproto.PrevoteType: 1132 return nil 1133 case tmproto.PrecommitType: 1134 return ps.PRS.CatchupCommit 1135 } 1136 } 1137 if ps.PRS.ProposalPOLRound == round { 1138 switch votesType { 1139 case tmproto.PrevoteType: 1140 return ps.PRS.ProposalPOL 1141 case tmproto.PrecommitType: 1142 return nil 1143 } 1144 } 1145 return nil 1146 } 1147 if ps.PRS.Height == height+1 { 1148 if ps.PRS.LastCommitRound == round { 1149 switch votesType { 1150 case tmproto.PrevoteType: 1151 return nil 1152 case tmproto.PrecommitType: 1153 return ps.PRS.LastCommit 1154 } 1155 } 1156 return nil 1157 } 1158 return nil 1159 } 1160 1161 // 'round': A round for which we have a +2/3 commit. 1162 func (ps *PeerState) ensureCatchupCommitRound(height int64, round int32, numValidators int) { 1163 if ps.PRS.Height != height { 1164 return 1165 } 1166 /* 1167 NOTE: This is wrong, 'round' could change. 1168 e.g. if orig round is not the same as block LastCommit round. 1169 if ps.CatchupCommitRound != -1 && ps.CatchupCommitRound != round { 1170 panic(fmt.Sprintf( 1171 "Conflicting CatchupCommitRound. Height: %v, 1172 Orig: %v, 1173 New: %v", 1174 height, 1175 ps.CatchupCommitRound, 1176 round)) 1177 } 1178 */ 1179 if ps.PRS.CatchupCommitRound == round { 1180 return // Nothing to do! 1181 } 1182 ps.PRS.CatchupCommitRound = round 1183 if round == ps.PRS.Round { 1184 ps.PRS.CatchupCommit = ps.PRS.Precommits 1185 } else { 1186 ps.PRS.CatchupCommit = bits.NewBitArray(numValidators) 1187 } 1188 } 1189 1190 // EnsureVoteBitArrays ensures the bit-arrays have been allocated for tracking 1191 // what votes this peer has received. 1192 // NOTE: It's important to make sure that numValidators actually matches 1193 // what the node sees as the number of validators for height. 1194 func (ps *PeerState) EnsureVoteBitArrays(height int64, numValidators int) { 1195 ps.mtx.Lock() 1196 defer ps.mtx.Unlock() 1197 ps.ensureVoteBitArrays(height, numValidators) 1198 } 1199 1200 func (ps *PeerState) ensureVoteBitArrays(height int64, numValidators int) { 1201 if ps.PRS.Height == height { 1202 if ps.PRS.Prevotes == nil { 1203 ps.PRS.Prevotes = bits.NewBitArray(numValidators) 1204 } 1205 if ps.PRS.Precommits == nil { 1206 ps.PRS.Precommits = bits.NewBitArray(numValidators) 1207 } 1208 if ps.PRS.CatchupCommit == nil { 1209 ps.PRS.CatchupCommit = bits.NewBitArray(numValidators) 1210 } 1211 if ps.PRS.ProposalPOL == nil { 1212 ps.PRS.ProposalPOL = bits.NewBitArray(numValidators) 1213 } 1214 } else if ps.PRS.Height == height+1 { 1215 if ps.PRS.LastCommit == nil { 1216 ps.PRS.LastCommit = bits.NewBitArray(numValidators) 1217 } 1218 } 1219 } 1220 1221 // RecordVote increments internal votes related statistics for this peer. 1222 // It returns the total number of added votes. 1223 func (ps *PeerState) RecordVote() int { 1224 ps.mtx.Lock() 1225 defer ps.mtx.Unlock() 1226 1227 ps.Stats.Votes++ 1228 1229 return ps.Stats.Votes 1230 } 1231 1232 // VotesSent returns the number of blocks for which peer has been sending us 1233 // votes. 1234 func (ps *PeerState) VotesSent() int { 1235 ps.mtx.Lock() 1236 defer ps.mtx.Unlock() 1237 1238 return ps.Stats.Votes 1239 } 1240 1241 // RecordBlockPart increments internal block part related statistics for this peer. 1242 // It returns the total number of added block parts. 1243 func (ps *PeerState) RecordBlockPart() int { 1244 ps.mtx.Lock() 1245 defer ps.mtx.Unlock() 1246 1247 ps.Stats.BlockParts++ 1248 return ps.Stats.BlockParts 1249 } 1250 1251 // BlockPartsSent returns the number of useful block parts the peer has sent us. 1252 func (ps *PeerState) BlockPartsSent() int { 1253 ps.mtx.Lock() 1254 defer ps.mtx.Unlock() 1255 1256 return ps.Stats.BlockParts 1257 } 1258 1259 // SetHasVote sets the given vote as known by the peer 1260 func (ps *PeerState) SetHasVote(vote *types.Vote) { 1261 ps.mtx.Lock() 1262 defer ps.mtx.Unlock() 1263 1264 ps.setHasVote(vote.Height, vote.Round, vote.Type, vote.ValidatorIndex) 1265 } 1266 1267 func (ps *PeerState) setHasVote(height int64, round int32, voteType tmproto.SignedMsgType, index int32) { 1268 logger := ps.logger.With( 1269 "peerH/R", 1270 fmt.Sprintf("%d/%d", ps.PRS.Height, ps.PRS.Round), 1271 "H/R", 1272 fmt.Sprintf("%d/%d", height, round)) 1273 logger.Debug("setHasVote", "type", voteType, "index", index) 1274 1275 // NOTE: some may be nil BitArrays -> no side effects. 1276 psVotes := ps.getVoteBitArray(height, round, voteType) 1277 if psVotes != nil { 1278 psVotes.SetIndex(int(index), true) 1279 } 1280 } 1281 1282 // ApplyNewRoundStepMessage updates the peer state for the new round. 1283 func (ps *PeerState) ApplyNewRoundStepMessage(msg *NewRoundStepMessage) { 1284 ps.mtx.Lock() 1285 defer ps.mtx.Unlock() 1286 1287 // Ignore duplicates or decreases 1288 if CompareHRS(msg.Height, msg.Round, msg.Step, ps.PRS.Height, ps.PRS.Round, ps.PRS.Step) <= 0 { 1289 return 1290 } 1291 1292 // Just remember these values. 1293 psHeight := ps.PRS.Height 1294 psRound := ps.PRS.Round 1295 psCatchupCommitRound := ps.PRS.CatchupCommitRound 1296 psCatchupCommit := ps.PRS.CatchupCommit 1297 1298 startTime := tmtime.Now().Add(-1 * time.Duration(msg.SecondsSinceStartTime) * time.Second) 1299 ps.PRS.Height = msg.Height 1300 ps.PRS.Round = msg.Round 1301 ps.PRS.Step = msg.Step 1302 ps.PRS.StartTime = startTime 1303 if psHeight != msg.Height || psRound != msg.Round { 1304 ps.PRS.Proposal = false 1305 ps.PRS.ProposalBlockPartSetHeader = types.PartSetHeader{} 1306 ps.PRS.ProposalBlockParts = nil 1307 ps.PRS.ProposalPOLRound = -1 1308 ps.PRS.ProposalPOL = nil 1309 // We'll update the BitArray capacity later. 1310 ps.PRS.Prevotes = nil 1311 ps.PRS.Precommits = nil 1312 } 1313 if psHeight == msg.Height && psRound != msg.Round && msg.Round == psCatchupCommitRound { 1314 // Peer caught up to CatchupCommitRound. 1315 // Preserve psCatchupCommit! 1316 // NOTE: We prefer to use prs.Precommits if 1317 // pr.Round matches pr.CatchupCommitRound. 1318 ps.PRS.Precommits = psCatchupCommit 1319 } 1320 if psHeight != msg.Height { 1321 // Shift Precommits to LastCommit. 1322 if psHeight+1 == msg.Height && psRound == msg.LastCommitRound { 1323 ps.PRS.LastCommitRound = msg.LastCommitRound 1324 ps.PRS.LastCommit = ps.PRS.Precommits 1325 } else { 1326 ps.PRS.LastCommitRound = msg.LastCommitRound 1327 ps.PRS.LastCommit = nil 1328 } 1329 // We'll update the BitArray capacity later. 1330 ps.PRS.CatchupCommitRound = -1 1331 ps.PRS.CatchupCommit = nil 1332 } 1333 } 1334 1335 // ApplyNewValidBlockMessage updates the peer state for the new valid block. 1336 func (ps *PeerState) ApplyNewValidBlockMessage(msg *NewValidBlockMessage) { 1337 ps.mtx.Lock() 1338 defer ps.mtx.Unlock() 1339 1340 if ps.PRS.Height != msg.Height { 1341 return 1342 } 1343 1344 if ps.PRS.Round != msg.Round && !msg.IsCommit { 1345 return 1346 } 1347 1348 ps.PRS.ProposalBlockPartSetHeader = msg.BlockPartSetHeader 1349 ps.PRS.ProposalBlockParts = msg.BlockParts 1350 } 1351 1352 // ApplyProposalPOLMessage updates the peer state for the new proposal POL. 1353 func (ps *PeerState) ApplyProposalPOLMessage(msg *ProposalPOLMessage) { 1354 ps.mtx.Lock() 1355 defer ps.mtx.Unlock() 1356 1357 if ps.PRS.Height != msg.Height { 1358 return 1359 } 1360 if ps.PRS.ProposalPOLRound != msg.ProposalPOLRound { 1361 return 1362 } 1363 1364 // TODO: Merge onto existing ps.PRS.ProposalPOL? 1365 // We might have sent some prevotes in the meantime. 1366 ps.PRS.ProposalPOL = msg.ProposalPOL 1367 } 1368 1369 // ApplyHasVoteMessage updates the peer state for the new vote. 1370 func (ps *PeerState) ApplyHasVoteMessage(msg *HasVoteMessage) { 1371 ps.mtx.Lock() 1372 defer ps.mtx.Unlock() 1373 1374 if ps.PRS.Height != msg.Height { 1375 return 1376 } 1377 1378 ps.setHasVote(msg.Height, msg.Round, msg.Type, msg.Index) 1379 } 1380 1381 // ApplyVoteSetBitsMessage updates the peer state for the bit-array of votes 1382 // it claims to have for the corresponding BlockID. 1383 // `ourVotes` is a BitArray of votes we have for msg.BlockID 1384 // NOTE: if ourVotes is nil (e.g. msg.Height < rs.Height), 1385 // we conservatively overwrite ps's votes w/ msg.Votes. 1386 func (ps *PeerState) ApplyVoteSetBitsMessage(msg *VoteSetBitsMessage, ourVotes *bits.BitArray) { 1387 ps.mtx.Lock() 1388 defer ps.mtx.Unlock() 1389 1390 votes := ps.getVoteBitArray(msg.Height, msg.Round, msg.Type) 1391 if votes != nil { 1392 if ourVotes == nil { 1393 votes.Update(msg.Votes) 1394 } else { 1395 otherVotes := votes.Sub(ourVotes) 1396 hasVotes := otherVotes.Or(msg.Votes) 1397 votes.Update(hasVotes) 1398 } 1399 } 1400 } 1401 1402 // String returns a string representation of the PeerState 1403 func (ps *PeerState) String() string { 1404 return ps.StringIndented("") 1405 } 1406 1407 // StringIndented returns a string representation of the PeerState 1408 func (ps *PeerState) StringIndented(indent string) string { 1409 ps.mtx.Lock() 1410 defer ps.mtx.Unlock() 1411 return fmt.Sprintf(`PeerState{ 1412 %s Key %v 1413 %s RoundState %v 1414 %s Stats %v 1415 %s}`, 1416 indent, ps.peer.ID(), 1417 indent, ps.PRS.StringIndented(indent+" "), 1418 indent, ps.Stats, 1419 indent) 1420 } 1421 1422 //----------------------------------------------------------------------------- 1423 // Messages 1424 1425 // Message is a message that can be sent and received on the Reactor 1426 type Message interface { 1427 ValidateBasic() error 1428 } 1429 1430 func init() { 1431 tmjson.RegisterType(&NewRoundStepMessage{}, "tendermint/NewRoundStepMessage") 1432 tmjson.RegisterType(&NewValidBlockMessage{}, "tendermint/NewValidBlockMessage") 1433 tmjson.RegisterType(&ProposalMessage{}, "tendermint/Proposal") 1434 tmjson.RegisterType(&ProposalPOLMessage{}, "tendermint/ProposalPOL") 1435 tmjson.RegisterType(&BlockPartMessage{}, "tendermint/BlockPart") 1436 tmjson.RegisterType(&VoteMessage{}, "tendermint/Vote") 1437 tmjson.RegisterType(&HasVoteMessage{}, "tendermint/HasVote") 1438 tmjson.RegisterType(&VoteSetMaj23Message{}, "tendermint/VoteSetMaj23") 1439 tmjson.RegisterType(&VoteSetBitsMessage{}, "tendermint/VoteSetBits") 1440 } 1441 1442 func decodeMsg(bz []byte) (msg Message, err error) { 1443 pb := &tmcons.Message{} 1444 if err = proto.Unmarshal(bz, pb); err != nil { 1445 return msg, err 1446 } 1447 1448 return MsgFromProto(pb) 1449 } 1450 1451 //------------------------------------- 1452 1453 // NewRoundStepMessage is sent for every step taken in the ConsensusState. 1454 // For every height/round/step transition 1455 type NewRoundStepMessage struct { 1456 Height int64 1457 Round int32 1458 Step cstypes.RoundStepType 1459 SecondsSinceStartTime int64 1460 LastCommitRound int32 1461 } 1462 1463 // ValidateBasic performs basic validation. 1464 func (m *NewRoundStepMessage) ValidateBasic() error { 1465 if m.Height < 0 { 1466 return errors.New("negative Height") 1467 } 1468 if m.Round < 0 { 1469 return errors.New("negative Round") 1470 } 1471 if !m.Step.IsValid() { 1472 return errors.New("invalid Step") 1473 } 1474 1475 // NOTE: SecondsSinceStartTime may be negative 1476 1477 // LastCommitRound will be -1 for the initial height, but we don't know what height this is 1478 // since it can be specified in genesis. The reactor will have to validate this via 1479 // ValidateHeight(). 1480 if m.LastCommitRound < -1 { 1481 return errors.New("invalid LastCommitRound (cannot be < -1)") 1482 } 1483 1484 return nil 1485 } 1486 1487 // ValidateHeight validates the height given the chain's initial height. 1488 func (m *NewRoundStepMessage) ValidateHeight(initialHeight int64) error { 1489 if m.Height < initialHeight { 1490 return fmt.Errorf("invalid Height %v (lower than initial height %v)", 1491 m.Height, initialHeight) 1492 } 1493 if m.Height == initialHeight && m.LastCommitRound != -1 { 1494 return fmt.Errorf("invalid LastCommitRound %v (must be -1 for initial height %v)", 1495 m.LastCommitRound, initialHeight) 1496 } 1497 if m.Height > initialHeight && m.LastCommitRound < 0 { 1498 return fmt.Errorf("LastCommitRound can only be negative for initial height %v", 1499 initialHeight) 1500 } 1501 return nil 1502 } 1503 1504 // String returns a string representation. 1505 func (m *NewRoundStepMessage) String() string { 1506 return fmt.Sprintf("[NewRoundStep H:%v R:%v S:%v LCR:%v]", 1507 m.Height, m.Round, m.Step, m.LastCommitRound) 1508 } 1509 1510 //------------------------------------- 1511 1512 // NewValidBlockMessage is sent when a validator observes a valid block B in some round r, 1513 // i.e., there is a Proposal for block B and 2/3+ prevotes for the block B in the round r. 1514 // In case the block is also committed, then IsCommit flag is set to true. 1515 type NewValidBlockMessage struct { 1516 Height int64 1517 Round int32 1518 BlockPartSetHeader types.PartSetHeader 1519 BlockParts *bits.BitArray 1520 IsCommit bool 1521 } 1522 1523 // ValidateBasic performs basic validation. 1524 func (m *NewValidBlockMessage) ValidateBasic() error { 1525 if m.Height < 0 { 1526 return errors.New("negative Height") 1527 } 1528 if m.Round < 0 { 1529 return errors.New("negative Round") 1530 } 1531 if err := m.BlockPartSetHeader.ValidateBasic(); err != nil { 1532 return fmt.Errorf("wrong BlockPartSetHeader: %v", err) 1533 } 1534 if m.BlockParts.Size() == 0 { 1535 return errors.New("empty blockParts") 1536 } 1537 if m.BlockParts.Size() != int(m.BlockPartSetHeader.Total) { 1538 return fmt.Errorf("blockParts bit array size %d not equal to BlockPartSetHeader.Total %d", 1539 m.BlockParts.Size(), 1540 m.BlockPartSetHeader.Total) 1541 } 1542 if m.BlockParts.Size() > int(types.MaxBlockPartsCount) { 1543 return fmt.Errorf("blockParts bit array is too big: %d, max: %d", m.BlockParts.Size(), types.MaxBlockPartsCount) 1544 } 1545 return nil 1546 } 1547 1548 // String returns a string representation. 1549 func (m *NewValidBlockMessage) String() string { 1550 return fmt.Sprintf("[ValidBlockMessage H:%v R:%v BP:%v BA:%v IsCommit:%v]", 1551 m.Height, m.Round, m.BlockPartSetHeader, m.BlockParts, m.IsCommit) 1552 } 1553 1554 //------------------------------------- 1555 1556 // ProposalMessage is sent when a new block is proposed. 1557 type ProposalMessage struct { 1558 Proposal *types.Proposal 1559 } 1560 1561 // ValidateBasic performs basic validation. 1562 func (m *ProposalMessage) ValidateBasic() error { 1563 return m.Proposal.ValidateBasic() 1564 } 1565 1566 // String returns a string representation. 1567 func (m *ProposalMessage) String() string { 1568 return fmt.Sprintf("[Proposal %v]", m.Proposal) 1569 } 1570 1571 //------------------------------------- 1572 1573 // ProposalPOLMessage is sent when a previous proposal is re-proposed. 1574 type ProposalPOLMessage struct { 1575 Height int64 1576 ProposalPOLRound int32 1577 ProposalPOL *bits.BitArray 1578 } 1579 1580 // ValidateBasic performs basic validation. 1581 func (m *ProposalPOLMessage) ValidateBasic() error { 1582 if m.Height < 0 { 1583 return errors.New("negative Height") 1584 } 1585 if m.ProposalPOLRound < 0 { 1586 return errors.New("negative ProposalPOLRound") 1587 } 1588 if m.ProposalPOL.Size() == 0 { 1589 return errors.New("empty ProposalPOL bit array") 1590 } 1591 if m.ProposalPOL.Size() > types.MaxVotesCount { 1592 return fmt.Errorf("proposalPOL bit array is too big: %d, max: %d", m.ProposalPOL.Size(), types.MaxVotesCount) 1593 } 1594 return nil 1595 } 1596 1597 // String returns a string representation. 1598 func (m *ProposalPOLMessage) String() string { 1599 return fmt.Sprintf("[ProposalPOL H:%v POLR:%v POL:%v]", m.Height, m.ProposalPOLRound, m.ProposalPOL) 1600 } 1601 1602 //------------------------------------- 1603 1604 // BlockPartMessage is sent when gossipping a piece of the proposed block. 1605 type BlockPartMessage struct { 1606 Height int64 1607 Round int32 1608 Part *types.Part 1609 } 1610 1611 // ValidateBasic performs basic validation. 1612 func (m *BlockPartMessage) ValidateBasic() error { 1613 if m.Height < 0 { 1614 return errors.New("negative Height") 1615 } 1616 if m.Round < 0 { 1617 return errors.New("negative Round") 1618 } 1619 if err := m.Part.ValidateBasic(); err != nil { 1620 return fmt.Errorf("wrong Part: %v", err) 1621 } 1622 return nil 1623 } 1624 1625 // String returns a string representation. 1626 func (m *BlockPartMessage) String() string { 1627 return fmt.Sprintf("[BlockPart H:%v R:%v P:%v]", m.Height, m.Round, m.Part) 1628 } 1629 1630 //------------------------------------- 1631 1632 // VoteMessage is sent when voting for a proposal (or lack thereof). 1633 type VoteMessage struct { 1634 Vote *types.Vote 1635 } 1636 1637 // ValidateBasic performs basic validation. 1638 func (m *VoteMessage) ValidateBasic() error { 1639 return m.Vote.ValidateBasic() 1640 } 1641 1642 // String returns a string representation. 1643 func (m *VoteMessage) String() string { 1644 return fmt.Sprintf("[Vote %v]", m.Vote) 1645 } 1646 1647 //------------------------------------- 1648 1649 // HasVoteMessage is sent to indicate that a particular vote has been received. 1650 type HasVoteMessage struct { 1651 Height int64 1652 Round int32 1653 Type tmproto.SignedMsgType 1654 Index int32 1655 } 1656 1657 // ValidateBasic performs basic validation. 1658 func (m *HasVoteMessage) ValidateBasic() error { 1659 if m.Height < 0 { 1660 return errors.New("negative Height") 1661 } 1662 if m.Round < 0 { 1663 return errors.New("negative Round") 1664 } 1665 if !types.IsVoteTypeValid(m.Type) { 1666 return errors.New("invalid Type") 1667 } 1668 if m.Index < 0 { 1669 return errors.New("negative Index") 1670 } 1671 return nil 1672 } 1673 1674 // String returns a string representation. 1675 func (m *HasVoteMessage) String() string { 1676 return fmt.Sprintf("[HasVote VI:%v V:{%v/%02d/%v}]", m.Index, m.Height, m.Round, m.Type) 1677 } 1678 1679 //------------------------------------- 1680 1681 // VoteSetMaj23Message is sent to indicate that a given BlockID has seen +2/3 votes. 1682 type VoteSetMaj23Message struct { 1683 Height int64 1684 Round int32 1685 Type tmproto.SignedMsgType 1686 BlockID types.BlockID 1687 } 1688 1689 // ValidateBasic performs basic validation. 1690 func (m *VoteSetMaj23Message) ValidateBasic() error { 1691 if m.Height < 0 { 1692 return errors.New("negative Height") 1693 } 1694 if m.Round < 0 { 1695 return errors.New("negative Round") 1696 } 1697 if !types.IsVoteTypeValid(m.Type) { 1698 return errors.New("invalid Type") 1699 } 1700 if err := m.BlockID.ValidateBasic(); err != nil { 1701 return fmt.Errorf("wrong BlockID: %v", err) 1702 } 1703 return nil 1704 } 1705 1706 // String returns a string representation. 1707 func (m *VoteSetMaj23Message) String() string { 1708 return fmt.Sprintf("[VSM23 %v/%02d/%v %v]", m.Height, m.Round, m.Type, m.BlockID) 1709 } 1710 1711 //------------------------------------- 1712 1713 // VoteSetBitsMessage is sent to communicate the bit-array of votes seen for the BlockID. 1714 type VoteSetBitsMessage struct { 1715 Height int64 1716 Round int32 1717 Type tmproto.SignedMsgType 1718 BlockID types.BlockID 1719 Votes *bits.BitArray 1720 } 1721 1722 // ValidateBasic performs basic validation. 1723 func (m *VoteSetBitsMessage) ValidateBasic() error { 1724 if m.Height < 0 { 1725 return errors.New("negative Height") 1726 } 1727 if !types.IsVoteTypeValid(m.Type) { 1728 return errors.New("invalid Type") 1729 } 1730 if err := m.BlockID.ValidateBasic(); err != nil { 1731 return fmt.Errorf("wrong BlockID: %v", err) 1732 } 1733 // NOTE: Votes.Size() can be zero if the node does not have any 1734 if m.Votes.Size() > types.MaxVotesCount { 1735 return fmt.Errorf("votes bit array is too big: %d, max: %d", m.Votes.Size(), types.MaxVotesCount) 1736 } 1737 return nil 1738 } 1739 1740 // String returns a string representation. 1741 func (m *VoteSetBitsMessage) String() string { 1742 return fmt.Sprintf("[VSB %v/%02d/%v %v %v]", m.Height, m.Round, m.Type, m.BlockID, m.Votes) 1743 } 1744 1745 //-------------------------------------