github.com/KYVENetwork/cometbft/v38@v38.0.3/evidence/reactor.go (about) 1 package evidence 2 3 import ( 4 "fmt" 5 "time" 6 7 "github.com/cosmos/gogoproto/proto" 8 9 clist "github.com/KYVENetwork/cometbft/v38/libs/clist" 10 "github.com/KYVENetwork/cometbft/v38/libs/log" 11 "github.com/KYVENetwork/cometbft/v38/p2p" 12 cmtproto "github.com/KYVENetwork/cometbft/v38/proto/cometbft/v38/types" 13 "github.com/KYVENetwork/cometbft/v38/types" 14 ) 15 16 const ( 17 EvidenceChannel = byte(0x38) 18 19 maxMsgSize = 1048576 // 1MB TODO make it configurable 20 21 // broadcast all uncommitted evidence this often. This sets when the reactor 22 // goes back to the start of the list and begins sending the evidence again. 23 // Most evidence should be committed in the very next block that is why we wait 24 // just over the block production rate before sending evidence again. 25 broadcastEvidenceIntervalS = 10 26 // If a message fails wait this much before sending it again 27 peerRetryMessageIntervalMS = 100 28 ) 29 30 // Reactor handles evpool evidence broadcasting amongst peers. 31 type Reactor struct { 32 p2p.BaseReactor 33 evpool *Pool 34 eventBus *types.EventBus 35 } 36 37 // NewReactor returns a new Reactor with the given config and evpool. 38 func NewReactor(evpool *Pool) *Reactor { 39 evR := &Reactor{ 40 evpool: evpool, 41 } 42 evR.BaseReactor = *p2p.NewBaseReactor("Evidence", evR) 43 return evR 44 } 45 46 // SetLogger sets the Logger on the reactor and the underlying Evidence. 47 func (evR *Reactor) SetLogger(l log.Logger) { 48 evR.Logger = l 49 evR.evpool.SetLogger(l) 50 } 51 52 // GetChannels implements Reactor. 53 // It returns the list of channels for this reactor. 54 func (evR *Reactor) GetChannels() []*p2p.ChannelDescriptor { 55 return []*p2p.ChannelDescriptor{ 56 { 57 ID: EvidenceChannel, 58 Priority: 6, 59 RecvMessageCapacity: maxMsgSize, 60 MessageType: &cmtproto.EvidenceList{}, 61 }, 62 } 63 } 64 65 // AddPeer implements Reactor. 66 func (evR *Reactor) AddPeer(peer p2p.Peer) { 67 go evR.broadcastEvidenceRoutine(peer) 68 } 69 70 // Receive implements Reactor. 71 // It adds any received evidence to the evpool. 72 func (evR *Reactor) Receive(e p2p.Envelope) { 73 evis, err := evidenceListFromProto(e.Message) 74 if err != nil { 75 evR.Logger.Error("Error decoding message", "src", e.Src, "chId", e.ChannelID, "err", err) 76 evR.Switch.StopPeerForError(e.Src, err) 77 return 78 } 79 80 for _, ev := range evis { 81 err := evR.evpool.AddEvidence(ev) 82 switch err.(type) { 83 case *types.ErrInvalidEvidence: 84 evR.Logger.Error(err.Error()) 85 // punish peer 86 evR.Switch.StopPeerForError(e.Src, err) 87 return 88 case nil: 89 default: 90 // continue to the next piece of evidence 91 evR.Logger.Error("Evidence has not been added", "evidence", evis, "err", err) 92 } 93 } 94 } 95 96 // SetEventBus implements events.Eventable. 97 func (evR *Reactor) SetEventBus(b *types.EventBus) { 98 evR.eventBus = b 99 } 100 101 // Modeled after the mempool routine. 102 // - Evidence accumulates in a clist. 103 // - Each peer has a routine that iterates through the clist, 104 // sending available evidence to the peer. 105 // - If we're waiting for new evidence and the list is not empty, 106 // start iterating from the beginning again. 107 func (evR *Reactor) broadcastEvidenceRoutine(peer p2p.Peer) { 108 var next *clist.CElement 109 for { 110 // This happens because the CElement we were looking at got garbage 111 // collected (removed). That is, .NextWait() returned nil. Go ahead and 112 // start from the beginning. 113 if next == nil { 114 select { 115 case <-evR.evpool.EvidenceWaitChan(): // Wait until evidence is available 116 if next = evR.evpool.EvidenceFront(); next == nil { 117 continue 118 } 119 case <-peer.Quit(): 120 return 121 case <-evR.Quit(): 122 return 123 } 124 } else if !peer.IsRunning() || !evR.IsRunning() { 125 return 126 } 127 128 ev := next.Value.(types.Evidence) 129 evis := evR.prepareEvidenceMessage(peer, ev) 130 if len(evis) > 0 { 131 evR.Logger.Debug("Gossiping evidence to peer", "ev", ev, "peer", peer) 132 evp, err := evidenceListToProto(evis) 133 if err != nil { 134 panic(err) 135 } 136 137 success := peer.Send(p2p.Envelope{ 138 ChannelID: EvidenceChannel, 139 Message: evp, 140 }) 141 if !success { 142 time.Sleep(peerRetryMessageIntervalMS * time.Millisecond) 143 continue 144 } 145 } 146 147 afterCh := time.After(time.Second * broadcastEvidenceIntervalS) 148 select { 149 case <-afterCh: 150 // start from the beginning every tick. 151 // TODO: only do this if we're at the end of the list! 152 next = nil 153 case <-next.NextWaitChan(): 154 // see the start of the for loop for nil check 155 next = next.Next() 156 case <-peer.Quit(): 157 return 158 case <-evR.Quit(): 159 return 160 } 161 } 162 } 163 164 // Returns the message to send to the peer, or nil if the evidence is invalid for the peer. 165 // If message is nil, we should sleep and try again. 166 func (evR Reactor) prepareEvidenceMessage( 167 peer p2p.Peer, 168 ev types.Evidence, 169 ) (evis []types.Evidence) { 170 171 // make sure the peer is up to date 172 evHeight := ev.Height() 173 peerState, ok := peer.Get(types.PeerStateKey).(PeerState) 174 if !ok { 175 // Peer does not have a state yet. We set it in the consensus reactor, but 176 // when we add peer in Switch, the order we call reactors#AddPeer is 177 // different every time due to us using a map. Sometimes other reactors 178 // will be initialized before the consensus reactor. We should wait a few 179 // milliseconds and retry. 180 return nil 181 } 182 183 // NOTE: We only send evidence to peers where 184 // peerHeight - maxAge < evidenceHeight < peerHeight 185 var ( 186 peerHeight = peerState.GetHeight() 187 params = evR.evpool.State().ConsensusParams.Evidence 188 ageNumBlocks = peerHeight - evHeight 189 ) 190 191 if peerHeight <= evHeight { // peer is behind. sleep while he catches up 192 return nil 193 } else if ageNumBlocks > params.MaxAgeNumBlocks { // evidence is too old relative to the peer, skip 194 195 // NOTE: if evidence is too old for an honest peer, then we're behind and 196 // either it already got committed or it never will! 197 evR.Logger.Info("Not sending peer old evidence", 198 "peerHeight", peerHeight, 199 "evHeight", evHeight, 200 "maxAgeNumBlocks", params.MaxAgeNumBlocks, 201 "lastBlockTime", evR.evpool.State().LastBlockTime, 202 "maxAgeDuration", params.MaxAgeDuration, 203 "peer", peer, 204 ) 205 206 return nil 207 } 208 209 // send evidence 210 return []types.Evidence{ev} 211 } 212 213 // PeerState describes the state of a peer. 214 type PeerState interface { 215 GetHeight() int64 216 } 217 218 // encodemsg takes a array of evidence 219 // returns the byte encoding of the List Message 220 func evidenceListToProto(evis []types.Evidence) (*cmtproto.EvidenceList, error) { 221 evi := make([]cmtproto.Evidence, len(evis)) 222 for i := 0; i < len(evis); i++ { 223 ev, err := types.EvidenceToProto(evis[i]) 224 if err != nil { 225 return nil, err 226 } 227 evi[i] = *ev 228 } 229 epl := cmtproto.EvidenceList{ 230 Evidence: evi, 231 } 232 return &epl, nil 233 } 234 235 func evidenceListFromProto(m proto.Message) ([]types.Evidence, error) { 236 lm := m.(*cmtproto.EvidenceList) 237 238 evis := make([]types.Evidence, len(lm.Evidence)) 239 for i := 0; i < len(lm.Evidence); i++ { 240 ev, err := types.EvidenceFromProto(&lm.Evidence[i]) 241 if err != nil { 242 return nil, err 243 } 244 evis[i] = ev 245 } 246 247 for i, ev := range evis { 248 if err := ev.ValidateBasic(); err != nil { 249 return nil, fmt.Errorf("invalid evidence (#%d): %v", i, err) 250 } 251 } 252 253 return evis, nil 254 }