github.com/badrootd/nibiru-cometbft@v0.37.5-0.20240307173500-2a75559eee9b/mempool/v0/reactor.go (about) 1 package v0 2 3 import ( 4 "context" 5 "errors" 6 "fmt" 7 "time" 8 9 cfg "github.com/badrootd/nibiru-cometbft/config" 10 "github.com/badrootd/nibiru-cometbft/libs/clist" 11 "github.com/badrootd/nibiru-cometbft/libs/log" 12 cmtsync "github.com/badrootd/nibiru-cometbft/libs/sync" 13 "github.com/badrootd/nibiru-cometbft/mempool" 14 "github.com/badrootd/nibiru-cometbft/p2p" 15 protomem "github.com/badrootd/nibiru-cometbft/proto/tendermint/mempool" 16 "github.com/badrootd/nibiru-cometbft/types" 17 "golang.org/x/sync/semaphore" 18 ) 19 20 // Reactor handles mempool tx broadcasting amongst peers. 21 // It maintains a map from peer ID to counter, to prevent gossiping txs to the 22 // peers you received it from. 23 type Reactor struct { 24 p2p.BaseReactor 25 config *cfg.MempoolConfig 26 mempool *CListMempool 27 ids *mempoolIDs 28 29 // Semaphores to keep track of how many connections to peers are active for broadcasting 30 // transactions. Each semaphore has a capacity that puts an upper bound on the number of 31 // connections for different groups of peers. 32 activePersistentPeersSemaphore *semaphore.Weighted 33 activeNonPersistentPeersSemaphore *semaphore.Weighted 34 } 35 36 type mempoolIDs struct { 37 mtx cmtsync.RWMutex 38 peerMap map[p2p.ID]uint16 39 nextID uint16 // assumes that a node will never have over 65536 active peers 40 activeIDs map[uint16]struct{} // used to check if a given peerID key is used, the value doesn't matter 41 } 42 43 // Reserve searches for the next unused ID and assigns it to the 44 // peer. 45 func (ids *mempoolIDs) ReserveForPeer(peer p2p.Peer) { 46 ids.mtx.Lock() 47 defer ids.mtx.Unlock() 48 49 curID := ids.nextPeerID() 50 ids.peerMap[peer.ID()] = curID 51 ids.activeIDs[curID] = struct{}{} 52 } 53 54 // nextPeerID returns the next unused peer ID to use. 55 // This assumes that ids's mutex is already locked. 56 func (ids *mempoolIDs) nextPeerID() uint16 { 57 if len(ids.activeIDs) == mempool.MaxActiveIDs { 58 panic(fmt.Sprintf("node has maximum %d active IDs and wanted to get one more", mempool.MaxActiveIDs)) 59 } 60 61 _, idExists := ids.activeIDs[ids.nextID] 62 for idExists { 63 ids.nextID++ 64 _, idExists = ids.activeIDs[ids.nextID] 65 } 66 curID := ids.nextID 67 ids.nextID++ 68 return curID 69 } 70 71 // Reclaim returns the ID reserved for the peer back to unused pool. 72 func (ids *mempoolIDs) Reclaim(peer p2p.Peer) { 73 ids.mtx.Lock() 74 defer ids.mtx.Unlock() 75 76 removedID, ok := ids.peerMap[peer.ID()] 77 if ok { 78 delete(ids.activeIDs, removedID) 79 delete(ids.peerMap, peer.ID()) 80 } 81 } 82 83 // GetForPeer returns an ID reserved for the peer. 84 func (ids *mempoolIDs) GetForPeer(peer p2p.Peer) uint16 { 85 ids.mtx.RLock() 86 defer ids.mtx.RUnlock() 87 88 return ids.peerMap[peer.ID()] 89 } 90 91 func newMempoolIDs() *mempoolIDs { 92 return &mempoolIDs{ 93 peerMap: make(map[p2p.ID]uint16), 94 activeIDs: map[uint16]struct{}{0: {}}, 95 nextID: 1, // reserve unknownPeerID(0) for mempoolReactor.BroadcastTx 96 } 97 } 98 99 // NewReactor returns a new Reactor with the given config and mempool. 100 func NewReactor(config *cfg.MempoolConfig, mempool *CListMempool) *Reactor { 101 memR := &Reactor{ 102 config: config, 103 mempool: mempool, 104 ids: newMempoolIDs(), 105 } 106 memR.BaseReactor = *p2p.NewBaseReactor("Mempool", memR) 107 memR.activePersistentPeersSemaphore = semaphore.NewWeighted(int64(memR.config.ExperimentalMaxGossipConnectionsToPersistentPeers)) 108 memR.activeNonPersistentPeersSemaphore = semaphore.NewWeighted(int64(memR.config.ExperimentalMaxGossipConnectionsToNonPersistentPeers)) 109 110 return memR 111 } 112 113 // InitPeer implements Reactor by creating a state for the peer. 114 func (memR *Reactor) InitPeer(peer p2p.Peer) p2p.Peer { 115 memR.ids.ReserveForPeer(peer) 116 return peer 117 } 118 119 // SetLogger sets the Logger on the reactor and the underlying mempool. 120 func (memR *Reactor) SetLogger(l log.Logger) { 121 memR.Logger = l 122 memR.mempool.SetLogger(l) 123 } 124 125 // OnStart implements p2p.BaseReactor. 126 func (memR *Reactor) OnStart() error { 127 if !memR.config.Broadcast { 128 memR.Logger.Info("Tx broadcasting is disabled") 129 } 130 return nil 131 } 132 133 // GetChannels implements Reactor by returning the list of channels for this 134 // reactor. 135 func (memR *Reactor) GetChannels() []*p2p.ChannelDescriptor { 136 largestTx := make([]byte, memR.config.MaxTxBytes) 137 batchMsg := protomem.Message{ 138 Sum: &protomem.Message_Txs{ 139 Txs: &protomem.Txs{Txs: [][]byte{largestTx}}, 140 }, 141 } 142 143 return []*p2p.ChannelDescriptor{ 144 { 145 ID: mempool.MempoolChannel, 146 Priority: 5, 147 RecvMessageCapacity: batchMsg.Size(), 148 MessageType: &protomem.Message{}, 149 }, 150 } 151 } 152 153 // AddPeer implements Reactor. 154 // It starts a broadcast routine ensuring all txs are forwarded to the given peer. 155 func (memR *Reactor) AddPeer(peer p2p.Peer) { 156 if memR.config.Broadcast { 157 go func() { 158 // Always forward transactions to unconditional peers. 159 if !memR.Switch.IsPeerUnconditional(peer.ID()) { 160 // Depending on the type of peer, we choose a semaphore to limit the gossiping peers. 161 var peerSemaphore *semaphore.Weighted 162 if peer.IsPersistent() && memR.config.ExperimentalMaxGossipConnectionsToPersistentPeers > 0 { 163 peerSemaphore = memR.activePersistentPeersSemaphore 164 } else if !peer.IsPersistent() && memR.config.ExperimentalMaxGossipConnectionsToNonPersistentPeers > 0 { 165 peerSemaphore = memR.activeNonPersistentPeersSemaphore 166 } 167 168 if peerSemaphore != nil { 169 for peer.IsRunning() { 170 // Block on the semaphore until a slot is available to start gossiping with this peer. 171 // Do not block indefinitely, in case the peer is disconnected before gossiping starts. 172 ctxTimeout, cancel := context.WithTimeout(context.TODO(), 30*time.Second) 173 // Block sending transactions to peer until one of the connections become 174 // available in the semaphore. 175 err := peerSemaphore.Acquire(ctxTimeout, 1) 176 cancel() 177 178 if err != nil { 179 continue 180 } 181 182 // Release semaphore to allow other peer to start sending transactions. 183 defer peerSemaphore.Release(1) 184 break 185 } 186 } 187 } 188 189 memR.mempool.metrics.ActiveOutboundConnections.Add(1) 190 defer memR.mempool.metrics.ActiveOutboundConnections.Add(-1) 191 memR.broadcastTxRoutine(peer) 192 }() 193 } 194 } 195 196 // RemovePeer implements Reactor. 197 func (memR *Reactor) RemovePeer(peer p2p.Peer, reason interface{}) { 198 memR.ids.Reclaim(peer) 199 // broadcast routine checks if peer is gone and returns 200 } 201 202 // Receive implements Reactor. 203 // It adds any received transactions to the mempool. 204 func (memR *Reactor) ReceiveEnvelope(e p2p.Envelope) { 205 memR.Logger.Debug("Receive", "src", e.Src, "chId", e.ChannelID, "msg", e.Message) 206 switch msg := e.Message.(type) { 207 case *protomem.Txs: 208 protoTxs := msg.GetTxs() 209 if len(protoTxs) == 0 { 210 memR.Logger.Error("received empty txs from peer", "src", e.Src) 211 return 212 } 213 txInfo := mempool.TxInfo{SenderID: memR.ids.GetForPeer(e.Src)} 214 if e.Src != nil { 215 txInfo.SenderP2PID = e.Src.ID() 216 } 217 218 var err error 219 for _, tx := range protoTxs { 220 ntx := types.Tx(tx) 221 err = memR.mempool.CheckTx(ntx, nil, txInfo) 222 if errors.Is(err, mempool.ErrTxInCache) { 223 memR.Logger.Debug("Tx already exists in cache", "tx", ntx.String()) 224 } else if err != nil { 225 memR.Logger.Info("Could not check tx", "tx", ntx.String(), "err", err) 226 } 227 } 228 default: 229 memR.Logger.Error("unknown message type", "src", e.Src, "chId", e.ChannelID, "msg", e.Message) 230 memR.Switch.StopPeerForError(e.Src, fmt.Errorf("mempool cannot handle message of type: %T", e.Message)) 231 return 232 } 233 234 // broadcasting happens from go routines per peer 235 } 236 237 // PeerState describes the state of a peer. 238 type PeerState interface { 239 GetHeight() int64 240 } 241 242 // Send new mempool txs to peer. 243 func (memR *Reactor) broadcastTxRoutine(peer p2p.Peer) { 244 peerID := memR.ids.GetForPeer(peer) 245 var next *clist.CElement 246 247 for { 248 // In case of both next.NextWaitChan() and peer.Quit() are variable at the same time 249 if !memR.IsRunning() || !peer.IsRunning() { 250 return 251 } 252 253 // This happens because the CElement we were looking at got garbage 254 // collected (removed). That is, .NextWait() returned nil. Go ahead and 255 // start from the beginning. 256 if next == nil { 257 select { 258 case <-memR.mempool.TxsWaitChan(): // Wait until a tx is available 259 if next = memR.mempool.TxsFront(); next == nil { 260 continue 261 } 262 case <-peer.Quit(): 263 return 264 case <-memR.Quit(): 265 return 266 } 267 } 268 269 // Make sure the peer is up to date. 270 peerState, ok := peer.Get(types.PeerStateKey).(PeerState) 271 if !ok { 272 // Peer does not have a state yet. We set it in the consensus reactor, but 273 // when we add peer in Switch, the order we call reactors#AddPeer is 274 // different every time due to us using a map. Sometimes other reactors 275 // will be initialized before the consensus reactor. We should wait a few 276 // milliseconds and retry. 277 time.Sleep(mempool.PeerCatchupSleepIntervalMS * time.Millisecond) 278 continue 279 } 280 281 // Allow for a lag of 1 block. 282 memTx := next.Value.(*mempoolTx) 283 if peerState.GetHeight() < memTx.Height()-1 { 284 time.Sleep(mempool.PeerCatchupSleepIntervalMS * time.Millisecond) 285 continue 286 } 287 288 // NOTE: Transaction batching was disabled due to 289 // https://github.com/tendermint/tendermint/issues/5796 290 291 if _, ok := memTx.senders.Load(peerID); !ok { 292 success := peer.SendEnvelope(p2p.Envelope{ 293 ChannelID: mempool.MempoolChannel, 294 Message: &protomem.Txs{Txs: [][]byte{memTx.tx}}, 295 }) 296 if !success { 297 time.Sleep(mempool.PeerCatchupSleepIntervalMS * time.Millisecond) 298 continue 299 } 300 } 301 302 select { 303 case <-next.NextWaitChan(): 304 // see the start of the for loop for nil check 305 next = next.Next() 306 case <-peer.Quit(): 307 return 308 case <-memR.Quit(): 309 return 310 } 311 } 312 } 313 314 // TxsMessage is a Message containing transactions. 315 type TxsMessage struct { 316 Txs []types.Tx 317 } 318 319 // String returns a string representation of the TxsMessage. 320 func (m *TxsMessage) String() string { 321 return fmt.Sprintf("[TxsMessage %v]", m.Txs) 322 }