github.com/badrootd/nibiru-cometbft@v0.37.5-0.20240307173500-2a75559eee9b/mempool/v0/reactor.go (about)

     1  package v0
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"fmt"
     7  	"time"
     8  
     9  	cfg "github.com/badrootd/nibiru-cometbft/config"
    10  	"github.com/badrootd/nibiru-cometbft/libs/clist"
    11  	"github.com/badrootd/nibiru-cometbft/libs/log"
    12  	cmtsync "github.com/badrootd/nibiru-cometbft/libs/sync"
    13  	"github.com/badrootd/nibiru-cometbft/mempool"
    14  	"github.com/badrootd/nibiru-cometbft/p2p"
    15  	protomem "github.com/badrootd/nibiru-cometbft/proto/tendermint/mempool"
    16  	"github.com/badrootd/nibiru-cometbft/types"
    17  	"golang.org/x/sync/semaphore"
    18  )
    19  
    20  // Reactor handles mempool tx broadcasting amongst peers.
    21  // It maintains a map from peer ID to counter, to prevent gossiping txs to the
    22  // peers you received it from.
    23  type Reactor struct {
    24  	p2p.BaseReactor
    25  	config  *cfg.MempoolConfig
    26  	mempool *CListMempool
    27  	ids     *mempoolIDs
    28  
    29  	// Semaphores to keep track of how many connections to peers are active for broadcasting
    30  	// transactions. Each semaphore has a capacity that puts an upper bound on the number of
    31  	// connections for different groups of peers.
    32  	activePersistentPeersSemaphore    *semaphore.Weighted
    33  	activeNonPersistentPeersSemaphore *semaphore.Weighted
    34  }
    35  
    36  type mempoolIDs struct {
    37  	mtx       cmtsync.RWMutex
    38  	peerMap   map[p2p.ID]uint16
    39  	nextID    uint16              // assumes that a node will never have over 65536 active peers
    40  	activeIDs map[uint16]struct{} // used to check if a given peerID key is used, the value doesn't matter
    41  }
    42  
    43  // Reserve searches for the next unused ID and assigns it to the
    44  // peer.
    45  func (ids *mempoolIDs) ReserveForPeer(peer p2p.Peer) {
    46  	ids.mtx.Lock()
    47  	defer ids.mtx.Unlock()
    48  
    49  	curID := ids.nextPeerID()
    50  	ids.peerMap[peer.ID()] = curID
    51  	ids.activeIDs[curID] = struct{}{}
    52  }
    53  
    54  // nextPeerID returns the next unused peer ID to use.
    55  // This assumes that ids's mutex is already locked.
    56  func (ids *mempoolIDs) nextPeerID() uint16 {
    57  	if len(ids.activeIDs) == mempool.MaxActiveIDs {
    58  		panic(fmt.Sprintf("node has maximum %d active IDs and wanted to get one more", mempool.MaxActiveIDs))
    59  	}
    60  
    61  	_, idExists := ids.activeIDs[ids.nextID]
    62  	for idExists {
    63  		ids.nextID++
    64  		_, idExists = ids.activeIDs[ids.nextID]
    65  	}
    66  	curID := ids.nextID
    67  	ids.nextID++
    68  	return curID
    69  }
    70  
    71  // Reclaim returns the ID reserved for the peer back to unused pool.
    72  func (ids *mempoolIDs) Reclaim(peer p2p.Peer) {
    73  	ids.mtx.Lock()
    74  	defer ids.mtx.Unlock()
    75  
    76  	removedID, ok := ids.peerMap[peer.ID()]
    77  	if ok {
    78  		delete(ids.activeIDs, removedID)
    79  		delete(ids.peerMap, peer.ID())
    80  	}
    81  }
    82  
    83  // GetForPeer returns an ID reserved for the peer.
    84  func (ids *mempoolIDs) GetForPeer(peer p2p.Peer) uint16 {
    85  	ids.mtx.RLock()
    86  	defer ids.mtx.RUnlock()
    87  
    88  	return ids.peerMap[peer.ID()]
    89  }
    90  
    91  func newMempoolIDs() *mempoolIDs {
    92  	return &mempoolIDs{
    93  		peerMap:   make(map[p2p.ID]uint16),
    94  		activeIDs: map[uint16]struct{}{0: {}},
    95  		nextID:    1, // reserve unknownPeerID(0) for mempoolReactor.BroadcastTx
    96  	}
    97  }
    98  
    99  // NewReactor returns a new Reactor with the given config and mempool.
   100  func NewReactor(config *cfg.MempoolConfig, mempool *CListMempool) *Reactor {
   101  	memR := &Reactor{
   102  		config:  config,
   103  		mempool: mempool,
   104  		ids:     newMempoolIDs(),
   105  	}
   106  	memR.BaseReactor = *p2p.NewBaseReactor("Mempool", memR)
   107  	memR.activePersistentPeersSemaphore = semaphore.NewWeighted(int64(memR.config.ExperimentalMaxGossipConnectionsToPersistentPeers))
   108  	memR.activeNonPersistentPeersSemaphore = semaphore.NewWeighted(int64(memR.config.ExperimentalMaxGossipConnectionsToNonPersistentPeers))
   109  
   110  	return memR
   111  }
   112  
   113  // InitPeer implements Reactor by creating a state for the peer.
   114  func (memR *Reactor) InitPeer(peer p2p.Peer) p2p.Peer {
   115  	memR.ids.ReserveForPeer(peer)
   116  	return peer
   117  }
   118  
   119  // SetLogger sets the Logger on the reactor and the underlying mempool.
   120  func (memR *Reactor) SetLogger(l log.Logger) {
   121  	memR.Logger = l
   122  	memR.mempool.SetLogger(l)
   123  }
   124  
   125  // OnStart implements p2p.BaseReactor.
   126  func (memR *Reactor) OnStart() error {
   127  	if !memR.config.Broadcast {
   128  		memR.Logger.Info("Tx broadcasting is disabled")
   129  	}
   130  	return nil
   131  }
   132  
   133  // GetChannels implements Reactor by returning the list of channels for this
   134  // reactor.
   135  func (memR *Reactor) GetChannels() []*p2p.ChannelDescriptor {
   136  	largestTx := make([]byte, memR.config.MaxTxBytes)
   137  	batchMsg := protomem.Message{
   138  		Sum: &protomem.Message_Txs{
   139  			Txs: &protomem.Txs{Txs: [][]byte{largestTx}},
   140  		},
   141  	}
   142  
   143  	return []*p2p.ChannelDescriptor{
   144  		{
   145  			ID:                  mempool.MempoolChannel,
   146  			Priority:            5,
   147  			RecvMessageCapacity: batchMsg.Size(),
   148  			MessageType:         &protomem.Message{},
   149  		},
   150  	}
   151  }
   152  
   153  // AddPeer implements Reactor.
   154  // It starts a broadcast routine ensuring all txs are forwarded to the given peer.
   155  func (memR *Reactor) AddPeer(peer p2p.Peer) {
   156  	if memR.config.Broadcast {
   157  		go func() {
   158  			// Always forward transactions to unconditional peers.
   159  			if !memR.Switch.IsPeerUnconditional(peer.ID()) {
   160  				// Depending on the type of peer, we choose a semaphore to limit the gossiping peers.
   161  				var peerSemaphore *semaphore.Weighted
   162  				if peer.IsPersistent() && memR.config.ExperimentalMaxGossipConnectionsToPersistentPeers > 0 {
   163  					peerSemaphore = memR.activePersistentPeersSemaphore
   164  				} else if !peer.IsPersistent() && memR.config.ExperimentalMaxGossipConnectionsToNonPersistentPeers > 0 {
   165  					peerSemaphore = memR.activeNonPersistentPeersSemaphore
   166  				}
   167  
   168  				if peerSemaphore != nil {
   169  					for peer.IsRunning() {
   170  						// Block on the semaphore until a slot is available to start gossiping with this peer.
   171  						// Do not block indefinitely, in case the peer is disconnected before gossiping starts.
   172  						ctxTimeout, cancel := context.WithTimeout(context.TODO(), 30*time.Second)
   173  						// Block sending transactions to peer until one of the connections become
   174  						// available in the semaphore.
   175  						err := peerSemaphore.Acquire(ctxTimeout, 1)
   176  						cancel()
   177  
   178  						if err != nil {
   179  							continue
   180  						}
   181  
   182  						// Release semaphore to allow other peer to start sending transactions.
   183  						defer peerSemaphore.Release(1)
   184  						break
   185  					}
   186  				}
   187  			}
   188  
   189  			memR.mempool.metrics.ActiveOutboundConnections.Add(1)
   190  			defer memR.mempool.metrics.ActiveOutboundConnections.Add(-1)
   191  			memR.broadcastTxRoutine(peer)
   192  		}()
   193  	}
   194  }
   195  
   196  // RemovePeer implements Reactor.
   197  func (memR *Reactor) RemovePeer(peer p2p.Peer, reason interface{}) {
   198  	memR.ids.Reclaim(peer)
   199  	// broadcast routine checks if peer is gone and returns
   200  }
   201  
   202  // Receive implements Reactor.
   203  // It adds any received transactions to the mempool.
   204  func (memR *Reactor) ReceiveEnvelope(e p2p.Envelope) {
   205  	memR.Logger.Debug("Receive", "src", e.Src, "chId", e.ChannelID, "msg", e.Message)
   206  	switch msg := e.Message.(type) {
   207  	case *protomem.Txs:
   208  		protoTxs := msg.GetTxs()
   209  		if len(protoTxs) == 0 {
   210  			memR.Logger.Error("received empty txs from peer", "src", e.Src)
   211  			return
   212  		}
   213  		txInfo := mempool.TxInfo{SenderID: memR.ids.GetForPeer(e.Src)}
   214  		if e.Src != nil {
   215  			txInfo.SenderP2PID = e.Src.ID()
   216  		}
   217  
   218  		var err error
   219  		for _, tx := range protoTxs {
   220  			ntx := types.Tx(tx)
   221  			err = memR.mempool.CheckTx(ntx, nil, txInfo)
   222  			if errors.Is(err, mempool.ErrTxInCache) {
   223  				memR.Logger.Debug("Tx already exists in cache", "tx", ntx.String())
   224  			} else if err != nil {
   225  				memR.Logger.Info("Could not check tx", "tx", ntx.String(), "err", err)
   226  			}
   227  		}
   228  	default:
   229  		memR.Logger.Error("unknown message type", "src", e.Src, "chId", e.ChannelID, "msg", e.Message)
   230  		memR.Switch.StopPeerForError(e.Src, fmt.Errorf("mempool cannot handle message of type: %T", e.Message))
   231  		return
   232  	}
   233  
   234  	// broadcasting happens from go routines per peer
   235  }
   236  
   237  // PeerState describes the state of a peer.
   238  type PeerState interface {
   239  	GetHeight() int64
   240  }
   241  
   242  // Send new mempool txs to peer.
   243  func (memR *Reactor) broadcastTxRoutine(peer p2p.Peer) {
   244  	peerID := memR.ids.GetForPeer(peer)
   245  	var next *clist.CElement
   246  
   247  	for {
   248  		// In case of both next.NextWaitChan() and peer.Quit() are variable at the same time
   249  		if !memR.IsRunning() || !peer.IsRunning() {
   250  			return
   251  		}
   252  
   253  		// This happens because the CElement we were looking at got garbage
   254  		// collected (removed). That is, .NextWait() returned nil. Go ahead and
   255  		// start from the beginning.
   256  		if next == nil {
   257  			select {
   258  			case <-memR.mempool.TxsWaitChan(): // Wait until a tx is available
   259  				if next = memR.mempool.TxsFront(); next == nil {
   260  					continue
   261  				}
   262  			case <-peer.Quit():
   263  				return
   264  			case <-memR.Quit():
   265  				return
   266  			}
   267  		}
   268  
   269  		// Make sure the peer is up to date.
   270  		peerState, ok := peer.Get(types.PeerStateKey).(PeerState)
   271  		if !ok {
   272  			// Peer does not have a state yet. We set it in the consensus reactor, but
   273  			// when we add peer in Switch, the order we call reactors#AddPeer is
   274  			// different every time due to us using a map. Sometimes other reactors
   275  			// will be initialized before the consensus reactor. We should wait a few
   276  			// milliseconds and retry.
   277  			time.Sleep(mempool.PeerCatchupSleepIntervalMS * time.Millisecond)
   278  			continue
   279  		}
   280  
   281  		// Allow for a lag of 1 block.
   282  		memTx := next.Value.(*mempoolTx)
   283  		if peerState.GetHeight() < memTx.Height()-1 {
   284  			time.Sleep(mempool.PeerCatchupSleepIntervalMS * time.Millisecond)
   285  			continue
   286  		}
   287  
   288  		// NOTE: Transaction batching was disabled due to
   289  		// https://github.com/tendermint/tendermint/issues/5796
   290  
   291  		if _, ok := memTx.senders.Load(peerID); !ok {
   292  			success := peer.SendEnvelope(p2p.Envelope{
   293  				ChannelID: mempool.MempoolChannel,
   294  				Message:   &protomem.Txs{Txs: [][]byte{memTx.tx}},
   295  			})
   296  			if !success {
   297  				time.Sleep(mempool.PeerCatchupSleepIntervalMS * time.Millisecond)
   298  				continue
   299  			}
   300  		}
   301  
   302  		select {
   303  		case <-next.NextWaitChan():
   304  			// see the start of the for loop for nil check
   305  			next = next.Next()
   306  		case <-peer.Quit():
   307  			return
   308  		case <-memR.Quit():
   309  			return
   310  		}
   311  	}
   312  }
   313  
   314  // TxsMessage is a Message containing transactions.
   315  type TxsMessage struct {
   316  	Txs []types.Tx
   317  }
   318  
   319  // String returns a string representation of the TxsMessage.
   320  func (m *TxsMessage) String() string {
   321  	return fmt.Sprintf("[TxsMessage %v]", m.Txs)
   322  }