github.com/neatio-net/neatio@v1.7.3-0.20231114194659-f4d7a2226baa/network/p2p/server.go (about)

     1  package p2p
     2  
     3  import (
     4  	"crypto/ecdsa"
     5  	"errors"
     6  	"fmt"
     7  	"net"
     8  	"sync"
     9  	"time"
    10  
    11  	"github.com/neatio-net/neatio/chain/log"
    12  	"github.com/neatio-net/neatio/network/p2p/discover"
    13  	"github.com/neatio-net/neatio/network/p2p/discv5"
    14  	"github.com/neatio-net/neatio/network/p2p/nat"
    15  	"github.com/neatio-net/neatio/network/p2p/netutil"
    16  	"github.com/neatio-net/neatio/utilities/common"
    17  	"github.com/neatio-net/neatio/utilities/common/mclock"
    18  	"github.com/neatio-net/neatio/utilities/event"
    19  	"github.com/neatio-net/neatio/utilities/rlp"
    20  )
    21  
    22  const (
    23  	defaultDialTimeout = 15 * time.Second
    24  
    25  	maxActiveDialTasks     = 16
    26  	defaultMaxPendingPeers = 50
    27  	defaultDialRatio       = 3
    28  
    29  	frameReadTimeout = 30 * time.Second
    30  
    31  	frameWriteTimeout = 20 * time.Second
    32  )
    33  
    34  var errServerStopped = errors.New("server stopped")
    35  
    36  type Config struct {
    37  	PrivateKey *ecdsa.PrivateKey `toml:"-"`
    38  
    39  	MaxPeers int
    40  
    41  	MaxPendingPeers int `toml:",omitempty"`
    42  
    43  	DialRatio int `toml:",omitempty"`
    44  
    45  	NoDiscovery bool
    46  
    47  	DiscoveryV5 bool `toml:",omitempty"`
    48  
    49  	Name string `toml:"-"`
    50  
    51  	BootstrapNodes []*discover.Node
    52  
    53  	BootstrapNodesV5 []*discv5.Node `toml:",omitempty"`
    54  
    55  	StaticNodes []*discover.Node
    56  
    57  	TrustedNodes []*discover.Node
    58  
    59  	LocalValidators []P2PValidator
    60  
    61  	Validators map[P2PValidator]*P2PValidatorNodeInfo
    62  
    63  	NetRestrict *netutil.Netlist `toml:",omitempty"`
    64  
    65  	NodeDatabase string `toml:",omitempty"`
    66  
    67  	Protocols []Protocol `toml:"-"`
    68  
    69  	ListenAddr string
    70  
    71  	NAT nat.Interface `toml:",omitempty"`
    72  
    73  	Dialer NodeDialer `toml:"-"`
    74  
    75  	NoDial bool `toml:",omitempty"`
    76  
    77  	EnableMsgEvents bool
    78  
    79  	Logger log.Logger `toml:",omitempty"`
    80  }
    81  
    82  type NodeInfoToSend struct {
    83  	valNodeInfo *P2PValidatorNodeInfo
    84  	action      uint64
    85  	p           *Peer
    86  }
    87  
    88  type Server struct {
    89  	Config
    90  
    91  	newTransport func(net.Conn) transport
    92  	newPeerHook  func(*Peer)
    93  
    94  	lock    sync.Mutex
    95  	running bool
    96  
    97  	ntab         discoverTable
    98  	listener     net.Listener
    99  	ourHandshake *protoHandshake
   100  	lastLookup   time.Time
   101  	DiscV5       *discv5.Network
   102  
   103  	peerOp     chan peerOpFunc
   104  	peerOpDone chan struct{}
   105  
   106  	quit          chan struct{}
   107  	addstatic     chan *discover.Node
   108  	removestatic  chan *discover.Node
   109  	posthandshake chan *conn
   110  	addpeer       chan *conn
   111  	delpeer       chan peerDrop
   112  	loopWG        sync.WaitGroup
   113  	peerFeed      event.Feed
   114  	log           log.Logger
   115  
   116  	events    chan *PeerEvent
   117  	eventsSub event.Subscription
   118  
   119  	nodeInfoLock sync.Mutex
   120  	nodeInfoList []*NodeInfoToSend
   121  }
   122  
   123  type peerOpFunc func(map[discover.NodeID]*Peer)
   124  
   125  type peerDrop struct {
   126  	*Peer
   127  	err       error
   128  	requested bool
   129  }
   130  
   131  type connFlag int
   132  
   133  const (
   134  	dynDialedConn connFlag = 1 << iota
   135  	staticDialedConn
   136  	inboundConn
   137  	trustedConn
   138  )
   139  
   140  type conn struct {
   141  	fd net.Conn
   142  	transport
   143  	flags connFlag
   144  	cont  chan error
   145  	id    discover.NodeID
   146  	caps  []Cap
   147  	name  string
   148  }
   149  
   150  type transport interface {
   151  	doEncHandshake(prv *ecdsa.PrivateKey, dialDest *discover.Node) (discover.NodeID, error)
   152  	doProtoHandshake(our *protoHandshake) (*protoHandshake, error)
   153  
   154  	MsgReadWriter
   155  
   156  	close(err error)
   157  }
   158  
   159  func (c *conn) String() string {
   160  	s := c.flags.String()
   161  	if (c.id != discover.NodeID{}) {
   162  		s += " " + c.id.String()
   163  	}
   164  	s += " " + c.fd.RemoteAddr().String()
   165  	return s
   166  }
   167  
   168  func (f connFlag) String() string {
   169  	s := ""
   170  	if f&trustedConn != 0 {
   171  		s += "-trusted"
   172  	}
   173  	if f&dynDialedConn != 0 {
   174  		s += "-dyndial"
   175  	}
   176  	if f&staticDialedConn != 0 {
   177  		s += "-staticdial"
   178  	}
   179  	if f&inboundConn != 0 {
   180  		s += "-inbound"
   181  	}
   182  	if s != "" {
   183  		s = s[1:]
   184  	}
   185  	return s
   186  }
   187  
   188  func (c *conn) is(f connFlag) bool {
   189  	return c.flags&f != 0
   190  }
   191  
   192  func (srv *Server) Peers() []*Peer {
   193  	var ps []*Peer
   194  	select {
   195  
   196  	case srv.peerOp <- func(peers map[discover.NodeID]*Peer) {
   197  		for _, p := range peers {
   198  			ps = append(ps, p)
   199  		}
   200  	}:
   201  		<-srv.peerOpDone
   202  	case <-srv.quit:
   203  	}
   204  	return ps
   205  }
   206  
   207  func (srv *Server) PeerCount() int {
   208  	var count int
   209  	select {
   210  	case srv.peerOp <- func(ps map[discover.NodeID]*Peer) { count = len(ps) }:
   211  		<-srv.peerOpDone
   212  	case <-srv.quit:
   213  	}
   214  	return count
   215  }
   216  
   217  func (srv *Server) AddPeer(node *discover.Node) {
   218  	select {
   219  	case srv.addstatic <- node:
   220  	case <-srv.quit:
   221  	}
   222  }
   223  
   224  func (srv *Server) RemovePeer(node *discover.Node) {
   225  	select {
   226  	case srv.removestatic <- node:
   227  	case <-srv.quit:
   228  	}
   229  }
   230  
   231  func (srv *Server) SubscribeEvents(ch chan *PeerEvent) event.Subscription {
   232  	return srv.peerFeed.Subscribe(ch)
   233  }
   234  
   235  func (srv *Server) Self() *discover.Node {
   236  	srv.lock.Lock()
   237  	defer srv.lock.Unlock()
   238  
   239  	if !srv.running {
   240  		return &discover.Node{IP: net.ParseIP("0.0.0.0")}
   241  	}
   242  	return srv.makeSelf(srv.listener, srv.ntab)
   243  }
   244  
   245  func (srv *Server) makeSelf(listener net.Listener, ntab discoverTable) *discover.Node {
   246  
   247  	if ntab == nil {
   248  
   249  		if listener == nil {
   250  			return &discover.Node{IP: net.ParseIP("0.0.0.0"), ID: discover.PubkeyID(&srv.PrivateKey.PublicKey)}
   251  		}
   252  
   253  		addr := listener.Addr().(*net.TCPAddr)
   254  		return &discover.Node{
   255  			ID:  discover.PubkeyID(&srv.PrivateKey.PublicKey),
   256  			IP:  addr.IP,
   257  			TCP: uint16(addr.Port),
   258  		}
   259  	}
   260  
   261  	return ntab.Self()
   262  }
   263  
   264  func (srv *Server) Stop() {
   265  	srv.lock.Lock()
   266  	if !srv.running {
   267  		srv.lock.Unlock()
   268  		return
   269  	}
   270  	srv.eventsSub.Unsubscribe()
   271  	srv.running = false
   272  	if srv.listener != nil {
   273  
   274  		srv.listener.Close()
   275  	}
   276  	close(srv.quit)
   277  	srv.lock.Unlock()
   278  	srv.loopWG.Wait()
   279  }
   280  
   281  type sharedUDPConn struct {
   282  	*net.UDPConn
   283  	unhandled chan discover.ReadPacket
   284  }
   285  
   286  func (s *sharedUDPConn) ReadFromUDP(b []byte) (n int, addr *net.UDPAddr, err error) {
   287  	packet, ok := <-s.unhandled
   288  	if !ok {
   289  		return 0, nil, fmt.Errorf("Connection was closed")
   290  	}
   291  	l := len(packet.Data)
   292  	if l > len(b) {
   293  		l = len(b)
   294  	}
   295  	copy(b[:l], packet.Data[:l])
   296  	return l, packet.Addr, nil
   297  }
   298  
   299  func (s *sharedUDPConn) Close() error {
   300  	return nil
   301  }
   302  
   303  func (srv *Server) Start() (err error) {
   304  	srv.lock.Lock()
   305  	defer srv.lock.Unlock()
   306  	if srv.running {
   307  		return errors.New("server already running")
   308  	}
   309  	srv.running = true
   310  	srv.log = srv.Config.Logger
   311  	if srv.log == nil {
   312  		srv.log = log.New()
   313  	}
   314  	srv.log.Info("Starting P2P networking...")
   315  
   316  	if srv.PrivateKey == nil {
   317  		return fmt.Errorf("Server.PrivateKey must be set to a non-nil key")
   318  	}
   319  	if srv.newTransport == nil {
   320  		srv.newTransport = newRLPX
   321  	}
   322  	if srv.Dialer == nil {
   323  		srv.Dialer = TCPDialer{&net.Dialer{Timeout: defaultDialTimeout}}
   324  	}
   325  	srv.quit = make(chan struct{})
   326  	srv.addpeer = make(chan *conn)
   327  	srv.delpeer = make(chan peerDrop)
   328  	srv.posthandshake = make(chan *conn)
   329  	srv.addstatic = make(chan *discover.Node)
   330  	srv.removestatic = make(chan *discover.Node)
   331  	srv.peerOp = make(chan peerOpFunc)
   332  	srv.peerOpDone = make(chan struct{})
   333  	srv.events = make(chan *PeerEvent)
   334  	srv.eventsSub = srv.SubscribeEvents(srv.events)
   335  
   336  	srv.nodeInfoList = make([]*NodeInfoToSend, 0)
   337  
   338  	var (
   339  		conn      *net.UDPConn
   340  		sconn     *sharedUDPConn
   341  		realaddr  *net.UDPAddr
   342  		unhandled chan discover.ReadPacket
   343  	)
   344  
   345  	if !srv.NoDiscovery || srv.DiscoveryV5 {
   346  		addr, err := net.ResolveUDPAddr("udp", srv.ListenAddr)
   347  		if err != nil {
   348  			return err
   349  		}
   350  		conn, err = net.ListenUDP("udp", addr)
   351  		if err != nil {
   352  			return err
   353  		}
   354  		realaddr = conn.LocalAddr().(*net.UDPAddr)
   355  		if srv.NAT != nil {
   356  			if !realaddr.IP.IsLoopback() {
   357  				go nat.Map(srv.NAT, srv.quit, "udp", realaddr.Port, realaddr.Port, "ethereum discovery")
   358  			}
   359  
   360  			if ext, err := srv.NAT.ExternalIP(); err == nil {
   361  				realaddr = &net.UDPAddr{IP: ext, Port: realaddr.Port}
   362  			}
   363  		}
   364  	}
   365  
   366  	if !srv.NoDiscovery && srv.DiscoveryV5 {
   367  		unhandled = make(chan discover.ReadPacket, 100)
   368  		sconn = &sharedUDPConn{conn, unhandled}
   369  	}
   370  
   371  	if !srv.NoDiscovery {
   372  		cfg := discover.Config{
   373  			PrivateKey:   srv.PrivateKey,
   374  			AnnounceAddr: realaddr,
   375  			NodeDBPath:   srv.NodeDatabase,
   376  			NetRestrict:  srv.NetRestrict,
   377  			Bootnodes:    srv.BootstrapNodes,
   378  			Unhandled:    unhandled,
   379  		}
   380  		ntab, err := discover.ListenUDP(conn, cfg)
   381  		if err != nil {
   382  			return err
   383  		}
   384  		srv.ntab = ntab
   385  	}
   386  
   387  	if srv.DiscoveryV5 {
   388  		var (
   389  			ntab *discv5.Network
   390  			err  error
   391  		)
   392  		if sconn != nil {
   393  			ntab, err = discv5.ListenUDP(srv.PrivateKey, sconn, realaddr, "", srv.NetRestrict)
   394  		} else {
   395  			ntab, err = discv5.ListenUDP(srv.PrivateKey, conn, realaddr, "", srv.NetRestrict)
   396  		}
   397  		if err != nil {
   398  			return err
   399  		}
   400  		if err := ntab.SetFallbackNodes(srv.BootstrapNodesV5); err != nil {
   401  			return err
   402  		}
   403  		srv.DiscV5 = ntab
   404  	}
   405  
   406  	dynPeers := srv.maxDialedConns()
   407  	dialer := newDialState(srv.StaticNodes, srv.BootstrapNodes, srv.ntab, dynPeers, srv.NetRestrict)
   408  
   409  	srv.ourHandshake = &protoHandshake{Version: baseProtocolVersion, Name: srv.Name, ID: discover.PubkeyID(&srv.PrivateKey.PublicKey)}
   410  	for _, p := range srv.Protocols {
   411  		srv.ourHandshake.Caps = append(srv.ourHandshake.Caps, p.cap())
   412  	}
   413  
   414  	if srv.ListenAddr != "" {
   415  		if err := srv.startListening(); err != nil {
   416  			return err
   417  		}
   418  	}
   419  	if srv.NoDial && srv.ListenAddr == "" {
   420  		srv.log.Warn("P2P server will be useless, neither dialing nor listening")
   421  	}
   422  
   423  	srv.loopWG.Add(1)
   424  	go srv.run(dialer)
   425  	srv.running = true
   426  
   427  	go srv.sendValidatorNodeInfoMessages()
   428  
   429  	return nil
   430  }
   431  
   432  func (srv *Server) AddChildProtocolCaps(sideProtocols []Protocol) {
   433  	for _, p := range sideProtocols {
   434  		srv.ourHandshake.Caps = append(srv.ourHandshake.Caps, p.cap())
   435  	}
   436  }
   437  
   438  func (srv *Server) startListening() error {
   439  
   440  	listener, err := net.Listen("tcp", srv.ListenAddr)
   441  	if err != nil {
   442  		return err
   443  	}
   444  	laddr := listener.Addr().(*net.TCPAddr)
   445  	srv.ListenAddr = laddr.String()
   446  	srv.listener = listener
   447  	srv.loopWG.Add(1)
   448  	go srv.listenLoop()
   449  
   450  	if !laddr.IP.IsLoopback() && srv.NAT != nil {
   451  		srv.loopWG.Add(1)
   452  		go func() {
   453  			nat.Map(srv.NAT, srv.quit, "tcp", laddr.Port, laddr.Port, "ethereum p2p")
   454  			srv.loopWG.Done()
   455  		}()
   456  	}
   457  	return nil
   458  }
   459  
   460  type dialer interface {
   461  	newTasks(running int, peers map[discover.NodeID]*Peer, now time.Time) []task
   462  	taskDone(task, time.Time)
   463  	addStatic(*discover.Node)
   464  	removeStatic(*discover.Node)
   465  }
   466  
   467  func (srv *Server) run(dialstate dialer) {
   468  	defer srv.loopWG.Done()
   469  	var (
   470  		peers        = make(map[discover.NodeID]*Peer)
   471  		inboundCount = 0
   472  		trusted      = make(map[discover.NodeID]bool, len(srv.TrustedNodes))
   473  		taskdone     = make(chan task, maxActiveDialTasks)
   474  		runningTasks []task
   475  		queuedTasks  []task
   476  	)
   477  
   478  	for _, n := range srv.TrustedNodes {
   479  		trusted[n.ID] = true
   480  	}
   481  
   482  	delTask := func(t task) {
   483  		for i := range runningTasks {
   484  			if runningTasks[i] == t {
   485  				runningTasks = append(runningTasks[:i], runningTasks[i+1:]...)
   486  				break
   487  			}
   488  		}
   489  	}
   490  
   491  	startTasks := func(ts []task) (rest []task) {
   492  		i := 0
   493  		for ; len(runningTasks) < maxActiveDialTasks && i < len(ts); i++ {
   494  			t := ts[i]
   495  			srv.log.Trace("New dial task", "task", t)
   496  			go func() { t.Do(srv); taskdone <- t }()
   497  			runningTasks = append(runningTasks, t)
   498  		}
   499  		return ts[i:]
   500  	}
   501  	scheduleTasks := func() {
   502  
   503  		queuedTasks = append(queuedTasks[:0], startTasks(queuedTasks)...)
   504  
   505  		if len(runningTasks) < maxActiveDialTasks {
   506  			nt := dialstate.newTasks(len(runningTasks)+len(queuedTasks), peers, time.Now())
   507  			queuedTasks = append(queuedTasks, startTasks(nt)...)
   508  		}
   509  	}
   510  
   511  running:
   512  	for {
   513  		scheduleTasks()
   514  
   515  		select {
   516  		case <-srv.quit:
   517  
   518  			break running
   519  		case n := <-srv.addstatic:
   520  
   521  			srv.log.Debug("Adding static node", "node", n)
   522  			dialstate.addStatic(n)
   523  		case n := <-srv.removestatic:
   524  
   525  			srv.log.Debug("Removing static node", "node", n)
   526  			dialstate.removeStatic(n)
   527  			if p, ok := peers[n.ID]; ok {
   528  				p.Disconnect(DiscRequested)
   529  			}
   530  		case op := <-srv.peerOp:
   531  
   532  			op(peers)
   533  			srv.peerOpDone <- struct{}{}
   534  		case t := <-taskdone:
   535  
   536  			srv.log.Trace("Dial task done", "task", t)
   537  			dialstate.taskDone(t, time.Now())
   538  			delTask(t)
   539  		case c := <-srv.posthandshake:
   540  
   541  			if trusted[c.id] {
   542  
   543  				c.flags |= trustedConn
   544  			}
   545  
   546  			select {
   547  			case c.cont <- srv.encHandshakeChecks(peers, inboundCount, c):
   548  			case <-srv.quit:
   549  				break running
   550  			}
   551  		case c := <-srv.addpeer:
   552  
   553  			err := srv.protoHandshakeChecks(peers, inboundCount, c)
   554  			if err == nil {
   555  
   556  				p := newPeer(c, srv.Protocols)
   557  
   558  				if srv.EnableMsgEvents {
   559  					p.events = &srv.peerFeed
   560  				}
   561  				name := truncateName(c.name)
   562  				srv.log.Debug("Adding p2p peer", "name", name, "addr", c.fd.RemoteAddr(), "peers", len(peers)+1)
   563  				go srv.runPeer(p)
   564  				peers[c.id] = p
   565  				if p.Inbound() {
   566  					inboundCount++
   567  				}
   568  
   569  				srv.validatorAddPeer(p)
   570  			}
   571  
   572  			select {
   573  			case c.cont <- err:
   574  			case <-srv.quit:
   575  				break running
   576  			}
   577  		case pd := <-srv.delpeer:
   578  
   579  			d := common.PrettyDuration(mclock.Now() - pd.created)
   580  			pd.log.Debug("Removing p2p peer", "duration", d, "peers", len(peers)-1, "req", pd.requested, "err", pd.err)
   581  			delete(peers, pd.ID())
   582  			if pd.Inbound() {
   583  				inboundCount--
   584  			}
   585  			srv.validatorDelPeer(pd.ID())
   586  
   587  		case evt := <-srv.events:
   588  			log.Debugf("peer events received: %v", evt)
   589  			switch evt.Type {
   590  
   591  			case PeerEventTypeRefreshValidator:
   592  				var valNodeInfo P2PValidatorNodeInfo
   593  				if err := rlp.DecodeBytes([]byte(evt.Protocol), &valNodeInfo); err != nil {
   594  					log.Debugf("rlp decode valNodeInfo failed with %v", err)
   595  				}
   596  
   597  				peerArr := make([]*Peer, 0)
   598  				for _, p := range peers {
   599  					peerArr = append(peerArr, p)
   600  				}
   601  
   602  				if err := srv.validatorAdd(valNodeInfo, peerArr, dialstate); err != nil {
   603  					log.Debugf("add valNodeInfo to local failed with %v", err)
   604  				}
   605  
   606  				log.Debugf("Got refresh validation node infomation from validation %v", valNodeInfo.Validator.Address.String())
   607  
   608  			case PeerEventTypeRemoveValidator:
   609  				var valNodeInfo P2PValidatorNodeInfo
   610  				if err := rlp.DecodeBytes([]byte(evt.Protocol), &valNodeInfo); err != nil {
   611  					log.Debugf("rlp decode valNodeInfo failed with %v", err)
   612  				}
   613  
   614  				peerArr := make([]*Peer, 0)
   615  				for _, p := range peers {
   616  					peerArr = append(peerArr, p)
   617  				}
   618  
   619  				if err := srv.validatorRemove(valNodeInfo, peerArr, dialstate); err != nil {
   620  					log.Debugf("remove valNodeInfo from local failed with %v", err)
   621  				}
   622  
   623  				log.Debugf("Got remove validation node infomation from validation %v", valNodeInfo.Validator.Address.String())
   624  
   625  			}
   626  		}
   627  	}
   628  
   629  	srv.log.Trace("P2P networking is spinning down")
   630  
   631  	if srv.ntab != nil {
   632  		srv.ntab.Close()
   633  	}
   634  	if srv.DiscV5 != nil {
   635  		srv.DiscV5.Close()
   636  	}
   637  
   638  	for _, p := range peers {
   639  		p.Disconnect(DiscQuitting)
   640  	}
   641  
   642  	for len(peers) > 0 {
   643  		p := <-srv.delpeer
   644  		p.log.Trace("<-delpeer (spindown)", "remainingTasks", len(runningTasks))
   645  		delete(peers, p.ID())
   646  	}
   647  }
   648  
   649  func (srv *Server) protoHandshakeChecks(peers map[discover.NodeID]*Peer, inboundCount int, c *conn) error {
   650  
   651  	if len(srv.Protocols) > 0 && countMatchingProtocols(srv.Protocols, c.caps) == 0 {
   652  		return DiscUselessPeer
   653  	}
   654  
   655  	return srv.encHandshakeChecks(peers, inboundCount, c)
   656  }
   657  
   658  func (srv *Server) encHandshakeChecks(peers map[discover.NodeID]*Peer, inboundCount int, c *conn) error {
   659  	switch {
   660  	case !c.is(trustedConn|staticDialedConn) && len(peers) >= srv.MaxPeers:
   661  		return DiscTooManyPeers
   662  	case !c.is(trustedConn) && c.is(inboundConn) && inboundCount >= srv.maxInboundConns():
   663  		return DiscTooManyPeers
   664  	case peers[c.id] != nil:
   665  		return DiscAlreadyConnected
   666  	case c.id == srv.Self().ID:
   667  		return DiscSelf
   668  	default:
   669  		return nil
   670  	}
   671  }
   672  
   673  func (srv *Server) maxInboundConns() int {
   674  	return srv.MaxPeers - srv.maxDialedConns()
   675  }
   676  
   677  func (srv *Server) maxDialedConns() int {
   678  	if srv.NoDiscovery || srv.NoDial {
   679  		return 0
   680  	}
   681  	r := srv.DialRatio
   682  	if r == 0 {
   683  		r = defaultDialRatio
   684  	}
   685  	return srv.MaxPeers / r
   686  }
   687  
   688  type tempError interface {
   689  	Temporary() bool
   690  }
   691  
   692  func (srv *Server) listenLoop() {
   693  	defer srv.loopWG.Done()
   694  
   695  	tokens := defaultMaxPendingPeers
   696  	if srv.MaxPendingPeers > 0 {
   697  		tokens = srv.MaxPendingPeers
   698  	}
   699  	slots := make(chan struct{}, tokens)
   700  	for i := 0; i < tokens; i++ {
   701  		slots <- struct{}{}
   702  	}
   703  
   704  	for {
   705  
   706  		<-slots
   707  
   708  		var (
   709  			fd  net.Conn
   710  			err error
   711  		)
   712  		for {
   713  			fd, err = srv.listener.Accept()
   714  			if tempErr, ok := err.(tempError); ok && tempErr.Temporary() {
   715  				srv.log.Debug("Temporary read error", "err", err)
   716  				continue
   717  			} else if err != nil {
   718  				srv.log.Debug("Read error", "err", err)
   719  				return
   720  			}
   721  			break
   722  		}
   723  
   724  		if srv.NetRestrict != nil {
   725  			if tcp, ok := fd.RemoteAddr().(*net.TCPAddr); ok && !srv.NetRestrict.Contains(tcp.IP) {
   726  				srv.log.Debug("Rejected conn (not whitelisted in NetRestrict)", "addr", fd.RemoteAddr())
   727  				fd.Close()
   728  				slots <- struct{}{}
   729  				continue
   730  			}
   731  		}
   732  
   733  		fd = newMeteredConn(fd, true)
   734  		srv.log.Trace("Accepted connection", "addr", fd.RemoteAddr())
   735  		go func() {
   736  			srv.SetupConn(fd, inboundConn, nil)
   737  			slots <- struct{}{}
   738  		}()
   739  	}
   740  }
   741  
   742  func (srv *Server) SetupConn(fd net.Conn, flags connFlag, dialDest *discover.Node) error {
   743  	self := srv.Self()
   744  	if self == nil {
   745  		return errors.New("shutdown")
   746  	}
   747  	c := &conn{fd: fd, transport: srv.newTransport(fd), flags: flags, cont: make(chan error)}
   748  	err := srv.setupConn(c, flags, dialDest)
   749  	if err != nil {
   750  		c.close(err)
   751  		srv.log.Trace("Setting up connection failed", "id", c.id, "err", err)
   752  	}
   753  	return err
   754  }
   755  
   756  func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *discover.Node) error {
   757  
   758  	srv.lock.Lock()
   759  	running := srv.running
   760  	srv.lock.Unlock()
   761  	if !running {
   762  		return errServerStopped
   763  	}
   764  
   765  	var err error
   766  	if c.id, err = c.doEncHandshake(srv.PrivateKey, dialDest); err != nil {
   767  		srv.log.Trace("Failed RLPx handshake", "addr", c.fd.RemoteAddr(), "conn", c.flags, "err", err)
   768  		return err
   769  	}
   770  	clog := srv.log.New("id", c.id, "addr", c.fd.RemoteAddr(), "conn", c.flags)
   771  
   772  	if dialDest != nil && c.id != dialDest.ID {
   773  		clog.Trace("Dialed identity mismatch", "want", c, dialDest.ID)
   774  		return DiscUnexpectedIdentity
   775  	}
   776  	err = srv.checkpoint(c, srv.posthandshake)
   777  	if err != nil {
   778  		clog.Trace("Rejected peer before protocol handshake", "err", err)
   779  		return err
   780  	}
   781  
   782  	phs, err := c.doProtoHandshake(srv.ourHandshake)
   783  	if err != nil {
   784  		clog.Trace("Failed proto handshake", "err", err)
   785  		return err
   786  	}
   787  	if phs.ID != c.id {
   788  		clog.Trace("Wrong devp2p handshake identity", "err", phs.ID)
   789  		return DiscUnexpectedIdentity
   790  	}
   791  	c.caps, c.name = phs.Caps, phs.Name
   792  	err = srv.checkpoint(c, srv.addpeer)
   793  	if err != nil {
   794  		clog.Trace("Rejected peer", "err", err)
   795  		return err
   796  	}
   797  
   798  	clog.Trace("connection set up", "inbound", dialDest == nil)
   799  	return nil
   800  }
   801  
   802  func truncateName(s string) string {
   803  	if len(s) > 20 {
   804  		return s[:20] + "..."
   805  	}
   806  	return s
   807  }
   808  
   809  func (srv *Server) checkpoint(c *conn, stage chan<- *conn) error {
   810  	select {
   811  	case stage <- c:
   812  	case <-srv.quit:
   813  		return errServerStopped
   814  	}
   815  	select {
   816  	case err := <-c.cont:
   817  		return err
   818  	case <-srv.quit:
   819  		return errServerStopped
   820  	}
   821  }
   822  
   823  func (srv *Server) runPeer(p *Peer) {
   824  	if srv.newPeerHook != nil {
   825  		srv.newPeerHook(p)
   826  	}
   827  
   828  	srv.peerFeed.Send(&PeerEvent{
   829  		Type: PeerEventTypeAdd,
   830  		Peer: p.ID(),
   831  	})
   832  
   833  	p.srvProtocols = &srv.Protocols
   834  
   835  	remoteRequested, err := p.run()
   836  
   837  	srv.peerFeed.Send(&PeerEvent{
   838  		Type:  PeerEventTypeDrop,
   839  		Peer:  p.ID(),
   840  		Error: err.Error(),
   841  	})
   842  
   843  	srv.delpeer <- peerDrop{p, err, remoteRequested}
   844  }
   845  
   846  type NodeInfo struct {
   847  	ID    string `json:"id"`
   848  	Name  string `json:"name"`
   849  	Enode string `json:"enode"`
   850  	IP    string `json:"ip"`
   851  	Ports struct {
   852  		Discovery int `json:"discovery"`
   853  		Listener  int `json:"listener"`
   854  	} `json:"ports"`
   855  	ListenAddr string                 `json:"listenAddr"`
   856  	Protocols  map[string]interface{} `json:"protocols"`
   857  }
   858  
   859  func (srv *Server) NodeInfo() *NodeInfo {
   860  	node := srv.Self()
   861  
   862  	info := &NodeInfo{
   863  		Name:       srv.Name,
   864  		Enode:      node.String(),
   865  		ID:         node.ID.String(),
   866  		IP:         node.IP.String(),
   867  		ListenAddr: srv.ListenAddr,
   868  		Protocols:  make(map[string]interface{}),
   869  	}
   870  	info.Ports.Discovery = int(node.UDP)
   871  	info.Ports.Listener = int(node.TCP)
   872  
   873  	for _, proto := range srv.Protocols {
   874  		if _, ok := info.Protocols[proto.Name]; !ok {
   875  			nodeInfo := interface{}("unknown")
   876  			if query := proto.NodeInfo; query != nil {
   877  				nodeInfo = proto.NodeInfo()
   878  			}
   879  			info.Protocols[proto.Name] = nodeInfo
   880  		}
   881  	}
   882  	return info
   883  }
   884  
   885  func (srv *Server) PeersInfo() []*PeerInfo {
   886  
   887  	infos := make([]*PeerInfo, 0, srv.PeerCount())
   888  	for _, peer := range srv.Peers() {
   889  		if peer != nil {
   890  			infos = append(infos, peer.Info())
   891  		}
   892  	}
   893  
   894  	for i := 0; i < len(infos); i++ {
   895  		for j := i + 1; j < len(infos); j++ {
   896  			if infos[i].ID > infos[j].ID {
   897  				infos[i], infos[j] = infos[j], infos[i]
   898  			}
   899  		}
   900  	}
   901  	return infos
   902  }
   903  
   904  func (srv *Server) BroadcastMsg(msgCode uint64, data interface{}) {
   905  	peers := srv.Peers()
   906  	for _, p := range peers {
   907  		Send(p.rw, BroadcastNewSideChainMsg, data)
   908  	}
   909  }
   910  
   911  func (srv *Server) AddLocalValidator(chainId string, address common.Address) {
   912  
   913  	log.Debug("AddLocalValidator")
   914  
   915  	validator := P2PValidator{
   916  		ChainId: chainId,
   917  		Address: address,
   918  	}
   919  
   920  	for i := 0; i < len(srv.LocalValidators); i++ {
   921  		if validator == srv.LocalValidators[i] {
   922  			return
   923  		}
   924  	}
   925  
   926  	srv.LocalValidators = append(srv.LocalValidators, validator)
   927  
   928  	srv.broadcastRefreshValidatorNodeInfo(&P2PValidatorNodeInfo{
   929  		Node:      *srv.Self(),
   930  		TimeStamp: time.Now(),
   931  		Validator: validator,
   932  		Original:  true,
   933  	}, nil)
   934  }
   935  
   936  func (srv *Server) RemoveLocalValidator(chainId string, address common.Address) {
   937  
   938  	log.Debug("RemoveLocalValidator")
   939  
   940  	validator := P2PValidator{
   941  		ChainId: chainId,
   942  		Address: address,
   943  	}
   944  
   945  	idx := -1
   946  	for i := 0; i < len(srv.LocalValidators); i++ {
   947  		if validator == srv.LocalValidators[i] {
   948  			idx = i
   949  			break
   950  		}
   951  	}
   952  
   953  	if idx < 0 {
   954  		return
   955  	}
   956  
   957  	srv.LocalValidators = append(srv.LocalValidators[:idx], srv.LocalValidators[idx+1:]...)
   958  
   959  	srv.broadcastRemoveValidatorNodeInfo(&P2PValidatorNodeInfo{
   960  		Node:      *srv.Self(),
   961  		TimeStamp: time.Now(),
   962  		Validator: validator,
   963  		Original:  true,
   964  	}, nil)
   965  }
   966  
   967  func (srv *Server) validatorAdd(valNodeInfo P2PValidatorNodeInfo, peers []*Peer, dialstate dialer) error {
   968  
   969  	log.Debug("validatorAdd")
   970  
   971  	validator := valNodeInfo.Validator
   972  
   973  	if srv.Self().ID == valNodeInfo.Node.ID {
   974  		return nil
   975  	}
   976  
   977  	if nodeInfo, ok := srv.Validators[validator]; ok {
   978  		con1 := valNodeInfo.Node.ID == nodeInfo.Node.ID
   979  		con2 := valNodeInfo.Node.IP.String() == nodeInfo.Node.IP.String()
   980  		log.Debugf("con1: %v, con2: %v", con1, con2)
   981  		if con1 && con2 {
   982  			log.Debug("validator found, not add")
   983  			return nil
   984  		}
   985  	}
   986  
   987  	log.Debug("validator not found")
   988  	srv.Validators[validator] = &valNodeInfo
   989  	inSameChain := false
   990  
   991  	for i := 0; i < len(srv.LocalValidators); i++ {
   992  		if validator.ChainId == srv.LocalValidators[i].ChainId {
   993  			inSameChain = true
   994  			break
   995  		}
   996  	}
   997  
   998  	notPeer := true
   999  	for _, p := range peers {
  1000  		if p.ID() == valNodeInfo.Node.ID {
  1001  			notPeer = false
  1002  			break
  1003  		}
  1004  	}
  1005  
  1006  	if inSameChain && notPeer {
  1007  		dialstate.addStatic(&valNodeInfo.Node)
  1008  	}
  1009  
  1010  	return nil
  1011  }
  1012  
  1013  func (srv *Server) validatorRemove(valNodeInfo P2PValidatorNodeInfo, peers []*Peer, dialstate dialer) error {
  1014  
  1015  	log.Debug("validatorRemove")
  1016  
  1017  	if srv.Self().ID == valNodeInfo.Node.ID {
  1018  		return nil
  1019  	}
  1020  
  1021  	validator := valNodeInfo.Validator
  1022  
  1023  	if _, ok := srv.Validators[validator]; !ok {
  1024  		return nil
  1025  	}
  1026  
  1027  	delete(srv.Validators, validator)
  1028  
  1029  	inSameChain := 0
  1030  
  1031  	for i := 0; i < len(srv.LocalValidators); i++ {
  1032  		if validator.ChainId == srv.LocalValidators[i].ChainId {
  1033  			inSameChain++
  1034  		}
  1035  	}
  1036  
  1037  	shouldRemove := false
  1038  
  1039  	if inSameChain == 1 {
  1040  		for _, p := range peers {
  1041  			if p.ID() == valNodeInfo.Node.ID {
  1042  
  1043  				shouldRemove = true
  1044  				break
  1045  			}
  1046  		}
  1047  	}
  1048  
  1049  	if shouldRemove {
  1050  		dialstate.removeStatic(&valNodeInfo.Node)
  1051  	}
  1052  
  1053  	return nil
  1054  }
  1055  
  1056  func (srv *Server) validatorAddPeer(peer *Peer) error {
  1057  
  1058  	log.Debug("validatorAddPeer")
  1059  
  1060  	sendList := make([]*NodeInfoToSend, 0)
  1061  
  1062  	var err error = nil
  1063  	for _, validatorNodeInfo := range srv.Validators {
  1064  
  1065  		if peer.ID() == validatorNodeInfo.Node.ID {
  1066  
  1067  			validatorNodeInfo.Node.IP = peer.RemoteAddr().(*net.TCPAddr).IP
  1068  			continue
  1069  		}
  1070  
  1071  		sendList = append(sendList, &NodeInfoToSend{
  1072  			valNodeInfo: validatorNodeInfo,
  1073  			action:      RefreshValidatorNodeInfoMsg,
  1074  			p:           peer,
  1075  		})
  1076  	}
  1077  
  1078  	node := *srv.Self()
  1079  	for i := 0; i < len(srv.LocalValidators); i++ {
  1080  
  1081  		sendList = append(sendList, &NodeInfoToSend{
  1082  			valNodeInfo: &P2PValidatorNodeInfo{
  1083  				Node:      node,
  1084  				TimeStamp: time.Now(),
  1085  				Validator: srv.LocalValidators[i],
  1086  				Original:  true,
  1087  			},
  1088  			action: RefreshValidatorNodeInfoMsg,
  1089  			p:      peer,
  1090  		})
  1091  	}
  1092  
  1093  	srv.addNodeInfoToSend(sendList)
  1094  
  1095  	return err
  1096  }
  1097  
  1098  func (srv *Server) validatorDelPeer(nodeId discover.NodeID) error {
  1099  
  1100  	log.Debug("validatorDelPeer")
  1101  
  1102  	srv.nodeInfoLock.Lock()
  1103  	defer srv.nodeInfoLock.Unlock()
  1104  
  1105  	tailIndex := 0
  1106  	for i := 0; i < len(srv.nodeInfoList); i++ {
  1107  
  1108  		nodeInfo := srv.nodeInfoList[i]
  1109  		if nodeInfo.valNodeInfo.Node.ID != nodeId {
  1110  			if i != tailIndex {
  1111  				srv.nodeInfoList[tailIndex] = nodeInfo
  1112  			}
  1113  			tailIndex++
  1114  		}
  1115  	}
  1116  
  1117  	removedCount := len(srv.nodeInfoList) - 1 - tailIndex
  1118  
  1119  	srv.nodeInfoList = srv.nodeInfoList[:tailIndex]
  1120  
  1121  	log.Debugf("removed %v node info to send to %v", removedCount, nodeId)
  1122  
  1123  	return nil
  1124  }
  1125  
  1126  func (srv *Server) broadcastRefreshValidatorNodeInfo(data *P2PValidatorNodeInfo, peers []*Peer) {
  1127  
  1128  	log.Debug("broadcastRefreshValidatorNodeInfo")
  1129  	if peers == nil {
  1130  		peers = srv.Peers()
  1131  	}
  1132  
  1133  	sendList := make([]*NodeInfoToSend, 0)
  1134  	for _, p := range peers {
  1135  
  1136  		sendList = append(sendList, &NodeInfoToSend{
  1137  			valNodeInfo: data,
  1138  			action:      RefreshValidatorNodeInfoMsg,
  1139  			p:           p,
  1140  		})
  1141  
  1142  	}
  1143  
  1144  	srv.addNodeInfoToSend(sendList)
  1145  }
  1146  
  1147  func (srv *Server) broadcastRemoveValidatorNodeInfo(data interface{}, peers []*Peer) {
  1148  
  1149  	log.Debug("broadcastRemoveValidatorNodeInfo")
  1150  	if peers == nil {
  1151  		peers = srv.Peers()
  1152  	}
  1153  
  1154  	sendList := make([]*NodeInfoToSend, 0)
  1155  	for _, p := range peers {
  1156  
  1157  		sendList = append(sendList, &NodeInfoToSend{
  1158  			valNodeInfo: data.(*P2PValidatorNodeInfo),
  1159  			action:      RemoveValidatorNodeInfoMsg,
  1160  			p:           p,
  1161  		})
  1162  
  1163  	}
  1164  
  1165  	srv.addNodeInfoToSend(sendList)
  1166  }
  1167  
  1168  func (srv *Server) addNodeInfoToSend(sendList []*NodeInfoToSend) {
  1169  
  1170  	srv.nodeInfoLock.Lock()
  1171  	defer srv.nodeInfoLock.Unlock()
  1172  
  1173  	srv.nodeInfoList = append(srv.nodeInfoList, sendList...)
  1174  }
  1175  
  1176  func (srv *Server) sendValidatorNodeInfoMessages() {
  1177  
  1178  	sleepDuration := 100 * time.Millisecond
  1179  
  1180  	for srv.running {
  1181  
  1182  		if len(srv.nodeInfoList) > 0 {
  1183  			srv.nodeInfoLock.Lock()
  1184  
  1185  			nodeInfo := srv.nodeInfoList[0]
  1186  			srv.nodeInfoList = srv.nodeInfoList[1:]
  1187  
  1188  			srv.nodeInfoLock.Unlock()
  1189  
  1190  			if nodeInfo != nil &&
  1191  				nodeInfo.valNodeInfo != nil &&
  1192  				nodeInfo.p != nil &&
  1193  				nodeInfo.p.rw != nil &&
  1194  				nodeInfo.p.rw.fd != nil {
  1195  
  1196  				Send(nodeInfo.p.rw, nodeInfo.action, nodeInfo.valNodeInfo)
  1197  
  1198  				log.Debugf("send node info (%v, %v) to %v",
  1199  					nodeInfo.valNodeInfo.Validator.Address.String(), nodeInfo.valNodeInfo.Node.ID,
  1200  					nodeInfo.p.ID())
  1201  			}
  1202  		}
  1203  
  1204  		time.Sleep(sleepDuration)
  1205  	}
  1206  }