github.com/intfoundation/intchain@v0.0.0-20220727031208-4316ad31ca73/p2p/server.go (about)

     1  // Copyright 2014 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  // Package p2p implements the INT Chain p2p network protocols.
    18  package p2p
    19  
    20  import (
    21  	"crypto/ecdsa"
    22  	"errors"
    23  	"fmt"
    24  	"net"
    25  	"sync"
    26  	"time"
    27  
    28  	"github.com/intfoundation/intchain/common"
    29  	"github.com/intfoundation/intchain/common/mclock"
    30  	"github.com/intfoundation/intchain/event"
    31  	"github.com/intfoundation/intchain/log"
    32  	"github.com/intfoundation/intchain/p2p/discover"
    33  	"github.com/intfoundation/intchain/p2p/discv5"
    34  	"github.com/intfoundation/intchain/p2p/nat"
    35  	"github.com/intfoundation/intchain/p2p/netutil"
    36  	"github.com/intfoundation/intchain/rlp"
    37  )
    38  
    39  const (
    40  	defaultDialTimeout = 15 * time.Second
    41  
    42  	// Connectivity defaults.
    43  	maxActiveDialTasks     = 16
    44  	defaultMaxPendingPeers = 50
    45  	defaultDialRatio       = 3
    46  
    47  	// Maximum time allowed for reading a complete message.
    48  	// This is effectively the amount of time a connection can be idle.
    49  	frameReadTimeout = 30 * time.Second
    50  
    51  	// Maximum amount of time allowed for writing a complete message.
    52  	frameWriteTimeout = 20 * time.Second
    53  )
    54  
    55  var errServerStopped = errors.New("server stopped")
    56  
    57  // Config holds Server options.
    58  type Config struct {
    59  	// This field must be set to a valid secp256k1 private key.
    60  	PrivateKey *ecdsa.PrivateKey `toml:"-"`
    61  
    62  	// MaxPeers is the maximum number of peers that can be
    63  	// connected. It must be greater than zero.
    64  	MaxPeers int
    65  
    66  	// MaxPendingPeers is the maximum number of peers that can be pending in the
    67  	// handshake phase, counted separately for inbound and outbound connections.
    68  	// Zero defaults to preset values.
    69  	MaxPendingPeers int `toml:",omitempty"`
    70  
    71  	// DialRatio controls the ratio of inbound to dialed connections.
    72  	// Example: a DialRatio of 2 allows 1/2 of connections to be dialed.
    73  	// Setting DialRatio to zero defaults it to 3.
    74  	DialRatio int `toml:",omitempty"`
    75  
    76  	// NoDiscovery can be used to disable the peer discovery mechanism.
    77  	// Disabling is useful for protocol debugging (manual topology).
    78  	NoDiscovery bool
    79  
    80  	// DiscoveryV5 specifies whether the the new topic-discovery based V5 discovery
    81  	// protocol should be started or not.
    82  	DiscoveryV5 bool `toml:",omitempty"`
    83  
    84  	// Name sets the node name of this server.
    85  	// Use common.MakeName to create a name that follows existing conventions.
    86  	Name string `toml:"-"`
    87  
    88  	// BootstrapNodes are used to establish connectivity
    89  	// with the rest of the network.
    90  	BootstrapNodes []*discover.Node
    91  
    92  	// BootstrapNodesV5 are used to establish connectivity
    93  	// with the rest of the network using the V5 discovery
    94  	// protocol.
    95  	BootstrapNodesV5 []*discv5.Node `toml:",omitempty"`
    96  
    97  	// Static nodes are used as pre-configured connections which are always
    98  	// maintained and re-connected on disconnects.
    99  	StaticNodes []*discover.Node
   100  
   101  	// Trusted nodes are used as pre-configured connections which are always
   102  	// allowed to connect, even above the peer limit.
   103  	TrustedNodes []*discover.Node
   104  
   105  	// Validators that this node acts as
   106  	LocalValidators []P2PValidator
   107  
   108  	// Validators set in all chains
   109  	// Should find nodes for all of them to support bls transport in IPBFT module
   110  	Validators map[P2PValidator]*P2PValidatorNodeInfo
   111  
   112  	// Connectivity can be restricted to certain IP networks.
   113  	// If this option is set to a non-nil value, only hosts which match one of the
   114  	// IP networks contained in the list are considered.
   115  	NetRestrict *netutil.Netlist `toml:",omitempty"`
   116  
   117  	// NodeDatabase is the path to the database containing the previously seen
   118  	// live nodes in the network.
   119  	NodeDatabase string `toml:",omitempty"`
   120  
   121  	// Protocols should contain the protocols supported
   122  	// by the server. Matching protocols are launched for
   123  	// each peer.
   124  	Protocols []Protocol `toml:"-"`
   125  
   126  	// If ListenAddr is set to a non-nil address, the server
   127  	// will listen for incoming connections.
   128  	//
   129  	// If the port is zero, the operating system will pick a port. The
   130  	// ListenAddr field will be updated with the actual address when
   131  	// the server is started.
   132  	ListenAddr string
   133  
   134  	// If set to a non-nil value, the given NAT port mapper
   135  	// is used to make the listening port available to the
   136  	// Internet.
   137  	NAT nat.Interface `toml:",omitempty"`
   138  
   139  	// If Dialer is set to a non-nil value, the given Dialer
   140  	// is used to dial outbound peer connections.
   141  	Dialer NodeDialer `toml:"-"`
   142  
   143  	// If NoDial is true, the server will not dial any peers.
   144  	NoDial bool `toml:",omitempty"`
   145  
   146  	// If EnableMsgEvents is set then the server will emit PeerEvents
   147  	// whenever a message is sent to or received from a peer
   148  	EnableMsgEvents bool
   149  
   150  	// Logger is a custom logger to use with the p2p.Server.
   151  	Logger log.Logger `toml:",omitempty"`
   152  }
   153  
   154  type NodeInfoToSend struct {
   155  	valNodeInfo *P2PValidatorNodeInfo
   156  	action      uint64
   157  	p           *Peer
   158  }
   159  
   160  // Server manages all peer connections.
   161  type Server struct {
   162  	// Config fields may not be modified while the server is running.
   163  	Config
   164  
   165  	// Hooks for testing. These are useful because we can inhibit
   166  	// the whole protocol stack.
   167  	newTransport func(net.Conn) transport
   168  	newPeerHook  func(*Peer)
   169  
   170  	lock    sync.Mutex // protects running
   171  	running bool
   172  
   173  	ntab         discoverTable
   174  	listener     net.Listener
   175  	ourHandshake *protoHandshake
   176  	lastLookup   time.Time
   177  	DiscV5       *discv5.Network
   178  
   179  	// These are for Peers, PeerCount (and nothing else).
   180  	peerOp     chan peerOpFunc
   181  	peerOpDone chan struct{}
   182  
   183  	quit          chan struct{}
   184  	addstatic     chan *discover.Node
   185  	removestatic  chan *discover.Node
   186  	posthandshake chan *conn
   187  	addpeer       chan *conn
   188  	delpeer       chan peerDrop
   189  	loopWG        sync.WaitGroup // loop, listenLoop
   190  	peerFeed      event.Feed
   191  	log           log.Logger
   192  
   193  	events    chan *PeerEvent
   194  	eventsSub event.Subscription
   195  
   196  	nodeInfoLock sync.Mutex // protects running
   197  	nodeInfoList []*NodeInfoToSend
   198  }
   199  
   200  type peerOpFunc func(map[discover.NodeID]*Peer)
   201  
   202  type peerDrop struct {
   203  	*Peer
   204  	err       error
   205  	requested bool // true if signaled by the peer
   206  }
   207  
   208  type connFlag int
   209  
   210  const (
   211  	dynDialedConn connFlag = 1 << iota
   212  	staticDialedConn
   213  	inboundConn
   214  	trustedConn
   215  )
   216  
   217  // conn wraps a network connection with information gathered
   218  // during the two handshakes.
   219  type conn struct {
   220  	fd net.Conn
   221  	transport
   222  	flags connFlag
   223  	cont  chan error      // The run loop uses cont to signal errors to SetupConn.
   224  	id    discover.NodeID // valid after the encryption handshake
   225  	caps  []Cap           // valid after the protocol handshake
   226  	name  string          // valid after the protocol handshake
   227  }
   228  
   229  type transport interface {
   230  	// The two handshakes.
   231  	doEncHandshake(prv *ecdsa.PrivateKey, dialDest *discover.Node) (discover.NodeID, error)
   232  	doProtoHandshake(our *protoHandshake) (*protoHandshake, error)
   233  	// The MsgReadWriter can only be used after the encryption
   234  	// handshake has completed. The code uses conn.id to track this
   235  	// by setting it to a non-nil value after the encryption handshake.
   236  	MsgReadWriter
   237  	// transports must provide Close because we use MsgPipe in some of
   238  	// the tests. Closing the actual network connection doesn't do
   239  	// anything in those tests because NsgPipe doesn't use it.
   240  	close(err error)
   241  }
   242  
   243  func (c *conn) String() string {
   244  	s := c.flags.String()
   245  	if (c.id != discover.NodeID{}) {
   246  		s += " " + c.id.String()
   247  	}
   248  	s += " " + c.fd.RemoteAddr().String()
   249  	return s
   250  }
   251  
   252  func (f connFlag) String() string {
   253  	s := ""
   254  	if f&trustedConn != 0 {
   255  		s += "-trusted"
   256  	}
   257  	if f&dynDialedConn != 0 {
   258  		s += "-dyndial"
   259  	}
   260  	if f&staticDialedConn != 0 {
   261  		s += "-staticdial"
   262  	}
   263  	if f&inboundConn != 0 {
   264  		s += "-inbound"
   265  	}
   266  	if s != "" {
   267  		s = s[1:]
   268  	}
   269  	return s
   270  }
   271  
   272  func (c *conn) is(f connFlag) bool {
   273  	return c.flags&f != 0
   274  }
   275  
   276  // Peers returns all connected peers.
   277  func (srv *Server) Peers() []*Peer {
   278  	var ps []*Peer
   279  	select {
   280  	// Note: We'd love to put this function into a variable but
   281  	// that seems to cause a weird compiler error in some
   282  	// environments.
   283  	case srv.peerOp <- func(peers map[discover.NodeID]*Peer) {
   284  		for _, p := range peers {
   285  			ps = append(ps, p)
   286  		}
   287  	}:
   288  		<-srv.peerOpDone
   289  	case <-srv.quit:
   290  	}
   291  	return ps
   292  }
   293  
   294  // PeerCount returns the number of connected peers.
   295  func (srv *Server) PeerCount() int {
   296  	var count int
   297  	select {
   298  	case srv.peerOp <- func(ps map[discover.NodeID]*Peer) { count = len(ps) }:
   299  		<-srv.peerOpDone
   300  	case <-srv.quit:
   301  	}
   302  	return count
   303  }
   304  
   305  // AddPeer connects to the given node and maintains the connection until the
   306  // server is shut down. If the connection fails for any reason, the server will
   307  // attempt to reconnect the peer.
   308  func (srv *Server) AddPeer(node *discover.Node) {
   309  	select {
   310  	case srv.addstatic <- node:
   311  	case <-srv.quit:
   312  	}
   313  }
   314  
   315  // RemovePeer disconnects from the given node
   316  func (srv *Server) RemovePeer(node *discover.Node) {
   317  	select {
   318  	case srv.removestatic <- node:
   319  	case <-srv.quit:
   320  	}
   321  }
   322  
   323  // SubscribePeers subscribes the given channel to peer events
   324  func (srv *Server) SubscribeEvents(ch chan *PeerEvent) event.Subscription {
   325  	return srv.peerFeed.Subscribe(ch)
   326  }
   327  
   328  // Self returns the local node's endpoint information.
   329  func (srv *Server) Self() *discover.Node {
   330  	srv.lock.Lock()
   331  	defer srv.lock.Unlock()
   332  
   333  	if !srv.running {
   334  		return &discover.Node{IP: net.ParseIP("0.0.0.0")}
   335  	}
   336  	return srv.makeSelf(srv.listener, srv.ntab)
   337  }
   338  
   339  func (srv *Server) makeSelf(listener net.Listener, ntab discoverTable) *discover.Node {
   340  	// If the server's not running, return an empty node.
   341  	// If the node is running but discovery is off, manually assemble the node infos.
   342  	if ntab == nil {
   343  		// Inbound connections disabled, use zero address.
   344  		if listener == nil {
   345  			return &discover.Node{IP: net.ParseIP("0.0.0.0"), ID: discover.PubkeyID(&srv.PrivateKey.PublicKey)}
   346  		}
   347  		// Otherwise inject the listener address too
   348  		addr := listener.Addr().(*net.TCPAddr)
   349  		return &discover.Node{
   350  			ID:  discover.PubkeyID(&srv.PrivateKey.PublicKey),
   351  			IP:  addr.IP,
   352  			TCP: uint16(addr.Port),
   353  		}
   354  	}
   355  	// Otherwise return the discovery node.
   356  	return ntab.Self()
   357  }
   358  
   359  // Stop terminates the server and all active peer connections.
   360  // It blocks until all active connections have been closed.
   361  func (srv *Server) Stop() {
   362  	srv.lock.Lock()
   363  	if !srv.running {
   364  		srv.lock.Unlock()
   365  		return
   366  	}
   367  	srv.eventsSub.Unsubscribe()
   368  	srv.running = false
   369  	if srv.listener != nil {
   370  		// this unblocks listener Accept
   371  		srv.listener.Close()
   372  	}
   373  	close(srv.quit)
   374  	srv.lock.Unlock()
   375  	srv.loopWG.Wait()
   376  }
   377  
   378  // sharedUDPConn implements a shared connection. Write sends messages to the underlying connection while read returns
   379  // messages that were found unprocessable and sent to the unhandled channel by the primary listener.
   380  type sharedUDPConn struct {
   381  	*net.UDPConn
   382  	unhandled chan discover.ReadPacket
   383  }
   384  
   385  // ReadFromUDP implements discv5.conn
   386  func (s *sharedUDPConn) ReadFromUDP(b []byte) (n int, addr *net.UDPAddr, err error) {
   387  	packet, ok := <-s.unhandled
   388  	if !ok {
   389  		return 0, nil, fmt.Errorf("Connection was closed")
   390  	}
   391  	l := len(packet.Data)
   392  	if l > len(b) {
   393  		l = len(b)
   394  	}
   395  	copy(b[:l], packet.Data[:l])
   396  	return l, packet.Addr, nil
   397  }
   398  
   399  // Close implements discv5.conn
   400  func (s *sharedUDPConn) Close() error {
   401  	return nil
   402  }
   403  
   404  // Start starts running the server.
   405  // Servers can not be re-used after stopping.
   406  func (srv *Server) Start() (err error) {
   407  	srv.lock.Lock()
   408  	defer srv.lock.Unlock()
   409  	if srv.running {
   410  		return errors.New("server already running")
   411  	}
   412  	srv.running = true
   413  	srv.log = srv.Config.Logger
   414  	if srv.log == nil {
   415  		srv.log = log.New()
   416  	}
   417  	srv.log.Info("Starting P2P networking")
   418  
   419  	// static fields
   420  	if srv.PrivateKey == nil {
   421  		return fmt.Errorf("Server.PrivateKey must be set to a non-nil key")
   422  	}
   423  	if srv.newTransport == nil {
   424  		srv.newTransport = newRLPX
   425  	}
   426  	if srv.Dialer == nil {
   427  		srv.Dialer = TCPDialer{&net.Dialer{Timeout: defaultDialTimeout}}
   428  	}
   429  	srv.quit = make(chan struct{})
   430  	srv.addpeer = make(chan *conn)
   431  	srv.delpeer = make(chan peerDrop)
   432  	srv.posthandshake = make(chan *conn)
   433  	srv.addstatic = make(chan *discover.Node)
   434  	srv.removestatic = make(chan *discover.Node)
   435  	srv.peerOp = make(chan peerOpFunc)
   436  	srv.peerOpDone = make(chan struct{})
   437  	srv.events = make(chan *PeerEvent)
   438  	srv.eventsSub = srv.SubscribeEvents(srv.events)
   439  
   440  	srv.nodeInfoList = make([]*NodeInfoToSend, 0)
   441  
   442  	var (
   443  		conn      *net.UDPConn
   444  		sconn     *sharedUDPConn
   445  		realaddr  *net.UDPAddr
   446  		unhandled chan discover.ReadPacket
   447  	)
   448  
   449  	if !srv.NoDiscovery || srv.DiscoveryV5 {
   450  		addr, err := net.ResolveUDPAddr("udp", srv.ListenAddr)
   451  		if err != nil {
   452  			return err
   453  		}
   454  		conn, err = net.ListenUDP("udp", addr)
   455  		if err != nil {
   456  			return err
   457  		}
   458  		realaddr = conn.LocalAddr().(*net.UDPAddr)
   459  		if srv.NAT != nil {
   460  			if !realaddr.IP.IsLoopback() {
   461  				go nat.Map(srv.NAT, srv.quit, "udp", realaddr.Port, realaddr.Port, "ethereum discovery")
   462  			}
   463  			// TODO: react to external IP changes over time.
   464  			if ext, err := srv.NAT.ExternalIP(); err == nil {
   465  				realaddr = &net.UDPAddr{IP: ext, Port: realaddr.Port}
   466  			}
   467  		}
   468  	}
   469  
   470  	if !srv.NoDiscovery && srv.DiscoveryV5 {
   471  		unhandled = make(chan discover.ReadPacket, 100)
   472  		sconn = &sharedUDPConn{conn, unhandled}
   473  	}
   474  
   475  	// node table
   476  	if !srv.NoDiscovery {
   477  		cfg := discover.Config{
   478  			PrivateKey:   srv.PrivateKey,
   479  			AnnounceAddr: realaddr,
   480  			NodeDBPath:   srv.NodeDatabase,
   481  			NetRestrict:  srv.NetRestrict,
   482  			Bootnodes:    srv.BootstrapNodes,
   483  			Unhandled:    unhandled,
   484  		}
   485  		ntab, err := discover.ListenUDP(conn, cfg)
   486  		if err != nil {
   487  			return err
   488  		}
   489  		srv.ntab = ntab
   490  	}
   491  
   492  	if srv.DiscoveryV5 {
   493  		var (
   494  			ntab *discv5.Network
   495  			err  error
   496  		)
   497  		if sconn != nil {
   498  			ntab, err = discv5.ListenUDP(srv.PrivateKey, sconn, realaddr, "", srv.NetRestrict) //srv.NodeDatabase)
   499  		} else {
   500  			ntab, err = discv5.ListenUDP(srv.PrivateKey, conn, realaddr, "", srv.NetRestrict) //srv.NodeDatabase)
   501  		}
   502  		if err != nil {
   503  			return err
   504  		}
   505  		if err := ntab.SetFallbackNodes(srv.BootstrapNodesV5); err != nil {
   506  			return err
   507  		}
   508  		srv.DiscV5 = ntab
   509  	}
   510  
   511  	dynPeers := srv.maxDialedConns()
   512  	dialer := newDialState(srv.StaticNodes, srv.BootstrapNodes, srv.ntab, dynPeers, srv.NetRestrict)
   513  
   514  	// handshake
   515  	srv.ourHandshake = &protoHandshake{Version: baseProtocolVersion, Name: srv.Name, ID: discover.PubkeyID(&srv.PrivateKey.PublicKey)}
   516  	for _, p := range srv.Protocols {
   517  		srv.ourHandshake.Caps = append(srv.ourHandshake.Caps, p.cap())
   518  	}
   519  	// listen/dial
   520  	if srv.ListenAddr != "" {
   521  		if err := srv.startListening(); err != nil {
   522  			return err
   523  		}
   524  	}
   525  	if srv.NoDial && srv.ListenAddr == "" {
   526  		srv.log.Warn("P2P server will be useless, neither dialing nor listening")
   527  	}
   528  
   529  	srv.loopWG.Add(1)
   530  	go srv.run(dialer)
   531  	srv.running = true
   532  
   533  	go srv.sendValidatorNodeInfoMessages()
   534  
   535  	return nil
   536  }
   537  
   538  // AddHandshakeCaps Add the Child Protocol Caps after create the child chain and before launch it
   539  func (srv *Server) AddChildProtocolCaps(childProtocols []Protocol) {
   540  	for _, p := range childProtocols {
   541  		srv.ourHandshake.Caps = append(srv.ourHandshake.Caps, p.cap())
   542  	}
   543  }
   544  
   545  func (srv *Server) startListening() error {
   546  	// Launch the TCP listener.
   547  	listener, err := net.Listen("tcp", srv.ListenAddr)
   548  	if err != nil {
   549  		return err
   550  	}
   551  	laddr := listener.Addr().(*net.TCPAddr)
   552  	srv.ListenAddr = laddr.String()
   553  	srv.listener = listener
   554  	srv.loopWG.Add(1)
   555  	go srv.listenLoop()
   556  	// Map the TCP listening port if NAT is configured.
   557  	if !laddr.IP.IsLoopback() && srv.NAT != nil {
   558  		srv.loopWG.Add(1)
   559  		go func() {
   560  			nat.Map(srv.NAT, srv.quit, "tcp", laddr.Port, laddr.Port, "ethereum p2p")
   561  			srv.loopWG.Done()
   562  		}()
   563  	}
   564  	return nil
   565  }
   566  
   567  type dialer interface {
   568  	newTasks(running int, peers map[discover.NodeID]*Peer, now time.Time) []task
   569  	taskDone(task, time.Time)
   570  	addStatic(*discover.Node)
   571  	removeStatic(*discover.Node)
   572  }
   573  
   574  func (srv *Server) run(dialstate dialer) {
   575  	defer srv.loopWG.Done()
   576  	var (
   577  		peers        = make(map[discover.NodeID]*Peer)
   578  		inboundCount = 0
   579  		trusted      = make(map[discover.NodeID]bool, len(srv.TrustedNodes))
   580  		taskdone     = make(chan task, maxActiveDialTasks)
   581  		runningTasks []task
   582  		queuedTasks  []task // tasks that can't run yet
   583  	)
   584  	// Put trusted nodes into a map to speed up checks.
   585  	// Trusted peers are loaded on startup and cannot be
   586  	// modified while the server is running.
   587  	for _, n := range srv.TrustedNodes {
   588  		trusted[n.ID] = true
   589  	}
   590  
   591  	// removes t from runningTasks
   592  	delTask := func(t task) {
   593  		for i := range runningTasks {
   594  			if runningTasks[i] == t {
   595  				runningTasks = append(runningTasks[:i], runningTasks[i+1:]...)
   596  				break
   597  			}
   598  		}
   599  	}
   600  	// starts until max number of active tasks is satisfied
   601  	startTasks := func(ts []task) (rest []task) {
   602  		i := 0
   603  		for ; len(runningTasks) < maxActiveDialTasks && i < len(ts); i++ {
   604  			t := ts[i]
   605  			srv.log.Trace("New dial task", "task", t)
   606  			go func() { t.Do(srv); taskdone <- t }()
   607  			runningTasks = append(runningTasks, t)
   608  		}
   609  		return ts[i:]
   610  	}
   611  	scheduleTasks := func() {
   612  		// Start from queue first.
   613  		queuedTasks = append(queuedTasks[:0], startTasks(queuedTasks)...)
   614  		// Query dialer for new tasks and start as many as possible now.
   615  		if len(runningTasks) < maxActiveDialTasks {
   616  			nt := dialstate.newTasks(len(runningTasks)+len(queuedTasks), peers, time.Now())
   617  			queuedTasks = append(queuedTasks, startTasks(nt)...)
   618  		}
   619  	}
   620  
   621  running:
   622  	for {
   623  		scheduleTasks()
   624  
   625  		select {
   626  		case <-srv.quit:
   627  			// The server was stopped. Run the cleanup logic.
   628  			break running
   629  		case n := <-srv.addstatic:
   630  			// This channel is used by AddPeer to add to the
   631  			// ephemeral static peer list. Add it to the dialer,
   632  			// it will keep the node connected.
   633  			srv.log.Debug("Adding static node", "node", n)
   634  			dialstate.addStatic(n)
   635  		case n := <-srv.removestatic:
   636  			// This channel is used by RemovePeer to send a
   637  			// disconnect request to a peer and begin the
   638  			// stop keeping the node connected
   639  			srv.log.Debug("Removing static node", "node", n)
   640  			dialstate.removeStatic(n)
   641  			if p, ok := peers[n.ID]; ok {
   642  				p.Disconnect(DiscRequested)
   643  			}
   644  		case op := <-srv.peerOp:
   645  			// This channel is used by Peers and PeerCount.
   646  			op(peers)
   647  			srv.peerOpDone <- struct{}{}
   648  		case t := <-taskdone:
   649  			// A task got done. Tell dialstate about it so it
   650  			// can update its state and remove it from the active
   651  			// tasks list.
   652  			srv.log.Trace("Dial task done", "task", t)
   653  			dialstate.taskDone(t, time.Now())
   654  			delTask(t)
   655  		case c := <-srv.posthandshake:
   656  			// A connection has passed the encryption handshake so
   657  			// the remote identity is known (but hasn't been verified yet).
   658  			if trusted[c.id] {
   659  				// Ensure that the trusted flag is set before checking against MaxPeers.
   660  				c.flags |= trustedConn
   661  			}
   662  			// TODO: track in-progress inbound node IDs (pre-Peer) to avoid dialing them.
   663  			select {
   664  			case c.cont <- srv.encHandshakeChecks(peers, inboundCount, c):
   665  			case <-srv.quit:
   666  				break running
   667  			}
   668  		case c := <-srv.addpeer:
   669  			// At this point the connection is past the protocol handshake.
   670  			// Its capabilities are known and the remote identity is verified.
   671  			err := srv.protoHandshakeChecks(peers, inboundCount, c)
   672  			if err == nil {
   673  				// The handshakes are done and it passed all checks.
   674  				p := newPeer(c, srv.Protocols)
   675  				// If message events are enabled, pass the peerFeed
   676  				// to the peer
   677  				if srv.EnableMsgEvents {
   678  					p.events = &srv.peerFeed
   679  				}
   680  				name := truncateName(c.name)
   681  				srv.log.Debug("Adding p2p peer", "name", name, "addr", c.fd.RemoteAddr(), "peers", len(peers)+1)
   682  				go srv.runPeer(p)
   683  				peers[c.id] = p
   684  				if p.Inbound() {
   685  					inboundCount++
   686  				}
   687  
   688  				srv.validatorAddPeer(p)
   689  			}
   690  			// The dialer logic relies on the assumption that
   691  			// dial tasks complete after the peer has been added or
   692  			// discarded. Unblock the task last.
   693  			select {
   694  			case c.cont <- err:
   695  			case <-srv.quit:
   696  				break running
   697  			}
   698  		case pd := <-srv.delpeer:
   699  			// A peer disconnected.
   700  			d := common.PrettyDuration(mclock.Now() - pd.created)
   701  			pd.log.Debug("Removing p2p peer", "duration", d, "peers", len(peers)-1, "req", pd.requested, "err", pd.err)
   702  			delete(peers, pd.ID())
   703  			if pd.Inbound() {
   704  				inboundCount--
   705  			}
   706  			srv.validatorDelPeer(pd.ID())
   707  
   708  		case evt := <-srv.events:
   709  			log.Debugf("peer events received: %v", evt)
   710  			switch evt.Type {
   711  
   712  			case PeerEventTypeRefreshValidator:
   713  				var valNodeInfo P2PValidatorNodeInfo
   714  				if err := rlp.DecodeBytes([]byte(evt.Protocol), &valNodeInfo); err != nil {
   715  					log.Debugf("rlp decode valNodeInfo failed with %v", err)
   716  				}
   717  
   718  				peerArr := make([]*Peer, 0)
   719  				for _, p := range peers {
   720  					peerArr = append(peerArr, p)
   721  				}
   722  
   723  				if err := srv.validatorAdd(valNodeInfo, peerArr, dialstate); err != nil {
   724  					log.Debugf("add valNodeInfo to local failed with %v", err)
   725  				}
   726  
   727  				log.Debugf("Got refresh validation node infomation from validation %v", valNodeInfo.Validator.Address.String())
   728  
   729  			case PeerEventTypeRemoveValidator:
   730  				var valNodeInfo P2PValidatorNodeInfo
   731  				if err := rlp.DecodeBytes([]byte(evt.Protocol), &valNodeInfo); err != nil {
   732  					log.Debugf("rlp decode valNodeInfo failed with %v", err)
   733  				}
   734  
   735  				peerArr := make([]*Peer, 0)
   736  				for _, p := range peers {
   737  					peerArr = append(peerArr, p)
   738  				}
   739  
   740  				if err := srv.validatorRemove(valNodeInfo, peerArr, dialstate); err != nil {
   741  					log.Debugf("remove valNodeInfo from local failed with %v", err)
   742  				}
   743  
   744  				log.Debugf("Got remove validation node infomation from validation %v", valNodeInfo.Validator.Address.String())
   745  
   746  			}
   747  		}
   748  	}
   749  
   750  	srv.log.Trace("P2P networking is spinning down")
   751  
   752  	// Terminate discovery. If there is a running lookup it will terminate soon.
   753  	if srv.ntab != nil {
   754  		srv.ntab.Close()
   755  	}
   756  	if srv.DiscV5 != nil {
   757  		srv.DiscV5.Close()
   758  	}
   759  	// Disconnect all peers.
   760  	for _, p := range peers {
   761  		p.Disconnect(DiscQuitting)
   762  	}
   763  	// Wait for peers to shut down. Pending connections and tasks are
   764  	// not handled here and will terminate soon-ish because srv.quit
   765  	// is closed.
   766  	for len(peers) > 0 {
   767  		p := <-srv.delpeer
   768  		p.log.Trace("<-delpeer (spindown)", "remainingTasks", len(runningTasks))
   769  		delete(peers, p.ID())
   770  	}
   771  }
   772  
   773  func (srv *Server) protoHandshakeChecks(peers map[discover.NodeID]*Peer, inboundCount int, c *conn) error {
   774  	// Drop connections with no matching protocols.
   775  	if len(srv.Protocols) > 0 && countMatchingProtocols(srv.Protocols, c.caps) == 0 {
   776  		return DiscUselessPeer
   777  	}
   778  	// Repeat the encryption handshake checks because the
   779  	// peer set might have changed between the handshakes.
   780  	return srv.encHandshakeChecks(peers, inboundCount, c)
   781  }
   782  
   783  func (srv *Server) encHandshakeChecks(peers map[discover.NodeID]*Peer, inboundCount int, c *conn) error {
   784  	switch {
   785  	case !c.is(trustedConn|staticDialedConn) && len(peers) >= srv.MaxPeers:
   786  		return DiscTooManyPeers
   787  	case !c.is(trustedConn) && c.is(inboundConn) && inboundCount >= srv.maxInboundConns():
   788  		return DiscTooManyPeers
   789  	case peers[c.id] != nil:
   790  		return DiscAlreadyConnected
   791  	case c.id == srv.Self().ID:
   792  		return DiscSelf
   793  	default:
   794  		return nil
   795  	}
   796  }
   797  
   798  func (srv *Server) maxInboundConns() int {
   799  	return srv.MaxPeers - srv.maxDialedConns()
   800  }
   801  
   802  func (srv *Server) maxDialedConns() int {
   803  	if srv.NoDiscovery || srv.NoDial {
   804  		return 0
   805  	}
   806  	r := srv.DialRatio
   807  	if r == 0 {
   808  		r = defaultDialRatio
   809  	}
   810  	return srv.MaxPeers / r
   811  }
   812  
   813  type tempError interface {
   814  	Temporary() bool
   815  }
   816  
   817  // listenLoop runs in its own goroutine and accepts
   818  // inbound connections.
   819  func (srv *Server) listenLoop() {
   820  	defer srv.loopWG.Done()
   821  	srv.log.Info("RLPx listener up", "self", srv.makeSelf(srv.listener, srv.ntab))
   822  
   823  	tokens := defaultMaxPendingPeers
   824  	if srv.MaxPendingPeers > 0 {
   825  		tokens = srv.MaxPendingPeers
   826  	}
   827  	slots := make(chan struct{}, tokens)
   828  	for i := 0; i < tokens; i++ {
   829  		slots <- struct{}{}
   830  	}
   831  
   832  	for {
   833  		// Wait for a handshake slot before accepting.
   834  		<-slots
   835  
   836  		var (
   837  			fd  net.Conn
   838  			err error
   839  		)
   840  		for {
   841  			fd, err = srv.listener.Accept()
   842  			if tempErr, ok := err.(tempError); ok && tempErr.Temporary() {
   843  				srv.log.Debug("Temporary read error", "err", err)
   844  				continue
   845  			} else if err != nil {
   846  				srv.log.Debug("Read error", "err", err)
   847  				return
   848  			}
   849  			break
   850  		}
   851  
   852  		// Reject connections that do not match NetRestrict.
   853  		if srv.NetRestrict != nil {
   854  			if tcp, ok := fd.RemoteAddr().(*net.TCPAddr); ok && !srv.NetRestrict.Contains(tcp.IP) {
   855  				srv.log.Debug("Rejected conn (not whitelisted in NetRestrict)", "addr", fd.RemoteAddr())
   856  				fd.Close()
   857  				slots <- struct{}{}
   858  				continue
   859  			}
   860  		}
   861  
   862  		fd = newMeteredConn(fd, true)
   863  		srv.log.Trace("Accepted connection", "addr", fd.RemoteAddr())
   864  		go func() {
   865  			srv.SetupConn(fd, inboundConn, nil)
   866  			slots <- struct{}{}
   867  		}()
   868  	}
   869  }
   870  
   871  // SetupConn runs the handshakes and attempts to add the connection
   872  // as a peer. It returns when the connection has been added as a peer
   873  // or the handshakes have failed.
   874  func (srv *Server) SetupConn(fd net.Conn, flags connFlag, dialDest *discover.Node) error {
   875  	self := srv.Self()
   876  	if self == nil {
   877  		return errors.New("shutdown")
   878  	}
   879  	c := &conn{fd: fd, transport: srv.newTransport(fd), flags: flags, cont: make(chan error)}
   880  	err := srv.setupConn(c, flags, dialDest)
   881  	if err != nil {
   882  		c.close(err)
   883  		srv.log.Trace("Setting up connection failed", "id", c.id, "err", err)
   884  	}
   885  	return err
   886  }
   887  
   888  func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *discover.Node) error {
   889  	// Prevent leftover pending conns from entering the handshake.
   890  	srv.lock.Lock()
   891  	running := srv.running
   892  	srv.lock.Unlock()
   893  	if !running {
   894  		return errServerStopped
   895  	}
   896  	// Run the encryption handshake.
   897  	var err error
   898  	if c.id, err = c.doEncHandshake(srv.PrivateKey, dialDest); err != nil {
   899  		srv.log.Trace("Failed RLPx handshake", "addr", c.fd.RemoteAddr(), "conn", c.flags, "err", err)
   900  		return err
   901  	}
   902  	clog := srv.log.New("id", c.id, "addr", c.fd.RemoteAddr(), "conn", c.flags)
   903  	// For dialed connections, check that the remote public key matches.
   904  	if dialDest != nil && c.id != dialDest.ID {
   905  		clog.Trace("Dialed identity mismatch", "want", c, dialDest.ID)
   906  		return DiscUnexpectedIdentity
   907  	}
   908  	err = srv.checkpoint(c, srv.posthandshake)
   909  	if err != nil {
   910  		clog.Trace("Rejected peer before protocol handshake", "err", err)
   911  		return err
   912  	}
   913  	// Run the protocol handshake
   914  	phs, err := c.doProtoHandshake(srv.ourHandshake)
   915  	if err != nil {
   916  		clog.Trace("Failed proto handshake", "err", err)
   917  		return err
   918  	}
   919  	if phs.ID != c.id {
   920  		clog.Trace("Wrong devp2p handshake identity", "err", phs.ID)
   921  		return DiscUnexpectedIdentity
   922  	}
   923  	c.caps, c.name = phs.Caps, phs.Name
   924  	err = srv.checkpoint(c, srv.addpeer)
   925  	if err != nil {
   926  		clog.Trace("Rejected peer", "err", err)
   927  		return err
   928  	}
   929  	// If the checks completed successfully, runPeer has now been
   930  	// launched by run.
   931  	clog.Trace("connection set up", "inbound", dialDest == nil)
   932  	return nil
   933  }
   934  
   935  func truncateName(s string) string {
   936  	if len(s) > 20 {
   937  		return s[:20] + "..."
   938  	}
   939  	return s
   940  }
   941  
   942  // checkpoint sends the conn to run, which performs the
   943  // post-handshake checks for the stage (posthandshake, addpeer).
   944  func (srv *Server) checkpoint(c *conn, stage chan<- *conn) error {
   945  	select {
   946  	case stage <- c:
   947  	case <-srv.quit:
   948  		return errServerStopped
   949  	}
   950  	select {
   951  	case err := <-c.cont:
   952  		return err
   953  	case <-srv.quit:
   954  		return errServerStopped
   955  	}
   956  }
   957  
   958  // runPeer runs in its own goroutine for each peer.
   959  // it waits until the Peer logic returns and removes
   960  // the peer.
   961  func (srv *Server) runPeer(p *Peer) {
   962  	if srv.newPeerHook != nil {
   963  		srv.newPeerHook(p)
   964  	}
   965  
   966  	// broadcast peer add
   967  	srv.peerFeed.Send(&PeerEvent{
   968  		Type: PeerEventTypeAdd,
   969  		Peer: p.ID(),
   970  	})
   971  
   972  	// Set the server protocol, this should link with p2p server's protocol and auto-update if changed
   973  	p.srvProtocols = &srv.Protocols
   974  
   975  	// run the protocol
   976  	remoteRequested, err := p.run()
   977  
   978  	// broadcast peer drop
   979  	srv.peerFeed.Send(&PeerEvent{
   980  		Type:  PeerEventTypeDrop,
   981  		Peer:  p.ID(),
   982  		Error: err.Error(),
   983  	})
   984  
   985  	// Note: run waits for existing peers to be sent on srv.delpeer
   986  	// before returning, so this send should not select on srv.quit.
   987  	srv.delpeer <- peerDrop{p, err, remoteRequested}
   988  }
   989  
   990  // NodeInfo represents a short summary of the information known about the host.
   991  type NodeInfo struct {
   992  	ID    string `json:"id"`    // Unique node identifier (also the encryption key)
   993  	Name  string `json:"name"`  // Name of the node, including client type, version, OS, custom data
   994  	Enode string `json:"enode"` // Enode URL for adding this peer from remote peers
   995  	IP    string `json:"ip"`    // IP address of the node
   996  	Ports struct {
   997  		Discovery int `json:"discovery"` // UDP listening port for discovery protocol
   998  		Listener  int `json:"listener"`  // TCP listening port for RLPx
   999  	} `json:"ports"`
  1000  	ListenAddr string                 `json:"listenAddr"`
  1001  	Protocols  map[string]interface{} `json:"protocols"`
  1002  }
  1003  
  1004  // NodeInfo gathers and returns a collection of metadata known about the host.
  1005  func (srv *Server) NodeInfo() *NodeInfo {
  1006  	node := srv.Self()
  1007  
  1008  	// Gather and assemble the generic node infos
  1009  	info := &NodeInfo{
  1010  		Name:       srv.Name,
  1011  		Enode:      node.String(),
  1012  		ID:         node.ID.String(),
  1013  		IP:         node.IP.String(),
  1014  		ListenAddr: srv.ListenAddr,
  1015  		Protocols:  make(map[string]interface{}),
  1016  	}
  1017  	info.Ports.Discovery = int(node.UDP)
  1018  	info.Ports.Listener = int(node.TCP)
  1019  
  1020  	// Gather all the running protocol infos (only once per protocol type)
  1021  	for _, proto := range srv.Protocols {
  1022  		if _, ok := info.Protocols[proto.Name]; !ok {
  1023  			nodeInfo := interface{}("unknown")
  1024  			if query := proto.NodeInfo; query != nil {
  1025  				nodeInfo = proto.NodeInfo()
  1026  			}
  1027  			info.Protocols[proto.Name] = nodeInfo
  1028  		}
  1029  	}
  1030  	return info
  1031  }
  1032  
  1033  // PeersInfo returns an array of metadata objects describing connected peers.
  1034  func (srv *Server) PeersInfo() []*PeerInfo {
  1035  	// Gather all the generic and sub-protocol specific infos
  1036  	infos := make([]*PeerInfo, 0, srv.PeerCount())
  1037  	for _, peer := range srv.Peers() {
  1038  		if peer != nil {
  1039  			infos = append(infos, peer.Info())
  1040  		}
  1041  	}
  1042  	// Sort the result array alphabetically by node identifier
  1043  	for i := 0; i < len(infos); i++ {
  1044  		for j := i + 1; j < len(infos); j++ {
  1045  			if infos[i].ID > infos[j].ID {
  1046  				infos[i], infos[j] = infos[j], infos[i]
  1047  			}
  1048  		}
  1049  	}
  1050  	return infos
  1051  }
  1052  
  1053  // BroadcastMsg broadcast the message to all connected peers, this is low level func compare with Eth Protocol Manager
  1054  func (srv *Server) BroadcastMsg(msgCode uint64, data interface{}) {
  1055  	peers := srv.Peers()
  1056  	for _, p := range peers {
  1057  		Send(p.rw, BroadcastNewChildChainMsg, data)
  1058  	}
  1059  }
  1060  
  1061  func (srv *Server) AddLocalValidator(chainId string, address common.Address) {
  1062  
  1063  	log.Debug("AddLocalValidator")
  1064  
  1065  	validator := P2PValidator{
  1066  		ChainId: chainId,
  1067  		Address: address,
  1068  	}
  1069  
  1070  	for i := 0; i < len(srv.LocalValidators); i++ {
  1071  		if validator == srv.LocalValidators[i] {
  1072  			return
  1073  		}
  1074  	}
  1075  
  1076  	srv.LocalValidators = append(srv.LocalValidators, validator)
  1077  
  1078  	srv.broadcastRefreshValidatorNodeInfo(&P2PValidatorNodeInfo{
  1079  		Node:      *srv.Self(),
  1080  		TimeStamp: time.Now(),
  1081  		Validator: validator,
  1082  		Original:  true,
  1083  	}, nil)
  1084  }
  1085  
  1086  func (srv *Server) RemoveLocalValidator(chainId string, address common.Address) {
  1087  
  1088  	log.Debug("RemoveLocalValidator")
  1089  
  1090  	validator := P2PValidator{
  1091  		ChainId: chainId,
  1092  		Address: address,
  1093  	}
  1094  
  1095  	idx := -1
  1096  	for i := 0; i < len(srv.LocalValidators); i++ {
  1097  		if validator == srv.LocalValidators[i] {
  1098  			idx = i
  1099  			break
  1100  		}
  1101  	}
  1102  
  1103  	if idx < 0 {
  1104  		return
  1105  	}
  1106  
  1107  	srv.LocalValidators = append(srv.LocalValidators[:idx], srv.LocalValidators[idx+1:]...)
  1108  
  1109  	srv.broadcastRemoveValidatorNodeInfo(&P2PValidatorNodeInfo{
  1110  		Node:      *srv.Self(),
  1111  		TimeStamp: time.Now(),
  1112  		Validator: validator,
  1113  		Original:  true,
  1114  	}, nil)
  1115  }
  1116  
  1117  func (srv *Server) validatorAdd(valNodeInfo P2PValidatorNodeInfo, peers []*Peer, dialstate dialer) error {
  1118  
  1119  	log.Debug("validatorAdd")
  1120  
  1121  	validator := valNodeInfo.Validator
  1122  
  1123  	//if the node info goes from this node, just skip
  1124  	if srv.Self().ID == valNodeInfo.Node.ID {
  1125  		return nil
  1126  	}
  1127  
  1128  	//if the node does exist, we skip it; this could also avoid repeated propagate
  1129  	if nodeInfo, ok := srv.Validators[validator]; ok {
  1130  		con1 := valNodeInfo.Node.ID == nodeInfo.Node.ID
  1131  		con2 := valNodeInfo.Node.IP.String() == nodeInfo.Node.IP.String()
  1132  		log.Debugf("con1: %v, con2: %v", con1, con2)
  1133  		if con1 && con2 /*not compare PORT*/ {
  1134  			log.Debug("validator found, not add")
  1135  			return nil
  1136  		}
  1137  	}
  1138  
  1139  	log.Debug("validator not found")
  1140  	srv.Validators[validator] = &valNodeInfo
  1141  	inSameChain := false
  1142  	//if this validator is in the same chains which we join as validator, connect to it
  1143  	for i := 0; i < len(srv.LocalValidators); i++ {
  1144  		if validator.ChainId == srv.LocalValidators[i].ChainId {
  1145  			inSameChain = true
  1146  			break
  1147  		}
  1148  	}
  1149  
  1150  	notPeer := true
  1151  	for _, p := range peers {
  1152  		if p.ID() == valNodeInfo.Node.ID {
  1153  			notPeer = false
  1154  			break
  1155  		}
  1156  	}
  1157  
  1158  	if inSameChain && notPeer {
  1159  		dialstate.addStatic(&valNodeInfo.Node)
  1160  	}
  1161  
  1162  	//broadcast this node info to peers
  1163  	//srv.broadcastRefreshValidatorNodeInfo(&valNodeInfo, peers)
  1164  
  1165  	return nil
  1166  }
  1167  
  1168  func (srv *Server) validatorRemove(valNodeInfo P2PValidatorNodeInfo, peers []*Peer, dialstate dialer) error {
  1169  
  1170  	log.Debug("validatorRemove")
  1171  
  1172  	//if the node info goes from this node, just skip
  1173  	if srv.Self().ID == valNodeInfo.Node.ID {
  1174  		return nil
  1175  	}
  1176  
  1177  	validator := valNodeInfo.Validator
  1178  	//if the node does not exist, we skip it
  1179  	// in some cases the msg may need send, but ignoring it does not matter
  1180  	if _, ok := srv.Validators[validator]; !ok {
  1181  		return nil
  1182  	}
  1183  
  1184  	delete(srv.Validators, validator)
  1185  
  1186  	inSameChain := 0
  1187  	//if this validator is in the same chains which we join as validator
  1188  	for i := 0; i < len(srv.LocalValidators); i++ {
  1189  		if validator.ChainId == srv.LocalValidators[i].ChainId {
  1190  			inSameChain++
  1191  		}
  1192  	}
  1193  
  1194  	shouldRemove := false
  1195  	//if there is only one validator in one chain connected with this peer, disconnect
  1196  	if inSameChain == 1 {
  1197  		for _, p := range peers {
  1198  			if p.ID() == valNodeInfo.Node.ID {
  1199  				//there peer is connected only for we are validator in one single chain, now can remove it
  1200  				shouldRemove = true
  1201  				break
  1202  			}
  1203  		}
  1204  	}
  1205  
  1206  	if shouldRemove {
  1207  		dialstate.removeStatic(&valNodeInfo.Node)
  1208  	}
  1209  
  1210  	//broadcast this node info to peers
  1211  	//srv.broadcastRemoveValidatorNodeInfo(&valNodeInfo, peers)
  1212  
  1213  	return nil
  1214  }
  1215  
  1216  func (srv *Server) validatorAddPeer(peer *Peer) error {
  1217  
  1218  	log.Debug("validatorAddPeer")
  1219  
  1220  	sendList := make([]*NodeInfoToSend, 0)
  1221  
  1222  	var err error = nil
  1223  	for _, validatorNodeInfo := range srv.Validators {
  1224  
  1225  		if peer.ID() == validatorNodeInfo.Node.ID {
  1226  			//refresh the node's ip, and no need to send the info back to the peer itself
  1227  			validatorNodeInfo.Node.IP = peer.RemoteAddr().(*net.TCPAddr).IP
  1228  			continue
  1229  		}
  1230  
  1231  		/*
  1232  			err1 := Send(peer.rw, RefreshValidatorNodeInfoMsg, validatorNodeInfo)
  1233  			if err == nil && err1 != nil {
  1234  				err = err1
  1235  			}
  1236  		*/
  1237  		sendList = append(sendList, &NodeInfoToSend{
  1238  			valNodeInfo: validatorNodeInfo,
  1239  			action:      RefreshValidatorNodeInfoMsg,
  1240  			p:           peer,
  1241  		})
  1242  	}
  1243  
  1244  	node := *srv.Self()
  1245  	for i := 0; i < len(srv.LocalValidators); i++ {
  1246  		/*
  1247  			err1 := Send(peer.rw, RefreshValidatorNodeInfoMsg, P2PValidatorNodeInfo{
  1248  				Node: node,
  1249  				TimeStamp: time.Now(),
  1250  				Validator: srv.LocalValidators[i],
  1251  				Original: true,
  1252  			})
  1253  			if err == nil && err1 != nil {
  1254  				err = err1
  1255  			}
  1256  		*/
  1257  		sendList = append(sendList, &NodeInfoToSend{
  1258  			valNodeInfo: &P2PValidatorNodeInfo{
  1259  				Node:      node,
  1260  				TimeStamp: time.Now(),
  1261  				Validator: srv.LocalValidators[i],
  1262  				Original:  true,
  1263  			},
  1264  			action: RefreshValidatorNodeInfoMsg,
  1265  			p:      peer,
  1266  		})
  1267  	}
  1268  
  1269  	srv.addNodeInfoToSend(sendList)
  1270  
  1271  	return err
  1272  }
  1273  
  1274  func (srv *Server) validatorDelPeer(nodeId discover.NodeID) error {
  1275  
  1276  	log.Debug("validatorDelPeer")
  1277  
  1278  	srv.nodeInfoLock.Lock()
  1279  	defer srv.nodeInfoLock.Unlock()
  1280  
  1281  	tailIndex := 0
  1282  	for i := 0; i < len(srv.nodeInfoList); i++ {
  1283  
  1284  		nodeInfo := srv.nodeInfoList[i]
  1285  		if nodeInfo.valNodeInfo.Node.ID != nodeId {
  1286  			if i != tailIndex {
  1287  				srv.nodeInfoList[tailIndex] = nodeInfo
  1288  			}
  1289  			tailIndex++
  1290  		}
  1291  	}
  1292  
  1293  	removedCount := len(srv.nodeInfoList) - 1 - tailIndex
  1294  
  1295  	srv.nodeInfoList = srv.nodeInfoList[:tailIndex]
  1296  
  1297  	log.Debugf("removed %v node info to send to %v", removedCount, nodeId)
  1298  
  1299  	return nil
  1300  }
  1301  
  1302  func (srv *Server) broadcastRefreshValidatorNodeInfo(data *P2PValidatorNodeInfo, peers []*Peer) {
  1303  
  1304  	log.Debug("broadcastRefreshValidatorNodeInfo")
  1305  	if peers == nil {
  1306  		peers = srv.Peers()
  1307  	}
  1308  
  1309  	sendList := make([]*NodeInfoToSend, 0)
  1310  	for _, p := range peers {
  1311  
  1312  		sendList = append(sendList, &NodeInfoToSend{
  1313  			valNodeInfo: data,
  1314  			action:      RefreshValidatorNodeInfoMsg,
  1315  			p:           p,
  1316  		})
  1317  		//Send(p.rw, RefreshValidatorNodeInfoMsg, data)
  1318  	}
  1319  
  1320  	srv.addNodeInfoToSend(sendList)
  1321  }
  1322  
  1323  func (srv *Server) broadcastRemoveValidatorNodeInfo(data interface{}, peers []*Peer) {
  1324  
  1325  	log.Debug("broadcastRemoveValidatorNodeInfo")
  1326  	if peers == nil {
  1327  		peers = srv.Peers()
  1328  	}
  1329  
  1330  	sendList := make([]*NodeInfoToSend, 0)
  1331  	for _, p := range peers {
  1332  
  1333  		sendList = append(sendList, &NodeInfoToSend{
  1334  			valNodeInfo: data.(*P2PValidatorNodeInfo),
  1335  			action:      RemoveValidatorNodeInfoMsg,
  1336  			p:           p,
  1337  		})
  1338  		//Send(p.rw, RemoveValidatorNodeInfoMsg, data)
  1339  	}
  1340  
  1341  	srv.addNodeInfoToSend(sendList)
  1342  }
  1343  
  1344  func (srv *Server) addNodeInfoToSend(sendList []*NodeInfoToSend) {
  1345  
  1346  	srv.nodeInfoLock.Lock()
  1347  	defer srv.nodeInfoLock.Unlock()
  1348  
  1349  	srv.nodeInfoList = append(srv.nodeInfoList, sendList...)
  1350  }
  1351  
  1352  //this function send validator information to otheres, every 100mimsecond send one
  1353  //currently just handle the refresh action, not remove action
  1354  func (srv *Server) sendValidatorNodeInfoMessages() {
  1355  
  1356  	sleepDuration := 100 * time.Millisecond // Time to sleep before send next message
  1357  
  1358  	for srv.running {
  1359  
  1360  		if len(srv.nodeInfoList) > 0 {
  1361  			srv.nodeInfoLock.Lock()
  1362  
  1363  			nodeInfo := srv.nodeInfoList[0]
  1364  			srv.nodeInfoList = srv.nodeInfoList[1:]
  1365  
  1366  			srv.nodeInfoLock.Unlock()
  1367  
  1368  			if nodeInfo != nil &&
  1369  				nodeInfo.valNodeInfo != nil &&
  1370  				nodeInfo.p != nil &&
  1371  				nodeInfo.p.rw != nil &&
  1372  				nodeInfo.p.rw.fd != nil {
  1373  
  1374  				Send(nodeInfo.p.rw, nodeInfo.action, nodeInfo.valNodeInfo)
  1375  
  1376  				log.Debugf("send node info (%v, %v) to %v",
  1377  					nodeInfo.valNodeInfo.Validator.Address.String(), nodeInfo.valNodeInfo.Node.ID,
  1378  					nodeInfo.p.ID())
  1379  			}
  1380  		}
  1381  
  1382  		time.Sleep(sleepDuration)
  1383  	}
  1384  }