github.com/intfoundation/intchain@v0.0.0-20220727031208-4316ad31ca73/p2p/server.go (about) 1 // Copyright 2014 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 // Package p2p implements the INT Chain p2p network protocols. 18 package p2p 19 20 import ( 21 "crypto/ecdsa" 22 "errors" 23 "fmt" 24 "net" 25 "sync" 26 "time" 27 28 "github.com/intfoundation/intchain/common" 29 "github.com/intfoundation/intchain/common/mclock" 30 "github.com/intfoundation/intchain/event" 31 "github.com/intfoundation/intchain/log" 32 "github.com/intfoundation/intchain/p2p/discover" 33 "github.com/intfoundation/intchain/p2p/discv5" 34 "github.com/intfoundation/intchain/p2p/nat" 35 "github.com/intfoundation/intchain/p2p/netutil" 36 "github.com/intfoundation/intchain/rlp" 37 ) 38 39 const ( 40 defaultDialTimeout = 15 * time.Second 41 42 // Connectivity defaults. 43 maxActiveDialTasks = 16 44 defaultMaxPendingPeers = 50 45 defaultDialRatio = 3 46 47 // Maximum time allowed for reading a complete message. 48 // This is effectively the amount of time a connection can be idle. 49 frameReadTimeout = 30 * time.Second 50 51 // Maximum amount of time allowed for writing a complete message. 52 frameWriteTimeout = 20 * time.Second 53 ) 54 55 var errServerStopped = errors.New("server stopped") 56 57 // Config holds Server options. 58 type Config struct { 59 // This field must be set to a valid secp256k1 private key. 60 PrivateKey *ecdsa.PrivateKey `toml:"-"` 61 62 // MaxPeers is the maximum number of peers that can be 63 // connected. It must be greater than zero. 64 MaxPeers int 65 66 // MaxPendingPeers is the maximum number of peers that can be pending in the 67 // handshake phase, counted separately for inbound and outbound connections. 68 // Zero defaults to preset values. 69 MaxPendingPeers int `toml:",omitempty"` 70 71 // DialRatio controls the ratio of inbound to dialed connections. 72 // Example: a DialRatio of 2 allows 1/2 of connections to be dialed. 73 // Setting DialRatio to zero defaults it to 3. 74 DialRatio int `toml:",omitempty"` 75 76 // NoDiscovery can be used to disable the peer discovery mechanism. 77 // Disabling is useful for protocol debugging (manual topology). 78 NoDiscovery bool 79 80 // DiscoveryV5 specifies whether the the new topic-discovery based V5 discovery 81 // protocol should be started or not. 82 DiscoveryV5 bool `toml:",omitempty"` 83 84 // Name sets the node name of this server. 85 // Use common.MakeName to create a name that follows existing conventions. 86 Name string `toml:"-"` 87 88 // BootstrapNodes are used to establish connectivity 89 // with the rest of the network. 90 BootstrapNodes []*discover.Node 91 92 // BootstrapNodesV5 are used to establish connectivity 93 // with the rest of the network using the V5 discovery 94 // protocol. 95 BootstrapNodesV5 []*discv5.Node `toml:",omitempty"` 96 97 // Static nodes are used as pre-configured connections which are always 98 // maintained and re-connected on disconnects. 99 StaticNodes []*discover.Node 100 101 // Trusted nodes are used as pre-configured connections which are always 102 // allowed to connect, even above the peer limit. 103 TrustedNodes []*discover.Node 104 105 // Validators that this node acts as 106 LocalValidators []P2PValidator 107 108 // Validators set in all chains 109 // Should find nodes for all of them to support bls transport in IPBFT module 110 Validators map[P2PValidator]*P2PValidatorNodeInfo 111 112 // Connectivity can be restricted to certain IP networks. 113 // If this option is set to a non-nil value, only hosts which match one of the 114 // IP networks contained in the list are considered. 115 NetRestrict *netutil.Netlist `toml:",omitempty"` 116 117 // NodeDatabase is the path to the database containing the previously seen 118 // live nodes in the network. 119 NodeDatabase string `toml:",omitempty"` 120 121 // Protocols should contain the protocols supported 122 // by the server. Matching protocols are launched for 123 // each peer. 124 Protocols []Protocol `toml:"-"` 125 126 // If ListenAddr is set to a non-nil address, the server 127 // will listen for incoming connections. 128 // 129 // If the port is zero, the operating system will pick a port. The 130 // ListenAddr field will be updated with the actual address when 131 // the server is started. 132 ListenAddr string 133 134 // If set to a non-nil value, the given NAT port mapper 135 // is used to make the listening port available to the 136 // Internet. 137 NAT nat.Interface `toml:",omitempty"` 138 139 // If Dialer is set to a non-nil value, the given Dialer 140 // is used to dial outbound peer connections. 141 Dialer NodeDialer `toml:"-"` 142 143 // If NoDial is true, the server will not dial any peers. 144 NoDial bool `toml:",omitempty"` 145 146 // If EnableMsgEvents is set then the server will emit PeerEvents 147 // whenever a message is sent to or received from a peer 148 EnableMsgEvents bool 149 150 // Logger is a custom logger to use with the p2p.Server. 151 Logger log.Logger `toml:",omitempty"` 152 } 153 154 type NodeInfoToSend struct { 155 valNodeInfo *P2PValidatorNodeInfo 156 action uint64 157 p *Peer 158 } 159 160 // Server manages all peer connections. 161 type Server struct { 162 // Config fields may not be modified while the server is running. 163 Config 164 165 // Hooks for testing. These are useful because we can inhibit 166 // the whole protocol stack. 167 newTransport func(net.Conn) transport 168 newPeerHook func(*Peer) 169 170 lock sync.Mutex // protects running 171 running bool 172 173 ntab discoverTable 174 listener net.Listener 175 ourHandshake *protoHandshake 176 lastLookup time.Time 177 DiscV5 *discv5.Network 178 179 // These are for Peers, PeerCount (and nothing else). 180 peerOp chan peerOpFunc 181 peerOpDone chan struct{} 182 183 quit chan struct{} 184 addstatic chan *discover.Node 185 removestatic chan *discover.Node 186 posthandshake chan *conn 187 addpeer chan *conn 188 delpeer chan peerDrop 189 loopWG sync.WaitGroup // loop, listenLoop 190 peerFeed event.Feed 191 log log.Logger 192 193 events chan *PeerEvent 194 eventsSub event.Subscription 195 196 nodeInfoLock sync.Mutex // protects running 197 nodeInfoList []*NodeInfoToSend 198 } 199 200 type peerOpFunc func(map[discover.NodeID]*Peer) 201 202 type peerDrop struct { 203 *Peer 204 err error 205 requested bool // true if signaled by the peer 206 } 207 208 type connFlag int 209 210 const ( 211 dynDialedConn connFlag = 1 << iota 212 staticDialedConn 213 inboundConn 214 trustedConn 215 ) 216 217 // conn wraps a network connection with information gathered 218 // during the two handshakes. 219 type conn struct { 220 fd net.Conn 221 transport 222 flags connFlag 223 cont chan error // The run loop uses cont to signal errors to SetupConn. 224 id discover.NodeID // valid after the encryption handshake 225 caps []Cap // valid after the protocol handshake 226 name string // valid after the protocol handshake 227 } 228 229 type transport interface { 230 // The two handshakes. 231 doEncHandshake(prv *ecdsa.PrivateKey, dialDest *discover.Node) (discover.NodeID, error) 232 doProtoHandshake(our *protoHandshake) (*protoHandshake, error) 233 // The MsgReadWriter can only be used after the encryption 234 // handshake has completed. The code uses conn.id to track this 235 // by setting it to a non-nil value after the encryption handshake. 236 MsgReadWriter 237 // transports must provide Close because we use MsgPipe in some of 238 // the tests. Closing the actual network connection doesn't do 239 // anything in those tests because NsgPipe doesn't use it. 240 close(err error) 241 } 242 243 func (c *conn) String() string { 244 s := c.flags.String() 245 if (c.id != discover.NodeID{}) { 246 s += " " + c.id.String() 247 } 248 s += " " + c.fd.RemoteAddr().String() 249 return s 250 } 251 252 func (f connFlag) String() string { 253 s := "" 254 if f&trustedConn != 0 { 255 s += "-trusted" 256 } 257 if f&dynDialedConn != 0 { 258 s += "-dyndial" 259 } 260 if f&staticDialedConn != 0 { 261 s += "-staticdial" 262 } 263 if f&inboundConn != 0 { 264 s += "-inbound" 265 } 266 if s != "" { 267 s = s[1:] 268 } 269 return s 270 } 271 272 func (c *conn) is(f connFlag) bool { 273 return c.flags&f != 0 274 } 275 276 // Peers returns all connected peers. 277 func (srv *Server) Peers() []*Peer { 278 var ps []*Peer 279 select { 280 // Note: We'd love to put this function into a variable but 281 // that seems to cause a weird compiler error in some 282 // environments. 283 case srv.peerOp <- func(peers map[discover.NodeID]*Peer) { 284 for _, p := range peers { 285 ps = append(ps, p) 286 } 287 }: 288 <-srv.peerOpDone 289 case <-srv.quit: 290 } 291 return ps 292 } 293 294 // PeerCount returns the number of connected peers. 295 func (srv *Server) PeerCount() int { 296 var count int 297 select { 298 case srv.peerOp <- func(ps map[discover.NodeID]*Peer) { count = len(ps) }: 299 <-srv.peerOpDone 300 case <-srv.quit: 301 } 302 return count 303 } 304 305 // AddPeer connects to the given node and maintains the connection until the 306 // server is shut down. If the connection fails for any reason, the server will 307 // attempt to reconnect the peer. 308 func (srv *Server) AddPeer(node *discover.Node) { 309 select { 310 case srv.addstatic <- node: 311 case <-srv.quit: 312 } 313 } 314 315 // RemovePeer disconnects from the given node 316 func (srv *Server) RemovePeer(node *discover.Node) { 317 select { 318 case srv.removestatic <- node: 319 case <-srv.quit: 320 } 321 } 322 323 // SubscribePeers subscribes the given channel to peer events 324 func (srv *Server) SubscribeEvents(ch chan *PeerEvent) event.Subscription { 325 return srv.peerFeed.Subscribe(ch) 326 } 327 328 // Self returns the local node's endpoint information. 329 func (srv *Server) Self() *discover.Node { 330 srv.lock.Lock() 331 defer srv.lock.Unlock() 332 333 if !srv.running { 334 return &discover.Node{IP: net.ParseIP("0.0.0.0")} 335 } 336 return srv.makeSelf(srv.listener, srv.ntab) 337 } 338 339 func (srv *Server) makeSelf(listener net.Listener, ntab discoverTable) *discover.Node { 340 // If the server's not running, return an empty node. 341 // If the node is running but discovery is off, manually assemble the node infos. 342 if ntab == nil { 343 // Inbound connections disabled, use zero address. 344 if listener == nil { 345 return &discover.Node{IP: net.ParseIP("0.0.0.0"), ID: discover.PubkeyID(&srv.PrivateKey.PublicKey)} 346 } 347 // Otherwise inject the listener address too 348 addr := listener.Addr().(*net.TCPAddr) 349 return &discover.Node{ 350 ID: discover.PubkeyID(&srv.PrivateKey.PublicKey), 351 IP: addr.IP, 352 TCP: uint16(addr.Port), 353 } 354 } 355 // Otherwise return the discovery node. 356 return ntab.Self() 357 } 358 359 // Stop terminates the server and all active peer connections. 360 // It blocks until all active connections have been closed. 361 func (srv *Server) Stop() { 362 srv.lock.Lock() 363 if !srv.running { 364 srv.lock.Unlock() 365 return 366 } 367 srv.eventsSub.Unsubscribe() 368 srv.running = false 369 if srv.listener != nil { 370 // this unblocks listener Accept 371 srv.listener.Close() 372 } 373 close(srv.quit) 374 srv.lock.Unlock() 375 srv.loopWG.Wait() 376 } 377 378 // sharedUDPConn implements a shared connection. Write sends messages to the underlying connection while read returns 379 // messages that were found unprocessable and sent to the unhandled channel by the primary listener. 380 type sharedUDPConn struct { 381 *net.UDPConn 382 unhandled chan discover.ReadPacket 383 } 384 385 // ReadFromUDP implements discv5.conn 386 func (s *sharedUDPConn) ReadFromUDP(b []byte) (n int, addr *net.UDPAddr, err error) { 387 packet, ok := <-s.unhandled 388 if !ok { 389 return 0, nil, fmt.Errorf("Connection was closed") 390 } 391 l := len(packet.Data) 392 if l > len(b) { 393 l = len(b) 394 } 395 copy(b[:l], packet.Data[:l]) 396 return l, packet.Addr, nil 397 } 398 399 // Close implements discv5.conn 400 func (s *sharedUDPConn) Close() error { 401 return nil 402 } 403 404 // Start starts running the server. 405 // Servers can not be re-used after stopping. 406 func (srv *Server) Start() (err error) { 407 srv.lock.Lock() 408 defer srv.lock.Unlock() 409 if srv.running { 410 return errors.New("server already running") 411 } 412 srv.running = true 413 srv.log = srv.Config.Logger 414 if srv.log == nil { 415 srv.log = log.New() 416 } 417 srv.log.Info("Starting P2P networking") 418 419 // static fields 420 if srv.PrivateKey == nil { 421 return fmt.Errorf("Server.PrivateKey must be set to a non-nil key") 422 } 423 if srv.newTransport == nil { 424 srv.newTransport = newRLPX 425 } 426 if srv.Dialer == nil { 427 srv.Dialer = TCPDialer{&net.Dialer{Timeout: defaultDialTimeout}} 428 } 429 srv.quit = make(chan struct{}) 430 srv.addpeer = make(chan *conn) 431 srv.delpeer = make(chan peerDrop) 432 srv.posthandshake = make(chan *conn) 433 srv.addstatic = make(chan *discover.Node) 434 srv.removestatic = make(chan *discover.Node) 435 srv.peerOp = make(chan peerOpFunc) 436 srv.peerOpDone = make(chan struct{}) 437 srv.events = make(chan *PeerEvent) 438 srv.eventsSub = srv.SubscribeEvents(srv.events) 439 440 srv.nodeInfoList = make([]*NodeInfoToSend, 0) 441 442 var ( 443 conn *net.UDPConn 444 sconn *sharedUDPConn 445 realaddr *net.UDPAddr 446 unhandled chan discover.ReadPacket 447 ) 448 449 if !srv.NoDiscovery || srv.DiscoveryV5 { 450 addr, err := net.ResolveUDPAddr("udp", srv.ListenAddr) 451 if err != nil { 452 return err 453 } 454 conn, err = net.ListenUDP("udp", addr) 455 if err != nil { 456 return err 457 } 458 realaddr = conn.LocalAddr().(*net.UDPAddr) 459 if srv.NAT != nil { 460 if !realaddr.IP.IsLoopback() { 461 go nat.Map(srv.NAT, srv.quit, "udp", realaddr.Port, realaddr.Port, "ethereum discovery") 462 } 463 // TODO: react to external IP changes over time. 464 if ext, err := srv.NAT.ExternalIP(); err == nil { 465 realaddr = &net.UDPAddr{IP: ext, Port: realaddr.Port} 466 } 467 } 468 } 469 470 if !srv.NoDiscovery && srv.DiscoveryV5 { 471 unhandled = make(chan discover.ReadPacket, 100) 472 sconn = &sharedUDPConn{conn, unhandled} 473 } 474 475 // node table 476 if !srv.NoDiscovery { 477 cfg := discover.Config{ 478 PrivateKey: srv.PrivateKey, 479 AnnounceAddr: realaddr, 480 NodeDBPath: srv.NodeDatabase, 481 NetRestrict: srv.NetRestrict, 482 Bootnodes: srv.BootstrapNodes, 483 Unhandled: unhandled, 484 } 485 ntab, err := discover.ListenUDP(conn, cfg) 486 if err != nil { 487 return err 488 } 489 srv.ntab = ntab 490 } 491 492 if srv.DiscoveryV5 { 493 var ( 494 ntab *discv5.Network 495 err error 496 ) 497 if sconn != nil { 498 ntab, err = discv5.ListenUDP(srv.PrivateKey, sconn, realaddr, "", srv.NetRestrict) //srv.NodeDatabase) 499 } else { 500 ntab, err = discv5.ListenUDP(srv.PrivateKey, conn, realaddr, "", srv.NetRestrict) //srv.NodeDatabase) 501 } 502 if err != nil { 503 return err 504 } 505 if err := ntab.SetFallbackNodes(srv.BootstrapNodesV5); err != nil { 506 return err 507 } 508 srv.DiscV5 = ntab 509 } 510 511 dynPeers := srv.maxDialedConns() 512 dialer := newDialState(srv.StaticNodes, srv.BootstrapNodes, srv.ntab, dynPeers, srv.NetRestrict) 513 514 // handshake 515 srv.ourHandshake = &protoHandshake{Version: baseProtocolVersion, Name: srv.Name, ID: discover.PubkeyID(&srv.PrivateKey.PublicKey)} 516 for _, p := range srv.Protocols { 517 srv.ourHandshake.Caps = append(srv.ourHandshake.Caps, p.cap()) 518 } 519 // listen/dial 520 if srv.ListenAddr != "" { 521 if err := srv.startListening(); err != nil { 522 return err 523 } 524 } 525 if srv.NoDial && srv.ListenAddr == "" { 526 srv.log.Warn("P2P server will be useless, neither dialing nor listening") 527 } 528 529 srv.loopWG.Add(1) 530 go srv.run(dialer) 531 srv.running = true 532 533 go srv.sendValidatorNodeInfoMessages() 534 535 return nil 536 } 537 538 // AddHandshakeCaps Add the Child Protocol Caps after create the child chain and before launch it 539 func (srv *Server) AddChildProtocolCaps(childProtocols []Protocol) { 540 for _, p := range childProtocols { 541 srv.ourHandshake.Caps = append(srv.ourHandshake.Caps, p.cap()) 542 } 543 } 544 545 func (srv *Server) startListening() error { 546 // Launch the TCP listener. 547 listener, err := net.Listen("tcp", srv.ListenAddr) 548 if err != nil { 549 return err 550 } 551 laddr := listener.Addr().(*net.TCPAddr) 552 srv.ListenAddr = laddr.String() 553 srv.listener = listener 554 srv.loopWG.Add(1) 555 go srv.listenLoop() 556 // Map the TCP listening port if NAT is configured. 557 if !laddr.IP.IsLoopback() && srv.NAT != nil { 558 srv.loopWG.Add(1) 559 go func() { 560 nat.Map(srv.NAT, srv.quit, "tcp", laddr.Port, laddr.Port, "ethereum p2p") 561 srv.loopWG.Done() 562 }() 563 } 564 return nil 565 } 566 567 type dialer interface { 568 newTasks(running int, peers map[discover.NodeID]*Peer, now time.Time) []task 569 taskDone(task, time.Time) 570 addStatic(*discover.Node) 571 removeStatic(*discover.Node) 572 } 573 574 func (srv *Server) run(dialstate dialer) { 575 defer srv.loopWG.Done() 576 var ( 577 peers = make(map[discover.NodeID]*Peer) 578 inboundCount = 0 579 trusted = make(map[discover.NodeID]bool, len(srv.TrustedNodes)) 580 taskdone = make(chan task, maxActiveDialTasks) 581 runningTasks []task 582 queuedTasks []task // tasks that can't run yet 583 ) 584 // Put trusted nodes into a map to speed up checks. 585 // Trusted peers are loaded on startup and cannot be 586 // modified while the server is running. 587 for _, n := range srv.TrustedNodes { 588 trusted[n.ID] = true 589 } 590 591 // removes t from runningTasks 592 delTask := func(t task) { 593 for i := range runningTasks { 594 if runningTasks[i] == t { 595 runningTasks = append(runningTasks[:i], runningTasks[i+1:]...) 596 break 597 } 598 } 599 } 600 // starts until max number of active tasks is satisfied 601 startTasks := func(ts []task) (rest []task) { 602 i := 0 603 for ; len(runningTasks) < maxActiveDialTasks && i < len(ts); i++ { 604 t := ts[i] 605 srv.log.Trace("New dial task", "task", t) 606 go func() { t.Do(srv); taskdone <- t }() 607 runningTasks = append(runningTasks, t) 608 } 609 return ts[i:] 610 } 611 scheduleTasks := func() { 612 // Start from queue first. 613 queuedTasks = append(queuedTasks[:0], startTasks(queuedTasks)...) 614 // Query dialer for new tasks and start as many as possible now. 615 if len(runningTasks) < maxActiveDialTasks { 616 nt := dialstate.newTasks(len(runningTasks)+len(queuedTasks), peers, time.Now()) 617 queuedTasks = append(queuedTasks, startTasks(nt)...) 618 } 619 } 620 621 running: 622 for { 623 scheduleTasks() 624 625 select { 626 case <-srv.quit: 627 // The server was stopped. Run the cleanup logic. 628 break running 629 case n := <-srv.addstatic: 630 // This channel is used by AddPeer to add to the 631 // ephemeral static peer list. Add it to the dialer, 632 // it will keep the node connected. 633 srv.log.Debug("Adding static node", "node", n) 634 dialstate.addStatic(n) 635 case n := <-srv.removestatic: 636 // This channel is used by RemovePeer to send a 637 // disconnect request to a peer and begin the 638 // stop keeping the node connected 639 srv.log.Debug("Removing static node", "node", n) 640 dialstate.removeStatic(n) 641 if p, ok := peers[n.ID]; ok { 642 p.Disconnect(DiscRequested) 643 } 644 case op := <-srv.peerOp: 645 // This channel is used by Peers and PeerCount. 646 op(peers) 647 srv.peerOpDone <- struct{}{} 648 case t := <-taskdone: 649 // A task got done. Tell dialstate about it so it 650 // can update its state and remove it from the active 651 // tasks list. 652 srv.log.Trace("Dial task done", "task", t) 653 dialstate.taskDone(t, time.Now()) 654 delTask(t) 655 case c := <-srv.posthandshake: 656 // A connection has passed the encryption handshake so 657 // the remote identity is known (but hasn't been verified yet). 658 if trusted[c.id] { 659 // Ensure that the trusted flag is set before checking against MaxPeers. 660 c.flags |= trustedConn 661 } 662 // TODO: track in-progress inbound node IDs (pre-Peer) to avoid dialing them. 663 select { 664 case c.cont <- srv.encHandshakeChecks(peers, inboundCount, c): 665 case <-srv.quit: 666 break running 667 } 668 case c := <-srv.addpeer: 669 // At this point the connection is past the protocol handshake. 670 // Its capabilities are known and the remote identity is verified. 671 err := srv.protoHandshakeChecks(peers, inboundCount, c) 672 if err == nil { 673 // The handshakes are done and it passed all checks. 674 p := newPeer(c, srv.Protocols) 675 // If message events are enabled, pass the peerFeed 676 // to the peer 677 if srv.EnableMsgEvents { 678 p.events = &srv.peerFeed 679 } 680 name := truncateName(c.name) 681 srv.log.Debug("Adding p2p peer", "name", name, "addr", c.fd.RemoteAddr(), "peers", len(peers)+1) 682 go srv.runPeer(p) 683 peers[c.id] = p 684 if p.Inbound() { 685 inboundCount++ 686 } 687 688 srv.validatorAddPeer(p) 689 } 690 // The dialer logic relies on the assumption that 691 // dial tasks complete after the peer has been added or 692 // discarded. Unblock the task last. 693 select { 694 case c.cont <- err: 695 case <-srv.quit: 696 break running 697 } 698 case pd := <-srv.delpeer: 699 // A peer disconnected. 700 d := common.PrettyDuration(mclock.Now() - pd.created) 701 pd.log.Debug("Removing p2p peer", "duration", d, "peers", len(peers)-1, "req", pd.requested, "err", pd.err) 702 delete(peers, pd.ID()) 703 if pd.Inbound() { 704 inboundCount-- 705 } 706 srv.validatorDelPeer(pd.ID()) 707 708 case evt := <-srv.events: 709 log.Debugf("peer events received: %v", evt) 710 switch evt.Type { 711 712 case PeerEventTypeRefreshValidator: 713 var valNodeInfo P2PValidatorNodeInfo 714 if err := rlp.DecodeBytes([]byte(evt.Protocol), &valNodeInfo); err != nil { 715 log.Debugf("rlp decode valNodeInfo failed with %v", err) 716 } 717 718 peerArr := make([]*Peer, 0) 719 for _, p := range peers { 720 peerArr = append(peerArr, p) 721 } 722 723 if err := srv.validatorAdd(valNodeInfo, peerArr, dialstate); err != nil { 724 log.Debugf("add valNodeInfo to local failed with %v", err) 725 } 726 727 log.Debugf("Got refresh validation node infomation from validation %v", valNodeInfo.Validator.Address.String()) 728 729 case PeerEventTypeRemoveValidator: 730 var valNodeInfo P2PValidatorNodeInfo 731 if err := rlp.DecodeBytes([]byte(evt.Protocol), &valNodeInfo); err != nil { 732 log.Debugf("rlp decode valNodeInfo failed with %v", err) 733 } 734 735 peerArr := make([]*Peer, 0) 736 for _, p := range peers { 737 peerArr = append(peerArr, p) 738 } 739 740 if err := srv.validatorRemove(valNodeInfo, peerArr, dialstate); err != nil { 741 log.Debugf("remove valNodeInfo from local failed with %v", err) 742 } 743 744 log.Debugf("Got remove validation node infomation from validation %v", valNodeInfo.Validator.Address.String()) 745 746 } 747 } 748 } 749 750 srv.log.Trace("P2P networking is spinning down") 751 752 // Terminate discovery. If there is a running lookup it will terminate soon. 753 if srv.ntab != nil { 754 srv.ntab.Close() 755 } 756 if srv.DiscV5 != nil { 757 srv.DiscV5.Close() 758 } 759 // Disconnect all peers. 760 for _, p := range peers { 761 p.Disconnect(DiscQuitting) 762 } 763 // Wait for peers to shut down. Pending connections and tasks are 764 // not handled here and will terminate soon-ish because srv.quit 765 // is closed. 766 for len(peers) > 0 { 767 p := <-srv.delpeer 768 p.log.Trace("<-delpeer (spindown)", "remainingTasks", len(runningTasks)) 769 delete(peers, p.ID()) 770 } 771 } 772 773 func (srv *Server) protoHandshakeChecks(peers map[discover.NodeID]*Peer, inboundCount int, c *conn) error { 774 // Drop connections with no matching protocols. 775 if len(srv.Protocols) > 0 && countMatchingProtocols(srv.Protocols, c.caps) == 0 { 776 return DiscUselessPeer 777 } 778 // Repeat the encryption handshake checks because the 779 // peer set might have changed between the handshakes. 780 return srv.encHandshakeChecks(peers, inboundCount, c) 781 } 782 783 func (srv *Server) encHandshakeChecks(peers map[discover.NodeID]*Peer, inboundCount int, c *conn) error { 784 switch { 785 case !c.is(trustedConn|staticDialedConn) && len(peers) >= srv.MaxPeers: 786 return DiscTooManyPeers 787 case !c.is(trustedConn) && c.is(inboundConn) && inboundCount >= srv.maxInboundConns(): 788 return DiscTooManyPeers 789 case peers[c.id] != nil: 790 return DiscAlreadyConnected 791 case c.id == srv.Self().ID: 792 return DiscSelf 793 default: 794 return nil 795 } 796 } 797 798 func (srv *Server) maxInboundConns() int { 799 return srv.MaxPeers - srv.maxDialedConns() 800 } 801 802 func (srv *Server) maxDialedConns() int { 803 if srv.NoDiscovery || srv.NoDial { 804 return 0 805 } 806 r := srv.DialRatio 807 if r == 0 { 808 r = defaultDialRatio 809 } 810 return srv.MaxPeers / r 811 } 812 813 type tempError interface { 814 Temporary() bool 815 } 816 817 // listenLoop runs in its own goroutine and accepts 818 // inbound connections. 819 func (srv *Server) listenLoop() { 820 defer srv.loopWG.Done() 821 srv.log.Info("RLPx listener up", "self", srv.makeSelf(srv.listener, srv.ntab)) 822 823 tokens := defaultMaxPendingPeers 824 if srv.MaxPendingPeers > 0 { 825 tokens = srv.MaxPendingPeers 826 } 827 slots := make(chan struct{}, tokens) 828 for i := 0; i < tokens; i++ { 829 slots <- struct{}{} 830 } 831 832 for { 833 // Wait for a handshake slot before accepting. 834 <-slots 835 836 var ( 837 fd net.Conn 838 err error 839 ) 840 for { 841 fd, err = srv.listener.Accept() 842 if tempErr, ok := err.(tempError); ok && tempErr.Temporary() { 843 srv.log.Debug("Temporary read error", "err", err) 844 continue 845 } else if err != nil { 846 srv.log.Debug("Read error", "err", err) 847 return 848 } 849 break 850 } 851 852 // Reject connections that do not match NetRestrict. 853 if srv.NetRestrict != nil { 854 if tcp, ok := fd.RemoteAddr().(*net.TCPAddr); ok && !srv.NetRestrict.Contains(tcp.IP) { 855 srv.log.Debug("Rejected conn (not whitelisted in NetRestrict)", "addr", fd.RemoteAddr()) 856 fd.Close() 857 slots <- struct{}{} 858 continue 859 } 860 } 861 862 fd = newMeteredConn(fd, true) 863 srv.log.Trace("Accepted connection", "addr", fd.RemoteAddr()) 864 go func() { 865 srv.SetupConn(fd, inboundConn, nil) 866 slots <- struct{}{} 867 }() 868 } 869 } 870 871 // SetupConn runs the handshakes and attempts to add the connection 872 // as a peer. It returns when the connection has been added as a peer 873 // or the handshakes have failed. 874 func (srv *Server) SetupConn(fd net.Conn, flags connFlag, dialDest *discover.Node) error { 875 self := srv.Self() 876 if self == nil { 877 return errors.New("shutdown") 878 } 879 c := &conn{fd: fd, transport: srv.newTransport(fd), flags: flags, cont: make(chan error)} 880 err := srv.setupConn(c, flags, dialDest) 881 if err != nil { 882 c.close(err) 883 srv.log.Trace("Setting up connection failed", "id", c.id, "err", err) 884 } 885 return err 886 } 887 888 func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *discover.Node) error { 889 // Prevent leftover pending conns from entering the handshake. 890 srv.lock.Lock() 891 running := srv.running 892 srv.lock.Unlock() 893 if !running { 894 return errServerStopped 895 } 896 // Run the encryption handshake. 897 var err error 898 if c.id, err = c.doEncHandshake(srv.PrivateKey, dialDest); err != nil { 899 srv.log.Trace("Failed RLPx handshake", "addr", c.fd.RemoteAddr(), "conn", c.flags, "err", err) 900 return err 901 } 902 clog := srv.log.New("id", c.id, "addr", c.fd.RemoteAddr(), "conn", c.flags) 903 // For dialed connections, check that the remote public key matches. 904 if dialDest != nil && c.id != dialDest.ID { 905 clog.Trace("Dialed identity mismatch", "want", c, dialDest.ID) 906 return DiscUnexpectedIdentity 907 } 908 err = srv.checkpoint(c, srv.posthandshake) 909 if err != nil { 910 clog.Trace("Rejected peer before protocol handshake", "err", err) 911 return err 912 } 913 // Run the protocol handshake 914 phs, err := c.doProtoHandshake(srv.ourHandshake) 915 if err != nil { 916 clog.Trace("Failed proto handshake", "err", err) 917 return err 918 } 919 if phs.ID != c.id { 920 clog.Trace("Wrong devp2p handshake identity", "err", phs.ID) 921 return DiscUnexpectedIdentity 922 } 923 c.caps, c.name = phs.Caps, phs.Name 924 err = srv.checkpoint(c, srv.addpeer) 925 if err != nil { 926 clog.Trace("Rejected peer", "err", err) 927 return err 928 } 929 // If the checks completed successfully, runPeer has now been 930 // launched by run. 931 clog.Trace("connection set up", "inbound", dialDest == nil) 932 return nil 933 } 934 935 func truncateName(s string) string { 936 if len(s) > 20 { 937 return s[:20] + "..." 938 } 939 return s 940 } 941 942 // checkpoint sends the conn to run, which performs the 943 // post-handshake checks for the stage (posthandshake, addpeer). 944 func (srv *Server) checkpoint(c *conn, stage chan<- *conn) error { 945 select { 946 case stage <- c: 947 case <-srv.quit: 948 return errServerStopped 949 } 950 select { 951 case err := <-c.cont: 952 return err 953 case <-srv.quit: 954 return errServerStopped 955 } 956 } 957 958 // runPeer runs in its own goroutine for each peer. 959 // it waits until the Peer logic returns and removes 960 // the peer. 961 func (srv *Server) runPeer(p *Peer) { 962 if srv.newPeerHook != nil { 963 srv.newPeerHook(p) 964 } 965 966 // broadcast peer add 967 srv.peerFeed.Send(&PeerEvent{ 968 Type: PeerEventTypeAdd, 969 Peer: p.ID(), 970 }) 971 972 // Set the server protocol, this should link with p2p server's protocol and auto-update if changed 973 p.srvProtocols = &srv.Protocols 974 975 // run the protocol 976 remoteRequested, err := p.run() 977 978 // broadcast peer drop 979 srv.peerFeed.Send(&PeerEvent{ 980 Type: PeerEventTypeDrop, 981 Peer: p.ID(), 982 Error: err.Error(), 983 }) 984 985 // Note: run waits for existing peers to be sent on srv.delpeer 986 // before returning, so this send should not select on srv.quit. 987 srv.delpeer <- peerDrop{p, err, remoteRequested} 988 } 989 990 // NodeInfo represents a short summary of the information known about the host. 991 type NodeInfo struct { 992 ID string `json:"id"` // Unique node identifier (also the encryption key) 993 Name string `json:"name"` // Name of the node, including client type, version, OS, custom data 994 Enode string `json:"enode"` // Enode URL for adding this peer from remote peers 995 IP string `json:"ip"` // IP address of the node 996 Ports struct { 997 Discovery int `json:"discovery"` // UDP listening port for discovery protocol 998 Listener int `json:"listener"` // TCP listening port for RLPx 999 } `json:"ports"` 1000 ListenAddr string `json:"listenAddr"` 1001 Protocols map[string]interface{} `json:"protocols"` 1002 } 1003 1004 // NodeInfo gathers and returns a collection of metadata known about the host. 1005 func (srv *Server) NodeInfo() *NodeInfo { 1006 node := srv.Self() 1007 1008 // Gather and assemble the generic node infos 1009 info := &NodeInfo{ 1010 Name: srv.Name, 1011 Enode: node.String(), 1012 ID: node.ID.String(), 1013 IP: node.IP.String(), 1014 ListenAddr: srv.ListenAddr, 1015 Protocols: make(map[string]interface{}), 1016 } 1017 info.Ports.Discovery = int(node.UDP) 1018 info.Ports.Listener = int(node.TCP) 1019 1020 // Gather all the running protocol infos (only once per protocol type) 1021 for _, proto := range srv.Protocols { 1022 if _, ok := info.Protocols[proto.Name]; !ok { 1023 nodeInfo := interface{}("unknown") 1024 if query := proto.NodeInfo; query != nil { 1025 nodeInfo = proto.NodeInfo() 1026 } 1027 info.Protocols[proto.Name] = nodeInfo 1028 } 1029 } 1030 return info 1031 } 1032 1033 // PeersInfo returns an array of metadata objects describing connected peers. 1034 func (srv *Server) PeersInfo() []*PeerInfo { 1035 // Gather all the generic and sub-protocol specific infos 1036 infos := make([]*PeerInfo, 0, srv.PeerCount()) 1037 for _, peer := range srv.Peers() { 1038 if peer != nil { 1039 infos = append(infos, peer.Info()) 1040 } 1041 } 1042 // Sort the result array alphabetically by node identifier 1043 for i := 0; i < len(infos); i++ { 1044 for j := i + 1; j < len(infos); j++ { 1045 if infos[i].ID > infos[j].ID { 1046 infos[i], infos[j] = infos[j], infos[i] 1047 } 1048 } 1049 } 1050 return infos 1051 } 1052 1053 // BroadcastMsg broadcast the message to all connected peers, this is low level func compare with Eth Protocol Manager 1054 func (srv *Server) BroadcastMsg(msgCode uint64, data interface{}) { 1055 peers := srv.Peers() 1056 for _, p := range peers { 1057 Send(p.rw, BroadcastNewChildChainMsg, data) 1058 } 1059 } 1060 1061 func (srv *Server) AddLocalValidator(chainId string, address common.Address) { 1062 1063 log.Debug("AddLocalValidator") 1064 1065 validator := P2PValidator{ 1066 ChainId: chainId, 1067 Address: address, 1068 } 1069 1070 for i := 0; i < len(srv.LocalValidators); i++ { 1071 if validator == srv.LocalValidators[i] { 1072 return 1073 } 1074 } 1075 1076 srv.LocalValidators = append(srv.LocalValidators, validator) 1077 1078 srv.broadcastRefreshValidatorNodeInfo(&P2PValidatorNodeInfo{ 1079 Node: *srv.Self(), 1080 TimeStamp: time.Now(), 1081 Validator: validator, 1082 Original: true, 1083 }, nil) 1084 } 1085 1086 func (srv *Server) RemoveLocalValidator(chainId string, address common.Address) { 1087 1088 log.Debug("RemoveLocalValidator") 1089 1090 validator := P2PValidator{ 1091 ChainId: chainId, 1092 Address: address, 1093 } 1094 1095 idx := -1 1096 for i := 0; i < len(srv.LocalValidators); i++ { 1097 if validator == srv.LocalValidators[i] { 1098 idx = i 1099 break 1100 } 1101 } 1102 1103 if idx < 0 { 1104 return 1105 } 1106 1107 srv.LocalValidators = append(srv.LocalValidators[:idx], srv.LocalValidators[idx+1:]...) 1108 1109 srv.broadcastRemoveValidatorNodeInfo(&P2PValidatorNodeInfo{ 1110 Node: *srv.Self(), 1111 TimeStamp: time.Now(), 1112 Validator: validator, 1113 Original: true, 1114 }, nil) 1115 } 1116 1117 func (srv *Server) validatorAdd(valNodeInfo P2PValidatorNodeInfo, peers []*Peer, dialstate dialer) error { 1118 1119 log.Debug("validatorAdd") 1120 1121 validator := valNodeInfo.Validator 1122 1123 //if the node info goes from this node, just skip 1124 if srv.Self().ID == valNodeInfo.Node.ID { 1125 return nil 1126 } 1127 1128 //if the node does exist, we skip it; this could also avoid repeated propagate 1129 if nodeInfo, ok := srv.Validators[validator]; ok { 1130 con1 := valNodeInfo.Node.ID == nodeInfo.Node.ID 1131 con2 := valNodeInfo.Node.IP.String() == nodeInfo.Node.IP.String() 1132 log.Debugf("con1: %v, con2: %v", con1, con2) 1133 if con1 && con2 /*not compare PORT*/ { 1134 log.Debug("validator found, not add") 1135 return nil 1136 } 1137 } 1138 1139 log.Debug("validator not found") 1140 srv.Validators[validator] = &valNodeInfo 1141 inSameChain := false 1142 //if this validator is in the same chains which we join as validator, connect to it 1143 for i := 0; i < len(srv.LocalValidators); i++ { 1144 if validator.ChainId == srv.LocalValidators[i].ChainId { 1145 inSameChain = true 1146 break 1147 } 1148 } 1149 1150 notPeer := true 1151 for _, p := range peers { 1152 if p.ID() == valNodeInfo.Node.ID { 1153 notPeer = false 1154 break 1155 } 1156 } 1157 1158 if inSameChain && notPeer { 1159 dialstate.addStatic(&valNodeInfo.Node) 1160 } 1161 1162 //broadcast this node info to peers 1163 //srv.broadcastRefreshValidatorNodeInfo(&valNodeInfo, peers) 1164 1165 return nil 1166 } 1167 1168 func (srv *Server) validatorRemove(valNodeInfo P2PValidatorNodeInfo, peers []*Peer, dialstate dialer) error { 1169 1170 log.Debug("validatorRemove") 1171 1172 //if the node info goes from this node, just skip 1173 if srv.Self().ID == valNodeInfo.Node.ID { 1174 return nil 1175 } 1176 1177 validator := valNodeInfo.Validator 1178 //if the node does not exist, we skip it 1179 // in some cases the msg may need send, but ignoring it does not matter 1180 if _, ok := srv.Validators[validator]; !ok { 1181 return nil 1182 } 1183 1184 delete(srv.Validators, validator) 1185 1186 inSameChain := 0 1187 //if this validator is in the same chains which we join as validator 1188 for i := 0; i < len(srv.LocalValidators); i++ { 1189 if validator.ChainId == srv.LocalValidators[i].ChainId { 1190 inSameChain++ 1191 } 1192 } 1193 1194 shouldRemove := false 1195 //if there is only one validator in one chain connected with this peer, disconnect 1196 if inSameChain == 1 { 1197 for _, p := range peers { 1198 if p.ID() == valNodeInfo.Node.ID { 1199 //there peer is connected only for we are validator in one single chain, now can remove it 1200 shouldRemove = true 1201 break 1202 } 1203 } 1204 } 1205 1206 if shouldRemove { 1207 dialstate.removeStatic(&valNodeInfo.Node) 1208 } 1209 1210 //broadcast this node info to peers 1211 //srv.broadcastRemoveValidatorNodeInfo(&valNodeInfo, peers) 1212 1213 return nil 1214 } 1215 1216 func (srv *Server) validatorAddPeer(peer *Peer) error { 1217 1218 log.Debug("validatorAddPeer") 1219 1220 sendList := make([]*NodeInfoToSend, 0) 1221 1222 var err error = nil 1223 for _, validatorNodeInfo := range srv.Validators { 1224 1225 if peer.ID() == validatorNodeInfo.Node.ID { 1226 //refresh the node's ip, and no need to send the info back to the peer itself 1227 validatorNodeInfo.Node.IP = peer.RemoteAddr().(*net.TCPAddr).IP 1228 continue 1229 } 1230 1231 /* 1232 err1 := Send(peer.rw, RefreshValidatorNodeInfoMsg, validatorNodeInfo) 1233 if err == nil && err1 != nil { 1234 err = err1 1235 } 1236 */ 1237 sendList = append(sendList, &NodeInfoToSend{ 1238 valNodeInfo: validatorNodeInfo, 1239 action: RefreshValidatorNodeInfoMsg, 1240 p: peer, 1241 }) 1242 } 1243 1244 node := *srv.Self() 1245 for i := 0; i < len(srv.LocalValidators); i++ { 1246 /* 1247 err1 := Send(peer.rw, RefreshValidatorNodeInfoMsg, P2PValidatorNodeInfo{ 1248 Node: node, 1249 TimeStamp: time.Now(), 1250 Validator: srv.LocalValidators[i], 1251 Original: true, 1252 }) 1253 if err == nil && err1 != nil { 1254 err = err1 1255 } 1256 */ 1257 sendList = append(sendList, &NodeInfoToSend{ 1258 valNodeInfo: &P2PValidatorNodeInfo{ 1259 Node: node, 1260 TimeStamp: time.Now(), 1261 Validator: srv.LocalValidators[i], 1262 Original: true, 1263 }, 1264 action: RefreshValidatorNodeInfoMsg, 1265 p: peer, 1266 }) 1267 } 1268 1269 srv.addNodeInfoToSend(sendList) 1270 1271 return err 1272 } 1273 1274 func (srv *Server) validatorDelPeer(nodeId discover.NodeID) error { 1275 1276 log.Debug("validatorDelPeer") 1277 1278 srv.nodeInfoLock.Lock() 1279 defer srv.nodeInfoLock.Unlock() 1280 1281 tailIndex := 0 1282 for i := 0; i < len(srv.nodeInfoList); i++ { 1283 1284 nodeInfo := srv.nodeInfoList[i] 1285 if nodeInfo.valNodeInfo.Node.ID != nodeId { 1286 if i != tailIndex { 1287 srv.nodeInfoList[tailIndex] = nodeInfo 1288 } 1289 tailIndex++ 1290 } 1291 } 1292 1293 removedCount := len(srv.nodeInfoList) - 1 - tailIndex 1294 1295 srv.nodeInfoList = srv.nodeInfoList[:tailIndex] 1296 1297 log.Debugf("removed %v node info to send to %v", removedCount, nodeId) 1298 1299 return nil 1300 } 1301 1302 func (srv *Server) broadcastRefreshValidatorNodeInfo(data *P2PValidatorNodeInfo, peers []*Peer) { 1303 1304 log.Debug("broadcastRefreshValidatorNodeInfo") 1305 if peers == nil { 1306 peers = srv.Peers() 1307 } 1308 1309 sendList := make([]*NodeInfoToSend, 0) 1310 for _, p := range peers { 1311 1312 sendList = append(sendList, &NodeInfoToSend{ 1313 valNodeInfo: data, 1314 action: RefreshValidatorNodeInfoMsg, 1315 p: p, 1316 }) 1317 //Send(p.rw, RefreshValidatorNodeInfoMsg, data) 1318 } 1319 1320 srv.addNodeInfoToSend(sendList) 1321 } 1322 1323 func (srv *Server) broadcastRemoveValidatorNodeInfo(data interface{}, peers []*Peer) { 1324 1325 log.Debug("broadcastRemoveValidatorNodeInfo") 1326 if peers == nil { 1327 peers = srv.Peers() 1328 } 1329 1330 sendList := make([]*NodeInfoToSend, 0) 1331 for _, p := range peers { 1332 1333 sendList = append(sendList, &NodeInfoToSend{ 1334 valNodeInfo: data.(*P2PValidatorNodeInfo), 1335 action: RemoveValidatorNodeInfoMsg, 1336 p: p, 1337 }) 1338 //Send(p.rw, RemoveValidatorNodeInfoMsg, data) 1339 } 1340 1341 srv.addNodeInfoToSend(sendList) 1342 } 1343 1344 func (srv *Server) addNodeInfoToSend(sendList []*NodeInfoToSend) { 1345 1346 srv.nodeInfoLock.Lock() 1347 defer srv.nodeInfoLock.Unlock() 1348 1349 srv.nodeInfoList = append(srv.nodeInfoList, sendList...) 1350 } 1351 1352 //this function send validator information to otheres, every 100mimsecond send one 1353 //currently just handle the refresh action, not remove action 1354 func (srv *Server) sendValidatorNodeInfoMessages() { 1355 1356 sleepDuration := 100 * time.Millisecond // Time to sleep before send next message 1357 1358 for srv.running { 1359 1360 if len(srv.nodeInfoList) > 0 { 1361 srv.nodeInfoLock.Lock() 1362 1363 nodeInfo := srv.nodeInfoList[0] 1364 srv.nodeInfoList = srv.nodeInfoList[1:] 1365 1366 srv.nodeInfoLock.Unlock() 1367 1368 if nodeInfo != nil && 1369 nodeInfo.valNodeInfo != nil && 1370 nodeInfo.p != nil && 1371 nodeInfo.p.rw != nil && 1372 nodeInfo.p.rw.fd != nil { 1373 1374 Send(nodeInfo.p.rw, nodeInfo.action, nodeInfo.valNodeInfo) 1375 1376 log.Debugf("send node info (%v, %v) to %v", 1377 nodeInfo.valNodeInfo.Validator.Address.String(), nodeInfo.valNodeInfo.Node.ID, 1378 nodeInfo.p.ID()) 1379 } 1380 } 1381 1382 time.Sleep(sleepDuration) 1383 } 1384 }