github.com/kisexp/xdchain@v0.0.0-20211206025815-490d6b732aa7/p2p/server.go (about) 1 // Copyright 2014 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 // Package p2p implements the Ethereum p2p network protocols. 18 package p2p 19 20 import ( 21 "bytes" 22 "crypto/ecdsa" 23 "encoding/hex" 24 "errors" 25 "fmt" 26 "net" 27 "sort" 28 "sync" 29 "sync/atomic" 30 "time" 31 32 "github.com/kisexp/xdchain/common" 33 "github.com/kisexp/xdchain/common/mclock" 34 "github.com/kisexp/xdchain/crypto" 35 "github.com/kisexp/xdchain/event" 36 "github.com/kisexp/xdchain/log" 37 "github.com/kisexp/xdchain/p2p/discover" 38 "github.com/kisexp/xdchain/p2p/discv5" 39 "github.com/kisexp/xdchain/p2p/enode" 40 "github.com/kisexp/xdchain/p2p/enr" 41 "github.com/kisexp/xdchain/p2p/nat" 42 "github.com/kisexp/xdchain/p2p/netutil" 43 "github.com/kisexp/xdchain/permission/core" 44 ) 45 46 const ( 47 defaultDialTimeout = 15 * time.Second 48 49 // This is the fairness knob for the discovery mixer. When looking for peers, we'll 50 // wait this long for a single source of candidates before moving on and trying other 51 // sources. 52 discmixTimeout = 5 * time.Second 53 54 // Connectivity defaults. 55 defaultMaxPendingPeers = 50 56 defaultDialRatio = 3 57 58 // This time limits inbound connection attempts per source IP. 59 inboundThrottleTime = 30 * time.Second 60 61 // Maximum time allowed for reading a complete message. 62 // This is effectively the amount of time a connection can be idle. 63 frameReadTimeout = 30 * time.Second 64 65 // Maximum amount of time allowed for writing a complete message. 66 frameWriteTimeout = 20 * time.Second 67 ) 68 69 var errServerStopped = errors.New("server stopped") 70 71 // Config holds Server options. 72 type Config struct { 73 // This field must be set to a valid secp256k1 private key. 74 PrivateKey *ecdsa.PrivateKey `toml:"-"` 75 76 // MaxPeers is the maximum number of peers that can be 77 // connected. It must be greater than zero. 78 MaxPeers int 79 80 // MaxPendingPeers is the maximum number of peers that can be pending in the 81 // handshake phase, counted separately for inbound and outbound connections. 82 // Zero defaults to preset values. 83 MaxPendingPeers int `toml:",omitempty"` 84 85 // DialRatio controls the ratio of inbound to dialed connections. 86 // Example: a DialRatio of 2 allows 1/2 of connections to be dialed. 87 // Setting DialRatio to zero defaults it to 3. 88 DialRatio int `toml:",omitempty"` 89 90 // NoDiscovery can be used to disable the peer discovery mechanism. 91 // Disabling is useful for protocol debugging (manual topology). 92 NoDiscovery bool 93 94 // DiscoveryV5 specifies whether the new topic-discovery based V5 discovery 95 // protocol should be started or not. 96 DiscoveryV5 bool `toml:",omitempty"` 97 98 // Name sets the node name of this server. 99 // Use common.MakeName to create a name that follows existing conventions. 100 Name string `toml:"-"` 101 102 // BootstrapNodes are used to establish connectivity 103 // with the rest of the network. 104 BootstrapNodes []*enode.Node 105 106 // BootstrapNodesV5 are used to establish connectivity 107 // with the rest of the network using the V5 discovery 108 // protocol. 109 BootstrapNodesV5 []*discv5.Node `toml:",omitempty"` 110 111 // Static nodes are used as pre-configured connections which are always 112 // maintained and re-connected on disconnects. 113 StaticNodes []*enode.Node 114 115 // Trusted nodes are used as pre-configured connections which are always 116 // allowed to connect, even above the peer limit. 117 TrustedNodes []*enode.Node 118 119 // Connectivity can be restricted to certain IP networks. 120 // If this option is set to a non-nil value, only hosts which match one of the 121 // IP networks contained in the list are considered. 122 NetRestrict *netutil.Netlist `toml:",omitempty"` 123 124 // NodeDatabase is the path to the database containing the previously seen 125 // live nodes in the network. 126 NodeDatabase string `toml:",omitempty"` 127 128 // Protocols should contain the protocols supported 129 // by the server. Matching protocols are launched for 130 // each peer. 131 Protocols []Protocol `toml:"-"` 132 133 // If ListenAddr is set to a non-nil address, the server 134 // will listen for incoming connections. 135 // 136 // If the port is zero, the operating system will pick a port. The 137 // ListenAddr field will be updated with the actual address when 138 // the server is started. 139 ListenAddr string 140 141 // If set to a non-nil value, the given NAT port mapper 142 // is used to make the listening port available to the 143 // Internet. 144 NAT nat.Interface `toml:",omitempty"` 145 146 // If Dialer is set to a non-nil value, the given Dialer 147 // is used to dial outbound peer connections. 148 Dialer NodeDialer `toml:"-"` 149 150 // If NoDial is true, the server will not dial any peers. 151 NoDial bool `toml:",omitempty"` 152 153 // If EnableMsgEvents is set then the server will emit PeerEvents 154 // whenever a message is sent to or received from a peer 155 EnableMsgEvents bool 156 157 EnableNodePermission bool `toml:",omitempty"` 158 159 DataDir string `toml:",omitempty"` 160 // Logger is a custom logger to use with the p2p.Server. 161 Logger log.Logger `toml:",omitempty"` 162 163 clock mclock.Clock 164 } 165 166 // Server manages all peer connections. 167 type Server struct { 168 // Config fields may not be modified while the server is running. 169 Config 170 171 // Hooks for testing. These are useful because we can inhibit 172 // the whole protocol stack. 173 newTransport func(net.Conn, *ecdsa.PublicKey) transport 174 newPeerHook func(*Peer) 175 listenFunc func(network, addr string) (net.Listener, error) 176 177 lock sync.Mutex // protects running 178 running bool 179 180 listener net.Listener 181 ourHandshake *protoHandshake 182 loopWG sync.WaitGroup // loop, listenLoop 183 peerFeed event.Feed 184 log log.Logger 185 186 nodedb *enode.DB 187 localnode *enode.LocalNode 188 ntab *discover.UDPv4 189 DiscV5 *discv5.Network 190 discmix *enode.FairMix 191 dialsched *dialScheduler 192 193 // Channels into the run loop. 194 quit chan struct{} 195 addtrusted chan *enode.Node 196 removetrusted chan *enode.Node 197 peerOp chan peerOpFunc 198 peerOpDone chan struct{} 199 delpeer chan peerDrop 200 checkpointPostHandshake chan *conn 201 checkpointAddPeer chan *conn 202 203 // State of run loop and listenLoop. 204 inboundHistory expHeap 205 206 // raft peers info 207 checkPeerInRaft func(*enode.Node) bool 208 209 // permissions - check if node is permissioned 210 isNodePermissionedFunc func(node *enode.Node, nodename string, currentNode string, datadir string, direction string) bool 211 } 212 213 type peerOpFunc func(map[enode.ID]*Peer) 214 215 type peerDrop struct { 216 *Peer 217 err error 218 requested bool // true if signaled by the peer 219 } 220 221 type connFlag int32 222 223 const ( 224 dynDialedConn connFlag = 1 << iota 225 staticDialedConn 226 inboundConn 227 trustedConn 228 ) 229 230 // conn wraps a network connection with information gathered 231 // during the two handshakes. 232 type conn struct { 233 fd net.Conn 234 transport 235 node *enode.Node 236 flags connFlag 237 cont chan error // The run loop uses cont to signal errors to SetupConn. 238 caps []Cap // valid after the protocol handshake 239 name string // valid after the protocol handshake 240 } 241 242 type transport interface { 243 // The two handshakes. 244 doEncHandshake(prv *ecdsa.PrivateKey) (*ecdsa.PublicKey, error) 245 doProtoHandshake(our *protoHandshake) (*protoHandshake, error) 246 // The MsgReadWriter can only be used after the encryption 247 // handshake has completed. The code uses conn.id to track this 248 // by setting it to a non-nil value after the encryption handshake. 249 MsgReadWriter 250 // transports must provide Close because we use MsgPipe in some of 251 // the tests. Closing the actual network connection doesn't do 252 // anything in those tests because MsgPipe doesn't use it. 253 close(err error) 254 } 255 256 func (c *conn) String() string { 257 s := c.flags.String() 258 if (c.node.ID() != enode.ID{}) { 259 s += " " + c.node.ID().String() 260 } 261 s += " " + c.fd.RemoteAddr().String() 262 return s 263 } 264 265 func (f connFlag) String() string { 266 s := "" 267 if f&trustedConn != 0 { 268 s += "-trusted" 269 } 270 if f&dynDialedConn != 0 { 271 s += "-dyndial" 272 } 273 if f&staticDialedConn != 0 { 274 s += "-staticdial" 275 } 276 if f&inboundConn != 0 { 277 s += "-inbound" 278 } 279 if s != "" { 280 s = s[1:] 281 } 282 return s 283 } 284 285 func (c *conn) is(f connFlag) bool { 286 flags := connFlag(atomic.LoadInt32((*int32)(&c.flags))) 287 return flags&f != 0 288 } 289 290 func (c *conn) set(f connFlag, val bool) { 291 for { 292 oldFlags := connFlag(atomic.LoadInt32((*int32)(&c.flags))) 293 flags := oldFlags 294 if val { 295 flags |= f 296 } else { 297 flags &= ^f 298 } 299 if atomic.CompareAndSwapInt32((*int32)(&c.flags), int32(oldFlags), int32(flags)) { 300 return 301 } 302 } 303 } 304 305 // LocalNode returns the local node record. 306 func (srv *Server) LocalNode() *enode.LocalNode { 307 return srv.localnode 308 } 309 310 // Peers returns all connected peers. 311 func (srv *Server) Peers() []*Peer { 312 var ps []*Peer 313 srv.doPeerOp(func(peers map[enode.ID]*Peer) { 314 for _, p := range peers { 315 ps = append(ps, p) 316 } 317 }) 318 return ps 319 } 320 321 // PeerCount returns the number of connected peers. 322 func (srv *Server) PeerCount() int { 323 var count int 324 srv.doPeerOp(func(ps map[enode.ID]*Peer) { 325 count = len(ps) 326 }) 327 return count 328 } 329 330 // AddPeer adds the given node to the static node set. When there is room in the peer set, 331 // the server will connect to the node. If the connection fails for any reason, the server 332 // will attempt to reconnect the peer. 333 func (srv *Server) AddPeer(node *enode.Node) { 334 srv.dialsched.addStatic(node) 335 } 336 337 // RemovePeer removes a node from the static node set. It also disconnects from the given 338 // node if it is currently connected as a peer. 339 // 340 // This method blocks until all protocols have exited and the peer is removed. Do not use 341 // RemovePeer in protocol implementations, call Disconnect on the Peer instead. 342 func (srv *Server) RemovePeer(node *enode.Node) { 343 var ( 344 ch chan *PeerEvent 345 sub event.Subscription 346 ) 347 // Disconnect the peer on the main loop. 348 srv.doPeerOp(func(peers map[enode.ID]*Peer) { 349 srv.dialsched.removeStatic(node) 350 if peer := peers[node.ID()]; peer != nil { 351 ch = make(chan *PeerEvent, 1) 352 sub = srv.peerFeed.Subscribe(ch) 353 peer.Disconnect(DiscRequested) 354 } 355 }) 356 // Wait for the peer connection to end. 357 if ch != nil { 358 defer sub.Unsubscribe() 359 for ev := range ch { 360 if ev.Peer == node.ID() && ev.Type == PeerEventTypeDrop { 361 return 362 } 363 } 364 } 365 } 366 367 // AddTrustedPeer adds the given node to a reserved whitelist which allows the 368 // node to always connect, even if the slot are full. 369 func (srv *Server) AddTrustedPeer(node *enode.Node) { 370 select { 371 case srv.addtrusted <- node: 372 case <-srv.quit: 373 } 374 } 375 376 // RemoveTrustedPeer removes the given node from the trusted peer set. 377 func (srv *Server) RemoveTrustedPeer(node *enode.Node) { 378 select { 379 case srv.removetrusted <- node: 380 case <-srv.quit: 381 } 382 } 383 384 // SubscribePeers subscribes the given channel to peer events 385 func (srv *Server) SubscribeEvents(ch chan *PeerEvent) event.Subscription { 386 return srv.peerFeed.Subscribe(ch) 387 } 388 389 // Self returns the local node's endpoint information. 390 func (srv *Server) Self() *enode.Node { 391 srv.lock.Lock() 392 ln := srv.localnode 393 srv.lock.Unlock() 394 395 if ln == nil { 396 return enode.NewV4(&srv.PrivateKey.PublicKey, net.ParseIP("0.0.0.0"), 0, 0) 397 } 398 return ln.Node() 399 } 400 401 // Stop terminates the server and all active peer connections. 402 // It blocks until all active connections have been closed. 403 func (srv *Server) Stop() { 404 srv.lock.Lock() 405 if !srv.running { 406 srv.lock.Unlock() 407 return 408 } 409 srv.running = false 410 if srv.listener != nil { 411 // this unblocks listener Accept 412 srv.listener.Close() 413 } 414 close(srv.quit) 415 srv.lock.Unlock() 416 srv.loopWG.Wait() 417 } 418 419 // sharedUDPConn implements a shared connection. Write sends messages to the underlying connection while read returns 420 // messages that were found unprocessable and sent to the unhandled channel by the primary listener. 421 type sharedUDPConn struct { 422 *net.UDPConn 423 unhandled chan discover.ReadPacket 424 } 425 426 // ReadFromUDP implements discv5.conn 427 func (s *sharedUDPConn) ReadFromUDP(b []byte) (n int, addr *net.UDPAddr, err error) { 428 packet, ok := <-s.unhandled 429 if !ok { 430 return 0, nil, errors.New("connection was closed") 431 } 432 l := len(packet.Data) 433 if l > len(b) { 434 l = len(b) 435 } 436 copy(b[:l], packet.Data[:l]) 437 return l, packet.Addr, nil 438 } 439 440 // Close implements discv5.conn 441 func (s *sharedUDPConn) Close() error { 442 return nil 443 } 444 445 // Start starts running the server. 446 // Servers can not be re-used after stopping. 447 func (srv *Server) Start() (err error) { 448 srv.lock.Lock() 449 defer srv.lock.Unlock() 450 if srv.running { 451 return errors.New("server already running") 452 } 453 srv.running = true 454 srv.log = srv.Config.Logger 455 if srv.log == nil { 456 srv.log = log.Root() 457 } 458 if srv.clock == nil { 459 srv.clock = mclock.System{} 460 } 461 if srv.NoDial && srv.ListenAddr == "" { 462 srv.log.Warn("P2P server will be useless, neither dialing nor listening") 463 } 464 465 // static fields 466 if srv.PrivateKey == nil { 467 return errors.New("Server.PrivateKey must be set to a non-nil key") 468 } 469 if srv.newTransport == nil { 470 srv.newTransport = newRLPX 471 } 472 if srv.listenFunc == nil { 473 srv.listenFunc = net.Listen 474 } 475 srv.quit = make(chan struct{}) 476 srv.delpeer = make(chan peerDrop) 477 srv.checkpointPostHandshake = make(chan *conn) 478 srv.checkpointAddPeer = make(chan *conn) 479 srv.addtrusted = make(chan *enode.Node) 480 srv.removetrusted = make(chan *enode.Node) 481 srv.peerOp = make(chan peerOpFunc) 482 srv.peerOpDone = make(chan struct{}) 483 484 if err := srv.setupLocalNode(); err != nil { 485 return err 486 } 487 if srv.ListenAddr != "" { 488 if err := srv.setupListening(); err != nil { 489 return err 490 } 491 } 492 if err := srv.setupDiscovery(); err != nil { 493 return err 494 } 495 srv.setupDialScheduler() 496 497 srv.loopWG.Add(1) 498 go srv.run() 499 return nil 500 } 501 502 func (srv *Server) setupLocalNode() error { 503 // Create the devp2p handshake. 504 pubkey := crypto.FromECDSAPub(&srv.PrivateKey.PublicKey) 505 srv.ourHandshake = &protoHandshake{Version: baseProtocolVersion, Name: srv.Name, ID: pubkey[1:]} 506 for _, p := range srv.Protocols { 507 srv.ourHandshake.Caps = append(srv.ourHandshake.Caps, p.cap()) 508 } 509 sort.Sort(capsByNameAndVersion(srv.ourHandshake.Caps)) 510 511 // Create the local node. 512 db, err := enode.OpenDB(srv.Config.NodeDatabase) 513 if err != nil { 514 return err 515 } 516 srv.nodedb = db 517 srv.localnode = enode.NewLocalNode(db, srv.PrivateKey) 518 srv.localnode.SetFallbackIP(net.IP{127, 0, 0, 1}) 519 // TODO: check conflicts 520 for _, p := range srv.Protocols { 521 for _, e := range p.Attributes { 522 srv.localnode.Set(e) 523 } 524 } 525 switch srv.NAT.(type) { 526 case nil: 527 // No NAT interface, do nothing. 528 case nat.ExtIP: 529 // ExtIP doesn't block, set the IP right away. 530 ip, _ := srv.NAT.ExternalIP() 531 srv.localnode.SetStaticIP(ip) 532 default: 533 // Ask the router about the IP. This takes a while and blocks startup, 534 // do it in the background. 535 srv.loopWG.Add(1) 536 go func() { 537 defer srv.loopWG.Done() 538 if ip, err := srv.NAT.ExternalIP(); err == nil { 539 srv.localnode.SetStaticIP(ip) 540 } 541 }() 542 } 543 return nil 544 } 545 546 func (srv *Server) setupDiscovery() error { 547 srv.discmix = enode.NewFairMix(discmixTimeout) 548 549 // Add protocol-specific discovery sources. 550 added := make(map[string]bool) 551 for _, proto := range srv.Protocols { 552 if proto.DialCandidates != nil && !added[proto.Name] { 553 srv.discmix.AddSource(proto.DialCandidates) 554 added[proto.Name] = true 555 } 556 } 557 558 // Don't listen on UDP endpoint if DHT is disabled. 559 if srv.NoDiscovery && !srv.DiscoveryV5 { 560 return nil 561 } 562 563 addr, err := net.ResolveUDPAddr("udp", srv.ListenAddr) 564 if err != nil { 565 return err 566 } 567 conn, err := net.ListenUDP("udp", addr) 568 if err != nil { 569 return err 570 } 571 realaddr := conn.LocalAddr().(*net.UDPAddr) 572 srv.log.Debug("UDP listener up", "addr", realaddr) 573 if srv.NAT != nil { 574 if !realaddr.IP.IsLoopback() { 575 srv.loopWG.Add(1) 576 go func() { 577 nat.Map(srv.NAT, srv.quit, "udp", realaddr.Port, realaddr.Port, "ethereum discovery") 578 srv.loopWG.Done() 579 }() 580 } 581 } 582 srv.localnode.SetFallbackUDP(realaddr.Port) 583 584 // Discovery V4 585 var unhandled chan discover.ReadPacket 586 var sconn *sharedUDPConn 587 if !srv.NoDiscovery { 588 if srv.DiscoveryV5 { 589 unhandled = make(chan discover.ReadPacket, 100) 590 sconn = &sharedUDPConn{conn, unhandled} 591 } 592 cfg := discover.Config{ 593 PrivateKey: srv.PrivateKey, 594 NetRestrict: srv.NetRestrict, 595 Bootnodes: srv.BootstrapNodes, 596 Unhandled: unhandled, 597 Log: srv.log, 598 } 599 ntab, err := discover.ListenUDP(conn, srv.localnode, cfg) 600 if err != nil { 601 return err 602 } 603 srv.ntab = ntab 604 srv.discmix.AddSource(ntab.RandomNodes()) 605 } 606 607 // Discovery V5 608 if srv.DiscoveryV5 { 609 var ntab *discv5.Network 610 var err error 611 if sconn != nil { 612 ntab, err = discv5.ListenUDP(srv.PrivateKey, sconn, "", srv.NetRestrict) 613 } else { 614 ntab, err = discv5.ListenUDP(srv.PrivateKey, conn, "", srv.NetRestrict) 615 } 616 if err != nil { 617 return err 618 } 619 if err := ntab.SetFallbackNodes(srv.BootstrapNodesV5); err != nil { 620 return err 621 } 622 srv.DiscV5 = ntab 623 } 624 return nil 625 } 626 627 func (srv *Server) setupDialScheduler() { 628 config := dialConfig{ 629 self: srv.localnode.ID(), 630 maxDialPeers: srv.maxDialedConns(), 631 maxActiveDials: srv.MaxPendingPeers, 632 log: srv.Logger, 633 netRestrict: srv.NetRestrict, 634 dialer: srv.Dialer, 635 clock: srv.clock, 636 } 637 if srv.ntab != nil { 638 config.resolver = srv.ntab 639 } 640 if config.dialer == nil { 641 config.dialer = tcpDialer{&net.Dialer{Timeout: defaultDialTimeout}} 642 } 643 srv.dialsched = newDialScheduler(config, srv.discmix, srv.SetupConn) 644 for _, n := range srv.StaticNodes { 645 srv.dialsched.addStatic(n) 646 } 647 } 648 649 func (srv *Server) maxInboundConns() int { 650 return srv.MaxPeers - srv.maxDialedConns() 651 } 652 653 func (srv *Server) maxDialedConns() (limit int) { 654 if srv.NoDial || srv.MaxPeers == 0 { 655 return 0 656 } 657 if srv.DialRatio == 0 { 658 limit = srv.MaxPeers / defaultDialRatio 659 } else { 660 limit = srv.MaxPeers / srv.DialRatio 661 } 662 if limit == 0 { 663 limit = 1 664 } 665 return limit 666 } 667 668 func (srv *Server) setupListening() error { 669 // Launch the listener. 670 listener, err := srv.listenFunc("tcp", srv.ListenAddr) 671 if err != nil { 672 return err 673 } 674 srv.listener = listener 675 srv.ListenAddr = listener.Addr().String() 676 677 // Update the local node record and map the TCP listening port if NAT is configured. 678 if tcp, ok := listener.Addr().(*net.TCPAddr); ok { 679 srv.localnode.Set(enr.TCP(tcp.Port)) 680 if !tcp.IP.IsLoopback() && srv.NAT != nil { 681 srv.loopWG.Add(1) 682 go func() { 683 nat.Map(srv.NAT, srv.quit, "tcp", tcp.Port, tcp.Port, "ethereum p2p") 684 srv.loopWG.Done() 685 }() 686 } 687 } 688 689 srv.loopWG.Add(1) 690 go srv.listenLoop() 691 return nil 692 } 693 694 // doPeerOp runs fn on the main loop. 695 func (srv *Server) doPeerOp(fn peerOpFunc) { 696 select { 697 case srv.peerOp <- fn: 698 <-srv.peerOpDone 699 case <-srv.quit: 700 } 701 } 702 703 // run is the main loop of the server. 704 func (srv *Server) run() { 705 srv.log.Info("Started P2P networking", "self", srv.localnode.Node().URLv4()) 706 defer srv.loopWG.Done() 707 defer srv.nodedb.Close() 708 defer srv.discmix.Close() 709 defer srv.dialsched.stop() 710 711 var ( 712 peers = make(map[enode.ID]*Peer) 713 inboundCount = 0 714 trusted = make(map[enode.ID]bool, len(srv.TrustedNodes)) 715 ) 716 // Put trusted nodes into a map to speed up checks. 717 // Trusted peers are loaded on startup or added via AddTrustedPeer RPC. 718 for _, n := range srv.TrustedNodes { 719 trusted[n.ID()] = true 720 } 721 722 running: 723 for { 724 select { 725 case <-srv.quit: 726 // The server was stopped. Run the cleanup logic. 727 break running 728 729 case n := <-srv.addtrusted: 730 // This channel is used by AddTrustedPeer to add a node 731 // to the trusted node set. 732 srv.log.Trace("Adding trusted node", "node", n) 733 trusted[n.ID()] = true 734 if p, ok := peers[n.ID()]; ok { 735 p.rw.set(trustedConn, true) 736 } 737 738 case n := <-srv.removetrusted: 739 // This channel is used by RemoveTrustedPeer to remove a node 740 // from the trusted node set. 741 srv.log.Trace("Removing trusted node", "node", n) 742 delete(trusted, n.ID()) 743 if p, ok := peers[n.ID()]; ok { 744 p.rw.set(trustedConn, false) 745 } 746 747 case op := <-srv.peerOp: 748 // This channel is used by Peers and PeerCount. 749 op(peers) 750 srv.peerOpDone <- struct{}{} 751 752 case c := <-srv.checkpointPostHandshake: 753 // A connection has passed the encryption handshake so 754 // the remote identity is known (but hasn't been verified yet). 755 if trusted[c.node.ID()] { 756 // Ensure that the trusted flag is set before checking against MaxPeers. 757 c.flags |= trustedConn 758 } 759 // TODO: track in-progress inbound node IDs (pre-Peer) to avoid dialing them. 760 c.cont <- srv.postHandshakeChecks(peers, inboundCount, c) 761 762 case c := <-srv.checkpointAddPeer: 763 // At this point the connection is past the protocol handshake. 764 // Its capabilities are known and the remote identity is verified. 765 err := srv.addPeerChecks(peers, inboundCount, c) 766 if err == nil { 767 // The handshakes are done and it passed all checks. 768 p := srv.launchPeer(c) 769 peers[c.node.ID()] = p 770 srv.log.Debug("Adding p2p peer", "peercount", len(peers), "id", p.ID(), "conn", c.flags, "addr", p.RemoteAddr(), "name", p.Name()) 771 srv.dialsched.peerAdded(c) 772 if p.Inbound() { 773 inboundCount++ 774 } 775 } 776 c.cont <- err 777 778 case pd := <-srv.delpeer: 779 // A peer disconnected. 780 d := common.PrettyDuration(mclock.Now() - pd.created) 781 delete(peers, pd.ID()) 782 srv.log.Debug("Removing p2p peer", "peercount", len(peers), "id", pd.ID(), "duration", d, "req", pd.requested, "err", pd.err) 783 srv.dialsched.peerRemoved(pd.rw) 784 if pd.Inbound() { 785 inboundCount-- 786 } 787 } 788 } 789 790 srv.log.Trace("P2P networking is spinning down") 791 792 // Terminate discovery. If there is a running lookup it will terminate soon. 793 if srv.ntab != nil { 794 srv.ntab.Close() 795 } 796 if srv.DiscV5 != nil { 797 srv.DiscV5.Close() 798 } 799 // Disconnect all peers. 800 for _, p := range peers { 801 p.Disconnect(DiscQuitting) 802 } 803 // Wait for peers to shut down. Pending connections and tasks are 804 // not handled here and will terminate soon-ish because srv.quit 805 // is closed. 806 for len(peers) > 0 { 807 p := <-srv.delpeer 808 p.log.Trace("<-delpeer (spindown)") 809 delete(peers, p.ID()) 810 } 811 } 812 813 func (srv *Server) postHandshakeChecks(peers map[enode.ID]*Peer, inboundCount int, c *conn) error { 814 switch { 815 case !c.is(trustedConn) && len(peers) >= srv.MaxPeers: 816 return DiscTooManyPeers 817 case !c.is(trustedConn) && c.is(inboundConn) && inboundCount >= srv.maxInboundConns(): 818 return DiscTooManyPeers 819 case peers[c.node.ID()] != nil: 820 return DiscAlreadyConnected 821 case c.node.ID() == srv.localnode.ID(): 822 return DiscSelf 823 default: 824 return nil 825 } 826 } 827 828 func (srv *Server) addPeerChecks(peers map[enode.ID]*Peer, inboundCount int, c *conn) error { 829 // Drop connections with no matching protocols. 830 if len(srv.Protocols) > 0 && countMatchingProtocols(srv.Protocols, c.caps) == 0 { 831 return DiscUselessPeer 832 } 833 // Repeat the post-handshake checks because the 834 // peer set might have changed since those checks were performed. 835 return srv.postHandshakeChecks(peers, inboundCount, c) 836 } 837 838 // listenLoop runs in its own goroutine and accepts 839 // inbound connections. 840 func (srv *Server) listenLoop() { 841 srv.log.Debug("TCP listener up", "addr", srv.listener.Addr()) 842 843 // The slots channel limits accepts of new connections. 844 tokens := defaultMaxPendingPeers 845 if srv.MaxPendingPeers > 0 { 846 tokens = srv.MaxPendingPeers 847 } 848 slots := make(chan struct{}, tokens) 849 for i := 0; i < tokens; i++ { 850 slots <- struct{}{} 851 } 852 853 // Wait for slots to be returned on exit. This ensures all connection goroutines 854 // are down before listenLoop returns. 855 defer srv.loopWG.Done() 856 defer func() { 857 for i := 0; i < cap(slots); i++ { 858 <-slots 859 } 860 }() 861 862 for { 863 // Wait for a free slot before accepting. 864 <-slots 865 866 var ( 867 fd net.Conn 868 err error 869 lastLog time.Time 870 ) 871 for { 872 fd, err = srv.listener.Accept() 873 if netutil.IsTemporaryError(err) { 874 if time.Since(lastLog) > 1*time.Second { 875 srv.log.Debug("Temporary read error", "err", err) 876 lastLog = time.Now() 877 } 878 time.Sleep(time.Millisecond * 200) 879 continue 880 } else if err != nil { 881 srv.log.Debug("Read error", "err", err) 882 slots <- struct{}{} 883 return 884 } 885 break 886 } 887 888 remoteIP := netutil.AddrIP(fd.RemoteAddr()) 889 if err := srv.checkInboundConn(fd, remoteIP); err != nil { 890 srv.log.Debug("Rejected inbound connnection", "addr", fd.RemoteAddr(), "err", err) 891 fd.Close() 892 slots <- struct{}{} 893 continue 894 } 895 if remoteIP != nil { 896 var addr *net.TCPAddr 897 if tcp, ok := fd.RemoteAddr().(*net.TCPAddr); ok { 898 addr = tcp 899 } 900 fd = newMeteredConn(fd, true, addr) 901 srv.log.Trace("Accepted connection", "addr", fd.RemoteAddr()) 902 } 903 go func() { 904 srv.SetupConn(fd, inboundConn, nil) 905 slots <- struct{}{} 906 }() 907 } 908 } 909 910 func (srv *Server) checkInboundConn(fd net.Conn, remoteIP net.IP) error { 911 if remoteIP == nil { 912 return nil 913 } 914 // Reject connections that do not match NetRestrict. 915 if srv.NetRestrict != nil && !srv.NetRestrict.Contains(remoteIP) { 916 return fmt.Errorf("not whitelisted in NetRestrict") 917 } 918 // Reject Internet peers that try too often. 919 now := srv.clock.Now() 920 srv.inboundHistory.expire(now, nil) 921 if !netutil.IsLAN(remoteIP) && srv.inboundHistory.contains(remoteIP.String()) { 922 return fmt.Errorf("too many attempts") 923 } 924 srv.inboundHistory.add(remoteIP.String(), now.Add(inboundThrottleTime)) 925 return nil 926 } 927 928 // SetupConn runs the handshakes and attempts to add the connection 929 // as a peer. It returns when the connection has been added as a peer 930 // or the handshakes have failed. 931 func (srv *Server) SetupConn(fd net.Conn, flags connFlag, dialDest *enode.Node) error { 932 c := &conn{fd: fd, flags: flags, cont: make(chan error)} 933 if dialDest == nil { 934 c.transport = srv.newTransport(fd, nil) 935 } else { 936 c.transport = srv.newTransport(fd, dialDest.Pubkey()) 937 } 938 939 err := srv.setupConn(c, flags, dialDest) 940 if err != nil { 941 c.close(err) 942 } 943 return err 944 } 945 946 func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) error { 947 // Prevent leftover pending conns from entering the handshake. 948 srv.lock.Lock() 949 running := srv.running 950 srv.lock.Unlock() 951 if !running { 952 return errServerStopped 953 } 954 955 // If dialing, figure out the remote public key. 956 var dialPubkey *ecdsa.PublicKey 957 if dialDest != nil { 958 dialPubkey = new(ecdsa.PublicKey) 959 if err := dialDest.Load((*enode.Secp256k1)(dialPubkey)); err != nil { 960 err = errors.New("dial destination doesn't have a secp256k1 public key") 961 srv.log.Trace("Setting up connection failed", "addr", c.fd.RemoteAddr(), "conn", c.flags, "err", err) 962 return err 963 } 964 } 965 966 // Run the RLPx handshake. 967 remotePubkey, err := c.doEncHandshake(srv.PrivateKey) 968 if err != nil { 969 srv.log.Trace("Failed RLPx handshake", "addr", c.fd.RemoteAddr(), "conn", c.flags, "err", err) 970 return err 971 } 972 973 if dialDest != nil { 974 c.node = dialDest 975 } else { 976 c.node = nodeFromConn(remotePubkey, c.fd) 977 } 978 clog := srv.log.New("id", c.node.ID(), "addr", c.fd.RemoteAddr(), "conn", c.flags) 979 980 // If raft is running, check if the dialing node is in the raft cluster 981 // Node doesn't belong to raft cluster is not allowed to join the p2p network 982 if srv.checkPeerInRaft != nil && !srv.checkPeerInRaft(c.node) { 983 node := c.node.ID().String() 984 log.Trace("incoming connection peer is not in the raft cluster", "enode.id", node) 985 return newPeerError(errNotInRaftCluster, "id=%s…%s", node[:4], node[len(node)-4:]) 986 } 987 988 //START - QUORUM Permissioning 989 currentNode := srv.NodeInfo().ID 990 cnodeName := srv.NodeInfo().Name 991 clog.Trace("Quorum permissioning", 992 "EnableNodePermission", srv.EnableNodePermission, 993 "DataDir", srv.DataDir, 994 "Current Node ID", currentNode, 995 "Node Name", cnodeName, 996 "Dialed Dest", dialDest, 997 "Connection ID", c.node.ID(), 998 "Connection String", c.node.ID().String()) 999 1000 if srv.EnableNodePermission { 1001 clog.Trace("Node Permissioning is Enabled.") 1002 nodeId := c.node.ID().String() 1003 node := c.node 1004 direction := "INCOMING" 1005 if dialDest != nil { 1006 node = dialDest 1007 nodeId = dialDest.ID().String() 1008 direction = "OUTGOING" 1009 log.Trace("Node Permissioning", "Connection Direction", direction) 1010 } 1011 1012 if srv.isNodePermissionedFunc == nil { 1013 if !core.IsNodePermissioned(nodeId, currentNode, srv.DataDir, direction) { 1014 return newPeerError(errPermissionDenied, "id=%s…%s %s id=%s…%s", currentNode[:4], currentNode[len(currentNode)-4:], direction, nodeId[:4], nodeId[len(nodeId)-4:]) 1015 } 1016 } else if !srv.isNodePermissionedFunc(node, nodeId, currentNode, srv.DataDir, direction) { 1017 return newPeerError(errPermissionDenied, "id=%s…%s %s id=%s…%s", currentNode[:4], currentNode[len(currentNode)-4:], direction, nodeId[:4], nodeId[len(nodeId)-4:]) 1018 } 1019 } else { 1020 clog.Trace("Node Permissioning is Disabled.") 1021 } 1022 1023 //END - QUORUM Permissioning 1024 1025 err = srv.checkpoint(c, srv.checkpointPostHandshake) 1026 if err != nil { 1027 clog.Trace("Rejected peer", "err", err) 1028 return err 1029 } 1030 1031 // Run the capability negotiation handshake. 1032 phs, err := c.doProtoHandshake(srv.ourHandshake) 1033 if err != nil { 1034 clog.Trace("Failed p2p handshake", "err", err) 1035 return err 1036 } 1037 if id := c.node.ID(); !bytes.Equal(crypto.Keccak256(phs.ID), id[:]) { 1038 clog.Trace("Wrong devp2p handshake identity", "phsid", hex.EncodeToString(phs.ID)) 1039 return DiscUnexpectedIdentity 1040 } 1041 c.caps, c.name = phs.Caps, phs.Name 1042 err = srv.checkpoint(c, srv.checkpointAddPeer) 1043 if err != nil { 1044 clog.Trace("Rejected peer", "err", err) 1045 return err 1046 } 1047 1048 return nil 1049 } 1050 1051 func nodeFromConn(pubkey *ecdsa.PublicKey, conn net.Conn) *enode.Node { 1052 var ip net.IP 1053 var port int 1054 if tcp, ok := conn.RemoteAddr().(*net.TCPAddr); ok { 1055 ip = tcp.IP 1056 port = tcp.Port 1057 } 1058 return enode.NewV4(pubkey, ip, port, port) 1059 } 1060 1061 // checkpoint sends the conn to run, which performs the 1062 // post-handshake checks for the stage (posthandshake, addpeer). 1063 func (srv *Server) checkpoint(c *conn, stage chan<- *conn) error { 1064 select { 1065 case stage <- c: 1066 case <-srv.quit: 1067 return errServerStopped 1068 } 1069 return <-c.cont 1070 } 1071 1072 func (srv *Server) launchPeer(c *conn) *Peer { 1073 p := newPeer(srv.log, c, srv.Protocols) 1074 if srv.EnableMsgEvents { 1075 // If message events are enabled, pass the peerFeed 1076 // to the peer. 1077 p.events = &srv.peerFeed 1078 } 1079 go srv.runPeer(p) 1080 return p 1081 } 1082 1083 // runPeer runs in its own goroutine for each peer. 1084 func (srv *Server) runPeer(p *Peer) { 1085 if srv.newPeerHook != nil { 1086 srv.newPeerHook(p) 1087 } 1088 srv.peerFeed.Send(&PeerEvent{ 1089 Type: PeerEventTypeAdd, 1090 Peer: p.ID(), 1091 RemoteAddress: p.RemoteAddr().String(), 1092 LocalAddress: p.LocalAddr().String(), 1093 }) 1094 1095 // Run the per-peer main loop. 1096 remoteRequested, err := p.run() 1097 1098 // Announce disconnect on the main loop to update the peer set. 1099 // The main loop waits for existing peers to be sent on srv.delpeer 1100 // before returning, so this send should not select on srv.quit. 1101 srv.delpeer <- peerDrop{p, err, remoteRequested} 1102 1103 // Broadcast peer drop to external subscribers. This needs to be 1104 // after the send to delpeer so subscribers have a consistent view of 1105 // the peer set (i.e. Server.Peers() doesn't include the peer when the 1106 // event is received. 1107 srv.peerFeed.Send(&PeerEvent{ 1108 Type: PeerEventTypeDrop, 1109 Peer: p.ID(), 1110 Error: err.Error(), 1111 RemoteAddress: p.RemoteAddr().String(), 1112 LocalAddress: p.LocalAddr().String(), 1113 }) 1114 } 1115 1116 // NodeInfo represents a short summary of the information known about the host. 1117 type NodeInfo struct { 1118 ID string `json:"id"` // Unique node identifier (also the encryption key) 1119 Name string `json:"name"` // Name of the node, including client type, version, OS, custom data 1120 Enode string `json:"enode"` // Enode URL for adding this peer from remote peers 1121 ENR string `json:"enr"` // Ethereum Node Record 1122 IP string `json:"ip"` // IP address of the node 1123 Ports struct { 1124 Discovery int `json:"discovery"` // UDP listening port for discovery protocol 1125 Listener int `json:"listener"` // TCP listening port for RLPx 1126 } `json:"ports"` 1127 ListenAddr string `json:"listenAddr"` 1128 Protocols map[string]interface{} `json:"protocols"` 1129 } 1130 1131 // NodeInfo gathers and returns a collection of metadata known about the host. 1132 func (srv *Server) NodeInfo() *NodeInfo { 1133 // Gather and assemble the generic node infos 1134 node := srv.Self() 1135 info := &NodeInfo{ 1136 Name: srv.Name, 1137 Enode: node.URLv4(), 1138 ID: node.ID().String(), 1139 IP: node.IP().String(), 1140 ListenAddr: srv.ListenAddr, 1141 Protocols: make(map[string]interface{}), 1142 } 1143 info.Ports.Discovery = node.UDP() 1144 info.Ports.Listener = node.TCP() 1145 info.ENR = node.String() 1146 1147 // Gather all the running protocol infos (only once per protocol type) 1148 for _, proto := range srv.Protocols { 1149 if _, ok := info.Protocols[proto.Name]; !ok { 1150 nodeInfo := interface{}("unknown") 1151 if query := proto.NodeInfo; query != nil { 1152 nodeInfo = proto.NodeInfo() 1153 } 1154 info.Protocols[proto.Name] = nodeInfo 1155 } 1156 } 1157 return info 1158 } 1159 1160 // PeersInfo returns an array of metadata objects describing connected peers. 1161 func (srv *Server) PeersInfo() []*PeerInfo { 1162 // Gather all the generic and sub-protocol specific infos 1163 infos := make([]*PeerInfo, 0, srv.PeerCount()) 1164 for _, peer := range srv.Peers() { 1165 if peer != nil { 1166 infos = append(infos, peer.Info()) 1167 } 1168 } 1169 // Sort the result array alphabetically by node identifier 1170 for i := 0; i < len(infos); i++ { 1171 for j := i + 1; j < len(infos); j++ { 1172 if infos[i].ID > infos[j].ID { 1173 infos[i], infos[j] = infos[j], infos[i] 1174 } 1175 } 1176 } 1177 return infos 1178 } 1179 1180 func (srv *Server) SetCheckPeerInRaft(f func(*enode.Node) bool) { 1181 srv.checkPeerInRaft = f 1182 } 1183 1184 func (srv *Server) SetIsNodePermissioned(f func(*enode.Node, string, string, string, string) bool) { 1185 if srv.isNodePermissionedFunc == nil { 1186 srv.isNodePermissionedFunc = f 1187 } 1188 }