github.com/ConsenSys/Quorum@v20.10.0+incompatible/p2p/server.go (about) 1 // Copyright 2014 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 // Package p2p implements the Ethereum p2p network protocols. 18 package p2p 19 20 import ( 21 "bytes" 22 "crypto/ecdsa" 23 "encoding/hex" 24 "errors" 25 "fmt" 26 "net" 27 "sort" 28 "sync" 29 "sync/atomic" 30 "time" 31 32 "github.com/ethereum/go-ethereum/common" 33 "github.com/ethereum/go-ethereum/common/mclock" 34 "github.com/ethereum/go-ethereum/crypto" 35 "github.com/ethereum/go-ethereum/event" 36 "github.com/ethereum/go-ethereum/log" 37 "github.com/ethereum/go-ethereum/p2p/discover" 38 "github.com/ethereum/go-ethereum/p2p/discv5" 39 "github.com/ethereum/go-ethereum/p2p/enode" 40 "github.com/ethereum/go-ethereum/p2p/enr" 41 "github.com/ethereum/go-ethereum/p2p/nat" 42 "github.com/ethereum/go-ethereum/p2p/netutil" 43 ) 44 45 const ( 46 defaultDialTimeout = 15 * time.Second 47 48 // This is the fairness knob for the discovery mixer. When looking for peers, we'll 49 // wait this long for a single source of candidates before moving on and trying other 50 // sources. 51 discmixTimeout = 5 * time.Second 52 53 // Connectivity defaults. 54 maxActiveDialTasks = 16 55 defaultMaxPendingPeers = 50 56 defaultDialRatio = 3 57 58 // This time limits inbound connection attempts per source IP. 59 inboundThrottleTime = 30 * time.Second 60 61 // Maximum time allowed for reading a complete message. 62 // This is effectively the amount of time a connection can be idle. 63 frameReadTimeout = 30 * time.Second 64 65 // Maximum amount of time allowed for writing a complete message. 66 frameWriteTimeout = 20 * time.Second 67 ) 68 69 var errServerStopped = errors.New("server stopped") 70 71 // Config holds Server options. 72 type Config struct { 73 // This field must be set to a valid secp256k1 private key. 74 PrivateKey *ecdsa.PrivateKey `toml:"-"` 75 76 // MaxPeers is the maximum number of peers that can be 77 // connected. It must be greater than zero. 78 MaxPeers int 79 80 // MaxPendingPeers is the maximum number of peers that can be pending in the 81 // handshake phase, counted separately for inbound and outbound connections. 82 // Zero defaults to preset values. 83 MaxPendingPeers int `toml:",omitempty"` 84 85 // DialRatio controls the ratio of inbound to dialed connections. 86 // Example: a DialRatio of 2 allows 1/2 of connections to be dialed. 87 // Setting DialRatio to zero defaults it to 3. 88 DialRatio int `toml:",omitempty"` 89 90 // NoDiscovery can be used to disable the peer discovery mechanism. 91 // Disabling is useful for protocol debugging (manual topology). 92 NoDiscovery bool 93 94 // DiscoveryV5 specifies whether the new topic-discovery based V5 discovery 95 // protocol should be started or not. 96 DiscoveryV5 bool `toml:",omitempty"` 97 98 // Name sets the node name of this server. 99 // Use common.MakeName to create a name that follows existing conventions. 100 Name string `toml:"-"` 101 102 // BootstrapNodes are used to establish connectivity 103 // with the rest of the network. 104 BootstrapNodes []*enode.Node 105 106 // BootstrapNodesV5 are used to establish connectivity 107 // with the rest of the network using the V5 discovery 108 // protocol. 109 BootstrapNodesV5 []*discv5.Node `toml:",omitempty"` 110 111 // Static nodes are used as pre-configured connections which are always 112 // maintained and re-connected on disconnects. 113 StaticNodes []*enode.Node 114 115 // Trusted nodes are used as pre-configured connections which are always 116 // allowed to connect, even above the peer limit. 117 TrustedNodes []*enode.Node 118 119 // Connectivity can be restricted to certain IP networks. 120 // If this option is set to a non-nil value, only hosts which match one of the 121 // IP networks contained in the list are considered. 122 NetRestrict *netutil.Netlist `toml:",omitempty"` 123 124 // NodeDatabase is the path to the database containing the previously seen 125 // live nodes in the network. 126 NodeDatabase string `toml:",omitempty"` 127 128 // Protocols should contain the protocols supported 129 // by the server. Matching protocols are launched for 130 // each peer. 131 Protocols []Protocol `toml:"-"` 132 133 // If ListenAddr is set to a non-nil address, the server 134 // will listen for incoming connections. 135 // 136 // If the port is zero, the operating system will pick a port. The 137 // ListenAddr field will be updated with the actual address when 138 // the server is started. 139 ListenAddr string 140 141 // If set to a non-nil value, the given NAT port mapper 142 // is used to make the listening port available to the 143 // Internet. 144 NAT nat.Interface `toml:",omitempty"` 145 146 // If Dialer is set to a non-nil value, the given Dialer 147 // is used to dial outbound peer connections. 148 Dialer NodeDialer `toml:"-"` 149 150 // If NoDial is true, the server will not dial any peers. 151 NoDial bool `toml:",omitempty"` 152 153 // If EnableMsgEvents is set then the server will emit PeerEvents 154 // whenever a message is sent to or received from a peer 155 EnableMsgEvents bool 156 157 EnableNodePermission bool `toml:",omitempty"` 158 159 DataDir string `toml:",omitempty"` 160 // Logger is a custom logger to use with the p2p.Server. 161 Logger log.Logger `toml:",omitempty"` 162 } 163 164 // Server manages all peer connections. 165 type Server struct { 166 // Config fields may not be modified while the server is running. 167 Config 168 169 // Hooks for testing. These are useful because we can inhibit 170 // the whole protocol stack. 171 newTransport func(net.Conn) transport 172 newPeerHook func(*Peer) 173 listenFunc func(network, addr string) (net.Listener, error) 174 175 lock sync.Mutex // protects running 176 running bool 177 178 listener net.Listener 179 ourHandshake *protoHandshake 180 loopWG sync.WaitGroup // loop, listenLoop 181 peerFeed event.Feed 182 log log.Logger 183 184 nodedb *enode.DB 185 localnode *enode.LocalNode 186 ntab *discover.UDPv4 187 DiscV5 *discv5.Network 188 discmix *enode.FairMix 189 190 staticNodeResolver nodeResolver 191 192 // Channels into the run loop. 193 quit chan struct{} 194 addstatic chan *enode.Node 195 removestatic chan *enode.Node 196 addtrusted chan *enode.Node 197 removetrusted chan *enode.Node 198 peerOp chan peerOpFunc 199 peerOpDone chan struct{} 200 delpeer chan peerDrop 201 checkpointPostHandshake chan *conn 202 checkpointAddPeer chan *conn 203 204 // State of run loop and listenLoop. 205 lastLookup time.Time 206 inboundHistory expHeap 207 208 // raft peers info 209 checkPeerInRaft func(*enode.Node) bool 210 } 211 212 type peerOpFunc func(map[enode.ID]*Peer) 213 214 type peerDrop struct { 215 *Peer 216 err error 217 requested bool // true if signaled by the peer 218 } 219 220 type connFlag int32 221 222 const ( 223 dynDialedConn connFlag = 1 << iota 224 staticDialedConn 225 inboundConn 226 trustedConn 227 ) 228 229 // conn wraps a network connection with information gathered 230 // during the two handshakes. 231 type conn struct { 232 fd net.Conn 233 transport 234 node *enode.Node 235 flags connFlag 236 cont chan error // The run loop uses cont to signal errors to SetupConn. 237 caps []Cap // valid after the protocol handshake 238 name string // valid after the protocol handshake 239 } 240 241 type transport interface { 242 // The two handshakes. 243 doEncHandshake(prv *ecdsa.PrivateKey, dialDest *ecdsa.PublicKey) (*ecdsa.PublicKey, error) 244 doProtoHandshake(our *protoHandshake) (*protoHandshake, error) 245 // The MsgReadWriter can only be used after the encryption 246 // handshake has completed. The code uses conn.id to track this 247 // by setting it to a non-nil value after the encryption handshake. 248 MsgReadWriter 249 // transports must provide Close because we use MsgPipe in some of 250 // the tests. Closing the actual network connection doesn't do 251 // anything in those tests because MsgPipe doesn't use it. 252 close(err error) 253 } 254 255 func (c *conn) String() string { 256 s := c.flags.String() 257 if (c.node.ID() != enode.ID{}) { 258 s += " " + c.node.ID().String() 259 } 260 s += " " + c.fd.RemoteAddr().String() 261 return s 262 } 263 264 func (f connFlag) String() string { 265 s := "" 266 if f&trustedConn != 0 { 267 s += "-trusted" 268 } 269 if f&dynDialedConn != 0 { 270 s += "-dyndial" 271 } 272 if f&staticDialedConn != 0 { 273 s += "-staticdial" 274 } 275 if f&inboundConn != 0 { 276 s += "-inbound" 277 } 278 if s != "" { 279 s = s[1:] 280 } 281 return s 282 } 283 284 func (c *conn) is(f connFlag) bool { 285 flags := connFlag(atomic.LoadInt32((*int32)(&c.flags))) 286 return flags&f != 0 287 } 288 289 func (c *conn) set(f connFlag, val bool) { 290 for { 291 oldFlags := connFlag(atomic.LoadInt32((*int32)(&c.flags))) 292 flags := oldFlags 293 if val { 294 flags |= f 295 } else { 296 flags &= ^f 297 } 298 if atomic.CompareAndSwapInt32((*int32)(&c.flags), int32(oldFlags), int32(flags)) { 299 return 300 } 301 } 302 } 303 304 // LocalNode returns the local node record. 305 func (srv *Server) LocalNode() *enode.LocalNode { 306 return srv.localnode 307 } 308 309 // Peers returns all connected peers. 310 func (srv *Server) Peers() []*Peer { 311 var ps []*Peer 312 select { 313 // Note: We'd love to put this function into a variable but 314 // that seems to cause a weird compiler error in some 315 // environments. 316 case srv.peerOp <- func(peers map[enode.ID]*Peer) { 317 for _, p := range peers { 318 ps = append(ps, p) 319 } 320 }: 321 <-srv.peerOpDone 322 case <-srv.quit: 323 } 324 return ps 325 } 326 327 // PeerCount returns the number of connected peers. 328 func (srv *Server) PeerCount() int { 329 var count int 330 select { 331 case srv.peerOp <- func(ps map[enode.ID]*Peer) { count = len(ps) }: 332 <-srv.peerOpDone 333 case <-srv.quit: 334 } 335 return count 336 } 337 338 // AddPeer connects to the given node and maintains the connection until the 339 // server is shut down. If the connection fails for any reason, the server will 340 // attempt to reconnect the peer. 341 func (srv *Server) AddPeer(node *enode.Node) { 342 select { 343 case srv.addstatic <- node: 344 case <-srv.quit: 345 } 346 } 347 348 // RemovePeer disconnects from the given node 349 func (srv *Server) RemovePeer(node *enode.Node) { 350 select { 351 case srv.removestatic <- node: 352 case <-srv.quit: 353 } 354 } 355 356 // AddTrustedPeer adds the given node to a reserved whitelist which allows the 357 // node to always connect, even if the slot are full. 358 func (srv *Server) AddTrustedPeer(node *enode.Node) { 359 select { 360 case srv.addtrusted <- node: 361 case <-srv.quit: 362 } 363 } 364 365 // RemoveTrustedPeer removes the given node from the trusted peer set. 366 func (srv *Server) RemoveTrustedPeer(node *enode.Node) { 367 select { 368 case srv.removetrusted <- node: 369 case <-srv.quit: 370 } 371 } 372 373 // SubscribePeers subscribes the given channel to peer events 374 func (srv *Server) SubscribeEvents(ch chan *PeerEvent) event.Subscription { 375 return srv.peerFeed.Subscribe(ch) 376 } 377 378 // Self returns the local node's endpoint information. 379 func (srv *Server) Self() *enode.Node { 380 srv.lock.Lock() 381 ln := srv.localnode 382 srv.lock.Unlock() 383 384 if ln == nil { 385 return enode.NewV4(&srv.PrivateKey.PublicKey, net.ParseIP("0.0.0.0"), 0, 0) 386 } 387 return ln.Node() 388 } 389 390 // Stop terminates the server and all active peer connections. 391 // It blocks until all active connections have been closed. 392 func (srv *Server) Stop() { 393 srv.lock.Lock() 394 if !srv.running { 395 srv.lock.Unlock() 396 return 397 } 398 srv.running = false 399 if srv.listener != nil { 400 // this unblocks listener Accept 401 srv.listener.Close() 402 } 403 close(srv.quit) 404 srv.lock.Unlock() 405 srv.loopWG.Wait() 406 } 407 408 // sharedUDPConn implements a shared connection. Write sends messages to the underlying connection while read returns 409 // messages that were found unprocessable and sent to the unhandled channel by the primary listener. 410 type sharedUDPConn struct { 411 *net.UDPConn 412 unhandled chan discover.ReadPacket 413 } 414 415 // ReadFromUDP implements discv5.conn 416 func (s *sharedUDPConn) ReadFromUDP(b []byte) (n int, addr *net.UDPAddr, err error) { 417 packet, ok := <-s.unhandled 418 if !ok { 419 return 0, nil, errors.New("Connection was closed") 420 } 421 l := len(packet.Data) 422 if l > len(b) { 423 l = len(b) 424 } 425 copy(b[:l], packet.Data[:l]) 426 return l, packet.Addr, nil 427 } 428 429 // Close implements discv5.conn 430 func (s *sharedUDPConn) Close() error { 431 return nil 432 } 433 434 // Start starts running the server. 435 // Servers can not be re-used after stopping. 436 func (srv *Server) Start() (err error) { 437 srv.lock.Lock() 438 defer srv.lock.Unlock() 439 if srv.running { 440 return errors.New("server already running") 441 } 442 srv.running = true 443 srv.log = srv.Config.Logger 444 if srv.log == nil { 445 srv.log = log.Root() 446 } 447 if srv.NoDial && srv.ListenAddr == "" { 448 srv.log.Warn("P2P server will be useless, neither dialing nor listening") 449 } 450 451 // static fields 452 if srv.PrivateKey == nil { 453 return errors.New("Server.PrivateKey must be set to a non-nil key") 454 } 455 if srv.newTransport == nil { 456 srv.newTransport = newRLPX 457 } 458 if srv.listenFunc == nil { 459 srv.listenFunc = net.Listen 460 } 461 if srv.Dialer == nil { 462 srv.Dialer = TCPDialer{&net.Dialer{Timeout: defaultDialTimeout}} 463 } 464 srv.quit = make(chan struct{}) 465 srv.delpeer = make(chan peerDrop) 466 srv.checkpointPostHandshake = make(chan *conn) 467 srv.checkpointAddPeer = make(chan *conn) 468 srv.addstatic = make(chan *enode.Node) 469 srv.removestatic = make(chan *enode.Node) 470 srv.addtrusted = make(chan *enode.Node) 471 srv.removetrusted = make(chan *enode.Node) 472 srv.peerOp = make(chan peerOpFunc) 473 srv.peerOpDone = make(chan struct{}) 474 475 if err := srv.setupLocalNode(); err != nil { 476 return err 477 } 478 if srv.ListenAddr != "" { 479 if err := srv.setupListening(); err != nil { 480 return err 481 } 482 } 483 if err := srv.setupDiscovery(); err != nil { 484 return err 485 } 486 487 dynPeers := srv.maxDialedConns() 488 dialer := newDialState(srv.localnode.ID(), dynPeers, &srv.Config) 489 srv.loopWG.Add(1) 490 go srv.run(dialer) 491 return nil 492 } 493 494 func (srv *Server) setupLocalNode() error { 495 // Create the devp2p handshake. 496 pubkey := crypto.FromECDSAPub(&srv.PrivateKey.PublicKey) 497 srv.ourHandshake = &protoHandshake{Version: baseProtocolVersion, Name: srv.Name, ID: pubkey[1:]} 498 for _, p := range srv.Protocols { 499 srv.ourHandshake.Caps = append(srv.ourHandshake.Caps, p.cap()) 500 } 501 sort.Sort(capsByNameAndVersion(srv.ourHandshake.Caps)) 502 503 // Create the local node. 504 db, err := enode.OpenDB(srv.Config.NodeDatabase) 505 if err != nil { 506 return err 507 } 508 srv.nodedb = db 509 srv.localnode = enode.NewLocalNode(db, srv.PrivateKey) 510 srv.localnode.SetFallbackIP(net.IP{127, 0, 0, 1}) 511 // TODO: check conflicts 512 for _, p := range srv.Protocols { 513 for _, e := range p.Attributes { 514 srv.localnode.Set(e) 515 } 516 } 517 switch srv.NAT.(type) { 518 case nil: 519 // No NAT interface, do nothing. 520 case nat.ExtIP: 521 // ExtIP doesn't block, set the IP right away. 522 ip, _ := srv.NAT.ExternalIP() 523 srv.localnode.SetStaticIP(ip) 524 default: 525 // Ask the router about the IP. This takes a while and blocks startup, 526 // do it in the background. 527 srv.loopWG.Add(1) 528 go func() { 529 defer srv.loopWG.Done() 530 if ip, err := srv.NAT.ExternalIP(); err == nil { 531 srv.localnode.SetStaticIP(ip) 532 } 533 }() 534 } 535 return nil 536 } 537 538 func (srv *Server) setupDiscovery() error { 539 srv.discmix = enode.NewFairMix(discmixTimeout) 540 541 // Add protocol-specific discovery sources. 542 added := make(map[string]bool) 543 for _, proto := range srv.Protocols { 544 if proto.DialCandidates != nil && !added[proto.Name] { 545 srv.discmix.AddSource(proto.DialCandidates) 546 added[proto.Name] = true 547 } 548 } 549 550 // Don't listen on UDP endpoint if DHT is disabled. 551 if srv.NoDiscovery && !srv.DiscoveryV5 { 552 return nil 553 } 554 555 addr, err := net.ResolveUDPAddr("udp", srv.ListenAddr) 556 if err != nil { 557 return err 558 } 559 conn, err := net.ListenUDP("udp", addr) 560 if err != nil { 561 return err 562 } 563 realaddr := conn.LocalAddr().(*net.UDPAddr) 564 srv.log.Debug("UDP listener up", "addr", realaddr) 565 if srv.NAT != nil { 566 if !realaddr.IP.IsLoopback() { 567 go nat.Map(srv.NAT, srv.quit, "udp", realaddr.Port, realaddr.Port, "ethereum discovery") 568 } 569 } 570 srv.localnode.SetFallbackUDP(realaddr.Port) 571 572 // Discovery V4 573 var unhandled chan discover.ReadPacket 574 var sconn *sharedUDPConn 575 if !srv.NoDiscovery { 576 if srv.DiscoveryV5 { 577 unhandled = make(chan discover.ReadPacket, 100) 578 sconn = &sharedUDPConn{conn, unhandled} 579 } 580 cfg := discover.Config{ 581 PrivateKey: srv.PrivateKey, 582 NetRestrict: srv.NetRestrict, 583 Bootnodes: srv.BootstrapNodes, 584 Unhandled: unhandled, 585 Log: srv.log, 586 } 587 ntab, err := discover.ListenUDP(conn, srv.localnode, cfg) 588 if err != nil { 589 return err 590 } 591 srv.ntab = ntab 592 srv.discmix.AddSource(ntab.RandomNodes()) 593 srv.staticNodeResolver = ntab 594 } 595 596 // Discovery V5 597 if srv.DiscoveryV5 { 598 var ntab *discv5.Network 599 var err error 600 if sconn != nil { 601 ntab, err = discv5.ListenUDP(srv.PrivateKey, sconn, "", srv.NetRestrict) 602 } else { 603 ntab, err = discv5.ListenUDP(srv.PrivateKey, conn, "", srv.NetRestrict) 604 } 605 if err != nil { 606 return err 607 } 608 if err := ntab.SetFallbackNodes(srv.BootstrapNodesV5); err != nil { 609 return err 610 } 611 srv.DiscV5 = ntab 612 } 613 return nil 614 } 615 616 func (srv *Server) setupListening() error { 617 // Launch the listener. 618 listener, err := srv.listenFunc("tcp", srv.ListenAddr) 619 if err != nil { 620 return err 621 } 622 srv.listener = listener 623 srv.ListenAddr = listener.Addr().String() 624 625 // Update the local node record and map the TCP listening port if NAT is configured. 626 if tcp, ok := listener.Addr().(*net.TCPAddr); ok { 627 srv.localnode.Set(enr.TCP(tcp.Port)) 628 if !tcp.IP.IsLoopback() && srv.NAT != nil { 629 srv.loopWG.Add(1) 630 go func() { 631 nat.Map(srv.NAT, srv.quit, "tcp", tcp.Port, tcp.Port, "ethereum p2p") 632 srv.loopWG.Done() 633 }() 634 } 635 } 636 637 srv.loopWG.Add(1) 638 go srv.listenLoop() 639 return nil 640 } 641 642 type dialer interface { 643 newTasks(running int, peers map[enode.ID]*Peer, now time.Time) []task 644 taskDone(task, time.Time) 645 addStatic(*enode.Node) 646 removeStatic(*enode.Node) 647 } 648 649 func (srv *Server) run(dialstate dialer) { 650 srv.log.Info("Started P2P networking", "self", srv.localnode.Node().URLv4()) 651 defer srv.loopWG.Done() 652 defer srv.nodedb.Close() 653 defer srv.discmix.Close() 654 655 var ( 656 peers = make(map[enode.ID]*Peer) 657 inboundCount = 0 658 trusted = make(map[enode.ID]bool, len(srv.TrustedNodes)) 659 taskdone = make(chan task, maxActiveDialTasks) 660 runningTasks []task 661 queuedTasks []task // tasks that can't run yet 662 ) 663 // Put trusted nodes into a map to speed up checks. 664 // Trusted peers are loaded on startup or added via AddTrustedPeer RPC. 665 for _, n := range srv.TrustedNodes { 666 trusted[n.ID()] = true 667 } 668 669 // removes t from runningTasks 670 delTask := func(t task) { 671 for i := range runningTasks { 672 if runningTasks[i] == t { 673 runningTasks = append(runningTasks[:i], runningTasks[i+1:]...) 674 break 675 } 676 } 677 } 678 // starts until max number of active tasks is satisfied 679 startTasks := func(ts []task) (rest []task) { 680 i := 0 681 for ; len(runningTasks) < maxActiveDialTasks && i < len(ts); i++ { 682 t := ts[i] 683 srv.log.Trace("New dial task", "task", t) 684 go func() { t.Do(srv); taskdone <- t }() 685 runningTasks = append(runningTasks, t) 686 } 687 return ts[i:] 688 } 689 scheduleTasks := func() { 690 // Start from queue first. 691 queuedTasks = append(queuedTasks[:0], startTasks(queuedTasks)...) 692 // Query dialer for new tasks and start as many as possible now. 693 if len(runningTasks) < maxActiveDialTasks { 694 nt := dialstate.newTasks(len(runningTasks)+len(queuedTasks), peers, time.Now()) 695 queuedTasks = append(queuedTasks, startTasks(nt)...) 696 } 697 } 698 699 running: 700 for { 701 scheduleTasks() 702 703 select { 704 case <-srv.quit: 705 // The server was stopped. Run the cleanup logic. 706 break running 707 708 case n := <-srv.addstatic: 709 // This channel is used by AddPeer to add to the 710 // ephemeral static peer list. Add it to the dialer, 711 // it will keep the node connected. 712 srv.log.Trace("Adding static node", "node", n) 713 dialstate.addStatic(n) 714 715 case n := <-srv.removestatic: 716 // This channel is used by RemovePeer to send a 717 // disconnect request to a peer and begin the 718 // stop keeping the node connected. 719 srv.log.Trace("Removing static node", "node", n) 720 dialstate.removeStatic(n) 721 if p, ok := peers[n.ID()]; ok { 722 p.Disconnect(DiscRequested) 723 } 724 725 case n := <-srv.addtrusted: 726 // This channel is used by AddTrustedPeer to add an enode 727 // to the trusted node set. 728 srv.log.Trace("Adding trusted node", "node", n) 729 trusted[n.ID()] = true 730 // Mark any already-connected peer as trusted 731 if p, ok := peers[n.ID()]; ok { 732 p.rw.set(trustedConn, true) 733 } 734 735 case n := <-srv.removetrusted: 736 // This channel is used by RemoveTrustedPeer to remove an enode 737 // from the trusted node set. 738 srv.log.Trace("Removing trusted node", "node", n) 739 delete(trusted, n.ID()) 740 741 // Unmark any already-connected peer as trusted 742 if p, ok := peers[n.ID()]; ok { 743 p.rw.set(trustedConn, false) 744 } 745 746 case op := <-srv.peerOp: 747 // This channel is used by Peers and PeerCount. 748 op(peers) 749 srv.peerOpDone <- struct{}{} 750 751 case t := <-taskdone: 752 // A task got done. Tell dialstate about it so it 753 // can update its state and remove it from the active 754 // tasks list. 755 srv.log.Trace("Dial task done", "task", t) 756 dialstate.taskDone(t, time.Now()) 757 delTask(t) 758 759 case c := <-srv.checkpointPostHandshake: 760 // A connection has passed the encryption handshake so 761 // the remote identity is known (but hasn't been verified yet). 762 if trusted[c.node.ID()] { 763 // Ensure that the trusted flag is set before checking against MaxPeers. 764 c.flags |= trustedConn 765 } 766 // TODO: track in-progress inbound node IDs (pre-Peer) to avoid dialing them. 767 c.cont <- srv.postHandshakeChecks(peers, inboundCount, c) 768 769 case c := <-srv.checkpointAddPeer: 770 // At this point the connection is past the protocol handshake. 771 // Its capabilities are known and the remote identity is verified. 772 err := srv.addPeerChecks(peers, inboundCount, c) 773 if err == nil { 774 // The handshakes are done and it passed all checks. 775 p := newPeer(srv.log, c, srv.Protocols) 776 // If message events are enabled, pass the peerFeed 777 // to the peer 778 if srv.EnableMsgEvents { 779 p.events = &srv.peerFeed 780 } 781 name := truncateName(c.name) 782 p.log.Debug("Adding p2p peer", "addr", p.RemoteAddr(), "peers", len(peers)+1, "name", name) 783 go srv.runPeer(p) 784 peers[c.node.ID()] = p 785 if p.Inbound() { 786 inboundCount++ 787 } 788 } 789 // The dialer logic relies on the assumption that 790 // dial tasks complete after the peer has been added or 791 // discarded. Unblock the task last. 792 c.cont <- err 793 794 case pd := <-srv.delpeer: 795 // A peer disconnected. 796 d := common.PrettyDuration(mclock.Now() - pd.created) 797 pd.log.Debug("Removing p2p peer", "addr", pd.RemoteAddr(), "peers", len(peers)-1, "duration", d, "req", pd.requested, "err", pd.err) 798 delete(peers, pd.ID()) 799 if pd.Inbound() { 800 inboundCount-- 801 } 802 } 803 } 804 805 srv.log.Trace("P2P networking is spinning down") 806 807 // Terminate discovery. If there is a running lookup it will terminate soon. 808 if srv.ntab != nil { 809 srv.ntab.Close() 810 } 811 if srv.DiscV5 != nil { 812 srv.DiscV5.Close() 813 } 814 // Disconnect all peers. 815 for _, p := range peers { 816 p.Disconnect(DiscQuitting) 817 } 818 // Wait for peers to shut down. Pending connections and tasks are 819 // not handled here and will terminate soon-ish because srv.quit 820 // is closed. 821 for len(peers) > 0 { 822 p := <-srv.delpeer 823 p.log.Trace("<-delpeer (spindown)", "remainingTasks", len(runningTasks)) 824 delete(peers, p.ID()) 825 } 826 } 827 828 func (srv *Server) postHandshakeChecks(peers map[enode.ID]*Peer, inboundCount int, c *conn) error { 829 switch { 830 case !c.is(trustedConn|staticDialedConn) && len(peers) >= srv.MaxPeers: 831 return DiscTooManyPeers 832 case !c.is(trustedConn) && c.is(inboundConn) && inboundCount >= srv.maxInboundConns(): 833 return DiscTooManyPeers 834 case peers[c.node.ID()] != nil: 835 return DiscAlreadyConnected 836 case c.node.ID() == srv.localnode.ID(): 837 return DiscSelf 838 default: 839 return nil 840 } 841 } 842 843 func (srv *Server) addPeerChecks(peers map[enode.ID]*Peer, inboundCount int, c *conn) error { 844 // Drop connections with no matching protocols. 845 if len(srv.Protocols) > 0 && countMatchingProtocols(srv.Protocols, c.caps) == 0 { 846 return DiscUselessPeer 847 } 848 // Repeat the post-handshake checks because the 849 // peer set might have changed since those checks were performed. 850 return srv.postHandshakeChecks(peers, inboundCount, c) 851 } 852 853 func (srv *Server) maxInboundConns() int { 854 return srv.MaxPeers - srv.maxDialedConns() 855 } 856 857 func (srv *Server) maxDialedConns() int { 858 if srv.NoDiscovery || srv.NoDial { 859 return 0 860 } 861 r := srv.DialRatio 862 if r == 0 { 863 r = defaultDialRatio 864 } 865 return srv.MaxPeers / r 866 } 867 868 // listenLoop runs in its own goroutine and accepts 869 // inbound connections. 870 func (srv *Server) listenLoop() { 871 defer srv.loopWG.Done() 872 srv.log.Debug("TCP listener up", "addr", srv.listener.Addr()) 873 874 tokens := defaultMaxPendingPeers 875 if srv.MaxPendingPeers > 0 { 876 tokens = srv.MaxPendingPeers 877 } 878 slots := make(chan struct{}, tokens) 879 for i := 0; i < tokens; i++ { 880 slots <- struct{}{} 881 } 882 883 for { 884 // Wait for a free slot before accepting. 885 <-slots 886 887 var ( 888 fd net.Conn 889 err error 890 ) 891 for { 892 fd, err = srv.listener.Accept() 893 if netutil.IsTemporaryError(err) { 894 srv.log.Debug("Temporary read error", "err", err) 895 continue 896 } else if err != nil { 897 srv.log.Debug("Read error", "err", err) 898 return 899 } 900 break 901 } 902 903 remoteIP := netutil.AddrIP(fd.RemoteAddr()) 904 if err := srv.checkInboundConn(fd, remoteIP); err != nil { 905 srv.log.Debug("Rejected inbound connnection", "addr", fd.RemoteAddr(), "err", err) 906 fd.Close() 907 slots <- struct{}{} 908 continue 909 } 910 if remoteIP != nil { 911 fd = newMeteredConn(fd, true, remoteIP) 912 } 913 srv.log.Trace("Accepted connection", "addr", fd.RemoteAddr()) 914 go func() { 915 srv.SetupConn(fd, inboundConn, nil) 916 slots <- struct{}{} 917 }() 918 } 919 } 920 921 func (srv *Server) checkInboundConn(fd net.Conn, remoteIP net.IP) error { 922 if remoteIP != nil { 923 // Reject connections that do not match NetRestrict. 924 if srv.NetRestrict != nil && !srv.NetRestrict.Contains(remoteIP) { 925 return fmt.Errorf("not whitelisted in NetRestrict") 926 } 927 // Reject Internet peers that try too often. 928 srv.inboundHistory.expire(time.Now()) 929 if !netutil.IsLAN(remoteIP) && srv.inboundHistory.contains(remoteIP.String()) { 930 return fmt.Errorf("too many attempts") 931 } 932 srv.inboundHistory.add(remoteIP.String(), time.Now().Add(inboundThrottleTime)) 933 } 934 return nil 935 } 936 937 // SetupConn runs the handshakes and attempts to add the connection 938 // as a peer. It returns when the connection has been added as a peer 939 // or the handshakes have failed. 940 func (srv *Server) SetupConn(fd net.Conn, flags connFlag, dialDest *enode.Node) error { 941 c := &conn{fd: fd, transport: srv.newTransport(fd), flags: flags, cont: make(chan error)} 942 err := srv.setupConn(c, flags, dialDest) 943 if err != nil { 944 c.close(err) 945 srv.log.Trace("Setting up connection failed", "addr", fd.RemoteAddr(), "err", err) 946 } 947 return err 948 } 949 950 func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) error { 951 // Prevent leftover pending conns from entering the handshake. 952 srv.lock.Lock() 953 running := srv.running 954 srv.lock.Unlock() 955 if !running { 956 return errServerStopped 957 } 958 959 // If dialing, figure out the remote public key. 960 var dialPubkey *ecdsa.PublicKey 961 if dialDest != nil { 962 dialPubkey = new(ecdsa.PublicKey) 963 if err := dialDest.Load((*enode.Secp256k1)(dialPubkey)); err != nil { 964 return errors.New("dial destination doesn't have a secp256k1 public key") 965 } 966 } 967 968 // Run the RLPx handshake. 969 remotePubkey, err := c.doEncHandshake(srv.PrivateKey, dialPubkey) 970 if err != nil { 971 srv.log.Trace("Failed RLPx handshake", "addr", c.fd.RemoteAddr(), "conn", c.flags, "err", err) 972 return err 973 } 974 975 if dialDest != nil { 976 // For dialed connections, check that the remote public key matches. 977 if dialPubkey.X.Cmp(remotePubkey.X) != 0 || dialPubkey.Y.Cmp(remotePubkey.Y) != 0 { 978 return DiscUnexpectedIdentity 979 } 980 c.node = dialDest 981 } else { 982 c.node = nodeFromConn(remotePubkey, c.fd) 983 } 984 clog := srv.log.New("id", c.node.ID(), "addr", c.fd.RemoteAddr(), "conn", c.flags) 985 986 // If raft is running, check if the dialing node is in the raft cluster 987 // Node doesn't belong to raft cluster is not allowed to join the p2p network 988 if srv.checkPeerInRaft != nil && !srv.checkPeerInRaft(c.node) { 989 node := c.node.ID().String() 990 log.Trace("incoming connection peer is not in the raft cluster", "enode.id", node) 991 return newPeerError(errNotInRaftCluster, "id=%s…%s", node[:4], node[len(node)-4:]) 992 } 993 994 //START - QUORUM Permissioning 995 currentNode := srv.NodeInfo().ID 996 cnodeName := srv.NodeInfo().Name 997 clog.Trace("Quorum permissioning", 998 "EnableNodePermission", srv.EnableNodePermission, 999 "DataDir", srv.DataDir, 1000 "Current Node ID", currentNode, 1001 "Node Name", cnodeName, 1002 "Dialed Dest", dialDest, 1003 "Connection ID", c.node.ID(), 1004 "Connection String", c.node.ID().String()) 1005 1006 if srv.EnableNodePermission { 1007 clog.Trace("Node Permissioning is Enabled.") 1008 node := c.node.ID().String() 1009 direction := "INCOMING" 1010 if dialDest != nil { 1011 node = dialDest.ID().String() 1012 direction = "OUTGOING" 1013 log.Trace("Node Permissioning", "Connection Direction", direction) 1014 } 1015 1016 if !isNodePermissioned(node, currentNode, srv.DataDir, direction) { 1017 return newPeerError(errPermissionDenied, "id=%s…%s %s id=%s…%s", currentNode[:4], currentNode[len(currentNode)-4:], direction, node[:4], node[len(node)-4:]) 1018 } 1019 } else { 1020 clog.Trace("Node Permissioning is Disabled.") 1021 } 1022 1023 //END - QUORUM Permissioning 1024 1025 if conn, ok := c.fd.(*meteredConn); ok { 1026 conn.handshakeDone(c.node.ID()) 1027 } 1028 1029 err = srv.checkpoint(c, srv.checkpointPostHandshake) 1030 if err != nil { 1031 clog.Trace("Rejected peer", "err", err) 1032 return err 1033 } 1034 1035 // Run the capability negotiation handshake. 1036 phs, err := c.doProtoHandshake(srv.ourHandshake) 1037 if err != nil { 1038 clog.Trace("Failed proto handshake", "err", err) 1039 return err 1040 } 1041 if id := c.node.ID(); !bytes.Equal(crypto.Keccak256(phs.ID), id[:]) { 1042 clog.Trace("Wrong devp2p handshake identity", "phsid", hex.EncodeToString(phs.ID)) 1043 return DiscUnexpectedIdentity 1044 } 1045 c.caps, c.name = phs.Caps, phs.Name 1046 err = srv.checkpoint(c, srv.checkpointAddPeer) 1047 if err != nil { 1048 clog.Trace("Rejected peer", "err", err) 1049 return err 1050 } 1051 1052 // If the checks completed successfully, the connection has been added as a peer and 1053 // runPeer has been launched. 1054 clog.Trace("Connection set up", "inbound", dialDest == nil) 1055 return nil 1056 } 1057 1058 func nodeFromConn(pubkey *ecdsa.PublicKey, conn net.Conn) *enode.Node { 1059 var ip net.IP 1060 var port int 1061 if tcp, ok := conn.RemoteAddr().(*net.TCPAddr); ok { 1062 ip = tcp.IP 1063 port = tcp.Port 1064 } 1065 return enode.NewV4(pubkey, ip, port, port) 1066 } 1067 1068 func truncateName(s string) string { 1069 if len(s) > 20 { 1070 return s[:20] + "..." 1071 } 1072 return s 1073 } 1074 1075 // checkpoint sends the conn to run, which performs the 1076 // post-handshake checks for the stage (posthandshake, addpeer). 1077 func (srv *Server) checkpoint(c *conn, stage chan<- *conn) error { 1078 select { 1079 case stage <- c: 1080 case <-srv.quit: 1081 return errServerStopped 1082 } 1083 return <-c.cont 1084 } 1085 1086 // runPeer runs in its own goroutine for each peer. 1087 // it waits until the Peer logic returns and removes 1088 // the peer. 1089 func (srv *Server) runPeer(p *Peer) { 1090 if srv.newPeerHook != nil { 1091 srv.newPeerHook(p) 1092 } 1093 1094 // broadcast peer add 1095 srv.peerFeed.Send(&PeerEvent{ 1096 Type: PeerEventTypeAdd, 1097 Peer: p.ID(), 1098 RemoteAddress: p.RemoteAddr().String(), 1099 LocalAddress: p.LocalAddr().String(), 1100 }) 1101 1102 // run the protocol 1103 remoteRequested, err := p.run() 1104 1105 // broadcast peer drop 1106 srv.peerFeed.Send(&PeerEvent{ 1107 Type: PeerEventTypeDrop, 1108 Peer: p.ID(), 1109 Error: err.Error(), 1110 RemoteAddress: p.RemoteAddr().String(), 1111 LocalAddress: p.LocalAddr().String(), 1112 }) 1113 1114 // Note: run waits for existing peers to be sent on srv.delpeer 1115 // before returning, so this send should not select on srv.quit. 1116 srv.delpeer <- peerDrop{p, err, remoteRequested} 1117 } 1118 1119 // NodeInfo represents a short summary of the information known about the host. 1120 type NodeInfo struct { 1121 ID string `json:"id"` // Unique node identifier (also the encryption key) 1122 Name string `json:"name"` // Name of the node, including client type, version, OS, custom data 1123 Enode string `json:"enode"` // Enode URL for adding this peer from remote peers 1124 ENR string `json:"enr"` // Ethereum Node Record 1125 IP string `json:"ip"` // IP address of the node 1126 Ports struct { 1127 Discovery int `json:"discovery"` // UDP listening port for discovery protocol 1128 Listener int `json:"listener"` // TCP listening port for RLPx 1129 } `json:"ports"` 1130 ListenAddr string `json:"listenAddr"` 1131 Protocols map[string]interface{} `json:"protocols"` 1132 } 1133 1134 // NodeInfo gathers and returns a collection of metadata known about the host. 1135 func (srv *Server) NodeInfo() *NodeInfo { 1136 // Gather and assemble the generic node infos 1137 node := srv.Self() 1138 info := &NodeInfo{ 1139 Name: srv.Name, 1140 Enode: node.URLv4(), 1141 ID: node.ID().String(), 1142 IP: node.IP().String(), 1143 ListenAddr: srv.ListenAddr, 1144 Protocols: make(map[string]interface{}), 1145 } 1146 info.Ports.Discovery = node.UDP() 1147 info.Ports.Listener = node.TCP() 1148 info.ENR = node.String() 1149 1150 // Gather all the running protocol infos (only once per protocol type) 1151 for _, proto := range srv.Protocols { 1152 if _, ok := info.Protocols[proto.Name]; !ok { 1153 nodeInfo := interface{}("unknown") 1154 if query := proto.NodeInfo; query != nil { 1155 nodeInfo = proto.NodeInfo() 1156 } 1157 info.Protocols[proto.Name] = nodeInfo 1158 } 1159 } 1160 return info 1161 } 1162 1163 // PeersInfo returns an array of metadata objects describing connected peers. 1164 func (srv *Server) PeersInfo() []*PeerInfo { 1165 // Gather all the generic and sub-protocol specific infos 1166 infos := make([]*PeerInfo, 0, srv.PeerCount()) 1167 for _, peer := range srv.Peers() { 1168 if peer != nil { 1169 infos = append(infos, peer.Info()) 1170 } 1171 } 1172 // Sort the result array alphabetically by node identifier 1173 for i := 0; i < len(infos); i++ { 1174 for j := i + 1; j < len(infos); j++ { 1175 if infos[i].ID > infos[j].ID { 1176 infos[i], infos[j] = infos[j], infos[i] 1177 } 1178 } 1179 } 1180 return infos 1181 } 1182 1183 func (srv *Server) SetCheckPeerInRaft(f func(*enode.Node) bool) { 1184 srv.checkPeerInRaft = f 1185 }