github.com/tdcblockchain/tdcblockchain@v0.0.0-20191111034745-805c65ade158/les/serverpool.go (about)

     1  // Copyright 2016 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package les
    18  
    19  import (
    20  	"crypto/ecdsa"
    21  	"fmt"
    22  	"io"
    23  	"math"
    24  	"math/rand"
    25  	"net"
    26  	"strconv"
    27  	"sync"
    28  	"time"
    29  
    30  	"github.com/ethereum/go-ethereum/common/mclock"
    31  	"github.com/ethereum/go-ethereum/crypto"
    32  	"github.com/ethereum/go-ethereum/ethdb"
    33  	"github.com/ethereum/go-ethereum/log"
    34  	"github.com/ethereum/go-ethereum/p2p"
    35  	"github.com/ethereum/go-ethereum/p2p/discv5"
    36  	"github.com/ethereum/go-ethereum/p2p/enode"
    37  	"github.com/ethereum/go-ethereum/rlp"
    38  )
    39  
    40  const (
    41  	// After a connection has been ended or timed out, there is a waiting period
    42  	// before it can be selected for connection again.
    43  	// waiting period = base delay * (1 + random(1))
    44  	// base delay = shortRetryDelay for the first shortRetryCnt times after a
    45  	// successful connection, after that longRetryDelay is applied
    46  	shortRetryCnt   = 5
    47  	shortRetryDelay = time.Second * 5
    48  	longRetryDelay  = time.Minute * 10
    49  	// maxNewEntries is the maximum number of newly discovered (never connected) nodes.
    50  	// If the limit is reached, the least recently discovered one is thrown out.
    51  	maxNewEntries = 1000
    52  	// maxKnownEntries is the maximum number of known (already connected) nodes.
    53  	// If the limit is reached, the least recently connected one is thrown out.
    54  	// (not that unlike new entries, known entries are persistent)
    55  	maxKnownEntries = 1000
    56  	// target for simultaneously connected servers
    57  	targetServerCount = 5
    58  	// target for servers selected from the known table
    59  	// (we leave room for trying new ones if there is any)
    60  	targetKnownSelect = 3
    61  	// after dialTimeout, consider the server unavailable and adjust statistics
    62  	dialTimeout = time.Second * 30
    63  	// targetConnTime is the minimum expected connection duration before a server
    64  	// drops a client without any specific reason
    65  	targetConnTime = time.Minute * 10
    66  	// new entry selection weight calculation based on most recent discovery time:
    67  	// unity until discoverExpireStart, then exponential decay with discoverExpireConst
    68  	discoverExpireStart = time.Minute * 20
    69  	discoverExpireConst = time.Minute * 20
    70  	// known entry selection weight is dropped by a factor of exp(-failDropLn) after
    71  	// each unsuccessful connection (restored after a successful one)
    72  	failDropLn = 0.1
    73  	// known node connection success and quality statistics have a long term average
    74  	// and a short term value which is adjusted exponentially with a factor of
    75  	// pstatRecentAdjust with each dial/connection and also returned exponentially
    76  	// to the average with the time constant pstatReturnToMeanTC
    77  	pstatReturnToMeanTC = time.Hour
    78  	// node address selection weight is dropped by a factor of exp(-addrFailDropLn) after
    79  	// each unsuccessful connection (restored after a successful one)
    80  	addrFailDropLn = math.Ln2
    81  	// responseScoreTC and delayScoreTC are exponential decay time constants for
    82  	// calculating selection chances from response times and block delay times
    83  	responseScoreTC = time.Millisecond * 100
    84  	delayScoreTC    = time.Second * 5
    85  	timeoutPow      = 10
    86  	// initStatsWeight is used to initialize previously unknown peers with good
    87  	// statistics to give a chance to prove themselves
    88  	initStatsWeight = 1
    89  )
    90  
    91  // connReq represents a request for peer connection.
    92  type connReq struct {
    93  	p      *peer
    94  	node   *enode.Node
    95  	result chan *poolEntry
    96  }
    97  
    98  // disconnReq represents a request for peer disconnection.
    99  type disconnReq struct {
   100  	entry   *poolEntry
   101  	stopped bool
   102  	done    chan struct{}
   103  }
   104  
   105  // registerReq represents a request for peer registration.
   106  type registerReq struct {
   107  	entry *poolEntry
   108  	done  chan struct{}
   109  }
   110  
   111  // serverPool implements a pool for storing and selecting newly discovered and already
   112  // known light server nodes. It received discovered nodes, stores statistics about
   113  // known nodes and takes care of always having enough good quality servers connected.
   114  type serverPool struct {
   115  	db     ethdb.Database
   116  	dbKey  []byte
   117  	server *p2p.Server
   118  	connWg sync.WaitGroup
   119  
   120  	topic discv5.Topic
   121  
   122  	discSetPeriod chan time.Duration
   123  	discNodes     chan *enode.Node
   124  	discLookups   chan bool
   125  
   126  	trustedNodes         map[enode.ID]*enode.Node
   127  	entries              map[enode.ID]*poolEntry
   128  	timeout, enableRetry chan *poolEntry
   129  	adjustStats          chan poolStatAdjust
   130  
   131  	knownQueue, newQueue       poolEntryQueue
   132  	knownSelect, newSelect     *weightedRandomSelect
   133  	knownSelected, newSelected int
   134  	fastDiscover               bool
   135  	connCh                     chan *connReq
   136  	disconnCh                  chan *disconnReq
   137  	registerCh                 chan *registerReq
   138  
   139  	closeCh chan struct{}
   140  	wg      sync.WaitGroup
   141  }
   142  
   143  // newServerPool creates a new serverPool instance
   144  func newServerPool(db ethdb.Database, ulcServers []string) *serverPool {
   145  	pool := &serverPool{
   146  		db:           db,
   147  		entries:      make(map[enode.ID]*poolEntry),
   148  		timeout:      make(chan *poolEntry, 1),
   149  		adjustStats:  make(chan poolStatAdjust, 100),
   150  		enableRetry:  make(chan *poolEntry, 1),
   151  		connCh:       make(chan *connReq),
   152  		disconnCh:    make(chan *disconnReq),
   153  		registerCh:   make(chan *registerReq),
   154  		closeCh:      make(chan struct{}),
   155  		knownSelect:  newWeightedRandomSelect(),
   156  		newSelect:    newWeightedRandomSelect(),
   157  		fastDiscover: true,
   158  		trustedNodes: parseTrustedNodes(ulcServers),
   159  	}
   160  
   161  	pool.knownQueue = newPoolEntryQueue(maxKnownEntries, pool.removeEntry)
   162  	pool.newQueue = newPoolEntryQueue(maxNewEntries, pool.removeEntry)
   163  	return pool
   164  }
   165  
   166  func (pool *serverPool) start(server *p2p.Server, topic discv5.Topic) {
   167  	pool.server = server
   168  	pool.topic = topic
   169  	pool.dbKey = append([]byte("serverPool/"), []byte(topic)...)
   170  	pool.loadNodes()
   171  	pool.connectToTrustedNodes()
   172  
   173  	if pool.server.DiscV5 != nil {
   174  		pool.discSetPeriod = make(chan time.Duration, 1)
   175  		pool.discNodes = make(chan *enode.Node, 100)
   176  		pool.discLookups = make(chan bool, 100)
   177  		go pool.discoverNodes()
   178  	}
   179  	pool.checkDial()
   180  	pool.wg.Add(1)
   181  	go pool.eventLoop()
   182  }
   183  
   184  func (pool *serverPool) stop() {
   185  	close(pool.closeCh)
   186  	pool.wg.Wait()
   187  }
   188  
   189  // discoverNodes wraps SearchTopic, converting result nodes to enode.Node.
   190  func (pool *serverPool) discoverNodes() {
   191  	ch := make(chan *discv5.Node)
   192  	go func() {
   193  		pool.server.DiscV5.SearchTopic(pool.topic, pool.discSetPeriod, ch, pool.discLookups)
   194  		close(ch)
   195  	}()
   196  	for n := range ch {
   197  		pubkey, err := decodePubkey64(n.ID[:])
   198  		if err != nil {
   199  			continue
   200  		}
   201  		pool.discNodes <- enode.NewV4(pubkey, n.IP, int(n.TCP), int(n.UDP))
   202  	}
   203  }
   204  
   205  // connect should be called upon any incoming connection. If the connection has been
   206  // dialed by the server pool recently, the appropriate pool entry is returned.
   207  // Otherwise, the connection should be rejected.
   208  // Note that whenever a connection has been accepted and a pool entry has been returned,
   209  // disconnect should also always be called.
   210  func (pool *serverPool) connect(p *peer, node *enode.Node) *poolEntry {
   211  	log.Debug("Connect new entry", "enode", p.id)
   212  	req := &connReq{p: p, node: node, result: make(chan *poolEntry, 1)}
   213  	select {
   214  	case pool.connCh <- req:
   215  	case <-pool.closeCh:
   216  		return nil
   217  	}
   218  	return <-req.result
   219  }
   220  
   221  // registered should be called after a successful handshake
   222  func (pool *serverPool) registered(entry *poolEntry) {
   223  	log.Debug("Registered new entry", "enode", entry.node.ID())
   224  	req := &registerReq{entry: entry, done: make(chan struct{})}
   225  	select {
   226  	case pool.registerCh <- req:
   227  	case <-pool.closeCh:
   228  		return
   229  	}
   230  	<-req.done
   231  }
   232  
   233  // disconnect should be called when ending a connection. Service quality statistics
   234  // can be updated optionally (not updated if no registration happened, in this case
   235  // only connection statistics are updated, just like in case of timeout)
   236  func (pool *serverPool) disconnect(entry *poolEntry) {
   237  	stopped := false
   238  	select {
   239  	case <-pool.closeCh:
   240  		stopped = true
   241  	default:
   242  	}
   243  	log.Debug("Disconnected old entry", "enode", entry.node.ID())
   244  	req := &disconnReq{entry: entry, stopped: stopped, done: make(chan struct{})}
   245  
   246  	// Block until disconnection request is served.
   247  	pool.disconnCh <- req
   248  	<-req.done
   249  }
   250  
   251  const (
   252  	pseBlockDelay = iota
   253  	pseResponseTime
   254  	pseResponseTimeout
   255  )
   256  
   257  // poolStatAdjust records are sent to adjust peer block delay/response time statistics
   258  type poolStatAdjust struct {
   259  	adjustType int
   260  	entry      *poolEntry
   261  	time       time.Duration
   262  }
   263  
   264  // adjustBlockDelay adjusts the block announce delay statistics of a node
   265  func (pool *serverPool) adjustBlockDelay(entry *poolEntry, time time.Duration) {
   266  	if entry == nil {
   267  		return
   268  	}
   269  	pool.adjustStats <- poolStatAdjust{pseBlockDelay, entry, time}
   270  }
   271  
   272  // adjustResponseTime adjusts the request response time statistics of a node
   273  func (pool *serverPool) adjustResponseTime(entry *poolEntry, time time.Duration, timeout bool) {
   274  	if entry == nil {
   275  		return
   276  	}
   277  	if timeout {
   278  		pool.adjustStats <- poolStatAdjust{pseResponseTimeout, entry, time}
   279  	} else {
   280  		pool.adjustStats <- poolStatAdjust{pseResponseTime, entry, time}
   281  	}
   282  }
   283  
   284  // eventLoop handles pool events and mutex locking for all internal functions
   285  func (pool *serverPool) eventLoop() {
   286  	defer pool.wg.Done()
   287  	lookupCnt := 0
   288  	var convTime mclock.AbsTime
   289  	if pool.discSetPeriod != nil {
   290  		pool.discSetPeriod <- time.Millisecond * 100
   291  	}
   292  
   293  	// disconnect updates service quality statistics depending on the connection time
   294  	// and disconnection initiator.
   295  	disconnect := func(req *disconnReq, stopped bool) {
   296  		// Handle peer disconnection requests.
   297  		entry := req.entry
   298  		if entry.state == psRegistered {
   299  			connAdjust := float64(mclock.Now()-entry.regTime) / float64(targetConnTime)
   300  			if connAdjust > 1 {
   301  				connAdjust = 1
   302  			}
   303  			if stopped {
   304  				// disconnect requested by ourselves.
   305  				entry.connectStats.add(1, connAdjust)
   306  			} else {
   307  				// disconnect requested by server side.
   308  				entry.connectStats.add(connAdjust, 1)
   309  			}
   310  		}
   311  		entry.state = psNotConnected
   312  
   313  		if entry.knownSelected {
   314  			pool.knownSelected--
   315  		} else {
   316  			pool.newSelected--
   317  		}
   318  		pool.setRetryDial(entry)
   319  		pool.connWg.Done()
   320  		close(req.done)
   321  	}
   322  
   323  	for {
   324  		select {
   325  		case entry := <-pool.timeout:
   326  			if !entry.removed {
   327  				pool.checkDialTimeout(entry)
   328  			}
   329  
   330  		case entry := <-pool.enableRetry:
   331  			if !entry.removed {
   332  				entry.delayedRetry = false
   333  				pool.updateCheckDial(entry)
   334  			}
   335  
   336  		case adj := <-pool.adjustStats:
   337  			switch adj.adjustType {
   338  			case pseBlockDelay:
   339  				adj.entry.delayStats.add(float64(adj.time), 1)
   340  			case pseResponseTime:
   341  				adj.entry.responseStats.add(float64(adj.time), 1)
   342  				adj.entry.timeoutStats.add(0, 1)
   343  			case pseResponseTimeout:
   344  				adj.entry.timeoutStats.add(1, 1)
   345  			}
   346  
   347  		case node := <-pool.discNodes:
   348  			if pool.trustedNodes[node.ID()] == nil {
   349  				entry := pool.findOrNewNode(node)
   350  				pool.updateCheckDial(entry)
   351  			}
   352  
   353  		case conv := <-pool.discLookups:
   354  			if conv {
   355  				if lookupCnt == 0 {
   356  					convTime = mclock.Now()
   357  				}
   358  				lookupCnt++
   359  				if pool.fastDiscover && (lookupCnt == 50 || time.Duration(mclock.Now()-convTime) > time.Minute) {
   360  					pool.fastDiscover = false
   361  					if pool.discSetPeriod != nil {
   362  						pool.discSetPeriod <- time.Minute
   363  					}
   364  				}
   365  			}
   366  
   367  		case req := <-pool.connCh:
   368  			if pool.trustedNodes[req.p.ID()] != nil {
   369  				// ignore trusted nodes
   370  				req.result <- &poolEntry{trusted: true}
   371  			} else {
   372  				// Handle peer connection requests.
   373  				entry := pool.entries[req.p.ID()]
   374  				if entry == nil {
   375  					entry = pool.findOrNewNode(req.node)
   376  				}
   377  				if entry.state == psConnected || entry.state == psRegistered {
   378  					req.result <- nil
   379  					continue
   380  				}
   381  				pool.connWg.Add(1)
   382  				entry.peer = req.p
   383  				entry.state = psConnected
   384  				addr := &poolEntryAddress{
   385  					ip:       req.node.IP(),
   386  					port:     uint16(req.node.TCP()),
   387  					lastSeen: mclock.Now(),
   388  				}
   389  				entry.lastConnected = addr
   390  				entry.addr = make(map[string]*poolEntryAddress)
   391  				entry.addr[addr.strKey()] = addr
   392  				entry.addrSelect = *newWeightedRandomSelect()
   393  				entry.addrSelect.update(addr)
   394  				req.result <- entry
   395  			}
   396  
   397  		case req := <-pool.registerCh:
   398  			if req.entry.trusted {
   399  				continue
   400  			}
   401  			// Handle peer registration requests.
   402  			entry := req.entry
   403  			entry.state = psRegistered
   404  			entry.regTime = mclock.Now()
   405  			if !entry.known {
   406  				pool.newQueue.remove(entry)
   407  				entry.known = true
   408  			}
   409  			pool.knownQueue.setLatest(entry)
   410  			entry.shortRetry = shortRetryCnt
   411  			close(req.done)
   412  
   413  		case req := <-pool.disconnCh:
   414  			if req.entry.trusted {
   415  				continue
   416  			}
   417  			// Handle peer disconnection requests.
   418  			disconnect(req, req.stopped)
   419  
   420  		case <-pool.closeCh:
   421  			if pool.discSetPeriod != nil {
   422  				close(pool.discSetPeriod)
   423  			}
   424  
   425  			// Spawn a goroutine to close the disconnCh after all connections are disconnected.
   426  			go func() {
   427  				pool.connWg.Wait()
   428  				close(pool.disconnCh)
   429  			}()
   430  
   431  			// Handle all remaining disconnection requests before exit.
   432  			for req := range pool.disconnCh {
   433  				disconnect(req, true)
   434  			}
   435  			pool.saveNodes()
   436  			return
   437  		}
   438  	}
   439  }
   440  
   441  func (pool *serverPool) findOrNewNode(node *enode.Node) *poolEntry {
   442  	now := mclock.Now()
   443  	entry := pool.entries[node.ID()]
   444  	if entry == nil {
   445  		log.Debug("Discovered new entry", "id", node.ID())
   446  		entry = &poolEntry{
   447  			node:       node,
   448  			addr:       make(map[string]*poolEntryAddress),
   449  			addrSelect: *newWeightedRandomSelect(),
   450  			shortRetry: shortRetryCnt,
   451  		}
   452  		pool.entries[node.ID()] = entry
   453  		// initialize previously unknown peers with good statistics to give a chance to prove themselves
   454  		entry.connectStats.add(1, initStatsWeight)
   455  		entry.delayStats.add(0, initStatsWeight)
   456  		entry.responseStats.add(0, initStatsWeight)
   457  		entry.timeoutStats.add(0, initStatsWeight)
   458  	}
   459  	entry.lastDiscovered = now
   460  	addr := &poolEntryAddress{ip: node.IP(), port: uint16(node.TCP())}
   461  	if a, ok := entry.addr[addr.strKey()]; ok {
   462  		addr = a
   463  	} else {
   464  		entry.addr[addr.strKey()] = addr
   465  	}
   466  	addr.lastSeen = now
   467  	entry.addrSelect.update(addr)
   468  	if !entry.known {
   469  		pool.newQueue.setLatest(entry)
   470  	}
   471  	return entry
   472  }
   473  
   474  // loadNodes loads known nodes and their statistics from the database
   475  func (pool *serverPool) loadNodes() {
   476  	enc, err := pool.db.Get(pool.dbKey)
   477  	if err != nil {
   478  		return
   479  	}
   480  	var list []*poolEntry
   481  	err = rlp.DecodeBytes(enc, &list)
   482  	if err != nil {
   483  		log.Debug("Failed to decode node list", "err", err)
   484  		return
   485  	}
   486  	for _, e := range list {
   487  		log.Debug("Loaded server stats", "id", e.node.ID(), "fails", e.lastConnected.fails,
   488  			"conn", fmt.Sprintf("%v/%v", e.connectStats.avg, e.connectStats.weight),
   489  			"delay", fmt.Sprintf("%v/%v", time.Duration(e.delayStats.avg), e.delayStats.weight),
   490  			"response", fmt.Sprintf("%v/%v", time.Duration(e.responseStats.avg), e.responseStats.weight),
   491  			"timeout", fmt.Sprintf("%v/%v", e.timeoutStats.avg, e.timeoutStats.weight))
   492  		pool.entries[e.node.ID()] = e
   493  		if pool.trustedNodes[e.node.ID()] == nil {
   494  			pool.knownQueue.setLatest(e)
   495  			pool.knownSelect.update((*knownEntry)(e))
   496  		}
   497  	}
   498  }
   499  
   500  // connectToTrustedNodes adds trusted server nodes as static trusted peers.
   501  //
   502  // Note: trusted nodes are not handled by the server pool logic, they are not
   503  // added to either the known or new selection pools. They are connected/reconnected
   504  // by p2p.Server whenever possible.
   505  func (pool *serverPool) connectToTrustedNodes() {
   506  	//connect to trusted nodes
   507  	for _, node := range pool.trustedNodes {
   508  		pool.server.AddTrustedPeer(node)
   509  		pool.server.AddPeer(node)
   510  		log.Debug("Added trusted node", "id", node.ID().String())
   511  	}
   512  }
   513  
   514  // parseTrustedNodes returns valid and parsed enodes
   515  func parseTrustedNodes(trustedNodes []string) map[enode.ID]*enode.Node {
   516  	nodes := make(map[enode.ID]*enode.Node)
   517  
   518  	for _, node := range trustedNodes {
   519  		node, err := enode.Parse(enode.ValidSchemes, node)
   520  		if err != nil {
   521  			log.Warn("Trusted node URL invalid", "enode", node, "err", err)
   522  			continue
   523  		}
   524  		nodes[node.ID()] = node
   525  	}
   526  	return nodes
   527  }
   528  
   529  // saveNodes saves known nodes and their statistics into the database. Nodes are
   530  // ordered from least to most recently connected.
   531  func (pool *serverPool) saveNodes() {
   532  	list := make([]*poolEntry, len(pool.knownQueue.queue))
   533  	for i := range list {
   534  		list[i] = pool.knownQueue.fetchOldest()
   535  	}
   536  	enc, err := rlp.EncodeToBytes(list)
   537  	if err == nil {
   538  		pool.db.Put(pool.dbKey, enc)
   539  	}
   540  }
   541  
   542  // removeEntry removes a pool entry when the entry count limit is reached.
   543  // Note that it is called by the new/known queues from which the entry has already
   544  // been removed so removing it from the queues is not necessary.
   545  func (pool *serverPool) removeEntry(entry *poolEntry) {
   546  	pool.newSelect.remove((*discoveredEntry)(entry))
   547  	pool.knownSelect.remove((*knownEntry)(entry))
   548  	entry.removed = true
   549  	delete(pool.entries, entry.node.ID())
   550  }
   551  
   552  // setRetryDial starts the timer which will enable dialing a certain node again
   553  func (pool *serverPool) setRetryDial(entry *poolEntry) {
   554  	delay := longRetryDelay
   555  	if entry.shortRetry > 0 {
   556  		entry.shortRetry--
   557  		delay = shortRetryDelay
   558  	}
   559  	delay += time.Duration(rand.Int63n(int64(delay) + 1))
   560  	entry.delayedRetry = true
   561  	go func() {
   562  		select {
   563  		case <-pool.closeCh:
   564  		case <-time.After(delay):
   565  			select {
   566  			case <-pool.closeCh:
   567  			case pool.enableRetry <- entry:
   568  			}
   569  		}
   570  	}()
   571  }
   572  
   573  // updateCheckDial is called when an entry can potentially be dialed again. It updates
   574  // its selection weights and checks if new dials can/should be made.
   575  func (pool *serverPool) updateCheckDial(entry *poolEntry) {
   576  	pool.newSelect.update((*discoveredEntry)(entry))
   577  	pool.knownSelect.update((*knownEntry)(entry))
   578  	pool.checkDial()
   579  }
   580  
   581  // checkDial checks if new dials can/should be made. It tries to select servers both
   582  // based on good statistics and recent discovery.
   583  func (pool *serverPool) checkDial() {
   584  	fillWithKnownSelects := !pool.fastDiscover
   585  	for pool.knownSelected < targetKnownSelect {
   586  		entry := pool.knownSelect.choose()
   587  		if entry == nil {
   588  			fillWithKnownSelects = false
   589  			break
   590  		}
   591  		pool.dial((*poolEntry)(entry.(*knownEntry)), true)
   592  	}
   593  	for pool.knownSelected+pool.newSelected < targetServerCount {
   594  		entry := pool.newSelect.choose()
   595  		if entry == nil {
   596  			break
   597  		}
   598  		pool.dial((*poolEntry)(entry.(*discoveredEntry)), false)
   599  	}
   600  	if fillWithKnownSelects {
   601  		// no more newly discovered nodes to select and since fast discover period
   602  		// is over, we probably won't find more in the near future so select more
   603  		// known entries if possible
   604  		for pool.knownSelected < targetServerCount {
   605  			entry := pool.knownSelect.choose()
   606  			if entry == nil {
   607  				break
   608  			}
   609  			pool.dial((*poolEntry)(entry.(*knownEntry)), true)
   610  		}
   611  	}
   612  }
   613  
   614  // dial initiates a new connection
   615  func (pool *serverPool) dial(entry *poolEntry, knownSelected bool) {
   616  	if pool.server == nil || entry.state != psNotConnected {
   617  		return
   618  	}
   619  	entry.state = psDialed
   620  	entry.knownSelected = knownSelected
   621  	if knownSelected {
   622  		pool.knownSelected++
   623  	} else {
   624  		pool.newSelected++
   625  	}
   626  	addr := entry.addrSelect.choose().(*poolEntryAddress)
   627  	log.Debug("Dialing new peer", "lesaddr", entry.node.ID().String()+"@"+addr.strKey(), "set", len(entry.addr), "known", knownSelected)
   628  	entry.dialed = addr
   629  	go func() {
   630  		pool.server.AddPeer(entry.node)
   631  		select {
   632  		case <-pool.closeCh:
   633  		case <-time.After(dialTimeout):
   634  			select {
   635  			case <-pool.closeCh:
   636  			case pool.timeout <- entry:
   637  			}
   638  		}
   639  	}()
   640  }
   641  
   642  // checkDialTimeout checks if the node is still in dialed state and if so, resets it
   643  // and adjusts connection statistics accordingly.
   644  func (pool *serverPool) checkDialTimeout(entry *poolEntry) {
   645  	if entry.state != psDialed {
   646  		return
   647  	}
   648  	log.Debug("Dial timeout", "lesaddr", entry.node.ID().String()+"@"+entry.dialed.strKey())
   649  	entry.state = psNotConnected
   650  	if entry.knownSelected {
   651  		pool.knownSelected--
   652  	} else {
   653  		pool.newSelected--
   654  	}
   655  	entry.connectStats.add(0, 1)
   656  	entry.dialed.fails++
   657  	pool.setRetryDial(entry)
   658  }
   659  
   660  const (
   661  	psNotConnected = iota
   662  	psDialed
   663  	psConnected
   664  	psRegistered
   665  )
   666  
   667  // poolEntry represents a server node and stores its current state and statistics.
   668  type poolEntry struct {
   669  	peer                  *peer
   670  	pubkey                [64]byte // secp256k1 key of the node
   671  	addr                  map[string]*poolEntryAddress
   672  	node                  *enode.Node
   673  	lastConnected, dialed *poolEntryAddress
   674  	addrSelect            weightedRandomSelect
   675  
   676  	lastDiscovered                mclock.AbsTime
   677  	known, knownSelected, trusted bool
   678  	connectStats, delayStats      poolStats
   679  	responseStats, timeoutStats   poolStats
   680  	state                         int
   681  	regTime                       mclock.AbsTime
   682  	queueIdx                      int
   683  	removed                       bool
   684  
   685  	delayedRetry bool
   686  	shortRetry   int
   687  }
   688  
   689  // poolEntryEnc is the RLP encoding of poolEntry.
   690  type poolEntryEnc struct {
   691  	Pubkey                     []byte
   692  	IP                         net.IP
   693  	Port                       uint16
   694  	Fails                      uint
   695  	CStat, DStat, RStat, TStat poolStats
   696  }
   697  
   698  func (e *poolEntry) EncodeRLP(w io.Writer) error {
   699  	return rlp.Encode(w, &poolEntryEnc{
   700  		Pubkey: encodePubkey64(e.node.Pubkey()),
   701  		IP:     e.lastConnected.ip,
   702  		Port:   e.lastConnected.port,
   703  		Fails:  e.lastConnected.fails,
   704  		CStat:  e.connectStats,
   705  		DStat:  e.delayStats,
   706  		RStat:  e.responseStats,
   707  		TStat:  e.timeoutStats,
   708  	})
   709  }
   710  
   711  func (e *poolEntry) DecodeRLP(s *rlp.Stream) error {
   712  	var entry poolEntryEnc
   713  	if err := s.Decode(&entry); err != nil {
   714  		return err
   715  	}
   716  	pubkey, err := decodePubkey64(entry.Pubkey)
   717  	if err != nil {
   718  		return err
   719  	}
   720  	addr := &poolEntryAddress{ip: entry.IP, port: entry.Port, fails: entry.Fails, lastSeen: mclock.Now()}
   721  	e.node = enode.NewV4(pubkey, entry.IP, int(entry.Port), int(entry.Port))
   722  	e.addr = make(map[string]*poolEntryAddress)
   723  	e.addr[addr.strKey()] = addr
   724  	e.addrSelect = *newWeightedRandomSelect()
   725  	e.addrSelect.update(addr)
   726  	e.lastConnected = addr
   727  	e.connectStats = entry.CStat
   728  	e.delayStats = entry.DStat
   729  	e.responseStats = entry.RStat
   730  	e.timeoutStats = entry.TStat
   731  	e.shortRetry = shortRetryCnt
   732  	e.known = true
   733  	return nil
   734  }
   735  
   736  func encodePubkey64(pub *ecdsa.PublicKey) []byte {
   737  	return crypto.FromECDSAPub(pub)[1:]
   738  }
   739  
   740  func decodePubkey64(b []byte) (*ecdsa.PublicKey, error) {
   741  	return crypto.UnmarshalPubkey(append([]byte{0x04}, b...))
   742  }
   743  
   744  // discoveredEntry implements wrsItem
   745  type discoveredEntry poolEntry
   746  
   747  // Weight calculates random selection weight for newly discovered entries
   748  func (e *discoveredEntry) Weight() int64 {
   749  	if e.state != psNotConnected || e.delayedRetry {
   750  		return 0
   751  	}
   752  	t := time.Duration(mclock.Now() - e.lastDiscovered)
   753  	if t <= discoverExpireStart {
   754  		return 1000000000
   755  	}
   756  	return int64(1000000000 * math.Exp(-float64(t-discoverExpireStart)/float64(discoverExpireConst)))
   757  }
   758  
   759  // knownEntry implements wrsItem
   760  type knownEntry poolEntry
   761  
   762  // Weight calculates random selection weight for known entries
   763  func (e *knownEntry) Weight() int64 {
   764  	if e.state != psNotConnected || !e.known || e.delayedRetry {
   765  		return 0
   766  	}
   767  	return int64(1000000000 * e.connectStats.recentAvg() * math.Exp(-float64(e.lastConnected.fails)*failDropLn-e.responseStats.recentAvg()/float64(responseScoreTC)-e.delayStats.recentAvg()/float64(delayScoreTC)) * math.Pow(1-e.timeoutStats.recentAvg(), timeoutPow))
   768  }
   769  
   770  // poolEntryAddress is a separate object because currently it is necessary to remember
   771  // multiple potential network addresses for a pool entry. This will be removed after
   772  // the final implementation of v5 discovery which will retrieve signed and serial
   773  // numbered advertisements, making it clear which IP/port is the latest one.
   774  type poolEntryAddress struct {
   775  	ip       net.IP
   776  	port     uint16
   777  	lastSeen mclock.AbsTime // last time it was discovered, connected or loaded from db
   778  	fails    uint           // connection failures since last successful connection (persistent)
   779  }
   780  
   781  func (a *poolEntryAddress) Weight() int64 {
   782  	t := time.Duration(mclock.Now() - a.lastSeen)
   783  	return int64(1000000*math.Exp(-float64(t)/float64(discoverExpireConst)-float64(a.fails)*addrFailDropLn)) + 1
   784  }
   785  
   786  func (a *poolEntryAddress) strKey() string {
   787  	return a.ip.String() + ":" + strconv.Itoa(int(a.port))
   788  }
   789  
   790  // poolStats implement statistics for a certain quantity with a long term average
   791  // and a short term value which is adjusted exponentially with a factor of
   792  // pstatRecentAdjust with each update and also returned exponentially to the
   793  // average with the time constant pstatReturnToMeanTC
   794  type poolStats struct {
   795  	sum, weight, avg, recent float64
   796  	lastRecalc               mclock.AbsTime
   797  }
   798  
   799  // init initializes stats with a long term sum/update count pair retrieved from the database
   800  func (s *poolStats) init(sum, weight float64) {
   801  	s.sum = sum
   802  	s.weight = weight
   803  	var avg float64
   804  	if weight > 0 {
   805  		avg = s.sum / weight
   806  	}
   807  	s.avg = avg
   808  	s.recent = avg
   809  	s.lastRecalc = mclock.Now()
   810  }
   811  
   812  // recalc recalculates recent value return-to-mean and long term average
   813  func (s *poolStats) recalc() {
   814  	now := mclock.Now()
   815  	s.recent = s.avg + (s.recent-s.avg)*math.Exp(-float64(now-s.lastRecalc)/float64(pstatReturnToMeanTC))
   816  	if s.sum == 0 {
   817  		s.avg = 0
   818  	} else {
   819  		if s.sum > s.weight*1e30 {
   820  			s.avg = 1e30
   821  		} else {
   822  			s.avg = s.sum / s.weight
   823  		}
   824  	}
   825  	s.lastRecalc = now
   826  }
   827  
   828  // add updates the stats with a new value
   829  func (s *poolStats) add(value, weight float64) {
   830  	s.weight += weight
   831  	s.sum += value * weight
   832  	s.recalc()
   833  }
   834  
   835  // recentAvg returns the short-term adjusted average
   836  func (s *poolStats) recentAvg() float64 {
   837  	s.recalc()
   838  	return s.recent
   839  }
   840  
   841  func (s *poolStats) EncodeRLP(w io.Writer) error {
   842  	return rlp.Encode(w, []interface{}{math.Float64bits(s.sum), math.Float64bits(s.weight)})
   843  }
   844  
   845  func (s *poolStats) DecodeRLP(st *rlp.Stream) error {
   846  	var stats struct {
   847  		SumUint, WeightUint uint64
   848  	}
   849  	if err := st.Decode(&stats); err != nil {
   850  		return err
   851  	}
   852  	s.init(math.Float64frombits(stats.SumUint), math.Float64frombits(stats.WeightUint))
   853  	return nil
   854  }
   855  
   856  // poolEntryQueue keeps track of its least recently accessed entries and removes
   857  // them when the number of entries reaches the limit
   858  type poolEntryQueue struct {
   859  	queue                  map[int]*poolEntry // known nodes indexed by their latest lastConnCnt value
   860  	newPtr, oldPtr, maxCnt int
   861  	removeFromPool         func(*poolEntry)
   862  }
   863  
   864  // newPoolEntryQueue returns a new poolEntryQueue
   865  func newPoolEntryQueue(maxCnt int, removeFromPool func(*poolEntry)) poolEntryQueue {
   866  	return poolEntryQueue{queue: make(map[int]*poolEntry), maxCnt: maxCnt, removeFromPool: removeFromPool}
   867  }
   868  
   869  // fetchOldest returns and removes the least recently accessed entry
   870  func (q *poolEntryQueue) fetchOldest() *poolEntry {
   871  	if len(q.queue) == 0 {
   872  		return nil
   873  	}
   874  	for {
   875  		if e := q.queue[q.oldPtr]; e != nil {
   876  			delete(q.queue, q.oldPtr)
   877  			q.oldPtr++
   878  			return e
   879  		}
   880  		q.oldPtr++
   881  	}
   882  }
   883  
   884  // remove removes an entry from the queue
   885  func (q *poolEntryQueue) remove(entry *poolEntry) {
   886  	if q.queue[entry.queueIdx] == entry {
   887  		delete(q.queue, entry.queueIdx)
   888  	}
   889  }
   890  
   891  // setLatest adds or updates a recently accessed entry. It also checks if an old entry
   892  // needs to be removed and removes it from the parent pool too with a callback function.
   893  func (q *poolEntryQueue) setLatest(entry *poolEntry) {
   894  	if q.queue[entry.queueIdx] == entry {
   895  		delete(q.queue, entry.queueIdx)
   896  	} else {
   897  		if len(q.queue) == q.maxCnt {
   898  			e := q.fetchOldest()
   899  			q.remove(e)
   900  			q.removeFromPool(e)
   901  		}
   902  	}
   903  	entry.queueIdx = q.newPtr
   904  	q.queue[entry.queueIdx] = entry
   905  	q.newPtr++
   906  }