github.com/tdcblockchain/tdcblockchain@v0.0.0-20191111034745-805c65ade158/les/distributor.go (about)

     1  // Copyright 2017 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package les
    18  
    19  import (
    20  	"container/list"
    21  	"sync"
    22  	"time"
    23  
    24  	"github.com/ethereum/go-ethereum/common/mclock"
    25  )
    26  
    27  // requestDistributor implements a mechanism that distributes requests to
    28  // suitable peers, obeying flow control rules and prioritizing them in creation
    29  // order (even when a resend is necessary).
    30  type requestDistributor struct {
    31  	clock        mclock.Clock
    32  	reqQueue     *list.List
    33  	lastReqOrder uint64
    34  	peers        map[distPeer]struct{}
    35  	peerLock     sync.RWMutex
    36  	loopChn      chan struct{}
    37  	loopNextSent bool
    38  	lock         sync.Mutex
    39  
    40  	closeCh chan struct{}
    41  	wg      sync.WaitGroup
    42  }
    43  
    44  // distPeer is an LES server peer interface for the request distributor.
    45  // waitBefore returns either the necessary waiting time before sending a request
    46  // with the given upper estimated cost or the estimated remaining relative buffer
    47  // value after sending such a request (in which case the request can be sent
    48  // immediately). At least one of these values is always zero.
    49  type distPeer interface {
    50  	waitBefore(uint64) (time.Duration, float64)
    51  	canQueue() bool
    52  	queueSend(f func())
    53  }
    54  
    55  // distReq is the request abstraction used by the distributor. It is based on
    56  // three callback functions:
    57  // - getCost returns the upper estimate of the cost of sending the request to a given peer
    58  // - canSend tells if the server peer is suitable to serve the request
    59  // - request prepares sending the request to the given peer and returns a function that
    60  // does the actual sending. Request order should be preserved but the callback itself should not
    61  // block until it is sent because other peers might still be able to receive requests while
    62  // one of them is blocking. Instead, the returned function is put in the peer's send queue.
    63  type distReq struct {
    64  	getCost func(distPeer) uint64
    65  	canSend func(distPeer) bool
    66  	request func(distPeer) func()
    67  
    68  	reqOrder     uint64
    69  	sentChn      chan distPeer
    70  	element      *list.Element
    71  	waitForPeers mclock.AbsTime
    72  	enterQueue   mclock.AbsTime
    73  }
    74  
    75  // newRequestDistributor creates a new request distributor
    76  func newRequestDistributor(peers *peerSet, clock mclock.Clock) *requestDistributor {
    77  	d := &requestDistributor{
    78  		clock:    clock,
    79  		reqQueue: list.New(),
    80  		loopChn:  make(chan struct{}, 2),
    81  		closeCh:  make(chan struct{}),
    82  		peers:    make(map[distPeer]struct{}),
    83  	}
    84  	if peers != nil {
    85  		peers.notify(d)
    86  	}
    87  	d.wg.Add(1)
    88  	go d.loop()
    89  	return d
    90  }
    91  
    92  // registerPeer implements peerSetNotify
    93  func (d *requestDistributor) registerPeer(p *peer) {
    94  	d.peerLock.Lock()
    95  	d.peers[p] = struct{}{}
    96  	d.peerLock.Unlock()
    97  }
    98  
    99  // unregisterPeer implements peerSetNotify
   100  func (d *requestDistributor) unregisterPeer(p *peer) {
   101  	d.peerLock.Lock()
   102  	delete(d.peers, p)
   103  	d.peerLock.Unlock()
   104  }
   105  
   106  // registerTestPeer adds a new test peer
   107  func (d *requestDistributor) registerTestPeer(p distPeer) {
   108  	d.peerLock.Lock()
   109  	d.peers[p] = struct{}{}
   110  	d.peerLock.Unlock()
   111  }
   112  
   113  // distMaxWait is the maximum waiting time after which further necessary waiting
   114  // times are recalculated based on new feedback from the servers
   115  const distMaxWait = time.Millisecond * 50
   116  
   117  // waitForPeers is the time window in which a request does not fail even if it
   118  // has no suitable peers to send to at the moment
   119  const waitForPeers = time.Second * 3
   120  
   121  // main event loop
   122  func (d *requestDistributor) loop() {
   123  	defer d.wg.Done()
   124  	for {
   125  		select {
   126  		case <-d.closeCh:
   127  			d.lock.Lock()
   128  			elem := d.reqQueue.Front()
   129  			for elem != nil {
   130  				req := elem.Value.(*distReq)
   131  				close(req.sentChn)
   132  				req.sentChn = nil
   133  				elem = elem.Next()
   134  			}
   135  			d.lock.Unlock()
   136  			return
   137  		case <-d.loopChn:
   138  			d.lock.Lock()
   139  			d.loopNextSent = false
   140  		loop:
   141  			for {
   142  				peer, req, wait := d.nextRequest()
   143  				if req != nil && wait == 0 {
   144  					chn := req.sentChn // save sentChn because remove sets it to nil
   145  					d.remove(req)
   146  					send := req.request(peer)
   147  					if send != nil {
   148  						peer.queueSend(send)
   149  						requestSendDelay.Update(time.Duration(d.clock.Now() - req.enterQueue))
   150  					}
   151  					chn <- peer
   152  					close(chn)
   153  				} else {
   154  					if wait == 0 {
   155  						// no request to send and nothing to wait for; the next
   156  						// queued request will wake up the loop
   157  						break loop
   158  					}
   159  					d.loopNextSent = true // a "next" signal has been sent, do not send another one until this one has been received
   160  					if wait > distMaxWait {
   161  						// waiting times may be reduced by incoming request replies, if it is too long, recalculate it periodically
   162  						wait = distMaxWait
   163  					}
   164  					go func() {
   165  						d.clock.Sleep(wait)
   166  						d.loopChn <- struct{}{}
   167  					}()
   168  					break loop
   169  				}
   170  			}
   171  			d.lock.Unlock()
   172  		}
   173  	}
   174  }
   175  
   176  // selectPeerItem represents a peer to be selected for a request by weightedRandomSelect
   177  type selectPeerItem struct {
   178  	peer   distPeer
   179  	req    *distReq
   180  	weight int64
   181  }
   182  
   183  // Weight implements wrsItem interface
   184  func (sp selectPeerItem) Weight() int64 {
   185  	return sp.weight
   186  }
   187  
   188  // nextRequest returns the next possible request from any peer, along with the
   189  // associated peer and necessary waiting time
   190  func (d *requestDistributor) nextRequest() (distPeer, *distReq, time.Duration) {
   191  	checkedPeers := make(map[distPeer]struct{})
   192  	elem := d.reqQueue.Front()
   193  	var (
   194  		bestWait time.Duration
   195  		sel      *weightedRandomSelect
   196  	)
   197  
   198  	d.peerLock.RLock()
   199  	defer d.peerLock.RUnlock()
   200  
   201  	peerCount := len(d.peers)
   202  	for (len(checkedPeers) < peerCount || elem == d.reqQueue.Front()) && elem != nil {
   203  		req := elem.Value.(*distReq)
   204  		canSend := false
   205  		now := d.clock.Now()
   206  		if req.waitForPeers > now {
   207  			canSend = true
   208  			wait := time.Duration(req.waitForPeers - now)
   209  			if bestWait == 0 || wait < bestWait {
   210  				bestWait = wait
   211  			}
   212  		}
   213  		for peer := range d.peers {
   214  			if _, ok := checkedPeers[peer]; !ok && peer.canQueue() && req.canSend(peer) {
   215  				canSend = true
   216  				cost := req.getCost(peer)
   217  				wait, bufRemain := peer.waitBefore(cost)
   218  				if wait == 0 {
   219  					if sel == nil {
   220  						sel = newWeightedRandomSelect()
   221  					}
   222  					sel.update(selectPeerItem{peer: peer, req: req, weight: int64(bufRemain*1000000) + 1})
   223  				} else {
   224  					if bestWait == 0 || wait < bestWait {
   225  						bestWait = wait
   226  					}
   227  				}
   228  				checkedPeers[peer] = struct{}{}
   229  			}
   230  		}
   231  		next := elem.Next()
   232  		if !canSend && elem == d.reqQueue.Front() {
   233  			close(req.sentChn)
   234  			d.remove(req)
   235  		}
   236  		elem = next
   237  	}
   238  
   239  	if sel != nil {
   240  		c := sel.choose().(selectPeerItem)
   241  		return c.peer, c.req, 0
   242  	}
   243  	return nil, nil, bestWait
   244  }
   245  
   246  // queue adds a request to the distribution queue, returns a channel where the
   247  // receiving peer is sent once the request has been sent (request callback returned).
   248  // If the request is cancelled or timed out without suitable peers, the channel is
   249  // closed without sending any peer references to it.
   250  func (d *requestDistributor) queue(r *distReq) chan distPeer {
   251  	d.lock.Lock()
   252  	defer d.lock.Unlock()
   253  
   254  	if r.reqOrder == 0 {
   255  		d.lastReqOrder++
   256  		r.reqOrder = d.lastReqOrder
   257  		r.waitForPeers = d.clock.Now() + mclock.AbsTime(waitForPeers)
   258  	}
   259  	// Assign the timestamp when the request is queued no matter it's
   260  	// a new one or re-queued one.
   261  	r.enterQueue = d.clock.Now()
   262  
   263  	back := d.reqQueue.Back()
   264  	if back == nil || r.reqOrder > back.Value.(*distReq).reqOrder {
   265  		r.element = d.reqQueue.PushBack(r)
   266  	} else {
   267  		before := d.reqQueue.Front()
   268  		for before.Value.(*distReq).reqOrder < r.reqOrder {
   269  			before = before.Next()
   270  		}
   271  		r.element = d.reqQueue.InsertBefore(r, before)
   272  	}
   273  
   274  	if !d.loopNextSent {
   275  		d.loopNextSent = true
   276  		d.loopChn <- struct{}{}
   277  	}
   278  
   279  	r.sentChn = make(chan distPeer, 1)
   280  	return r.sentChn
   281  }
   282  
   283  // cancel removes a request from the queue if it has not been sent yet (returns
   284  // false if it has been sent already). It is guaranteed that the callback functions
   285  // will not be called after cancel returns.
   286  func (d *requestDistributor) cancel(r *distReq) bool {
   287  	d.lock.Lock()
   288  	defer d.lock.Unlock()
   289  
   290  	if r.sentChn == nil {
   291  		return false
   292  	}
   293  
   294  	close(r.sentChn)
   295  	d.remove(r)
   296  	return true
   297  }
   298  
   299  // remove removes a request from the queue
   300  func (d *requestDistributor) remove(r *distReq) {
   301  	r.sentChn = nil
   302  	if r.element != nil {
   303  		d.reqQueue.Remove(r.element)
   304  		r.element = nil
   305  	}
   306  }
   307  
   308  func (d *requestDistributor) close() {
   309  	close(d.closeCh)
   310  	d.wg.Wait()
   311  }