github.com/tdcblockchain/tdcblockchain@v0.0.0-20191111034745-805c65ade158/les/distributor.go (about) 1 // Copyright 2017 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package les 18 19 import ( 20 "container/list" 21 "sync" 22 "time" 23 24 "github.com/ethereum/go-ethereum/common/mclock" 25 ) 26 27 // requestDistributor implements a mechanism that distributes requests to 28 // suitable peers, obeying flow control rules and prioritizing them in creation 29 // order (even when a resend is necessary). 30 type requestDistributor struct { 31 clock mclock.Clock 32 reqQueue *list.List 33 lastReqOrder uint64 34 peers map[distPeer]struct{} 35 peerLock sync.RWMutex 36 loopChn chan struct{} 37 loopNextSent bool 38 lock sync.Mutex 39 40 closeCh chan struct{} 41 wg sync.WaitGroup 42 } 43 44 // distPeer is an LES server peer interface for the request distributor. 45 // waitBefore returns either the necessary waiting time before sending a request 46 // with the given upper estimated cost or the estimated remaining relative buffer 47 // value after sending such a request (in which case the request can be sent 48 // immediately). At least one of these values is always zero. 49 type distPeer interface { 50 waitBefore(uint64) (time.Duration, float64) 51 canQueue() bool 52 queueSend(f func()) 53 } 54 55 // distReq is the request abstraction used by the distributor. It is based on 56 // three callback functions: 57 // - getCost returns the upper estimate of the cost of sending the request to a given peer 58 // - canSend tells if the server peer is suitable to serve the request 59 // - request prepares sending the request to the given peer and returns a function that 60 // does the actual sending. Request order should be preserved but the callback itself should not 61 // block until it is sent because other peers might still be able to receive requests while 62 // one of them is blocking. Instead, the returned function is put in the peer's send queue. 63 type distReq struct { 64 getCost func(distPeer) uint64 65 canSend func(distPeer) bool 66 request func(distPeer) func() 67 68 reqOrder uint64 69 sentChn chan distPeer 70 element *list.Element 71 waitForPeers mclock.AbsTime 72 enterQueue mclock.AbsTime 73 } 74 75 // newRequestDistributor creates a new request distributor 76 func newRequestDistributor(peers *peerSet, clock mclock.Clock) *requestDistributor { 77 d := &requestDistributor{ 78 clock: clock, 79 reqQueue: list.New(), 80 loopChn: make(chan struct{}, 2), 81 closeCh: make(chan struct{}), 82 peers: make(map[distPeer]struct{}), 83 } 84 if peers != nil { 85 peers.notify(d) 86 } 87 d.wg.Add(1) 88 go d.loop() 89 return d 90 } 91 92 // registerPeer implements peerSetNotify 93 func (d *requestDistributor) registerPeer(p *peer) { 94 d.peerLock.Lock() 95 d.peers[p] = struct{}{} 96 d.peerLock.Unlock() 97 } 98 99 // unregisterPeer implements peerSetNotify 100 func (d *requestDistributor) unregisterPeer(p *peer) { 101 d.peerLock.Lock() 102 delete(d.peers, p) 103 d.peerLock.Unlock() 104 } 105 106 // registerTestPeer adds a new test peer 107 func (d *requestDistributor) registerTestPeer(p distPeer) { 108 d.peerLock.Lock() 109 d.peers[p] = struct{}{} 110 d.peerLock.Unlock() 111 } 112 113 // distMaxWait is the maximum waiting time after which further necessary waiting 114 // times are recalculated based on new feedback from the servers 115 const distMaxWait = time.Millisecond * 50 116 117 // waitForPeers is the time window in which a request does not fail even if it 118 // has no suitable peers to send to at the moment 119 const waitForPeers = time.Second * 3 120 121 // main event loop 122 func (d *requestDistributor) loop() { 123 defer d.wg.Done() 124 for { 125 select { 126 case <-d.closeCh: 127 d.lock.Lock() 128 elem := d.reqQueue.Front() 129 for elem != nil { 130 req := elem.Value.(*distReq) 131 close(req.sentChn) 132 req.sentChn = nil 133 elem = elem.Next() 134 } 135 d.lock.Unlock() 136 return 137 case <-d.loopChn: 138 d.lock.Lock() 139 d.loopNextSent = false 140 loop: 141 for { 142 peer, req, wait := d.nextRequest() 143 if req != nil && wait == 0 { 144 chn := req.sentChn // save sentChn because remove sets it to nil 145 d.remove(req) 146 send := req.request(peer) 147 if send != nil { 148 peer.queueSend(send) 149 requestSendDelay.Update(time.Duration(d.clock.Now() - req.enterQueue)) 150 } 151 chn <- peer 152 close(chn) 153 } else { 154 if wait == 0 { 155 // no request to send and nothing to wait for; the next 156 // queued request will wake up the loop 157 break loop 158 } 159 d.loopNextSent = true // a "next" signal has been sent, do not send another one until this one has been received 160 if wait > distMaxWait { 161 // waiting times may be reduced by incoming request replies, if it is too long, recalculate it periodically 162 wait = distMaxWait 163 } 164 go func() { 165 d.clock.Sleep(wait) 166 d.loopChn <- struct{}{} 167 }() 168 break loop 169 } 170 } 171 d.lock.Unlock() 172 } 173 } 174 } 175 176 // selectPeerItem represents a peer to be selected for a request by weightedRandomSelect 177 type selectPeerItem struct { 178 peer distPeer 179 req *distReq 180 weight int64 181 } 182 183 // Weight implements wrsItem interface 184 func (sp selectPeerItem) Weight() int64 { 185 return sp.weight 186 } 187 188 // nextRequest returns the next possible request from any peer, along with the 189 // associated peer and necessary waiting time 190 func (d *requestDistributor) nextRequest() (distPeer, *distReq, time.Duration) { 191 checkedPeers := make(map[distPeer]struct{}) 192 elem := d.reqQueue.Front() 193 var ( 194 bestWait time.Duration 195 sel *weightedRandomSelect 196 ) 197 198 d.peerLock.RLock() 199 defer d.peerLock.RUnlock() 200 201 peerCount := len(d.peers) 202 for (len(checkedPeers) < peerCount || elem == d.reqQueue.Front()) && elem != nil { 203 req := elem.Value.(*distReq) 204 canSend := false 205 now := d.clock.Now() 206 if req.waitForPeers > now { 207 canSend = true 208 wait := time.Duration(req.waitForPeers - now) 209 if bestWait == 0 || wait < bestWait { 210 bestWait = wait 211 } 212 } 213 for peer := range d.peers { 214 if _, ok := checkedPeers[peer]; !ok && peer.canQueue() && req.canSend(peer) { 215 canSend = true 216 cost := req.getCost(peer) 217 wait, bufRemain := peer.waitBefore(cost) 218 if wait == 0 { 219 if sel == nil { 220 sel = newWeightedRandomSelect() 221 } 222 sel.update(selectPeerItem{peer: peer, req: req, weight: int64(bufRemain*1000000) + 1}) 223 } else { 224 if bestWait == 0 || wait < bestWait { 225 bestWait = wait 226 } 227 } 228 checkedPeers[peer] = struct{}{} 229 } 230 } 231 next := elem.Next() 232 if !canSend && elem == d.reqQueue.Front() { 233 close(req.sentChn) 234 d.remove(req) 235 } 236 elem = next 237 } 238 239 if sel != nil { 240 c := sel.choose().(selectPeerItem) 241 return c.peer, c.req, 0 242 } 243 return nil, nil, bestWait 244 } 245 246 // queue adds a request to the distribution queue, returns a channel where the 247 // receiving peer is sent once the request has been sent (request callback returned). 248 // If the request is cancelled or timed out without suitable peers, the channel is 249 // closed without sending any peer references to it. 250 func (d *requestDistributor) queue(r *distReq) chan distPeer { 251 d.lock.Lock() 252 defer d.lock.Unlock() 253 254 if r.reqOrder == 0 { 255 d.lastReqOrder++ 256 r.reqOrder = d.lastReqOrder 257 r.waitForPeers = d.clock.Now() + mclock.AbsTime(waitForPeers) 258 } 259 // Assign the timestamp when the request is queued no matter it's 260 // a new one or re-queued one. 261 r.enterQueue = d.clock.Now() 262 263 back := d.reqQueue.Back() 264 if back == nil || r.reqOrder > back.Value.(*distReq).reqOrder { 265 r.element = d.reqQueue.PushBack(r) 266 } else { 267 before := d.reqQueue.Front() 268 for before.Value.(*distReq).reqOrder < r.reqOrder { 269 before = before.Next() 270 } 271 r.element = d.reqQueue.InsertBefore(r, before) 272 } 273 274 if !d.loopNextSent { 275 d.loopNextSent = true 276 d.loopChn <- struct{}{} 277 } 278 279 r.sentChn = make(chan distPeer, 1) 280 return r.sentChn 281 } 282 283 // cancel removes a request from the queue if it has not been sent yet (returns 284 // false if it has been sent already). It is guaranteed that the callback functions 285 // will not be called after cancel returns. 286 func (d *requestDistributor) cancel(r *distReq) bool { 287 d.lock.Lock() 288 defer d.lock.Unlock() 289 290 if r.sentChn == nil { 291 return false 292 } 293 294 close(r.sentChn) 295 d.remove(r) 296 return true 297 } 298 299 // remove removes a request from the queue 300 func (d *requestDistributor) remove(r *distReq) { 301 r.sentChn = nil 302 if r.element != nil { 303 d.reqQueue.Remove(r.element) 304 r.element = nil 305 } 306 } 307 308 func (d *requestDistributor) close() { 309 close(d.closeCh) 310 d.wg.Wait() 311 }