github.com/JFJun/bsc@v1.0.0/les/clientpool.go (about) 1 // Copyright 2019 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package les 18 19 import ( 20 "bytes" 21 "encoding/binary" 22 "fmt" 23 "io" 24 "math" 25 "sync" 26 "time" 27 28 "github.com/JFJun/bsc/common" 29 "github.com/JFJun/bsc/common/mclock" 30 "github.com/JFJun/bsc/common/prque" 31 "github.com/JFJun/bsc/ethdb" 32 "github.com/JFJun/bsc/log" 33 "github.com/JFJun/bsc/p2p/enode" 34 "github.com/JFJun/bsc/rlp" 35 lru "github.com/hashicorp/golang-lru" 36 ) 37 38 const ( 39 negBalanceExpTC = time.Hour // time constant for exponentially reducing negative balance 40 fixedPointMultiplier = 0x1000000 // constant to convert logarithms to fixed point format 41 lazyQueueRefresh = time.Second * 10 // refresh period of the connected queue 42 persistCumulativeTimeRefresh = time.Minute * 5 // refresh period of the cumulative running time persistence 43 posBalanceCacheLimit = 8192 // the maximum number of cached items in positive balance queue 44 negBalanceCacheLimit = 8192 // the maximum number of cached items in negative balance queue 45 46 // connectedBias is applied to already connected clients So that 47 // already connected client won't be kicked out very soon and we 48 // can ensure all connected clients can have enough time to request 49 // or sync some data. 50 // 51 // todo(rjl493456442) make it configurable. It can be the option of 52 // free trial time! 53 connectedBias = time.Minute * 3 54 ) 55 56 // clientPool implements a client database that assigns a priority to each client 57 // based on a positive and negative balance. Positive balance is externally assigned 58 // to prioritized clients and is decreased with connection time and processed 59 // requests (unless the price factors are zero). If the positive balance is zero 60 // then negative balance is accumulated. 61 // 62 // Balance tracking and priority calculation for connected clients is done by 63 // balanceTracker. connectedQueue ensures that clients with the lowest positive or 64 // highest negative balance get evicted when the total capacity allowance is full 65 // and new clients with a better balance want to connect. 66 // 67 // Already connected nodes receive a small bias in their favor in order to avoid 68 // accepting and instantly kicking out clients. In theory, we try to ensure that 69 // each client can have several minutes of connection time. 70 // 71 // Balances of disconnected clients are stored in nodeDB including positive balance 72 // and negative banalce. Negative balance is transformed into a logarithmic form 73 // with a constantly shifting linear offset in order to implement an exponential 74 // decrease. Besides nodeDB will have a background thread to check the negative 75 // balance of disconnected client. If the balance is low enough, then the record 76 // will be dropped. 77 type clientPool struct { 78 ndb *nodeDB 79 lock sync.Mutex 80 clock mclock.Clock 81 stopCh chan struct{} 82 closed bool 83 removePeer func(enode.ID) 84 85 connectedMap map[enode.ID]*clientInfo 86 connectedQueue *prque.LazyQueue 87 88 defaultPosFactors, defaultNegFactors priceFactors 89 90 connLimit int // The maximum number of connections that clientpool can support 91 capLimit uint64 // The maximum cumulative capacity that clientpool can support 92 connectedCap uint64 // The sum of the capacity of the current clientpool connected 93 priorityConnected uint64 // The sum of the capacity of currently connected priority clients 94 freeClientCap uint64 // The capacity value of each free client 95 startTime mclock.AbsTime // The timestamp at which the clientpool started running 96 cumulativeTime int64 // The cumulative running time of clientpool at the start point. 97 disableBias bool // Disable connection bias(used in testing) 98 } 99 100 // clientPoolPeer represents a client peer in the pool. 101 // Positive balances are assigned to node key while negative balances are assigned 102 // to freeClientId. Currently network IP address without port is used because 103 // clients have a limited access to IP addresses while new node keys can be easily 104 // generated so it would be useless to assign a negative value to them. 105 type clientPoolPeer interface { 106 ID() enode.ID 107 freeClientId() string 108 updateCapacity(uint64) 109 freezeClient() 110 } 111 112 // clientInfo represents a connected client 113 type clientInfo struct { 114 address string 115 id enode.ID 116 connectedAt mclock.AbsTime 117 capacity uint64 118 priority bool 119 pool *clientPool 120 peer clientPoolPeer 121 queueIndex int // position in connectedQueue 122 balanceTracker balanceTracker 123 posFactors, negFactors priceFactors 124 balanceMetaInfo string 125 } 126 127 // connSetIndex callback updates clientInfo item index in connectedQueue 128 func connSetIndex(a interface{}, index int) { 129 a.(*clientInfo).queueIndex = index 130 } 131 132 // connPriority callback returns actual priority of clientInfo item in connectedQueue 133 func connPriority(a interface{}, now mclock.AbsTime) int64 { 134 c := a.(*clientInfo) 135 return c.balanceTracker.getPriority(now) 136 } 137 138 // connMaxPriority callback returns estimated maximum priority of clientInfo item in connectedQueue 139 func connMaxPriority(a interface{}, until mclock.AbsTime) int64 { 140 c := a.(*clientInfo) 141 pri := c.balanceTracker.estimatedPriority(until, true) 142 c.balanceTracker.addCallback(balanceCallbackQueue, pri+1, func() { 143 c.pool.lock.Lock() 144 if c.queueIndex != -1 { 145 c.pool.connectedQueue.Update(c.queueIndex) 146 } 147 c.pool.lock.Unlock() 148 }) 149 return pri 150 } 151 152 // priceFactors determine the pricing policy (may apply either to positive or 153 // negative balances which may have different factors). 154 // - timeFactor is cost unit per nanosecond of connection time 155 // - capacityFactor is cost unit per nanosecond of connection time per 1000000 capacity 156 // - requestFactor is cost unit per request "realCost" unit 157 type priceFactors struct { 158 timeFactor, capacityFactor, requestFactor float64 159 } 160 161 // newClientPool creates a new client pool 162 func newClientPool(db ethdb.Database, freeClientCap uint64, clock mclock.Clock, removePeer func(enode.ID)) *clientPool { 163 ndb := newNodeDB(db, clock) 164 pool := &clientPool{ 165 ndb: ndb, 166 clock: clock, 167 connectedMap: make(map[enode.ID]*clientInfo), 168 connectedQueue: prque.NewLazyQueue(connSetIndex, connPriority, connMaxPriority, clock, lazyQueueRefresh), 169 freeClientCap: freeClientCap, 170 removePeer: removePeer, 171 startTime: clock.Now(), 172 cumulativeTime: ndb.getCumulativeTime(), 173 stopCh: make(chan struct{}), 174 } 175 // If the negative balance of free client is even lower than 1, 176 // delete this entry. 177 ndb.nbEvictCallBack = func(now mclock.AbsTime, b negBalance) bool { 178 balance := math.Exp(float64(b.logValue-pool.logOffset(now)) / fixedPointMultiplier) 179 return balance <= 1 180 } 181 go func() { 182 for { 183 select { 184 case <-clock.After(lazyQueueRefresh): 185 pool.lock.Lock() 186 pool.connectedQueue.Refresh() 187 pool.lock.Unlock() 188 case <-clock.After(persistCumulativeTimeRefresh): 189 pool.ndb.setCumulativeTime(pool.logOffset(clock.Now())) 190 case <-pool.stopCh: 191 return 192 } 193 } 194 }() 195 return pool 196 } 197 198 // stop shuts the client pool down 199 func (f *clientPool) stop() { 200 close(f.stopCh) 201 f.lock.Lock() 202 f.closed = true 203 f.lock.Unlock() 204 f.ndb.setCumulativeTime(f.logOffset(f.clock.Now())) 205 f.ndb.close() 206 } 207 208 // connect should be called after a successful handshake. If the connection was 209 // rejected, there is no need to call disconnect. 210 func (f *clientPool) connect(peer clientPoolPeer, capacity uint64) bool { 211 f.lock.Lock() 212 defer f.lock.Unlock() 213 214 // Short circuit if clientPool is already closed. 215 if f.closed { 216 return false 217 } 218 // Dedup connected peers. 219 id, freeID := peer.ID(), peer.freeClientId() 220 if _, ok := f.connectedMap[id]; ok { 221 clientRejectedMeter.Mark(1) 222 log.Debug("Client already connected", "address", freeID, "id", peerIdToString(id)) 223 return false 224 } 225 // Create a clientInfo but do not add it yet 226 var ( 227 posBalance uint64 228 negBalance uint64 229 now = f.clock.Now() 230 ) 231 pb := f.ndb.getOrNewPB(id) 232 posBalance = pb.value 233 234 nb := f.ndb.getOrNewNB(freeID) 235 if nb.logValue != 0 { 236 negBalance = uint64(math.Exp(float64(nb.logValue-f.logOffset(now))/fixedPointMultiplier) * float64(time.Second)) 237 } 238 e := &clientInfo{ 239 pool: f, 240 peer: peer, 241 address: freeID, 242 queueIndex: -1, 243 id: id, 244 connectedAt: now, 245 priority: posBalance != 0, 246 posFactors: f.defaultPosFactors, 247 negFactors: f.defaultNegFactors, 248 balanceMetaInfo: pb.meta, 249 } 250 // If the client is a free client, assign with a low free capacity, 251 // Otherwise assign with the given value(priority client) 252 if !e.priority || capacity == 0 { 253 capacity = f.freeClientCap 254 } 255 e.capacity = capacity 256 257 // Starts a balance tracker 258 e.balanceTracker.init(f.clock, capacity) 259 e.balanceTracker.setBalance(posBalance, negBalance) 260 e.updatePriceFactors() 261 262 // If the number of clients already connected in the clientpool exceeds its 263 // capacity, evict some clients with lowest priority. 264 // 265 // If the priority of the newly added client is lower than the priority of 266 // all connected clients, the client is rejected. 267 newCapacity := f.connectedCap + capacity 268 newCount := f.connectedQueue.Size() + 1 269 if newCapacity > f.capLimit || newCount > f.connLimit { 270 var ( 271 kickList []*clientInfo 272 kickPriority int64 273 ) 274 f.connectedQueue.MultiPop(func(data interface{}, priority int64) bool { 275 c := data.(*clientInfo) 276 kickList = append(kickList, c) 277 kickPriority = priority 278 newCapacity -= c.capacity 279 newCount-- 280 return newCapacity > f.capLimit || newCount > f.connLimit 281 }) 282 bias := connectedBias 283 if f.disableBias { 284 bias = 0 285 } 286 if newCapacity > f.capLimit || newCount > f.connLimit || (e.balanceTracker.estimatedPriority(now+mclock.AbsTime(bias), false)-kickPriority) > 0 { 287 for _, c := range kickList { 288 f.connectedQueue.Push(c) 289 } 290 clientRejectedMeter.Mark(1) 291 log.Debug("Client rejected", "address", freeID, "id", peerIdToString(id)) 292 return false 293 } 294 // accept new client, drop old ones 295 for _, c := range kickList { 296 f.dropClient(c, now, true) 297 } 298 } 299 300 // Register new client to connection queue. 301 f.connectedMap[id] = e 302 f.connectedQueue.Push(e) 303 f.connectedCap += e.capacity 304 305 // If the current client is a paid client, monitor the status of client, 306 // downgrade it to normal client if positive balance is used up. 307 if e.priority { 308 f.priorityConnected += capacity 309 e.balanceTracker.addCallback(balanceCallbackZero, 0, func() { f.balanceExhausted(id) }) 310 } 311 // If the capacity of client is not the default value(free capacity), notify 312 // it to update capacity. 313 if e.capacity != f.freeClientCap { 314 e.peer.updateCapacity(e.capacity) 315 } 316 totalConnectedGauge.Update(int64(f.connectedCap)) 317 clientConnectedMeter.Mark(1) 318 log.Debug("Client accepted", "address", freeID) 319 return true 320 } 321 322 // disconnect should be called when a connection is terminated. If the disconnection 323 // was initiated by the pool itself using disconnectFn then calling disconnect is 324 // not necessary but permitted. 325 func (f *clientPool) disconnect(p clientPoolPeer) { 326 f.lock.Lock() 327 defer f.lock.Unlock() 328 329 // Short circuit if client pool is already closed. 330 if f.closed { 331 return 332 } 333 // Short circuit if the peer hasn't been registered. 334 e := f.connectedMap[p.ID()] 335 if e == nil { 336 log.Debug("Client not connected", "address", p.freeClientId(), "id", peerIdToString(p.ID())) 337 return 338 } 339 f.dropClient(e, f.clock.Now(), false) 340 } 341 342 // forClients iterates through a list of clients, calling the callback for each one. 343 // If a client is not connected then clientInfo is nil. If the specified list is empty 344 // then the callback is called for all connected clients. 345 func (f *clientPool) forClients(ids []enode.ID, callback func(*clientInfo, enode.ID) error) error { 346 f.lock.Lock() 347 defer f.lock.Unlock() 348 349 if len(ids) > 0 { 350 for _, id := range ids { 351 if err := callback(f.connectedMap[id], id); err != nil { 352 return err 353 } 354 } 355 } else { 356 for _, c := range f.connectedMap { 357 if err := callback(c, c.id); err != nil { 358 return err 359 } 360 } 361 } 362 return nil 363 } 364 365 // setDefaultFactors sets the default price factors applied to subsequently connected clients 366 func (f *clientPool) setDefaultFactors(posFactors, negFactors priceFactors) { 367 f.lock.Lock() 368 defer f.lock.Unlock() 369 370 f.defaultPosFactors = posFactors 371 f.defaultNegFactors = negFactors 372 } 373 374 // dropClient removes a client from the connected queue and finalizes its balance. 375 // If kick is true then it also initiates the disconnection. 376 func (f *clientPool) dropClient(e *clientInfo, now mclock.AbsTime, kick bool) { 377 if _, ok := f.connectedMap[e.id]; !ok { 378 return 379 } 380 f.finalizeBalance(e, now) 381 f.connectedQueue.Remove(e.queueIndex) 382 delete(f.connectedMap, e.id) 383 f.connectedCap -= e.capacity 384 if e.priority { 385 f.priorityConnected -= e.capacity 386 } 387 totalConnectedGauge.Update(int64(f.connectedCap)) 388 if kick { 389 clientKickedMeter.Mark(1) 390 log.Debug("Client kicked out", "address", e.address) 391 f.removePeer(e.id) 392 } else { 393 clientDisconnectedMeter.Mark(1) 394 log.Debug("Client disconnected", "address", e.address) 395 } 396 } 397 398 // capacityInfo returns the total capacity allowance, the total capacity of connected 399 // clients and the total capacity of connected and prioritized clients 400 func (f *clientPool) capacityInfo() (uint64, uint64, uint64) { 401 f.lock.Lock() 402 defer f.lock.Unlock() 403 404 return f.capLimit, f.connectedCap, f.priorityConnected 405 } 406 407 // finalizeBalance stops the balance tracker, retrieves the final balances and 408 // stores them in posBalanceQueue and negBalanceQueue 409 func (f *clientPool) finalizeBalance(c *clientInfo, now mclock.AbsTime) { 410 c.balanceTracker.stop(now) 411 pos, neg := c.balanceTracker.getBalance(now) 412 413 pb, nb := f.ndb.getOrNewPB(c.id), f.ndb.getOrNewNB(c.address) 414 pb.value = pos 415 f.ndb.setPB(c.id, pb) 416 417 neg /= uint64(time.Second) // Convert the expanse to second level. 418 if neg > 1 { 419 nb.logValue = int64(math.Log(float64(neg))*fixedPointMultiplier) + f.logOffset(now) 420 f.ndb.setNB(c.address, nb) 421 } else { 422 f.ndb.delNB(c.address) // Negative balance is small enough, drop it directly. 423 } 424 } 425 426 // balanceExhausted callback is called by balanceTracker when positive balance is exhausted. 427 // It revokes priority status and also reduces the client capacity if necessary. 428 func (f *clientPool) balanceExhausted(id enode.ID) { 429 f.lock.Lock() 430 defer f.lock.Unlock() 431 432 c := f.connectedMap[id] 433 if c == nil || !c.priority { 434 return 435 } 436 if c.priority { 437 f.priorityConnected -= c.capacity 438 } 439 c.priority = false 440 if c.capacity != f.freeClientCap { 441 f.connectedCap += f.freeClientCap - c.capacity 442 totalConnectedGauge.Update(int64(f.connectedCap)) 443 c.capacity = f.freeClientCap 444 c.balanceTracker.setCapacity(c.capacity) 445 c.peer.updateCapacity(c.capacity) 446 } 447 pb := f.ndb.getOrNewPB(id) 448 pb.value = 0 449 f.ndb.setPB(id, pb) 450 } 451 452 // setConnLimit sets the maximum number and total capacity of connected clients, 453 // dropping some of them if necessary. 454 func (f *clientPool) setLimits(totalConn int, totalCap uint64) { 455 f.lock.Lock() 456 defer f.lock.Unlock() 457 458 f.connLimit = totalConn 459 f.capLimit = totalCap 460 if f.connectedCap > f.capLimit || f.connectedQueue.Size() > f.connLimit { 461 f.connectedQueue.MultiPop(func(data interface{}, priority int64) bool { 462 f.dropClient(data.(*clientInfo), mclock.Now(), true) 463 return f.connectedCap > f.capLimit || f.connectedQueue.Size() > f.connLimit 464 }) 465 } 466 } 467 468 // setCapacity sets the assigned capacity of a connected client 469 func (f *clientPool) setCapacity(c *clientInfo, capacity uint64) error { 470 if f.connectedMap[c.id] != c { 471 return fmt.Errorf("client %064x is not connected", c.id[:]) 472 } 473 if c.capacity == capacity { 474 return nil 475 } 476 if !c.priority { 477 return errNoPriority 478 } 479 oldCapacity := c.capacity 480 c.capacity = capacity 481 f.connectedCap += capacity - oldCapacity 482 c.balanceTracker.setCapacity(capacity) 483 f.connectedQueue.Update(c.queueIndex) 484 if f.connectedCap > f.capLimit { 485 var kickList []*clientInfo 486 kick := true 487 f.connectedQueue.MultiPop(func(data interface{}, priority int64) bool { 488 client := data.(*clientInfo) 489 kickList = append(kickList, client) 490 f.connectedCap -= client.capacity 491 if client == c { 492 kick = false 493 } 494 return kick && (f.connectedCap > f.capLimit) 495 }) 496 if kick { 497 now := mclock.Now() 498 for _, c := range kickList { 499 f.dropClient(c, now, true) 500 } 501 } else { 502 c.capacity = oldCapacity 503 c.balanceTracker.setCapacity(oldCapacity) 504 for _, c := range kickList { 505 f.connectedCap += c.capacity 506 f.connectedQueue.Push(c) 507 } 508 return errNoPriority 509 } 510 } 511 totalConnectedGauge.Update(int64(f.connectedCap)) 512 f.priorityConnected += capacity - oldCapacity 513 c.updatePriceFactors() 514 c.peer.updateCapacity(c.capacity) 515 return nil 516 } 517 518 // requestCost feeds request cost after serving a request from the given peer. 519 func (f *clientPool) requestCost(p *clientPeer, cost uint64) { 520 f.lock.Lock() 521 defer f.lock.Unlock() 522 523 info, exist := f.connectedMap[p.ID()] 524 if !exist || f.closed { 525 return 526 } 527 info.balanceTracker.requestCost(cost) 528 } 529 530 // logOffset calculates the time-dependent offset for the logarithmic 531 // representation of negative balance 532 // 533 // From another point of view, the result returned by the function represents 534 // the total time that the clientpool is cumulatively running(total_hours/multiplier). 535 func (f *clientPool) logOffset(now mclock.AbsTime) int64 { 536 // Note: fixedPointMultiplier acts as a multiplier here; the reason for dividing the divisor 537 // is to avoid int64 overflow. We assume that int64(negBalanceExpTC) >> fixedPointMultiplier. 538 cumulativeTime := int64((time.Duration(now - f.startTime)) / (negBalanceExpTC / fixedPointMultiplier)) 539 return f.cumulativeTime + cumulativeTime 540 } 541 542 // setClientPriceFactors sets the pricing factors for an individual connected client 543 func (c *clientInfo) updatePriceFactors() { 544 c.balanceTracker.setFactors(true, c.negFactors.timeFactor+float64(c.capacity)*c.negFactors.capacityFactor/1000000, c.negFactors.requestFactor) 545 c.balanceTracker.setFactors(false, c.posFactors.timeFactor+float64(c.capacity)*c.posFactors.capacityFactor/1000000, c.posFactors.requestFactor) 546 } 547 548 // getPosBalance retrieves a single positive balance entry from cache or the database 549 func (f *clientPool) getPosBalance(id enode.ID) posBalance { 550 f.lock.Lock() 551 defer f.lock.Unlock() 552 553 return f.ndb.getOrNewPB(id) 554 } 555 556 // addBalance updates the balance of a client (either overwrites it or adds to it). 557 // It also updates the balance meta info string. 558 func (f *clientPool) addBalance(id enode.ID, amount int64, meta string) (uint64, uint64, error) { 559 f.lock.Lock() 560 defer f.lock.Unlock() 561 562 pb := f.ndb.getOrNewPB(id) 563 var negBalance uint64 564 c := f.connectedMap[id] 565 if c != nil { 566 pb.value, negBalance = c.balanceTracker.getBalance(f.clock.Now()) 567 } 568 oldBalance := pb.value 569 if amount > 0 { 570 if amount > maxBalance || pb.value > maxBalance-uint64(amount) { 571 return oldBalance, oldBalance, errBalanceOverflow 572 } 573 pb.value += uint64(amount) 574 } else { 575 if uint64(-amount) > pb.value { 576 pb.value = 0 577 } else { 578 pb.value -= uint64(-amount) 579 } 580 } 581 pb.meta = meta 582 f.ndb.setPB(id, pb) 583 if c != nil { 584 c.balanceTracker.setBalance(pb.value, negBalance) 585 if !c.priority && pb.value > 0 { 586 // The capacity should be adjusted based on the requirement, 587 // but we have no idea about the new capacity, need a second 588 // call to udpate it. 589 c.priority = true 590 f.priorityConnected += c.capacity 591 c.balanceTracker.addCallback(balanceCallbackZero, 0, func() { f.balanceExhausted(id) }) 592 } 593 // if balance is set to zero then reverting to non-priority status 594 // is handled by the balanceExhausted callback 595 c.balanceMetaInfo = meta 596 } 597 return oldBalance, pb.value, nil 598 } 599 600 // posBalance represents a recently accessed positive balance entry 601 type posBalance struct { 602 value uint64 603 meta string 604 } 605 606 // EncodeRLP implements rlp.Encoder 607 func (e *posBalance) EncodeRLP(w io.Writer) error { 608 return rlp.Encode(w, []interface{}{e.value, e.meta}) 609 } 610 611 // DecodeRLP implements rlp.Decoder 612 func (e *posBalance) DecodeRLP(s *rlp.Stream) error { 613 var entry struct { 614 Value uint64 615 Meta string 616 } 617 if err := s.Decode(&entry); err != nil { 618 return err 619 } 620 e.value = entry.Value 621 e.meta = entry.Meta 622 return nil 623 } 624 625 // negBalance represents a negative balance entry of a disconnected client 626 type negBalance struct{ logValue int64 } 627 628 // EncodeRLP implements rlp.Encoder 629 func (e *negBalance) EncodeRLP(w io.Writer) error { 630 return rlp.Encode(w, []interface{}{uint64(e.logValue)}) 631 } 632 633 // DecodeRLP implements rlp.Decoder 634 func (e *negBalance) DecodeRLP(s *rlp.Stream) error { 635 var entry struct { 636 LogValue uint64 637 } 638 if err := s.Decode(&entry); err != nil { 639 return err 640 } 641 e.logValue = int64(entry.LogValue) 642 return nil 643 } 644 645 const ( 646 // nodeDBVersion is the version identifier of the node data in db 647 // 648 // Changelog: 649 // * Replace `lastTotal` with `meta` in positive balance: version 0=>1 650 nodeDBVersion = 1 651 652 // dbCleanupCycle is the cycle of db for useless data cleanup 653 dbCleanupCycle = time.Hour 654 ) 655 656 var ( 657 positiveBalancePrefix = []byte("pb:") // dbVersion(uint16 big endian) + positiveBalancePrefix + id -> balance 658 negativeBalancePrefix = []byte("nb:") // dbVersion(uint16 big endian) + negativeBalancePrefix + ip -> balance 659 cumulativeRunningTimeKey = []byte("cumulativeTime:") // dbVersion(uint16 big endian) + cumulativeRunningTimeKey -> cumulativeTime 660 ) 661 662 type nodeDB struct { 663 db ethdb.Database 664 pcache *lru.Cache 665 ncache *lru.Cache 666 auxbuf []byte // 37-byte auxiliary buffer for key encoding 667 verbuf [2]byte // 2-byte auxiliary buffer for db version 668 nbEvictCallBack func(mclock.AbsTime, negBalance) bool // Callback to determine whether the negative balance can be evicted. 669 clock mclock.Clock 670 closeCh chan struct{} 671 cleanupHook func() // Test hook used for testing 672 } 673 674 func newNodeDB(db ethdb.Database, clock mclock.Clock) *nodeDB { 675 pcache, _ := lru.New(posBalanceCacheLimit) 676 ncache, _ := lru.New(negBalanceCacheLimit) 677 ndb := &nodeDB{ 678 db: db, 679 pcache: pcache, 680 ncache: ncache, 681 auxbuf: make([]byte, 37), 682 clock: clock, 683 closeCh: make(chan struct{}), 684 } 685 binary.BigEndian.PutUint16(ndb.verbuf[:], uint16(nodeDBVersion)) 686 go ndb.expirer() 687 return ndb 688 } 689 690 func (db *nodeDB) close() { 691 close(db.closeCh) 692 } 693 694 func (db *nodeDB) getPrefix(neg bool) []byte { 695 prefix := positiveBalancePrefix 696 if neg { 697 prefix = negativeBalancePrefix 698 } 699 return append(db.verbuf[:], prefix...) 700 } 701 702 func (db *nodeDB) key(id []byte, neg bool) []byte { 703 prefix := positiveBalancePrefix 704 if neg { 705 prefix = negativeBalancePrefix 706 } 707 if len(prefix)+len(db.verbuf)+len(id) > len(db.auxbuf) { 708 db.auxbuf = append(db.auxbuf, make([]byte, len(prefix)+len(db.verbuf)+len(id)-len(db.auxbuf))...) 709 } 710 copy(db.auxbuf[:len(db.verbuf)], db.verbuf[:]) 711 copy(db.auxbuf[len(db.verbuf):len(db.verbuf)+len(prefix)], prefix) 712 copy(db.auxbuf[len(prefix)+len(db.verbuf):len(prefix)+len(db.verbuf)+len(id)], id) 713 return db.auxbuf[:len(prefix)+len(db.verbuf)+len(id)] 714 } 715 716 func (db *nodeDB) getCumulativeTime() int64 { 717 blob, err := db.db.Get(append(cumulativeRunningTimeKey, db.verbuf[:]...)) 718 if err != nil || len(blob) == 0 { 719 return 0 720 } 721 return int64(binary.BigEndian.Uint64(blob)) 722 } 723 724 func (db *nodeDB) setCumulativeTime(v int64) { 725 binary.BigEndian.PutUint64(db.auxbuf[:8], uint64(v)) 726 db.db.Put(append(cumulativeRunningTimeKey, db.verbuf[:]...), db.auxbuf[:8]) 727 } 728 729 func (db *nodeDB) getOrNewPB(id enode.ID) posBalance { 730 key := db.key(id.Bytes(), false) 731 item, exist := db.pcache.Get(string(key)) 732 if exist { 733 return item.(posBalance) 734 } 735 var balance posBalance 736 if enc, err := db.db.Get(key); err == nil { 737 if err := rlp.DecodeBytes(enc, &balance); err != nil { 738 log.Error("Failed to decode positive balance", "err", err) 739 } 740 } 741 db.pcache.Add(string(key), balance) 742 return balance 743 } 744 745 func (db *nodeDB) setPB(id enode.ID, b posBalance) { 746 if b.value == 0 && len(b.meta) == 0 { 747 db.delPB(id) 748 return 749 } 750 key := db.key(id.Bytes(), false) 751 enc, err := rlp.EncodeToBytes(&(b)) 752 if err != nil { 753 log.Error("Failed to encode positive balance", "err", err) 754 return 755 } 756 db.db.Put(key, enc) 757 db.pcache.Add(string(key), b) 758 } 759 760 func (db *nodeDB) delPB(id enode.ID) { 761 key := db.key(id.Bytes(), false) 762 db.db.Delete(key) 763 db.pcache.Remove(string(key)) 764 } 765 766 // getPosBalanceIDs returns a lexicographically ordered list of IDs of accounts 767 // with a positive balance 768 func (db *nodeDB) getPosBalanceIDs(start, stop enode.ID, maxCount int) (result []enode.ID) { 769 if maxCount <= 0 { 770 return 771 } 772 prefix := db.getPrefix(false) 773 it := db.db.NewIterator(prefix, start.Bytes()) 774 defer it.Release() 775 for i := len(stop[:]) - 1; i >= 0; i-- { 776 stop[i]-- 777 if stop[i] != 255 { 778 break 779 } 780 } 781 stopKey := db.key(stop.Bytes(), false) 782 keyLen := len(stopKey) 783 784 for it.Next() { 785 var id enode.ID 786 if len(it.Key()) != keyLen || bytes.Compare(it.Key(), stopKey) == 1 { 787 return 788 } 789 copy(id[:], it.Key()[keyLen-len(id):]) 790 result = append(result, id) 791 if len(result) == maxCount { 792 return 793 } 794 } 795 return 796 } 797 798 func (db *nodeDB) getOrNewNB(id string) negBalance { 799 key := db.key([]byte(id), true) 800 item, exist := db.ncache.Get(string(key)) 801 if exist { 802 return item.(negBalance) 803 } 804 var balance negBalance 805 if enc, err := db.db.Get(key); err == nil { 806 if err := rlp.DecodeBytes(enc, &balance); err != nil { 807 log.Error("Failed to decode negative balance", "err", err) 808 } 809 } 810 db.ncache.Add(string(key), balance) 811 return balance 812 } 813 814 func (db *nodeDB) setNB(id string, b negBalance) { 815 key := db.key([]byte(id), true) 816 enc, err := rlp.EncodeToBytes(&(b)) 817 if err != nil { 818 log.Error("Failed to encode negative balance", "err", err) 819 return 820 } 821 db.db.Put(key, enc) 822 db.ncache.Add(string(key), b) 823 } 824 825 func (db *nodeDB) delNB(id string) { 826 key := db.key([]byte(id), true) 827 db.db.Delete(key) 828 db.ncache.Remove(string(key)) 829 } 830 831 func (db *nodeDB) expirer() { 832 for { 833 select { 834 case <-db.clock.After(dbCleanupCycle): 835 db.expireNodes() 836 case <-db.closeCh: 837 return 838 } 839 } 840 } 841 842 // expireNodes iterates the whole node db and checks whether the negative balance 843 // entry can deleted. 844 // 845 // The rationale behind this is: server doesn't need to keep the negative balance 846 // records if they are low enough. 847 func (db *nodeDB) expireNodes() { 848 var ( 849 visited int 850 deleted int 851 start = time.Now() 852 prefix = db.getPrefix(true) 853 ) 854 iter := db.db.NewIterator(prefix, nil) 855 for iter.Next() { 856 visited += 1 857 var balance negBalance 858 if err := rlp.DecodeBytes(iter.Value(), &balance); err != nil { 859 log.Error("Failed to decode negative balance", "err", err) 860 continue 861 } 862 if db.nbEvictCallBack != nil && db.nbEvictCallBack(db.clock.Now(), balance) { 863 deleted += 1 864 db.db.Delete(iter.Key()) 865 } 866 } 867 // Invoke testing hook if it's not nil. 868 if db.cleanupHook != nil { 869 db.cleanupHook() 870 } 871 log.Debug("Expire nodes", "visited", visited, "deleted", deleted, "elapsed", common.PrettyDuration(time.Since(start))) 872 }