github.com/vipernet-xyz/tendermint-core@v0.32.0/blockchain/v0/pool.go (about) 1 package v0 2 3 import ( 4 "errors" 5 "fmt" 6 "math" 7 "sync" 8 "sync/atomic" 9 "time" 10 11 flow "github.com/tendermint/tendermint/libs/flowrate" 12 "github.com/tendermint/tendermint/libs/log" 13 "github.com/tendermint/tendermint/libs/service" 14 15 "github.com/tendermint/tendermint/p2p" 16 "github.com/tendermint/tendermint/types" 17 ) 18 19 /* 20 eg, L = latency = 0.1s 21 P = num peers = 10 22 FN = num full nodes 23 BS = 1kB block size 24 CB = 1 Mbit/s = 128 kB/s 25 CB/P = 12.8 kB 26 B/S = CB/P/BS = 12.8 blocks/s 27 28 12.8 * 0.1 = 1.28 blocks on conn 29 */ 30 31 const ( 32 requestIntervalMS = 2 33 maxTotalRequesters = 600 34 maxPendingRequests = maxTotalRequesters 35 maxPendingRequestsPerPeer = 20 36 37 // Minimum recv rate to ensure we're receiving blocks from a peer fast 38 // enough. If a peer is not sending us data at at least that rate, we 39 // consider them to have timedout and we disconnect. 40 // 41 // Assuming a DSL connection (not a good choice) 128 Kbps (upload) ~ 15 KB/s, 42 // sending data across atlantic ~ 7.5 KB/s. 43 minRecvRate = 7680 44 45 // Maximum difference between current and new block's height. 46 maxDiffBetweenCurrentAndReceivedBlockHeight = 100 47 ) 48 49 var peerTimeout = 15 * time.Second // not const so we can override with tests 50 51 /* 52 Peers self report their heights when we join the block pool. 53 Starting from our latest pool.height, we request blocks 54 in sequence from peers that reported higher heights than ours. 55 Every so often we ask peers what height they're on so we can keep going. 56 57 Requests are continuously made for blocks of higher heights until 58 the limit is reached. If most of the requests have no available peers, and we 59 are not at peer limits, we can probably switch to consensus reactor 60 */ 61 62 // BlockPool keeps track of the fast sync peers, block requests and block responses. 63 type BlockPool struct { 64 service.BaseService 65 startTime time.Time 66 67 mtx sync.Mutex 68 // block requests 69 requesters map[int64]*bpRequester 70 height int64 // the lowest key in requesters. 71 // peers 72 peers map[p2p.ID]*bpPeer 73 maxPeerHeight int64 // the biggest reported height 74 75 // atomic 76 numPending int32 // number of requests pending assignment or block response 77 78 requestsCh chan<- BlockRequest 79 errorsCh chan<- peerError 80 } 81 82 // NewBlockPool returns a new BlockPool with the height equal to start. Block 83 // requests and errors will be sent to requestsCh and errorsCh accordingly. 84 func NewBlockPool(start int64, requestsCh chan<- BlockRequest, errorsCh chan<- peerError) *BlockPool { 85 bp := &BlockPool{ 86 peers: make(map[p2p.ID]*bpPeer), 87 88 requesters: make(map[int64]*bpRequester), 89 height: start, 90 numPending: 0, 91 92 requestsCh: requestsCh, 93 errorsCh: errorsCh, 94 } 95 bp.BaseService = *service.NewBaseService(nil, "BlockPool", bp) 96 return bp 97 } 98 99 // OnStart implements service.Service by spawning requesters routine and recording 100 // pool's start time. 101 func (pool *BlockPool) OnStart() error { 102 go pool.makeRequestersRoutine() 103 pool.startTime = time.Now() 104 return nil 105 } 106 107 // spawns requesters as needed 108 func (pool *BlockPool) makeRequestersRoutine() { 109 for { 110 if !pool.IsRunning() { 111 break 112 } 113 114 _, numPending, lenRequesters := pool.GetStatus() 115 switch { 116 case numPending >= maxPendingRequests: 117 // sleep for a bit. 118 time.Sleep(requestIntervalMS * time.Millisecond) 119 // check for timed out peers 120 pool.removeTimedoutPeers() 121 case lenRequesters >= maxTotalRequesters: 122 // sleep for a bit. 123 time.Sleep(requestIntervalMS * time.Millisecond) 124 // check for timed out peers 125 pool.removeTimedoutPeers() 126 default: 127 // request for more blocks. 128 pool.makeNextRequester() 129 } 130 } 131 } 132 133 func (pool *BlockPool) removeTimedoutPeers() { 134 pool.mtx.Lock() 135 defer pool.mtx.Unlock() 136 137 for _, peer := range pool.peers { 138 if !peer.didTimeout && peer.numPending > 0 { 139 curRate := peer.recvMonitor.Status().CurRate 140 // curRate can be 0 on start 141 if curRate != 0 && curRate < minRecvRate { 142 err := errors.New("peer is not sending us data fast enough") 143 pool.sendError(err, peer.id) 144 pool.Logger.Error("SendTimeout", "peer", peer.id, 145 "reason", err, 146 "curRate", fmt.Sprintf("%d KB/s", curRate/1024), 147 "minRate", fmt.Sprintf("%d KB/s", minRecvRate/1024)) 148 peer.didTimeout = true 149 } 150 } 151 if peer.didTimeout { 152 pool.removePeer(peer.id) 153 } 154 } 155 } 156 157 // GetStatus returns pool's height, numPending requests and the number of 158 // requesters. 159 func (pool *BlockPool) GetStatus() (height int64, numPending int32, lenRequesters int) { 160 pool.mtx.Lock() 161 defer pool.mtx.Unlock() 162 163 return pool.height, atomic.LoadInt32(&pool.numPending), len(pool.requesters) 164 } 165 166 // IsCaughtUp returns true if this node is caught up, false - otherwise. 167 // TODO: relax conditions, prevent abuse. 168 func (pool *BlockPool) IsCaughtUp() bool { 169 pool.mtx.Lock() 170 defer pool.mtx.Unlock() 171 172 // Need at least 1 peer to be considered caught up. 173 if len(pool.peers) == 0 { 174 pool.Logger.Debug("Blockpool has no peers") 175 return false 176 } 177 178 // Some conditions to determine if we're caught up. 179 // Ensures we've either received a block or waited some amount of time, 180 // and that we're synced to the highest known height. 181 // Note we use maxPeerHeight - 1 because to sync block H requires block H+1 182 // to verify the LastCommit. 183 receivedBlockOrTimedOut := pool.height > 0 || time.Since(pool.startTime) > 5*time.Second 184 ourChainIsLongestAmongPeers := pool.maxPeerHeight == 0 || pool.height >= (pool.maxPeerHeight-1) 185 isCaughtUp := receivedBlockOrTimedOut && ourChainIsLongestAmongPeers 186 return isCaughtUp 187 } 188 189 // PeekTwoBlocks returns blocks at pool.height and pool.height+1. 190 // We need to see the second block's Commit to validate the first block. 191 // So we peek two blocks at a time. 192 // The caller will verify the commit. 193 func (pool *BlockPool) PeekTwoBlocks() (first *types.Block, second *types.Block) { 194 pool.mtx.Lock() 195 defer pool.mtx.Unlock() 196 197 if r := pool.requesters[pool.height]; r != nil { 198 first = r.getBlock() 199 } 200 if r := pool.requesters[pool.height+1]; r != nil { 201 second = r.getBlock() 202 } 203 return 204 } 205 206 // PopRequest pops the first block at pool.height. 207 // It must have been validated by 'second'.Commit from PeekTwoBlocks(). 208 func (pool *BlockPool) PopRequest() { 209 pool.mtx.Lock() 210 defer pool.mtx.Unlock() 211 212 if r := pool.requesters[pool.height]; r != nil { 213 /* The block can disappear at any time, due to removePeer(). 214 if r := pool.requesters[pool.height]; r == nil || r.block == nil { 215 PanicSanity("PopRequest() requires a valid block") 216 } 217 */ 218 r.Stop() 219 delete(pool.requesters, pool.height) 220 pool.height++ 221 } else { 222 panic(fmt.Sprintf("Expected requester to pop, got nothing at height %v", pool.height)) 223 } 224 } 225 226 // RedoRequest invalidates the block at pool.height, 227 // Remove the peer and redo request from others. 228 // Returns the ID of the removed peer. 229 func (pool *BlockPool) RedoRequest(height int64) p2p.ID { 230 pool.mtx.Lock() 231 defer pool.mtx.Unlock() 232 233 request := pool.requesters[height] 234 peerID := request.getPeerID() 235 if peerID != p2p.ID("") { 236 // RemovePeer will redo all requesters associated with this peer. 237 pool.removePeer(peerID) 238 } 239 return peerID 240 } 241 242 // AddBlock validates that the block comes from the peer it was expected from and calls the requester to store it. 243 // TODO: ensure that blocks come in order for each peer. 244 func (pool *BlockPool) AddBlock(peerID p2p.ID, block *types.Block, blockSize int) { 245 pool.mtx.Lock() 246 defer pool.mtx.Unlock() 247 248 requester := pool.requesters[block.Height] 249 if requester == nil { 250 pool.Logger.Info( 251 "peer sent us a block we didn't expect", 252 "peer", 253 peerID, 254 "curHeight", 255 pool.height, 256 "blockHeight", 257 block.Height) 258 diff := pool.height - block.Height 259 if diff < 0 { 260 diff *= -1 261 } 262 if diff > maxDiffBetweenCurrentAndReceivedBlockHeight { 263 pool.sendError(errors.New("peer sent us a block we didn't expect with a height too far ahead/behind"), peerID) 264 } 265 return 266 } 267 268 if requester.setBlock(block, peerID) { 269 atomic.AddInt32(&pool.numPending, -1) 270 peer := pool.peers[peerID] 271 if peer != nil { 272 peer.decrPending(blockSize) 273 } 274 } else { 275 pool.Logger.Info("invalid peer", "peer", peerID, "blockHeight", block.Height) 276 pool.sendError(errors.New("invalid peer"), peerID) 277 } 278 } 279 280 // MaxPeerHeight returns the highest reported height. 281 func (pool *BlockPool) MaxPeerHeight() int64 { 282 pool.mtx.Lock() 283 defer pool.mtx.Unlock() 284 return pool.maxPeerHeight 285 } 286 287 // SetPeerRange sets the peer's alleged blockchain base and height. 288 func (pool *BlockPool) SetPeerRange(peerID p2p.ID, base int64, height int64) { 289 pool.mtx.Lock() 290 defer pool.mtx.Unlock() 291 292 peer := pool.peers[peerID] 293 if peer != nil { 294 peer.base = base 295 peer.height = height 296 } else { 297 peer = newBPPeer(pool, peerID, base, height) 298 peer.setLogger(pool.Logger.With("peer", peerID)) 299 pool.peers[peerID] = peer 300 } 301 302 if height > pool.maxPeerHeight { 303 pool.maxPeerHeight = height 304 } 305 } 306 307 // RemovePeer removes the peer with peerID from the pool. If there's no peer 308 // with peerID, function is a no-op. 309 func (pool *BlockPool) RemovePeer(peerID p2p.ID) { 310 pool.mtx.Lock() 311 defer pool.mtx.Unlock() 312 313 pool.removePeer(peerID) 314 } 315 316 func (pool *BlockPool) removePeer(peerID p2p.ID) { 317 for _, requester := range pool.requesters { 318 if requester.getPeerID() == peerID { 319 requester.redo(peerID) 320 } 321 } 322 323 peer, ok := pool.peers[peerID] 324 if ok { 325 if peer.timeout != nil { 326 peer.timeout.Stop() 327 } 328 329 delete(pool.peers, peerID) 330 331 // Find a new peer with the biggest height and update maxPeerHeight if the 332 // peer's height was the biggest. 333 if peer.height == pool.maxPeerHeight { 334 pool.updateMaxPeerHeight() 335 } 336 } 337 } 338 339 // If no peers are left, maxPeerHeight is set to 0. 340 func (pool *BlockPool) updateMaxPeerHeight() { 341 var max int64 342 for _, peer := range pool.peers { 343 if peer.height > max { 344 max = peer.height 345 } 346 } 347 pool.maxPeerHeight = max 348 } 349 350 // Pick an available peer with the given height available. 351 // If no peers are available, returns nil. 352 func (pool *BlockPool) pickIncrAvailablePeer(height int64) *bpPeer { 353 pool.mtx.Lock() 354 defer pool.mtx.Unlock() 355 356 for _, peer := range pool.peers { 357 if peer.didTimeout { 358 pool.removePeer(peer.id) 359 continue 360 } 361 if peer.numPending >= maxPendingRequestsPerPeer { 362 continue 363 } 364 if height < peer.base || height > peer.height { 365 continue 366 } 367 peer.incrPending() 368 return peer 369 } 370 return nil 371 } 372 373 func (pool *BlockPool) makeNextRequester() { 374 pool.mtx.Lock() 375 defer pool.mtx.Unlock() 376 377 nextHeight := pool.height + pool.requestersLen() 378 if nextHeight > pool.maxPeerHeight { 379 return 380 } 381 382 request := newBPRequester(pool, nextHeight) 383 384 pool.requesters[nextHeight] = request 385 atomic.AddInt32(&pool.numPending, 1) 386 387 err := request.Start() 388 if err != nil { 389 request.Logger.Error("Error starting request", "err", err) 390 } 391 } 392 393 func (pool *BlockPool) requestersLen() int64 { 394 return int64(len(pool.requesters)) 395 } 396 397 func (pool *BlockPool) sendRequest(height int64, peerID p2p.ID) { 398 if !pool.IsRunning() { 399 return 400 } 401 pool.requestsCh <- BlockRequest{height, peerID} 402 } 403 404 func (pool *BlockPool) sendError(err error, peerID p2p.ID) { 405 if !pool.IsRunning() { 406 return 407 } 408 pool.errorsCh <- peerError{err, peerID} 409 } 410 411 // for debugging purposes 412 //nolint:unused 413 func (pool *BlockPool) debug() string { 414 pool.mtx.Lock() 415 defer pool.mtx.Unlock() 416 417 str := "" 418 nextHeight := pool.height + pool.requestersLen() 419 for h := pool.height; h < nextHeight; h++ { 420 if pool.requesters[h] == nil { 421 str += fmt.Sprintf("H(%v):X ", h) 422 } else { 423 str += fmt.Sprintf("H(%v):", h) 424 str += fmt.Sprintf("B?(%v) ", pool.requesters[h].block != nil) 425 } 426 } 427 return str 428 } 429 430 //------------------------------------- 431 432 type bpPeer struct { 433 didTimeout bool 434 numPending int32 435 height int64 436 base int64 437 pool *BlockPool 438 id p2p.ID 439 recvMonitor *flow.Monitor 440 441 timeout *time.Timer 442 443 logger log.Logger 444 } 445 446 func newBPPeer(pool *BlockPool, peerID p2p.ID, base int64, height int64) *bpPeer { 447 peer := &bpPeer{ 448 pool: pool, 449 id: peerID, 450 base: base, 451 height: height, 452 numPending: 0, 453 logger: log.NewNopLogger(), 454 } 455 return peer 456 } 457 458 func (peer *bpPeer) setLogger(l log.Logger) { 459 peer.logger = l 460 } 461 462 func (peer *bpPeer) resetMonitor() { 463 peer.recvMonitor = flow.New(time.Second, time.Second*40) 464 initialValue := float64(minRecvRate) * math.E 465 peer.recvMonitor.SetREMA(initialValue) 466 } 467 468 func (peer *bpPeer) resetTimeout() { 469 if peer.timeout == nil { 470 peer.timeout = time.AfterFunc(peerTimeout, peer.onTimeout) 471 } else { 472 peer.timeout.Reset(peerTimeout) 473 } 474 } 475 476 func (peer *bpPeer) incrPending() { 477 if peer.numPending == 0 { 478 peer.resetMonitor() 479 peer.resetTimeout() 480 } 481 peer.numPending++ 482 } 483 484 func (peer *bpPeer) decrPending(recvSize int) { 485 peer.numPending-- 486 if peer.numPending == 0 { 487 peer.timeout.Stop() 488 } else { 489 peer.recvMonitor.Update(recvSize) 490 peer.resetTimeout() 491 } 492 } 493 494 func (peer *bpPeer) onTimeout() { 495 peer.pool.mtx.Lock() 496 defer peer.pool.mtx.Unlock() 497 498 err := errors.New("peer did not send us anything") 499 peer.pool.sendError(err, peer.id) 500 peer.logger.Error("SendTimeout", "reason", err, "timeout", peerTimeout) 501 peer.didTimeout = true 502 } 503 504 //------------------------------------- 505 506 type bpRequester struct { 507 service.BaseService 508 pool *BlockPool 509 height int64 510 gotBlockCh chan struct{} 511 redoCh chan p2p.ID //redo may send multitime, add peerId to identify repeat 512 513 mtx sync.Mutex 514 peerID p2p.ID 515 block *types.Block 516 } 517 518 func newBPRequester(pool *BlockPool, height int64) *bpRequester { 519 bpr := &bpRequester{ 520 pool: pool, 521 height: height, 522 gotBlockCh: make(chan struct{}, 1), 523 redoCh: make(chan p2p.ID, 1), 524 525 peerID: "", 526 block: nil, 527 } 528 bpr.BaseService = *service.NewBaseService(nil, "bpRequester", bpr) 529 return bpr 530 } 531 532 func (bpr *bpRequester) OnStart() error { 533 go bpr.requestRoutine() 534 return nil 535 } 536 537 // Returns true if the peer matches and block doesn't already exist. 538 func (bpr *bpRequester) setBlock(block *types.Block, peerID p2p.ID) bool { 539 bpr.mtx.Lock() 540 if bpr.block != nil || bpr.peerID != peerID { 541 bpr.mtx.Unlock() 542 return false 543 } 544 bpr.block = block 545 bpr.mtx.Unlock() 546 547 select { 548 case bpr.gotBlockCh <- struct{}{}: 549 default: 550 } 551 return true 552 } 553 554 func (bpr *bpRequester) getBlock() *types.Block { 555 bpr.mtx.Lock() 556 defer bpr.mtx.Unlock() 557 return bpr.block 558 } 559 560 func (bpr *bpRequester) getPeerID() p2p.ID { 561 bpr.mtx.Lock() 562 defer bpr.mtx.Unlock() 563 return bpr.peerID 564 } 565 566 // This is called from the requestRoutine, upon redo(). 567 func (bpr *bpRequester) reset() { 568 bpr.mtx.Lock() 569 defer bpr.mtx.Unlock() 570 571 if bpr.block != nil { 572 atomic.AddInt32(&bpr.pool.numPending, 1) 573 } 574 575 bpr.peerID = "" 576 bpr.block = nil 577 } 578 579 // Tells bpRequester to pick another peer and try again. 580 // NOTE: Nonblocking, and does nothing if another redo 581 // was already requested. 582 func (bpr *bpRequester) redo(peerID p2p.ID) { 583 select { 584 case bpr.redoCh <- peerID: 585 default: 586 } 587 } 588 589 // Responsible for making more requests as necessary 590 // Returns only when a block is found (e.g. AddBlock() is called) 591 func (bpr *bpRequester) requestRoutine() { 592 OUTER_LOOP: 593 for { 594 // Pick a peer to send request to. 595 var peer *bpPeer 596 PICK_PEER_LOOP: 597 for { 598 if !bpr.IsRunning() || !bpr.pool.IsRunning() { 599 return 600 } 601 peer = bpr.pool.pickIncrAvailablePeer(bpr.height) 602 if peer == nil { 603 //log.Info("No peers available", "height", height) 604 time.Sleep(requestIntervalMS * time.Millisecond) 605 continue PICK_PEER_LOOP 606 } 607 break PICK_PEER_LOOP 608 } 609 bpr.mtx.Lock() 610 bpr.peerID = peer.id 611 bpr.mtx.Unlock() 612 613 // Send request and wait. 614 bpr.pool.sendRequest(bpr.height, peer.id) 615 WAIT_LOOP: 616 for { 617 select { 618 case <-bpr.pool.Quit(): 619 bpr.Stop() 620 return 621 case <-bpr.Quit(): 622 return 623 case peerID := <-bpr.redoCh: 624 if peerID == bpr.peerID { 625 bpr.reset() 626 continue OUTER_LOOP 627 } else { 628 continue WAIT_LOOP 629 } 630 case <-bpr.gotBlockCh: 631 // We got a block! 632 // Continue the for-loop and wait til Quit. 633 continue WAIT_LOOP 634 } 635 } 636 } 637 } 638 639 // BlockRequest stores a block request identified by the block Height and the PeerID responsible for 640 // delivering the block 641 type BlockRequest struct { 642 Height int64 643 PeerID p2p.ID 644 }