github.com/sberex/go-sberex@v1.8.2-0.20181113200658-ed96ac38f7d7/eth/downloader/queue.go (about) 1 // This file is part of the go-sberex library. The go-sberex library is 2 // free software: you can redistribute it and/or modify it under the terms 3 // of the GNU Lesser General Public License as published by the Free 4 // Software Foundation, either version 3 of the License, or (at your option) 5 // any later version. 6 // 7 // The go-sberex library is distributed in the hope that it will be useful, 8 // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser 10 // General Public License <http://www.gnu.org/licenses/> for more details. 11 12 // Contains the block download scheduler to collect download tasks and schedule 13 // them in an ordered, and throttled way. 14 15 package downloader 16 17 import ( 18 "errors" 19 "fmt" 20 "sync" 21 "time" 22 23 "github.com/Sberex/go-sberex/common" 24 "github.com/Sberex/go-sberex/core/types" 25 "github.com/Sberex/go-sberex/log" 26 "github.com/Sberex/go-sberex/metrics" 27 "gopkg.in/karalabe/cookiejar.v2/collections/prque" 28 ) 29 30 var ( 31 blockCacheItems = 8192 // Maximum number of blocks to cache before throttling the download 32 blockCacheMemory = 64 * 1024 * 1024 // Maximum amount of memory to use for block caching 33 blockCacheSizeWeight = 0.1 // Multiplier to approximate the average block size based on past ones 34 ) 35 36 var ( 37 errNoFetchesPending = errors.New("no fetches pending") 38 errStaleDelivery = errors.New("stale delivery") 39 ) 40 41 // fetchRequest is a currently running data retrieval operation. 42 type fetchRequest struct { 43 Peer *peerConnection // Peer to which the request was sent 44 From uint64 // [eth/62] Requested chain element index (used for skeleton fills only) 45 Headers []*types.Header // [eth/62] Requested headers, sorted by request order 46 Time time.Time // Time when the request was made 47 } 48 49 // fetchResult is a struct collecting partial results from data fetchers until 50 // all outstanding pieces complete and the result as a whole can be processed. 51 type fetchResult struct { 52 Pending int // Number of data fetches still pending 53 Hash common.Hash // Hash of the header to prevent recalculating 54 55 Header *types.Header 56 Uncles []*types.Header 57 Transactions types.Transactions 58 Receipts types.Receipts 59 } 60 61 // queue represents hashes that are either need fetching or are being fetched 62 type queue struct { 63 mode SyncMode // Synchronisation mode to decide on the block parts to schedule for fetching 64 65 // Headers are "special", they download in batches, supported by a skeleton chain 66 headerHead common.Hash // [eth/62] Hash of the last queued header to verify order 67 headerTaskPool map[uint64]*types.Header // [eth/62] Pending header retrieval tasks, mapping starting indexes to skeleton headers 68 headerTaskQueue *prque.Prque // [eth/62] Priority queue of the skeleton indexes to fetch the filling headers for 69 headerPeerMiss map[string]map[uint64]struct{} // [eth/62] Set of per-peer header batches known to be unavailable 70 headerPendPool map[string]*fetchRequest // [eth/62] Currently pending header retrieval operations 71 headerResults []*types.Header // [eth/62] Result cache accumulating the completed headers 72 headerProced int // [eth/62] Number of headers already processed from the results 73 headerOffset uint64 // [eth/62] Number of the first header in the result cache 74 headerContCh chan bool // [eth/62] Channel to notify when header download finishes 75 76 // All data retrievals below are based on an already assembles header chain 77 blockTaskPool map[common.Hash]*types.Header // [eth/62] Pending block (body) retrieval tasks, mapping hashes to headers 78 blockTaskQueue *prque.Prque // [eth/62] Priority queue of the headers to fetch the blocks (bodies) for 79 blockPendPool map[string]*fetchRequest // [eth/62] Currently pending block (body) retrieval operations 80 blockDonePool map[common.Hash]struct{} // [eth/62] Set of the completed block (body) fetches 81 82 receiptTaskPool map[common.Hash]*types.Header // [eth/63] Pending receipt retrieval tasks, mapping hashes to headers 83 receiptTaskQueue *prque.Prque // [eth/63] Priority queue of the headers to fetch the receipts for 84 receiptPendPool map[string]*fetchRequest // [eth/63] Currently pending receipt retrieval operations 85 receiptDonePool map[common.Hash]struct{} // [eth/63] Set of the completed receipt fetches 86 87 resultCache []*fetchResult // Downloaded but not yet delivered fetch results 88 resultOffset uint64 // Offset of the first cached fetch result in the block chain 89 resultSize common.StorageSize // Approximate size of a block (exponential moving average) 90 91 lock *sync.Mutex 92 active *sync.Cond 93 closed bool 94 } 95 96 // newQueue creates a new download queue for scheduling block retrieval. 97 func newQueue() *queue { 98 lock := new(sync.Mutex) 99 return &queue{ 100 headerPendPool: make(map[string]*fetchRequest), 101 headerContCh: make(chan bool), 102 blockTaskPool: make(map[common.Hash]*types.Header), 103 blockTaskQueue: prque.New(), 104 blockPendPool: make(map[string]*fetchRequest), 105 blockDonePool: make(map[common.Hash]struct{}), 106 receiptTaskPool: make(map[common.Hash]*types.Header), 107 receiptTaskQueue: prque.New(), 108 receiptPendPool: make(map[string]*fetchRequest), 109 receiptDonePool: make(map[common.Hash]struct{}), 110 resultCache: make([]*fetchResult, blockCacheItems), 111 active: sync.NewCond(lock), 112 lock: lock, 113 } 114 } 115 116 // Reset clears out the queue contents. 117 func (q *queue) Reset() { 118 q.lock.Lock() 119 defer q.lock.Unlock() 120 121 q.closed = false 122 q.mode = FullSync 123 124 q.headerHead = common.Hash{} 125 q.headerPendPool = make(map[string]*fetchRequest) 126 127 q.blockTaskPool = make(map[common.Hash]*types.Header) 128 q.blockTaskQueue.Reset() 129 q.blockPendPool = make(map[string]*fetchRequest) 130 q.blockDonePool = make(map[common.Hash]struct{}) 131 132 q.receiptTaskPool = make(map[common.Hash]*types.Header) 133 q.receiptTaskQueue.Reset() 134 q.receiptPendPool = make(map[string]*fetchRequest) 135 q.receiptDonePool = make(map[common.Hash]struct{}) 136 137 q.resultCache = make([]*fetchResult, blockCacheItems) 138 q.resultOffset = 0 139 } 140 141 // Close marks the end of the sync, unblocking WaitResults. 142 // It may be called even if the queue is already closed. 143 func (q *queue) Close() { 144 q.lock.Lock() 145 q.closed = true 146 q.lock.Unlock() 147 q.active.Broadcast() 148 } 149 150 // PendingHeaders retrieves the number of header requests pending for retrieval. 151 func (q *queue) PendingHeaders() int { 152 q.lock.Lock() 153 defer q.lock.Unlock() 154 155 return q.headerTaskQueue.Size() 156 } 157 158 // PendingBlocks retrieves the number of block (body) requests pending for retrieval. 159 func (q *queue) PendingBlocks() int { 160 q.lock.Lock() 161 defer q.lock.Unlock() 162 163 return q.blockTaskQueue.Size() 164 } 165 166 // PendingReceipts retrieves the number of block receipts pending for retrieval. 167 func (q *queue) PendingReceipts() int { 168 q.lock.Lock() 169 defer q.lock.Unlock() 170 171 return q.receiptTaskQueue.Size() 172 } 173 174 // InFlightHeaders retrieves whether there are header fetch requests currently 175 // in flight. 176 func (q *queue) InFlightHeaders() bool { 177 q.lock.Lock() 178 defer q.lock.Unlock() 179 180 return len(q.headerPendPool) > 0 181 } 182 183 // InFlightBlocks retrieves whether there are block fetch requests currently in 184 // flight. 185 func (q *queue) InFlightBlocks() bool { 186 q.lock.Lock() 187 defer q.lock.Unlock() 188 189 return len(q.blockPendPool) > 0 190 } 191 192 // InFlightReceipts retrieves whether there are receipt fetch requests currently 193 // in flight. 194 func (q *queue) InFlightReceipts() bool { 195 q.lock.Lock() 196 defer q.lock.Unlock() 197 198 return len(q.receiptPendPool) > 0 199 } 200 201 // Idle returns if the queue is fully idle or has some data still inside. 202 func (q *queue) Idle() bool { 203 q.lock.Lock() 204 defer q.lock.Unlock() 205 206 queued := q.blockTaskQueue.Size() + q.receiptTaskQueue.Size() 207 pending := len(q.blockPendPool) + len(q.receiptPendPool) 208 cached := len(q.blockDonePool) + len(q.receiptDonePool) 209 210 return (queued + pending + cached) == 0 211 } 212 213 // ShouldThrottleBlocks checks if the download should be throttled (active block (body) 214 // fetches exceed block cache). 215 func (q *queue) ShouldThrottleBlocks() bool { 216 q.lock.Lock() 217 defer q.lock.Unlock() 218 219 return q.resultSlots(q.blockPendPool, q.blockDonePool) <= 0 220 } 221 222 // ShouldThrottleReceipts checks if the download should be throttled (active receipt 223 // fetches exceed block cache). 224 func (q *queue) ShouldThrottleReceipts() bool { 225 q.lock.Lock() 226 defer q.lock.Unlock() 227 228 return q.resultSlots(q.receiptPendPool, q.receiptDonePool) <= 0 229 } 230 231 // resultSlots calculates the number of results slots available for requests 232 // whilst adhering to both the item and the memory limit too of the results 233 // cache. 234 func (q *queue) resultSlots(pendPool map[string]*fetchRequest, donePool map[common.Hash]struct{}) int { 235 // Calculate the maximum length capped by the memory limit 236 limit := len(q.resultCache) 237 if common.StorageSize(len(q.resultCache))*q.resultSize > common.StorageSize(blockCacheMemory) { 238 limit = int((common.StorageSize(blockCacheMemory) + q.resultSize - 1) / q.resultSize) 239 } 240 // Calculate the number of slots already finished 241 finished := 0 242 for _, result := range q.resultCache[:limit] { 243 if result == nil { 244 break 245 } 246 if _, ok := donePool[result.Hash]; ok { 247 finished++ 248 } 249 } 250 // Calculate the number of slots currently downloading 251 pending := 0 252 for _, request := range pendPool { 253 for _, header := range request.Headers { 254 if header.Number.Uint64() < q.resultOffset+uint64(limit) { 255 pending++ 256 } 257 } 258 } 259 // Return the free slots to distribute 260 return limit - finished - pending 261 } 262 263 // ScheduleSkeleton adds a batch of header retrieval tasks to the queue to fill 264 // up an already retrieved header skeleton. 265 func (q *queue) ScheduleSkeleton(from uint64, skeleton []*types.Header) { 266 q.lock.Lock() 267 defer q.lock.Unlock() 268 269 // No skeleton retrieval can be in progress, fail hard if so (huge implementation bug) 270 if q.headerResults != nil { 271 panic("skeleton assembly already in progress") 272 } 273 // Shedule all the header retrieval tasks for the skeleton assembly 274 q.headerTaskPool = make(map[uint64]*types.Header) 275 q.headerTaskQueue = prque.New() 276 q.headerPeerMiss = make(map[string]map[uint64]struct{}) // Reset availability to correct invalid chains 277 q.headerResults = make([]*types.Header, len(skeleton)*MaxHeaderFetch) 278 q.headerProced = 0 279 q.headerOffset = from 280 q.headerContCh = make(chan bool, 1) 281 282 for i, header := range skeleton { 283 index := from + uint64(i*MaxHeaderFetch) 284 285 q.headerTaskPool[index] = header 286 q.headerTaskQueue.Push(index, -float32(index)) 287 } 288 } 289 290 // RetrieveHeaders retrieves the header chain assemble based on the scheduled 291 // skeleton. 292 func (q *queue) RetrieveHeaders() ([]*types.Header, int) { 293 q.lock.Lock() 294 defer q.lock.Unlock() 295 296 headers, proced := q.headerResults, q.headerProced 297 q.headerResults, q.headerProced = nil, 0 298 299 return headers, proced 300 } 301 302 // Schedule adds a set of headers for the download queue for scheduling, returning 303 // the new headers encountered. 304 func (q *queue) Schedule(headers []*types.Header, from uint64) []*types.Header { 305 q.lock.Lock() 306 defer q.lock.Unlock() 307 308 // Insert all the headers prioritised by the contained block number 309 inserts := make([]*types.Header, 0, len(headers)) 310 for _, header := range headers { 311 // Make sure chain order is honoured and preserved throughout 312 hash := header.Hash() 313 if header.Number == nil || header.Number.Uint64() != from { 314 log.Warn("Header broke chain ordering", "number", header.Number, "hash", hash, "expected", from) 315 break 316 } 317 if q.headerHead != (common.Hash{}) && q.headerHead != header.ParentHash { 318 log.Warn("Header broke chain ancestry", "number", header.Number, "hash", hash) 319 break 320 } 321 // Make sure no duplicate requests are executed 322 if _, ok := q.blockTaskPool[hash]; ok { 323 log.Warn("Header already scheduled for block fetch", "number", header.Number, "hash", hash) 324 continue 325 } 326 if _, ok := q.receiptTaskPool[hash]; ok { 327 log.Warn("Header already scheduled for receipt fetch", "number", header.Number, "hash", hash) 328 continue 329 } 330 // Queue the header for content retrieval 331 q.blockTaskPool[hash] = header 332 q.blockTaskQueue.Push(header, -float32(header.Number.Uint64())) 333 334 if q.mode == FastSync { 335 q.receiptTaskPool[hash] = header 336 q.receiptTaskQueue.Push(header, -float32(header.Number.Uint64())) 337 } 338 inserts = append(inserts, header) 339 q.headerHead = hash 340 from++ 341 } 342 return inserts 343 } 344 345 // Results retrieves and permanently removes a batch of fetch results from 346 // the cache. the result slice will be empty if the queue has been closed. 347 func (q *queue) Results(block bool) []*fetchResult { 348 q.lock.Lock() 349 defer q.lock.Unlock() 350 351 // Count the number of items available for processing 352 nproc := q.countProcessableItems() 353 for nproc == 0 && !q.closed { 354 if !block { 355 return nil 356 } 357 q.active.Wait() 358 nproc = q.countProcessableItems() 359 } 360 // Since we have a batch limit, don't pull more into "dangling" memory 361 if nproc > maxResultsProcess { 362 nproc = maxResultsProcess 363 } 364 results := make([]*fetchResult, nproc) 365 copy(results, q.resultCache[:nproc]) 366 if len(results) > 0 { 367 // Mark results as done before dropping them from the cache. 368 for _, result := range results { 369 hash := result.Header.Hash() 370 delete(q.blockDonePool, hash) 371 delete(q.receiptDonePool, hash) 372 } 373 // Delete the results from the cache and clear the tail. 374 copy(q.resultCache, q.resultCache[nproc:]) 375 for i := len(q.resultCache) - nproc; i < len(q.resultCache); i++ { 376 q.resultCache[i] = nil 377 } 378 // Advance the expected block number of the first cache entry. 379 q.resultOffset += uint64(nproc) 380 381 // Recalculate the result item weights to prevent memory exhaustion 382 for _, result := range results { 383 size := result.Header.Size() 384 for _, uncle := range result.Uncles { 385 size += uncle.Size() 386 } 387 for _, receipt := range result.Receipts { 388 size += receipt.Size() 389 } 390 for _, tx := range result.Transactions { 391 size += tx.Size() 392 } 393 q.resultSize = common.StorageSize(blockCacheSizeWeight)*size + (1-common.StorageSize(blockCacheSizeWeight))*q.resultSize 394 } 395 } 396 return results 397 } 398 399 // countProcessableItems counts the processable items. 400 func (q *queue) countProcessableItems() int { 401 for i, result := range q.resultCache { 402 if result == nil || result.Pending > 0 { 403 return i 404 } 405 } 406 return len(q.resultCache) 407 } 408 409 // ReserveHeaders reserves a set of headers for the given peer, skipping any 410 // previously failed batches. 411 func (q *queue) ReserveHeaders(p *peerConnection, count int) *fetchRequest { 412 q.lock.Lock() 413 defer q.lock.Unlock() 414 415 // Short circuit if the peer's already downloading something (sanity check to 416 // not corrupt state) 417 if _, ok := q.headerPendPool[p.id]; ok { 418 return nil 419 } 420 // Retrieve a batch of hashes, skipping previously failed ones 421 send, skip := uint64(0), []uint64{} 422 for send == 0 && !q.headerTaskQueue.Empty() { 423 from, _ := q.headerTaskQueue.Pop() 424 if q.headerPeerMiss[p.id] != nil { 425 if _, ok := q.headerPeerMiss[p.id][from.(uint64)]; ok { 426 skip = append(skip, from.(uint64)) 427 continue 428 } 429 } 430 send = from.(uint64) 431 } 432 // Merge all the skipped batches back 433 for _, from := range skip { 434 q.headerTaskQueue.Push(from, -float32(from)) 435 } 436 // Assemble and return the block download request 437 if send == 0 { 438 return nil 439 } 440 request := &fetchRequest{ 441 Peer: p, 442 From: send, 443 Time: time.Now(), 444 } 445 q.headerPendPool[p.id] = request 446 return request 447 } 448 449 // ReserveBodies reserves a set of body fetches for the given peer, skipping any 450 // previously failed downloads. Beside the next batch of needed fetches, it also 451 // returns a flag whether empty blocks were queued requiring processing. 452 func (q *queue) ReserveBodies(p *peerConnection, count int) (*fetchRequest, bool, error) { 453 isNoop := func(header *types.Header) bool { 454 return header.TxHash == types.EmptyRootHash && header.UncleHash == types.EmptyUncleHash 455 } 456 q.lock.Lock() 457 defer q.lock.Unlock() 458 459 return q.reserveHeaders(p, count, q.blockTaskPool, q.blockTaskQueue, q.blockPendPool, q.blockDonePool, isNoop) 460 } 461 462 // ReserveReceipts reserves a set of receipt fetches for the given peer, skipping 463 // any previously failed downloads. Beside the next batch of needed fetches, it 464 // also returns a flag whether empty receipts were queued requiring importing. 465 func (q *queue) ReserveReceipts(p *peerConnection, count int) (*fetchRequest, bool, error) { 466 isNoop := func(header *types.Header) bool { 467 return header.ReceiptHash == types.EmptyRootHash 468 } 469 q.lock.Lock() 470 defer q.lock.Unlock() 471 472 return q.reserveHeaders(p, count, q.receiptTaskPool, q.receiptTaskQueue, q.receiptPendPool, q.receiptDonePool, isNoop) 473 } 474 475 // reserveHeaders reserves a set of data download operations for a given peer, 476 // skipping any previously failed ones. This method is a generic version used 477 // by the individual special reservation functions. 478 // 479 // Note, this method expects the queue lock to be already held for writing. The 480 // reason the lock is not obtained in here is because the parameters already need 481 // to access the queue, so they already need a lock anyway. 482 func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common.Hash]*types.Header, taskQueue *prque.Prque, 483 pendPool map[string]*fetchRequest, donePool map[common.Hash]struct{}, isNoop func(*types.Header) bool) (*fetchRequest, bool, error) { 484 // Short circuit if the pool has been depleted, or if the peer's already 485 // downloading something (sanity check not to corrupt state) 486 if taskQueue.Empty() { 487 return nil, false, nil 488 } 489 if _, ok := pendPool[p.id]; ok { 490 return nil, false, nil 491 } 492 // Calculate an upper limit on the items we might fetch (i.e. throttling) 493 space := q.resultSlots(pendPool, donePool) 494 495 // Retrieve a batch of tasks, skipping previously failed ones 496 send := make([]*types.Header, 0, count) 497 skip := make([]*types.Header, 0) 498 499 progress := false 500 for proc := 0; proc < space && len(send) < count && !taskQueue.Empty(); proc++ { 501 header := taskQueue.PopItem().(*types.Header) 502 hash := header.Hash() 503 504 // If we're the first to request this task, initialise the result container 505 index := int(header.Number.Int64() - int64(q.resultOffset)) 506 if index >= len(q.resultCache) || index < 0 { 507 common.Report("index allocation went beyond available resultCache space") 508 return nil, false, errInvalidChain 509 } 510 if q.resultCache[index] == nil { 511 components := 1 512 if q.mode == FastSync { 513 components = 2 514 } 515 q.resultCache[index] = &fetchResult{ 516 Pending: components, 517 Hash: hash, 518 Header: header, 519 } 520 } 521 // If this fetch task is a noop, skip this fetch operation 522 if isNoop(header) { 523 donePool[hash] = struct{}{} 524 delete(taskPool, hash) 525 526 space, proc = space-1, proc-1 527 q.resultCache[index].Pending-- 528 progress = true 529 continue 530 } 531 // Otherwise unless the peer is known not to have the data, add to the retrieve list 532 if p.Lacks(hash) { 533 skip = append(skip, header) 534 } else { 535 send = append(send, header) 536 } 537 } 538 // Merge all the skipped headers back 539 for _, header := range skip { 540 taskQueue.Push(header, -float32(header.Number.Uint64())) 541 } 542 if progress { 543 // Wake WaitResults, resultCache was modified 544 q.active.Signal() 545 } 546 // Assemble and return the block download request 547 if len(send) == 0 { 548 return nil, progress, nil 549 } 550 request := &fetchRequest{ 551 Peer: p, 552 Headers: send, 553 Time: time.Now(), 554 } 555 pendPool[p.id] = request 556 557 return request, progress, nil 558 } 559 560 // CancelHeaders aborts a fetch request, returning all pending skeleton indexes to the queue. 561 func (q *queue) CancelHeaders(request *fetchRequest) { 562 q.cancel(request, q.headerTaskQueue, q.headerPendPool) 563 } 564 565 // CancelBodies aborts a body fetch request, returning all pending headers to the 566 // task queue. 567 func (q *queue) CancelBodies(request *fetchRequest) { 568 q.cancel(request, q.blockTaskQueue, q.blockPendPool) 569 } 570 571 // CancelReceipts aborts a body fetch request, returning all pending headers to 572 // the task queue. 573 func (q *queue) CancelReceipts(request *fetchRequest) { 574 q.cancel(request, q.receiptTaskQueue, q.receiptPendPool) 575 } 576 577 // Cancel aborts a fetch request, returning all pending hashes to the task queue. 578 func (q *queue) cancel(request *fetchRequest, taskQueue *prque.Prque, pendPool map[string]*fetchRequest) { 579 q.lock.Lock() 580 defer q.lock.Unlock() 581 582 if request.From > 0 { 583 taskQueue.Push(request.From, -float32(request.From)) 584 } 585 for _, header := range request.Headers { 586 taskQueue.Push(header, -float32(header.Number.Uint64())) 587 } 588 delete(pendPool, request.Peer.id) 589 } 590 591 // Revoke cancels all pending requests belonging to a given peer. This method is 592 // meant to be called during a peer drop to quickly reassign owned data fetches 593 // to remaining nodes. 594 func (q *queue) Revoke(peerId string) { 595 q.lock.Lock() 596 defer q.lock.Unlock() 597 598 if request, ok := q.blockPendPool[peerId]; ok { 599 for _, header := range request.Headers { 600 q.blockTaskQueue.Push(header, -float32(header.Number.Uint64())) 601 } 602 delete(q.blockPendPool, peerId) 603 } 604 if request, ok := q.receiptPendPool[peerId]; ok { 605 for _, header := range request.Headers { 606 q.receiptTaskQueue.Push(header, -float32(header.Number.Uint64())) 607 } 608 delete(q.receiptPendPool, peerId) 609 } 610 } 611 612 // ExpireHeaders checks for in flight requests that exceeded a timeout allowance, 613 // canceling them and returning the responsible peers for penalisation. 614 func (q *queue) ExpireHeaders(timeout time.Duration) map[string]int { 615 q.lock.Lock() 616 defer q.lock.Unlock() 617 618 return q.expire(timeout, q.headerPendPool, q.headerTaskQueue, headerTimeoutMeter) 619 } 620 621 // ExpireBodies checks for in flight block body requests that exceeded a timeout 622 // allowance, canceling them and returning the responsible peers for penalisation. 623 func (q *queue) ExpireBodies(timeout time.Duration) map[string]int { 624 q.lock.Lock() 625 defer q.lock.Unlock() 626 627 return q.expire(timeout, q.blockPendPool, q.blockTaskQueue, bodyTimeoutMeter) 628 } 629 630 // ExpireReceipts checks for in flight receipt requests that exceeded a timeout 631 // allowance, canceling them and returning the responsible peers for penalisation. 632 func (q *queue) ExpireReceipts(timeout time.Duration) map[string]int { 633 q.lock.Lock() 634 defer q.lock.Unlock() 635 636 return q.expire(timeout, q.receiptPendPool, q.receiptTaskQueue, receiptTimeoutMeter) 637 } 638 639 // expire is the generic check that move expired tasks from a pending pool back 640 // into a task pool, returning all entities caught with expired tasks. 641 // 642 // Note, this method expects the queue lock to be already held. The 643 // reason the lock is not obtained in here is because the parameters already need 644 // to access the queue, so they already need a lock anyway. 645 func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest, taskQueue *prque.Prque, timeoutMeter metrics.Meter) map[string]int { 646 // Iterate over the expired requests and return each to the queue 647 expiries := make(map[string]int) 648 for id, request := range pendPool { 649 if time.Since(request.Time) > timeout { 650 // Update the metrics with the timeout 651 timeoutMeter.Mark(1) 652 653 // Return any non satisfied requests to the pool 654 if request.From > 0 { 655 taskQueue.Push(request.From, -float32(request.From)) 656 } 657 for _, header := range request.Headers { 658 taskQueue.Push(header, -float32(header.Number.Uint64())) 659 } 660 // Add the peer to the expiry report along the the number of failed requests 661 expiries[id] = len(request.Headers) 662 } 663 } 664 // Remove the expired requests from the pending pool 665 for id := range expiries { 666 delete(pendPool, id) 667 } 668 return expiries 669 } 670 671 // DeliverHeaders injects a header retrieval response into the header results 672 // cache. This method either accepts all headers it received, or none of them 673 // if they do not map correctly to the skeleton. 674 // 675 // If the headers are accepted, the method makes an attempt to deliver the set 676 // of ready headers to the processor to keep the pipeline full. However it will 677 // not block to prevent stalling other pending deliveries. 678 func (q *queue) DeliverHeaders(id string, headers []*types.Header, headerProcCh chan []*types.Header) (int, error) { 679 q.lock.Lock() 680 defer q.lock.Unlock() 681 682 // Short circuit if the data was never requested 683 request := q.headerPendPool[id] 684 if request == nil { 685 return 0, errNoFetchesPending 686 } 687 headerReqTimer.UpdateSince(request.Time) 688 delete(q.headerPendPool, id) 689 690 // Ensure headers can be mapped onto the skeleton chain 691 target := q.headerTaskPool[request.From].Hash() 692 693 accepted := len(headers) == MaxHeaderFetch 694 if accepted { 695 if headers[0].Number.Uint64() != request.From { 696 log.Trace("First header broke chain ordering", "peer", id, "number", headers[0].Number, "hash", headers[0].Hash(), request.From) 697 accepted = false 698 } else if headers[len(headers)-1].Hash() != target { 699 log.Trace("Last header broke skeleton structure ", "peer", id, "number", headers[len(headers)-1].Number, "hash", headers[len(headers)-1].Hash(), "expected", target) 700 accepted = false 701 } 702 } 703 if accepted { 704 for i, header := range headers[1:] { 705 hash := header.Hash() 706 if want := request.From + 1 + uint64(i); header.Number.Uint64() != want { 707 log.Warn("Header broke chain ordering", "peer", id, "number", header.Number, "hash", hash, "expected", want) 708 accepted = false 709 break 710 } 711 if headers[i].Hash() != header.ParentHash { 712 log.Warn("Header broke chain ancestry", "peer", id, "number", header.Number, "hash", hash) 713 accepted = false 714 break 715 } 716 } 717 } 718 // If the batch of headers wasn't accepted, mark as unavailable 719 if !accepted { 720 log.Trace("Skeleton filling not accepted", "peer", id, "from", request.From) 721 722 miss := q.headerPeerMiss[id] 723 if miss == nil { 724 q.headerPeerMiss[id] = make(map[uint64]struct{}) 725 miss = q.headerPeerMiss[id] 726 } 727 miss[request.From] = struct{}{} 728 729 q.headerTaskQueue.Push(request.From, -float32(request.From)) 730 return 0, errors.New("delivery not accepted") 731 } 732 // Clean up a successful fetch and try to deliver any sub-results 733 copy(q.headerResults[request.From-q.headerOffset:], headers) 734 delete(q.headerTaskPool, request.From) 735 736 ready := 0 737 for q.headerProced+ready < len(q.headerResults) && q.headerResults[q.headerProced+ready] != nil { 738 ready += MaxHeaderFetch 739 } 740 if ready > 0 { 741 // Headers are ready for delivery, gather them and push forward (non blocking) 742 process := make([]*types.Header, ready) 743 copy(process, q.headerResults[q.headerProced:q.headerProced+ready]) 744 745 select { 746 case headerProcCh <- process: 747 log.Trace("Pre-scheduled new headers", "peer", id, "count", len(process), "from", process[0].Number) 748 q.headerProced += len(process) 749 default: 750 } 751 } 752 // Check for termination and return 753 if len(q.headerTaskPool) == 0 { 754 q.headerContCh <- false 755 } 756 return len(headers), nil 757 } 758 759 // DeliverBodies injects a block body retrieval response into the results queue. 760 // The method returns the number of blocks bodies accepted from the delivery and 761 // also wakes any threads waiting for data delivery. 762 func (q *queue) DeliverBodies(id string, txLists [][]*types.Transaction, uncleLists [][]*types.Header) (int, error) { 763 q.lock.Lock() 764 defer q.lock.Unlock() 765 766 reconstruct := func(header *types.Header, index int, result *fetchResult) error { 767 if types.DeriveSha(types.Transactions(txLists[index])) != header.TxHash || types.CalcUncleHash(uncleLists[index]) != header.UncleHash { 768 return errInvalidBody 769 } 770 result.Transactions = txLists[index] 771 result.Uncles = uncleLists[index] 772 return nil 773 } 774 return q.deliver(id, q.blockTaskPool, q.blockTaskQueue, q.blockPendPool, q.blockDonePool, bodyReqTimer, len(txLists), reconstruct) 775 } 776 777 // DeliverReceipts injects a receipt retrieval response into the results queue. 778 // The method returns the number of transaction receipts accepted from the delivery 779 // and also wakes any threads waiting for data delivery. 780 func (q *queue) DeliverReceipts(id string, receiptList [][]*types.Receipt) (int, error) { 781 q.lock.Lock() 782 defer q.lock.Unlock() 783 784 reconstruct := func(header *types.Header, index int, result *fetchResult) error { 785 if types.DeriveSha(types.Receipts(receiptList[index])) != header.ReceiptHash { 786 return errInvalidReceipt 787 } 788 result.Receipts = receiptList[index] 789 return nil 790 } 791 return q.deliver(id, q.receiptTaskPool, q.receiptTaskQueue, q.receiptPendPool, q.receiptDonePool, receiptReqTimer, len(receiptList), reconstruct) 792 } 793 794 // deliver injects a data retrieval response into the results queue. 795 // 796 // Note, this method expects the queue lock to be already held for writing. The 797 // reason the lock is not obtained in here is because the parameters already need 798 // to access the queue, so they already need a lock anyway. 799 func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, taskQueue *prque.Prque, 800 pendPool map[string]*fetchRequest, donePool map[common.Hash]struct{}, reqTimer metrics.Timer, 801 results int, reconstruct func(header *types.Header, index int, result *fetchResult) error) (int, error) { 802 803 // Short circuit if the data was never requested 804 request := pendPool[id] 805 if request == nil { 806 return 0, errNoFetchesPending 807 } 808 reqTimer.UpdateSince(request.Time) 809 delete(pendPool, id) 810 811 // If no data items were retrieved, mark them as unavailable for the origin peer 812 if results == 0 { 813 for _, header := range request.Headers { 814 request.Peer.MarkLacking(header.Hash()) 815 } 816 } 817 // Assemble each of the results with their headers and retrieved data parts 818 var ( 819 accepted int 820 failure error 821 useful bool 822 ) 823 for i, header := range request.Headers { 824 // Short circuit assembly if no more fetch results are found 825 if i >= results { 826 break 827 } 828 // Reconstruct the next result if contents match up 829 index := int(header.Number.Int64() - int64(q.resultOffset)) 830 if index >= len(q.resultCache) || index < 0 || q.resultCache[index] == nil { 831 failure = errInvalidChain 832 break 833 } 834 if err := reconstruct(header, i, q.resultCache[index]); err != nil { 835 failure = err 836 break 837 } 838 hash := header.Hash() 839 840 donePool[hash] = struct{}{} 841 q.resultCache[index].Pending-- 842 useful = true 843 accepted++ 844 845 // Clean up a successful fetch 846 request.Headers[i] = nil 847 delete(taskPool, hash) 848 } 849 // Return all failed or missing fetches to the queue 850 for _, header := range request.Headers { 851 if header != nil { 852 taskQueue.Push(header, -float32(header.Number.Uint64())) 853 } 854 } 855 // Wake up WaitResults 856 if accepted > 0 { 857 q.active.Signal() 858 } 859 // If none of the data was good, it's a stale delivery 860 switch { 861 case failure == nil || failure == errInvalidChain: 862 return accepted, failure 863 case useful: 864 return accepted, fmt.Errorf("partial failure: %v", failure) 865 default: 866 return accepted, errStaleDelivery 867 } 868 } 869 870 // Prepare configures the result cache to allow accepting and caching inbound 871 // fetch results. 872 func (q *queue) Prepare(offset uint64, mode SyncMode) { 873 q.lock.Lock() 874 defer q.lock.Unlock() 875 876 // Prepare the queue for sync results 877 if q.resultOffset < offset { 878 q.resultOffset = offset 879 } 880 q.mode = mode 881 }