github.com/jeffallen/go-ethereum@v1.1.4-0.20150910155051-571d3236c49c/eth/fetcher/fetcher_test.go (about) 1 // Copyright 2015 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package fetcher 18 19 import ( 20 "errors" 21 "math/big" 22 "sync" 23 "sync/atomic" 24 "testing" 25 "time" 26 27 "github.com/ethereum/go-ethereum/common" 28 "github.com/ethereum/go-ethereum/core" 29 "github.com/ethereum/go-ethereum/core/types" 30 "github.com/ethereum/go-ethereum/ethdb" 31 "github.com/ethereum/go-ethereum/params" 32 ) 33 34 var ( 35 testdb, _ = ethdb.NewMemDatabase() 36 genesis = core.GenesisBlockForTesting(testdb, common.Address{}, big.NewInt(0)) 37 unknownBlock = types.NewBlock(&types.Header{GasLimit: params.GenesisGasLimit}, nil, nil, nil) 38 ) 39 40 // makeChain creates a chain of n blocks starting at and including parent. 41 // the returned hash chain is ordered head->parent. 42 func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block) { 43 blocks := core.GenerateChain(parent, testdb, n, func(i int, gen *core.BlockGen) { 44 gen.SetCoinbase(common.Address{seed}) 45 }) 46 hashes := make([]common.Hash, n+1) 47 hashes[len(hashes)-1] = parent.Hash() 48 blockm := make(map[common.Hash]*types.Block, n+1) 49 blockm[parent.Hash()] = parent 50 for i, b := range blocks { 51 hashes[len(hashes)-i-2] = b.Hash() 52 blockm[b.Hash()] = b 53 } 54 return hashes, blockm 55 } 56 57 // fetcherTester is a test simulator for mocking out local block chain. 58 type fetcherTester struct { 59 fetcher *Fetcher 60 61 hashes []common.Hash // Hash chain belonging to the tester 62 blocks map[common.Hash]*types.Block // Blocks belonging to the tester 63 64 lock sync.RWMutex 65 } 66 67 // newTester creates a new fetcher test mocker. 68 func newTester() *fetcherTester { 69 tester := &fetcherTester{ 70 hashes: []common.Hash{genesis.Hash()}, 71 blocks: map[common.Hash]*types.Block{genesis.Hash(): genesis}, 72 } 73 tester.fetcher = New(tester.getBlock, tester.verifyBlock, tester.broadcastBlock, tester.chainHeight, tester.insertChain, tester.dropPeer) 74 tester.fetcher.Start() 75 76 return tester 77 } 78 79 // getBlock retrieves a block from the tester's block chain. 80 func (f *fetcherTester) getBlock(hash common.Hash) *types.Block { 81 f.lock.RLock() 82 defer f.lock.RUnlock() 83 84 return f.blocks[hash] 85 } 86 87 // verifyBlock is a nop placeholder for the block header verification. 88 func (f *fetcherTester) verifyBlock(block *types.Block, parent *types.Block) error { 89 return nil 90 } 91 92 // broadcastBlock is a nop placeholder for the block broadcasting. 93 func (f *fetcherTester) broadcastBlock(block *types.Block, propagate bool) { 94 } 95 96 // chainHeight retrieves the current height (block number) of the chain. 97 func (f *fetcherTester) chainHeight() uint64 { 98 f.lock.RLock() 99 defer f.lock.RUnlock() 100 101 return f.blocks[f.hashes[len(f.hashes)-1]].NumberU64() 102 } 103 104 // insertChain injects a new blocks into the simulated chain. 105 func (f *fetcherTester) insertChain(blocks types.Blocks) (int, error) { 106 f.lock.Lock() 107 defer f.lock.Unlock() 108 109 for i, block := range blocks { 110 // Make sure the parent in known 111 if _, ok := f.blocks[block.ParentHash()]; !ok { 112 return i, errors.New("unknown parent") 113 } 114 // Discard any new blocks if the same height already exists 115 if block.NumberU64() <= f.blocks[f.hashes[len(f.hashes)-1]].NumberU64() { 116 return i, nil 117 } 118 // Otherwise build our current chain 119 f.hashes = append(f.hashes, block.Hash()) 120 f.blocks[block.Hash()] = block 121 } 122 return 0, nil 123 } 124 125 // dropPeer is a nop placeholder for the peer removal. 126 func (f *fetcherTester) dropPeer(peer string) { 127 } 128 129 // peerFetcher retrieves a fetcher associated with a simulated peer. 130 func (f *fetcherTester) makeFetcher(blocks map[common.Hash]*types.Block) blockRequesterFn { 131 closure := make(map[common.Hash]*types.Block) 132 for hash, block := range blocks { 133 closure[hash] = block 134 } 135 // Create a function that returns blocks from the closure 136 return func(hashes []common.Hash) error { 137 // Gather the blocks to return 138 blocks := make([]*types.Block, 0, len(hashes)) 139 for _, hash := range hashes { 140 if block, ok := closure[hash]; ok { 141 blocks = append(blocks, block) 142 } 143 } 144 // Return on a new thread 145 go f.fetcher.Filter(blocks) 146 147 return nil 148 } 149 } 150 151 // verifyImportEvent verifies that one single event arrive on an import channel. 152 func verifyImportEvent(t *testing.T, imported chan *types.Block) { 153 select { 154 case <-imported: 155 case <-time.After(time.Second): 156 t.Fatalf("import timeout") 157 } 158 } 159 160 // verifyImportCount verifies that exactly count number of events arrive on an 161 // import hook channel. 162 func verifyImportCount(t *testing.T, imported chan *types.Block, count int) { 163 for i := 0; i < count; i++ { 164 select { 165 case <-imported: 166 case <-time.After(time.Second): 167 t.Fatalf("block %d: import timeout", i) 168 } 169 } 170 verifyImportDone(t, imported) 171 } 172 173 // verifyImportDone verifies that no more events are arriving on an import channel. 174 func verifyImportDone(t *testing.T, imported chan *types.Block) { 175 select { 176 case <-imported: 177 t.Fatalf("extra block imported") 178 case <-time.After(50 * time.Millisecond): 179 } 180 } 181 182 // Tests that a fetcher accepts block announcements and initiates retrievals for 183 // them, successfully importing into the local chain. 184 func TestSequentialAnnouncements(t *testing.T) { 185 // Create a chain of blocks to import 186 targetBlocks := 4 * hashLimit 187 hashes, blocks := makeChain(targetBlocks, 0, genesis) 188 189 tester := newTester() 190 fetcher := tester.makeFetcher(blocks) 191 192 // Iteratively announce blocks until all are imported 193 imported := make(chan *types.Block) 194 tester.fetcher.importedHook = func(block *types.Block) { imported <- block } 195 196 for i := len(hashes) - 2; i >= 0; i-- { 197 tester.fetcher.Notify("valid", hashes[i], time.Now().Add(-arriveTimeout), fetcher) 198 verifyImportEvent(t, imported) 199 } 200 verifyImportDone(t, imported) 201 } 202 203 // Tests that if blocks are announced by multiple peers (or even the same buggy 204 // peer), they will only get downloaded at most once. 205 func TestConcurrentAnnouncements(t *testing.T) { 206 // Create a chain of blocks to import 207 targetBlocks := 4 * hashLimit 208 hashes, blocks := makeChain(targetBlocks, 0, genesis) 209 210 // Assemble a tester with a built in counter for the requests 211 tester := newTester() 212 fetcher := tester.makeFetcher(blocks) 213 214 counter := uint32(0) 215 wrapper := func(hashes []common.Hash) error { 216 atomic.AddUint32(&counter, uint32(len(hashes))) 217 return fetcher(hashes) 218 } 219 // Iteratively announce blocks until all are imported 220 imported := make(chan *types.Block) 221 tester.fetcher.importedHook = func(block *types.Block) { imported <- block } 222 223 for i := len(hashes) - 2; i >= 0; i-- { 224 tester.fetcher.Notify("first", hashes[i], time.Now().Add(-arriveTimeout), wrapper) 225 tester.fetcher.Notify("second", hashes[i], time.Now().Add(-arriveTimeout+time.Millisecond), wrapper) 226 tester.fetcher.Notify("second", hashes[i], time.Now().Add(-arriveTimeout-time.Millisecond), wrapper) 227 228 verifyImportEvent(t, imported) 229 } 230 verifyImportDone(t, imported) 231 232 // Make sure no blocks were retrieved twice 233 if int(counter) != targetBlocks { 234 t.Fatalf("retrieval count mismatch: have %v, want %v", counter, targetBlocks) 235 } 236 } 237 238 // Tests that announcements arriving while a previous is being fetched still 239 // results in a valid import. 240 func TestOverlappingAnnouncements(t *testing.T) { 241 // Create a chain of blocks to import 242 targetBlocks := 4 * hashLimit 243 hashes, blocks := makeChain(targetBlocks, 0, genesis) 244 245 tester := newTester() 246 fetcher := tester.makeFetcher(blocks) 247 248 // Iteratively announce blocks, but overlap them continuously 249 fetching := make(chan []common.Hash) 250 imported := make(chan *types.Block, len(hashes)-1) 251 tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- hashes } 252 tester.fetcher.importedHook = func(block *types.Block) { imported <- block } 253 254 for i := len(hashes) - 2; i >= 0; i-- { 255 tester.fetcher.Notify("valid", hashes[i], time.Now().Add(-arriveTimeout), fetcher) 256 select { 257 case <-fetching: 258 case <-time.After(time.Second): 259 t.Fatalf("hash %d: announce timeout", len(hashes)-i) 260 } 261 } 262 // Wait for all the imports to complete and check count 263 verifyImportCount(t, imported, len(hashes)-1) 264 } 265 266 // Tests that announces already being retrieved will not be duplicated. 267 func TestPendingDeduplication(t *testing.T) { 268 // Create a hash and corresponding block 269 hashes, blocks := makeChain(1, 0, genesis) 270 271 // Assemble a tester with a built in counter and delayed fetcher 272 tester := newTester() 273 fetcher := tester.makeFetcher(blocks) 274 275 delay := 50 * time.Millisecond 276 counter := uint32(0) 277 wrapper := func(hashes []common.Hash) error { 278 atomic.AddUint32(&counter, uint32(len(hashes))) 279 280 // Simulate a long running fetch 281 go func() { 282 time.Sleep(delay) 283 fetcher(hashes) 284 }() 285 return nil 286 } 287 // Announce the same block many times until it's fetched (wait for any pending ops) 288 for tester.getBlock(hashes[0]) == nil { 289 tester.fetcher.Notify("repeater", hashes[0], time.Now().Add(-arriveTimeout), wrapper) 290 time.Sleep(time.Millisecond) 291 } 292 time.Sleep(delay) 293 294 // Check that all blocks were imported and none fetched twice 295 if imported := len(tester.blocks); imported != 2 { 296 t.Fatalf("synchronised block mismatch: have %v, want %v", imported, 2) 297 } 298 if int(counter) != 1 { 299 t.Fatalf("retrieval count mismatch: have %v, want %v", counter, 1) 300 } 301 } 302 303 // Tests that announcements retrieved in a random order are cached and eventually 304 // imported when all the gaps are filled in. 305 func TestRandomArrivalImport(t *testing.T) { 306 // Create a chain of blocks to import, and choose one to delay 307 targetBlocks := maxQueueDist 308 hashes, blocks := makeChain(targetBlocks, 0, genesis) 309 skip := targetBlocks / 2 310 311 tester := newTester() 312 fetcher := tester.makeFetcher(blocks) 313 314 // Iteratively announce blocks, skipping one entry 315 imported := make(chan *types.Block, len(hashes)-1) 316 tester.fetcher.importedHook = func(block *types.Block) { imported <- block } 317 318 for i := len(hashes) - 1; i >= 0; i-- { 319 if i != skip { 320 tester.fetcher.Notify("valid", hashes[i], time.Now().Add(-arriveTimeout), fetcher) 321 time.Sleep(time.Millisecond) 322 } 323 } 324 // Finally announce the skipped entry and check full import 325 tester.fetcher.Notify("valid", hashes[skip], time.Now().Add(-arriveTimeout), fetcher) 326 verifyImportCount(t, imported, len(hashes)-1) 327 } 328 329 // Tests that direct block enqueues (due to block propagation vs. hash announce) 330 // are correctly schedule, filling and import queue gaps. 331 func TestQueueGapFill(t *testing.T) { 332 // Create a chain of blocks to import, and choose one to not announce at all 333 targetBlocks := maxQueueDist 334 hashes, blocks := makeChain(targetBlocks, 0, genesis) 335 skip := targetBlocks / 2 336 337 tester := newTester() 338 fetcher := tester.makeFetcher(blocks) 339 340 // Iteratively announce blocks, skipping one entry 341 imported := make(chan *types.Block, len(hashes)-1) 342 tester.fetcher.importedHook = func(block *types.Block) { imported <- block } 343 344 for i := len(hashes) - 1; i >= 0; i-- { 345 if i != skip { 346 tester.fetcher.Notify("valid", hashes[i], time.Now().Add(-arriveTimeout), fetcher) 347 time.Sleep(time.Millisecond) 348 } 349 } 350 // Fill the missing block directly as if propagated 351 tester.fetcher.Enqueue("valid", blocks[hashes[skip]]) 352 verifyImportCount(t, imported, len(hashes)-1) 353 } 354 355 // Tests that blocks arriving from various sources (multiple propagations, hash 356 // announces, etc) do not get scheduled for import multiple times. 357 func TestImportDeduplication(t *testing.T) { 358 // Create two blocks to import (one for duplication, the other for stalling) 359 hashes, blocks := makeChain(2, 0, genesis) 360 361 // Create the tester and wrap the importer with a counter 362 tester := newTester() 363 fetcher := tester.makeFetcher(blocks) 364 365 counter := uint32(0) 366 tester.fetcher.insertChain = func(blocks types.Blocks) (int, error) { 367 atomic.AddUint32(&counter, uint32(len(blocks))) 368 return tester.insertChain(blocks) 369 } 370 // Instrument the fetching and imported events 371 fetching := make(chan []common.Hash) 372 imported := make(chan *types.Block, len(hashes)-1) 373 tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- hashes } 374 tester.fetcher.importedHook = func(block *types.Block) { imported <- block } 375 376 // Announce the duplicating block, wait for retrieval, and also propagate directly 377 tester.fetcher.Notify("valid", hashes[0], time.Now().Add(-arriveTimeout), fetcher) 378 <-fetching 379 380 tester.fetcher.Enqueue("valid", blocks[hashes[0]]) 381 tester.fetcher.Enqueue("valid", blocks[hashes[0]]) 382 tester.fetcher.Enqueue("valid", blocks[hashes[0]]) 383 384 // Fill the missing block directly as if propagated, and check import uniqueness 385 tester.fetcher.Enqueue("valid", blocks[hashes[1]]) 386 verifyImportCount(t, imported, 2) 387 388 if counter != 2 { 389 t.Fatalf("import invocation count mismatch: have %v, want %v", counter, 2) 390 } 391 } 392 393 // Tests that blocks with numbers much lower or higher than out current head get 394 // discarded no prevent wasting resources on useless blocks from faulty peers. 395 func TestDistantDiscarding(t *testing.T) { 396 // Create a long chain to import 397 hashes, blocks := makeChain(3*maxQueueDist, 0, genesis) 398 head := hashes[len(hashes)/2] 399 400 // Create a tester and simulate a head block being the middle of the above chain 401 tester := newTester() 402 tester.hashes = []common.Hash{head} 403 tester.blocks = map[common.Hash]*types.Block{head: blocks[head]} 404 405 // Ensure that a block with a lower number than the threshold is discarded 406 tester.fetcher.Enqueue("lower", blocks[hashes[0]]) 407 time.Sleep(10 * time.Millisecond) 408 if !tester.fetcher.queue.Empty() { 409 t.Fatalf("fetcher queued stale block") 410 } 411 // Ensure that a block with a higher number than the threshold is discarded 412 tester.fetcher.Enqueue("higher", blocks[hashes[len(hashes)-1]]) 413 time.Sleep(10 * time.Millisecond) 414 if !tester.fetcher.queue.Empty() { 415 t.Fatalf("fetcher queued future block") 416 } 417 } 418 419 // Tests that a peer is unable to use unbounded memory with sending infinite 420 // block announcements to a node, but that even in the face of such an attack, 421 // the fetcher remains operational. 422 func TestHashMemoryExhaustionAttack(t *testing.T) { 423 // Create a tester with instrumented import hooks 424 tester := newTester() 425 426 imported := make(chan *types.Block) 427 tester.fetcher.importedHook = func(block *types.Block) { imported <- block } 428 429 // Create a valid chain and an infinite junk chain 430 targetBlocks := hashLimit + 2*maxQueueDist 431 hashes, blocks := makeChain(targetBlocks, 0, genesis) 432 valid := tester.makeFetcher(blocks) 433 434 attack, _ := makeChain(targetBlocks, 0, unknownBlock) 435 attacker := tester.makeFetcher(nil) 436 437 // Feed the tester a huge hashset from the attacker, and a limited from the valid peer 438 for i := 0; i < len(attack); i++ { 439 if i < maxQueueDist { 440 tester.fetcher.Notify("valid", hashes[len(hashes)-2-i], time.Now(), valid) 441 } 442 tester.fetcher.Notify("attacker", attack[i], time.Now(), attacker) 443 } 444 if len(tester.fetcher.announced) != hashLimit+maxQueueDist { 445 t.Fatalf("queued announce count mismatch: have %d, want %d", len(tester.fetcher.announced), hashLimit+maxQueueDist) 446 } 447 // Wait for fetches to complete 448 verifyImportCount(t, imported, maxQueueDist) 449 450 // Feed the remaining valid hashes to ensure DOS protection state remains clean 451 for i := len(hashes) - maxQueueDist - 2; i >= 0; i-- { 452 tester.fetcher.Notify("valid", hashes[i], time.Now().Add(-arriveTimeout), valid) 453 verifyImportEvent(t, imported) 454 } 455 verifyImportDone(t, imported) 456 } 457 458 // Tests that blocks sent to the fetcher (either through propagation or via hash 459 // announces and retrievals) don't pile up indefinitely, exhausting available 460 // system memory. 461 func TestBlockMemoryExhaustionAttack(t *testing.T) { 462 // Create a tester with instrumented import hooks 463 tester := newTester() 464 465 imported := make(chan *types.Block) 466 tester.fetcher.importedHook = func(block *types.Block) { imported <- block } 467 468 // Create a valid chain and a batch of dangling (but in range) blocks 469 targetBlocks := hashLimit + 2*maxQueueDist 470 hashes, blocks := makeChain(targetBlocks, 0, genesis) 471 attack := make(map[common.Hash]*types.Block) 472 for i := byte(0); len(attack) < blockLimit+2*maxQueueDist; i++ { 473 hashes, blocks := makeChain(maxQueueDist-1, i, unknownBlock) 474 for _, hash := range hashes[:maxQueueDist-2] { 475 attack[hash] = blocks[hash] 476 } 477 } 478 // Try to feed all the attacker blocks make sure only a limited batch is accepted 479 for _, block := range attack { 480 tester.fetcher.Enqueue("attacker", block) 481 } 482 time.Sleep(200 * time.Millisecond) 483 if queued := tester.fetcher.queue.Size(); queued != blockLimit { 484 t.Fatalf("queued block count mismatch: have %d, want %d", queued, blockLimit) 485 } 486 // Queue up a batch of valid blocks, and check that a new peer is allowed to do so 487 for i := 0; i < maxQueueDist-1; i++ { 488 tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-3-i]]) 489 } 490 time.Sleep(100 * time.Millisecond) 491 if queued := tester.fetcher.queue.Size(); queued != blockLimit+maxQueueDist-1 { 492 t.Fatalf("queued block count mismatch: have %d, want %d", queued, blockLimit+maxQueueDist-1) 493 } 494 // Insert the missing piece (and sanity check the import) 495 tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-2]]) 496 verifyImportCount(t, imported, maxQueueDist) 497 498 // Insert the remaining blocks in chunks to ensure clean DOS protection 499 for i := maxQueueDist; i < len(hashes)-1; i++ { 500 tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-2-i]]) 501 verifyImportEvent(t, imported) 502 } 503 verifyImportDone(t, imported) 504 }