github.com/jeffallen/go-ethereum@v1.1.4-0.20150910155051-571d3236c49c/eth/fetcher/fetcher_test.go (about)

     1  // Copyright 2015 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package fetcher
    18  
    19  import (
    20  	"errors"
    21  	"math/big"
    22  	"sync"
    23  	"sync/atomic"
    24  	"testing"
    25  	"time"
    26  
    27  	"github.com/ethereum/go-ethereum/common"
    28  	"github.com/ethereum/go-ethereum/core"
    29  	"github.com/ethereum/go-ethereum/core/types"
    30  	"github.com/ethereum/go-ethereum/ethdb"
    31  	"github.com/ethereum/go-ethereum/params"
    32  )
    33  
    34  var (
    35  	testdb, _    = ethdb.NewMemDatabase()
    36  	genesis      = core.GenesisBlockForTesting(testdb, common.Address{}, big.NewInt(0))
    37  	unknownBlock = types.NewBlock(&types.Header{GasLimit: params.GenesisGasLimit}, nil, nil, nil)
    38  )
    39  
    40  // makeChain creates a chain of n blocks starting at and including parent.
    41  // the returned hash chain is ordered head->parent.
    42  func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block) {
    43  	blocks := core.GenerateChain(parent, testdb, n, func(i int, gen *core.BlockGen) {
    44  		gen.SetCoinbase(common.Address{seed})
    45  	})
    46  	hashes := make([]common.Hash, n+1)
    47  	hashes[len(hashes)-1] = parent.Hash()
    48  	blockm := make(map[common.Hash]*types.Block, n+1)
    49  	blockm[parent.Hash()] = parent
    50  	for i, b := range blocks {
    51  		hashes[len(hashes)-i-2] = b.Hash()
    52  		blockm[b.Hash()] = b
    53  	}
    54  	return hashes, blockm
    55  }
    56  
    57  // fetcherTester is a test simulator for mocking out local block chain.
    58  type fetcherTester struct {
    59  	fetcher *Fetcher
    60  
    61  	hashes []common.Hash                // Hash chain belonging to the tester
    62  	blocks map[common.Hash]*types.Block // Blocks belonging to the tester
    63  
    64  	lock sync.RWMutex
    65  }
    66  
    67  // newTester creates a new fetcher test mocker.
    68  func newTester() *fetcherTester {
    69  	tester := &fetcherTester{
    70  		hashes: []common.Hash{genesis.Hash()},
    71  		blocks: map[common.Hash]*types.Block{genesis.Hash(): genesis},
    72  	}
    73  	tester.fetcher = New(tester.getBlock, tester.verifyBlock, tester.broadcastBlock, tester.chainHeight, tester.insertChain, tester.dropPeer)
    74  	tester.fetcher.Start()
    75  
    76  	return tester
    77  }
    78  
    79  // getBlock retrieves a block from the tester's block chain.
    80  func (f *fetcherTester) getBlock(hash common.Hash) *types.Block {
    81  	f.lock.RLock()
    82  	defer f.lock.RUnlock()
    83  
    84  	return f.blocks[hash]
    85  }
    86  
    87  // verifyBlock is a nop placeholder for the block header verification.
    88  func (f *fetcherTester) verifyBlock(block *types.Block, parent *types.Block) error {
    89  	return nil
    90  }
    91  
    92  // broadcastBlock is a nop placeholder for the block broadcasting.
    93  func (f *fetcherTester) broadcastBlock(block *types.Block, propagate bool) {
    94  }
    95  
    96  // chainHeight retrieves the current height (block number) of the chain.
    97  func (f *fetcherTester) chainHeight() uint64 {
    98  	f.lock.RLock()
    99  	defer f.lock.RUnlock()
   100  
   101  	return f.blocks[f.hashes[len(f.hashes)-1]].NumberU64()
   102  }
   103  
   104  // insertChain injects a new blocks into the simulated chain.
   105  func (f *fetcherTester) insertChain(blocks types.Blocks) (int, error) {
   106  	f.lock.Lock()
   107  	defer f.lock.Unlock()
   108  
   109  	for i, block := range blocks {
   110  		// Make sure the parent in known
   111  		if _, ok := f.blocks[block.ParentHash()]; !ok {
   112  			return i, errors.New("unknown parent")
   113  		}
   114  		// Discard any new blocks if the same height already exists
   115  		if block.NumberU64() <= f.blocks[f.hashes[len(f.hashes)-1]].NumberU64() {
   116  			return i, nil
   117  		}
   118  		// Otherwise build our current chain
   119  		f.hashes = append(f.hashes, block.Hash())
   120  		f.blocks[block.Hash()] = block
   121  	}
   122  	return 0, nil
   123  }
   124  
   125  // dropPeer is a nop placeholder for the peer removal.
   126  func (f *fetcherTester) dropPeer(peer string) {
   127  }
   128  
   129  // peerFetcher retrieves a fetcher associated with a simulated peer.
   130  func (f *fetcherTester) makeFetcher(blocks map[common.Hash]*types.Block) blockRequesterFn {
   131  	closure := make(map[common.Hash]*types.Block)
   132  	for hash, block := range blocks {
   133  		closure[hash] = block
   134  	}
   135  	// Create a function that returns blocks from the closure
   136  	return func(hashes []common.Hash) error {
   137  		// Gather the blocks to return
   138  		blocks := make([]*types.Block, 0, len(hashes))
   139  		for _, hash := range hashes {
   140  			if block, ok := closure[hash]; ok {
   141  				blocks = append(blocks, block)
   142  			}
   143  		}
   144  		// Return on a new thread
   145  		go f.fetcher.Filter(blocks)
   146  
   147  		return nil
   148  	}
   149  }
   150  
   151  // verifyImportEvent verifies that one single event arrive on an import channel.
   152  func verifyImportEvent(t *testing.T, imported chan *types.Block) {
   153  	select {
   154  	case <-imported:
   155  	case <-time.After(time.Second):
   156  		t.Fatalf("import timeout")
   157  	}
   158  }
   159  
   160  // verifyImportCount verifies that exactly count number of events arrive on an
   161  // import hook channel.
   162  func verifyImportCount(t *testing.T, imported chan *types.Block, count int) {
   163  	for i := 0; i < count; i++ {
   164  		select {
   165  		case <-imported:
   166  		case <-time.After(time.Second):
   167  			t.Fatalf("block %d: import timeout", i)
   168  		}
   169  	}
   170  	verifyImportDone(t, imported)
   171  }
   172  
   173  // verifyImportDone verifies that no more events are arriving on an import channel.
   174  func verifyImportDone(t *testing.T, imported chan *types.Block) {
   175  	select {
   176  	case <-imported:
   177  		t.Fatalf("extra block imported")
   178  	case <-time.After(50 * time.Millisecond):
   179  	}
   180  }
   181  
   182  // Tests that a fetcher accepts block announcements and initiates retrievals for
   183  // them, successfully importing into the local chain.
   184  func TestSequentialAnnouncements(t *testing.T) {
   185  	// Create a chain of blocks to import
   186  	targetBlocks := 4 * hashLimit
   187  	hashes, blocks := makeChain(targetBlocks, 0, genesis)
   188  
   189  	tester := newTester()
   190  	fetcher := tester.makeFetcher(blocks)
   191  
   192  	// Iteratively announce blocks until all are imported
   193  	imported := make(chan *types.Block)
   194  	tester.fetcher.importedHook = func(block *types.Block) { imported <- block }
   195  
   196  	for i := len(hashes) - 2; i >= 0; i-- {
   197  		tester.fetcher.Notify("valid", hashes[i], time.Now().Add(-arriveTimeout), fetcher)
   198  		verifyImportEvent(t, imported)
   199  	}
   200  	verifyImportDone(t, imported)
   201  }
   202  
   203  // Tests that if blocks are announced by multiple peers (or even the same buggy
   204  // peer), they will only get downloaded at most once.
   205  func TestConcurrentAnnouncements(t *testing.T) {
   206  	// Create a chain of blocks to import
   207  	targetBlocks := 4 * hashLimit
   208  	hashes, blocks := makeChain(targetBlocks, 0, genesis)
   209  
   210  	// Assemble a tester with a built in counter for the requests
   211  	tester := newTester()
   212  	fetcher := tester.makeFetcher(blocks)
   213  
   214  	counter := uint32(0)
   215  	wrapper := func(hashes []common.Hash) error {
   216  		atomic.AddUint32(&counter, uint32(len(hashes)))
   217  		return fetcher(hashes)
   218  	}
   219  	// Iteratively announce blocks until all are imported
   220  	imported := make(chan *types.Block)
   221  	tester.fetcher.importedHook = func(block *types.Block) { imported <- block }
   222  
   223  	for i := len(hashes) - 2; i >= 0; i-- {
   224  		tester.fetcher.Notify("first", hashes[i], time.Now().Add(-arriveTimeout), wrapper)
   225  		tester.fetcher.Notify("second", hashes[i], time.Now().Add(-arriveTimeout+time.Millisecond), wrapper)
   226  		tester.fetcher.Notify("second", hashes[i], time.Now().Add(-arriveTimeout-time.Millisecond), wrapper)
   227  
   228  		verifyImportEvent(t, imported)
   229  	}
   230  	verifyImportDone(t, imported)
   231  
   232  	// Make sure no blocks were retrieved twice
   233  	if int(counter) != targetBlocks {
   234  		t.Fatalf("retrieval count mismatch: have %v, want %v", counter, targetBlocks)
   235  	}
   236  }
   237  
   238  // Tests that announcements arriving while a previous is being fetched still
   239  // results in a valid import.
   240  func TestOverlappingAnnouncements(t *testing.T) {
   241  	// Create a chain of blocks to import
   242  	targetBlocks := 4 * hashLimit
   243  	hashes, blocks := makeChain(targetBlocks, 0, genesis)
   244  
   245  	tester := newTester()
   246  	fetcher := tester.makeFetcher(blocks)
   247  
   248  	// Iteratively announce blocks, but overlap them continuously
   249  	fetching := make(chan []common.Hash)
   250  	imported := make(chan *types.Block, len(hashes)-1)
   251  	tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- hashes }
   252  	tester.fetcher.importedHook = func(block *types.Block) { imported <- block }
   253  
   254  	for i := len(hashes) - 2; i >= 0; i-- {
   255  		tester.fetcher.Notify("valid", hashes[i], time.Now().Add(-arriveTimeout), fetcher)
   256  		select {
   257  		case <-fetching:
   258  		case <-time.After(time.Second):
   259  			t.Fatalf("hash %d: announce timeout", len(hashes)-i)
   260  		}
   261  	}
   262  	// Wait for all the imports to complete and check count
   263  	verifyImportCount(t, imported, len(hashes)-1)
   264  }
   265  
   266  // Tests that announces already being retrieved will not be duplicated.
   267  func TestPendingDeduplication(t *testing.T) {
   268  	// Create a hash and corresponding block
   269  	hashes, blocks := makeChain(1, 0, genesis)
   270  
   271  	// Assemble a tester with a built in counter and delayed fetcher
   272  	tester := newTester()
   273  	fetcher := tester.makeFetcher(blocks)
   274  
   275  	delay := 50 * time.Millisecond
   276  	counter := uint32(0)
   277  	wrapper := func(hashes []common.Hash) error {
   278  		atomic.AddUint32(&counter, uint32(len(hashes)))
   279  
   280  		// Simulate a long running fetch
   281  		go func() {
   282  			time.Sleep(delay)
   283  			fetcher(hashes)
   284  		}()
   285  		return nil
   286  	}
   287  	// Announce the same block many times until it's fetched (wait for any pending ops)
   288  	for tester.getBlock(hashes[0]) == nil {
   289  		tester.fetcher.Notify("repeater", hashes[0], time.Now().Add(-arriveTimeout), wrapper)
   290  		time.Sleep(time.Millisecond)
   291  	}
   292  	time.Sleep(delay)
   293  
   294  	// Check that all blocks were imported and none fetched twice
   295  	if imported := len(tester.blocks); imported != 2 {
   296  		t.Fatalf("synchronised block mismatch: have %v, want %v", imported, 2)
   297  	}
   298  	if int(counter) != 1 {
   299  		t.Fatalf("retrieval count mismatch: have %v, want %v", counter, 1)
   300  	}
   301  }
   302  
   303  // Tests that announcements retrieved in a random order are cached and eventually
   304  // imported when all the gaps are filled in.
   305  func TestRandomArrivalImport(t *testing.T) {
   306  	// Create a chain of blocks to import, and choose one to delay
   307  	targetBlocks := maxQueueDist
   308  	hashes, blocks := makeChain(targetBlocks, 0, genesis)
   309  	skip := targetBlocks / 2
   310  
   311  	tester := newTester()
   312  	fetcher := tester.makeFetcher(blocks)
   313  
   314  	// Iteratively announce blocks, skipping one entry
   315  	imported := make(chan *types.Block, len(hashes)-1)
   316  	tester.fetcher.importedHook = func(block *types.Block) { imported <- block }
   317  
   318  	for i := len(hashes) - 1; i >= 0; i-- {
   319  		if i != skip {
   320  			tester.fetcher.Notify("valid", hashes[i], time.Now().Add(-arriveTimeout), fetcher)
   321  			time.Sleep(time.Millisecond)
   322  		}
   323  	}
   324  	// Finally announce the skipped entry and check full import
   325  	tester.fetcher.Notify("valid", hashes[skip], time.Now().Add(-arriveTimeout), fetcher)
   326  	verifyImportCount(t, imported, len(hashes)-1)
   327  }
   328  
   329  // Tests that direct block enqueues (due to block propagation vs. hash announce)
   330  // are correctly schedule, filling and import queue gaps.
   331  func TestQueueGapFill(t *testing.T) {
   332  	// Create a chain of blocks to import, and choose one to not announce at all
   333  	targetBlocks := maxQueueDist
   334  	hashes, blocks := makeChain(targetBlocks, 0, genesis)
   335  	skip := targetBlocks / 2
   336  
   337  	tester := newTester()
   338  	fetcher := tester.makeFetcher(blocks)
   339  
   340  	// Iteratively announce blocks, skipping one entry
   341  	imported := make(chan *types.Block, len(hashes)-1)
   342  	tester.fetcher.importedHook = func(block *types.Block) { imported <- block }
   343  
   344  	for i := len(hashes) - 1; i >= 0; i-- {
   345  		if i != skip {
   346  			tester.fetcher.Notify("valid", hashes[i], time.Now().Add(-arriveTimeout), fetcher)
   347  			time.Sleep(time.Millisecond)
   348  		}
   349  	}
   350  	// Fill the missing block directly as if propagated
   351  	tester.fetcher.Enqueue("valid", blocks[hashes[skip]])
   352  	verifyImportCount(t, imported, len(hashes)-1)
   353  }
   354  
   355  // Tests that blocks arriving from various sources (multiple propagations, hash
   356  // announces, etc) do not get scheduled for import multiple times.
   357  func TestImportDeduplication(t *testing.T) {
   358  	// Create two blocks to import (one for duplication, the other for stalling)
   359  	hashes, blocks := makeChain(2, 0, genesis)
   360  
   361  	// Create the tester and wrap the importer with a counter
   362  	tester := newTester()
   363  	fetcher := tester.makeFetcher(blocks)
   364  
   365  	counter := uint32(0)
   366  	tester.fetcher.insertChain = func(blocks types.Blocks) (int, error) {
   367  		atomic.AddUint32(&counter, uint32(len(blocks)))
   368  		return tester.insertChain(blocks)
   369  	}
   370  	// Instrument the fetching and imported events
   371  	fetching := make(chan []common.Hash)
   372  	imported := make(chan *types.Block, len(hashes)-1)
   373  	tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- hashes }
   374  	tester.fetcher.importedHook = func(block *types.Block) { imported <- block }
   375  
   376  	// Announce the duplicating block, wait for retrieval, and also propagate directly
   377  	tester.fetcher.Notify("valid", hashes[0], time.Now().Add(-arriveTimeout), fetcher)
   378  	<-fetching
   379  
   380  	tester.fetcher.Enqueue("valid", blocks[hashes[0]])
   381  	tester.fetcher.Enqueue("valid", blocks[hashes[0]])
   382  	tester.fetcher.Enqueue("valid", blocks[hashes[0]])
   383  
   384  	// Fill the missing block directly as if propagated, and check import uniqueness
   385  	tester.fetcher.Enqueue("valid", blocks[hashes[1]])
   386  	verifyImportCount(t, imported, 2)
   387  
   388  	if counter != 2 {
   389  		t.Fatalf("import invocation count mismatch: have %v, want %v", counter, 2)
   390  	}
   391  }
   392  
   393  // Tests that blocks with numbers much lower or higher than out current head get
   394  // discarded no prevent wasting resources on useless blocks from faulty peers.
   395  func TestDistantDiscarding(t *testing.T) {
   396  	// Create a long chain to import
   397  	hashes, blocks := makeChain(3*maxQueueDist, 0, genesis)
   398  	head := hashes[len(hashes)/2]
   399  
   400  	// Create a tester and simulate a head block being the middle of the above chain
   401  	tester := newTester()
   402  	tester.hashes = []common.Hash{head}
   403  	tester.blocks = map[common.Hash]*types.Block{head: blocks[head]}
   404  
   405  	// Ensure that a block with a lower number than the threshold is discarded
   406  	tester.fetcher.Enqueue("lower", blocks[hashes[0]])
   407  	time.Sleep(10 * time.Millisecond)
   408  	if !tester.fetcher.queue.Empty() {
   409  		t.Fatalf("fetcher queued stale block")
   410  	}
   411  	// Ensure that a block with a higher number than the threshold is discarded
   412  	tester.fetcher.Enqueue("higher", blocks[hashes[len(hashes)-1]])
   413  	time.Sleep(10 * time.Millisecond)
   414  	if !tester.fetcher.queue.Empty() {
   415  		t.Fatalf("fetcher queued future block")
   416  	}
   417  }
   418  
   419  // Tests that a peer is unable to use unbounded memory with sending infinite
   420  // block announcements to a node, but that even in the face of such an attack,
   421  // the fetcher remains operational.
   422  func TestHashMemoryExhaustionAttack(t *testing.T) {
   423  	// Create a tester with instrumented import hooks
   424  	tester := newTester()
   425  
   426  	imported := make(chan *types.Block)
   427  	tester.fetcher.importedHook = func(block *types.Block) { imported <- block }
   428  
   429  	// Create a valid chain and an infinite junk chain
   430  	targetBlocks := hashLimit + 2*maxQueueDist
   431  	hashes, blocks := makeChain(targetBlocks, 0, genesis)
   432  	valid := tester.makeFetcher(blocks)
   433  
   434  	attack, _ := makeChain(targetBlocks, 0, unknownBlock)
   435  	attacker := tester.makeFetcher(nil)
   436  
   437  	// Feed the tester a huge hashset from the attacker, and a limited from the valid peer
   438  	for i := 0; i < len(attack); i++ {
   439  		if i < maxQueueDist {
   440  			tester.fetcher.Notify("valid", hashes[len(hashes)-2-i], time.Now(), valid)
   441  		}
   442  		tester.fetcher.Notify("attacker", attack[i], time.Now(), attacker)
   443  	}
   444  	if len(tester.fetcher.announced) != hashLimit+maxQueueDist {
   445  		t.Fatalf("queued announce count mismatch: have %d, want %d", len(tester.fetcher.announced), hashLimit+maxQueueDist)
   446  	}
   447  	// Wait for fetches to complete
   448  	verifyImportCount(t, imported, maxQueueDist)
   449  
   450  	// Feed the remaining valid hashes to ensure DOS protection state remains clean
   451  	for i := len(hashes) - maxQueueDist - 2; i >= 0; i-- {
   452  		tester.fetcher.Notify("valid", hashes[i], time.Now().Add(-arriveTimeout), valid)
   453  		verifyImportEvent(t, imported)
   454  	}
   455  	verifyImportDone(t, imported)
   456  }
   457  
   458  // Tests that blocks sent to the fetcher (either through propagation or via hash
   459  // announces and retrievals) don't pile up indefinitely, exhausting available
   460  // system memory.
   461  func TestBlockMemoryExhaustionAttack(t *testing.T) {
   462  	// Create a tester with instrumented import hooks
   463  	tester := newTester()
   464  
   465  	imported := make(chan *types.Block)
   466  	tester.fetcher.importedHook = func(block *types.Block) { imported <- block }
   467  
   468  	// Create a valid chain and a batch of dangling (but in range) blocks
   469  	targetBlocks := hashLimit + 2*maxQueueDist
   470  	hashes, blocks := makeChain(targetBlocks, 0, genesis)
   471  	attack := make(map[common.Hash]*types.Block)
   472  	for i := byte(0); len(attack) < blockLimit+2*maxQueueDist; i++ {
   473  		hashes, blocks := makeChain(maxQueueDist-1, i, unknownBlock)
   474  		for _, hash := range hashes[:maxQueueDist-2] {
   475  			attack[hash] = blocks[hash]
   476  		}
   477  	}
   478  	// Try to feed all the attacker blocks make sure only a limited batch is accepted
   479  	for _, block := range attack {
   480  		tester.fetcher.Enqueue("attacker", block)
   481  	}
   482  	time.Sleep(200 * time.Millisecond)
   483  	if queued := tester.fetcher.queue.Size(); queued != blockLimit {
   484  		t.Fatalf("queued block count mismatch: have %d, want %d", queued, blockLimit)
   485  	}
   486  	// Queue up a batch of valid blocks, and check that a new peer is allowed to do so
   487  	for i := 0; i < maxQueueDist-1; i++ {
   488  		tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-3-i]])
   489  	}
   490  	time.Sleep(100 * time.Millisecond)
   491  	if queued := tester.fetcher.queue.Size(); queued != blockLimit+maxQueueDist-1 {
   492  		t.Fatalf("queued block count mismatch: have %d, want %d", queued, blockLimit+maxQueueDist-1)
   493  	}
   494  	// Insert the missing piece (and sanity check the import)
   495  	tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-2]])
   496  	verifyImportCount(t, imported, maxQueueDist)
   497  
   498  	// Insert the remaining blocks in chunks to ensure clean DOS protection
   499  	for i := maxQueueDist; i < len(hashes)-1; i++ {
   500  		tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-2-i]])
   501  		verifyImportEvent(t, imported)
   502  	}
   503  	verifyImportDone(t, imported)
   504  }