github.com/Cleverse/go-ethereum@v0.0.0-20220927095127-45113064e7f2/les/downloader/queue_test.go (about)

     1  // Copyright 2015 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package downloader
    18  
    19  import (
    20  	"fmt"
    21  	"math/big"
    22  	"math/rand"
    23  	"sync"
    24  	"testing"
    25  	"time"
    26  
    27  	"github.com/ethereum/go-ethereum/common"
    28  	"github.com/ethereum/go-ethereum/consensus/ethash"
    29  	"github.com/ethereum/go-ethereum/core"
    30  	"github.com/ethereum/go-ethereum/core/rawdb"
    31  	"github.com/ethereum/go-ethereum/core/types"
    32  	"github.com/ethereum/go-ethereum/log"
    33  	"github.com/ethereum/go-ethereum/params"
    34  )
    35  
    36  var (
    37  	testdb  = rawdb.NewMemoryDatabase()
    38  	genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000000000))
    39  )
    40  
    41  // makeChain creates a chain of n blocks starting at and including parent.
    42  // the returned hash chain is ordered head->parent. In addition, every 3rd block
    43  // contains a transaction and every 5th an uncle to allow testing correct block
    44  // reassembly.
    45  func makeChain(n int, seed byte, parent *types.Block, empty bool) ([]*types.Block, []types.Receipts) {
    46  	blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, func(i int, block *core.BlockGen) {
    47  		block.SetCoinbase(common.Address{seed})
    48  		// Add one tx to every secondblock
    49  		if !empty && i%2 == 0 {
    50  			signer := types.MakeSigner(params.TestChainConfig, block.Number())
    51  			tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, block.BaseFee(), nil), signer, testKey)
    52  			if err != nil {
    53  				panic(err)
    54  			}
    55  			block.AddTx(tx)
    56  		}
    57  	})
    58  	return blocks, receipts
    59  }
    60  
    61  type chainData struct {
    62  	blocks []*types.Block
    63  	offset int
    64  }
    65  
    66  var chain *chainData
    67  var emptyChain *chainData
    68  
    69  func init() {
    70  	// Create a chain of blocks to import
    71  	targetBlocks := 128
    72  	blocks, _ := makeChain(targetBlocks, 0, genesis, false)
    73  	chain = &chainData{blocks, 0}
    74  
    75  	blocks, _ = makeChain(targetBlocks, 0, genesis, true)
    76  	emptyChain = &chainData{blocks, 0}
    77  }
    78  
    79  func (chain *chainData) headers() []*types.Header {
    80  	hdrs := make([]*types.Header, len(chain.blocks))
    81  	for i, b := range chain.blocks {
    82  		hdrs[i] = b.Header()
    83  	}
    84  	return hdrs
    85  }
    86  
    87  func (chain *chainData) Len() int {
    88  	return len(chain.blocks)
    89  }
    90  
    91  func dummyPeer(id string) *peerConnection {
    92  	p := &peerConnection{
    93  		id:      id,
    94  		lacking: make(map[common.Hash]struct{}),
    95  	}
    96  	return p
    97  }
    98  
    99  func TestBasics(t *testing.T) {
   100  	numOfBlocks := len(emptyChain.blocks)
   101  	numOfReceipts := len(emptyChain.blocks) / 2
   102  
   103  	q := newQueue(10, 10)
   104  	if !q.Idle() {
   105  		t.Errorf("new queue should be idle")
   106  	}
   107  	q.Prepare(1, FastSync)
   108  	if res := q.Results(false); len(res) != 0 {
   109  		t.Fatal("new queue should have 0 results")
   110  	}
   111  
   112  	// Schedule a batch of headers
   113  	q.Schedule(chain.headers(), 1)
   114  	if q.Idle() {
   115  		t.Errorf("queue should not be idle")
   116  	}
   117  	if got, exp := q.PendingBlocks(), chain.Len(); got != exp {
   118  		t.Errorf("wrong pending block count, got %d, exp %d", got, exp)
   119  	}
   120  	// Only non-empty receipts get added to task-queue
   121  	if got, exp := q.PendingReceipts(), 64; got != exp {
   122  		t.Errorf("wrong pending receipt count, got %d, exp %d", got, exp)
   123  	}
   124  	// Items are now queued for downloading, next step is that we tell the
   125  	// queue that a certain peer will deliver them for us
   126  	{
   127  		peer := dummyPeer("peer-1")
   128  		fetchReq, _, throttle := q.ReserveBodies(peer, 50)
   129  		if !throttle {
   130  			// queue size is only 10, so throttling should occur
   131  			t.Fatal("should throttle")
   132  		}
   133  		// But we should still get the first things to fetch
   134  		if got, exp := len(fetchReq.Headers), 5; got != exp {
   135  			t.Fatalf("expected %d requests, got %d", exp, got)
   136  		}
   137  		if got, exp := fetchReq.Headers[0].Number.Uint64(), uint64(1); got != exp {
   138  			t.Fatalf("expected header %d, got %d", exp, got)
   139  		}
   140  	}
   141  	if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got {
   142  		t.Errorf("expected block task queue to be %d, got %d", exp, got)
   143  	}
   144  	if exp, got := q.receiptTaskQueue.Size(), numOfReceipts; exp != got {
   145  		t.Errorf("expected receipt task queue to be %d, got %d", exp, got)
   146  	}
   147  	{
   148  		peer := dummyPeer("peer-2")
   149  		fetchReq, _, throttle := q.ReserveBodies(peer, 50)
   150  
   151  		// The second peer should hit throttling
   152  		if !throttle {
   153  			t.Fatalf("should not throttle")
   154  		}
   155  		// And not get any fetches at all, since it was throttled to begin with
   156  		if fetchReq != nil {
   157  			t.Fatalf("should have no fetches, got %d", len(fetchReq.Headers))
   158  		}
   159  	}
   160  	if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got {
   161  		t.Errorf("expected block task queue to be %d, got %d", exp, got)
   162  	}
   163  	if exp, got := q.receiptTaskQueue.Size(), numOfReceipts; exp != got {
   164  		t.Errorf("expected receipt task queue to be %d, got %d", exp, got)
   165  	}
   166  	{
   167  		// The receipt delivering peer should not be affected
   168  		// by the throttling of body deliveries
   169  		peer := dummyPeer("peer-3")
   170  		fetchReq, _, throttle := q.ReserveReceipts(peer, 50)
   171  		if !throttle {
   172  			// queue size is only 10, so throttling should occur
   173  			t.Fatal("should throttle")
   174  		}
   175  		// But we should still get the first things to fetch
   176  		if got, exp := len(fetchReq.Headers), 5; got != exp {
   177  			t.Fatalf("expected %d requests, got %d", exp, got)
   178  		}
   179  		if got, exp := fetchReq.Headers[0].Number.Uint64(), uint64(1); got != exp {
   180  			t.Fatalf("expected header %d, got %d", exp, got)
   181  		}
   182  	}
   183  	if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got {
   184  		t.Errorf("expected block task queue to be %d, got %d", exp, got)
   185  	}
   186  	if exp, got := q.receiptTaskQueue.Size(), numOfReceipts-5; exp != got {
   187  		t.Errorf("expected receipt task queue to be %d, got %d", exp, got)
   188  	}
   189  	if got, exp := q.resultCache.countCompleted(), 0; got != exp {
   190  		t.Errorf("wrong processable count, got %d, exp %d", got, exp)
   191  	}
   192  }
   193  
   194  func TestEmptyBlocks(t *testing.T) {
   195  	numOfBlocks := len(emptyChain.blocks)
   196  
   197  	q := newQueue(10, 10)
   198  
   199  	q.Prepare(1, FastSync)
   200  	// Schedule a batch of headers
   201  	q.Schedule(emptyChain.headers(), 1)
   202  	if q.Idle() {
   203  		t.Errorf("queue should not be idle")
   204  	}
   205  	if got, exp := q.PendingBlocks(), len(emptyChain.blocks); got != exp {
   206  		t.Errorf("wrong pending block count, got %d, exp %d", got, exp)
   207  	}
   208  	if got, exp := q.PendingReceipts(), 0; got != exp {
   209  		t.Errorf("wrong pending receipt count, got %d, exp %d", got, exp)
   210  	}
   211  	// They won't be processable, because the fetchresults haven't been
   212  	// created yet
   213  	if got, exp := q.resultCache.countCompleted(), 0; got != exp {
   214  		t.Errorf("wrong processable count, got %d, exp %d", got, exp)
   215  	}
   216  
   217  	// Items are now queued for downloading, next step is that we tell the
   218  	// queue that a certain peer will deliver them for us
   219  	// That should trigger all of them to suddenly become 'done'
   220  	{
   221  		// Reserve blocks
   222  		peer := dummyPeer("peer-1")
   223  		fetchReq, _, _ := q.ReserveBodies(peer, 50)
   224  
   225  		// there should be nothing to fetch, blocks are empty
   226  		if fetchReq != nil {
   227  			t.Fatal("there should be no body fetch tasks remaining")
   228  		}
   229  	}
   230  	if q.blockTaskQueue.Size() != numOfBlocks-10 {
   231  		t.Errorf("expected block task queue to be %d, got %d", numOfBlocks-10, q.blockTaskQueue.Size())
   232  	}
   233  	if q.receiptTaskQueue.Size() != 0 {
   234  		t.Errorf("expected receipt task queue to be %d, got %d", 0, q.receiptTaskQueue.Size())
   235  	}
   236  	{
   237  		peer := dummyPeer("peer-3")
   238  		fetchReq, _, _ := q.ReserveReceipts(peer, 50)
   239  
   240  		// there should be nothing to fetch, blocks are empty
   241  		if fetchReq != nil {
   242  			t.Fatal("there should be no body fetch tasks remaining")
   243  		}
   244  	}
   245  	if q.blockTaskQueue.Size() != numOfBlocks-10 {
   246  		t.Errorf("expected block task queue to be %d, got %d", numOfBlocks-10, q.blockTaskQueue.Size())
   247  	}
   248  	if q.receiptTaskQueue.Size() != 0 {
   249  		t.Errorf("expected receipt task queue to be %d, got %d", 0, q.receiptTaskQueue.Size())
   250  	}
   251  	if got, exp := q.resultCache.countCompleted(), 10; got != exp {
   252  		t.Errorf("wrong processable count, got %d, exp %d", got, exp)
   253  	}
   254  }
   255  
   256  // XTestDelivery does some more extensive testing of events that happen,
   257  // blocks that become known and peers that make reservations and deliveries.
   258  // disabled since it's not really a unit-test, but can be executed to test
   259  // some more advanced scenarios
   260  func XTestDelivery(t *testing.T) {
   261  	// the outside network, holding blocks
   262  	blo, rec := makeChain(128, 0, genesis, false)
   263  	world := newNetwork()
   264  	world.receipts = rec
   265  	world.chain = blo
   266  	world.progress(10)
   267  	if false {
   268  		log.Root().SetHandler(log.StdoutHandler)
   269  	}
   270  	q := newQueue(10, 10)
   271  	var wg sync.WaitGroup
   272  	q.Prepare(1, FastSync)
   273  	wg.Add(1)
   274  	go func() {
   275  		// deliver headers
   276  		defer wg.Done()
   277  		c := 1
   278  		for {
   279  			//fmt.Printf("getting headers from %d\n", c)
   280  			hdrs := world.headers(c)
   281  			l := len(hdrs)
   282  			//fmt.Printf("scheduling %d headers, first %d last %d\n",
   283  			//	l, hdrs[0].Number.Uint64(), hdrs[len(hdrs)-1].Number.Uint64())
   284  			q.Schedule(hdrs, uint64(c))
   285  			c += l
   286  		}
   287  	}()
   288  	wg.Add(1)
   289  	go func() {
   290  		// collect results
   291  		defer wg.Done()
   292  		tot := 0
   293  		for {
   294  			res := q.Results(true)
   295  			tot += len(res)
   296  			fmt.Printf("got %d results, %d tot\n", len(res), tot)
   297  			// Now we can forget about these
   298  			world.forget(res[len(res)-1].Header.Number.Uint64())
   299  		}
   300  	}()
   301  	wg.Add(1)
   302  	go func() {
   303  		defer wg.Done()
   304  		// reserve body fetch
   305  		i := 4
   306  		for {
   307  			peer := dummyPeer(fmt.Sprintf("peer-%d", i))
   308  			f, _, _ := q.ReserveBodies(peer, rand.Intn(30))
   309  			if f != nil {
   310  				var emptyList []*types.Header
   311  				var txs [][]*types.Transaction
   312  				var uncles [][]*types.Header
   313  				numToSkip := rand.Intn(len(f.Headers))
   314  				for _, hdr := range f.Headers[0 : len(f.Headers)-numToSkip] {
   315  					txs = append(txs, world.getTransactions(hdr.Number.Uint64()))
   316  					uncles = append(uncles, emptyList)
   317  				}
   318  				time.Sleep(100 * time.Millisecond)
   319  				_, err := q.DeliverBodies(peer.id, txs, uncles)
   320  				if err != nil {
   321  					fmt.Printf("delivered %d bodies %v\n", len(txs), err)
   322  				}
   323  			} else {
   324  				i++
   325  				time.Sleep(200 * time.Millisecond)
   326  			}
   327  		}
   328  	}()
   329  	go func() {
   330  		defer wg.Done()
   331  		// reserve receiptfetch
   332  		peer := dummyPeer("peer-3")
   333  		for {
   334  			f, _, _ := q.ReserveReceipts(peer, rand.Intn(50))
   335  			if f != nil {
   336  				var rcs [][]*types.Receipt
   337  				for _, hdr := range f.Headers {
   338  					rcs = append(rcs, world.getReceipts(hdr.Number.Uint64()))
   339  				}
   340  				_, err := q.DeliverReceipts(peer.id, rcs)
   341  				if err != nil {
   342  					fmt.Printf("delivered %d receipts %v\n", len(rcs), err)
   343  				}
   344  				time.Sleep(100 * time.Millisecond)
   345  			} else {
   346  				time.Sleep(200 * time.Millisecond)
   347  			}
   348  		}
   349  	}()
   350  	wg.Add(1)
   351  	go func() {
   352  		defer wg.Done()
   353  		for i := 0; i < 50; i++ {
   354  			time.Sleep(300 * time.Millisecond)
   355  			//world.tick()
   356  			//fmt.Printf("trying to progress\n")
   357  			world.progress(rand.Intn(100))
   358  		}
   359  		for i := 0; i < 50; i++ {
   360  			time.Sleep(2990 * time.Millisecond)
   361  		}
   362  	}()
   363  	wg.Add(1)
   364  	go func() {
   365  		defer wg.Done()
   366  		for {
   367  			time.Sleep(990 * time.Millisecond)
   368  			fmt.Printf("world block tip is %d\n",
   369  				world.chain[len(world.chain)-1].Header().Number.Uint64())
   370  			fmt.Println(q.Stats())
   371  		}
   372  	}()
   373  	wg.Wait()
   374  }
   375  
   376  func newNetwork() *network {
   377  	var l sync.RWMutex
   378  	return &network{
   379  		cond:   sync.NewCond(&l),
   380  		offset: 1, // block 1 is at blocks[0]
   381  	}
   382  }
   383  
   384  // represents the network
   385  type network struct {
   386  	offset   int
   387  	chain    []*types.Block
   388  	receipts []types.Receipts
   389  	lock     sync.RWMutex
   390  	cond     *sync.Cond
   391  }
   392  
   393  func (n *network) getTransactions(blocknum uint64) types.Transactions {
   394  	index := blocknum - uint64(n.offset)
   395  	return n.chain[index].Transactions()
   396  }
   397  func (n *network) getReceipts(blocknum uint64) types.Receipts {
   398  	index := blocknum - uint64(n.offset)
   399  	if got := n.chain[index].Header().Number.Uint64(); got != blocknum {
   400  		fmt.Printf("Err, got %d exp %d\n", got, blocknum)
   401  		panic("sd")
   402  	}
   403  	return n.receipts[index]
   404  }
   405  
   406  func (n *network) forget(blocknum uint64) {
   407  	index := blocknum - uint64(n.offset)
   408  	n.chain = n.chain[index:]
   409  	n.receipts = n.receipts[index:]
   410  	n.offset = int(blocknum)
   411  }
   412  func (n *network) progress(numBlocks int) {
   413  	n.lock.Lock()
   414  	defer n.lock.Unlock()
   415  	//fmt.Printf("progressing...\n")
   416  	newBlocks, newR := makeChain(numBlocks, 0, n.chain[len(n.chain)-1], false)
   417  	n.chain = append(n.chain, newBlocks...)
   418  	n.receipts = append(n.receipts, newR...)
   419  	n.cond.Broadcast()
   420  }
   421  
   422  func (n *network) headers(from int) []*types.Header {
   423  	numHeaders := 128
   424  	var hdrs []*types.Header
   425  	index := from - n.offset
   426  
   427  	for index >= len(n.chain) {
   428  		// wait for progress
   429  		n.cond.L.Lock()
   430  		//fmt.Printf("header going into wait\n")
   431  		n.cond.Wait()
   432  		index = from - n.offset
   433  		n.cond.L.Unlock()
   434  	}
   435  	n.lock.RLock()
   436  	defer n.lock.RUnlock()
   437  	for i, b := range n.chain[index:] {
   438  		hdrs = append(hdrs, b.Header())
   439  		if i >= numHeaders {
   440  			break
   441  		}
   442  	}
   443  	return hdrs
   444  }