github.com/Cleverse/go-ethereum@v0.0.0-20220927095127-45113064e7f2/eth/downloader/queue_test.go (about) 1 // Copyright 2015 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package downloader 18 19 import ( 20 "fmt" 21 "math/big" 22 "math/rand" 23 "sync" 24 "testing" 25 "time" 26 27 "github.com/ethereum/go-ethereum/common" 28 "github.com/ethereum/go-ethereum/consensus/ethash" 29 "github.com/ethereum/go-ethereum/core" 30 "github.com/ethereum/go-ethereum/core/rawdb" 31 "github.com/ethereum/go-ethereum/core/types" 32 "github.com/ethereum/go-ethereum/log" 33 "github.com/ethereum/go-ethereum/params" 34 "github.com/ethereum/go-ethereum/trie" 35 ) 36 37 var ( 38 testdb = rawdb.NewMemoryDatabase() 39 genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000000000)) 40 ) 41 42 // makeChain creates a chain of n blocks starting at and including parent. 43 // the returned hash chain is ordered head->parent. In addition, every 3rd block 44 // contains a transaction and every 5th an uncle to allow testing correct block 45 // reassembly. 46 func makeChain(n int, seed byte, parent *types.Block, empty bool) ([]*types.Block, []types.Receipts) { 47 blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, func(i int, block *core.BlockGen) { 48 block.SetCoinbase(common.Address{seed}) 49 // Add one tx to every secondblock 50 if !empty && i%2 == 0 { 51 signer := types.MakeSigner(params.TestChainConfig, block.Number()) 52 tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, block.BaseFee(), nil), signer, testKey) 53 if err != nil { 54 panic(err) 55 } 56 block.AddTx(tx) 57 } 58 }) 59 return blocks, receipts 60 } 61 62 type chainData struct { 63 blocks []*types.Block 64 offset int 65 } 66 67 var chain *chainData 68 var emptyChain *chainData 69 70 func init() { 71 // Create a chain of blocks to import 72 targetBlocks := 128 73 blocks, _ := makeChain(targetBlocks, 0, genesis, false) 74 chain = &chainData{blocks, 0} 75 76 blocks, _ = makeChain(targetBlocks, 0, genesis, true) 77 emptyChain = &chainData{blocks, 0} 78 } 79 80 func (chain *chainData) headers() []*types.Header { 81 hdrs := make([]*types.Header, len(chain.blocks)) 82 for i, b := range chain.blocks { 83 hdrs[i] = b.Header() 84 } 85 return hdrs 86 } 87 88 func (chain *chainData) Len() int { 89 return len(chain.blocks) 90 } 91 92 func dummyPeer(id string) *peerConnection { 93 p := &peerConnection{ 94 id: id, 95 lacking: make(map[common.Hash]struct{}), 96 } 97 return p 98 } 99 100 func TestBasics(t *testing.T) { 101 numOfBlocks := len(emptyChain.blocks) 102 numOfReceipts := len(emptyChain.blocks) / 2 103 104 q := newQueue(10, 10) 105 if !q.Idle() { 106 t.Errorf("new queue should be idle") 107 } 108 q.Prepare(1, SnapSync) 109 if res := q.Results(false); len(res) != 0 { 110 t.Fatal("new queue should have 0 results") 111 } 112 113 // Schedule a batch of headers 114 headers := chain.headers() 115 hashes := make([]common.Hash, len(headers)) 116 for i, header := range headers { 117 hashes[i] = header.Hash() 118 } 119 q.Schedule(headers, hashes, 1) 120 if q.Idle() { 121 t.Errorf("queue should not be idle") 122 } 123 if got, exp := q.PendingBodies(), chain.Len(); got != exp { 124 t.Errorf("wrong pending block count, got %d, exp %d", got, exp) 125 } 126 // Only non-empty receipts get added to task-queue 127 if got, exp := q.PendingReceipts(), 64; got != exp { 128 t.Errorf("wrong pending receipt count, got %d, exp %d", got, exp) 129 } 130 // Items are now queued for downloading, next step is that we tell the 131 // queue that a certain peer will deliver them for us 132 { 133 peer := dummyPeer("peer-1") 134 fetchReq, _, throttle := q.ReserveBodies(peer, 50) 135 if !throttle { 136 // queue size is only 10, so throttling should occur 137 t.Fatal("should throttle") 138 } 139 // But we should still get the first things to fetch 140 if got, exp := len(fetchReq.Headers), 5; got != exp { 141 t.Fatalf("expected %d requests, got %d", exp, got) 142 } 143 if got, exp := fetchReq.Headers[0].Number.Uint64(), uint64(1); got != exp { 144 t.Fatalf("expected header %d, got %d", exp, got) 145 } 146 } 147 if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got { 148 t.Errorf("expected block task queue to be %d, got %d", exp, got) 149 } 150 if exp, got := q.receiptTaskQueue.Size(), numOfReceipts; exp != got { 151 t.Errorf("expected receipt task queue to be %d, got %d", exp, got) 152 } 153 { 154 peer := dummyPeer("peer-2") 155 fetchReq, _, throttle := q.ReserveBodies(peer, 50) 156 157 // The second peer should hit throttling 158 if !throttle { 159 t.Fatalf("should not throttle") 160 } 161 // And not get any fetches at all, since it was throttled to begin with 162 if fetchReq != nil { 163 t.Fatalf("should have no fetches, got %d", len(fetchReq.Headers)) 164 } 165 } 166 if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got { 167 t.Errorf("expected block task queue to be %d, got %d", exp, got) 168 } 169 if exp, got := q.receiptTaskQueue.Size(), numOfReceipts; exp != got { 170 t.Errorf("expected receipt task queue to be %d, got %d", exp, got) 171 } 172 { 173 // The receipt delivering peer should not be affected 174 // by the throttling of body deliveries 175 peer := dummyPeer("peer-3") 176 fetchReq, _, throttle := q.ReserveReceipts(peer, 50) 177 if !throttle { 178 // queue size is only 10, so throttling should occur 179 t.Fatal("should throttle") 180 } 181 // But we should still get the first things to fetch 182 if got, exp := len(fetchReq.Headers), 5; got != exp { 183 t.Fatalf("expected %d requests, got %d", exp, got) 184 } 185 if got, exp := fetchReq.Headers[0].Number.Uint64(), uint64(1); got != exp { 186 t.Fatalf("expected header %d, got %d", exp, got) 187 } 188 } 189 if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got { 190 t.Errorf("expected block task queue to be %d, got %d", exp, got) 191 } 192 if exp, got := q.receiptTaskQueue.Size(), numOfReceipts-5; exp != got { 193 t.Errorf("expected receipt task queue to be %d, got %d", exp, got) 194 } 195 if got, exp := q.resultCache.countCompleted(), 0; got != exp { 196 t.Errorf("wrong processable count, got %d, exp %d", got, exp) 197 } 198 } 199 200 func TestEmptyBlocks(t *testing.T) { 201 numOfBlocks := len(emptyChain.blocks) 202 203 q := newQueue(10, 10) 204 205 q.Prepare(1, SnapSync) 206 207 // Schedule a batch of headers 208 headers := emptyChain.headers() 209 hashes := make([]common.Hash, len(headers)) 210 for i, header := range headers { 211 hashes[i] = header.Hash() 212 } 213 q.Schedule(headers, hashes, 1) 214 if q.Idle() { 215 t.Errorf("queue should not be idle") 216 } 217 if got, exp := q.PendingBodies(), len(emptyChain.blocks); got != exp { 218 t.Errorf("wrong pending block count, got %d, exp %d", got, exp) 219 } 220 if got, exp := q.PendingReceipts(), 0; got != exp { 221 t.Errorf("wrong pending receipt count, got %d, exp %d", got, exp) 222 } 223 // They won't be processable, because the fetchresults haven't been 224 // created yet 225 if got, exp := q.resultCache.countCompleted(), 0; got != exp { 226 t.Errorf("wrong processable count, got %d, exp %d", got, exp) 227 } 228 229 // Items are now queued for downloading, next step is that we tell the 230 // queue that a certain peer will deliver them for us 231 // That should trigger all of them to suddenly become 'done' 232 { 233 // Reserve blocks 234 peer := dummyPeer("peer-1") 235 fetchReq, _, _ := q.ReserveBodies(peer, 50) 236 237 // there should be nothing to fetch, blocks are empty 238 if fetchReq != nil { 239 t.Fatal("there should be no body fetch tasks remaining") 240 } 241 } 242 if q.blockTaskQueue.Size() != numOfBlocks-10 { 243 t.Errorf("expected block task queue to be %d, got %d", numOfBlocks-10, q.blockTaskQueue.Size()) 244 } 245 if q.receiptTaskQueue.Size() != 0 { 246 t.Errorf("expected receipt task queue to be %d, got %d", 0, q.receiptTaskQueue.Size()) 247 } 248 { 249 peer := dummyPeer("peer-3") 250 fetchReq, _, _ := q.ReserveReceipts(peer, 50) 251 252 // there should be nothing to fetch, blocks are empty 253 if fetchReq != nil { 254 t.Fatal("there should be no body fetch tasks remaining") 255 } 256 } 257 if q.blockTaskQueue.Size() != numOfBlocks-10 { 258 t.Errorf("expected block task queue to be %d, got %d", numOfBlocks-10, q.blockTaskQueue.Size()) 259 } 260 if q.receiptTaskQueue.Size() != 0 { 261 t.Errorf("expected receipt task queue to be %d, got %d", 0, q.receiptTaskQueue.Size()) 262 } 263 if got, exp := q.resultCache.countCompleted(), 10; got != exp { 264 t.Errorf("wrong processable count, got %d, exp %d", got, exp) 265 } 266 } 267 268 // XTestDelivery does some more extensive testing of events that happen, 269 // blocks that become known and peers that make reservations and deliveries. 270 // disabled since it's not really a unit-test, but can be executed to test 271 // some more advanced scenarios 272 func XTestDelivery(t *testing.T) { 273 // the outside network, holding blocks 274 blo, rec := makeChain(128, 0, genesis, false) 275 world := newNetwork() 276 world.receipts = rec 277 world.chain = blo 278 world.progress(10) 279 if false { 280 log.Root().SetHandler(log.StdoutHandler) 281 } 282 q := newQueue(10, 10) 283 var wg sync.WaitGroup 284 q.Prepare(1, SnapSync) 285 wg.Add(1) 286 go func() { 287 // deliver headers 288 defer wg.Done() 289 c := 1 290 for { 291 //fmt.Printf("getting headers from %d\n", c) 292 headers := world.headers(c) 293 hashes := make([]common.Hash, len(headers)) 294 for i, header := range headers { 295 hashes[i] = header.Hash() 296 } 297 l := len(headers) 298 //fmt.Printf("scheduling %d headers, first %d last %d\n", 299 // l, headers[0].Number.Uint64(), headers[len(headers)-1].Number.Uint64()) 300 q.Schedule(headers, hashes, uint64(c)) 301 c += l 302 } 303 }() 304 wg.Add(1) 305 go func() { 306 // collect results 307 defer wg.Done() 308 tot := 0 309 for { 310 res := q.Results(true) 311 tot += len(res) 312 fmt.Printf("got %d results, %d tot\n", len(res), tot) 313 // Now we can forget about these 314 world.forget(res[len(res)-1].Header.Number.Uint64()) 315 } 316 }() 317 wg.Add(1) 318 go func() { 319 defer wg.Done() 320 // reserve body fetch 321 i := 4 322 for { 323 peer := dummyPeer(fmt.Sprintf("peer-%d", i)) 324 f, _, _ := q.ReserveBodies(peer, rand.Intn(30)) 325 if f != nil { 326 var ( 327 emptyList []*types.Header 328 txset [][]*types.Transaction 329 uncleset [][]*types.Header 330 ) 331 numToSkip := rand.Intn(len(f.Headers)) 332 for _, hdr := range f.Headers[0 : len(f.Headers)-numToSkip] { 333 txset = append(txset, world.getTransactions(hdr.Number.Uint64())) 334 uncleset = append(uncleset, emptyList) 335 } 336 var ( 337 txsHashes = make([]common.Hash, len(txset)) 338 uncleHashes = make([]common.Hash, len(uncleset)) 339 ) 340 hasher := trie.NewStackTrie(nil) 341 for i, txs := range txset { 342 txsHashes[i] = types.DeriveSha(types.Transactions(txs), hasher) 343 } 344 for i, uncles := range uncleset { 345 uncleHashes[i] = types.CalcUncleHash(uncles) 346 } 347 time.Sleep(100 * time.Millisecond) 348 _, err := q.DeliverBodies(peer.id, txset, txsHashes, uncleset, uncleHashes) 349 if err != nil { 350 fmt.Printf("delivered %d bodies %v\n", len(txset), err) 351 } 352 } else { 353 i++ 354 time.Sleep(200 * time.Millisecond) 355 } 356 } 357 }() 358 go func() { 359 defer wg.Done() 360 // reserve receiptfetch 361 peer := dummyPeer("peer-3") 362 for { 363 f, _, _ := q.ReserveReceipts(peer, rand.Intn(50)) 364 if f != nil { 365 var rcs [][]*types.Receipt 366 for _, hdr := range f.Headers { 367 rcs = append(rcs, world.getReceipts(hdr.Number.Uint64())) 368 } 369 hasher := trie.NewStackTrie(nil) 370 hashes := make([]common.Hash, len(rcs)) 371 for i, receipt := range rcs { 372 hashes[i] = types.DeriveSha(types.Receipts(receipt), hasher) 373 } 374 _, err := q.DeliverReceipts(peer.id, rcs, hashes) 375 if err != nil { 376 fmt.Printf("delivered %d receipts %v\n", len(rcs), err) 377 } 378 time.Sleep(100 * time.Millisecond) 379 } else { 380 time.Sleep(200 * time.Millisecond) 381 } 382 } 383 }() 384 wg.Add(1) 385 go func() { 386 defer wg.Done() 387 for i := 0; i < 50; i++ { 388 time.Sleep(300 * time.Millisecond) 389 //world.tick() 390 //fmt.Printf("trying to progress\n") 391 world.progress(rand.Intn(100)) 392 } 393 for i := 0; i < 50; i++ { 394 time.Sleep(2990 * time.Millisecond) 395 } 396 }() 397 wg.Add(1) 398 go func() { 399 defer wg.Done() 400 for { 401 time.Sleep(990 * time.Millisecond) 402 fmt.Printf("world block tip is %d\n", 403 world.chain[len(world.chain)-1].Header().Number.Uint64()) 404 fmt.Println(q.Stats()) 405 } 406 }() 407 wg.Wait() 408 } 409 410 func newNetwork() *network { 411 var l sync.RWMutex 412 return &network{ 413 cond: sync.NewCond(&l), 414 offset: 1, // block 1 is at blocks[0] 415 } 416 } 417 418 // represents the network 419 type network struct { 420 offset int 421 chain []*types.Block 422 receipts []types.Receipts 423 lock sync.RWMutex 424 cond *sync.Cond 425 } 426 427 func (n *network) getTransactions(blocknum uint64) types.Transactions { 428 index := blocknum - uint64(n.offset) 429 return n.chain[index].Transactions() 430 } 431 func (n *network) getReceipts(blocknum uint64) types.Receipts { 432 index := blocknum - uint64(n.offset) 433 if got := n.chain[index].Header().Number.Uint64(); got != blocknum { 434 fmt.Printf("Err, got %d exp %d\n", got, blocknum) 435 panic("sd") 436 } 437 return n.receipts[index] 438 } 439 440 func (n *network) forget(blocknum uint64) { 441 index := blocknum - uint64(n.offset) 442 n.chain = n.chain[index:] 443 n.receipts = n.receipts[index:] 444 n.offset = int(blocknum) 445 } 446 func (n *network) progress(numBlocks int) { 447 n.lock.Lock() 448 defer n.lock.Unlock() 449 //fmt.Printf("progressing...\n") 450 newBlocks, newR := makeChain(numBlocks, 0, n.chain[len(n.chain)-1], false) 451 n.chain = append(n.chain, newBlocks...) 452 n.receipts = append(n.receipts, newR...) 453 n.cond.Broadcast() 454 } 455 456 func (n *network) headers(from int) []*types.Header { 457 numHeaders := 128 458 var hdrs []*types.Header 459 index := from - n.offset 460 461 for index >= len(n.chain) { 462 // wait for progress 463 n.cond.L.Lock() 464 //fmt.Printf("header going into wait\n") 465 n.cond.Wait() 466 index = from - n.offset 467 n.cond.L.Unlock() 468 } 469 n.lock.RLock() 470 defer n.lock.RUnlock() 471 for i, b := range n.chain[index:] { 472 hdrs = append(hdrs, b.Header()) 473 if i >= numHeaders { 474 break 475 } 476 } 477 return hdrs 478 }