github.com/Cleverse/go-ethereum@v0.0.0-20220927095127-45113064e7f2/les/downloader/queue_test.go (about) 1 // Copyright 2015 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package downloader 18 19 import ( 20 "fmt" 21 "math/big" 22 "math/rand" 23 "sync" 24 "testing" 25 "time" 26 27 "github.com/ethereum/go-ethereum/common" 28 "github.com/ethereum/go-ethereum/consensus/ethash" 29 "github.com/ethereum/go-ethereum/core" 30 "github.com/ethereum/go-ethereum/core/rawdb" 31 "github.com/ethereum/go-ethereum/core/types" 32 "github.com/ethereum/go-ethereum/log" 33 "github.com/ethereum/go-ethereum/params" 34 ) 35 36 var ( 37 testdb = rawdb.NewMemoryDatabase() 38 genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000000000)) 39 ) 40 41 // makeChain creates a chain of n blocks starting at and including parent. 42 // the returned hash chain is ordered head->parent. In addition, every 3rd block 43 // contains a transaction and every 5th an uncle to allow testing correct block 44 // reassembly. 45 func makeChain(n int, seed byte, parent *types.Block, empty bool) ([]*types.Block, []types.Receipts) { 46 blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, func(i int, block *core.BlockGen) { 47 block.SetCoinbase(common.Address{seed}) 48 // Add one tx to every secondblock 49 if !empty && i%2 == 0 { 50 signer := types.MakeSigner(params.TestChainConfig, block.Number()) 51 tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, block.BaseFee(), nil), signer, testKey) 52 if err != nil { 53 panic(err) 54 } 55 block.AddTx(tx) 56 } 57 }) 58 return blocks, receipts 59 } 60 61 type chainData struct { 62 blocks []*types.Block 63 offset int 64 } 65 66 var chain *chainData 67 var emptyChain *chainData 68 69 func init() { 70 // Create a chain of blocks to import 71 targetBlocks := 128 72 blocks, _ := makeChain(targetBlocks, 0, genesis, false) 73 chain = &chainData{blocks, 0} 74 75 blocks, _ = makeChain(targetBlocks, 0, genesis, true) 76 emptyChain = &chainData{blocks, 0} 77 } 78 79 func (chain *chainData) headers() []*types.Header { 80 hdrs := make([]*types.Header, len(chain.blocks)) 81 for i, b := range chain.blocks { 82 hdrs[i] = b.Header() 83 } 84 return hdrs 85 } 86 87 func (chain *chainData) Len() int { 88 return len(chain.blocks) 89 } 90 91 func dummyPeer(id string) *peerConnection { 92 p := &peerConnection{ 93 id: id, 94 lacking: make(map[common.Hash]struct{}), 95 } 96 return p 97 } 98 99 func TestBasics(t *testing.T) { 100 numOfBlocks := len(emptyChain.blocks) 101 numOfReceipts := len(emptyChain.blocks) / 2 102 103 q := newQueue(10, 10) 104 if !q.Idle() { 105 t.Errorf("new queue should be idle") 106 } 107 q.Prepare(1, FastSync) 108 if res := q.Results(false); len(res) != 0 { 109 t.Fatal("new queue should have 0 results") 110 } 111 112 // Schedule a batch of headers 113 q.Schedule(chain.headers(), 1) 114 if q.Idle() { 115 t.Errorf("queue should not be idle") 116 } 117 if got, exp := q.PendingBlocks(), chain.Len(); got != exp { 118 t.Errorf("wrong pending block count, got %d, exp %d", got, exp) 119 } 120 // Only non-empty receipts get added to task-queue 121 if got, exp := q.PendingReceipts(), 64; got != exp { 122 t.Errorf("wrong pending receipt count, got %d, exp %d", got, exp) 123 } 124 // Items are now queued for downloading, next step is that we tell the 125 // queue that a certain peer will deliver them for us 126 { 127 peer := dummyPeer("peer-1") 128 fetchReq, _, throttle := q.ReserveBodies(peer, 50) 129 if !throttle { 130 // queue size is only 10, so throttling should occur 131 t.Fatal("should throttle") 132 } 133 // But we should still get the first things to fetch 134 if got, exp := len(fetchReq.Headers), 5; got != exp { 135 t.Fatalf("expected %d requests, got %d", exp, got) 136 } 137 if got, exp := fetchReq.Headers[0].Number.Uint64(), uint64(1); got != exp { 138 t.Fatalf("expected header %d, got %d", exp, got) 139 } 140 } 141 if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got { 142 t.Errorf("expected block task queue to be %d, got %d", exp, got) 143 } 144 if exp, got := q.receiptTaskQueue.Size(), numOfReceipts; exp != got { 145 t.Errorf("expected receipt task queue to be %d, got %d", exp, got) 146 } 147 { 148 peer := dummyPeer("peer-2") 149 fetchReq, _, throttle := q.ReserveBodies(peer, 50) 150 151 // The second peer should hit throttling 152 if !throttle { 153 t.Fatalf("should not throttle") 154 } 155 // And not get any fetches at all, since it was throttled to begin with 156 if fetchReq != nil { 157 t.Fatalf("should have no fetches, got %d", len(fetchReq.Headers)) 158 } 159 } 160 if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got { 161 t.Errorf("expected block task queue to be %d, got %d", exp, got) 162 } 163 if exp, got := q.receiptTaskQueue.Size(), numOfReceipts; exp != got { 164 t.Errorf("expected receipt task queue to be %d, got %d", exp, got) 165 } 166 { 167 // The receipt delivering peer should not be affected 168 // by the throttling of body deliveries 169 peer := dummyPeer("peer-3") 170 fetchReq, _, throttle := q.ReserveReceipts(peer, 50) 171 if !throttle { 172 // queue size is only 10, so throttling should occur 173 t.Fatal("should throttle") 174 } 175 // But we should still get the first things to fetch 176 if got, exp := len(fetchReq.Headers), 5; got != exp { 177 t.Fatalf("expected %d requests, got %d", exp, got) 178 } 179 if got, exp := fetchReq.Headers[0].Number.Uint64(), uint64(1); got != exp { 180 t.Fatalf("expected header %d, got %d", exp, got) 181 } 182 } 183 if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got { 184 t.Errorf("expected block task queue to be %d, got %d", exp, got) 185 } 186 if exp, got := q.receiptTaskQueue.Size(), numOfReceipts-5; exp != got { 187 t.Errorf("expected receipt task queue to be %d, got %d", exp, got) 188 } 189 if got, exp := q.resultCache.countCompleted(), 0; got != exp { 190 t.Errorf("wrong processable count, got %d, exp %d", got, exp) 191 } 192 } 193 194 func TestEmptyBlocks(t *testing.T) { 195 numOfBlocks := len(emptyChain.blocks) 196 197 q := newQueue(10, 10) 198 199 q.Prepare(1, FastSync) 200 // Schedule a batch of headers 201 q.Schedule(emptyChain.headers(), 1) 202 if q.Idle() { 203 t.Errorf("queue should not be idle") 204 } 205 if got, exp := q.PendingBlocks(), len(emptyChain.blocks); got != exp { 206 t.Errorf("wrong pending block count, got %d, exp %d", got, exp) 207 } 208 if got, exp := q.PendingReceipts(), 0; got != exp { 209 t.Errorf("wrong pending receipt count, got %d, exp %d", got, exp) 210 } 211 // They won't be processable, because the fetchresults haven't been 212 // created yet 213 if got, exp := q.resultCache.countCompleted(), 0; got != exp { 214 t.Errorf("wrong processable count, got %d, exp %d", got, exp) 215 } 216 217 // Items are now queued for downloading, next step is that we tell the 218 // queue that a certain peer will deliver them for us 219 // That should trigger all of them to suddenly become 'done' 220 { 221 // Reserve blocks 222 peer := dummyPeer("peer-1") 223 fetchReq, _, _ := q.ReserveBodies(peer, 50) 224 225 // there should be nothing to fetch, blocks are empty 226 if fetchReq != nil { 227 t.Fatal("there should be no body fetch tasks remaining") 228 } 229 } 230 if q.blockTaskQueue.Size() != numOfBlocks-10 { 231 t.Errorf("expected block task queue to be %d, got %d", numOfBlocks-10, q.blockTaskQueue.Size()) 232 } 233 if q.receiptTaskQueue.Size() != 0 { 234 t.Errorf("expected receipt task queue to be %d, got %d", 0, q.receiptTaskQueue.Size()) 235 } 236 { 237 peer := dummyPeer("peer-3") 238 fetchReq, _, _ := q.ReserveReceipts(peer, 50) 239 240 // there should be nothing to fetch, blocks are empty 241 if fetchReq != nil { 242 t.Fatal("there should be no body fetch tasks remaining") 243 } 244 } 245 if q.blockTaskQueue.Size() != numOfBlocks-10 { 246 t.Errorf("expected block task queue to be %d, got %d", numOfBlocks-10, q.blockTaskQueue.Size()) 247 } 248 if q.receiptTaskQueue.Size() != 0 { 249 t.Errorf("expected receipt task queue to be %d, got %d", 0, q.receiptTaskQueue.Size()) 250 } 251 if got, exp := q.resultCache.countCompleted(), 10; got != exp { 252 t.Errorf("wrong processable count, got %d, exp %d", got, exp) 253 } 254 } 255 256 // XTestDelivery does some more extensive testing of events that happen, 257 // blocks that become known and peers that make reservations and deliveries. 258 // disabled since it's not really a unit-test, but can be executed to test 259 // some more advanced scenarios 260 func XTestDelivery(t *testing.T) { 261 // the outside network, holding blocks 262 blo, rec := makeChain(128, 0, genesis, false) 263 world := newNetwork() 264 world.receipts = rec 265 world.chain = blo 266 world.progress(10) 267 if false { 268 log.Root().SetHandler(log.StdoutHandler) 269 } 270 q := newQueue(10, 10) 271 var wg sync.WaitGroup 272 q.Prepare(1, FastSync) 273 wg.Add(1) 274 go func() { 275 // deliver headers 276 defer wg.Done() 277 c := 1 278 for { 279 //fmt.Printf("getting headers from %d\n", c) 280 hdrs := world.headers(c) 281 l := len(hdrs) 282 //fmt.Printf("scheduling %d headers, first %d last %d\n", 283 // l, hdrs[0].Number.Uint64(), hdrs[len(hdrs)-1].Number.Uint64()) 284 q.Schedule(hdrs, uint64(c)) 285 c += l 286 } 287 }() 288 wg.Add(1) 289 go func() { 290 // collect results 291 defer wg.Done() 292 tot := 0 293 for { 294 res := q.Results(true) 295 tot += len(res) 296 fmt.Printf("got %d results, %d tot\n", len(res), tot) 297 // Now we can forget about these 298 world.forget(res[len(res)-1].Header.Number.Uint64()) 299 } 300 }() 301 wg.Add(1) 302 go func() { 303 defer wg.Done() 304 // reserve body fetch 305 i := 4 306 for { 307 peer := dummyPeer(fmt.Sprintf("peer-%d", i)) 308 f, _, _ := q.ReserveBodies(peer, rand.Intn(30)) 309 if f != nil { 310 var emptyList []*types.Header 311 var txs [][]*types.Transaction 312 var uncles [][]*types.Header 313 numToSkip := rand.Intn(len(f.Headers)) 314 for _, hdr := range f.Headers[0 : len(f.Headers)-numToSkip] { 315 txs = append(txs, world.getTransactions(hdr.Number.Uint64())) 316 uncles = append(uncles, emptyList) 317 } 318 time.Sleep(100 * time.Millisecond) 319 _, err := q.DeliverBodies(peer.id, txs, uncles) 320 if err != nil { 321 fmt.Printf("delivered %d bodies %v\n", len(txs), err) 322 } 323 } else { 324 i++ 325 time.Sleep(200 * time.Millisecond) 326 } 327 } 328 }() 329 go func() { 330 defer wg.Done() 331 // reserve receiptfetch 332 peer := dummyPeer("peer-3") 333 for { 334 f, _, _ := q.ReserveReceipts(peer, rand.Intn(50)) 335 if f != nil { 336 var rcs [][]*types.Receipt 337 for _, hdr := range f.Headers { 338 rcs = append(rcs, world.getReceipts(hdr.Number.Uint64())) 339 } 340 _, err := q.DeliverReceipts(peer.id, rcs) 341 if err != nil { 342 fmt.Printf("delivered %d receipts %v\n", len(rcs), err) 343 } 344 time.Sleep(100 * time.Millisecond) 345 } else { 346 time.Sleep(200 * time.Millisecond) 347 } 348 } 349 }() 350 wg.Add(1) 351 go func() { 352 defer wg.Done() 353 for i := 0; i < 50; i++ { 354 time.Sleep(300 * time.Millisecond) 355 //world.tick() 356 //fmt.Printf("trying to progress\n") 357 world.progress(rand.Intn(100)) 358 } 359 for i := 0; i < 50; i++ { 360 time.Sleep(2990 * time.Millisecond) 361 } 362 }() 363 wg.Add(1) 364 go func() { 365 defer wg.Done() 366 for { 367 time.Sleep(990 * time.Millisecond) 368 fmt.Printf("world block tip is %d\n", 369 world.chain[len(world.chain)-1].Header().Number.Uint64()) 370 fmt.Println(q.Stats()) 371 } 372 }() 373 wg.Wait() 374 } 375 376 func newNetwork() *network { 377 var l sync.RWMutex 378 return &network{ 379 cond: sync.NewCond(&l), 380 offset: 1, // block 1 is at blocks[0] 381 } 382 } 383 384 // represents the network 385 type network struct { 386 offset int 387 chain []*types.Block 388 receipts []types.Receipts 389 lock sync.RWMutex 390 cond *sync.Cond 391 } 392 393 func (n *network) getTransactions(blocknum uint64) types.Transactions { 394 index := blocknum - uint64(n.offset) 395 return n.chain[index].Transactions() 396 } 397 func (n *network) getReceipts(blocknum uint64) types.Receipts { 398 index := blocknum - uint64(n.offset) 399 if got := n.chain[index].Header().Number.Uint64(); got != blocknum { 400 fmt.Printf("Err, got %d exp %d\n", got, blocknum) 401 panic("sd") 402 } 403 return n.receipts[index] 404 } 405 406 func (n *network) forget(blocknum uint64) { 407 index := blocknum - uint64(n.offset) 408 n.chain = n.chain[index:] 409 n.receipts = n.receipts[index:] 410 n.offset = int(blocknum) 411 } 412 func (n *network) progress(numBlocks int) { 413 n.lock.Lock() 414 defer n.lock.Unlock() 415 //fmt.Printf("progressing...\n") 416 newBlocks, newR := makeChain(numBlocks, 0, n.chain[len(n.chain)-1], false) 417 n.chain = append(n.chain, newBlocks...) 418 n.receipts = append(n.receipts, newR...) 419 n.cond.Broadcast() 420 } 421 422 func (n *network) headers(from int) []*types.Header { 423 numHeaders := 128 424 var hdrs []*types.Header 425 index := from - n.offset 426 427 for index >= len(n.chain) { 428 // wait for progress 429 n.cond.L.Lock() 430 //fmt.Printf("header going into wait\n") 431 n.cond.Wait() 432 index = from - n.offset 433 n.cond.L.Unlock() 434 } 435 n.lock.RLock() 436 defer n.lock.RUnlock() 437 for i, b := range n.chain[index:] { 438 hdrs = append(hdrs, b.Header()) 439 if i >= numHeaders { 440 break 441 } 442 } 443 return hdrs 444 }