github.com/muhammedhassanm/blockchain@v0.0.0-20200120143007-697261defd4d/go-ethereum-master/swarm/network_test.go (about) 1 // Copyright 2018 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package swarm 18 19 import ( 20 "context" 21 "flag" 22 "fmt" 23 "io/ioutil" 24 "math/rand" 25 "os" 26 "sync" 27 "sync/atomic" 28 "testing" 29 "time" 30 31 "github.com/ethereum/go-ethereum/common" 32 "github.com/ethereum/go-ethereum/crypto" 33 "github.com/ethereum/go-ethereum/log" 34 "github.com/ethereum/go-ethereum/node" 35 "github.com/ethereum/go-ethereum/p2p/discover" 36 "github.com/ethereum/go-ethereum/p2p/simulations" 37 "github.com/ethereum/go-ethereum/p2p/simulations/adapters" 38 "github.com/ethereum/go-ethereum/swarm/api" 39 "github.com/ethereum/go-ethereum/swarm/network" 40 "github.com/ethereum/go-ethereum/swarm/storage" 41 colorable "github.com/mattn/go-colorable" 42 ) 43 44 var ( 45 loglevel = flag.Int("loglevel", 2, "verbosity of logs") 46 longrunning = flag.Bool("longrunning", false, "do run long-running tests") 47 waitKademlia = flag.Bool("waitkademlia", false, "wait for healthy kademlia before checking files availability") 48 ) 49 50 func init() { 51 rand.Seed(time.Now().UnixNano()) 52 53 flag.Parse() 54 55 log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) 56 } 57 58 // TestSwarmNetwork runs a series of test simulations with 59 // static and dynamic Swarm nodes in network simulation, by 60 // uploading files to every node and retrieving them. 61 func TestSwarmNetwork(t *testing.T) { 62 for _, tc := range []struct { 63 name string 64 steps []testSwarmNetworkStep 65 options *testSwarmNetworkOptions 66 disabled bool 67 }{ 68 { 69 name: "10_nodes", 70 steps: []testSwarmNetworkStep{ 71 { 72 nodeCount: 10, 73 }, 74 }, 75 options: &testSwarmNetworkOptions{ 76 Timeout: 45 * time.Second, 77 }, 78 }, 79 { 80 name: "10_nodes_skip_check", 81 steps: []testSwarmNetworkStep{ 82 { 83 nodeCount: 10, 84 }, 85 }, 86 options: &testSwarmNetworkOptions{ 87 Timeout: 45 * time.Second, 88 SkipCheck: true, 89 }, 90 }, 91 { 92 name: "100_nodes", 93 steps: []testSwarmNetworkStep{ 94 { 95 nodeCount: 100, 96 }, 97 }, 98 options: &testSwarmNetworkOptions{ 99 Timeout: 3 * time.Minute, 100 }, 101 disabled: !*longrunning, 102 }, 103 { 104 name: "100_nodes_skip_check", 105 steps: []testSwarmNetworkStep{ 106 { 107 nodeCount: 100, 108 }, 109 }, 110 options: &testSwarmNetworkOptions{ 111 Timeout: 3 * time.Minute, 112 SkipCheck: true, 113 }, 114 disabled: !*longrunning, 115 }, 116 { 117 name: "inc_node_count", 118 steps: []testSwarmNetworkStep{ 119 { 120 nodeCount: 2, 121 }, 122 { 123 nodeCount: 5, 124 }, 125 { 126 nodeCount: 10, 127 }, 128 }, 129 options: &testSwarmNetworkOptions{ 130 Timeout: 90 * time.Second, 131 }, 132 disabled: !*longrunning, 133 }, 134 { 135 name: "dec_node_count", 136 steps: []testSwarmNetworkStep{ 137 { 138 nodeCount: 10, 139 }, 140 { 141 nodeCount: 6, 142 }, 143 { 144 nodeCount: 3, 145 }, 146 }, 147 options: &testSwarmNetworkOptions{ 148 Timeout: 90 * time.Second, 149 }, 150 disabled: !*longrunning, 151 }, 152 { 153 name: "dec_inc_node_count", 154 steps: []testSwarmNetworkStep{ 155 { 156 nodeCount: 5, 157 }, 158 { 159 nodeCount: 3, 160 }, 161 { 162 nodeCount: 10, 163 }, 164 }, 165 options: &testSwarmNetworkOptions{ 166 Timeout: 90 * time.Second, 167 }, 168 }, 169 { 170 name: "inc_dec_node_count", 171 steps: []testSwarmNetworkStep{ 172 { 173 nodeCount: 3, 174 }, 175 { 176 nodeCount: 5, 177 }, 178 { 179 nodeCount: 25, 180 }, 181 { 182 nodeCount: 10, 183 }, 184 { 185 nodeCount: 4, 186 }, 187 }, 188 options: &testSwarmNetworkOptions{ 189 Timeout: 5 * time.Minute, 190 }, 191 disabled: !*longrunning, 192 }, 193 { 194 name: "inc_dec_node_count_skip_check", 195 steps: []testSwarmNetworkStep{ 196 { 197 nodeCount: 3, 198 }, 199 { 200 nodeCount: 5, 201 }, 202 { 203 nodeCount: 25, 204 }, 205 { 206 nodeCount: 10, 207 }, 208 { 209 nodeCount: 4, 210 }, 211 }, 212 options: &testSwarmNetworkOptions{ 213 Timeout: 5 * time.Minute, 214 SkipCheck: true, 215 }, 216 disabled: !*longrunning, 217 }, 218 } { 219 if tc.disabled { 220 continue 221 } 222 t.Run(tc.name, func(t *testing.T) { 223 testSwarmNetwork(t, tc.options, tc.steps...) 224 }) 225 } 226 } 227 228 // testSwarmNetworkStep is the configuration 229 // for the state of the simulation network. 230 type testSwarmNetworkStep struct { 231 // number of swarm nodes that must be in the Up state 232 nodeCount int 233 } 234 235 // file represents the file uploaded on a particular node. 236 type file struct { 237 addr storage.Address 238 data string 239 nodeID discover.NodeID 240 } 241 242 // check represents a reference to a file that is retrieved 243 // from a particular node. 244 type check struct { 245 key string 246 nodeID discover.NodeID 247 } 248 249 // testSwarmNetworkOptions contains optional parameters for running 250 // testSwarmNetwork. 251 type testSwarmNetworkOptions struct { 252 Timeout time.Duration 253 SkipCheck bool 254 } 255 256 // testSwarmNetwork is a helper function used for testing different 257 // static and dynamic Swarm network simulations. 258 // It is responsible for: 259 // - Setting up a Swarm network simulation, and updates the number of nodes within the network on every step according to steps. 260 // - Uploading a unique file to every node on every step. 261 // - May wait for Kademlia on every node to be healthy. 262 // - Checking if a file is retrievable from all nodes. 263 func testSwarmNetwork(t *testing.T, o *testSwarmNetworkOptions, steps ...testSwarmNetworkStep) { 264 dir, err := ioutil.TempDir("", "swarm-network-test") 265 if err != nil { 266 t.Fatal(err) 267 } 268 defer os.RemoveAll(dir) 269 270 if o == nil { 271 o = new(testSwarmNetworkOptions) 272 } 273 274 ctx := context.Background() 275 if o.Timeout > 0 { 276 var cancel context.CancelFunc 277 ctx, cancel = context.WithTimeout(ctx, o.Timeout) 278 defer cancel() 279 } 280 281 swarms := make(map[discover.NodeID]*Swarm) 282 files := make([]file, 0) 283 284 services := map[string]adapters.ServiceFunc{ 285 "swarm": func(ctx *adapters.ServiceContext) (node.Service, error) { 286 config := api.NewConfig() 287 288 dir, err := ioutil.TempDir(dir, "node") 289 if err != nil { 290 return nil, err 291 } 292 293 config.Path = dir 294 295 privkey, err := crypto.GenerateKey() 296 if err != nil { 297 return nil, err 298 } 299 300 config.Init(privkey) 301 config.DeliverySkipCheck = o.SkipCheck 302 303 s, err := NewSwarm(config, nil) 304 if err != nil { 305 return nil, err 306 } 307 log.Info("new swarm", "bzzKey", config.BzzKey, "baseAddr", fmt.Sprintf("%x", s.bzz.BaseAddr())) 308 swarms[ctx.Config.ID] = s 309 return s, nil 310 }, 311 } 312 313 a := adapters.NewSimAdapter(services) 314 net := simulations.NewNetwork(a, &simulations.NetworkConfig{ 315 ID: "0", 316 DefaultService: "swarm", 317 }) 318 defer net.Shutdown() 319 320 trigger := make(chan discover.NodeID) 321 322 sim := simulations.NewSimulation(net) 323 324 for i, step := range steps { 325 log.Debug("test sync step", "n", i+1, "nodes", step.nodeCount) 326 327 change := step.nodeCount - len(allNodeIDs(net)) 328 329 if change > 0 { 330 _, err := addNodes(change, net) 331 if err != nil { 332 t.Fatal(err) 333 } 334 } else if change < 0 { 335 err := removeNodes(-change, net) 336 if err != nil { 337 t.Fatal(err) 338 } 339 } else { 340 t.Logf("step %v: no change in nodes", i) 341 continue 342 } 343 344 nodeIDs := allNodeIDs(net) 345 shuffle(len(nodeIDs), func(i, j int) { 346 nodeIDs[i], nodeIDs[j] = nodeIDs[j], nodeIDs[i] 347 }) 348 for _, id := range nodeIDs { 349 key, data, err := uploadFile(swarms[id]) 350 if err != nil { 351 t.Fatal(err) 352 } 353 log.Trace("file uploaded", "node", id, "key", key.String()) 354 files = append(files, file{ 355 addr: key, 356 data: data, 357 nodeID: id, 358 }) 359 } 360 361 // Prepare PeerPot map for checking Kademlia health 362 var ppmap map[string]*network.PeerPot 363 nIDs := allNodeIDs(net) 364 addrs := make([][]byte, len(nIDs)) 365 if *waitKademlia { 366 for i, id := range nIDs { 367 addrs[i] = swarms[id].bzz.BaseAddr() 368 } 369 ppmap = network.NewPeerPotMap(2, addrs) 370 } 371 372 var checkStatusM sync.Map 373 var nodeStatusM sync.Map 374 var totalFoundCount uint64 375 376 result := sim.Run(ctx, &simulations.Step{ 377 Action: func(ctx context.Context) error { 378 if *waitKademlia { 379 // Wait for healthy Kademlia on every node before checking files 380 ticker := time.NewTicker(200 * time.Millisecond) 381 defer ticker.Stop() 382 383 for range ticker.C { 384 healthy := true 385 log.Debug("kademlia health check", "node count", len(nIDs), "addr count", len(addrs)) 386 for i, id := range nIDs { 387 swarm := swarms[id] 388 //PeerPot for this node 389 addr := common.Bytes2Hex(swarm.bzz.BaseAddr()) 390 pp := ppmap[addr] 391 //call Healthy RPC 392 h := swarm.bzz.Healthy(pp) 393 //print info 394 log.Debug(swarm.bzz.String()) 395 log.Debug("kademlia", "empty bins", pp.EmptyBins, "gotNN", h.GotNN, "knowNN", h.KnowNN, "full", h.Full) 396 log.Debug("kademlia", "health", h.GotNN && h.KnowNN && h.Full, "addr", fmt.Sprintf("%x", swarm.bzz.BaseAddr()), "id", id, "i", i) 397 log.Debug("kademlia", "ill condition", !h.GotNN || !h.Full, "addr", fmt.Sprintf("%x", swarm.bzz.BaseAddr()), "id", id, "i", i) 398 if !h.GotNN || !h.Full { 399 healthy = false 400 break 401 } 402 } 403 if healthy { 404 break 405 } 406 } 407 } 408 409 go func() { 410 // File retrieval check is repeated until all uploaded files are retrieved from all nodes 411 // or until the timeout is reached. 412 for { 413 if retrieve(net, files, swarms, trigger, &checkStatusM, &nodeStatusM, &totalFoundCount) == 0 { 414 return 415 } 416 } 417 }() 418 return nil 419 }, 420 Trigger: trigger, 421 Expect: &simulations.Expectation{ 422 Nodes: allNodeIDs(net), 423 Check: func(ctx context.Context, id discover.NodeID) (bool, error) { 424 // The check is done by a goroutine in the action function. 425 return true, nil 426 }, 427 }, 428 }) 429 if result.Error != nil { 430 t.Fatal(result.Error) 431 } 432 log.Debug("done: test sync step", "n", i+1, "nodes", step.nodeCount) 433 } 434 } 435 436 // allNodeIDs is returning NodeID for every node that is Up. 437 func allNodeIDs(net *simulations.Network) (nodes []discover.NodeID) { 438 for _, n := range net.GetNodes() { 439 if n.Up { 440 nodes = append(nodes, n.ID()) 441 } 442 } 443 return 444 } 445 446 // addNodes adds a number of nodes to the network. 447 func addNodes(count int, net *simulations.Network) (ids []discover.NodeID, err error) { 448 for i := 0; i < count; i++ { 449 nodeIDs := allNodeIDs(net) 450 l := len(nodeIDs) 451 nodeconf := adapters.RandomNodeConfig() 452 node, err := net.NewNodeWithConfig(nodeconf) 453 if err != nil { 454 return nil, fmt.Errorf("create node: %v", err) 455 } 456 err = net.Start(node.ID()) 457 if err != nil { 458 return nil, fmt.Errorf("start node: %v", err) 459 } 460 461 log.Debug("created node", "id", node.ID()) 462 463 // connect nodes in a chain 464 if l > 0 { 465 var otherNodeID discover.NodeID 466 for i := l - 1; i >= 0; i-- { 467 n := net.GetNode(nodeIDs[i]) 468 if n.Up { 469 otherNodeID = n.ID() 470 break 471 } 472 } 473 log.Debug("connect nodes", "one", node.ID(), "other", otherNodeID) 474 if err := net.Connect(node.ID(), otherNodeID); err != nil { 475 return nil, err 476 } 477 } 478 ids = append(ids, node.ID()) 479 } 480 return ids, nil 481 } 482 483 // removeNodes stops a random nodes in the network. 484 func removeNodes(count int, net *simulations.Network) error { 485 for i := 0; i < count; i++ { 486 // allNodeIDs are returning only the Up nodes. 487 nodeIDs := allNodeIDs(net) 488 if len(nodeIDs) == 0 { 489 break 490 } 491 node := net.GetNode(nodeIDs[rand.Intn(len(nodeIDs))]) 492 if err := node.Stop(); err != nil { 493 return err 494 } 495 log.Debug("removed node", "id", node.ID()) 496 } 497 return nil 498 } 499 500 // uploadFile, uploads a short file to the swarm instance 501 // using the api.Put method. 502 func uploadFile(swarm *Swarm) (storage.Address, string, error) { 503 b := make([]byte, 8) 504 _, err := rand.Read(b) 505 if err != nil { 506 return nil, "", err 507 } 508 // File data is very short, but it is ensured that its 509 // uniqueness is very certain. 510 data := fmt.Sprintf("test content %s %x", time.Now().Round(0), b) 511 ctx := context.TODO() 512 k, wait, err := swarm.api.Put(ctx, data, "text/plain", false) 513 if err != nil { 514 return nil, "", err 515 } 516 if wait != nil { 517 err = wait(ctx) 518 } 519 return k, data, err 520 } 521 522 // retrieve is the function that is used for checking the availability of 523 // uploaded files in testSwarmNetwork test helper function. 524 func retrieve( 525 net *simulations.Network, 526 files []file, 527 swarms map[discover.NodeID]*Swarm, 528 trigger chan discover.NodeID, 529 checkStatusM *sync.Map, 530 nodeStatusM *sync.Map, 531 totalFoundCount *uint64, 532 ) (missing uint64) { 533 shuffle(len(files), func(i, j int) { 534 files[i], files[j] = files[j], files[i] 535 }) 536 537 var totalWg sync.WaitGroup 538 errc := make(chan error) 539 540 nodeIDs := allNodeIDs(net) 541 542 totalCheckCount := len(nodeIDs) * len(files) 543 544 for _, id := range nodeIDs { 545 if _, ok := nodeStatusM.Load(id); ok { 546 continue 547 } 548 start := time.Now() 549 var checkCount uint64 550 var foundCount uint64 551 552 totalWg.Add(1) 553 554 var wg sync.WaitGroup 555 556 for _, f := range files { 557 swarm := swarms[id] 558 559 checkKey := check{ 560 key: f.addr.String(), 561 nodeID: id, 562 } 563 if n, ok := checkStatusM.Load(checkKey); ok && n.(int) == 0 { 564 continue 565 } 566 567 checkCount++ 568 wg.Add(1) 569 go func(f file, id discover.NodeID) { 570 defer wg.Done() 571 572 log.Debug("api get: check file", "node", id.String(), "key", f.addr.String(), "total files found", atomic.LoadUint64(totalFoundCount)) 573 574 r, _, _, _, err := swarm.api.Get(context.TODO(), f.addr, "/") 575 if err != nil { 576 errc <- fmt.Errorf("api get: node %s, key %s, kademlia %s: %v", id, f.addr, swarm.bzz.Hive, err) 577 return 578 } 579 d, err := ioutil.ReadAll(r) 580 if err != nil { 581 errc <- fmt.Errorf("api get: read response: node %s, key %s: kademlia %s: %v", id, f.addr, swarm.bzz.Hive, err) 582 return 583 } 584 data := string(d) 585 if data != f.data { 586 errc <- fmt.Errorf("file contend missmatch: node %s, key %s, expected %q, got %q", id, f.addr, f.data, data) 587 return 588 } 589 checkStatusM.Store(checkKey, 0) 590 atomic.AddUint64(&foundCount, 1) 591 log.Info("api get: file found", "node", id.String(), "key", f.addr.String(), "content", data, "files found", atomic.LoadUint64(&foundCount)) 592 }(f, id) 593 } 594 595 go func(id discover.NodeID) { 596 defer totalWg.Done() 597 wg.Wait() 598 599 atomic.AddUint64(totalFoundCount, foundCount) 600 601 if foundCount == checkCount { 602 log.Info("all files are found for node", "id", id.String(), "duration", time.Since(start)) 603 nodeStatusM.Store(id, 0) 604 trigger <- id 605 return 606 } 607 log.Debug("files missing for node", "id", id.String(), "check", checkCount, "found", foundCount) 608 }(id) 609 610 } 611 612 go func() { 613 totalWg.Wait() 614 close(errc) 615 }() 616 617 var errCount int 618 for err := range errc { 619 if err != nil { 620 errCount++ 621 } 622 log.Warn(err.Error()) 623 } 624 625 log.Info("check stats", "total check count", totalCheckCount, "total files found", atomic.LoadUint64(totalFoundCount), "total errors", errCount) 626 627 return uint64(totalCheckCount) - atomic.LoadUint64(totalFoundCount) 628 } 629 630 // Backported from stdlib https://golang.org/src/math/rand/rand.go?s=11175:11215#L333 631 // 632 // Replace with rand.Shuffle from go 1.10 when go 1.9 support is dropped. 633 // 634 // shuffle pseudo-randomizes the order of elements. 635 // n is the number of elements. Shuffle panics if n < 0. 636 // swap swaps the elements with indexes i and j. 637 func shuffle(n int, swap func(i, j int)) { 638 if n < 0 { 639 panic("invalid argument to Shuffle") 640 } 641 642 // Fisher-Yates shuffle: https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle 643 // Shuffle really ought not be called with n that doesn't fit in 32 bits. 644 // Not only will it take a very long time, but with 2³¹! possible permutations, 645 // there's no way that any PRNG can have a big enough internal state to 646 // generate even a minuscule percentage of the possible permutations. 647 // Nevertheless, the right API signature accepts an int n, so handle it as best we can. 648 i := n - 1 649 for ; i > 1<<31-1-1; i-- { 650 j := int(rand.Int63n(int64(i + 1))) 651 swap(i, j) 652 } 653 for ; i > 0; i-- { 654 j := int(rand.Int31n(int32(i + 1))) 655 swap(i, j) 656 } 657 }