github.com/oskarth/go-ethereum@v1.6.8-0.20191013093314-dac24a9d3494/swarm/network/simulations/discovery/discovery_test.go (about) 1 // Copyright 2018 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package discovery 18 19 import ( 20 "context" 21 "encoding/json" 22 "errors" 23 "flag" 24 "fmt" 25 "io/ioutil" 26 "math/rand" 27 "os" 28 "path" 29 "strings" 30 "sync" 31 "testing" 32 "time" 33 34 "github.com/ethereum/go-ethereum/log" 35 "github.com/ethereum/go-ethereum/node" 36 "github.com/ethereum/go-ethereum/p2p" 37 "github.com/ethereum/go-ethereum/p2p/enode" 38 "github.com/ethereum/go-ethereum/p2p/simulations" 39 "github.com/ethereum/go-ethereum/p2p/simulations/adapters" 40 "github.com/ethereum/go-ethereum/swarm/network" 41 "github.com/ethereum/go-ethereum/swarm/state" 42 colorable "github.com/mattn/go-colorable" 43 ) 44 45 // serviceName is used with the exec adapter so the exec'd binary knows which 46 // service to execute 47 const serviceName = "discovery" 48 const testMinProxBinSize = 2 49 const discoveryPersistenceDatadir = "discovery_persistence_test_store" 50 51 var discoveryPersistencePath = path.Join(os.TempDir(), discoveryPersistenceDatadir) 52 var discoveryEnabled = true 53 var persistenceEnabled = false 54 55 var services = adapters.Services{ 56 serviceName: newService, 57 } 58 59 func cleanDbStores() error { 60 entries, err := ioutil.ReadDir(os.TempDir()) 61 if err != nil { 62 return err 63 } 64 65 for _, f := range entries { 66 if strings.HasPrefix(f.Name(), discoveryPersistenceDatadir) { 67 os.RemoveAll(path.Join(os.TempDir(), f.Name())) 68 } 69 } 70 return nil 71 72 } 73 74 func getDbStore(nodeID string) (*state.DBStore, error) { 75 if _, err := os.Stat(discoveryPersistencePath + "_" + nodeID); os.IsNotExist(err) { 76 log.Info(fmt.Sprintf("directory for nodeID %s does not exist. creating...", nodeID)) 77 ioutil.TempDir("", discoveryPersistencePath+"_"+nodeID) 78 } 79 log.Info(fmt.Sprintf("opening storage directory for nodeID %s", nodeID)) 80 store, err := state.NewDBStore(discoveryPersistencePath + "_" + nodeID) 81 if err != nil { 82 return nil, err 83 } 84 return store, nil 85 } 86 87 var ( 88 nodeCount = flag.Int("nodes", 10, "number of nodes to create (default 10)") 89 initCount = flag.Int("conns", 1, "number of originally connected peers (default 1)") 90 snapshotFile = flag.String("snapshot", "", "create snapshot") 91 loglevel = flag.Int("loglevel", 3, "verbosity of logs") 92 rawlog = flag.Bool("rawlog", false, "remove terminal formatting from logs") 93 ) 94 95 func init() { 96 flag.Parse() 97 // register the discovery service which will run as a devp2p 98 // protocol when using the exec adapter 99 adapters.RegisterServices(services) 100 101 log.PrintOrigins(true) 102 log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(!*rawlog)))) 103 } 104 105 // Benchmarks to test the average time it takes for an N-node ring 106 // to full a healthy kademlia topology 107 func BenchmarkDiscovery_8_1(b *testing.B) { benchmarkDiscovery(b, 8, 1) } 108 func BenchmarkDiscovery_16_1(b *testing.B) { benchmarkDiscovery(b, 16, 1) } 109 func BenchmarkDiscovery_32_1(b *testing.B) { benchmarkDiscovery(b, 32, 1) } 110 func BenchmarkDiscovery_64_1(b *testing.B) { benchmarkDiscovery(b, 64, 1) } 111 func BenchmarkDiscovery_128_1(b *testing.B) { benchmarkDiscovery(b, 128, 1) } 112 func BenchmarkDiscovery_256_1(b *testing.B) { benchmarkDiscovery(b, 256, 1) } 113 114 func BenchmarkDiscovery_8_2(b *testing.B) { benchmarkDiscovery(b, 8, 2) } 115 func BenchmarkDiscovery_16_2(b *testing.B) { benchmarkDiscovery(b, 16, 2) } 116 func BenchmarkDiscovery_32_2(b *testing.B) { benchmarkDiscovery(b, 32, 2) } 117 func BenchmarkDiscovery_64_2(b *testing.B) { benchmarkDiscovery(b, 64, 2) } 118 func BenchmarkDiscovery_128_2(b *testing.B) { benchmarkDiscovery(b, 128, 2) } 119 func BenchmarkDiscovery_256_2(b *testing.B) { benchmarkDiscovery(b, 256, 2) } 120 121 func BenchmarkDiscovery_8_4(b *testing.B) { benchmarkDiscovery(b, 8, 4) } 122 func BenchmarkDiscovery_16_4(b *testing.B) { benchmarkDiscovery(b, 16, 4) } 123 func BenchmarkDiscovery_32_4(b *testing.B) { benchmarkDiscovery(b, 32, 4) } 124 func BenchmarkDiscovery_64_4(b *testing.B) { benchmarkDiscovery(b, 64, 4) } 125 func BenchmarkDiscovery_128_4(b *testing.B) { benchmarkDiscovery(b, 128, 4) } 126 func BenchmarkDiscovery_256_4(b *testing.B) { benchmarkDiscovery(b, 256, 4) } 127 128 func TestDiscoverySimulationExecAdapter(t *testing.T) { 129 testDiscoverySimulationExecAdapter(t, *nodeCount, *initCount) 130 } 131 132 func testDiscoverySimulationExecAdapter(t *testing.T, nodes, conns int) { 133 baseDir, err := ioutil.TempDir("", "swarm-test") 134 if err != nil { 135 t.Fatal(err) 136 } 137 defer os.RemoveAll(baseDir) 138 testDiscoverySimulation(t, nodes, conns, adapters.NewExecAdapter(baseDir)) 139 } 140 141 func TestDiscoverySimulationSimAdapter(t *testing.T) { 142 testDiscoverySimulationSimAdapter(t, *nodeCount, *initCount) 143 } 144 145 func TestDiscoveryPersistenceSimulationSimAdapter(t *testing.T) { 146 testDiscoveryPersistenceSimulationSimAdapter(t, *nodeCount, *initCount) 147 } 148 149 func testDiscoveryPersistenceSimulationSimAdapter(t *testing.T, nodes, conns int) { 150 testDiscoveryPersistenceSimulation(t, nodes, conns, adapters.NewSimAdapter(services)) 151 } 152 153 func testDiscoverySimulationSimAdapter(t *testing.T, nodes, conns int) { 154 testDiscoverySimulation(t, nodes, conns, adapters.NewSimAdapter(services)) 155 } 156 157 func testDiscoverySimulation(t *testing.T, nodes, conns int, adapter adapters.NodeAdapter) { 158 startedAt := time.Now() 159 result, err := discoverySimulation(nodes, conns, adapter) 160 if err != nil { 161 t.Fatalf("Setting up simulation failed: %v", err) 162 } 163 if result.Error != nil { 164 t.Fatalf("Simulation failed: %s", result.Error) 165 } 166 t.Logf("Simulation with %d nodes passed in %s", nodes, result.FinishedAt.Sub(result.StartedAt)) 167 var min, max time.Duration 168 var sum int 169 for _, pass := range result.Passes { 170 duration := pass.Sub(result.StartedAt) 171 if sum == 0 || duration < min { 172 min = duration 173 } 174 if duration > max { 175 max = duration 176 } 177 sum += int(duration.Nanoseconds()) 178 } 179 t.Logf("Min: %s, Max: %s, Average: %s", min, max, time.Duration(sum/len(result.Passes))*time.Nanosecond) 180 finishedAt := time.Now() 181 t.Logf("Setup: %s, shutdown: %s", result.StartedAt.Sub(startedAt), finishedAt.Sub(result.FinishedAt)) 182 } 183 184 func testDiscoveryPersistenceSimulation(t *testing.T, nodes, conns int, adapter adapters.NodeAdapter) map[int][]byte { 185 persistenceEnabled = true 186 discoveryEnabled = true 187 188 result, err := discoveryPersistenceSimulation(nodes, conns, adapter) 189 190 if err != nil { 191 t.Fatalf("Setting up simulation failed: %v", err) 192 } 193 if result.Error != nil { 194 t.Fatalf("Simulation failed: %s", result.Error) 195 } 196 t.Logf("Simulation with %d nodes passed in %s", nodes, result.FinishedAt.Sub(result.StartedAt)) 197 // set the discovery and persistence flags again to default so other 198 // tests will not be affected 199 discoveryEnabled = true 200 persistenceEnabled = false 201 return nil 202 } 203 204 func benchmarkDiscovery(b *testing.B, nodes, conns int) { 205 for i := 0; i < b.N; i++ { 206 result, err := discoverySimulation(nodes, conns, adapters.NewSimAdapter(services)) 207 if err != nil { 208 b.Fatalf("setting up simulation failed: %v", err) 209 } 210 if result.Error != nil { 211 b.Logf("simulation failed: %s", result.Error) 212 } 213 } 214 } 215 216 func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simulations.StepResult, error) { 217 // create network 218 net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{ 219 ID: "0", 220 DefaultService: serviceName, 221 }) 222 defer net.Shutdown() 223 trigger := make(chan enode.ID) 224 ids := make([]enode.ID, nodes) 225 for i := 0; i < nodes; i++ { 226 conf := adapters.RandomNodeConfig() 227 node, err := net.NewNodeWithConfig(conf) 228 if err != nil { 229 return nil, fmt.Errorf("error starting node: %s", err) 230 } 231 if err := net.Start(node.ID()); err != nil { 232 return nil, fmt.Errorf("error starting node %s: %s", node.ID().TerminalString(), err) 233 } 234 if err := triggerChecks(trigger, net, node.ID()); err != nil { 235 return nil, fmt.Errorf("error triggering checks for node %s: %s", node.ID().TerminalString(), err) 236 } 237 ids[i] = node.ID() 238 } 239 240 // run a simulation which connects the 10 nodes in a ring and waits 241 // for full peer discovery 242 var addrs [][]byte 243 action := func(ctx context.Context) error { 244 return nil 245 } 246 wg := sync.WaitGroup{} 247 for i := range ids { 248 // collect the overlay addresses, to 249 addrs = append(addrs, ids[i].Bytes()) 250 for j := 0; j < conns; j++ { 251 var k int 252 if j == 0 { 253 k = (i + 1) % len(ids) 254 } else { 255 k = rand.Intn(len(ids)) 256 } 257 wg.Add(1) 258 go func(i, k int) { 259 defer wg.Done() 260 net.Connect(ids[i], ids[k]) 261 }(i, k) 262 } 263 } 264 wg.Wait() 265 log.Debug(fmt.Sprintf("nodes: %v", len(addrs))) 266 // construct the peer pot, so that kademlia health can be checked 267 ppmap := network.NewPeerPotMap(testMinProxBinSize, addrs) 268 check := func(ctx context.Context, id enode.ID) (bool, error) { 269 select { 270 case <-ctx.Done(): 271 return false, ctx.Err() 272 default: 273 } 274 275 node := net.GetNode(id) 276 if node == nil { 277 return false, fmt.Errorf("unknown node: %s", id) 278 } 279 client, err := node.Client() 280 if err != nil { 281 return false, fmt.Errorf("error getting node client: %s", err) 282 } 283 healthy := &network.Health{} 284 if err := client.Call(&healthy, "hive_healthy", ppmap[id.String()]); err != nil { 285 return false, fmt.Errorf("error getting node health: %s", err) 286 } 287 log.Debug(fmt.Sprintf("node %4s healthy: got nearest neighbours: %v, know nearest neighbours: %v, saturated: %v\n%v", id, healthy.GotNN, healthy.KnowNN, healthy.Full, healthy.Hive)) 288 return healthy.KnowNN && healthy.GotNN && healthy.Full, nil 289 } 290 291 // 64 nodes ~ 1min 292 // 128 nodes ~ 293 timeout := 300 * time.Second 294 ctx, cancel := context.WithTimeout(context.Background(), timeout) 295 defer cancel() 296 result := simulations.NewSimulation(net).Run(ctx, &simulations.Step{ 297 Action: action, 298 Trigger: trigger, 299 Expect: &simulations.Expectation{ 300 Nodes: ids, 301 Check: check, 302 }, 303 }) 304 if result.Error != nil { 305 return result, nil 306 } 307 308 if *snapshotFile != "" { 309 snap, err := net.Snapshot() 310 if err != nil { 311 return nil, errors.New("no shapshot dude") 312 } 313 jsonsnapshot, err := json.Marshal(snap) 314 if err != nil { 315 return nil, fmt.Errorf("corrupt json snapshot: %v", err) 316 } 317 log.Info("writing snapshot", "file", *snapshotFile) 318 err = ioutil.WriteFile(*snapshotFile, jsonsnapshot, 0755) 319 if err != nil { 320 return nil, err 321 } 322 } 323 return result, nil 324 } 325 326 func discoveryPersistenceSimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simulations.StepResult, error) { 327 cleanDbStores() 328 defer cleanDbStores() 329 330 // create network 331 net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{ 332 ID: "0", 333 DefaultService: serviceName, 334 }) 335 defer net.Shutdown() 336 trigger := make(chan enode.ID) 337 ids := make([]enode.ID, nodes) 338 var addrs [][]byte 339 340 for i := 0; i < nodes; i++ { 341 conf := adapters.RandomNodeConfig() 342 node, err := net.NewNodeWithConfig(conf) 343 if err != nil { 344 panic(err) 345 } 346 if err != nil { 347 return nil, fmt.Errorf("error starting node: %s", err) 348 } 349 if err := net.Start(node.ID()); err != nil { 350 return nil, fmt.Errorf("error starting node %s: %s", node.ID().TerminalString(), err) 351 } 352 if err := triggerChecks(trigger, net, node.ID()); err != nil { 353 return nil, fmt.Errorf("error triggering checks for node %s: %s", node.ID().TerminalString(), err) 354 } 355 ids[i] = node.ID() 356 a := ids[i].Bytes() 357 358 addrs = append(addrs, a) 359 } 360 361 // run a simulation which connects the 10 nodes in a ring and waits 362 // for full peer discovery 363 ppmap := network.NewPeerPotMap(testMinProxBinSize, addrs) 364 365 var restartTime time.Time 366 367 action := func(ctx context.Context) error { 368 ticker := time.NewTicker(500 * time.Millisecond) 369 370 for range ticker.C { 371 isHealthy := true 372 for _, id := range ids { 373 //call Healthy RPC 374 node := net.GetNode(id) 375 if node == nil { 376 return fmt.Errorf("unknown node: %s", id) 377 } 378 client, err := node.Client() 379 if err != nil { 380 return fmt.Errorf("error getting node client: %s", err) 381 } 382 healthy := &network.Health{} 383 addr := id.String() 384 if err := client.Call(&healthy, "hive_healthy", ppmap[addr]); err != nil { 385 return fmt.Errorf("error getting node health: %s", err) 386 } 387 388 log.Info(fmt.Sprintf("NODE: %s, IS HEALTHY: %t", addr, healthy.GotNN && healthy.KnowNN && healthy.Full)) 389 if !healthy.GotNN || !healthy.Full { 390 isHealthy = false 391 break 392 } 393 } 394 if isHealthy { 395 break 396 } 397 } 398 ticker.Stop() 399 400 log.Info("reached healthy kademlia. starting to shutdown nodes.") 401 shutdownStarted := time.Now() 402 // stop all ids, then start them again 403 for _, id := range ids { 404 node := net.GetNode(id) 405 406 if err := net.Stop(node.ID()); err != nil { 407 return fmt.Errorf("error stopping node %s: %s", node.ID().TerminalString(), err) 408 } 409 } 410 log.Info(fmt.Sprintf("shutting down nodes took: %s", time.Since(shutdownStarted))) 411 persistenceEnabled = true 412 discoveryEnabled = false 413 restartTime = time.Now() 414 for _, id := range ids { 415 node := net.GetNode(id) 416 if err := net.Start(node.ID()); err != nil { 417 return fmt.Errorf("error starting node %s: %s", node.ID().TerminalString(), err) 418 } 419 if err := triggerChecks(trigger, net, node.ID()); err != nil { 420 return fmt.Errorf("error triggering checks for node %s: %s", node.ID().TerminalString(), err) 421 } 422 } 423 424 log.Info(fmt.Sprintf("restarting nodes took: %s", time.Since(restartTime))) 425 426 return nil 427 } 428 //connects in a chain 429 wg := sync.WaitGroup{} 430 //connects in a ring 431 for i := range ids { 432 for j := 1; j <= conns; j++ { 433 k := (i + j) % len(ids) 434 if k == i { 435 k = (k + 1) % len(ids) 436 } 437 wg.Add(1) 438 go func(i, k int) { 439 defer wg.Done() 440 net.Connect(ids[i], ids[k]) 441 }(i, k) 442 } 443 } 444 wg.Wait() 445 log.Debug(fmt.Sprintf("nodes: %v", len(addrs))) 446 // construct the peer pot, so that kademlia health can be checked 447 check := func(ctx context.Context, id enode.ID) (bool, error) { 448 select { 449 case <-ctx.Done(): 450 return false, ctx.Err() 451 default: 452 } 453 454 node := net.GetNode(id) 455 if node == nil { 456 return false, fmt.Errorf("unknown node: %s", id) 457 } 458 client, err := node.Client() 459 if err != nil { 460 return false, fmt.Errorf("error getting node client: %s", err) 461 } 462 healthy := &network.Health{} 463 if err := client.Call(&healthy, "hive_healthy", ppmap[id.String()]); err != nil { 464 return false, fmt.Errorf("error getting node health: %s", err) 465 } 466 log.Info(fmt.Sprintf("node %4s healthy: got nearest neighbours: %v, know nearest neighbours: %v, saturated: %v", id, healthy.GotNN, healthy.KnowNN, healthy.Full)) 467 468 return healthy.KnowNN && healthy.GotNN && healthy.Full, nil 469 } 470 471 // 64 nodes ~ 1min 472 // 128 nodes ~ 473 timeout := 300 * time.Second 474 ctx, cancel := context.WithTimeout(context.Background(), timeout) 475 defer cancel() 476 result := simulations.NewSimulation(net).Run(ctx, &simulations.Step{ 477 Action: action, 478 Trigger: trigger, 479 Expect: &simulations.Expectation{ 480 Nodes: ids, 481 Check: check, 482 }, 483 }) 484 if result.Error != nil { 485 return result, nil 486 } 487 488 return result, nil 489 } 490 491 // triggerChecks triggers a simulation step check whenever a peer is added or 492 // removed from the given node, and also every second to avoid a race between 493 // peer events and kademlia becoming healthy 494 func triggerChecks(trigger chan enode.ID, net *simulations.Network, id enode.ID) error { 495 node := net.GetNode(id) 496 if node == nil { 497 return fmt.Errorf("unknown node: %s", id) 498 } 499 client, err := node.Client() 500 if err != nil { 501 return err 502 } 503 events := make(chan *p2p.PeerEvent) 504 sub, err := client.Subscribe(context.Background(), "admin", events, "peerEvents") 505 if err != nil { 506 return fmt.Errorf("error getting peer events for node %v: %s", id, err) 507 } 508 go func() { 509 defer sub.Unsubscribe() 510 511 tick := time.NewTicker(time.Second) 512 defer tick.Stop() 513 514 for { 515 select { 516 case <-events: 517 trigger <- id 518 case <-tick.C: 519 trigger <- id 520 case err := <-sub.Err(): 521 if err != nil { 522 log.Error(fmt.Sprintf("error getting peer events for node %v", id), "err", err) 523 } 524 return 525 } 526 } 527 }() 528 return nil 529 } 530 531 func newService(ctx *adapters.ServiceContext) (node.Service, error) { 532 addr := network.NewAddr(ctx.Config.Node()) 533 534 kp := network.NewKadParams() 535 kp.MinProxBinSize = testMinProxBinSize 536 537 if ctx.Config.Reachable != nil { 538 kp.Reachable = func(o *network.BzzAddr) bool { 539 return ctx.Config.Reachable(o.ID()) 540 } 541 } 542 kad := network.NewKademlia(addr.Over(), kp) 543 hp := network.NewHiveParams() 544 hp.KeepAliveInterval = time.Duration(200) * time.Millisecond 545 hp.Discovery = discoveryEnabled 546 547 log.Info(fmt.Sprintf("discovery for nodeID %s is %t", ctx.Config.ID.String(), hp.Discovery)) 548 549 config := &network.BzzConfig{ 550 OverlayAddr: addr.Over(), 551 UnderlayAddr: addr.Under(), 552 HiveParams: hp, 553 } 554 555 if persistenceEnabled { 556 log.Info(fmt.Sprintf("persistence enabled for nodeID %s", ctx.Config.ID.String())) 557 store, err := getDbStore(ctx.Config.ID.String()) 558 if err != nil { 559 return nil, err 560 } 561 return network.NewBzz(config, kad, store, nil, nil), nil 562 } 563 564 return network.NewBzz(config, kad, nil, nil, nil), nil 565 }