github.com/MagHErmit/tendermint@v0.282.1/test/e2e/pkg/testnet.go (about) 1 // nolint: gosec 2 package e2e 3 4 import ( 5 "errors" 6 "fmt" 7 "io" 8 "math/rand" 9 "net" 10 "path/filepath" 11 "sort" 12 "strconv" 13 "strings" 14 15 "github.com/MagHErmit/tendermint/crypto" 16 "github.com/MagHErmit/tendermint/crypto/ed25519" 17 "github.com/MagHErmit/tendermint/crypto/secp256k1" 18 rpchttp "github.com/MagHErmit/tendermint/rpc/client/http" 19 mcs "github.com/MagHErmit/tendermint/test/maverick/consensus" 20 ) 21 22 const ( 23 randomSeed int64 = 2308084734268 24 proxyPortFirst uint32 = 5701 25 networkIPv4 = "10.186.73.0/24" 26 networkIPv6 = "fd80:b10c::/48" 27 ) 28 29 type Mode string 30 type Protocol string 31 type Perturbation string 32 33 const ( 34 ModeValidator Mode = "validator" 35 ModeFull Mode = "full" 36 ModeLight Mode = "light" 37 ModeSeed Mode = "seed" 38 39 ProtocolBuiltin Protocol = "builtin" 40 ProtocolFile Protocol = "file" 41 ProtocolGRPC Protocol = "grpc" 42 ProtocolTCP Protocol = "tcp" 43 ProtocolUNIX Protocol = "unix" 44 45 PerturbationDisconnect Perturbation = "disconnect" 46 PerturbationKill Perturbation = "kill" 47 PerturbationPause Perturbation = "pause" 48 PerturbationRestart Perturbation = "restart" 49 ) 50 51 // Testnet represents a single testnet. 52 type Testnet struct { 53 Name string 54 File string 55 Dir string 56 IP *net.IPNet 57 InitialHeight int64 58 InitialState map[string]string 59 Validators map[*Node]int64 60 ValidatorUpdates map[int64]map[*Node]int64 61 Nodes []*Node 62 KeyType string 63 ABCIProtocol string 64 } 65 66 // Node represents a Tendermint node in a testnet. 67 type Node struct { 68 Name string 69 Testnet *Testnet 70 Mode Mode 71 PrivvalKey crypto.PrivKey 72 NodeKey crypto.PrivKey 73 IP net.IP 74 ProxyPort uint32 75 StartAt int64 76 FastSync string 77 StateSync bool 78 Mempool string 79 Database string 80 ABCIProtocol Protocol 81 PrivvalProtocol Protocol 82 PersistInterval uint64 83 SnapshotInterval uint64 84 RetainBlocks uint64 85 Seeds []*Node 86 PersistentPeers []*Node 87 Perturbations []Perturbation 88 Misbehaviors map[int64]string 89 } 90 91 // LoadTestnet loads a testnet from a manifest file, using the filename to 92 // determine the testnet name and directory (from the basename of the file). 93 // The testnet generation must be deterministic, since it is generated 94 // separately by the runner and the test cases. For this reason, testnets use a 95 // random seed to generate e.g. keys. 96 func LoadTestnet(file string) (*Testnet, error) { 97 manifest, err := LoadManifest(file) 98 if err != nil { 99 return nil, err 100 } 101 dir := strings.TrimSuffix(file, filepath.Ext(file)) 102 103 // Set up resource generators. These must be deterministic. 104 netAddress := networkIPv4 105 if manifest.IPv6 { 106 netAddress = networkIPv6 107 } 108 _, ipNet, err := net.ParseCIDR(netAddress) 109 if err != nil { 110 return nil, fmt.Errorf("invalid IP network address %q: %w", netAddress, err) 111 } 112 113 ipGen := newIPGenerator(ipNet) 114 keyGen := newKeyGenerator(randomSeed) 115 proxyPortGen := newPortGenerator(proxyPortFirst) 116 117 testnet := &Testnet{ 118 Name: filepath.Base(dir), 119 File: file, 120 Dir: dir, 121 IP: ipGen.Network(), 122 InitialHeight: 1, 123 InitialState: manifest.InitialState, 124 Validators: map[*Node]int64{}, 125 ValidatorUpdates: map[int64]map[*Node]int64{}, 126 Nodes: []*Node{}, 127 ABCIProtocol: manifest.ABCIProtocol, 128 } 129 if len(manifest.KeyType) != 0 { 130 testnet.KeyType = manifest.KeyType 131 } 132 if manifest.InitialHeight > 0 { 133 testnet.InitialHeight = manifest.InitialHeight 134 } 135 if testnet.ABCIProtocol == "" { 136 testnet.ABCIProtocol = string(ProtocolBuiltin) 137 } 138 139 // Set up nodes, in alphabetical order (IPs and ports get same order). 140 nodeNames := []string{} 141 for name := range manifest.Nodes { 142 nodeNames = append(nodeNames, name) 143 } 144 sort.Strings(nodeNames) 145 146 for _, name := range nodeNames { 147 nodeManifest := manifest.Nodes[name] 148 node := &Node{ 149 Name: name, 150 Testnet: testnet, 151 PrivvalKey: keyGen.Generate(manifest.KeyType), 152 NodeKey: keyGen.Generate("ed25519"), 153 IP: ipGen.Next(), 154 ProxyPort: proxyPortGen.Next(), 155 Mode: ModeValidator, 156 Database: "goleveldb", 157 ABCIProtocol: Protocol(testnet.ABCIProtocol), 158 PrivvalProtocol: ProtocolFile, 159 StartAt: nodeManifest.StartAt, 160 FastSync: nodeManifest.FastSync, 161 Mempool: nodeManifest.Mempool, 162 StateSync: nodeManifest.StateSync, 163 PersistInterval: 1, 164 SnapshotInterval: nodeManifest.SnapshotInterval, 165 RetainBlocks: nodeManifest.RetainBlocks, 166 Perturbations: []Perturbation{}, 167 Misbehaviors: make(map[int64]string), 168 } 169 if node.StartAt == testnet.InitialHeight { 170 node.StartAt = 0 // normalize to 0 for initial nodes, since code expects this 171 } 172 if nodeManifest.Mode != "" { 173 node.Mode = Mode(nodeManifest.Mode) 174 } 175 if node.Mode == ModeLight { 176 node.ABCIProtocol = ProtocolBuiltin 177 } 178 if nodeManifest.Database != "" { 179 node.Database = nodeManifest.Database 180 } 181 if nodeManifest.PrivvalProtocol != "" { 182 node.PrivvalProtocol = Protocol(nodeManifest.PrivvalProtocol) 183 } 184 if nodeManifest.PersistInterval != nil { 185 node.PersistInterval = *nodeManifest.PersistInterval 186 } 187 for _, p := range nodeManifest.Perturb { 188 node.Perturbations = append(node.Perturbations, Perturbation(p)) 189 } 190 for heightString, misbehavior := range nodeManifest.Misbehaviors { 191 height, err := strconv.ParseInt(heightString, 10, 64) 192 if err != nil { 193 return nil, fmt.Errorf("unable to parse height %s to int64: %w", heightString, err) 194 } 195 node.Misbehaviors[height] = misbehavior 196 } 197 testnet.Nodes = append(testnet.Nodes, node) 198 } 199 200 // We do a second pass to set up seeds and persistent peers, which allows graph cycles. 201 for _, node := range testnet.Nodes { 202 nodeManifest, ok := manifest.Nodes[node.Name] 203 if !ok { 204 return nil, fmt.Errorf("failed to look up manifest for node %q", node.Name) 205 } 206 for _, seedName := range nodeManifest.Seeds { 207 seed := testnet.LookupNode(seedName) 208 if seed == nil { 209 return nil, fmt.Errorf("unknown seed %q for node %q", seedName, node.Name) 210 } 211 node.Seeds = append(node.Seeds, seed) 212 } 213 for _, peerName := range nodeManifest.PersistentPeers { 214 peer := testnet.LookupNode(peerName) 215 if peer == nil { 216 return nil, fmt.Errorf("unknown persistent peer %q for node %q", peerName, node.Name) 217 } 218 node.PersistentPeers = append(node.PersistentPeers, peer) 219 } 220 221 // If there are no seeds or persistent peers specified, default to persistent 222 // connections to all other nodes. 223 if len(node.PersistentPeers) == 0 && len(node.Seeds) == 0 { 224 for _, peer := range testnet.Nodes { 225 if peer.Name == node.Name { 226 continue 227 } 228 node.PersistentPeers = append(node.PersistentPeers, peer) 229 } 230 } 231 } 232 233 // Set up genesis validators. If not specified explicitly, use all validator nodes. 234 if manifest.Validators != nil { 235 for validatorName, power := range *manifest.Validators { 236 validator := testnet.LookupNode(validatorName) 237 if validator == nil { 238 return nil, fmt.Errorf("unknown validator %q", validatorName) 239 } 240 testnet.Validators[validator] = power 241 } 242 } else { 243 for _, node := range testnet.Nodes { 244 if node.Mode == ModeValidator { 245 testnet.Validators[node] = 100 246 } 247 } 248 } 249 250 // Set up validator updates. 251 for heightStr, validators := range manifest.ValidatorUpdates { 252 height, err := strconv.Atoi(heightStr) 253 if err != nil { 254 return nil, fmt.Errorf("invalid validator update height %q: %w", height, err) 255 } 256 valUpdate := map[*Node]int64{} 257 for name, power := range validators { 258 node := testnet.LookupNode(name) 259 if node == nil { 260 return nil, fmt.Errorf("unknown validator %q for update at height %v", name, height) 261 } 262 valUpdate[node] = power 263 } 264 testnet.ValidatorUpdates[int64(height)] = valUpdate 265 } 266 267 return testnet, testnet.Validate() 268 } 269 270 // Validate validates a testnet. 271 func (t Testnet) Validate() error { 272 if t.Name == "" { 273 return errors.New("network has no name") 274 } 275 if t.IP == nil { 276 return errors.New("network has no IP") 277 } 278 if len(t.Nodes) == 0 { 279 return errors.New("network has no nodes") 280 } 281 for _, node := range t.Nodes { 282 if err := node.Validate(t); err != nil { 283 return fmt.Errorf("invalid node %q: %w", node.Name, err) 284 } 285 } 286 return nil 287 } 288 289 // Validate validates a node. 290 func (n Node) Validate(testnet Testnet) error { 291 if n.Name == "" { 292 return errors.New("node has no name") 293 } 294 if n.IP == nil { 295 return errors.New("node has no IP address") 296 } 297 if !testnet.IP.Contains(n.IP) { 298 return fmt.Errorf("node IP %v is not in testnet network %v", n.IP, testnet.IP) 299 } 300 if n.ProxyPort > 0 { 301 if n.ProxyPort <= 1024 { 302 return fmt.Errorf("local port %v must be >1024", n.ProxyPort) 303 } 304 for _, peer := range testnet.Nodes { 305 if peer.Name != n.Name && peer.ProxyPort == n.ProxyPort { 306 return fmt.Errorf("peer %q also has local port %v", peer.Name, n.ProxyPort) 307 } 308 } 309 } 310 switch n.FastSync { 311 case "", "v0", "v1", "v2": 312 default: 313 return fmt.Errorf("invalid fast sync setting %q", n.FastSync) 314 315 } 316 switch n.Mempool { 317 case "", "v0", "v1": 318 default: 319 return fmt.Errorf("invalid mempool version %q", n.Mempool) 320 } 321 switch n.Database { 322 case "goleveldb", "cleveldb", "boltdb", "rocksdb", "badgerdb": 323 default: 324 return fmt.Errorf("invalid database setting %q", n.Database) 325 } 326 switch n.ABCIProtocol { 327 case ProtocolBuiltin, ProtocolUNIX, ProtocolTCP, ProtocolGRPC: 328 default: 329 return fmt.Errorf("invalid ABCI protocol setting %q", n.ABCIProtocol) 330 } 331 if n.Mode == ModeLight && n.ABCIProtocol != ProtocolBuiltin { 332 return errors.New("light client must use builtin protocol") 333 } 334 switch n.PrivvalProtocol { 335 case ProtocolFile, ProtocolUNIX, ProtocolTCP: 336 default: 337 return fmt.Errorf("invalid privval protocol setting %q", n.PrivvalProtocol) 338 } 339 340 if n.StartAt > 0 && n.StartAt < n.Testnet.InitialHeight { 341 return fmt.Errorf("cannot start at height %v lower than initial height %v", 342 n.StartAt, n.Testnet.InitialHeight) 343 } 344 if n.StateSync && n.StartAt == 0 { 345 return errors.New("state synced nodes cannot start at the initial height") 346 } 347 if n.PersistInterval == 0 && n.RetainBlocks > 0 { 348 return errors.New("persist_interval=0 requires retain_blocks=0") 349 } 350 if n.PersistInterval > 1 && n.RetainBlocks > 0 && n.RetainBlocks < n.PersistInterval { 351 return errors.New("persist_interval must be less than or equal to retain_blocks") 352 } 353 if n.SnapshotInterval > 0 && n.RetainBlocks > 0 && n.RetainBlocks < n.SnapshotInterval { 354 return errors.New("snapshot_interval must be less than er equal to retain_blocks") 355 } 356 357 for _, perturbation := range n.Perturbations { 358 switch perturbation { 359 case PerturbationDisconnect, PerturbationKill, PerturbationPause, PerturbationRestart: 360 default: 361 return fmt.Errorf("invalid perturbation %q", perturbation) 362 } 363 } 364 365 if (n.PrivvalProtocol != "file" || n.Mode != "validator") && len(n.Misbehaviors) != 0 { 366 return errors.New("must be using \"file\" privval protocol to implement misbehaviors") 367 } 368 369 for height, misbehavior := range n.Misbehaviors { 370 if height < n.StartAt { 371 return fmt.Errorf("misbehavior height %d is below node start height %d", 372 height, n.StartAt) 373 } 374 if height < testnet.InitialHeight { 375 return fmt.Errorf("misbehavior height %d is below network initial height %d", 376 height, testnet.InitialHeight) 377 } 378 exists := false 379 for possibleBehaviors := range mcs.MisbehaviorList { 380 if possibleBehaviors == misbehavior { 381 exists = true 382 } 383 } 384 if !exists { 385 return fmt.Errorf("misbehavior %s does not exist", misbehavior) 386 } 387 } 388 389 return nil 390 } 391 392 // LookupNode looks up a node by name. For now, simply do a linear search. 393 func (t Testnet) LookupNode(name string) *Node { 394 for _, node := range t.Nodes { 395 if node.Name == name { 396 return node 397 } 398 } 399 return nil 400 } 401 402 // ArchiveNodes returns a list of archive nodes that start at the initial height 403 // and contain the entire blockchain history. They are used e.g. as light client 404 // RPC servers. 405 func (t Testnet) ArchiveNodes() []*Node { 406 nodes := []*Node{} 407 for _, node := range t.Nodes { 408 if !node.Stateless() && node.StartAt == 0 && node.RetainBlocks == 0 { 409 nodes = append(nodes, node) 410 } 411 } 412 return nodes 413 } 414 415 // RandomNode returns a random non-seed node. 416 func (t Testnet) RandomNode() *Node { 417 for { 418 node := t.Nodes[rand.Intn(len(t.Nodes))] 419 if node.Mode != ModeSeed { 420 return node 421 } 422 } 423 } 424 425 // IPv6 returns true if the testnet is an IPv6 network. 426 func (t Testnet) IPv6() bool { 427 return t.IP.IP.To4() == nil 428 } 429 430 // HasPerturbations returns whether the network has any perturbations. 431 func (t Testnet) HasPerturbations() bool { 432 for _, node := range t.Nodes { 433 if len(node.Perturbations) > 0 { 434 return true 435 } 436 } 437 return false 438 } 439 440 // LastMisbehaviorHeight returns the height of the last misbehavior. 441 func (t Testnet) LastMisbehaviorHeight() int64 { 442 lastHeight := int64(0) 443 for _, node := range t.Nodes { 444 for height := range node.Misbehaviors { 445 if height > lastHeight { 446 lastHeight = height 447 } 448 } 449 } 450 return lastHeight 451 } 452 453 // Address returns a P2P endpoint address for the node. 454 func (n Node) AddressP2P(withID bool) string { 455 ip := n.IP.String() 456 if n.IP.To4() == nil { 457 // IPv6 addresses must be wrapped in [] to avoid conflict with : port separator 458 ip = fmt.Sprintf("[%v]", ip) 459 } 460 addr := fmt.Sprintf("%v:26656", ip) 461 if withID { 462 addr = fmt.Sprintf("%x@%v", n.NodeKey.PubKey().Address().Bytes(), addr) 463 } 464 return addr 465 } 466 467 // Address returns an RPC endpoint address for the node. 468 func (n Node) AddressRPC() string { 469 ip := n.IP.String() 470 if n.IP.To4() == nil { 471 // IPv6 addresses must be wrapped in [] to avoid conflict with : port separator 472 ip = fmt.Sprintf("[%v]", ip) 473 } 474 return fmt.Sprintf("%v:26657", ip) 475 } 476 477 // Client returns an RPC client for a node. 478 func (n Node) Client() (*rpchttp.HTTP, error) { 479 return rpchttp.New(fmt.Sprintf("http://127.0.0.1:%v", n.ProxyPort), "/websocket") 480 } 481 482 // Stateless returns true if the node is either a seed node or a light node 483 func (n Node) Stateless() bool { 484 return n.Mode == ModeLight || n.Mode == ModeSeed 485 } 486 487 // keyGenerator generates pseudorandom Ed25519 keys based on a seed. 488 type keyGenerator struct { 489 random *rand.Rand 490 } 491 492 func newKeyGenerator(seed int64) *keyGenerator { 493 return &keyGenerator{ 494 random: rand.New(rand.NewSource(seed)), 495 } 496 } 497 498 func (g *keyGenerator) Generate(keyType string) crypto.PrivKey { 499 seed := make([]byte, ed25519.SeedSize) 500 501 _, err := io.ReadFull(g.random, seed) 502 if err != nil { 503 panic(err) // this shouldn't happen 504 } 505 switch keyType { 506 case "secp256k1": 507 return secp256k1.GenPrivKeySecp256k1(seed) 508 case "", "ed25519": 509 return ed25519.GenPrivKeyFromSecret(seed) 510 default: 511 panic("KeyType not supported") // should not make it this far 512 } 513 } 514 515 // portGenerator generates local Docker proxy ports for each node. 516 type portGenerator struct { 517 nextPort uint32 518 } 519 520 func newPortGenerator(firstPort uint32) *portGenerator { 521 return &portGenerator{nextPort: firstPort} 522 } 523 524 func (g *portGenerator) Next() uint32 { 525 port := g.nextPort 526 g.nextPort++ 527 if g.nextPort == 0 { 528 panic("port overflow") 529 } 530 return port 531 } 532 533 // ipGenerator generates sequential IP addresses for each node, using a random 534 // network address. 535 type ipGenerator struct { 536 network *net.IPNet 537 nextIP net.IP 538 } 539 540 func newIPGenerator(network *net.IPNet) *ipGenerator { 541 nextIP := make([]byte, len(network.IP)) 542 copy(nextIP, network.IP) 543 gen := &ipGenerator{network: network, nextIP: nextIP} 544 // Skip network and gateway addresses 545 gen.Next() 546 gen.Next() 547 return gen 548 } 549 550 func (g *ipGenerator) Network() *net.IPNet { 551 n := &net.IPNet{ 552 IP: make([]byte, len(g.network.IP)), 553 Mask: make([]byte, len(g.network.Mask)), 554 } 555 copy(n.IP, g.network.IP) 556 copy(n.Mask, g.network.Mask) 557 return n 558 } 559 560 func (g *ipGenerator) Next() net.IP { 561 ip := make([]byte, len(g.nextIP)) 562 copy(ip, g.nextIP) 563 for i := len(g.nextIP) - 1; i >= 0; i-- { 564 g.nextIP[i]++ 565 if g.nextIP[i] != 0 { 566 break 567 } 568 } 569 return ip 570 }