github.com/hechain20/hechain@v0.0.0-20220316014945-b544036ba106/integration/raft/config_test.go (about) 1 /* 2 Copyright hechain All Rights Reserved. 3 4 SPDX-License-Identifier: Apache-2.0 5 */ 6 7 package raft 8 9 import ( 10 "bytes" 11 "crypto/x509" 12 "encoding/pem" 13 "fmt" 14 "io/ioutil" 15 "os" 16 "path" 17 "path/filepath" 18 "strings" 19 "syscall" 20 "time" 21 22 docker "github.com/fsouza/go-dockerclient" 23 "github.com/golang/protobuf/proto" 24 "github.com/hechain20/hechain/common/crypto/tlsgen" 25 "github.com/hechain20/hechain/integration/nwo" 26 "github.com/hechain20/hechain/integration/nwo/commands" 27 "github.com/hechain20/hechain/integration/ordererclient" 28 "github.com/hechain20/hechain/internal/configtxgen/encoder" 29 "github.com/hechain20/hechain/internal/configtxgen/genesisconfig" 30 "github.com/hechain20/hechain/protoutil" 31 "github.com/hyperledger/fabric-protos-go/common" 32 "github.com/hyperledger/fabric-protos-go/msp" 33 protosorderer "github.com/hyperledger/fabric-protos-go/orderer" 34 "github.com/hyperledger/fabric-protos-go/orderer/etcdraft" 35 . "github.com/onsi/ginkgo" 36 . "github.com/onsi/gomega" 37 "github.com/onsi/gomega/gbytes" 38 "github.com/onsi/gomega/gexec" 39 "github.com/tedsuo/ifrit" 40 "github.com/tedsuo/ifrit/ginkgomon" 41 ) 42 43 var _ = Describe("EndToEnd reconfiguration and onboarding", func() { 44 var ( 45 testDir string 46 client *docker.Client 47 network *nwo.Network 48 peer *nwo.Peer 49 50 networkProcess ifrit.Process 51 ordererProcesses []ifrit.Process 52 ordererRunners []*ginkgomon.Runner 53 ) 54 55 BeforeEach(func() { 56 ordererRunners = nil 57 ordererProcesses = nil 58 59 var err error 60 testDir, err = ioutil.TempDir("", "e2e-etcdraft_reconfig") 61 Expect(err).NotTo(HaveOccurred()) 62 63 client, err = docker.NewClientFromEnv() 64 Expect(err).NotTo(HaveOccurred()) 65 }) 66 67 AfterEach(func() { 68 if networkProcess != nil { 69 networkProcess.Signal(syscall.SIGTERM) 70 Eventually(networkProcess.Wait(), network.EventuallyTimeout).Should(Receive()) 71 } 72 if network != nil { 73 network.Cleanup() 74 } 75 for _, ordererInstance := range ordererProcesses { 76 ordererInstance.Signal(syscall.SIGTERM) 77 Eventually(ordererInstance.Wait(), network.EventuallyTimeout).Should(Receive()) 78 } 79 os.RemoveAll(testDir) 80 }) 81 82 Describe("three node etcdraft network with 2 orgs", func() { 83 BeforeEach(func() { 84 network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, StartPort(), components) 85 network.GenerateConfigTree() 86 network.Bootstrap() 87 88 networkRunner := network.NetworkGroupRunner() 89 networkProcess = ifrit.Invoke(networkRunner) 90 Eventually(networkProcess.Ready(), network.EventuallyTimeout).Should(BeClosed()) 91 }) 92 93 // This tests: 94 // 95 // 1. channel creation with raft orderer, 96 // 2. all the nodes on three-node raft cluster are in sync wrt blocks, 97 // 3. raft orderer processes type A config updates and delivers the 98 // config blocks to the peers. 99 It("executes an etcdraft network with 2 orgs and three orderer nodes", func() { 100 orderer1 := network.Orderer("orderer1") 101 orderer2 := network.Orderer("orderer2") 102 orderer3 := network.Orderer("orderer3") 103 peer := network.Peer("Org1", "peer0") 104 blockFile1 := filepath.Join(testDir, "newest_orderer1_block.pb") 105 blockFile2 := filepath.Join(testDir, "newest_orderer2_block.pb") 106 blockFile3 := filepath.Join(testDir, "newest_orderer3_block.pb") 107 108 By("Ordering service system channel is ready") 109 assertBlockReception(map[string]int{ 110 "systemchannel": 0, 111 }, []*nwo.Orderer{orderer1, orderer2, orderer3}, peer, network) 112 113 fetchLatestBlock := func(targetOrderer *nwo.Orderer, blockFile string) { 114 c := commands.ChannelFetch{ 115 ChannelID: "testchannel", 116 Block: "newest", 117 OutputFile: blockFile, 118 } 119 if targetOrderer != nil { 120 c.Orderer = network.OrdererAddress(targetOrderer, nwo.ListenPort) 121 } 122 sess, err := network.PeerAdminSession(peer, c) 123 Expect(err).NotTo(HaveOccurred()) 124 Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(0)) 125 } 126 127 By("Creating a new channel") 128 network.CreateChannel("testchannel", orderer1, peer) 129 130 // the above can work even if the orderer nodes are not in the same Raft 131 // cluster; we need to verify all the three orderer nodes are in sync wrt 132 // blocks. 133 By("Fetching the latest blocks from all the orderer nodes and testing them for equality") 134 fetchLatestBlock(orderer1, blockFile1) 135 fetchLatestBlock(orderer2, blockFile2) 136 fetchLatestBlock(orderer3, blockFile3) 137 b1 := nwo.UnmarshalBlockFromFile(blockFile1) 138 b2 := nwo.UnmarshalBlockFromFile(blockFile2) 139 b3 := nwo.UnmarshalBlockFromFile(blockFile3) 140 Expect(protoutil.BlockHeaderBytes(b1.Header)).To(Equal(protoutil.BlockHeaderBytes(b2.Header))) 141 Expect(protoutil.BlockHeaderBytes(b2.Header)).To(Equal(protoutil.BlockHeaderBytes(b3.Header))) 142 }) 143 }) 144 145 Describe("Invalid Raft config metadata", func() { 146 It("refuses to start orderer or rejects config update", func() { 147 By("Creating malformed genesis block") 148 network = nwo.New(nwo.BasicEtcdRaft(), testDir, client, StartPort(), components) 149 network.GenerateConfigTree() 150 network.Bootstrap() 151 152 sysProfile := genesisconfig.Load(network.SystemChannel.Profile, network.RootDir) 153 Expect(sysProfile.Orderer).NotTo(BeNil()) 154 sysProfile.Orderer.EtcdRaft.Options.ElectionTick = sysProfile.Orderer.EtcdRaft.Options.HeartbeatTick 155 pgen := encoder.New(sysProfile) 156 genesisBlock := pgen.GenesisBlockForChannel(network.SystemChannel.Name) 157 data, err := proto.Marshal(genesisBlock) 158 Expect(err).NotTo(HaveOccurred()) 159 err = ioutil.WriteFile(network.OutputBlockPath(network.SystemChannel.Name), data, 0o644) 160 Expect(err).NotTo(HaveOccurred()) 161 162 By("Starting orderer with malformed genesis block") 163 ordererRunner := network.OrdererGroupRunner() 164 process := ifrit.Invoke(ordererRunner) 165 Eventually(process.Wait(), network.EventuallyTimeout).Should(Receive()) // orderer process should exit 166 network.Cleanup() 167 os.RemoveAll(testDir) 168 169 By("Starting orderer with correct genesis block") 170 testDir, err = ioutil.TempDir("", "e2e") 171 Expect(err).NotTo(HaveOccurred()) 172 network = nwo.New(nwo.BasicEtcdRaft(), testDir, client, StartPort(), components) 173 network.GenerateConfigTree() 174 network.Bootstrap() 175 176 orderer := network.Orderer("orderer") 177 runner := network.OrdererRunner(orderer) 178 process = ifrit.Invoke(runner) 179 Eventually(process.Ready, network.EventuallyTimeout).Should(BeClosed()) 180 defer func() { 181 process.Signal(syscall.SIGTERM) 182 Eventually(process.Wait(), network.EventuallyTimeout).Should(Receive()) 183 }() 184 185 By("Waiting for system channel to be ready") 186 findLeader([]*ginkgomon.Runner{runner}) 187 188 By("Creating malformed channel creation config tx") 189 channel := "testchannel" 190 sysProfile = genesisconfig.Load(network.SystemChannel.Profile, network.RootDir) 191 Expect(sysProfile.Orderer).NotTo(BeNil()) 192 appProfile := genesisconfig.Load(network.ProfileForChannel(channel), network.RootDir) 193 Expect(appProfile).NotTo(BeNil()) 194 o := *sysProfile.Orderer 195 appProfile.Orderer = &o 196 appProfile.Orderer.EtcdRaft = proto.Clone(sysProfile.Orderer.EtcdRaft).(*etcdraft.ConfigMetadata) 197 appProfile.Orderer.EtcdRaft.Options.HeartbeatTick = appProfile.Orderer.EtcdRaft.Options.ElectionTick 198 configtx, err := encoder.MakeChannelCreationTransactionWithSystemChannelContext(channel, nil, appProfile, sysProfile) 199 Expect(err).NotTo(HaveOccurred()) 200 data, err = proto.Marshal(configtx) 201 Expect(err).NotTo(HaveOccurred()) 202 err = ioutil.WriteFile(network.CreateChannelTxPath(channel), data, 0o644) 203 Expect(err).NotTo(HaveOccurred()) 204 205 By("Submitting malformed channel creation config tx to orderer") 206 org1Peer0 := network.Peer("Org1", "peer0") 207 org2Peer0 := network.Peer("Org2", "peer0") 208 209 exitCode := network.CreateChannelExitCode(channel, orderer, org1Peer0, org1Peer0, org2Peer0, orderer) 210 Expect(exitCode).NotTo(Equal(0)) 211 Consistently(process.Wait()).ShouldNot(Receive()) // malformed tx should not crash orderer 212 Expect(runner.Err()).To(gbytes.Say(`invalid new config metadata: ElectionTick \(10\) must be greater than HeartbeatTick \(10\)`)) 213 214 By("Submitting channel config update with illegal value") 215 channel = network.SystemChannel.Name 216 config := nwo.GetConfig(network, org1Peer0, orderer, channel) 217 updatedConfig := proto.Clone(config).(*common.Config) 218 219 consensusTypeConfigValue := updatedConfig.ChannelGroup.Groups["Orderer"].Values["ConsensusType"] 220 consensusTypeValue := &protosorderer.ConsensusType{} 221 Expect(proto.Unmarshal(consensusTypeConfigValue.Value, consensusTypeValue)).To(Succeed()) 222 223 metadata := &etcdraft.ConfigMetadata{} 224 Expect(proto.Unmarshal(consensusTypeValue.Metadata, metadata)).To(Succeed()) 225 226 metadata.Options.HeartbeatTick = 10 227 metadata.Options.ElectionTick = 10 228 229 newMetadata, err := proto.Marshal(metadata) 230 Expect(err).NotTo(HaveOccurred()) 231 consensusTypeValue.Metadata = newMetadata 232 233 updatedConfig.ChannelGroup.Groups["Orderer"].Values["ConsensusType"] = &common.ConfigValue{ 234 ModPolicy: "Admins", 235 Value: protoutil.MarshalOrPanic(consensusTypeValue), 236 } 237 238 sess := nwo.UpdateOrdererConfigSession(network, orderer, channel, config, updatedConfig, org1Peer0, orderer) 239 Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(1)) 240 Expect(sess.Err).To(gbytes.Say(`invalid new config metadata: ElectionTick \(10\) must be greater than HeartbeatTick \(10\)`)) 241 }) 242 }) 243 244 When("a single node cluster is expanded", func() { 245 It("is still possible to onboard the new cluster member and then another one with a different TLS root CA", func() { 246 launch := func(o *nwo.Orderer) { 247 runner := network.OrdererRunner(o) 248 ordererRunners = append(ordererRunners, runner) 249 250 process := ifrit.Invoke(runner) 251 Eventually(process.Ready(), network.EventuallyTimeout).Should(BeClosed()) 252 ordererProcesses = append(ordererProcesses, process) 253 } 254 255 network = nwo.New(nwo.BasicEtcdRaft(), testDir, client, StartPort(), components) 256 network.GenerateConfigTree() 257 network.Bootstrap() 258 259 orderer := network.Orderer("orderer") 260 peer = network.Peer("Org1", "peer0") 261 262 By("Launching the orderer") 263 launch(orderer) 264 265 By("Checking that it elected itself as a leader") 266 findLeader(ordererRunners) 267 268 By("Extending the network configuration to add a new orderer") 269 orderer2 := &nwo.Orderer{ 270 Name: "orderer2", 271 Organization: "OrdererOrg", 272 } 273 ports := nwo.Ports{} 274 for _, portName := range nwo.OrdererPortNames() { 275 ports[portName] = network.ReservePort() 276 } 277 network.PortsByOrdererID[orderer2.ID()] = ports 278 network.Orderers = append(network.Orderers, orderer2) 279 network.GenerateOrdererConfig(orderer2) 280 extendNetwork(network) 281 282 secondOrdererCertificatePath := filepath.Join(network.OrdererLocalTLSDir(orderer2), "server.crt") 283 secondOrdererCertificate, err := ioutil.ReadFile(secondOrdererCertificatePath) 284 Expect(err).NotTo(HaveOccurred()) 285 286 By("Adding the second orderer") 287 addConsenter(network, peer, orderer, "systemchannel", etcdraft.Consenter{ 288 ServerTlsCert: secondOrdererCertificate, 289 ClientTlsCert: secondOrdererCertificate, 290 Host: "127.0.0.1", 291 Port: uint32(network.OrdererPort(orderer2, nwo.ClusterPort)), 292 }) 293 294 By("Obtaining the last config block from the orderer") 295 // Get the last config block of the system channel 296 configBlock := nwo.GetConfigBlock(network, peer, orderer, "systemchannel") 297 // Plant it in the file system of orderer2, the new node to be onboarded. 298 err = ioutil.WriteFile(filepath.Join(testDir, "systemchannel_block.pb"), protoutil.MarshalOrPanic(configBlock), 0o644) 299 Expect(err).NotTo(HaveOccurred()) 300 301 By("Waiting for the existing orderer to relinquish its leadership") 302 Eventually(ordererRunners[0].Err(), network.EventuallyTimeout).Should(gbytes.Say("1 stepped down to follower since quorum is not active")) 303 Eventually(ordererRunners[0].Err(), network.EventuallyTimeout).Should(gbytes.Say("No leader is present, cluster size is 2")) 304 By("Launching the second orderer") 305 launch(orderer2) 306 By("Waiting for a leader to be re-elected") 307 findLeader(ordererRunners) 308 309 // In the next part of the test we're going to bring up a third node 310 // with a different TLS root CA. We're then going to remove the TLS 311 // root CA and restart the orderer, to ensure that we can dynamically 312 // update TLS root CAs in Raft while membership stays the same. 313 314 By("Creating configuration for a third orderer with a different TLS root CA") 315 orderer3 := &nwo.Orderer{ 316 Name: "orderer3", 317 Organization: "OrdererOrg", 318 } 319 ports = nwo.Ports{} 320 for _, portName := range nwo.OrdererPortNames() { 321 ports[portName] = network.ReservePort() 322 } 323 network.PortsByOrdererID[orderer3.ID()] = ports 324 network.Orderers = append(network.Orderers, orderer3) 325 network.GenerateOrdererConfig(orderer3) 326 327 tmpDir, err := ioutil.TempDir("", "e2e-etcfraft_reconfig") 328 Expect(err).NotTo(HaveOccurred()) 329 defer os.RemoveAll(tmpDir) 330 331 sess, err := network.Cryptogen(commands.Generate{ 332 Config: network.CryptoConfigPath(), 333 Output: tmpDir, 334 }) 335 Expect(err).NotTo(HaveOccurred()) 336 Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(0)) 337 338 name := network.Orderers[0].Name 339 domain := network.Organization(network.Orderers[0].Organization).Domain 340 nameDomain := fmt.Sprintf("%s.%s", name, domain) 341 ordererTLSPath := filepath.Join(tmpDir, "ordererOrganizations", domain, "orderers", nameDomain, "tls") 342 343 caCertPath := filepath.Join(tmpDir, "ordererOrganizations", domain, "tlsca", fmt.Sprintf("tlsca.%s-cert.pem", domain)) 344 345 caCert, err := ioutil.ReadFile(caCertPath) 346 Expect(err).NotTo(HaveOccurred()) 347 348 thirdOrdererCertificatePath := filepath.Join(ordererTLSPath, "server.crt") 349 thirdOrdererCertificate, err := ioutil.ReadFile(thirdOrdererCertificatePath) 350 Expect(err).NotTo(HaveOccurred()) 351 352 By("Updating it on the file system") 353 err = ioutil.WriteFile(caCertPath, caCert, 0o644) 354 Expect(err).NotTo(HaveOccurred()) 355 err = ioutil.WriteFile(thirdOrdererCertificatePath, thirdOrdererCertificate, 0o644) 356 Expect(err).NotTo(HaveOccurred()) 357 358 By("Overwriting the TLS directory of the new orderer") 359 for _, fileName := range []string{"server.crt", "server.key", "ca.crt"} { 360 dst := filepath.Join(network.OrdererLocalTLSDir(orderer3), fileName) 361 362 data, err := ioutil.ReadFile(filepath.Join(ordererTLSPath, fileName)) 363 Expect(err).NotTo(HaveOccurred()) 364 365 err = ioutil.WriteFile(dst, data, 0o644) 366 Expect(err).NotTo(HaveOccurred()) 367 } 368 369 By("Obtaining the last config block from the orderer once more to update the bootstrap file") 370 configBlock = nwo.GetConfigBlock(network, peer, orderer, "systemchannel") 371 err = ioutil.WriteFile(filepath.Join(testDir, "systemchannel_block.pb"), protoutil.MarshalOrPanic(configBlock), 0o644) 372 Expect(err).NotTo(HaveOccurred()) 373 374 By("Launching orderer3") 375 launch(orderer3) 376 377 By("Expanding the TLS root CA certificates and adding orderer3 to the channel") 378 updateOrdererMSPAndConsensusMetadata(network, peer, orderer, "systemchannel", "OrdererOrg", func(config msp.FabricMSPConfig) msp.FabricMSPConfig { 379 config.TlsRootCerts = append(config.TlsRootCerts, caCert) 380 return config 381 }, 382 func(metadata *etcdraft.ConfigMetadata) { 383 metadata.Consenters = append(metadata.Consenters, &etcdraft.Consenter{ 384 ServerTlsCert: thirdOrdererCertificate, 385 ClientTlsCert: thirdOrdererCertificate, 386 Host: "127.0.0.1", 387 Port: uint32(network.OrdererPort(orderer3, nwo.ClusterPort)), 388 }) 389 }, 390 ) 391 392 By("Waiting for orderer3 to see the leader") 393 findLeader([]*ginkgomon.Runner{ordererRunners[2]}) 394 395 By("Attemping to add a consenter with invalid certs") 396 // create new certs that are not in the channel config 397 ca, err := tlsgen.NewCA() 398 Expect(err).NotTo(HaveOccurred()) 399 client, err := ca.NewClientCertKeyPair() 400 Expect(err).NotTo(HaveOccurred()) 401 402 newConsenterCertPem, _ := pem.Decode(client.Cert) 403 newConsenterCert, err := x509.ParseCertificate(newConsenterCertPem.Bytes) 404 Expect(err).NotTo(HaveOccurred()) 405 406 newConsenterHost := "127.0.0.1" 407 newConsenterPort := uint32(network.OrdererPort(orderer3, nwo.ListenPort)) 408 409 current, updated := consenterAdder( 410 network, 411 peer, 412 orderer, 413 "systemchannel", 414 etcdraft.Consenter{ 415 ServerTlsCert: client.Cert, 416 ClientTlsCert: client.Cert, 417 Host: newConsenterHost, 418 Port: newConsenterPort, 419 }, 420 ) 421 sess = nwo.UpdateOrdererConfigSession(network, orderer, network.SystemChannel.Name, current, updated, peer, orderer) 422 Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(1)) 423 Expect(sess.Err).To(gbytes.Say(fmt.Sprintf("BAD_REQUEST -- error applying config update to existing channel 'systemchannel': consensus metadata update for channel config update is invalid: invalid new config metadata: consenter %s:%d has invalid certificate: verifying tls client cert with serial number %d: x509: certificate signed by unknown authority", newConsenterHost, newConsenterPort, newConsenterCert.SerialNumber))) 424 }) 425 }) 426 427 When("a single node cluster has the tick interval overridden", func() { 428 It("reflects this in its startup logs", func() { 429 network = nwo.New(nwo.BasicEtcdRaft(), testDir, client, StartPort(), components) 430 network.GenerateConfigTree() 431 network.Bootstrap() 432 433 orderer := network.Orderer("orderer") 434 ordererConfig := network.ReadOrdererConfig(orderer) 435 ordererConfig.Consensus["TickIntervalOverride"] = "642ms" 436 network.WriteOrdererConfig(orderer, ordererConfig) 437 438 By("Launching the orderer") 439 runner := network.OrdererRunner(orderer) 440 ordererRunners = append(ordererRunners, runner) 441 442 process := ifrit.Invoke(runner) 443 Eventually(process.Ready(), network.EventuallyTimeout).Should(BeClosed()) 444 ordererProcesses = append(ordererProcesses, process) 445 446 Eventually(runner.Err()).Should(gbytes.Say("TickIntervalOverride is set, overriding channel configuration tick interval to 642ms")) 447 }) 448 }) 449 450 When("the orderer certificates are all rotated", func() { 451 It("is possible to rotate certificate by adding & removing cert in single config", func() { 452 layout := nwo.MultiNodeEtcdRaft() 453 network = nwo.New(layout, testDir, client, StartPort(), components) 454 network.GenerateConfigTree() 455 network.Bootstrap() 456 457 o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3") 458 orderers := []*nwo.Orderer{o1, o2, o3} 459 peer = network.Peer("Org1", "peer0") 460 461 By("Launching the orderers") 462 for _, o := range orderers { 463 runner := network.OrdererRunner(o) 464 ordererRunners = append(ordererRunners, runner) 465 process := ifrit.Invoke(runner) 466 ordererProcesses = append(ordererProcesses, process) 467 } 468 469 for _, ordererProc := range ordererProcesses { 470 Eventually(ordererProc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 471 } 472 473 By("Finding leader") 474 leader := findLeader(ordererRunners) 475 leaderIndex := leader - 1 476 blockSeq := 0 477 478 By("Checking that all orderers are online") 479 assertBlockReception(map[string]int{ 480 "systemchannel": blockSeq, 481 }, orderers, peer, network) 482 483 By("Preparing new certificates for the orderer nodes") 484 extendNetwork(network) 485 certificateRotations := refreshOrdererPEMs(network) 486 487 swap := func(o *nwo.Orderer, certificate []byte, c etcdraft.Consenter) { 488 updateEtcdRaftMetadata(network, peer, o, network.SystemChannel.Name, func(metadata *etcdraft.ConfigMetadata) { 489 var newConsenters []*etcdraft.Consenter 490 for _, consenter := range metadata.Consenters { 491 if bytes.Equal(consenter.ClientTlsCert, certificate) || bytes.Equal(consenter.ServerTlsCert, certificate) { 492 continue 493 } 494 newConsenters = append(newConsenters, consenter) 495 } 496 newConsenters = append(newConsenters, &c) 497 498 metadata.Consenters = newConsenters 499 }) 500 blockSeq++ 501 } 502 503 rotate := func(target int) { 504 // submit a config tx to rotate the cert of an orderer. 505 // The orderer being rotated is going to be unavailable 506 // eventually, therefore submitter of tx is different 507 // from the target, so the configuration can be reliably 508 // checked. 509 submitter := (target + 1) % 3 510 rotation := certificateRotations[target] 511 targetOrderer := network.Orderers[target] 512 remainder := func() []*nwo.Orderer { 513 var ret []*nwo.Orderer 514 for i, o := range network.Orderers { 515 if i == target { 516 continue 517 } 518 ret = append(ret, o) 519 } 520 return ret 521 }() 522 submitterOrderer := network.Orderers[submitter] 523 port := network.OrdererPort(targetOrderer, nwo.ClusterPort) 524 525 fmt.Fprintf(GinkgoWriter, "Rotating certificate of orderer node %d\n", target+1) 526 swap(submitterOrderer, rotation.oldCert, etcdraft.Consenter{ 527 ServerTlsCert: rotation.newCert, 528 ClientTlsCert: rotation.newCert, 529 Host: "127.0.0.1", 530 Port: uint32(port), 531 }) 532 533 By("Waiting for all orderers to sync") 534 assertBlockReception(map[string]int{ 535 "systemchannel": blockSeq, 536 }, remainder, peer, network) 537 538 By("Waiting for rotated node to be unavailable") 539 c := commands.ChannelFetch{ 540 ChannelID: network.SystemChannel.Name, 541 Block: "newest", 542 OutputFile: "/dev/null", 543 Orderer: network.OrdererAddress(targetOrderer, nwo.ClusterPort), 544 } 545 Eventually(func() string { 546 sess, err := network.OrdererAdminSession(targetOrderer, peer, c) 547 Expect(err).NotTo(HaveOccurred()) 548 Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit()) 549 if sess.ExitCode() != 0 { 550 return fmt.Sprintf("exit code is %d: %s", sess.ExitCode(), string(sess.Err.Contents())) 551 } 552 sessErr := string(sess.Err.Contents()) 553 expected := fmt.Sprintf("Received block: %d", blockSeq) 554 if strings.Contains(sessErr, expected) { 555 return "" 556 } 557 return sessErr 558 }, network.EventuallyTimeout, time.Second).ShouldNot(BeEmpty()) 559 560 By("Killing the orderer") 561 ordererProcesses[target].Signal(syscall.SIGTERM) 562 Eventually(ordererProcesses[target].Wait(), network.EventuallyTimeout).Should(Receive()) 563 564 By("Starting the orderer again") 565 ordererRunner := network.OrdererRunner(targetOrderer) 566 ordererRunners = append(ordererRunners, ordererRunner) 567 ordererProcesses[target] = ifrit.Invoke(ordererRunner) 568 Eventually(ordererProcesses[target].Ready(), network.EventuallyTimeout).Should(BeClosed()) 569 570 By("And waiting for it to stabilize") 571 assertBlockReception(map[string]int{ 572 "systemchannel": blockSeq, 573 }, orderers, peer, network) 574 } 575 576 By(fmt.Sprintf("Rotating cert on leader %d", leader)) 577 rotate(leaderIndex) 578 579 By("Rotating certificates of other orderer nodes") 580 for i := range certificateRotations { 581 if i != leaderIndex { 582 rotate(i) 583 } 584 } 585 }) 586 587 It("is still possible to onboard new orderers", func() { 588 // In this test, we have 3 OSNs and we rotate their TLS certificates one by one, 589 // by adding the future certificate to the channel, killing the OSN to make it 590 // grab the new certificate, and then removing the old certificate from the channel. 591 592 // After we completely rotate all the certificates, we put the last config block 593 // of the system channel into the file system of orderer4, and then launch it, 594 // and ensure it onboards and pulls channels testchannel only, and not testchannel2 595 // which it is not part of. 596 597 // Consenter i after its certificate is rotated is denoted as consenter i' 598 // The blocks of channels contain the following updates: 599 // | system channel height | testchannel height | update description 600 // ------------------------------------------------------------------------ 601 // 0 | 2 | 1 | adding consenter 1' 602 // 1 | 3 | 2 | removing consenter 1 603 // 2 | 4 | 3 | adding consenter 2' 604 // 3 | 5 | 4 | removing consenter 2 605 // 4 | 6 | 5 | adding consenter 3' 606 // 5 | 7 | 6 | removing consenter 3 607 // 6 | 8 | 6 | creating channel testchannel2 608 // 7 | 9 | 6 | creating channel testchannel3 609 // 8 | 10 | 7 | adding consenter 4 610 611 layout := nwo.MultiNodeEtcdRaft() 612 layout.Channels = append(layout.Channels, &nwo.Channel{ 613 Name: "testchannel2", 614 Profile: "TwoOrgsChannel", 615 }, &nwo.Channel{ 616 Name: "testchannel3", 617 Profile: "TwoOrgsChannel", 618 }) 619 620 network = nwo.New(layout, testDir, client, StartPort(), components) 621 network.GenerateConfigTree() 622 network.Bootstrap() 623 624 o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3") 625 orderers := []*nwo.Orderer{o1, o2, o3} 626 peer = network.Peer("Org1", "peer0") 627 628 By("Launching the orderers") 629 for _, o := range orderers { 630 runner := network.OrdererRunner(o) 631 ordererRunners = append(ordererRunners, runner) 632 process := ifrit.Invoke(runner) 633 ordererProcesses = append(ordererProcesses, process) 634 } 635 636 for _, ordererProc := range ordererProcesses { 637 Eventually(ordererProc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 638 } 639 640 By("Checking that all orderers are online") 641 assertBlockReception(map[string]int{ 642 "systemchannel": 0, 643 }, orderers, peer, network) 644 645 By("Creating a channel and checking that all orderers got the channel creation") 646 network.CreateChannel("testchannel", network.Orderers[0], peer) 647 assertBlockReception(map[string]int{ 648 "systemchannel": 1, 649 "testchannel": 0, 650 }, orderers, peer, network) 651 652 By("Preparing new certificates for the orderer nodes") 653 extendNetwork(network) 654 certificateRotations := refreshOrdererPEMs(network) 655 656 expectedBlockNumPerChannel := []map[string]int{ 657 {"systemchannel": 2, "testchannel": 1}, 658 {"systemchannel": 3, "testchannel": 2}, 659 {"systemchannel": 4, "testchannel": 3}, 660 {"systemchannel": 5, "testchannel": 4}, 661 {"systemchannel": 6, "testchannel": 5}, 662 {"systemchannel": 7, "testchannel": 6}, 663 } 664 665 for i, rotation := range certificateRotations { 666 o := network.Orderers[i] 667 port := network.OrdererPort(o, nwo.ClusterPort) 668 669 By(fmt.Sprintf("Adding the future certificate of orderer node %d", i)) 670 for _, channelName := range []string{"systemchannel", "testchannel"} { 671 addConsenter(network, peer, o, channelName, etcdraft.Consenter{ 672 ServerTlsCert: rotation.newCert, 673 ClientTlsCert: rotation.newCert, 674 Host: "127.0.0.1", 675 Port: uint32(port), 676 }) 677 } 678 679 By("Waiting for all orderers to sync") 680 assertBlockReception(expectedBlockNumPerChannel[i*2], orderers, peer, network) 681 682 By("Killing the orderer") 683 ordererProcesses[i].Signal(syscall.SIGTERM) 684 Eventually(ordererProcesses[i].Wait(), network.EventuallyTimeout).Should(Receive()) 685 686 By("Starting the orderer again") 687 ordererRunner := network.OrdererRunner(orderers[i]) 688 ordererRunners = append(ordererRunners, ordererRunner) 689 ordererProcesses[i] = ifrit.Invoke(ordererRunner) 690 Eventually(ordererProcesses[i].Ready(), network.EventuallyTimeout).Should(BeClosed()) 691 692 By("And waiting for it to stabilize") 693 assertBlockReception(expectedBlockNumPerChannel[i*2], orderers, peer, network) 694 695 By("Removing the previous certificate of the old orderer") 696 for _, channelName := range []string{"systemchannel", "testchannel"} { 697 removeConsenter(network, peer, network.Orderers[(i+1)%len(network.Orderers)], channelName, rotation.oldCert) 698 } 699 700 By("Waiting for all orderers to sync") 701 assertBlockReception(expectedBlockNumPerChannel[i*2+1], orderers, peer, network) 702 } 703 704 By("Creating testchannel2") 705 network.CreateChannel("testchannel2", network.Orderers[0], peer) 706 assertBlockReception(map[string]int{ 707 "systemchannel": 8, 708 }, orderers, peer, network) 709 710 By("Creating testchannel3") 711 network.CreateChannel("testchannel3", network.Orderers[0], peer) 712 assertBlockReception(map[string]int{ 713 "systemchannel": 9, 714 }, orderers, peer, network) 715 716 o4 := &nwo.Orderer{ 717 Name: "orderer4", 718 Organization: "OrdererOrg", 719 } 720 721 By("Configuring orderer4 in the network") 722 ports := nwo.Ports{} 723 for _, portName := range nwo.OrdererPortNames() { 724 ports[portName] = network.ReservePort() 725 } 726 network.PortsByOrdererID[o4.ID()] = ports 727 728 network.Orderers = append(network.Orderers, o4) 729 network.GenerateOrdererConfig(network.Orderer("orderer4")) 730 731 By("Adding orderer4 to the channels") 732 orderer4CertificatePath := filepath.Join(network.OrdererLocalTLSDir(o4), "server.crt") 733 orderer4Certificate, err := ioutil.ReadFile(orderer4CertificatePath) 734 Expect(err).NotTo(HaveOccurred()) 735 for _, channel := range []string{"systemchannel", "testchannel"} { 736 addConsenter(network, peer, o1, channel, etcdraft.Consenter{ 737 ServerTlsCert: orderer4Certificate, 738 ClientTlsCert: orderer4Certificate, 739 Host: "127.0.0.1", 740 Port: uint32(network.OrdererPort(o4, nwo.ClusterPort)), 741 }) 742 } 743 744 By("Ensuring all orderers know about orderer4's addition") 745 assertBlockReception(map[string]int{ 746 "systemchannel": 10, 747 "testchannel": 7, 748 }, orderers, peer, network) 749 750 By("Broadcasting envelope to testchannel") 751 env := CreateBroadcastEnvelope(network, peer, "testchannel", []byte("hello")) 752 resp, err := ordererclient.Broadcast(network, o1, env) 753 Expect(err).NotTo(HaveOccurred()) 754 Expect(resp.Status).To(Equal(common.Status_SUCCESS)) 755 756 assertBlockReception(map[string]int{ 757 "testchannel": 8, 758 }, orderers, peer, network) 759 760 By("Corrupting the readers policy of testchannel3") 761 revokeReaderAccess(network, "testchannel3", o3, peer) 762 763 // Get the last config block of the system channel 764 configBlock := nwo.GetConfigBlock(network, peer, o1, "systemchannel") 765 // Plant it in the file system of orderer4, the new node to be onboarded. 766 err = ioutil.WriteFile(filepath.Join(testDir, "systemchannel_block.pb"), protoutil.MarshalOrPanic(configBlock), 0o644) 767 Expect(err).NotTo(HaveOccurred()) 768 769 By("Launching orderer4") 770 orderers = append(orderers, o4) 771 orderer4Runner := network.OrdererRunner(o4) 772 ordererRunners = append(ordererRunners, orderer4Runner) 773 // Spawn orderer4's process 774 o4process := ifrit.Invoke(orderer4Runner) 775 Eventually(o4process.Ready(), network.EventuallyTimeout).Should(BeClosed()) 776 ordererProcesses = append(ordererProcesses, o4process) 777 778 By("And waiting for it to sync with the rest of the orderers") 779 assertBlockReception(map[string]int{ 780 "systemchannel": 10, 781 "testchannel": 8, 782 }, orderers, peer, network) 783 784 By("Ensuring orderer4 doesn't serve testchannel2 and testchannel3") 785 env = CreateBroadcastEnvelope(network, peer, "testchannel2", []byte("hello")) 786 resp, err = ordererclient.Broadcast(network, o4, env) 787 Expect(err).NotTo(HaveOccurred()) 788 Expect(resp.Status).To(Equal(common.Status_SERVICE_UNAVAILABLE)) 789 790 env = CreateBroadcastEnvelope(network, peer, "testchannel3", []byte("hello")) 791 resp, err = ordererclient.Broadcast(network, o4, env) 792 Expect(err).NotTo(HaveOccurred()) 793 Expect(resp.Status).To(Equal(common.Status_SERVICE_UNAVAILABLE)) 794 795 belongRegex := `\QI do not belong to channel testchannel2 or am forbidden pulling it (not in the channel), skipping chain retrieval\E` 796 forbiddenRegex := `\QI do not belong to channel testchannel3 or am forbidden pulling it (forbidden pulling the channel), skipping chain retrieval\E` 797 798 Expect(orderer4Runner.Err()).To(gbytes.Say(belongRegex + "|" + forbiddenRegex)) 799 Expect(orderer4Runner.Err()).To(gbytes.Say(belongRegex + "|" + forbiddenRegex)) 800 801 By("Adding orderer4 to testchannel2") 802 addConsenter(network, peer, o1, "testchannel2", etcdraft.Consenter{ 803 ServerTlsCert: orderer4Certificate, 804 ClientTlsCert: orderer4Certificate, 805 Host: "127.0.0.1", 806 Port: uint32(network.OrdererPort(o4, nwo.ClusterPort)), 807 }) 808 809 By("Waiting for orderer4 and to replicate testchannel2") 810 assertBlockReception(map[string]int{ 811 "testchannel2": 1, 812 }, []*nwo.Orderer{o4}, peer, network) 813 814 By("Ensuring orderer4 doesn't have any errors in the logs") 815 Consistently(orderer4Runner.Err()).ShouldNot(gbytes.Say("ERRO")) 816 817 By("Submitting a transaction through orderer4") 818 env = CreateBroadcastEnvelope(network, peer, "testchannel2", []byte("hello")) 819 resp, err = ordererclient.Broadcast(network, o4, env) 820 Expect(err).NotTo(HaveOccurred()) 821 Expect(resp.Status).To(Equal(common.Status_SUCCESS)) 822 823 By("And ensuring it is propagated amongst all orderers") 824 assertBlockReception(map[string]int{ 825 "testchannel2": 2, 826 }, orderers, peer, network) 827 }) 828 }) 829 830 When("an orderer channel is created with a subset of nodes", func() { 831 It("is still possible to onboard a new orderer to the channel", func() { 832 network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, StartPort(), components) 833 network.Profiles = append(network.Profiles, &nwo.Profile{ 834 Name: "myprofile", 835 Consortium: "SampleConsortium", 836 Orderers: []string{"orderer1"}, 837 Organizations: []string{"Org1"}, 838 }) 839 network.Channels = append(network.Channels, &nwo.Channel{ 840 Name: "mychannel", 841 Profile: "myprofile", 842 BaseProfile: "SampleDevModeEtcdRaft", 843 }) 844 845 network.GenerateConfigTree() 846 network.Bootstrap() 847 848 o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3") 849 orderers := []*nwo.Orderer{o1, o2, o3} 850 peer = network.Peer("Org1", "peer0") 851 852 By("Launching the orderers") 853 for _, o := range orderers { 854 runner := network.OrdererRunner(o) 855 ordererRunners = append(ordererRunners, runner) 856 process := ifrit.Invoke(runner) 857 ordererProcesses = append(ordererProcesses, process) 858 } 859 860 for _, ordererProc := range ordererProcesses { 861 Eventually(ordererProc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 862 } 863 864 By("Waiting for the system channel to be available") 865 assertBlockReception(map[string]int{ 866 "systemchannel": 0, 867 }, orderers, peer, network) 868 869 By("Creating a channel with a subset of orderers") 870 network.CreateChannel("mychannel", o1, peer, peer, o1) 871 872 By("Waiting for the channel to be available") 873 assertBlockReception(map[string]int{ 874 "mychannel": 0, 875 }, []*nwo.Orderer{o1}, peer, network) 876 877 By("Ensuring only orderer1 services the channel") 878 ensureEvicted(o2, peer, network, "mychannel") 879 ensureEvicted(o3, peer, network, "mychannel") 880 881 By("Adding orderer2 to the channel") 882 ordererCertificatePath := filepath.Join(network.OrdererLocalTLSDir(o2), "server.crt") 883 ordererCertificate, err := ioutil.ReadFile(ordererCertificatePath) 884 Expect(err).NotTo(HaveOccurred()) 885 886 addConsenter(network, peer, o1, "mychannel", etcdraft.Consenter{ 887 ServerTlsCert: ordererCertificate, 888 ClientTlsCert: ordererCertificate, 889 Host: "127.0.0.1", 890 Port: uint32(network.OrdererPort(o2, nwo.ClusterPort)), 891 }) 892 893 By("Waiting for orderer2 to join the channel") 894 assertBlockReception(map[string]int{ 895 "mychannel": 1, 896 }, []*nwo.Orderer{o1, o2}, peer, network) 897 898 By("Adding orderer3 to the channel") 899 ordererCertificatePath = filepath.Join(network.OrdererLocalTLSDir(o3), "server.crt") 900 ordererCertificate, err = ioutil.ReadFile(ordererCertificatePath) 901 Expect(err).NotTo(HaveOccurred()) 902 addConsenter(network, peer, o1, "mychannel", etcdraft.Consenter{ 903 ServerTlsCert: ordererCertificate, 904 ClientTlsCert: ordererCertificate, 905 Host: "127.0.0.1", 906 Port: uint32(network.OrdererPort(o3, nwo.ClusterPort)), 907 }) 908 909 By("Waiting for orderer3 to join the channel") 910 assertBlockReception(map[string]int{ 911 "mychannel": 2, 912 }, orderers, peer, network) 913 }) 914 }) 915 916 When("orderer cluster is not healthy", func() { 917 var o1, o2 *nwo.Orderer 918 919 BeforeEach(func() { 920 network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, StartPort(), components) 921 network.GenerateConfigTree() 922 network.Bootstrap() 923 924 o1, o2 = network.Orderer("orderer1"), network.Orderer("orderer2") 925 orderers := []*nwo.Orderer{o1, o2} 926 By("Launching the orderers") 927 for _, o := range orderers { 928 runner := network.OrdererRunner(o) 929 ordererRunners = append(ordererRunners, runner) 930 process := ifrit.Invoke(runner) 931 ordererProcesses = append(ordererProcesses, process) 932 } 933 934 for _, ordererProc := range ordererProcesses { 935 Eventually(ordererProc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 936 } 937 }) 938 939 AfterEach(func() { 940 for _, ordererInstance := range ordererProcesses { 941 ordererInstance.Signal(syscall.SIGTERM) 942 Eventually(ordererInstance.Wait(), network.EventuallyTimeout).Should(Receive()) 943 } 944 }) 945 946 It("refuses to reconfig if it results in quorum loss", func() { 947 By("Waiting for them to elect a leader") 948 findLeader(ordererRunners) 949 950 extendNetwork(network) 951 certificatesOfOrderers := refreshOrdererPEMs(network) 952 953 By("Removing alive node from 2/3 cluster") 954 peer := network.Peer("Org1", "peer0") 955 current, updated := consenterRemover(network, peer, o2, network.SystemChannel.Name, certificatesOfOrderers[1].oldCert) 956 Eventually(func() []byte { 957 sess := nwo.UpdateOrdererConfigSession(network, o2, network.SystemChannel.Name, current, updated, peer, o2) 958 Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(1)) 959 return sess.Err.Contents() 960 }, network.EventuallyTimeout).Should(ContainSubstring("2 out of 3 nodes are alive, configuration will result in quorum loss")) 961 962 By("Adding node to 2/3 cluster") 963 current, updated = consenterAdder( 964 network, 965 peer, 966 o2, 967 network.SystemChannel.Name, 968 etcdraft.Consenter{ 969 ServerTlsCert: certificatesOfOrderers[0].newCert, 970 ClientTlsCert: certificatesOfOrderers[0].newCert, 971 Host: "127.0.0.1", 972 Port: uint32(network.OrdererPort(o1, nwo.ListenPort)), 973 }, 974 ) 975 sess := nwo.UpdateOrdererConfigSession(network, o2, network.SystemChannel.Name, current, updated, peer, o2) 976 Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(1)) 977 Expect(string(sess.Err.Contents())).To(ContainSubstring("2 out of 3 nodes are alive, configuration will result in quorum loss")) 978 }) 979 }) 980 981 When("an orderer node is evicted", func() { 982 BeforeEach(func() { 983 ordererRunners = nil 984 ordererProcesses = nil 985 986 network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, StartPort(), components) 987 network.GenerateConfigTree() 988 network.Bootstrap() 989 990 o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3") 991 orderers := []*nwo.Orderer{o1, o2, o3} 992 peer = network.Peer("Org1", "peer0") 993 994 By("Launching the orderers") 995 for _, o := range orderers { 996 runner := network.OrdererRunner(o, "FABRIC_LOGGING_SPEC=orderer.consensus.etcdraft=debug:info") 997 ordererRunners = append(ordererRunners, runner) 998 process := ifrit.Invoke(runner) 999 ordererProcesses = append(ordererProcesses, process) 1000 } 1001 1002 for _, ordererProc := range ordererProcesses { 1003 Eventually(ordererProc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 1004 } 1005 }) 1006 1007 AfterEach(func() { 1008 for _, ordererInstance := range ordererProcesses { 1009 ordererInstance.Signal(syscall.SIGTERM) 1010 Eventually(ordererInstance.Wait(), network.EventuallyTimeout).Should(Receive()) 1011 } 1012 }) 1013 1014 It("doesn't complain and does it obediently", func() { 1015 o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3") 1016 orderers := []*nwo.Orderer{o1, o2, o3} 1017 1018 By("Waiting for them to elect a leader") 1019 firstEvictedNode := findLeader(ordererRunners) - 1 1020 1021 By("Removing the leader from 3-node channel") 1022 server1CertBytes, err := ioutil.ReadFile(filepath.Join(network.OrdererLocalTLSDir(orderers[firstEvictedNode]), "server.crt")) 1023 Expect(err).To(Not(HaveOccurred())) 1024 1025 removeConsenter(network, peer, network.Orderers[(firstEvictedNode+1)%3], "systemchannel", server1CertBytes) 1026 1027 var survivedOrdererRunners []*ginkgomon.Runner 1028 for i := range orderers { 1029 if i == firstEvictedNode { 1030 continue 1031 } 1032 1033 survivedOrdererRunners = append(survivedOrdererRunners, ordererRunners[i]) 1034 } 1035 1036 secondEvictedNode := findLeader(survivedOrdererRunners) - 1 1037 1038 var survivor int 1039 for i := range orderers { 1040 if i != firstEvictedNode && i != secondEvictedNode { 1041 survivor = i 1042 break 1043 } 1044 } 1045 1046 const stopMsg = "Raft node stopped channel=systemchannel" 1047 fmt.Fprintln(GinkgoWriter, "Ensuring the evicted orderer stops rafting on channel systemchannel") 1048 Eventually(ordererRunners[firstEvictedNode].Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say(stopMsg)) 1049 1050 By("Ensuring the evicted orderer now doesn't serve clients") 1051 ensureEvicted(orderers[firstEvictedNode], peer, network, "systemchannel") 1052 1053 By("Removing the leader from 2-node channel") 1054 server2CertBytes, err := ioutil.ReadFile(filepath.Join(network.OrdererLocalTLSDir(orderers[secondEvictedNode]), "server.crt")) 1055 Expect(err).To(Not(HaveOccurred())) 1056 1057 removeConsenter(network, peer, orderers[survivor], "systemchannel", server2CertBytes) 1058 findLeader([]*ginkgomon.Runner{ordererRunners[survivor]}) 1059 1060 fmt.Fprintln(GinkgoWriter, "Ensuring the other orderer detect the eviction of the node on channel systemchannel") 1061 Eventually(ordererRunners[secondEvictedNode].Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say(stopMsg)) 1062 1063 By("Ensuring the evicted orderer now doesn't serve clients") 1064 ensureEvicted(orderers[secondEvictedNode], peer, network, "systemchannel") 1065 1066 By("Re-adding first evicted orderer") 1067 addConsenter(network, peer, network.Orderers[survivor], "systemchannel", etcdraft.Consenter{ 1068 Host: "127.0.0.1", 1069 Port: uint32(network.OrdererPort(orderers[firstEvictedNode], nwo.ClusterPort)), 1070 ClientTlsCert: server1CertBytes, 1071 ServerTlsCert: server1CertBytes, 1072 }) 1073 1074 By("Ensuring re-added orderer starts serving system channel") 1075 assertBlockReception(map[string]int{ 1076 "systemchannel": 3, 1077 }, []*nwo.Orderer{orderers[firstEvictedNode]}, peer, network) 1078 1079 env := CreateBroadcastEnvelope(network, orderers[secondEvictedNode], network.SystemChannel.Name, []byte("foo")) 1080 resp, err := ordererclient.Broadcast(network, orderers[survivor], env) 1081 Expect(err).NotTo(HaveOccurred()) 1082 Expect(resp.Status).To(Equal(common.Status_SUCCESS)) 1083 }) 1084 1085 When("an evicted node is added back while it's offline", func() { 1086 It("can start with correct new raft id", func() { 1087 o1 := network.Orderer("orderer1") 1088 o2 := network.Orderer("orderer2") 1089 o3 := network.Orderer("orderer3") 1090 1091 By("Waiting for them to elect a leader") 1092 findLeader(ordererRunners) 1093 1094 By("Creating an application channel testchannel") 1095 network.CreateChannel("testchannel", o1, peer) 1096 1097 assertBlockReception(map[string]int{ 1098 "testchannel": 0, 1099 "systemchannel": 1, 1100 }, []*nwo.Orderer{o1, o2, o3}, peer, network) 1101 1102 By("Killing the first orderer") 1103 ordererProcesses[0].Signal(syscall.SIGTERM) 1104 Eventually(ordererProcesses[0].Wait(), network.EventuallyTimeout).Should(Receive()) 1105 1106 // We need to wait for stabilization, as we might have killed the leader OSN. 1107 By("Waiting for the channel to stabilize after killing the orderer") 1108 assertBlockReception(map[string]int{ 1109 "testchannel": 0, 1110 "systemchannel": 1, 1111 }, []*nwo.Orderer{o2, o3}, peer, network) 1112 1113 By("observing active nodes to shrink") 1114 o2Runner := ordererRunners[1] 1115 Eventually(o2Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("Current active nodes in cluster are: \\[2 3\\]")) 1116 1117 By("Removing the first orderer from the application channel") 1118 server1CertBytes, err := ioutil.ReadFile(filepath.Join(network.OrdererLocalTLSDir(o1), "server.crt")) 1119 Expect(err).To(Not(HaveOccurred())) 1120 removeConsenter(network, peer, o2, "testchannel", server1CertBytes) 1121 1122 By("Adding the evicted orderer back to the application channel") 1123 addConsenter(network, peer, o2, "testchannel", etcdraft.Consenter{ 1124 ServerTlsCert: server1CertBytes, 1125 ClientTlsCert: server1CertBytes, 1126 Host: "127.0.0.1", 1127 Port: uint32(network.OrdererPort(o1, nwo.ClusterPort)), 1128 }) 1129 1130 By("Removing the first orderer from the application channel again") 1131 removeConsenter(network, peer, o2, "testchannel", server1CertBytes) 1132 1133 By("Adding the evicted orderer back to the application channel again") 1134 addConsenter(network, peer, o2, "testchannel", etcdraft.Consenter{ 1135 ServerTlsCert: server1CertBytes, 1136 ClientTlsCert: server1CertBytes, 1137 Host: "127.0.0.1", 1138 Port: uint32(network.OrdererPort(o1, nwo.ClusterPort)), 1139 }) 1140 1141 By("Starting the first orderer again") 1142 o1Runner := network.OrdererRunner(o1) 1143 ordererProcesses[0] = ifrit.Invoke(o1Runner) 1144 Eventually(ordererProcesses[0].Ready(), network.EventuallyTimeout).Should(BeClosed()) 1145 1146 By("Submitting tx") 1147 env := CreateBroadcastEnvelope(network, o2, "testchannel", []byte("foo")) 1148 resp, err := ordererclient.Broadcast(network, o2, env) 1149 Expect(err).NotTo(HaveOccurred()) 1150 Expect(resp.Status).To(Equal(common.Status_SUCCESS)) 1151 1152 By("Waiting for the channel to stabilize") 1153 assertBlockReception(map[string]int{ 1154 "testchannel": 5, 1155 }, []*nwo.Orderer{o1, o2, o3}, peer, network) 1156 }) 1157 }) 1158 1159 It("notices it even if it is down at the time of its eviction", func() { 1160 o1 := network.Orderer("orderer1") 1161 o2 := network.Orderer("orderer2") 1162 o3 := network.Orderer("orderer3") 1163 1164 orderers := []*nwo.Orderer{o1, o2, o3} 1165 1166 By("Waiting for them to elect a leader") 1167 findLeader(ordererRunners) 1168 1169 By("Creating a channel") 1170 network.CreateChannel("testchannel", o1, peer) 1171 1172 assertBlockReception(map[string]int{ 1173 "testchannel": 0, 1174 "systemchannel": 1, 1175 }, []*nwo.Orderer{o1, o2, o3}, peer, network) 1176 1177 By("Killing the orderer") 1178 ordererProcesses[0].Signal(syscall.SIGTERM) 1179 Eventually(ordererProcesses[0].Wait(), network.EventuallyTimeout).Should(Receive()) 1180 1181 // We need to wait for stabilization, as we might have killed the leader OSN. 1182 By("Waiting for the channel to stabilize after killing the orderer") 1183 assertBlockReception(map[string]int{ 1184 "testchannel": 0, 1185 "systemchannel": 1, 1186 }, []*nwo.Orderer{o2, o3}, peer, network) 1187 1188 By("Removing the first orderer from an application channel") 1189 o1cert, err := ioutil.ReadFile(path.Join(network.OrdererLocalTLSDir(o1), "server.crt")) 1190 Expect(err).ToNot(HaveOccurred()) 1191 removeConsenter(network, peer, o2, "testchannel", o1cert) 1192 1193 By("Starting the orderer again") 1194 ordererRunner := network.OrdererRunner(orderers[0]) 1195 ordererRunners[0] = ordererRunner 1196 ordererProcesses[0] = ifrit.Invoke(ordererRunner) 1197 Eventually(ordererProcesses[0].Ready(), network.EventuallyTimeout).Should(BeClosed()) 1198 1199 By("Ensuring the remaining OSNs reject authentication") 1200 Eventually(ordererRunners[1].Err(), time.Minute, time.Second).Should(gbytes.Say("certificate extracted from TLS connection isn't authorized")) 1201 Eventually(ordererRunners[2].Err(), time.Minute, time.Second).Should(gbytes.Say("certificate extracted from TLS connection isn't authorized")) 1202 1203 By("Ensuring it detects its eviction") 1204 evictionDetection := gbytes.Say(`Detected our own eviction from the channel in block \[1\] channel=testchannel`) 1205 Eventually(ordererRunner.Err(), time.Minute, time.Second).Should(evictionDetection) 1206 1207 By("Ensuring all blocks are pulled up to the block that evicts the OSN") 1208 Eventually(ordererRunner.Err(), time.Minute, time.Second).Should(gbytes.Say("Periodic check is stopping. channel=testchannel")) 1209 Eventually(ordererRunner.Err(), time.Minute, time.Second).Should(gbytes.Say("Pulled all blocks up to eviction block. channel=testchannel")) 1210 1211 By("Killing the evicted orderer") 1212 ordererProcesses[0].Signal(syscall.SIGTERM) 1213 Eventually(ordererProcesses[0].Wait(), network.EventuallyTimeout).Should(Receive()) 1214 1215 By("Starting the evicted orderer again") 1216 ordererRunner = network.OrdererRunner(orderers[0]) 1217 ordererRunners[0] = ordererRunner 1218 ordererProcesses[0] = ifrit.Invoke(ordererRunner) 1219 Eventually(ordererProcesses[0].Ready(), network.EventuallyTimeout).Should(BeClosed()) 1220 1221 By("Ensuring the evicted orderer starts up marked the channel is inactive") 1222 Eventually(ordererRunner.Err(), time.Minute, time.Second).Should(gbytes.Say("Found 1 inactive chains")) 1223 1224 iDoNotBelong := "I do not belong to channel testchannel or am forbidden pulling it" 1225 Eventually(ordererRunner.Err(), time.Minute, time.Second).Should(gbytes.Say(iDoNotBelong)) 1226 1227 By("Adding the evicted orderer back to the application channel") 1228 addConsenter(network, peer, o2, "testchannel", etcdraft.Consenter{ 1229 ServerTlsCert: o1cert, 1230 ClientTlsCert: o1cert, 1231 Host: "127.0.0.1", 1232 Port: uint32(network.OrdererPort(orderers[0], nwo.ClusterPort)), 1233 }) 1234 1235 By("Ensuring the re-added orderer joins the Raft cluster") 1236 findLeader([]*ginkgomon.Runner{ordererRunner}) 1237 }) 1238 }) 1239 1240 When("an orderer node is joined", func() { 1241 It("isn't influenced by outdated orderers", func() { 1242 // This test checks that if a lagged is not aware of newly added nodes, 1243 // among which leader is present, it eventually pulls config block from 1244 // the orderer it knows, gets the certificates from it and participate 1245 // in consensus again. 1246 // 1247 // Steps: 1248 // Initial nodes in cluster: <1, 2, 3, 4> 1249 // - start <1, 2, 3> 1250 // - add <5, 6, 7>, start <5, 6, 7> 1251 // - kill <1> 1252 // - submit a tx, so that Raft index on <1> is behind <5, 6, 7> and <2, 3> 1253 // - kill <2, 3> 1254 // - start <1> and <4>. Since <1> is behind <5, 6, 7>, leader is certainly 1255 // going to be elected from <5, 6, 7> 1256 // - assert that even <4> is not aware of leader, it can pull config block 1257 // from <1>, and start participating in consensus. 1258 1259 orderers := make([]*nwo.Orderer, 7) 1260 ordererRunners = make([]*ginkgomon.Runner, 7) 1261 ordererProcesses = make([]ifrit.Process, 7) 1262 1263 for i := range orderers { 1264 orderers[i] = &nwo.Orderer{ 1265 Name: fmt.Sprintf("orderer%d", i+1), 1266 Organization: "OrdererOrg", 1267 } 1268 } 1269 1270 layout := nwo.MultiNodeEtcdRaft() 1271 layout.Orderers = orderers[:4] 1272 layout.Profiles[0].Orderers = []string{"orderer1", "orderer2", "orderer3", "orderer4"} 1273 1274 network = nwo.New(layout, testDir, client, StartPort(), components) 1275 network.GenerateConfigTree() 1276 network.Bootstrap() 1277 1278 peer = network.Peer("Org1", "peer0") 1279 1280 launch := func(i int) { 1281 runner := network.OrdererRunner(orderers[i]) 1282 ordererRunners[i] = runner 1283 process := ifrit.Invoke(runner) 1284 Eventually(process.Ready(), network.EventuallyTimeout).Should(BeClosed()) 1285 ordererProcesses[i] = process 1286 } 1287 1288 By("Launching 3 out of 4 orderers") 1289 for i := range orderers[:3] { 1290 launch(i) 1291 } 1292 1293 leader := findLeader(ordererRunners[:3]) 1294 1295 By("Checking that all orderers are online") 1296 assertBlockReception(map[string]int{ 1297 "systemchannel": 0, 1298 }, orderers[:3], peer, network) 1299 1300 By("Configuring orderer[5, 6, 7] in the network") 1301 extendNetwork(network) 1302 1303 for _, o := range orderers[4:7] { 1304 ports := nwo.Ports{} 1305 for _, portName := range nwo.OrdererPortNames() { 1306 ports[portName] = network.ReservePort() 1307 } 1308 network.PortsByOrdererID[o.ID()] = ports 1309 1310 network.Orderers = append(network.Orderers, o) 1311 network.GenerateOrdererConfig(o) 1312 } 1313 1314 // Backup previous system channel block 1315 genesisBootBlock, err := ioutil.ReadFile(filepath.Join(testDir, "systemchannel_block.pb")) 1316 Expect(err).NotTo(HaveOccurred()) 1317 restoreBootBlock := func() { 1318 err = ioutil.WriteFile(filepath.Join(testDir, "systemchannel_block.pb"), genesisBootBlock, 0o644) 1319 Expect(err).NotTo(HaveOccurred()) 1320 } 1321 1322 blockNum := 0 // there's only one block in channel - genesis 1323 for _, i := range []int{4, 5, 6} { 1324 By(fmt.Sprintf("Adding orderer%d", i+1)) 1325 ordererCertificatePath := filepath.Join(network.OrdererLocalTLSDir(orderers[i]), "server.crt") 1326 ordererCertificate, err := ioutil.ReadFile(ordererCertificatePath) 1327 Expect(err).NotTo(HaveOccurred()) 1328 1329 addConsenter(network, peer, orderers[0], "systemchannel", etcdraft.Consenter{ 1330 ServerTlsCert: ordererCertificate, 1331 ClientTlsCert: ordererCertificate, 1332 Host: "127.0.0.1", 1333 Port: uint32(network.OrdererPort(orderers[i], nwo.ClusterPort)), 1334 }) 1335 blockNum++ 1336 1337 // Get the last config block of the system channel 1338 configBlock := nwo.GetConfigBlock(network, peer, orderers[0], "systemchannel") 1339 // Plant it in the file system of orderer, the new node to be onboarded. 1340 err = ioutil.WriteFile(filepath.Join(testDir, "systemchannel_block.pb"), protoutil.MarshalOrPanic(configBlock), 0o644) 1341 Expect(err).NotTo(HaveOccurred()) 1342 1343 By(fmt.Sprintf("Launching orderer%d", i+1)) 1344 launch(i) 1345 1346 By(fmt.Sprintf("Checking that orderer%d has onboarded the network", i+1)) 1347 assertBlockReception(map[string]int{ 1348 "systemchannel": blockNum, 1349 }, []*nwo.Orderer{orderers[i]}, peer, network) 1350 } 1351 1352 Expect(findLeader(ordererRunners[4:])).To(Equal(leader)) 1353 1354 // Later on, when we start [1, 4, 5, 6, 7], we want to make sure that leader 1355 // is elected from [5, 6, 7], who are unknown to [4]. So we can assert that 1356 // [4] suspects its own eviction, pulls block from [1], and join the cluster. 1357 // Otherwise, if [1] is elected, and all other nodes already knew it, [4] may 1358 // simply replicate missing blocks with Raft, instead of pulling from others 1359 // triggered by eviction suspector. 1360 1361 By("Killing orderer1") 1362 ordererProcesses[0].Signal(syscall.SIGTERM) 1363 Eventually(ordererProcesses[0].Wait(), network.EventuallyTimeout).Should(Receive()) 1364 1365 By("Submitting another tx to increment Raft index on alive orderers") 1366 if leader == 1 { 1367 // if orderer1 was leader, we should expect a new leader being elected before going forward 1368 findLeader([]*ginkgomon.Runner{ordererRunners[4]}) 1369 } 1370 1371 env := CreateBroadcastEnvelope(network, orderers[4], network.SystemChannel.Name, []byte("hello")) 1372 resp, err := ordererclient.Broadcast(network, orderers[4], env) 1373 Expect(err).NotTo(HaveOccurred()) 1374 Expect(resp.Status).To(Equal(common.Status_SUCCESS)) 1375 blockNum++ 1376 1377 assertBlockReception(map[string]int{ 1378 "systemchannel": blockNum, 1379 }, []*nwo.Orderer{orderers[1], orderers[2], orderers[4], orderers[5], orderers[6]}, peer, network) // alive orderers: 2, 3, 5, 6, 7 1380 1381 By("Killing orderer[2,3]") 1382 for _, i := range []int{1, 2} { 1383 ordererProcesses[i].Signal(syscall.SIGTERM) 1384 Eventually(ordererProcesses[i].Wait(), network.EventuallyTimeout).Should(Receive()) 1385 } 1386 1387 By("Launching the orderer that was never started") 1388 restoreBootBlock() 1389 launch(3) 1390 1391 By("Launching orderer1") 1392 launch(0) 1393 1394 By("Waiting until it suspects its eviction from the channel") 1395 Eventually(ordererRunners[3].Err(), time.Minute, time.Second).Should(gbytes.Say("Suspecting our own eviction from the channel")) 1396 1397 By("Making sure 4/7 orderers form quorum and serve request") 1398 assertBlockReception(map[string]int{ 1399 "systemchannel": blockNum, 1400 }, []*nwo.Orderer{orderers[3], orderers[4], orderers[5], orderers[6]}, peer, network) // alive orderers: 4, 5, 6, 7 1401 }) 1402 }) 1403 1404 It("can create a channel that contains a subset of orderers in system channel", func() { 1405 config := nwo.BasicEtcdRaft() 1406 config.Orderers = []*nwo.Orderer{ 1407 {Name: "orderer1", Organization: "OrdererOrg"}, 1408 {Name: "orderer2", Organization: "OrdererOrg"}, 1409 {Name: "orderer3", Organization: "OrdererOrg"}, 1410 } 1411 config.Profiles = []*nwo.Profile{{ 1412 Name: "SampleDevModeEtcdRaft", 1413 Orderers: []string{"orderer1", "orderer2", "orderer3"}, 1414 }, { 1415 Name: "ThreeOrdererChannel", 1416 Consortium: "SampleConsortium", 1417 Organizations: []string{"Org1", "Org2"}, 1418 Orderers: []string{"orderer1", "orderer2", "orderer3"}, 1419 }, { 1420 Name: "SingleOrdererChannel", 1421 Consortium: "SampleConsortium", 1422 Organizations: []string{"Org1", "Org2"}, 1423 Orderers: []string{"orderer1"}, 1424 }} 1425 config.Channels = []*nwo.Channel{ 1426 {Name: "single-orderer-channel", Profile: "SingleOrdererChannel", BaseProfile: "SampleDevModeEtcdRaft"}, 1427 {Name: "three-orderer-channel", Profile: "ThreeOrdererChannel"}, 1428 } 1429 1430 network = nwo.New(config, testDir, client, StartPort(), components) 1431 o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3") 1432 orderers := []*nwo.Orderer{o1, o2, o3} 1433 peer = network.Peer("Org1", "peer0") 1434 1435 network.GenerateConfigTree() 1436 network.Bootstrap() 1437 1438 By("Launching the orderers") 1439 for _, o := range orderers { 1440 runner := network.OrdererRunner(o) 1441 ordererRunners = append(ordererRunners, runner) 1442 process := ifrit.Invoke(runner) 1443 ordererProcesses = append(ordererProcesses, process) 1444 } 1445 1446 for _, ordererProc := range ordererProcesses { 1447 Eventually(ordererProc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 1448 } 1449 1450 By("Creating an application channel with a subset of orderers in system channel") 1451 additionalPeer := network.Peer("Org2", "peer0") 1452 network.CreateChannel("single-orderer-channel", network.Orderers[0], peer, additionalPeer, network.Orderers[0]) 1453 1454 By("Creating another channel via the orderer that is in system channel but not app channel") 1455 network.CreateChannel("three-orderer-channel", network.Orderers[2], peer) 1456 }) 1457 1458 It("can add a new orderer organization", func() { 1459 network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, StartPort(), components) 1460 o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3") 1461 orderers := []*nwo.Orderer{o1, o2, o3} 1462 1463 network.GenerateConfigTree() 1464 network.Bootstrap() 1465 1466 By("Launching the orderers") 1467 for _, o := range orderers { 1468 runner := network.OrdererRunner(o) 1469 ordererRunners = append(ordererRunners, runner) 1470 process := ifrit.Invoke(runner) 1471 ordererProcesses = append(ordererProcesses, process) 1472 } 1473 1474 for _, ordererProc := range ordererProcesses { 1475 Eventually(ordererProc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 1476 } 1477 1478 By("Waiting for system channel to be ready") 1479 findLeader(ordererRunners) 1480 1481 peer := network.Peer("Org1", "peer0") 1482 channel := "systemchannel" 1483 1484 config := nwo.GetConfig(network, peer, o1, channel) 1485 updatedConfig := proto.Clone(config).(*common.Config) 1486 1487 ordererOrg := updatedConfig.ChannelGroup.Groups["Orderer"].Groups["OrdererOrg"] 1488 mspConfig := &msp.MSPConfig{} 1489 err := proto.Unmarshal(ordererOrg.Values["MSP"].Value, mspConfig) 1490 Expect(err).NotTo(HaveOccurred()) 1491 1492 fabMSPConfig := &msp.FabricMSPConfig{} 1493 err = proto.Unmarshal(mspConfig.Config, fabMSPConfig) 1494 Expect(err).NotTo(HaveOccurred()) 1495 1496 fabMSPConfig.Name = "OrdererMSP2" 1497 1498 mspConfig.Config, _ = proto.Marshal(fabMSPConfig) 1499 updatedConfig.ChannelGroup.Groups["Orderer"].Groups["OrdererMSP2"] = &common.ConfigGroup{ 1500 Values: map[string]*common.ConfigValue{ 1501 "MSP": { 1502 Value: protoutil.MarshalOrPanic(mspConfig), 1503 ModPolicy: "Admins", 1504 }, 1505 }, 1506 ModPolicy: "Admins", 1507 } 1508 1509 nwo.UpdateOrdererConfig(network, o1, channel, config, updatedConfig, peer, o1) 1510 }) 1511 }) 1512 1513 func ensureEvicted(evictedOrderer *nwo.Orderer, submitter *nwo.Peer, network *nwo.Network, channel string) { 1514 c := commands.ChannelFetch{ 1515 ChannelID: channel, 1516 Block: "newest", 1517 OutputFile: "/dev/null", 1518 Orderer: network.OrdererAddress(evictedOrderer, nwo.ListenPort), 1519 } 1520 1521 sess, err := network.OrdererAdminSession(evictedOrderer, submitter, c) 1522 Expect(err).NotTo(HaveOccurred()) 1523 1524 Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit()) 1525 Expect(sess.Err).To(gbytes.Say("SERVICE_UNAVAILABLE")) 1526 } 1527 1528 var extendedCryptoConfig = `--- 1529 OrdererOrgs: 1530 - Name: OrdererOrg 1531 Domain: example.com 1532 EnableNodeOUs: false 1533 CA: 1534 Hostname: ca 1535 Specs: 1536 - Hostname: orderer1 1537 SANS: 1538 - localhost 1539 - 127.0.0.1 1540 - ::1 1541 - Hostname: orderer1new 1542 SANS: 1543 - localhost 1544 - 127.0.0.1 1545 - ::1 1546 - Hostname: orderer2 1547 SANS: 1548 - localhost 1549 - 127.0.0.1 1550 - ::1 1551 - Hostname: orderer2new 1552 SANS: 1553 - localhost 1554 - 127.0.0.1 1555 - ::1 1556 - Hostname: orderer3 1557 SANS: 1558 - localhost 1559 - 127.0.0.1 1560 - ::1 1561 - Hostname: orderer3new 1562 SANS: 1563 - localhost 1564 - 127.0.0.1 1565 - ::1 1566 - Hostname: orderer4 1567 SANS: 1568 - localhost 1569 - 127.0.0.1 1570 - ::1 1571 - Hostname: orderer5 1572 SANS: 1573 - localhost 1574 - 127.0.0.1 1575 - ::1 1576 - Hostname: orderer6 1577 SANS: 1578 - localhost 1579 - 127.0.0.1 1580 - ::1 1581 - Hostname: orderer7 1582 SANS: 1583 - localhost 1584 - 127.0.0.1 1585 - ::1 1586 ` 1587 1588 type certificateChange struct { 1589 srcFile string 1590 dstFile string 1591 oldCert []byte 1592 oldKey []byte 1593 newCert []byte 1594 } 1595 1596 // extendNetwork rotates adds an additional orderer 1597 func extendNetwork(n *nwo.Network) { 1598 // Overwrite the current crypto-config with additional orderers 1599 cryptoConfigYAML, err := ioutil.TempFile("", "crypto-config.yaml") 1600 Expect(err).NotTo(HaveOccurred()) 1601 defer os.Remove(cryptoConfigYAML.Name()) 1602 1603 err = ioutil.WriteFile(cryptoConfigYAML.Name(), []byte(extendedCryptoConfig), 0o644) 1604 Expect(err).NotTo(HaveOccurred()) 1605 1606 // Invoke cryptogen extend to add new orderers 1607 sess, err := n.Cryptogen(commands.Extend{ 1608 Config: cryptoConfigYAML.Name(), 1609 Input: n.CryptoPath(), 1610 }) 1611 Expect(err).NotTo(HaveOccurred()) 1612 Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(0)) 1613 } 1614 1615 // refreshOrdererPEMs rotates all TLS certificates of all nodes, 1616 // and returns the deltas 1617 func refreshOrdererPEMs(n *nwo.Network) []*certificateChange { 1618 var fileChanges []*certificateChange 1619 // Populate source to destination files 1620 err := filepath.Walk(n.CryptoPath(), func(path string, info os.FileInfo, err error) error { 1621 if !strings.Contains(path, "/tls/") { 1622 return nil 1623 } 1624 if strings.Contains(path, "new") { 1625 fileChanges = append(fileChanges, &certificateChange{ 1626 srcFile: path, 1627 dstFile: strings.Replace(path, "new", "", -1), 1628 }) 1629 } 1630 return nil 1631 }) 1632 Expect(err).NotTo(HaveOccurred()) 1633 1634 var serverCertChanges []*certificateChange 1635 1636 // Overwrite the destination files with the contents of the source files. 1637 for _, certChange := range fileChanges { 1638 previousCertBytes, err := ioutil.ReadFile(certChange.dstFile) 1639 Expect(err).NotTo(HaveOccurred()) 1640 1641 newCertBytes, err := ioutil.ReadFile(certChange.srcFile) 1642 Expect(err).NotTo(HaveOccurred()) 1643 1644 err = ioutil.WriteFile(certChange.dstFile, newCertBytes, 0o644) 1645 Expect(err).NotTo(HaveOccurred()) 1646 1647 if !strings.Contains(certChange.dstFile, "server.crt") { 1648 continue 1649 } 1650 1651 // Read the previous key file 1652 previousKeyBytes, err := ioutil.ReadFile(strings.Replace(certChange.dstFile, "server.crt", "server.key", -1)) 1653 Expect(err).NotTo(HaveOccurred()) 1654 1655 serverCertChanges = append(serverCertChanges, certChange) 1656 certChange.newCert = newCertBytes 1657 certChange.oldCert = previousCertBytes 1658 certChange.oldKey = previousKeyBytes 1659 } 1660 return serverCertChanges 1661 } 1662 1663 // assertBlockReception asserts that the given orderers have the expected 1664 // newest block number for the specified channels 1665 func assertBlockReception(expectedBlockNumPerChannel map[string]int, orderers []*nwo.Orderer, p *nwo.Peer, n *nwo.Network) { 1666 for channelName, blockNum := range expectedBlockNumPerChannel { 1667 for _, orderer := range orderers { 1668 waitForBlockReception(orderer, p, n, channelName, blockNum) 1669 } 1670 } 1671 } 1672 1673 func waitForBlockReception(o *nwo.Orderer, submitter *nwo.Peer, network *nwo.Network, channelName string, blockNum int) { 1674 c := commands.ChannelFetch{ 1675 ChannelID: channelName, 1676 Block: "newest", 1677 OutputFile: "/dev/null", 1678 Orderer: network.OrdererAddress(o, nwo.ListenPort), 1679 } 1680 Eventually(func() string { 1681 sess, err := network.OrdererAdminSession(o, submitter, c) 1682 Expect(err).NotTo(HaveOccurred()) 1683 Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit()) 1684 if sess.ExitCode() != 0 { 1685 return fmt.Sprintf("exit code is %d: %s", sess.ExitCode(), string(sess.Err.Contents())) 1686 } 1687 sessErr := string(sess.Err.Contents()) 1688 expected := fmt.Sprintf("Received block: %d", blockNum) 1689 if strings.Contains(sessErr, expected) { 1690 return "" 1691 } 1692 return sessErr 1693 }, network.EventuallyTimeout, time.Second).Should(BeEmpty()) 1694 } 1695 1696 func revokeReaderAccess(network *nwo.Network, channel string, orderer *nwo.Orderer, peer *nwo.Peer) { 1697 config := nwo.GetConfig(network, peer, orderer, channel) 1698 updatedConfig := proto.Clone(config).(*common.Config) 1699 1700 // set the policy 1701 adminPolicy := protoutil.MarshalOrPanic(&common.ImplicitMetaPolicy{ 1702 SubPolicy: "Admins", 1703 Rule: common.ImplicitMetaPolicy_MAJORITY, 1704 }) 1705 updatedConfig.ChannelGroup.Groups["Orderer"].Policies["Readers"].Policy.Value = adminPolicy 1706 nwo.UpdateOrdererConfig(network, orderer, channel, config, updatedConfig, peer, orderer) 1707 } 1708 1709 // consenterAdder constructs configs that can be used by `UpdateOrdererConfig` 1710 // to add a consenter. 1711 func consenterAdder(n *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel string, consenter etcdraft.Consenter) (current, updated *common.Config) { 1712 config := nwo.GetConfig(n, peer, orderer, channel) 1713 updatedConfig := proto.Clone(config).(*common.Config) 1714 1715 consensusTypeConfigValue := updatedConfig.ChannelGroup.Groups["Orderer"].Values["ConsensusType"] 1716 consensusTypeValue := &protosorderer.ConsensusType{} 1717 err := proto.Unmarshal(consensusTypeConfigValue.Value, consensusTypeValue) 1718 Expect(err).NotTo(HaveOccurred()) 1719 1720 metadata := &etcdraft.ConfigMetadata{} 1721 err = proto.Unmarshal(consensusTypeValue.Metadata, metadata) 1722 Expect(err).NotTo(HaveOccurred()) 1723 1724 metadata.Consenters = append(metadata.Consenters, &consenter) 1725 1726 consensusTypeValue.Metadata, err = proto.Marshal(metadata) 1727 Expect(err).NotTo(HaveOccurred()) 1728 1729 updatedConfig.ChannelGroup.Groups["Orderer"].Values["ConsensusType"] = &common.ConfigValue{ 1730 ModPolicy: "Admins", 1731 Value: protoutil.MarshalOrPanic(consensusTypeValue), 1732 } 1733 1734 return config, updatedConfig 1735 } 1736 1737 // consenterRemover constructs configs that can be used by 1738 // `UpdateOrdererConfig` to remove a consenter. 1739 func consenterRemover(n *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel string, certificate []byte) (current, updated *common.Config) { 1740 config := nwo.GetConfig(n, peer, orderer, channel) 1741 updatedConfig := proto.Clone(config).(*common.Config) 1742 1743 consensusTypeConfigValue := updatedConfig.ChannelGroup.Groups["Orderer"].Values["ConsensusType"] 1744 consensusTypeValue := &protosorderer.ConsensusType{} 1745 err := proto.Unmarshal(consensusTypeConfigValue.Value, consensusTypeValue) 1746 Expect(err).NotTo(HaveOccurred()) 1747 1748 metadata := &etcdraft.ConfigMetadata{} 1749 err = proto.Unmarshal(consensusTypeValue.Metadata, metadata) 1750 Expect(err).NotTo(HaveOccurred()) 1751 1752 var newConsenters []*etcdraft.Consenter 1753 for _, consenter := range metadata.Consenters { 1754 if bytes.Equal(consenter.ClientTlsCert, certificate) || bytes.Equal(consenter.ServerTlsCert, certificate) { 1755 continue 1756 } 1757 newConsenters = append(newConsenters, consenter) 1758 } 1759 1760 metadata.Consenters = newConsenters 1761 consensusTypeValue.Metadata, err = proto.Marshal(metadata) 1762 Expect(err).NotTo(HaveOccurred()) 1763 1764 updatedConfig.ChannelGroup.Groups["Orderer"].Values["ConsensusType"] = &common.ConfigValue{ 1765 ModPolicy: "Admins", 1766 Value: protoutil.MarshalOrPanic(consensusTypeValue), 1767 } 1768 1769 return config, updatedConfig 1770 } 1771 1772 // addConsenter adds a new consenter to the given channel. 1773 func addConsenter(n *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel string, consenter etcdraft.Consenter) { 1774 updateEtcdRaftMetadata(n, peer, orderer, channel, func(metadata *etcdraft.ConfigMetadata) { 1775 metadata.Consenters = append(metadata.Consenters, &consenter) 1776 }) 1777 } 1778 1779 // removeConsenter removes a consenter with the given certificate in PEM format 1780 // from the given channel. 1781 func removeConsenter(n *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel string, certificate []byte) { 1782 updateEtcdRaftMetadata(n, peer, orderer, channel, func(metadata *etcdraft.ConfigMetadata) { 1783 var newConsenters []*etcdraft.Consenter 1784 for _, consenter := range metadata.Consenters { 1785 if bytes.Equal(consenter.ClientTlsCert, certificate) || bytes.Equal(consenter.ServerTlsCert, certificate) { 1786 continue 1787 } 1788 newConsenters = append(newConsenters, consenter) 1789 } 1790 1791 metadata.Consenters = newConsenters 1792 }) 1793 } 1794 1795 // updateEtcdRaftMetadata executes a config update that updates the etcdraft 1796 // metadata according to the given function f. 1797 func updateEtcdRaftMetadata(network *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel string, f func(md *etcdraft.ConfigMetadata)) { 1798 nwo.UpdateConsensusMetadata(network, peer, orderer, channel, func(originalMetadata []byte) []byte { 1799 metadata := &etcdraft.ConfigMetadata{} 1800 err := proto.Unmarshal(originalMetadata, metadata) 1801 Expect(err).NotTo(HaveOccurred()) 1802 1803 f(metadata) 1804 1805 newMetadata, err := proto.Marshal(metadata) 1806 Expect(err).NotTo(HaveOccurred()) 1807 return newMetadata 1808 }) 1809 } 1810 1811 func mutateConsensusMetadata(originalMetadata []byte, f func(md *etcdraft.ConfigMetadata)) []byte { 1812 metadata := &etcdraft.ConfigMetadata{} 1813 err := proto.Unmarshal(originalMetadata, metadata) 1814 Expect(err).NotTo(HaveOccurred()) 1815 1816 f(metadata) 1817 1818 newMetadata, err := proto.Marshal(metadata) 1819 Expect(err).NotTo(HaveOccurred()) 1820 return newMetadata 1821 } 1822 1823 func updateOrdererMSPAndConsensusMetadata(network *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel, orgID string, mutateMSP nwo.MSPMutator, f func(md *etcdraft.ConfigMetadata)) { 1824 config := nwo.GetConfig(network, peer, orderer, channel) 1825 updatedConfig := proto.Clone(config).(*common.Config) 1826 1827 // Unpack the MSP config 1828 rawMSPConfig := updatedConfig.ChannelGroup.Groups["Orderer"].Groups[orgID].Values["MSP"] 1829 mspConfig := &msp.MSPConfig{} 1830 err := proto.Unmarshal(rawMSPConfig.Value, mspConfig) 1831 Expect(err).NotTo(HaveOccurred()) 1832 1833 fabricConfig := &msp.FabricMSPConfig{} 1834 err = proto.Unmarshal(mspConfig.Config, fabricConfig) 1835 Expect(err).NotTo(HaveOccurred()) 1836 1837 // Mutate it as we are asked 1838 *fabricConfig = mutateMSP(*fabricConfig) 1839 1840 // Wrap it back into the config 1841 mspConfig.Config = protoutil.MarshalOrPanic(fabricConfig) 1842 rawMSPConfig.Value = protoutil.MarshalOrPanic(mspConfig) 1843 1844 consensusTypeConfigValue := updatedConfig.ChannelGroup.Groups["Orderer"].Values["ConsensusType"] 1845 consensusTypeValue := &protosorderer.ConsensusType{} 1846 err = proto.Unmarshal(consensusTypeConfigValue.Value, consensusTypeValue) 1847 Expect(err).NotTo(HaveOccurred()) 1848 1849 consensusTypeValue.Metadata = mutateConsensusMetadata(consensusTypeValue.Metadata, f) 1850 1851 updatedConfig.ChannelGroup.Groups["Orderer"].Values["ConsensusType"] = &common.ConfigValue{ 1852 ModPolicy: "Admins", 1853 Value: protoutil.MarshalOrPanic(consensusTypeValue), 1854 } 1855 1856 nwo.UpdateOrdererConfig(network, orderer, channel, config, updatedConfig, peer, orderer) 1857 }