github.com/hechain20/hechain@v0.0.0-20220316014945-b544036ba106/integration/raft/config_test.go (about)

     1  /*
     2  Copyright hechain All Rights Reserved.
     3  
     4  SPDX-License-Identifier: Apache-2.0
     5  */
     6  
     7  package raft
     8  
     9  import (
    10  	"bytes"
    11  	"crypto/x509"
    12  	"encoding/pem"
    13  	"fmt"
    14  	"io/ioutil"
    15  	"os"
    16  	"path"
    17  	"path/filepath"
    18  	"strings"
    19  	"syscall"
    20  	"time"
    21  
    22  	docker "github.com/fsouza/go-dockerclient"
    23  	"github.com/golang/protobuf/proto"
    24  	"github.com/hechain20/hechain/common/crypto/tlsgen"
    25  	"github.com/hechain20/hechain/integration/nwo"
    26  	"github.com/hechain20/hechain/integration/nwo/commands"
    27  	"github.com/hechain20/hechain/integration/ordererclient"
    28  	"github.com/hechain20/hechain/internal/configtxgen/encoder"
    29  	"github.com/hechain20/hechain/internal/configtxgen/genesisconfig"
    30  	"github.com/hechain20/hechain/protoutil"
    31  	"github.com/hyperledger/fabric-protos-go/common"
    32  	"github.com/hyperledger/fabric-protos-go/msp"
    33  	protosorderer "github.com/hyperledger/fabric-protos-go/orderer"
    34  	"github.com/hyperledger/fabric-protos-go/orderer/etcdraft"
    35  	. "github.com/onsi/ginkgo"
    36  	. "github.com/onsi/gomega"
    37  	"github.com/onsi/gomega/gbytes"
    38  	"github.com/onsi/gomega/gexec"
    39  	"github.com/tedsuo/ifrit"
    40  	"github.com/tedsuo/ifrit/ginkgomon"
    41  )
    42  
    43  var _ = Describe("EndToEnd reconfiguration and onboarding", func() {
    44  	var (
    45  		testDir string
    46  		client  *docker.Client
    47  		network *nwo.Network
    48  		peer    *nwo.Peer
    49  
    50  		networkProcess   ifrit.Process
    51  		ordererProcesses []ifrit.Process
    52  		ordererRunners   []*ginkgomon.Runner
    53  	)
    54  
    55  	BeforeEach(func() {
    56  		ordererRunners = nil
    57  		ordererProcesses = nil
    58  
    59  		var err error
    60  		testDir, err = ioutil.TempDir("", "e2e-etcdraft_reconfig")
    61  		Expect(err).NotTo(HaveOccurred())
    62  
    63  		client, err = docker.NewClientFromEnv()
    64  		Expect(err).NotTo(HaveOccurred())
    65  	})
    66  
    67  	AfterEach(func() {
    68  		if networkProcess != nil {
    69  			networkProcess.Signal(syscall.SIGTERM)
    70  			Eventually(networkProcess.Wait(), network.EventuallyTimeout).Should(Receive())
    71  		}
    72  		if network != nil {
    73  			network.Cleanup()
    74  		}
    75  		for _, ordererInstance := range ordererProcesses {
    76  			ordererInstance.Signal(syscall.SIGTERM)
    77  			Eventually(ordererInstance.Wait(), network.EventuallyTimeout).Should(Receive())
    78  		}
    79  		os.RemoveAll(testDir)
    80  	})
    81  
    82  	Describe("three node etcdraft network with 2 orgs", func() {
    83  		BeforeEach(func() {
    84  			network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, StartPort(), components)
    85  			network.GenerateConfigTree()
    86  			network.Bootstrap()
    87  
    88  			networkRunner := network.NetworkGroupRunner()
    89  			networkProcess = ifrit.Invoke(networkRunner)
    90  			Eventually(networkProcess.Ready(), network.EventuallyTimeout).Should(BeClosed())
    91  		})
    92  
    93  		// This tests:
    94  		//
    95  		// 1. channel creation with raft orderer,
    96  		// 2. all the nodes on three-node raft cluster are in sync wrt blocks,
    97  		// 3. raft orderer processes type A config updates and delivers the
    98  		//    config blocks to the peers.
    99  		It("executes an etcdraft network with 2 orgs and three orderer nodes", func() {
   100  			orderer1 := network.Orderer("orderer1")
   101  			orderer2 := network.Orderer("orderer2")
   102  			orderer3 := network.Orderer("orderer3")
   103  			peer := network.Peer("Org1", "peer0")
   104  			blockFile1 := filepath.Join(testDir, "newest_orderer1_block.pb")
   105  			blockFile2 := filepath.Join(testDir, "newest_orderer2_block.pb")
   106  			blockFile3 := filepath.Join(testDir, "newest_orderer3_block.pb")
   107  
   108  			By("Ordering service system channel is ready")
   109  			assertBlockReception(map[string]int{
   110  				"systemchannel": 0,
   111  			}, []*nwo.Orderer{orderer1, orderer2, orderer3}, peer, network)
   112  
   113  			fetchLatestBlock := func(targetOrderer *nwo.Orderer, blockFile string) {
   114  				c := commands.ChannelFetch{
   115  					ChannelID:  "testchannel",
   116  					Block:      "newest",
   117  					OutputFile: blockFile,
   118  				}
   119  				if targetOrderer != nil {
   120  					c.Orderer = network.OrdererAddress(targetOrderer, nwo.ListenPort)
   121  				}
   122  				sess, err := network.PeerAdminSession(peer, c)
   123  				Expect(err).NotTo(HaveOccurred())
   124  				Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(0))
   125  			}
   126  
   127  			By("Creating a new channel")
   128  			network.CreateChannel("testchannel", orderer1, peer)
   129  
   130  			// the above can work even if the orderer nodes are not in the same Raft
   131  			// cluster; we need to verify all the three orderer nodes are in sync wrt
   132  			// blocks.
   133  			By("Fetching the latest blocks from all the orderer nodes and testing them for equality")
   134  			fetchLatestBlock(orderer1, blockFile1)
   135  			fetchLatestBlock(orderer2, blockFile2)
   136  			fetchLatestBlock(orderer3, blockFile3)
   137  			b1 := nwo.UnmarshalBlockFromFile(blockFile1)
   138  			b2 := nwo.UnmarshalBlockFromFile(blockFile2)
   139  			b3 := nwo.UnmarshalBlockFromFile(blockFile3)
   140  			Expect(protoutil.BlockHeaderBytes(b1.Header)).To(Equal(protoutil.BlockHeaderBytes(b2.Header)))
   141  			Expect(protoutil.BlockHeaderBytes(b2.Header)).To(Equal(protoutil.BlockHeaderBytes(b3.Header)))
   142  		})
   143  	})
   144  
   145  	Describe("Invalid Raft config metadata", func() {
   146  		It("refuses to start orderer or rejects config update", func() {
   147  			By("Creating malformed genesis block")
   148  			network = nwo.New(nwo.BasicEtcdRaft(), testDir, client, StartPort(), components)
   149  			network.GenerateConfigTree()
   150  			network.Bootstrap()
   151  
   152  			sysProfile := genesisconfig.Load(network.SystemChannel.Profile, network.RootDir)
   153  			Expect(sysProfile.Orderer).NotTo(BeNil())
   154  			sysProfile.Orderer.EtcdRaft.Options.ElectionTick = sysProfile.Orderer.EtcdRaft.Options.HeartbeatTick
   155  			pgen := encoder.New(sysProfile)
   156  			genesisBlock := pgen.GenesisBlockForChannel(network.SystemChannel.Name)
   157  			data, err := proto.Marshal(genesisBlock)
   158  			Expect(err).NotTo(HaveOccurred())
   159  			err = ioutil.WriteFile(network.OutputBlockPath(network.SystemChannel.Name), data, 0o644)
   160  			Expect(err).NotTo(HaveOccurred())
   161  
   162  			By("Starting orderer with malformed genesis block")
   163  			ordererRunner := network.OrdererGroupRunner()
   164  			process := ifrit.Invoke(ordererRunner)
   165  			Eventually(process.Wait(), network.EventuallyTimeout).Should(Receive()) // orderer process should exit
   166  			network.Cleanup()
   167  			os.RemoveAll(testDir)
   168  
   169  			By("Starting orderer with correct genesis block")
   170  			testDir, err = ioutil.TempDir("", "e2e")
   171  			Expect(err).NotTo(HaveOccurred())
   172  			network = nwo.New(nwo.BasicEtcdRaft(), testDir, client, StartPort(), components)
   173  			network.GenerateConfigTree()
   174  			network.Bootstrap()
   175  
   176  			orderer := network.Orderer("orderer")
   177  			runner := network.OrdererRunner(orderer)
   178  			process = ifrit.Invoke(runner)
   179  			Eventually(process.Ready, network.EventuallyTimeout).Should(BeClosed())
   180  			defer func() {
   181  				process.Signal(syscall.SIGTERM)
   182  				Eventually(process.Wait(), network.EventuallyTimeout).Should(Receive())
   183  			}()
   184  
   185  			By("Waiting for system channel to be ready")
   186  			findLeader([]*ginkgomon.Runner{runner})
   187  
   188  			By("Creating malformed channel creation config tx")
   189  			channel := "testchannel"
   190  			sysProfile = genesisconfig.Load(network.SystemChannel.Profile, network.RootDir)
   191  			Expect(sysProfile.Orderer).NotTo(BeNil())
   192  			appProfile := genesisconfig.Load(network.ProfileForChannel(channel), network.RootDir)
   193  			Expect(appProfile).NotTo(BeNil())
   194  			o := *sysProfile.Orderer
   195  			appProfile.Orderer = &o
   196  			appProfile.Orderer.EtcdRaft = proto.Clone(sysProfile.Orderer.EtcdRaft).(*etcdraft.ConfigMetadata)
   197  			appProfile.Orderer.EtcdRaft.Options.HeartbeatTick = appProfile.Orderer.EtcdRaft.Options.ElectionTick
   198  			configtx, err := encoder.MakeChannelCreationTransactionWithSystemChannelContext(channel, nil, appProfile, sysProfile)
   199  			Expect(err).NotTo(HaveOccurred())
   200  			data, err = proto.Marshal(configtx)
   201  			Expect(err).NotTo(HaveOccurred())
   202  			err = ioutil.WriteFile(network.CreateChannelTxPath(channel), data, 0o644)
   203  			Expect(err).NotTo(HaveOccurred())
   204  
   205  			By("Submitting malformed channel creation config tx to orderer")
   206  			org1Peer0 := network.Peer("Org1", "peer0")
   207  			org2Peer0 := network.Peer("Org2", "peer0")
   208  
   209  			exitCode := network.CreateChannelExitCode(channel, orderer, org1Peer0, org1Peer0, org2Peer0, orderer)
   210  			Expect(exitCode).NotTo(Equal(0))
   211  			Consistently(process.Wait()).ShouldNot(Receive()) // malformed tx should not crash orderer
   212  			Expect(runner.Err()).To(gbytes.Say(`invalid new config metadata: ElectionTick \(10\) must be greater than HeartbeatTick \(10\)`))
   213  
   214  			By("Submitting channel config update with illegal value")
   215  			channel = network.SystemChannel.Name
   216  			config := nwo.GetConfig(network, org1Peer0, orderer, channel)
   217  			updatedConfig := proto.Clone(config).(*common.Config)
   218  
   219  			consensusTypeConfigValue := updatedConfig.ChannelGroup.Groups["Orderer"].Values["ConsensusType"]
   220  			consensusTypeValue := &protosorderer.ConsensusType{}
   221  			Expect(proto.Unmarshal(consensusTypeConfigValue.Value, consensusTypeValue)).To(Succeed())
   222  
   223  			metadata := &etcdraft.ConfigMetadata{}
   224  			Expect(proto.Unmarshal(consensusTypeValue.Metadata, metadata)).To(Succeed())
   225  
   226  			metadata.Options.HeartbeatTick = 10
   227  			metadata.Options.ElectionTick = 10
   228  
   229  			newMetadata, err := proto.Marshal(metadata)
   230  			Expect(err).NotTo(HaveOccurred())
   231  			consensusTypeValue.Metadata = newMetadata
   232  
   233  			updatedConfig.ChannelGroup.Groups["Orderer"].Values["ConsensusType"] = &common.ConfigValue{
   234  				ModPolicy: "Admins",
   235  				Value:     protoutil.MarshalOrPanic(consensusTypeValue),
   236  			}
   237  
   238  			sess := nwo.UpdateOrdererConfigSession(network, orderer, channel, config, updatedConfig, org1Peer0, orderer)
   239  			Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(1))
   240  			Expect(sess.Err).To(gbytes.Say(`invalid new config metadata: ElectionTick \(10\) must be greater than HeartbeatTick \(10\)`))
   241  		})
   242  	})
   243  
   244  	When("a single node cluster is expanded", func() {
   245  		It("is still possible to onboard the new cluster member and then another one with a different TLS root CA", func() {
   246  			launch := func(o *nwo.Orderer) {
   247  				runner := network.OrdererRunner(o)
   248  				ordererRunners = append(ordererRunners, runner)
   249  
   250  				process := ifrit.Invoke(runner)
   251  				Eventually(process.Ready(), network.EventuallyTimeout).Should(BeClosed())
   252  				ordererProcesses = append(ordererProcesses, process)
   253  			}
   254  
   255  			network = nwo.New(nwo.BasicEtcdRaft(), testDir, client, StartPort(), components)
   256  			network.GenerateConfigTree()
   257  			network.Bootstrap()
   258  
   259  			orderer := network.Orderer("orderer")
   260  			peer = network.Peer("Org1", "peer0")
   261  
   262  			By("Launching the orderer")
   263  			launch(orderer)
   264  
   265  			By("Checking that it elected itself as a leader")
   266  			findLeader(ordererRunners)
   267  
   268  			By("Extending the network configuration to add a new orderer")
   269  			orderer2 := &nwo.Orderer{
   270  				Name:         "orderer2",
   271  				Organization: "OrdererOrg",
   272  			}
   273  			ports := nwo.Ports{}
   274  			for _, portName := range nwo.OrdererPortNames() {
   275  				ports[portName] = network.ReservePort()
   276  			}
   277  			network.PortsByOrdererID[orderer2.ID()] = ports
   278  			network.Orderers = append(network.Orderers, orderer2)
   279  			network.GenerateOrdererConfig(orderer2)
   280  			extendNetwork(network)
   281  
   282  			secondOrdererCertificatePath := filepath.Join(network.OrdererLocalTLSDir(orderer2), "server.crt")
   283  			secondOrdererCertificate, err := ioutil.ReadFile(secondOrdererCertificatePath)
   284  			Expect(err).NotTo(HaveOccurred())
   285  
   286  			By("Adding the second orderer")
   287  			addConsenter(network, peer, orderer, "systemchannel", etcdraft.Consenter{
   288  				ServerTlsCert: secondOrdererCertificate,
   289  				ClientTlsCert: secondOrdererCertificate,
   290  				Host:          "127.0.0.1",
   291  				Port:          uint32(network.OrdererPort(orderer2, nwo.ClusterPort)),
   292  			})
   293  
   294  			By("Obtaining the last config block from the orderer")
   295  			// Get the last config block of the system channel
   296  			configBlock := nwo.GetConfigBlock(network, peer, orderer, "systemchannel")
   297  			// Plant it in the file system of orderer2, the new node to be onboarded.
   298  			err = ioutil.WriteFile(filepath.Join(testDir, "systemchannel_block.pb"), protoutil.MarshalOrPanic(configBlock), 0o644)
   299  			Expect(err).NotTo(HaveOccurred())
   300  
   301  			By("Waiting for the existing orderer to relinquish its leadership")
   302  			Eventually(ordererRunners[0].Err(), network.EventuallyTimeout).Should(gbytes.Say("1 stepped down to follower since quorum is not active"))
   303  			Eventually(ordererRunners[0].Err(), network.EventuallyTimeout).Should(gbytes.Say("No leader is present, cluster size is 2"))
   304  			By("Launching the second orderer")
   305  			launch(orderer2)
   306  			By("Waiting for a leader to be re-elected")
   307  			findLeader(ordererRunners)
   308  
   309  			// In the next part of the test we're going to bring up a third node
   310  			// with a different TLS root CA. We're then going to remove the TLS
   311  			// root CA and restart the orderer, to ensure that we can dynamically
   312  			// update TLS root CAs in Raft while membership stays the same.
   313  
   314  			By("Creating configuration for a third orderer with a different TLS root CA")
   315  			orderer3 := &nwo.Orderer{
   316  				Name:         "orderer3",
   317  				Organization: "OrdererOrg",
   318  			}
   319  			ports = nwo.Ports{}
   320  			for _, portName := range nwo.OrdererPortNames() {
   321  				ports[portName] = network.ReservePort()
   322  			}
   323  			network.PortsByOrdererID[orderer3.ID()] = ports
   324  			network.Orderers = append(network.Orderers, orderer3)
   325  			network.GenerateOrdererConfig(orderer3)
   326  
   327  			tmpDir, err := ioutil.TempDir("", "e2e-etcfraft_reconfig")
   328  			Expect(err).NotTo(HaveOccurred())
   329  			defer os.RemoveAll(tmpDir)
   330  
   331  			sess, err := network.Cryptogen(commands.Generate{
   332  				Config: network.CryptoConfigPath(),
   333  				Output: tmpDir,
   334  			})
   335  			Expect(err).NotTo(HaveOccurred())
   336  			Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(0))
   337  
   338  			name := network.Orderers[0].Name
   339  			domain := network.Organization(network.Orderers[0].Organization).Domain
   340  			nameDomain := fmt.Sprintf("%s.%s", name, domain)
   341  			ordererTLSPath := filepath.Join(tmpDir, "ordererOrganizations", domain, "orderers", nameDomain, "tls")
   342  
   343  			caCertPath := filepath.Join(tmpDir, "ordererOrganizations", domain, "tlsca", fmt.Sprintf("tlsca.%s-cert.pem", domain))
   344  
   345  			caCert, err := ioutil.ReadFile(caCertPath)
   346  			Expect(err).NotTo(HaveOccurred())
   347  
   348  			thirdOrdererCertificatePath := filepath.Join(ordererTLSPath, "server.crt")
   349  			thirdOrdererCertificate, err := ioutil.ReadFile(thirdOrdererCertificatePath)
   350  			Expect(err).NotTo(HaveOccurred())
   351  
   352  			By("Updating it on the file system")
   353  			err = ioutil.WriteFile(caCertPath, caCert, 0o644)
   354  			Expect(err).NotTo(HaveOccurred())
   355  			err = ioutil.WriteFile(thirdOrdererCertificatePath, thirdOrdererCertificate, 0o644)
   356  			Expect(err).NotTo(HaveOccurred())
   357  
   358  			By("Overwriting the TLS directory of the new orderer")
   359  			for _, fileName := range []string{"server.crt", "server.key", "ca.crt"} {
   360  				dst := filepath.Join(network.OrdererLocalTLSDir(orderer3), fileName)
   361  
   362  				data, err := ioutil.ReadFile(filepath.Join(ordererTLSPath, fileName))
   363  				Expect(err).NotTo(HaveOccurred())
   364  
   365  				err = ioutil.WriteFile(dst, data, 0o644)
   366  				Expect(err).NotTo(HaveOccurred())
   367  			}
   368  
   369  			By("Obtaining the last config block from the orderer once more to update the bootstrap file")
   370  			configBlock = nwo.GetConfigBlock(network, peer, orderer, "systemchannel")
   371  			err = ioutil.WriteFile(filepath.Join(testDir, "systemchannel_block.pb"), protoutil.MarshalOrPanic(configBlock), 0o644)
   372  			Expect(err).NotTo(HaveOccurred())
   373  
   374  			By("Launching orderer3")
   375  			launch(orderer3)
   376  
   377  			By("Expanding the TLS root CA certificates and adding orderer3 to the channel")
   378  			updateOrdererMSPAndConsensusMetadata(network, peer, orderer, "systemchannel", "OrdererOrg", func(config msp.FabricMSPConfig) msp.FabricMSPConfig {
   379  				config.TlsRootCerts = append(config.TlsRootCerts, caCert)
   380  				return config
   381  			},
   382  				func(metadata *etcdraft.ConfigMetadata) {
   383  					metadata.Consenters = append(metadata.Consenters, &etcdraft.Consenter{
   384  						ServerTlsCert: thirdOrdererCertificate,
   385  						ClientTlsCert: thirdOrdererCertificate,
   386  						Host:          "127.0.0.1",
   387  						Port:          uint32(network.OrdererPort(orderer3, nwo.ClusterPort)),
   388  					})
   389  				},
   390  			)
   391  
   392  			By("Waiting for orderer3 to see the leader")
   393  			findLeader([]*ginkgomon.Runner{ordererRunners[2]})
   394  
   395  			By("Attemping to add a consenter with invalid certs")
   396  			// create new certs that are not in the channel config
   397  			ca, err := tlsgen.NewCA()
   398  			Expect(err).NotTo(HaveOccurred())
   399  			client, err := ca.NewClientCertKeyPair()
   400  			Expect(err).NotTo(HaveOccurred())
   401  
   402  			newConsenterCertPem, _ := pem.Decode(client.Cert)
   403  			newConsenterCert, err := x509.ParseCertificate(newConsenterCertPem.Bytes)
   404  			Expect(err).NotTo(HaveOccurred())
   405  
   406  			newConsenterHost := "127.0.0.1"
   407  			newConsenterPort := uint32(network.OrdererPort(orderer3, nwo.ListenPort))
   408  
   409  			current, updated := consenterAdder(
   410  				network,
   411  				peer,
   412  				orderer,
   413  				"systemchannel",
   414  				etcdraft.Consenter{
   415  					ServerTlsCert: client.Cert,
   416  					ClientTlsCert: client.Cert,
   417  					Host:          newConsenterHost,
   418  					Port:          newConsenterPort,
   419  				},
   420  			)
   421  			sess = nwo.UpdateOrdererConfigSession(network, orderer, network.SystemChannel.Name, current, updated, peer, orderer)
   422  			Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(1))
   423  			Expect(sess.Err).To(gbytes.Say(fmt.Sprintf("BAD_REQUEST -- error applying config update to existing channel 'systemchannel': consensus metadata update for channel config update is invalid: invalid new config metadata: consenter %s:%d has invalid certificate: verifying tls client cert with serial number %d: x509: certificate signed by unknown authority", newConsenterHost, newConsenterPort, newConsenterCert.SerialNumber)))
   424  		})
   425  	})
   426  
   427  	When("a single node cluster has the tick interval overridden", func() {
   428  		It("reflects this in its startup logs", func() {
   429  			network = nwo.New(nwo.BasicEtcdRaft(), testDir, client, StartPort(), components)
   430  			network.GenerateConfigTree()
   431  			network.Bootstrap()
   432  
   433  			orderer := network.Orderer("orderer")
   434  			ordererConfig := network.ReadOrdererConfig(orderer)
   435  			ordererConfig.Consensus["TickIntervalOverride"] = "642ms"
   436  			network.WriteOrdererConfig(orderer, ordererConfig)
   437  
   438  			By("Launching the orderer")
   439  			runner := network.OrdererRunner(orderer)
   440  			ordererRunners = append(ordererRunners, runner)
   441  
   442  			process := ifrit.Invoke(runner)
   443  			Eventually(process.Ready(), network.EventuallyTimeout).Should(BeClosed())
   444  			ordererProcesses = append(ordererProcesses, process)
   445  
   446  			Eventually(runner.Err()).Should(gbytes.Say("TickIntervalOverride is set, overriding channel configuration tick interval to 642ms"))
   447  		})
   448  	})
   449  
   450  	When("the orderer certificates are all rotated", func() {
   451  		It("is possible to rotate certificate by adding & removing cert in single config", func() {
   452  			layout := nwo.MultiNodeEtcdRaft()
   453  			network = nwo.New(layout, testDir, client, StartPort(), components)
   454  			network.GenerateConfigTree()
   455  			network.Bootstrap()
   456  
   457  			o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3")
   458  			orderers := []*nwo.Orderer{o1, o2, o3}
   459  			peer = network.Peer("Org1", "peer0")
   460  
   461  			By("Launching the orderers")
   462  			for _, o := range orderers {
   463  				runner := network.OrdererRunner(o)
   464  				ordererRunners = append(ordererRunners, runner)
   465  				process := ifrit.Invoke(runner)
   466  				ordererProcesses = append(ordererProcesses, process)
   467  			}
   468  
   469  			for _, ordererProc := range ordererProcesses {
   470  				Eventually(ordererProc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   471  			}
   472  
   473  			By("Finding leader")
   474  			leader := findLeader(ordererRunners)
   475  			leaderIndex := leader - 1
   476  			blockSeq := 0
   477  
   478  			By("Checking that all orderers are online")
   479  			assertBlockReception(map[string]int{
   480  				"systemchannel": blockSeq,
   481  			}, orderers, peer, network)
   482  
   483  			By("Preparing new certificates for the orderer nodes")
   484  			extendNetwork(network)
   485  			certificateRotations := refreshOrdererPEMs(network)
   486  
   487  			swap := func(o *nwo.Orderer, certificate []byte, c etcdraft.Consenter) {
   488  				updateEtcdRaftMetadata(network, peer, o, network.SystemChannel.Name, func(metadata *etcdraft.ConfigMetadata) {
   489  					var newConsenters []*etcdraft.Consenter
   490  					for _, consenter := range metadata.Consenters {
   491  						if bytes.Equal(consenter.ClientTlsCert, certificate) || bytes.Equal(consenter.ServerTlsCert, certificate) {
   492  							continue
   493  						}
   494  						newConsenters = append(newConsenters, consenter)
   495  					}
   496  					newConsenters = append(newConsenters, &c)
   497  
   498  					metadata.Consenters = newConsenters
   499  				})
   500  				blockSeq++
   501  			}
   502  
   503  			rotate := func(target int) {
   504  				// submit a config tx to rotate the cert of an orderer.
   505  				// The orderer being rotated is going to be unavailable
   506  				// eventually, therefore submitter of tx is different
   507  				// from the target, so the configuration can be reliably
   508  				// checked.
   509  				submitter := (target + 1) % 3
   510  				rotation := certificateRotations[target]
   511  				targetOrderer := network.Orderers[target]
   512  				remainder := func() []*nwo.Orderer {
   513  					var ret []*nwo.Orderer
   514  					for i, o := range network.Orderers {
   515  						if i == target {
   516  							continue
   517  						}
   518  						ret = append(ret, o)
   519  					}
   520  					return ret
   521  				}()
   522  				submitterOrderer := network.Orderers[submitter]
   523  				port := network.OrdererPort(targetOrderer, nwo.ClusterPort)
   524  
   525  				fmt.Fprintf(GinkgoWriter, "Rotating certificate of orderer node %d\n", target+1)
   526  				swap(submitterOrderer, rotation.oldCert, etcdraft.Consenter{
   527  					ServerTlsCert: rotation.newCert,
   528  					ClientTlsCert: rotation.newCert,
   529  					Host:          "127.0.0.1",
   530  					Port:          uint32(port),
   531  				})
   532  
   533  				By("Waiting for all orderers to sync")
   534  				assertBlockReception(map[string]int{
   535  					"systemchannel": blockSeq,
   536  				}, remainder, peer, network)
   537  
   538  				By("Waiting for rotated node to be unavailable")
   539  				c := commands.ChannelFetch{
   540  					ChannelID:  network.SystemChannel.Name,
   541  					Block:      "newest",
   542  					OutputFile: "/dev/null",
   543  					Orderer:    network.OrdererAddress(targetOrderer, nwo.ClusterPort),
   544  				}
   545  				Eventually(func() string {
   546  					sess, err := network.OrdererAdminSession(targetOrderer, peer, c)
   547  					Expect(err).NotTo(HaveOccurred())
   548  					Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit())
   549  					if sess.ExitCode() != 0 {
   550  						return fmt.Sprintf("exit code is %d: %s", sess.ExitCode(), string(sess.Err.Contents()))
   551  					}
   552  					sessErr := string(sess.Err.Contents())
   553  					expected := fmt.Sprintf("Received block: %d", blockSeq)
   554  					if strings.Contains(sessErr, expected) {
   555  						return ""
   556  					}
   557  					return sessErr
   558  				}, network.EventuallyTimeout, time.Second).ShouldNot(BeEmpty())
   559  
   560  				By("Killing the orderer")
   561  				ordererProcesses[target].Signal(syscall.SIGTERM)
   562  				Eventually(ordererProcesses[target].Wait(), network.EventuallyTimeout).Should(Receive())
   563  
   564  				By("Starting the orderer again")
   565  				ordererRunner := network.OrdererRunner(targetOrderer)
   566  				ordererRunners = append(ordererRunners, ordererRunner)
   567  				ordererProcesses[target] = ifrit.Invoke(ordererRunner)
   568  				Eventually(ordererProcesses[target].Ready(), network.EventuallyTimeout).Should(BeClosed())
   569  
   570  				By("And waiting for it to stabilize")
   571  				assertBlockReception(map[string]int{
   572  					"systemchannel": blockSeq,
   573  				}, orderers, peer, network)
   574  			}
   575  
   576  			By(fmt.Sprintf("Rotating cert on leader %d", leader))
   577  			rotate(leaderIndex)
   578  
   579  			By("Rotating certificates of other orderer nodes")
   580  			for i := range certificateRotations {
   581  				if i != leaderIndex {
   582  					rotate(i)
   583  				}
   584  			}
   585  		})
   586  
   587  		It("is still possible to onboard new orderers", func() {
   588  			// In this test, we have 3 OSNs and we rotate their TLS certificates one by one,
   589  			// by adding the future certificate to the channel, killing the OSN to make it
   590  			// grab the new certificate, and then removing the old certificate from the channel.
   591  
   592  			// After we completely rotate all the certificates, we put the last config block
   593  			// of the system channel into the file system of orderer4, and then launch it,
   594  			// and ensure it onboards and pulls channels testchannel only, and not testchannel2
   595  			// which it is not part of.
   596  
   597  			// Consenter i after its certificate is rotated is denoted as consenter i'
   598  			// The blocks of channels contain the following updates:
   599  			//    | system channel height | testchannel  height  | update description
   600  			// ------------------------------------------------------------------------
   601  			// 0  |            2          |         1            | adding consenter 1'
   602  			// 1  |            3          |         2            | removing consenter 1
   603  			// 2  |            4          |         3            | adding consenter 2'
   604  			// 3  |            5          |         4            | removing consenter 2
   605  			// 4  |            6          |         5            | adding consenter 3'
   606  			// 5  |            7          |         6            | removing consenter 3
   607  			// 6  |            8          |         6            | creating channel testchannel2
   608  			// 7  |            9          |         6            | creating channel testchannel3
   609  			// 8  |            10         |         7            | adding consenter 4
   610  
   611  			layout := nwo.MultiNodeEtcdRaft()
   612  			layout.Channels = append(layout.Channels, &nwo.Channel{
   613  				Name:    "testchannel2",
   614  				Profile: "TwoOrgsChannel",
   615  			}, &nwo.Channel{
   616  				Name:    "testchannel3",
   617  				Profile: "TwoOrgsChannel",
   618  			})
   619  
   620  			network = nwo.New(layout, testDir, client, StartPort(), components)
   621  			network.GenerateConfigTree()
   622  			network.Bootstrap()
   623  
   624  			o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3")
   625  			orderers := []*nwo.Orderer{o1, o2, o3}
   626  			peer = network.Peer("Org1", "peer0")
   627  
   628  			By("Launching the orderers")
   629  			for _, o := range orderers {
   630  				runner := network.OrdererRunner(o)
   631  				ordererRunners = append(ordererRunners, runner)
   632  				process := ifrit.Invoke(runner)
   633  				ordererProcesses = append(ordererProcesses, process)
   634  			}
   635  
   636  			for _, ordererProc := range ordererProcesses {
   637  				Eventually(ordererProc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   638  			}
   639  
   640  			By("Checking that all orderers are online")
   641  			assertBlockReception(map[string]int{
   642  				"systemchannel": 0,
   643  			}, orderers, peer, network)
   644  
   645  			By("Creating a channel and checking that all orderers got the channel creation")
   646  			network.CreateChannel("testchannel", network.Orderers[0], peer)
   647  			assertBlockReception(map[string]int{
   648  				"systemchannel": 1,
   649  				"testchannel":   0,
   650  			}, orderers, peer, network)
   651  
   652  			By("Preparing new certificates for the orderer nodes")
   653  			extendNetwork(network)
   654  			certificateRotations := refreshOrdererPEMs(network)
   655  
   656  			expectedBlockNumPerChannel := []map[string]int{
   657  				{"systemchannel": 2, "testchannel": 1},
   658  				{"systemchannel": 3, "testchannel": 2},
   659  				{"systemchannel": 4, "testchannel": 3},
   660  				{"systemchannel": 5, "testchannel": 4},
   661  				{"systemchannel": 6, "testchannel": 5},
   662  				{"systemchannel": 7, "testchannel": 6},
   663  			}
   664  
   665  			for i, rotation := range certificateRotations {
   666  				o := network.Orderers[i]
   667  				port := network.OrdererPort(o, nwo.ClusterPort)
   668  
   669  				By(fmt.Sprintf("Adding the future certificate of orderer node %d", i))
   670  				for _, channelName := range []string{"systemchannel", "testchannel"} {
   671  					addConsenter(network, peer, o, channelName, etcdraft.Consenter{
   672  						ServerTlsCert: rotation.newCert,
   673  						ClientTlsCert: rotation.newCert,
   674  						Host:          "127.0.0.1",
   675  						Port:          uint32(port),
   676  					})
   677  				}
   678  
   679  				By("Waiting for all orderers to sync")
   680  				assertBlockReception(expectedBlockNumPerChannel[i*2], orderers, peer, network)
   681  
   682  				By("Killing the orderer")
   683  				ordererProcesses[i].Signal(syscall.SIGTERM)
   684  				Eventually(ordererProcesses[i].Wait(), network.EventuallyTimeout).Should(Receive())
   685  
   686  				By("Starting the orderer again")
   687  				ordererRunner := network.OrdererRunner(orderers[i])
   688  				ordererRunners = append(ordererRunners, ordererRunner)
   689  				ordererProcesses[i] = ifrit.Invoke(ordererRunner)
   690  				Eventually(ordererProcesses[i].Ready(), network.EventuallyTimeout).Should(BeClosed())
   691  
   692  				By("And waiting for it to stabilize")
   693  				assertBlockReception(expectedBlockNumPerChannel[i*2], orderers, peer, network)
   694  
   695  				By("Removing the previous certificate of the old orderer")
   696  				for _, channelName := range []string{"systemchannel", "testchannel"} {
   697  					removeConsenter(network, peer, network.Orderers[(i+1)%len(network.Orderers)], channelName, rotation.oldCert)
   698  				}
   699  
   700  				By("Waiting for all orderers to sync")
   701  				assertBlockReception(expectedBlockNumPerChannel[i*2+1], orderers, peer, network)
   702  			}
   703  
   704  			By("Creating testchannel2")
   705  			network.CreateChannel("testchannel2", network.Orderers[0], peer)
   706  			assertBlockReception(map[string]int{
   707  				"systemchannel": 8,
   708  			}, orderers, peer, network)
   709  
   710  			By("Creating testchannel3")
   711  			network.CreateChannel("testchannel3", network.Orderers[0], peer)
   712  			assertBlockReception(map[string]int{
   713  				"systemchannel": 9,
   714  			}, orderers, peer, network)
   715  
   716  			o4 := &nwo.Orderer{
   717  				Name:         "orderer4",
   718  				Organization: "OrdererOrg",
   719  			}
   720  
   721  			By("Configuring orderer4 in the network")
   722  			ports := nwo.Ports{}
   723  			for _, portName := range nwo.OrdererPortNames() {
   724  				ports[portName] = network.ReservePort()
   725  			}
   726  			network.PortsByOrdererID[o4.ID()] = ports
   727  
   728  			network.Orderers = append(network.Orderers, o4)
   729  			network.GenerateOrdererConfig(network.Orderer("orderer4"))
   730  
   731  			By("Adding orderer4 to the channels")
   732  			orderer4CertificatePath := filepath.Join(network.OrdererLocalTLSDir(o4), "server.crt")
   733  			orderer4Certificate, err := ioutil.ReadFile(orderer4CertificatePath)
   734  			Expect(err).NotTo(HaveOccurred())
   735  			for _, channel := range []string{"systemchannel", "testchannel"} {
   736  				addConsenter(network, peer, o1, channel, etcdraft.Consenter{
   737  					ServerTlsCert: orderer4Certificate,
   738  					ClientTlsCert: orderer4Certificate,
   739  					Host:          "127.0.0.1",
   740  					Port:          uint32(network.OrdererPort(o4, nwo.ClusterPort)),
   741  				})
   742  			}
   743  
   744  			By("Ensuring all orderers know about orderer4's addition")
   745  			assertBlockReception(map[string]int{
   746  				"systemchannel": 10,
   747  				"testchannel":   7,
   748  			}, orderers, peer, network)
   749  
   750  			By("Broadcasting envelope to testchannel")
   751  			env := CreateBroadcastEnvelope(network, peer, "testchannel", []byte("hello"))
   752  			resp, err := ordererclient.Broadcast(network, o1, env)
   753  			Expect(err).NotTo(HaveOccurred())
   754  			Expect(resp.Status).To(Equal(common.Status_SUCCESS))
   755  
   756  			assertBlockReception(map[string]int{
   757  				"testchannel": 8,
   758  			}, orderers, peer, network)
   759  
   760  			By("Corrupting the readers policy of testchannel3")
   761  			revokeReaderAccess(network, "testchannel3", o3, peer)
   762  
   763  			// Get the last config block of the system channel
   764  			configBlock := nwo.GetConfigBlock(network, peer, o1, "systemchannel")
   765  			// Plant it in the file system of orderer4, the new node to be onboarded.
   766  			err = ioutil.WriteFile(filepath.Join(testDir, "systemchannel_block.pb"), protoutil.MarshalOrPanic(configBlock), 0o644)
   767  			Expect(err).NotTo(HaveOccurred())
   768  
   769  			By("Launching orderer4")
   770  			orderers = append(orderers, o4)
   771  			orderer4Runner := network.OrdererRunner(o4)
   772  			ordererRunners = append(ordererRunners, orderer4Runner)
   773  			// Spawn orderer4's process
   774  			o4process := ifrit.Invoke(orderer4Runner)
   775  			Eventually(o4process.Ready(), network.EventuallyTimeout).Should(BeClosed())
   776  			ordererProcesses = append(ordererProcesses, o4process)
   777  
   778  			By("And waiting for it to sync with the rest of the orderers")
   779  			assertBlockReception(map[string]int{
   780  				"systemchannel": 10,
   781  				"testchannel":   8,
   782  			}, orderers, peer, network)
   783  
   784  			By("Ensuring orderer4 doesn't serve testchannel2 and testchannel3")
   785  			env = CreateBroadcastEnvelope(network, peer, "testchannel2", []byte("hello"))
   786  			resp, err = ordererclient.Broadcast(network, o4, env)
   787  			Expect(err).NotTo(HaveOccurred())
   788  			Expect(resp.Status).To(Equal(common.Status_SERVICE_UNAVAILABLE))
   789  
   790  			env = CreateBroadcastEnvelope(network, peer, "testchannel3", []byte("hello"))
   791  			resp, err = ordererclient.Broadcast(network, o4, env)
   792  			Expect(err).NotTo(HaveOccurred())
   793  			Expect(resp.Status).To(Equal(common.Status_SERVICE_UNAVAILABLE))
   794  
   795  			belongRegex := `\QI do not belong to channel testchannel2 or am forbidden pulling it (not in the channel), skipping chain retrieval\E`
   796  			forbiddenRegex := `\QI do not belong to channel testchannel3 or am forbidden pulling it (forbidden pulling the channel), skipping chain retrieval\E`
   797  
   798  			Expect(orderer4Runner.Err()).To(gbytes.Say(belongRegex + "|" + forbiddenRegex))
   799  			Expect(orderer4Runner.Err()).To(gbytes.Say(belongRegex + "|" + forbiddenRegex))
   800  
   801  			By("Adding orderer4 to testchannel2")
   802  			addConsenter(network, peer, o1, "testchannel2", etcdraft.Consenter{
   803  				ServerTlsCert: orderer4Certificate,
   804  				ClientTlsCert: orderer4Certificate,
   805  				Host:          "127.0.0.1",
   806  				Port:          uint32(network.OrdererPort(o4, nwo.ClusterPort)),
   807  			})
   808  
   809  			By("Waiting for orderer4 and to replicate testchannel2")
   810  			assertBlockReception(map[string]int{
   811  				"testchannel2": 1,
   812  			}, []*nwo.Orderer{o4}, peer, network)
   813  
   814  			By("Ensuring orderer4 doesn't have any errors in the logs")
   815  			Consistently(orderer4Runner.Err()).ShouldNot(gbytes.Say("ERRO"))
   816  
   817  			By("Submitting a transaction through orderer4")
   818  			env = CreateBroadcastEnvelope(network, peer, "testchannel2", []byte("hello"))
   819  			resp, err = ordererclient.Broadcast(network, o4, env)
   820  			Expect(err).NotTo(HaveOccurred())
   821  			Expect(resp.Status).To(Equal(common.Status_SUCCESS))
   822  
   823  			By("And ensuring it is propagated amongst all orderers")
   824  			assertBlockReception(map[string]int{
   825  				"testchannel2": 2,
   826  			}, orderers, peer, network)
   827  		})
   828  	})
   829  
   830  	When("an orderer channel is created with a subset of nodes", func() {
   831  		It("is still possible to onboard a new orderer to the channel", func() {
   832  			network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, StartPort(), components)
   833  			network.Profiles = append(network.Profiles, &nwo.Profile{
   834  				Name:          "myprofile",
   835  				Consortium:    "SampleConsortium",
   836  				Orderers:      []string{"orderer1"},
   837  				Organizations: []string{"Org1"},
   838  			})
   839  			network.Channels = append(network.Channels, &nwo.Channel{
   840  				Name:        "mychannel",
   841  				Profile:     "myprofile",
   842  				BaseProfile: "SampleDevModeEtcdRaft",
   843  			})
   844  
   845  			network.GenerateConfigTree()
   846  			network.Bootstrap()
   847  
   848  			o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3")
   849  			orderers := []*nwo.Orderer{o1, o2, o3}
   850  			peer = network.Peer("Org1", "peer0")
   851  
   852  			By("Launching the orderers")
   853  			for _, o := range orderers {
   854  				runner := network.OrdererRunner(o)
   855  				ordererRunners = append(ordererRunners, runner)
   856  				process := ifrit.Invoke(runner)
   857  				ordererProcesses = append(ordererProcesses, process)
   858  			}
   859  
   860  			for _, ordererProc := range ordererProcesses {
   861  				Eventually(ordererProc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   862  			}
   863  
   864  			By("Waiting for the system channel to be available")
   865  			assertBlockReception(map[string]int{
   866  				"systemchannel": 0,
   867  			}, orderers, peer, network)
   868  
   869  			By("Creating a channel with a subset of orderers")
   870  			network.CreateChannel("mychannel", o1, peer, peer, o1)
   871  
   872  			By("Waiting for the channel to be available")
   873  			assertBlockReception(map[string]int{
   874  				"mychannel": 0,
   875  			}, []*nwo.Orderer{o1}, peer, network)
   876  
   877  			By("Ensuring only orderer1 services the channel")
   878  			ensureEvicted(o2, peer, network, "mychannel")
   879  			ensureEvicted(o3, peer, network, "mychannel")
   880  
   881  			By("Adding orderer2 to the channel")
   882  			ordererCertificatePath := filepath.Join(network.OrdererLocalTLSDir(o2), "server.crt")
   883  			ordererCertificate, err := ioutil.ReadFile(ordererCertificatePath)
   884  			Expect(err).NotTo(HaveOccurred())
   885  
   886  			addConsenter(network, peer, o1, "mychannel", etcdraft.Consenter{
   887  				ServerTlsCert: ordererCertificate,
   888  				ClientTlsCert: ordererCertificate,
   889  				Host:          "127.0.0.1",
   890  				Port:          uint32(network.OrdererPort(o2, nwo.ClusterPort)),
   891  			})
   892  
   893  			By("Waiting for orderer2 to join the channel")
   894  			assertBlockReception(map[string]int{
   895  				"mychannel": 1,
   896  			}, []*nwo.Orderer{o1, o2}, peer, network)
   897  
   898  			By("Adding orderer3 to the channel")
   899  			ordererCertificatePath = filepath.Join(network.OrdererLocalTLSDir(o3), "server.crt")
   900  			ordererCertificate, err = ioutil.ReadFile(ordererCertificatePath)
   901  			Expect(err).NotTo(HaveOccurred())
   902  			addConsenter(network, peer, o1, "mychannel", etcdraft.Consenter{
   903  				ServerTlsCert: ordererCertificate,
   904  				ClientTlsCert: ordererCertificate,
   905  				Host:          "127.0.0.1",
   906  				Port:          uint32(network.OrdererPort(o3, nwo.ClusterPort)),
   907  			})
   908  
   909  			By("Waiting for orderer3 to join the channel")
   910  			assertBlockReception(map[string]int{
   911  				"mychannel": 2,
   912  			}, orderers, peer, network)
   913  		})
   914  	})
   915  
   916  	When("orderer cluster is not healthy", func() {
   917  		var o1, o2 *nwo.Orderer
   918  
   919  		BeforeEach(func() {
   920  			network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, StartPort(), components)
   921  			network.GenerateConfigTree()
   922  			network.Bootstrap()
   923  
   924  			o1, o2 = network.Orderer("orderer1"), network.Orderer("orderer2")
   925  			orderers := []*nwo.Orderer{o1, o2}
   926  			By("Launching the orderers")
   927  			for _, o := range orderers {
   928  				runner := network.OrdererRunner(o)
   929  				ordererRunners = append(ordererRunners, runner)
   930  				process := ifrit.Invoke(runner)
   931  				ordererProcesses = append(ordererProcesses, process)
   932  			}
   933  
   934  			for _, ordererProc := range ordererProcesses {
   935  				Eventually(ordererProc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   936  			}
   937  		})
   938  
   939  		AfterEach(func() {
   940  			for _, ordererInstance := range ordererProcesses {
   941  				ordererInstance.Signal(syscall.SIGTERM)
   942  				Eventually(ordererInstance.Wait(), network.EventuallyTimeout).Should(Receive())
   943  			}
   944  		})
   945  
   946  		It("refuses to reconfig if it results in quorum loss", func() {
   947  			By("Waiting for them to elect a leader")
   948  			findLeader(ordererRunners)
   949  
   950  			extendNetwork(network)
   951  			certificatesOfOrderers := refreshOrdererPEMs(network)
   952  
   953  			By("Removing alive node from 2/3 cluster")
   954  			peer := network.Peer("Org1", "peer0")
   955  			current, updated := consenterRemover(network, peer, o2, network.SystemChannel.Name, certificatesOfOrderers[1].oldCert)
   956  			Eventually(func() []byte {
   957  				sess := nwo.UpdateOrdererConfigSession(network, o2, network.SystemChannel.Name, current, updated, peer, o2)
   958  				Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(1))
   959  				return sess.Err.Contents()
   960  			}, network.EventuallyTimeout).Should(ContainSubstring("2 out of 3 nodes are alive, configuration will result in quorum loss"))
   961  
   962  			By("Adding node to 2/3 cluster")
   963  			current, updated = consenterAdder(
   964  				network,
   965  				peer,
   966  				o2,
   967  				network.SystemChannel.Name,
   968  				etcdraft.Consenter{
   969  					ServerTlsCert: certificatesOfOrderers[0].newCert,
   970  					ClientTlsCert: certificatesOfOrderers[0].newCert,
   971  					Host:          "127.0.0.1",
   972  					Port:          uint32(network.OrdererPort(o1, nwo.ListenPort)),
   973  				},
   974  			)
   975  			sess := nwo.UpdateOrdererConfigSession(network, o2, network.SystemChannel.Name, current, updated, peer, o2)
   976  			Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(1))
   977  			Expect(string(sess.Err.Contents())).To(ContainSubstring("2 out of 3 nodes are alive, configuration will result in quorum loss"))
   978  		})
   979  	})
   980  
   981  	When("an orderer node is evicted", func() {
   982  		BeforeEach(func() {
   983  			ordererRunners = nil
   984  			ordererProcesses = nil
   985  
   986  			network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, StartPort(), components)
   987  			network.GenerateConfigTree()
   988  			network.Bootstrap()
   989  
   990  			o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3")
   991  			orderers := []*nwo.Orderer{o1, o2, o3}
   992  			peer = network.Peer("Org1", "peer0")
   993  
   994  			By("Launching the orderers")
   995  			for _, o := range orderers {
   996  				runner := network.OrdererRunner(o, "FABRIC_LOGGING_SPEC=orderer.consensus.etcdraft=debug:info")
   997  				ordererRunners = append(ordererRunners, runner)
   998  				process := ifrit.Invoke(runner)
   999  				ordererProcesses = append(ordererProcesses, process)
  1000  			}
  1001  
  1002  			for _, ordererProc := range ordererProcesses {
  1003  				Eventually(ordererProc.Ready(), network.EventuallyTimeout).Should(BeClosed())
  1004  			}
  1005  		})
  1006  
  1007  		AfterEach(func() {
  1008  			for _, ordererInstance := range ordererProcesses {
  1009  				ordererInstance.Signal(syscall.SIGTERM)
  1010  				Eventually(ordererInstance.Wait(), network.EventuallyTimeout).Should(Receive())
  1011  			}
  1012  		})
  1013  
  1014  		It("doesn't complain and does it obediently", func() {
  1015  			o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3")
  1016  			orderers := []*nwo.Orderer{o1, o2, o3}
  1017  
  1018  			By("Waiting for them to elect a leader")
  1019  			firstEvictedNode := findLeader(ordererRunners) - 1
  1020  
  1021  			By("Removing the leader from 3-node channel")
  1022  			server1CertBytes, err := ioutil.ReadFile(filepath.Join(network.OrdererLocalTLSDir(orderers[firstEvictedNode]), "server.crt"))
  1023  			Expect(err).To(Not(HaveOccurred()))
  1024  
  1025  			removeConsenter(network, peer, network.Orderers[(firstEvictedNode+1)%3], "systemchannel", server1CertBytes)
  1026  
  1027  			var survivedOrdererRunners []*ginkgomon.Runner
  1028  			for i := range orderers {
  1029  				if i == firstEvictedNode {
  1030  					continue
  1031  				}
  1032  
  1033  				survivedOrdererRunners = append(survivedOrdererRunners, ordererRunners[i])
  1034  			}
  1035  
  1036  			secondEvictedNode := findLeader(survivedOrdererRunners) - 1
  1037  
  1038  			var survivor int
  1039  			for i := range orderers {
  1040  				if i != firstEvictedNode && i != secondEvictedNode {
  1041  					survivor = i
  1042  					break
  1043  				}
  1044  			}
  1045  
  1046  			const stopMsg = "Raft node stopped channel=systemchannel"
  1047  			fmt.Fprintln(GinkgoWriter, "Ensuring the evicted orderer stops rafting on channel systemchannel")
  1048  			Eventually(ordererRunners[firstEvictedNode].Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say(stopMsg))
  1049  
  1050  			By("Ensuring the evicted orderer now doesn't serve clients")
  1051  			ensureEvicted(orderers[firstEvictedNode], peer, network, "systemchannel")
  1052  
  1053  			By("Removing the leader from 2-node channel")
  1054  			server2CertBytes, err := ioutil.ReadFile(filepath.Join(network.OrdererLocalTLSDir(orderers[secondEvictedNode]), "server.crt"))
  1055  			Expect(err).To(Not(HaveOccurred()))
  1056  
  1057  			removeConsenter(network, peer, orderers[survivor], "systemchannel", server2CertBytes)
  1058  			findLeader([]*ginkgomon.Runner{ordererRunners[survivor]})
  1059  
  1060  			fmt.Fprintln(GinkgoWriter, "Ensuring the other orderer detect the eviction of the node on channel systemchannel")
  1061  			Eventually(ordererRunners[secondEvictedNode].Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say(stopMsg))
  1062  
  1063  			By("Ensuring the evicted orderer now doesn't serve clients")
  1064  			ensureEvicted(orderers[secondEvictedNode], peer, network, "systemchannel")
  1065  
  1066  			By("Re-adding first evicted orderer")
  1067  			addConsenter(network, peer, network.Orderers[survivor], "systemchannel", etcdraft.Consenter{
  1068  				Host:          "127.0.0.1",
  1069  				Port:          uint32(network.OrdererPort(orderers[firstEvictedNode], nwo.ClusterPort)),
  1070  				ClientTlsCert: server1CertBytes,
  1071  				ServerTlsCert: server1CertBytes,
  1072  			})
  1073  
  1074  			By("Ensuring re-added orderer starts serving system channel")
  1075  			assertBlockReception(map[string]int{
  1076  				"systemchannel": 3,
  1077  			}, []*nwo.Orderer{orderers[firstEvictedNode]}, peer, network)
  1078  
  1079  			env := CreateBroadcastEnvelope(network, orderers[secondEvictedNode], network.SystemChannel.Name, []byte("foo"))
  1080  			resp, err := ordererclient.Broadcast(network, orderers[survivor], env)
  1081  			Expect(err).NotTo(HaveOccurred())
  1082  			Expect(resp.Status).To(Equal(common.Status_SUCCESS))
  1083  		})
  1084  
  1085  		When("an evicted node is added back while it's offline", func() {
  1086  			It("can start with correct new raft id", func() {
  1087  				o1 := network.Orderer("orderer1")
  1088  				o2 := network.Orderer("orderer2")
  1089  				o3 := network.Orderer("orderer3")
  1090  
  1091  				By("Waiting for them to elect a leader")
  1092  				findLeader(ordererRunners)
  1093  
  1094  				By("Creating an application channel testchannel")
  1095  				network.CreateChannel("testchannel", o1, peer)
  1096  
  1097  				assertBlockReception(map[string]int{
  1098  					"testchannel":   0,
  1099  					"systemchannel": 1,
  1100  				}, []*nwo.Orderer{o1, o2, o3}, peer, network)
  1101  
  1102  				By("Killing the first orderer")
  1103  				ordererProcesses[0].Signal(syscall.SIGTERM)
  1104  				Eventually(ordererProcesses[0].Wait(), network.EventuallyTimeout).Should(Receive())
  1105  
  1106  				// We need to wait for stabilization, as we might have killed the leader OSN.
  1107  				By("Waiting for the channel to stabilize after killing the orderer")
  1108  				assertBlockReception(map[string]int{
  1109  					"testchannel":   0,
  1110  					"systemchannel": 1,
  1111  				}, []*nwo.Orderer{o2, o3}, peer, network)
  1112  
  1113  				By("observing active nodes to shrink")
  1114  				o2Runner := ordererRunners[1]
  1115  				Eventually(o2Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("Current active nodes in cluster are: \\[2 3\\]"))
  1116  
  1117  				By("Removing the first orderer from the application channel")
  1118  				server1CertBytes, err := ioutil.ReadFile(filepath.Join(network.OrdererLocalTLSDir(o1), "server.crt"))
  1119  				Expect(err).To(Not(HaveOccurred()))
  1120  				removeConsenter(network, peer, o2, "testchannel", server1CertBytes)
  1121  
  1122  				By("Adding the evicted orderer back to the application channel")
  1123  				addConsenter(network, peer, o2, "testchannel", etcdraft.Consenter{
  1124  					ServerTlsCert: server1CertBytes,
  1125  					ClientTlsCert: server1CertBytes,
  1126  					Host:          "127.0.0.1",
  1127  					Port:          uint32(network.OrdererPort(o1, nwo.ClusterPort)),
  1128  				})
  1129  
  1130  				By("Removing the first orderer from the application channel again")
  1131  				removeConsenter(network, peer, o2, "testchannel", server1CertBytes)
  1132  
  1133  				By("Adding the evicted orderer back to the application channel again")
  1134  				addConsenter(network, peer, o2, "testchannel", etcdraft.Consenter{
  1135  					ServerTlsCert: server1CertBytes,
  1136  					ClientTlsCert: server1CertBytes,
  1137  					Host:          "127.0.0.1",
  1138  					Port:          uint32(network.OrdererPort(o1, nwo.ClusterPort)),
  1139  				})
  1140  
  1141  				By("Starting the first orderer again")
  1142  				o1Runner := network.OrdererRunner(o1)
  1143  				ordererProcesses[0] = ifrit.Invoke(o1Runner)
  1144  				Eventually(ordererProcesses[0].Ready(), network.EventuallyTimeout).Should(BeClosed())
  1145  
  1146  				By("Submitting tx")
  1147  				env := CreateBroadcastEnvelope(network, o2, "testchannel", []byte("foo"))
  1148  				resp, err := ordererclient.Broadcast(network, o2, env)
  1149  				Expect(err).NotTo(HaveOccurred())
  1150  				Expect(resp.Status).To(Equal(common.Status_SUCCESS))
  1151  
  1152  				By("Waiting for the channel to stabilize")
  1153  				assertBlockReception(map[string]int{
  1154  					"testchannel": 5,
  1155  				}, []*nwo.Orderer{o1, o2, o3}, peer, network)
  1156  			})
  1157  		})
  1158  
  1159  		It("notices it even if it is down at the time of its eviction", func() {
  1160  			o1 := network.Orderer("orderer1")
  1161  			o2 := network.Orderer("orderer2")
  1162  			o3 := network.Orderer("orderer3")
  1163  
  1164  			orderers := []*nwo.Orderer{o1, o2, o3}
  1165  
  1166  			By("Waiting for them to elect a leader")
  1167  			findLeader(ordererRunners)
  1168  
  1169  			By("Creating a channel")
  1170  			network.CreateChannel("testchannel", o1, peer)
  1171  
  1172  			assertBlockReception(map[string]int{
  1173  				"testchannel":   0,
  1174  				"systemchannel": 1,
  1175  			}, []*nwo.Orderer{o1, o2, o3}, peer, network)
  1176  
  1177  			By("Killing the orderer")
  1178  			ordererProcesses[0].Signal(syscall.SIGTERM)
  1179  			Eventually(ordererProcesses[0].Wait(), network.EventuallyTimeout).Should(Receive())
  1180  
  1181  			// We need to wait for stabilization, as we might have killed the leader OSN.
  1182  			By("Waiting for the channel to stabilize after killing the orderer")
  1183  			assertBlockReception(map[string]int{
  1184  				"testchannel":   0,
  1185  				"systemchannel": 1,
  1186  			}, []*nwo.Orderer{o2, o3}, peer, network)
  1187  
  1188  			By("Removing the first orderer from an application channel")
  1189  			o1cert, err := ioutil.ReadFile(path.Join(network.OrdererLocalTLSDir(o1), "server.crt"))
  1190  			Expect(err).ToNot(HaveOccurred())
  1191  			removeConsenter(network, peer, o2, "testchannel", o1cert)
  1192  
  1193  			By("Starting the orderer again")
  1194  			ordererRunner := network.OrdererRunner(orderers[0])
  1195  			ordererRunners[0] = ordererRunner
  1196  			ordererProcesses[0] = ifrit.Invoke(ordererRunner)
  1197  			Eventually(ordererProcesses[0].Ready(), network.EventuallyTimeout).Should(BeClosed())
  1198  
  1199  			By("Ensuring the remaining OSNs reject authentication")
  1200  			Eventually(ordererRunners[1].Err(), time.Minute, time.Second).Should(gbytes.Say("certificate extracted from TLS connection isn't authorized"))
  1201  			Eventually(ordererRunners[2].Err(), time.Minute, time.Second).Should(gbytes.Say("certificate extracted from TLS connection isn't authorized"))
  1202  
  1203  			By("Ensuring it detects its eviction")
  1204  			evictionDetection := gbytes.Say(`Detected our own eviction from the channel in block \[1\] channel=testchannel`)
  1205  			Eventually(ordererRunner.Err(), time.Minute, time.Second).Should(evictionDetection)
  1206  
  1207  			By("Ensuring all blocks are pulled up to the block that evicts the OSN")
  1208  			Eventually(ordererRunner.Err(), time.Minute, time.Second).Should(gbytes.Say("Periodic check is stopping. channel=testchannel"))
  1209  			Eventually(ordererRunner.Err(), time.Minute, time.Second).Should(gbytes.Say("Pulled all blocks up to eviction block. channel=testchannel"))
  1210  
  1211  			By("Killing the evicted orderer")
  1212  			ordererProcesses[0].Signal(syscall.SIGTERM)
  1213  			Eventually(ordererProcesses[0].Wait(), network.EventuallyTimeout).Should(Receive())
  1214  
  1215  			By("Starting the evicted orderer again")
  1216  			ordererRunner = network.OrdererRunner(orderers[0])
  1217  			ordererRunners[0] = ordererRunner
  1218  			ordererProcesses[0] = ifrit.Invoke(ordererRunner)
  1219  			Eventually(ordererProcesses[0].Ready(), network.EventuallyTimeout).Should(BeClosed())
  1220  
  1221  			By("Ensuring the evicted orderer starts up marked the channel is inactive")
  1222  			Eventually(ordererRunner.Err(), time.Minute, time.Second).Should(gbytes.Say("Found 1 inactive chains"))
  1223  
  1224  			iDoNotBelong := "I do not belong to channel testchannel or am forbidden pulling it"
  1225  			Eventually(ordererRunner.Err(), time.Minute, time.Second).Should(gbytes.Say(iDoNotBelong))
  1226  
  1227  			By("Adding the evicted orderer back to the application channel")
  1228  			addConsenter(network, peer, o2, "testchannel", etcdraft.Consenter{
  1229  				ServerTlsCert: o1cert,
  1230  				ClientTlsCert: o1cert,
  1231  				Host:          "127.0.0.1",
  1232  				Port:          uint32(network.OrdererPort(orderers[0], nwo.ClusterPort)),
  1233  			})
  1234  
  1235  			By("Ensuring the re-added orderer joins the Raft cluster")
  1236  			findLeader([]*ginkgomon.Runner{ordererRunner})
  1237  		})
  1238  	})
  1239  
  1240  	When("an orderer node is joined", func() {
  1241  		It("isn't influenced by outdated orderers", func() {
  1242  			// This test checks that if a lagged is not aware of newly added nodes,
  1243  			// among which leader is present, it eventually pulls config block from
  1244  			// the orderer it knows, gets the certificates from it and participate
  1245  			// in consensus again.
  1246  			//
  1247  			// Steps:
  1248  			// Initial nodes in cluster: <1, 2, 3, 4>
  1249  			// - start <1, 2, 3>
  1250  			// - add <5, 6, 7>, start <5, 6, 7>
  1251  			// - kill <1>
  1252  			// - submit a tx, so that Raft index on <1> is behind <5, 6, 7> and <2, 3>
  1253  			// - kill <2, 3>
  1254  			// - start <1> and <4>. Since <1> is behind <5, 6, 7>, leader is certainly
  1255  			//   going to be elected from <5, 6, 7>
  1256  			// - assert that even <4> is not aware of leader, it can pull config block
  1257  			//   from <1>, and start participating in consensus.
  1258  
  1259  			orderers := make([]*nwo.Orderer, 7)
  1260  			ordererRunners = make([]*ginkgomon.Runner, 7)
  1261  			ordererProcesses = make([]ifrit.Process, 7)
  1262  
  1263  			for i := range orderers {
  1264  				orderers[i] = &nwo.Orderer{
  1265  					Name:         fmt.Sprintf("orderer%d", i+1),
  1266  					Organization: "OrdererOrg",
  1267  				}
  1268  			}
  1269  
  1270  			layout := nwo.MultiNodeEtcdRaft()
  1271  			layout.Orderers = orderers[:4]
  1272  			layout.Profiles[0].Orderers = []string{"orderer1", "orderer2", "orderer3", "orderer4"}
  1273  
  1274  			network = nwo.New(layout, testDir, client, StartPort(), components)
  1275  			network.GenerateConfigTree()
  1276  			network.Bootstrap()
  1277  
  1278  			peer = network.Peer("Org1", "peer0")
  1279  
  1280  			launch := func(i int) {
  1281  				runner := network.OrdererRunner(orderers[i])
  1282  				ordererRunners[i] = runner
  1283  				process := ifrit.Invoke(runner)
  1284  				Eventually(process.Ready(), network.EventuallyTimeout).Should(BeClosed())
  1285  				ordererProcesses[i] = process
  1286  			}
  1287  
  1288  			By("Launching 3 out of 4 orderers")
  1289  			for i := range orderers[:3] {
  1290  				launch(i)
  1291  			}
  1292  
  1293  			leader := findLeader(ordererRunners[:3])
  1294  
  1295  			By("Checking that all orderers are online")
  1296  			assertBlockReception(map[string]int{
  1297  				"systemchannel": 0,
  1298  			}, orderers[:3], peer, network)
  1299  
  1300  			By("Configuring orderer[5, 6, 7] in the network")
  1301  			extendNetwork(network)
  1302  
  1303  			for _, o := range orderers[4:7] {
  1304  				ports := nwo.Ports{}
  1305  				for _, portName := range nwo.OrdererPortNames() {
  1306  					ports[portName] = network.ReservePort()
  1307  				}
  1308  				network.PortsByOrdererID[o.ID()] = ports
  1309  
  1310  				network.Orderers = append(network.Orderers, o)
  1311  				network.GenerateOrdererConfig(o)
  1312  			}
  1313  
  1314  			// Backup previous system channel block
  1315  			genesisBootBlock, err := ioutil.ReadFile(filepath.Join(testDir, "systemchannel_block.pb"))
  1316  			Expect(err).NotTo(HaveOccurred())
  1317  			restoreBootBlock := func() {
  1318  				err = ioutil.WriteFile(filepath.Join(testDir, "systemchannel_block.pb"), genesisBootBlock, 0o644)
  1319  				Expect(err).NotTo(HaveOccurred())
  1320  			}
  1321  
  1322  			blockNum := 0 // there's only one block in channel - genesis
  1323  			for _, i := range []int{4, 5, 6} {
  1324  				By(fmt.Sprintf("Adding orderer%d", i+1))
  1325  				ordererCertificatePath := filepath.Join(network.OrdererLocalTLSDir(orderers[i]), "server.crt")
  1326  				ordererCertificate, err := ioutil.ReadFile(ordererCertificatePath)
  1327  				Expect(err).NotTo(HaveOccurred())
  1328  
  1329  				addConsenter(network, peer, orderers[0], "systemchannel", etcdraft.Consenter{
  1330  					ServerTlsCert: ordererCertificate,
  1331  					ClientTlsCert: ordererCertificate,
  1332  					Host:          "127.0.0.1",
  1333  					Port:          uint32(network.OrdererPort(orderers[i], nwo.ClusterPort)),
  1334  				})
  1335  				blockNum++
  1336  
  1337  				// Get the last config block of the system channel
  1338  				configBlock := nwo.GetConfigBlock(network, peer, orderers[0], "systemchannel")
  1339  				// Plant it in the file system of orderer, the new node to be onboarded.
  1340  				err = ioutil.WriteFile(filepath.Join(testDir, "systemchannel_block.pb"), protoutil.MarshalOrPanic(configBlock), 0o644)
  1341  				Expect(err).NotTo(HaveOccurred())
  1342  
  1343  				By(fmt.Sprintf("Launching orderer%d", i+1))
  1344  				launch(i)
  1345  
  1346  				By(fmt.Sprintf("Checking that orderer%d has onboarded the network", i+1))
  1347  				assertBlockReception(map[string]int{
  1348  					"systemchannel": blockNum,
  1349  				}, []*nwo.Orderer{orderers[i]}, peer, network)
  1350  			}
  1351  
  1352  			Expect(findLeader(ordererRunners[4:])).To(Equal(leader))
  1353  
  1354  			// Later on, when we start [1, 4, 5, 6, 7], we want to make sure that leader
  1355  			// is elected from [5, 6, 7], who are unknown to [4]. So we can assert that
  1356  			// [4] suspects its own eviction, pulls block from [1], and join the cluster.
  1357  			// Otherwise, if [1] is elected, and all other nodes already knew it, [4] may
  1358  			// simply replicate missing blocks with Raft, instead of pulling from others
  1359  			// triggered by eviction suspector.
  1360  
  1361  			By("Killing orderer1")
  1362  			ordererProcesses[0].Signal(syscall.SIGTERM)
  1363  			Eventually(ordererProcesses[0].Wait(), network.EventuallyTimeout).Should(Receive())
  1364  
  1365  			By("Submitting another tx to increment Raft index on alive orderers")
  1366  			if leader == 1 {
  1367  				// if orderer1 was leader, we should expect a new leader being elected before going forward
  1368  				findLeader([]*ginkgomon.Runner{ordererRunners[4]})
  1369  			}
  1370  
  1371  			env := CreateBroadcastEnvelope(network, orderers[4], network.SystemChannel.Name, []byte("hello"))
  1372  			resp, err := ordererclient.Broadcast(network, orderers[4], env)
  1373  			Expect(err).NotTo(HaveOccurred())
  1374  			Expect(resp.Status).To(Equal(common.Status_SUCCESS))
  1375  			blockNum++
  1376  
  1377  			assertBlockReception(map[string]int{
  1378  				"systemchannel": blockNum,
  1379  			}, []*nwo.Orderer{orderers[1], orderers[2], orderers[4], orderers[5], orderers[6]}, peer, network) // alive orderers: 2, 3, 5, 6, 7
  1380  
  1381  			By("Killing orderer[2,3]")
  1382  			for _, i := range []int{1, 2} {
  1383  				ordererProcesses[i].Signal(syscall.SIGTERM)
  1384  				Eventually(ordererProcesses[i].Wait(), network.EventuallyTimeout).Should(Receive())
  1385  			}
  1386  
  1387  			By("Launching the orderer that was never started")
  1388  			restoreBootBlock()
  1389  			launch(3)
  1390  
  1391  			By("Launching orderer1")
  1392  			launch(0)
  1393  
  1394  			By("Waiting until it suspects its eviction from the channel")
  1395  			Eventually(ordererRunners[3].Err(), time.Minute, time.Second).Should(gbytes.Say("Suspecting our own eviction from the channel"))
  1396  
  1397  			By("Making sure 4/7 orderers form quorum and serve request")
  1398  			assertBlockReception(map[string]int{
  1399  				"systemchannel": blockNum,
  1400  			}, []*nwo.Orderer{orderers[3], orderers[4], orderers[5], orderers[6]}, peer, network) // alive orderers: 4, 5, 6, 7
  1401  		})
  1402  	})
  1403  
  1404  	It("can create a channel that contains a subset of orderers in system channel", func() {
  1405  		config := nwo.BasicEtcdRaft()
  1406  		config.Orderers = []*nwo.Orderer{
  1407  			{Name: "orderer1", Organization: "OrdererOrg"},
  1408  			{Name: "orderer2", Organization: "OrdererOrg"},
  1409  			{Name: "orderer3", Organization: "OrdererOrg"},
  1410  		}
  1411  		config.Profiles = []*nwo.Profile{{
  1412  			Name:     "SampleDevModeEtcdRaft",
  1413  			Orderers: []string{"orderer1", "orderer2", "orderer3"},
  1414  		}, {
  1415  			Name:          "ThreeOrdererChannel",
  1416  			Consortium:    "SampleConsortium",
  1417  			Organizations: []string{"Org1", "Org2"},
  1418  			Orderers:      []string{"orderer1", "orderer2", "orderer3"},
  1419  		}, {
  1420  			Name:          "SingleOrdererChannel",
  1421  			Consortium:    "SampleConsortium",
  1422  			Organizations: []string{"Org1", "Org2"},
  1423  			Orderers:      []string{"orderer1"},
  1424  		}}
  1425  		config.Channels = []*nwo.Channel{
  1426  			{Name: "single-orderer-channel", Profile: "SingleOrdererChannel", BaseProfile: "SampleDevModeEtcdRaft"},
  1427  			{Name: "three-orderer-channel", Profile: "ThreeOrdererChannel"},
  1428  		}
  1429  
  1430  		network = nwo.New(config, testDir, client, StartPort(), components)
  1431  		o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3")
  1432  		orderers := []*nwo.Orderer{o1, o2, o3}
  1433  		peer = network.Peer("Org1", "peer0")
  1434  
  1435  		network.GenerateConfigTree()
  1436  		network.Bootstrap()
  1437  
  1438  		By("Launching the orderers")
  1439  		for _, o := range orderers {
  1440  			runner := network.OrdererRunner(o)
  1441  			ordererRunners = append(ordererRunners, runner)
  1442  			process := ifrit.Invoke(runner)
  1443  			ordererProcesses = append(ordererProcesses, process)
  1444  		}
  1445  
  1446  		for _, ordererProc := range ordererProcesses {
  1447  			Eventually(ordererProc.Ready(), network.EventuallyTimeout).Should(BeClosed())
  1448  		}
  1449  
  1450  		By("Creating an application channel with a subset of orderers in system channel")
  1451  		additionalPeer := network.Peer("Org2", "peer0")
  1452  		network.CreateChannel("single-orderer-channel", network.Orderers[0], peer, additionalPeer, network.Orderers[0])
  1453  
  1454  		By("Creating another channel via the orderer that is in system channel but not app channel")
  1455  		network.CreateChannel("three-orderer-channel", network.Orderers[2], peer)
  1456  	})
  1457  
  1458  	It("can add a new orderer organization", func() {
  1459  		network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, StartPort(), components)
  1460  		o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3")
  1461  		orderers := []*nwo.Orderer{o1, o2, o3}
  1462  
  1463  		network.GenerateConfigTree()
  1464  		network.Bootstrap()
  1465  
  1466  		By("Launching the orderers")
  1467  		for _, o := range orderers {
  1468  			runner := network.OrdererRunner(o)
  1469  			ordererRunners = append(ordererRunners, runner)
  1470  			process := ifrit.Invoke(runner)
  1471  			ordererProcesses = append(ordererProcesses, process)
  1472  		}
  1473  
  1474  		for _, ordererProc := range ordererProcesses {
  1475  			Eventually(ordererProc.Ready(), network.EventuallyTimeout).Should(BeClosed())
  1476  		}
  1477  
  1478  		By("Waiting for system channel to be ready")
  1479  		findLeader(ordererRunners)
  1480  
  1481  		peer := network.Peer("Org1", "peer0")
  1482  		channel := "systemchannel"
  1483  
  1484  		config := nwo.GetConfig(network, peer, o1, channel)
  1485  		updatedConfig := proto.Clone(config).(*common.Config)
  1486  
  1487  		ordererOrg := updatedConfig.ChannelGroup.Groups["Orderer"].Groups["OrdererOrg"]
  1488  		mspConfig := &msp.MSPConfig{}
  1489  		err := proto.Unmarshal(ordererOrg.Values["MSP"].Value, mspConfig)
  1490  		Expect(err).NotTo(HaveOccurred())
  1491  
  1492  		fabMSPConfig := &msp.FabricMSPConfig{}
  1493  		err = proto.Unmarshal(mspConfig.Config, fabMSPConfig)
  1494  		Expect(err).NotTo(HaveOccurred())
  1495  
  1496  		fabMSPConfig.Name = "OrdererMSP2"
  1497  
  1498  		mspConfig.Config, _ = proto.Marshal(fabMSPConfig)
  1499  		updatedConfig.ChannelGroup.Groups["Orderer"].Groups["OrdererMSP2"] = &common.ConfigGroup{
  1500  			Values: map[string]*common.ConfigValue{
  1501  				"MSP": {
  1502  					Value:     protoutil.MarshalOrPanic(mspConfig),
  1503  					ModPolicy: "Admins",
  1504  				},
  1505  			},
  1506  			ModPolicy: "Admins",
  1507  		}
  1508  
  1509  		nwo.UpdateOrdererConfig(network, o1, channel, config, updatedConfig, peer, o1)
  1510  	})
  1511  })
  1512  
  1513  func ensureEvicted(evictedOrderer *nwo.Orderer, submitter *nwo.Peer, network *nwo.Network, channel string) {
  1514  	c := commands.ChannelFetch{
  1515  		ChannelID:  channel,
  1516  		Block:      "newest",
  1517  		OutputFile: "/dev/null",
  1518  		Orderer:    network.OrdererAddress(evictedOrderer, nwo.ListenPort),
  1519  	}
  1520  
  1521  	sess, err := network.OrdererAdminSession(evictedOrderer, submitter, c)
  1522  	Expect(err).NotTo(HaveOccurred())
  1523  
  1524  	Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit())
  1525  	Expect(sess.Err).To(gbytes.Say("SERVICE_UNAVAILABLE"))
  1526  }
  1527  
  1528  var extendedCryptoConfig = `---
  1529  OrdererOrgs:
  1530  - Name: OrdererOrg
  1531    Domain: example.com
  1532    EnableNodeOUs: false
  1533    CA:
  1534      Hostname: ca
  1535    Specs:
  1536    - Hostname: orderer1
  1537      SANS:
  1538      - localhost
  1539      - 127.0.0.1
  1540      - ::1
  1541    - Hostname: orderer1new
  1542      SANS:
  1543      - localhost
  1544      - 127.0.0.1
  1545      - ::1
  1546    - Hostname: orderer2
  1547      SANS:
  1548      - localhost
  1549      - 127.0.0.1
  1550      - ::1
  1551    - Hostname: orderer2new
  1552      SANS:
  1553      - localhost
  1554      - 127.0.0.1
  1555      - ::1
  1556    - Hostname: orderer3
  1557      SANS:
  1558      - localhost
  1559      - 127.0.0.1
  1560      - ::1
  1561    - Hostname: orderer3new
  1562      SANS:
  1563      - localhost
  1564      - 127.0.0.1
  1565      - ::1
  1566    - Hostname: orderer4
  1567      SANS:
  1568      - localhost
  1569      - 127.0.0.1
  1570      - ::1
  1571    - Hostname: orderer5
  1572      SANS:
  1573      - localhost
  1574      - 127.0.0.1
  1575      - ::1
  1576    - Hostname: orderer6
  1577      SANS:
  1578      - localhost
  1579      - 127.0.0.1
  1580      - ::1
  1581    - Hostname: orderer7
  1582      SANS:
  1583      - localhost
  1584      - 127.0.0.1
  1585      - ::1
  1586  `
  1587  
  1588  type certificateChange struct {
  1589  	srcFile string
  1590  	dstFile string
  1591  	oldCert []byte
  1592  	oldKey  []byte
  1593  	newCert []byte
  1594  }
  1595  
  1596  // extendNetwork rotates adds an additional orderer
  1597  func extendNetwork(n *nwo.Network) {
  1598  	// Overwrite the current crypto-config with additional orderers
  1599  	cryptoConfigYAML, err := ioutil.TempFile("", "crypto-config.yaml")
  1600  	Expect(err).NotTo(HaveOccurred())
  1601  	defer os.Remove(cryptoConfigYAML.Name())
  1602  
  1603  	err = ioutil.WriteFile(cryptoConfigYAML.Name(), []byte(extendedCryptoConfig), 0o644)
  1604  	Expect(err).NotTo(HaveOccurred())
  1605  
  1606  	// Invoke cryptogen extend to add new orderers
  1607  	sess, err := n.Cryptogen(commands.Extend{
  1608  		Config: cryptoConfigYAML.Name(),
  1609  		Input:  n.CryptoPath(),
  1610  	})
  1611  	Expect(err).NotTo(HaveOccurred())
  1612  	Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(0))
  1613  }
  1614  
  1615  // refreshOrdererPEMs rotates all TLS certificates of all nodes,
  1616  // and returns the deltas
  1617  func refreshOrdererPEMs(n *nwo.Network) []*certificateChange {
  1618  	var fileChanges []*certificateChange
  1619  	// Populate source to destination files
  1620  	err := filepath.Walk(n.CryptoPath(), func(path string, info os.FileInfo, err error) error {
  1621  		if !strings.Contains(path, "/tls/") {
  1622  			return nil
  1623  		}
  1624  		if strings.Contains(path, "new") {
  1625  			fileChanges = append(fileChanges, &certificateChange{
  1626  				srcFile: path,
  1627  				dstFile: strings.Replace(path, "new", "", -1),
  1628  			})
  1629  		}
  1630  		return nil
  1631  	})
  1632  	Expect(err).NotTo(HaveOccurred())
  1633  
  1634  	var serverCertChanges []*certificateChange
  1635  
  1636  	// Overwrite the destination files with the contents of the source files.
  1637  	for _, certChange := range fileChanges {
  1638  		previousCertBytes, err := ioutil.ReadFile(certChange.dstFile)
  1639  		Expect(err).NotTo(HaveOccurred())
  1640  
  1641  		newCertBytes, err := ioutil.ReadFile(certChange.srcFile)
  1642  		Expect(err).NotTo(HaveOccurred())
  1643  
  1644  		err = ioutil.WriteFile(certChange.dstFile, newCertBytes, 0o644)
  1645  		Expect(err).NotTo(HaveOccurred())
  1646  
  1647  		if !strings.Contains(certChange.dstFile, "server.crt") {
  1648  			continue
  1649  		}
  1650  
  1651  		// Read the previous key file
  1652  		previousKeyBytes, err := ioutil.ReadFile(strings.Replace(certChange.dstFile, "server.crt", "server.key", -1))
  1653  		Expect(err).NotTo(HaveOccurred())
  1654  
  1655  		serverCertChanges = append(serverCertChanges, certChange)
  1656  		certChange.newCert = newCertBytes
  1657  		certChange.oldCert = previousCertBytes
  1658  		certChange.oldKey = previousKeyBytes
  1659  	}
  1660  	return serverCertChanges
  1661  }
  1662  
  1663  // assertBlockReception asserts that the given orderers have the expected
  1664  // newest block number for the specified channels
  1665  func assertBlockReception(expectedBlockNumPerChannel map[string]int, orderers []*nwo.Orderer, p *nwo.Peer, n *nwo.Network) {
  1666  	for channelName, blockNum := range expectedBlockNumPerChannel {
  1667  		for _, orderer := range orderers {
  1668  			waitForBlockReception(orderer, p, n, channelName, blockNum)
  1669  		}
  1670  	}
  1671  }
  1672  
  1673  func waitForBlockReception(o *nwo.Orderer, submitter *nwo.Peer, network *nwo.Network, channelName string, blockNum int) {
  1674  	c := commands.ChannelFetch{
  1675  		ChannelID:  channelName,
  1676  		Block:      "newest",
  1677  		OutputFile: "/dev/null",
  1678  		Orderer:    network.OrdererAddress(o, nwo.ListenPort),
  1679  	}
  1680  	Eventually(func() string {
  1681  		sess, err := network.OrdererAdminSession(o, submitter, c)
  1682  		Expect(err).NotTo(HaveOccurred())
  1683  		Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit())
  1684  		if sess.ExitCode() != 0 {
  1685  			return fmt.Sprintf("exit code is %d: %s", sess.ExitCode(), string(sess.Err.Contents()))
  1686  		}
  1687  		sessErr := string(sess.Err.Contents())
  1688  		expected := fmt.Sprintf("Received block: %d", blockNum)
  1689  		if strings.Contains(sessErr, expected) {
  1690  			return ""
  1691  		}
  1692  		return sessErr
  1693  	}, network.EventuallyTimeout, time.Second).Should(BeEmpty())
  1694  }
  1695  
  1696  func revokeReaderAccess(network *nwo.Network, channel string, orderer *nwo.Orderer, peer *nwo.Peer) {
  1697  	config := nwo.GetConfig(network, peer, orderer, channel)
  1698  	updatedConfig := proto.Clone(config).(*common.Config)
  1699  
  1700  	// set the policy
  1701  	adminPolicy := protoutil.MarshalOrPanic(&common.ImplicitMetaPolicy{
  1702  		SubPolicy: "Admins",
  1703  		Rule:      common.ImplicitMetaPolicy_MAJORITY,
  1704  	})
  1705  	updatedConfig.ChannelGroup.Groups["Orderer"].Policies["Readers"].Policy.Value = adminPolicy
  1706  	nwo.UpdateOrdererConfig(network, orderer, channel, config, updatedConfig, peer, orderer)
  1707  }
  1708  
  1709  // consenterAdder constructs configs that can be used by `UpdateOrdererConfig`
  1710  // to add a consenter.
  1711  func consenterAdder(n *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel string, consenter etcdraft.Consenter) (current, updated *common.Config) {
  1712  	config := nwo.GetConfig(n, peer, orderer, channel)
  1713  	updatedConfig := proto.Clone(config).(*common.Config)
  1714  
  1715  	consensusTypeConfigValue := updatedConfig.ChannelGroup.Groups["Orderer"].Values["ConsensusType"]
  1716  	consensusTypeValue := &protosorderer.ConsensusType{}
  1717  	err := proto.Unmarshal(consensusTypeConfigValue.Value, consensusTypeValue)
  1718  	Expect(err).NotTo(HaveOccurred())
  1719  
  1720  	metadata := &etcdraft.ConfigMetadata{}
  1721  	err = proto.Unmarshal(consensusTypeValue.Metadata, metadata)
  1722  	Expect(err).NotTo(HaveOccurred())
  1723  
  1724  	metadata.Consenters = append(metadata.Consenters, &consenter)
  1725  
  1726  	consensusTypeValue.Metadata, err = proto.Marshal(metadata)
  1727  	Expect(err).NotTo(HaveOccurred())
  1728  
  1729  	updatedConfig.ChannelGroup.Groups["Orderer"].Values["ConsensusType"] = &common.ConfigValue{
  1730  		ModPolicy: "Admins",
  1731  		Value:     protoutil.MarshalOrPanic(consensusTypeValue),
  1732  	}
  1733  
  1734  	return config, updatedConfig
  1735  }
  1736  
  1737  // consenterRemover constructs configs that can be used by
  1738  // `UpdateOrdererConfig` to remove a consenter.
  1739  func consenterRemover(n *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel string, certificate []byte) (current, updated *common.Config) {
  1740  	config := nwo.GetConfig(n, peer, orderer, channel)
  1741  	updatedConfig := proto.Clone(config).(*common.Config)
  1742  
  1743  	consensusTypeConfigValue := updatedConfig.ChannelGroup.Groups["Orderer"].Values["ConsensusType"]
  1744  	consensusTypeValue := &protosorderer.ConsensusType{}
  1745  	err := proto.Unmarshal(consensusTypeConfigValue.Value, consensusTypeValue)
  1746  	Expect(err).NotTo(HaveOccurred())
  1747  
  1748  	metadata := &etcdraft.ConfigMetadata{}
  1749  	err = proto.Unmarshal(consensusTypeValue.Metadata, metadata)
  1750  	Expect(err).NotTo(HaveOccurred())
  1751  
  1752  	var newConsenters []*etcdraft.Consenter
  1753  	for _, consenter := range metadata.Consenters {
  1754  		if bytes.Equal(consenter.ClientTlsCert, certificate) || bytes.Equal(consenter.ServerTlsCert, certificate) {
  1755  			continue
  1756  		}
  1757  		newConsenters = append(newConsenters, consenter)
  1758  	}
  1759  
  1760  	metadata.Consenters = newConsenters
  1761  	consensusTypeValue.Metadata, err = proto.Marshal(metadata)
  1762  	Expect(err).NotTo(HaveOccurred())
  1763  
  1764  	updatedConfig.ChannelGroup.Groups["Orderer"].Values["ConsensusType"] = &common.ConfigValue{
  1765  		ModPolicy: "Admins",
  1766  		Value:     protoutil.MarshalOrPanic(consensusTypeValue),
  1767  	}
  1768  
  1769  	return config, updatedConfig
  1770  }
  1771  
  1772  // addConsenter adds a new consenter to the given channel.
  1773  func addConsenter(n *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel string, consenter etcdraft.Consenter) {
  1774  	updateEtcdRaftMetadata(n, peer, orderer, channel, func(metadata *etcdraft.ConfigMetadata) {
  1775  		metadata.Consenters = append(metadata.Consenters, &consenter)
  1776  	})
  1777  }
  1778  
  1779  // removeConsenter removes a consenter with the given certificate in PEM format
  1780  // from the given channel.
  1781  func removeConsenter(n *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel string, certificate []byte) {
  1782  	updateEtcdRaftMetadata(n, peer, orderer, channel, func(metadata *etcdraft.ConfigMetadata) {
  1783  		var newConsenters []*etcdraft.Consenter
  1784  		for _, consenter := range metadata.Consenters {
  1785  			if bytes.Equal(consenter.ClientTlsCert, certificate) || bytes.Equal(consenter.ServerTlsCert, certificate) {
  1786  				continue
  1787  			}
  1788  			newConsenters = append(newConsenters, consenter)
  1789  		}
  1790  
  1791  		metadata.Consenters = newConsenters
  1792  	})
  1793  }
  1794  
  1795  // updateEtcdRaftMetadata executes a config update that updates the etcdraft
  1796  // metadata according to the given function f.
  1797  func updateEtcdRaftMetadata(network *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel string, f func(md *etcdraft.ConfigMetadata)) {
  1798  	nwo.UpdateConsensusMetadata(network, peer, orderer, channel, func(originalMetadata []byte) []byte {
  1799  		metadata := &etcdraft.ConfigMetadata{}
  1800  		err := proto.Unmarshal(originalMetadata, metadata)
  1801  		Expect(err).NotTo(HaveOccurred())
  1802  
  1803  		f(metadata)
  1804  
  1805  		newMetadata, err := proto.Marshal(metadata)
  1806  		Expect(err).NotTo(HaveOccurred())
  1807  		return newMetadata
  1808  	})
  1809  }
  1810  
  1811  func mutateConsensusMetadata(originalMetadata []byte, f func(md *etcdraft.ConfigMetadata)) []byte {
  1812  	metadata := &etcdraft.ConfigMetadata{}
  1813  	err := proto.Unmarshal(originalMetadata, metadata)
  1814  	Expect(err).NotTo(HaveOccurred())
  1815  
  1816  	f(metadata)
  1817  
  1818  	newMetadata, err := proto.Marshal(metadata)
  1819  	Expect(err).NotTo(HaveOccurred())
  1820  	return newMetadata
  1821  }
  1822  
  1823  func updateOrdererMSPAndConsensusMetadata(network *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel, orgID string, mutateMSP nwo.MSPMutator, f func(md *etcdraft.ConfigMetadata)) {
  1824  	config := nwo.GetConfig(network, peer, orderer, channel)
  1825  	updatedConfig := proto.Clone(config).(*common.Config)
  1826  
  1827  	// Unpack the MSP config
  1828  	rawMSPConfig := updatedConfig.ChannelGroup.Groups["Orderer"].Groups[orgID].Values["MSP"]
  1829  	mspConfig := &msp.MSPConfig{}
  1830  	err := proto.Unmarshal(rawMSPConfig.Value, mspConfig)
  1831  	Expect(err).NotTo(HaveOccurred())
  1832  
  1833  	fabricConfig := &msp.FabricMSPConfig{}
  1834  	err = proto.Unmarshal(mspConfig.Config, fabricConfig)
  1835  	Expect(err).NotTo(HaveOccurred())
  1836  
  1837  	// Mutate it as we are asked
  1838  	*fabricConfig = mutateMSP(*fabricConfig)
  1839  
  1840  	// Wrap it back into the config
  1841  	mspConfig.Config = protoutil.MarshalOrPanic(fabricConfig)
  1842  	rawMSPConfig.Value = protoutil.MarshalOrPanic(mspConfig)
  1843  
  1844  	consensusTypeConfigValue := updatedConfig.ChannelGroup.Groups["Orderer"].Values["ConsensusType"]
  1845  	consensusTypeValue := &protosorderer.ConsensusType{}
  1846  	err = proto.Unmarshal(consensusTypeConfigValue.Value, consensusTypeValue)
  1847  	Expect(err).NotTo(HaveOccurred())
  1848  
  1849  	consensusTypeValue.Metadata = mutateConsensusMetadata(consensusTypeValue.Metadata, f)
  1850  
  1851  	updatedConfig.ChannelGroup.Groups["Orderer"].Values["ConsensusType"] = &common.ConfigValue{
  1852  		ModPolicy: "Admins",
  1853  		Value:     protoutil.MarshalOrPanic(consensusTypeValue),
  1854  	}
  1855  
  1856  	nwo.UpdateOrdererConfig(network, orderer, channel, config, updatedConfig, peer, orderer)
  1857  }