github.com/hechain20/hechain@v0.0.0-20220316014945-b544036ba106/integration/kafka/migration_test.go (about)

     1  /*
     2  Copyright hechain All Rights Reserved.
     3  
     4  SPDX-License-Identifier: Apache-2.0
     5  */
     6  
     7  package kafka
     8  
     9  import (
    10  	"errors"
    11  	"fmt"
    12  	"io/ioutil"
    13  	"os"
    14  	"path/filepath"
    15  	"strings"
    16  	"syscall"
    17  	"time"
    18  
    19  	docker "github.com/fsouza/go-dockerclient"
    20  	"github.com/golang/protobuf/proto"
    21  	"github.com/hechain20/hechain/integration/nwo"
    22  	"github.com/hechain20/hechain/integration/nwo/commands"
    23  	"github.com/hechain20/hechain/integration/ordererclient"
    24  	"github.com/hechain20/hechain/protoutil"
    25  	"github.com/hyperledger/fabric-protos-go/common"
    26  	protosorderer "github.com/hyperledger/fabric-protos-go/orderer"
    27  	protosraft "github.com/hyperledger/fabric-protos-go/orderer/etcdraft"
    28  	. "github.com/onsi/ginkgo"
    29  	. "github.com/onsi/gomega"
    30  	"github.com/onsi/gomega/gbytes"
    31  	"github.com/onsi/gomega/gexec"
    32  	"github.com/tedsuo/ifrit"
    33  	"github.com/tedsuo/ifrit/ginkgomon"
    34  )
    35  
    36  var _ = Describe("Kafka2RaftMigration", func() {
    37  	var (
    38  		testDir string
    39  		client  *docker.Client
    40  		network *nwo.Network
    41  
    42  		process                ifrit.Process
    43  		brokerProc             ifrit.Process
    44  		o1Proc, o2Proc, o3Proc ifrit.Process
    45  
    46  		o1Runner, o2Runner, o3Runner *ginkgomon.Runner
    47  	)
    48  
    49  	BeforeEach(func() {
    50  		var err error
    51  		testDir, err = ioutil.TempDir("", "kafka2raft-migration")
    52  		Expect(err).NotTo(HaveOccurred())
    53  
    54  		client, err = docker.NewClientFromEnv()
    55  		Expect(err).NotTo(HaveOccurred())
    56  	})
    57  
    58  	AfterEach(func() {
    59  		if process != nil {
    60  			process.Signal(syscall.SIGTERM)
    61  			Eventually(process.Wait(), network.EventuallyTimeout).Should(Receive())
    62  		}
    63  
    64  		for _, oProc := range []ifrit.Process{o1Proc, o2Proc, o3Proc} {
    65  			if oProc != nil {
    66  				oProc.Signal(syscall.SIGTERM)
    67  				Eventually(oProc.Wait(), network.EventuallyTimeout).Should(Receive())
    68  			}
    69  		}
    70  
    71  		if brokerProc != nil {
    72  			brokerProc.Signal(syscall.SIGTERM)
    73  			Eventually(brokerProc.Wait(), network.EventuallyTimeout).Should(Receive())
    74  		}
    75  
    76  		if network != nil {
    77  			network.Cleanup()
    78  		}
    79  
    80  		_ = os.RemoveAll(testDir)
    81  	})
    82  
    83  	// These tests execute the migration config updates on Kafka based system, and verify that these config update
    84  	// have the desired effect. They focus on testing that filtering and the blocking of transitions into, in, and
    85  	// out-of maintenance mode are enforced. These tests use a single node.
    86  	Describe("Kafka to Raft migration kafka side", func() {
    87  		var (
    88  			orderer                                  *nwo.Orderer
    89  			peer                                     *nwo.Peer
    90  			syschannel, channel1, channel2, channel3 string
    91  			raftMetadata                             []byte
    92  		)
    93  
    94  		BeforeEach(func() {
    95  			network = nwo.New(kafka2RaftMultiChannel(), testDir, client, StartPort(), components)
    96  			network.GenerateConfigTree()
    97  			network.Bootstrap()
    98  
    99  			networkRunner := network.NetworkGroupRunner()
   100  			process = ifrit.Invoke(networkRunner)
   101  			Eventually(process.Ready(), network.EventuallyTimeout).Should(BeClosed())
   102  
   103  			orderer = network.Orderer("orderer")
   104  			peer = network.Peer("Org1", "peer0")
   105  
   106  			syschannel = network.SystemChannel.Name
   107  			channel1 = "testchannel1"
   108  			channel2 = "testchannel2"
   109  			channel3 = "testchannel3"
   110  
   111  			raftMetadata = prepareRaftMetadata(network)
   112  
   113  			By("Create & join first channel, deploy and invoke chaincode")
   114  			network.CreateChannel(channel1, orderer, peer)
   115  		})
   116  
   117  		// This test executes the "green path" migration config updates on Kafka based system with a system channel
   118  		// and a standard channel, and verifies that these config updates have the desired effect.
   119  		//
   120  		// The green path is entering maintenance mode, and then changing the consensus type.
   121  		// In maintenance mode we check that channel creation is blocked and that normal transactions are blocked.
   122  		//
   123  		// We also check that after entering maintenance mode, we can exit it without making any
   124  		// changes - the "abort path".
   125  		It("executes kafka2raft green path", func() {
   126  			//=== The abort path ======================================================================================
   127  			//=== Step 1: Config update on system channel, MAINTENANCE ===
   128  			By("1) Config update on system channel, State=MAINTENANCE, enter maintenance-mode")
   129  			config, updatedConfig := prepareTransition(network, peer, orderer, syschannel,
   130  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   131  				"kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   132  			nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   133  
   134  			By("1) Verify: system channel1 config changed")
   135  			sysStartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel)
   136  			Expect(sysStartBlockNum).ToNot(Equal(0))
   137  			config = nwo.GetConfig(network, peer, orderer, syschannel)
   138  			consensusTypeValue := extractOrdererConsensusType(config)
   139  			validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE)
   140  
   141  			By("1) Verify: new channels cannot be created")
   142  			exitCode := network.CreateChannelExitCode(channel2, orderer, peer)
   143  			Expect(exitCode).ToNot(Equal(0))
   144  
   145  			//=== Step 2: Config update on standard channel, MAINTENANCE ===
   146  			By("2) Config update on standard channel, State=MAINTENANCE, enter maintenance-mode")
   147  			config, updatedConfig = prepareTransition(network, peer, orderer, channel1,
   148  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   149  				"kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   150  			nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer)
   151  
   152  			By("2) Verify: standard channel config changed")
   153  			std1EntryBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1)
   154  			Expect(std1EntryBlockNum).ToNot(Equal(0))
   155  			config = nwo.GetConfig(network, peer, orderer, channel1)
   156  			consensusTypeValue = extractOrdererConsensusType(config)
   157  			validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE)
   158  
   159  			By("2) Verify: Normal TX's on standard channel are blocked")
   160  			assertTxFailed(network, orderer, channel1)
   161  
   162  			// In maintenance mode deliver requests are open to those entities that satisfy the /Channel/Orderer/Readers policy
   163  			By("2) Verify: delivery request from peer is blocked")
   164  			err := checkPeerDeliverRequest(orderer, peer, network, channel1)
   165  			Expect(err).To(MatchError(errors.New("FORBIDDEN")))
   166  
   167  			//=== Step 3: config update on system channel, State=NORMAL, abort ===
   168  			By("3) Config update on system channel, State=NORMAL, exit maintenance-mode - abort path")
   169  			config, updatedConfig = prepareTransition(network, peer, orderer, syschannel,
   170  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   171  				"kafka", nil, protosorderer.ConsensusType_STATE_NORMAL)
   172  			nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   173  
   174  			By("3) Verify: system channel config changed")
   175  			sysBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel)
   176  			Expect(sysBlockNum).To(Equal(sysStartBlockNum + 1))
   177  
   178  			By("3) Verify: create new channel, executing transaction")
   179  			network.CreateChannel(channel2, orderer, peer)
   180  
   181  			assertBlockCreation(network, orderer, peer, channel2, 1)
   182  
   183  			By("3) Verify: delivery request from peer is not blocked on new channel")
   184  			err = checkPeerDeliverRequest(orderer, peer, network, channel2)
   185  			Expect(err).NotTo(HaveOccurred())
   186  
   187  			//=== Step 4: config update on standard channel, State=NORMAL, abort ===
   188  			By("4) Config update on standard channel, State=NORMAL, exit maintenance-mode - abort path")
   189  			config, updatedConfig = prepareTransition(network, peer, orderer, channel1,
   190  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   191  				"kafka", nil, protosorderer.ConsensusType_STATE_NORMAL)
   192  			nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer)
   193  
   194  			By("4) Verify: standard channel config changed")
   195  			std1BlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1)
   196  			Expect(std1BlockNum).To(Equal(std1EntryBlockNum + 1))
   197  
   198  			By("4) Verify: standard channel delivery requests from peer unblocked")
   199  			err = checkPeerDeliverRequest(orderer, peer, network, channel1)
   200  			Expect(err).NotTo(HaveOccurred())
   201  
   202  			By("4) Verify: Normal TX's on standard channel are permitted again")
   203  			assertBlockCreation(network, orderer, nil, channel1, 3)
   204  
   205  			//=== The green path ======================================================================================
   206  			//=== Step 5: Config update on system channel, MAINTENANCE, again ===
   207  			By("5) Config update on system channel, State=MAINTENANCE, enter maintenance-mode again")
   208  			config, updatedConfig = prepareTransition(network, peer, orderer, syschannel,
   209  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   210  				"kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   211  			nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   212  
   213  			By("5) Verify: system channel config changed")
   214  			sysStartBlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel)
   215  			Expect(sysStartBlockNum).ToNot(Equal(0))
   216  			config = nwo.GetConfig(network, peer, orderer, syschannel)
   217  			consensusTypeValue = extractOrdererConsensusType(config)
   218  			validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE)
   219  
   220  			//=== Step 6: Config update on standard channel1, MAINTENANCE, again ===
   221  			By("6) Config update on standard channel1, State=MAINTENANCE, enter maintenance-mode again")
   222  			config, updatedConfig = prepareTransition(network, peer, orderer, channel1,
   223  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   224  				"kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   225  			nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer)
   226  
   227  			By("6) Verify: standard channel config changed")
   228  			std1EntryBlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1)
   229  			Expect(std1EntryBlockNum).ToNot(Equal(0))
   230  
   231  			config = nwo.GetConfig(network, peer, orderer, channel1)
   232  			consensusTypeValue = extractOrdererConsensusType(config)
   233  			validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE)
   234  
   235  			By("6) Verify: delivery request from peer is blocked")
   236  			err = checkPeerDeliverRequest(orderer, peer, network, channel1)
   237  			Expect(err).To(MatchError(errors.New("FORBIDDEN")))
   238  
   239  			By("6) Verify: Normal TX's on standard channel are blocked")
   240  			assertTxFailed(network, orderer, channel1)
   241  
   242  			//=== Step 7: Config update on standard channel2, MAINTENANCE ===
   243  			By("7) Config update on standard channel2, State=MAINTENANCE, enter maintenance-mode again")
   244  			config, updatedConfig = prepareTransition(network, peer, orderer, channel2,
   245  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   246  				"kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   247  			nwo.UpdateOrdererConfig(network, orderer, channel2, config, updatedConfig, peer, orderer)
   248  
   249  			By("7) Verify: standard channel config changed")
   250  			std2EntryBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel2)
   251  			Expect(std2EntryBlockNum).ToNot(Equal(0))
   252  
   253  			config = nwo.GetConfig(network, peer, orderer, channel2)
   254  			consensusTypeValue = extractOrdererConsensusType(config)
   255  			validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE)
   256  
   257  			By("7) Verify: delivery request from peer is blocked")
   258  			err = checkPeerDeliverRequest(orderer, peer, network, channel2)
   259  			Expect(err).To(MatchError(errors.New("FORBIDDEN")))
   260  
   261  			By("7) Verify: Normal TX's on standard channel are blocked")
   262  			assertTxFailed(network, orderer, channel2)
   263  
   264  			//=== Step 8: config update on system channel, State=MAINTENANCE, type=etcdraft ===
   265  			By("8) Config update on system channel, State=MAINTENANCE, type=etcdraft")
   266  			config, updatedConfig = prepareTransition(network, peer, orderer, syschannel,
   267  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   268  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE)
   269  			nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   270  
   271  			By("8) Verify: system channel config changed")
   272  			sysBlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel)
   273  			Expect(sysBlockNum).To(Equal(sysStartBlockNum + 1))
   274  
   275  			By("8) Verify: new channels cannot be created")
   276  			exitCode = network.CreateChannelExitCode(channel3, orderer, peer)
   277  			Expect(exitCode).ToNot(Equal(0))
   278  
   279  			//=== Step 9: config update on standard channel1, State=MAINTENANCE, type=etcdraft ===
   280  			By("9) Config update on standard channel1, State=MAINTENANCE, type=etcdraft")
   281  			config, updatedConfig = prepareTransition(network, peer, orderer, channel1,
   282  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   283  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE)
   284  			nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer)
   285  
   286  			By("9) Verify: standard channel config changed")
   287  			std1BlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1)
   288  			Expect(std1BlockNum).To(Equal(std1EntryBlockNum + 1))
   289  
   290  			By("9) Verify: delivery request from peer is blocked")
   291  			err = checkPeerDeliverRequest(orderer, peer, network, channel1)
   292  			Expect(err).To(MatchError(errors.New("FORBIDDEN")))
   293  
   294  			By("9) Verify: Normal TX's on standard channel are blocked")
   295  			assertTxFailed(network, orderer, channel1)
   296  
   297  			//=== Step 10: config update on standard channel2, State=MAINTENANCE, type=etcdraft ===
   298  			By("10) Config update on standard channel2, State=MAINTENANCE, type=etcdraft")
   299  			config, updatedConfig = prepareTransition(network, peer, orderer, channel2,
   300  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   301  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE)
   302  			nwo.UpdateOrdererConfig(network, orderer, channel2, config, updatedConfig, peer, orderer)
   303  
   304  			By("10) Verify: standard channel config changed")
   305  			std2BlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel2)
   306  			Expect(std2BlockNum).To(Equal(std2EntryBlockNum + 1))
   307  
   308  			By("10) Verify: delivery request from peer is blocked")
   309  			err = checkPeerDeliverRequest(orderer, peer, network, channel2)
   310  			Expect(err).To(MatchError(errors.New("FORBIDDEN")))
   311  
   312  			By("10) Verify: Normal TX's on standard channel are blocked")
   313  			assertTxFailed(network, orderer, channel2)
   314  		})
   315  
   316  		// This test executes the migration flow and checks that forbidden transitions are rejected.
   317  		// These transitions are enforced by the maintenance filter:
   318  		// - Entry to & exit from maintenance mode can only change ConsensusType.State.
   319  		// - In maintenance mode one can only change ConsensusType.Type & ConsensusType.Metadata.
   320  		// - ConsensusType.Type can only change from "kafka" to "etcdraft", and only in maintenance mode.
   321  		It("executes kafka2raft forbidden transitions", func() {
   322  			//=== Step 1: ===
   323  			By("1) Config update on system channel, changing both ConsensusType State & Type is forbidden")
   324  			assertTransitionFailed(network, peer, orderer, syschannel,
   325  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   326  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE)
   327  
   328  			//=== Step 2: ===
   329  			By("2) Config update on standard channel, changing both ConsensusType State & Type is forbidden")
   330  			assertTransitionFailed(network, peer, orderer, channel1,
   331  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   332  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE)
   333  
   334  			//=== Step 3: ===
   335  			By("3) Config update on system channel, changing both ConsensusType State & some other value is forbidden")
   336  			config, updatedConfig := prepareTransition(network, peer, orderer, syschannel,
   337  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   338  				"kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   339  			updateConfigWithBatchTimeout(updatedConfig)
   340  			updateOrdererConfigFailed(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   341  
   342  			//=== Step 4: ===
   343  			By("4) Config update on standard channel, both ConsensusType State & some other value is forbidden")
   344  			config, updatedConfig = prepareTransition(network, peer, orderer, channel1,
   345  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   346  				"kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   347  			updateConfigWithBatchTimeout(updatedConfig)
   348  			updateOrdererConfigFailed(network, orderer, channel1, config, updatedConfig, peer, orderer)
   349  
   350  			//=== Step 5: ===
   351  			By("5) Config update on system channel, State=MAINTENANCE, enter maintenance-mode")
   352  			config, updatedConfig = prepareTransition(network, peer, orderer, syschannel,
   353  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   354  				"kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   355  			nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   356  
   357  			By("5) Verify: system channel config changed")
   358  			sysStartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel)
   359  			Expect(sysStartBlockNum).ToNot(Equal(0))
   360  			config = nwo.GetConfig(network, peer, orderer, syschannel)
   361  			consensusTypeValue := extractOrdererConsensusType(config)
   362  			validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE)
   363  
   364  			//=== Step 6: ===
   365  			By("6) Config update on standard channel, State=MAINTENANCE, enter maintenance-mode")
   366  			config, updatedConfig = prepareTransition(network, peer, orderer, channel1,
   367  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   368  				"kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   369  			nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer)
   370  
   371  			By("6) Verify: standard channel config changed")
   372  			std1StartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1)
   373  			Expect(std1StartBlockNum).ToNot(Equal(0))
   374  			config = nwo.GetConfig(network, peer, orderer, channel1)
   375  			consensusTypeValue = extractOrdererConsensusType(config)
   376  			validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE)
   377  
   378  			//=== Step 7: ===
   379  			By("7) Config update on system channel, change ConsensusType.Type to unsupported type, forbidden")
   380  			assertTransitionFailed(network, peer, orderer, syschannel,
   381  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   382  				"melville", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   383  
   384  			//=== Step 8: ===
   385  			By("8) Config update on standard channel, change ConsensusType.Type to unsupported type, forbidden")
   386  			assertTransitionFailed(network, peer, orderer, channel1,
   387  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   388  				"hesse", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   389  
   390  			//=== Step 9: ===
   391  			By("9) Config update on system channel, change ConsensusType.Type and State, forbidden")
   392  			assertTransitionFailed(network, peer, orderer, syschannel,
   393  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   394  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL)
   395  
   396  			//=== Step 10: ===
   397  			By("10) Config update on standard channel, change ConsensusType.Type and State, forbidden")
   398  			assertTransitionFailed(network, peer, orderer, channel1,
   399  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   400  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL)
   401  
   402  			//=== Step 11: ===
   403  			By("11) Config update on system channel, changing both ConsensusType.Type and other value is permitted")
   404  			config, updatedConfig = prepareTransition(network, peer, orderer, syschannel,
   405  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   406  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE)
   407  			updateConfigWithBatchTimeout(updatedConfig)
   408  			nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   409  
   410  			By("11) Verify: system channel config changed")
   411  			sysBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel)
   412  			Expect(sysBlockNum).To(Equal(sysStartBlockNum + 1))
   413  			config = nwo.GetConfig(network, peer, orderer, syschannel)
   414  			consensusTypeValue = extractOrdererConsensusType(config)
   415  			validateConsensusTypeValue(consensusTypeValue, "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE)
   416  
   417  			//=== Step 12: ===
   418  			By("12) Config update on standard channel, changing both ConsensusType.Type and other value is permitted")
   419  			config, updatedConfig = prepareTransition(network, peer, orderer, channel1,
   420  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   421  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE)
   422  			updateConfigWithBatchTimeout(updatedConfig)
   423  			nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer)
   424  
   425  			By("12) Verify: standard channel config changed")
   426  			std1BlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1)
   427  			Expect(std1BlockNum).To(Equal(std1StartBlockNum + 1))
   428  			config = nwo.GetConfig(network, peer, orderer, channel1)
   429  			consensusTypeValue = extractOrdererConsensusType(config)
   430  			validateConsensusTypeValue(consensusTypeValue, "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE)
   431  
   432  			//=== Step 13: ===
   433  			By("13) Config update on system channel, changing value other than ConsensusType.Type is permitted")
   434  			config = nwo.GetConfig(network, peer, orderer, syschannel)
   435  			consensusTypeValue = extractOrdererConsensusType(config)
   436  			validateConsensusTypeValue(consensusTypeValue, "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE)
   437  			updatedConfig = proto.Clone(config).(*common.Config)
   438  			updateConfigWithBatchTimeout(updatedConfig)
   439  			nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   440  
   441  			By("13) Verify: system channel config changed")
   442  			sysBlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel)
   443  			Expect(sysBlockNum).To(Equal(sysStartBlockNum + 2))
   444  
   445  			//=== Step 14: ===
   446  			By("14) Config update on standard channel, changing value other than ConsensusType.Type is permitted")
   447  			config = nwo.GetConfig(network, peer, orderer, channel1)
   448  			consensusTypeValue = extractOrdererConsensusType(config)
   449  			validateConsensusTypeValue(consensusTypeValue, "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE)
   450  			updatedConfig = proto.Clone(config).(*common.Config)
   451  			updateConfigWithBatchTimeout(updatedConfig)
   452  			nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer)
   453  
   454  			By("14) Verify: standard channel config changed")
   455  			std1BlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1)
   456  			Expect(std1BlockNum).To(Equal(std1StartBlockNum + 2))
   457  
   458  			//=== Step 15: ===
   459  			By("15) Config update on system channel, changing both ConsensusType State & some other value is forbidden")
   460  			config, updatedConfig = prepareTransition(network, peer, orderer, syschannel,
   461  				"etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE,
   462  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL)
   463  			updateConfigWithBatchTimeout(updatedConfig)
   464  			updateOrdererConfigFailed(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   465  
   466  			//=== Step 16: ===
   467  			By("16) Config update on standard channel, both ConsensusType State & some other value is forbidden")
   468  			config, updatedConfig = prepareTransition(network, peer, orderer, channel1,
   469  				"etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE,
   470  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL)
   471  			updateConfigWithBatchTimeout(updatedConfig)
   472  			updateOrdererConfigFailed(network, orderer, channel1, config, updatedConfig, peer, orderer)
   473  		})
   474  	})
   475  
   476  	// These tests execute the migration config updates on Kafka based system, restart the orderer onto a Raft-based
   477  	// system, and verifies that the newly restarted orderer cluster performs as expected.
   478  	Describe("Kafka to Raft migration raft side", func() {
   479  		var (
   480  			o1, o2, o3                     *nwo.Orderer
   481  			peer                           *nwo.Peer
   482  			syschannel, channel1, channel2 string
   483  			raftMetadata                   []byte
   484  		)
   485  
   486  		BeforeEach(func() {
   487  			network = nwo.New(kafka2RaftMultiNode(), testDir, client, StartPort(), components)
   488  			network.GenerateConfigTree()
   489  			network.Bootstrap()
   490  
   491  			o1, o2, o3 = network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3")
   492  			peer = network.Peer("Org1", "peer0")
   493  
   494  			brokerGroup := network.BrokerGroupRunner()
   495  			brokerProc = ifrit.Invoke(brokerGroup)
   496  			Eventually(brokerProc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   497  
   498  			o1Runner = network.OrdererRunner(o1)
   499  			o2Runner = network.OrdererRunner(o2)
   500  			o3Runner = network.OrdererRunner(o3)
   501  
   502  			o1Proc = ifrit.Invoke(o1Runner)
   503  			o2Proc = ifrit.Invoke(o2Runner)
   504  			o3Proc = ifrit.Invoke(o3Runner)
   505  
   506  			Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   507  			Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   508  			Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   509  
   510  			raftMetadata = prepareRaftMetadata(network)
   511  
   512  			syschannel = network.SystemChannel.Name
   513  			channel1 = "testchannel1"
   514  			channel2 = "testchannel2"
   515  
   516  			By("Create & join first channel, deploy and invoke chaincode")
   517  			network.CreateChannel(channel1, o1, peer)
   518  		})
   519  
   520  		// This test executes the "green path" migration config updates on Kafka based system
   521  		// with a three orderers, a system channel and two standard channels.
   522  		// It then restarts the orderers onto a Raft-based system, and verifies that the
   523  		// newly restarted orderers perform as expected.
   524  		It("executes bootstrap to raft - multi node", func() {
   525  			//=== Step 1: Config update on system channel, MAINTENANCE ===
   526  			By("1) Config update on system channel, State=MAINTENANCE")
   527  			config, updatedConfig := prepareTransition(network, peer, o1, syschannel,
   528  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   529  				"kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   530  			nwo.UpdateOrdererConfig(network, o1, syschannel, config, updatedConfig, peer, o1)
   531  
   532  			By("1) Verify: system channel config changed")
   533  			sysStartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, o1, syschannel)
   534  			Expect(sysStartBlockNum).ToNot(Equal(0))
   535  
   536  			config = nwo.GetConfig(network, peer, o1, syschannel)
   537  			consensusTypeValue := extractOrdererConsensusType(config)
   538  			validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE)
   539  
   540  			//=== Step 2: Config update on standard channel, MAINTENANCE ===
   541  			By("2) Config update on standard channel, State=MAINTENANCE")
   542  			config, updatedConfig = prepareTransition(network, peer, o1, channel1,
   543  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   544  				"kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   545  			nwo.UpdateOrdererConfig(network, o1, channel1, config, updatedConfig, peer, o1)
   546  
   547  			By("2) Verify: standard channel config changed")
   548  			chan1StartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, o1, channel1)
   549  			Expect(chan1StartBlockNum).ToNot(Equal(0))
   550  
   551  			config = nwo.GetConfig(network, peer, o1, channel1)
   552  			consensusTypeValue = extractOrdererConsensusType(config)
   553  			validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE)
   554  
   555  			//=== Step 3: config update on system channel, State=MAINTENANCE, type=etcdraft ===
   556  			By("3) Config update on system channel, State=MAINTENANCE, type=etcdraft")
   557  			config, updatedConfig = prepareTransition(network, peer, o1, syschannel,
   558  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   559  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE)
   560  			nwo.UpdateOrdererConfig(network, o1, syschannel, config, updatedConfig, peer, o1)
   561  
   562  			By("3) Verify: system channel config changed")
   563  			sysBlockNum := nwo.CurrentConfigBlockNumber(network, peer, o1, syschannel)
   564  			Expect(sysBlockNum).To(Equal(sysStartBlockNum + 1))
   565  
   566  			//=== Step 4: config update on standard channel, State=MAINTENANCE, type=etcdraft ===
   567  			By("4) Config update on standard channel, State=MAINTENANCE, type=etcdraft")
   568  			config, updatedConfig = prepareTransition(network, peer, o1, channel1,
   569  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   570  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE)
   571  			nwo.UpdateOrdererConfig(network, o1, channel1, config, updatedConfig, peer, o1)
   572  
   573  			By("4) Verify: standard channel config changed")
   574  			chan1BlockNum := nwo.CurrentConfigBlockNumber(network, peer, o1, channel1)
   575  			Expect(chan1BlockNum).To(Equal(chan1StartBlockNum + 1))
   576  
   577  			//=== Step 5: kill ===
   578  			By("5) killing orderer1,2,3")
   579  			for _, oProc := range []ifrit.Process{o1Proc, o2Proc, o3Proc} {
   580  				if oProc != nil {
   581  					oProc.Signal(syscall.SIGKILL)
   582  					Eventually(oProc.Wait(), network.EventuallyTimeout).Should(Receive(MatchError("exit status 137")))
   583  				}
   584  			}
   585  
   586  			//=== Step 6: restart ===
   587  			By("6) restarting orderer1,2,3")
   588  			network.Consensus.Type = "etcdraft"
   589  			o1Runner = network.OrdererRunner(o1)
   590  			o2Runner = network.OrdererRunner(o2)
   591  			o3Runner = network.OrdererRunner(o3)
   592  
   593  			o1Proc = ifrit.Invoke(o1Runner)
   594  			o2Proc = ifrit.Invoke(o2Runner)
   595  			o3Proc = ifrit.Invoke(o3Runner)
   596  
   597  			Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   598  			Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   599  			Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   600  
   601  			assertBlockReception(
   602  				map[string]int{
   603  					syschannel: int(sysBlockNum),
   604  					channel1:   int(chan1BlockNum),
   605  				},
   606  				[]*nwo.Orderer{o1, o2, o3},
   607  				peer,
   608  				network,
   609  			)
   610  
   611  			Eventually(o1Runner.Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say("Raft leader changed: 0 -> "))
   612  			Eventually(o2Runner.Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say("Raft leader changed: 0 -> "))
   613  			Eventually(o3Runner.Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say("Raft leader changed: 0 -> "))
   614  
   615  			By("7) System channel still in maintenance, State=MAINTENANCE, cannot create new channels")
   616  			exitCode := network.CreateChannelExitCode(channel2, o1, peer)
   617  			Expect(exitCode).ToNot(Equal(0))
   618  
   619  			By("8) Standard channel still in maintenance, State=MAINTENANCE, normal TX's blocked, delivery to peers blocked")
   620  			assertTxFailed(network, o1, channel1)
   621  
   622  			err := checkPeerDeliverRequest(o1, peer, network, channel1)
   623  			Expect(err).To(MatchError(errors.New("FORBIDDEN")))
   624  
   625  			By("9) Release - executing config transaction on system channel with restarted orderer")
   626  			config, updatedConfig = prepareTransition(network, peer, o1, syschannel,
   627  				"etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE,
   628  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL)
   629  			nwo.UpdateOrdererConfig(network, o1, syschannel, config, updatedConfig, peer, o1)
   630  
   631  			By("9) Verify: system channel config changed")
   632  			sysBlockNum = nwo.CurrentConfigBlockNumber(network, peer, o1, syschannel)
   633  			Expect(sysBlockNum).To(Equal(sysStartBlockNum + 2))
   634  
   635  			By("10) Release - executing config transaction on standard channel with restarted orderer")
   636  			config, updatedConfig = prepareTransition(network, peer, o1, channel1,
   637  				"etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE,
   638  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL)
   639  			nwo.UpdateOrdererConfig(network, o1, channel1, config, updatedConfig, peer, o1)
   640  
   641  			By("10) Verify: standard channel config changed")
   642  			chan1BlockNum = nwo.CurrentConfigBlockNumber(network, peer, o1, channel1)
   643  			Expect(chan1BlockNum).To(Equal(chan1StartBlockNum + 2))
   644  
   645  			By("11) Executing transaction on standard channel with restarted orderer")
   646  			assertBlockCreation(network, o1, peer, channel1, chan1StartBlockNum+3)
   647  			assertBlockCreation(network, o1, nil, channel1, chan1StartBlockNum+4)
   648  
   649  			By("12) Create new channel, executing transaction with restarted orderer")
   650  			network.CreateChannel(channel2, o1, peer)
   651  
   652  			chan2StartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, o1, channel2)
   653  			Expect(chan2StartBlockNum).ToNot(Equal(0))
   654  
   655  			assertBlockCreation(network, o1, peer, channel2, chan2StartBlockNum+1)
   656  			assertBlockCreation(network, o1, nil, channel2, chan2StartBlockNum+2)
   657  
   658  			By("Extending the network configuration to add a new orderer")
   659  			o4 := &nwo.Orderer{
   660  				Name:         "orderer4",
   661  				Organization: "OrdererOrg",
   662  			}
   663  			ports := nwo.Ports{}
   664  			for _, portName := range nwo.OrdererPortNames() {
   665  				ports[portName] = network.ReservePort()
   666  			}
   667  			network.PortsByOrdererID[o4.ID()] = ports
   668  			network.Orderers = append(network.Orderers, o4)
   669  			network.GenerateOrdererConfig(o4)
   670  			extendNetwork(network)
   671  
   672  			fourthOrdererCertificatePath := filepath.Join(network.OrdererLocalTLSDir(o4), "server.crt")
   673  			fourthOrdererCertificate, err := ioutil.ReadFile(fourthOrdererCertificatePath)
   674  			Expect(err).NotTo(HaveOccurred())
   675  
   676  			By("Adding the fourth orderer to the system channel")
   677  			addConsenter(network, peer, o1, "systemchannel", protosraft.Consenter{
   678  				ServerTlsCert: fourthOrdererCertificate,
   679  				ClientTlsCert: fourthOrdererCertificate,
   680  				Host:          "127.0.0.1",
   681  				Port:          uint32(network.OrdererPort(o4, nwo.ClusterPort)),
   682  			})
   683  
   684  			By("Obtaining the last config block from an orderer")
   685  			// Get the last config block of the system channel
   686  			configBlock := nwo.GetConfigBlock(network, peer, o1, "systemchannel")
   687  			// Plant it in the file system of orderer4, the new node to be onboarded.
   688  			err = ioutil.WriteFile(filepath.Join(testDir, "systemchannel_block.pb"), protoutil.MarshalOrPanic(configBlock), 0o644)
   689  			Expect(err).NotTo(HaveOccurred())
   690  
   691  			By("Launching the fourth orderer")
   692  			o4Runner := network.OrdererRunner(o4)
   693  			o4Process := ifrit.Invoke(o4Runner)
   694  
   695  			defer func() {
   696  				o4Process.Signal(syscall.SIGTERM)
   697  				Eventually(o4Process.Wait(), network.EventuallyTimeout).Should(Receive())
   698  			}()
   699  
   700  			Eventually(o4Process.Ready(), network.EventuallyTimeout).Should(BeClosed())
   701  
   702  			By("Waiting for the orderer to figure out it was migrated")
   703  			Eventually(o4Runner.Err(), time.Minute, time.Second).Should(gbytes.Say("This node was migrated from Kafka to Raft, skipping activation of Kafka chain"))
   704  
   705  			By("Adding the fourth orderer to the application channel")
   706  			addConsenter(network, peer, o1, channel1, protosraft.Consenter{
   707  				ServerTlsCert: fourthOrdererCertificate,
   708  				ClientTlsCert: fourthOrdererCertificate,
   709  				Host:          "127.0.0.1",
   710  				Port:          uint32(network.OrdererPort(o4, nwo.ClusterPort)),
   711  			})
   712  
   713  			chan1BlockNum = nwo.CurrentConfigBlockNumber(network, peer, o1, channel1)
   714  
   715  			By("Ensuring the added orderer has synced the application channel")
   716  			assertBlockReception(
   717  				map[string]int{
   718  					channel1: int(chan1BlockNum),
   719  				},
   720  				[]*nwo.Orderer{o4},
   721  				peer,
   722  				network,
   723  			)
   724  		})
   725  	})
   726  
   727  	// These tests execute the migration config updates on a solo based system, restart the orderer onto a Raft-based
   728  	// system, and verifies that the newly restarted orderer (single node) cluster performs as expected.
   729  	Describe("Solo to Raft migration", func() {
   730  		var (
   731  			orderer                        *nwo.Orderer
   732  			peer                           *nwo.Peer
   733  			syschannel, channel1, channel2 string
   734  			raftMetadata                   []byte
   735  		)
   736  
   737  		BeforeEach(func() {
   738  			network = nwo.New(solo2RaftMultiChannel(), testDir, client, StartPort(), components)
   739  			network.GenerateConfigTree()
   740  			network.Bootstrap()
   741  
   742  			orderer = network.Orderer("orderer")
   743  			peer = network.Peer("Org1", "peer0")
   744  
   745  			brokerGroup := network.BrokerGroupRunner()
   746  			brokerProc = ifrit.Invoke(brokerGroup)
   747  			Eventually(brokerProc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   748  
   749  			o1Runner = network.OrdererRunner(orderer)
   750  
   751  			o1Proc = ifrit.Invoke(o1Runner)
   752  			Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   753  
   754  			raftMetadata = prepareRaftMetadata(network)
   755  
   756  			syschannel = network.SystemChannel.Name
   757  			channel1 = "testchannel1"
   758  			channel2 = "testchannel2"
   759  
   760  			By("Create & join first channel, deploy and invoke chaincode")
   761  			network.CreateChannel(channel1, orderer, peer)
   762  		})
   763  
   764  		It("executes bootstrap to raft - single node", func() {
   765  			//=== Step 1: Config update on system channel, MAINTENANCE ===
   766  			By("1) Config update on system channel, State=MAINTENANCE")
   767  			config, updatedConfig := prepareTransition(network, peer, orderer, syschannel,
   768  				"solo", protosorderer.ConsensusType_STATE_NORMAL,
   769  				"solo", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   770  			nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   771  
   772  			By("1) Verify: system channel config changed")
   773  			sysStartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel)
   774  			Expect(sysStartBlockNum).ToNot(Equal(0))
   775  
   776  			config = nwo.GetConfig(network, peer, orderer, syschannel)
   777  			consensusTypeValue := extractOrdererConsensusType(config)
   778  			validateConsensusTypeValue(consensusTypeValue, "solo", protosorderer.ConsensusType_STATE_MAINTENANCE)
   779  
   780  			//=== Step 2: Config update on standard channel, MAINTENANCE ===
   781  			By("2) Config update on standard channel, State=MAINTENANCE")
   782  			config, updatedConfig = prepareTransition(network, peer, orderer, channel1,
   783  				"solo", protosorderer.ConsensusType_STATE_NORMAL,
   784  				"solo", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   785  			nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer)
   786  
   787  			By("2) Verify: standard channel config changed")
   788  			chan1StartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1)
   789  			Expect(chan1StartBlockNum).ToNot(Equal(0))
   790  
   791  			config = nwo.GetConfig(network, peer, orderer, channel1)
   792  			consensusTypeValue = extractOrdererConsensusType(config)
   793  			validateConsensusTypeValue(consensusTypeValue, "solo", protosorderer.ConsensusType_STATE_MAINTENANCE)
   794  
   795  			//=== Step 3: config update on system channel, State=MAINTENANCE, type=etcdraft ===
   796  			By("3) Config update on system channel, State=MAINTENANCE, type=etcdraft")
   797  			config, updatedConfig = prepareTransition(network, peer, orderer, syschannel,
   798  				"solo", protosorderer.ConsensusType_STATE_MAINTENANCE,
   799  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE)
   800  			nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   801  
   802  			By("3) Verify: system channel config changed")
   803  			sysBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel)
   804  			Expect(sysBlockNum).To(Equal(sysStartBlockNum + 1))
   805  
   806  			//=== Step 4: config update on standard channel, State=MAINTENANCE, type=etcdraft ===
   807  			By("4) Config update on standard channel, State=MAINTENANCE, type=etcdraft")
   808  			config, updatedConfig = prepareTransition(network, peer, orderer, channel1,
   809  				"solo", protosorderer.ConsensusType_STATE_MAINTENANCE,
   810  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE)
   811  			nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer)
   812  
   813  			By("4) Verify: standard channel config changed")
   814  			chan1BlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1)
   815  			Expect(chan1BlockNum).To(Equal(chan1StartBlockNum + 1))
   816  
   817  			//=== Step 5: kill ===
   818  			By("5) killing orderer1")
   819  			o1Proc.Signal(syscall.SIGKILL)
   820  			Eventually(o1Proc.Wait(), network.EventuallyTimeout).Should(Receive(MatchError("exit status 137")))
   821  
   822  			//=== Step 6: restart ===
   823  			By("6) restarting orderer1")
   824  			network.Consensus.Type = "etcdraft"
   825  
   826  			o1Runner = network.OrdererRunner(orderer)
   827  			o1Proc = ifrit.Invoke(o1Runner)
   828  
   829  			Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   830  
   831  			assertBlockReception(
   832  				map[string]int{
   833  					syschannel: int(sysBlockNum),
   834  					channel1:   int(chan1BlockNum),
   835  				},
   836  				[]*nwo.Orderer{orderer},
   837  				peer,
   838  				network,
   839  			)
   840  
   841  			Eventually(o1Runner.Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say("Raft leader changed: 0 -> "))
   842  			Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   843  
   844  			By("7) System channel still in maintenance, State=MAINTENANCE, cannot create new channels")
   845  			exitCode := network.CreateChannelExitCode(channel2, orderer, peer)
   846  			Expect(exitCode).ToNot(Equal(0))
   847  
   848  			By("8) Standard channel still in maintenance, State=MAINTENANCE, normal TX's blocked, delivery to peers blocked")
   849  			assertTxFailed(network, orderer, channel1)
   850  
   851  			err := checkPeerDeliverRequest(orderer, peer, network, channel1)
   852  			Expect(err).To(MatchError(errors.New("FORBIDDEN")))
   853  
   854  			By("9) Release - executing config transaction on system channel with restarted orderer")
   855  			config, updatedConfig = prepareTransition(network, peer, orderer, syschannel,
   856  				"etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE,
   857  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL)
   858  			nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   859  
   860  			By("9) Verify: system channel config changed")
   861  			sysBlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel)
   862  			Expect(sysBlockNum).To(Equal(sysStartBlockNum + 2))
   863  
   864  			By("10) Release - executing config transaction on standard channel with restarted orderer")
   865  			config, updatedConfig = prepareTransition(network, peer, orderer, channel1,
   866  				"etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE,
   867  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL)
   868  			nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer)
   869  
   870  			By("10) Verify: standard channel config changed")
   871  			chan1BlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1)
   872  			Expect(chan1BlockNum).To(Equal(chan1StartBlockNum + 2))
   873  
   874  			By("11) Executing transaction on standard channel with restarted orderer")
   875  			assertBlockCreation(network, orderer, peer, channel1, chan1StartBlockNum+3)
   876  			assertBlockCreation(network, orderer, nil, channel1, chan1StartBlockNum+4)
   877  
   878  			By("12) Create new channel, executing transaction with restarted orderer")
   879  			network.CreateChannel(channel2, orderer, peer)
   880  
   881  			assertBlockCreation(network, orderer, peer, channel2, 1)
   882  			assertBlockCreation(network, orderer, nil, channel2, 2)
   883  
   884  			By("13) Extending the network configuration to add a new orderer")
   885  			// Add another orderer
   886  			orderer2 := &nwo.Orderer{
   887  				Name:         "orderer2",
   888  				Organization: "OrdererOrg",
   889  			}
   890  			ports := nwo.Ports{}
   891  			for _, portName := range nwo.OrdererPortNames() {
   892  				ports[portName] = network.ReservePort()
   893  			}
   894  			network.PortsByOrdererID[orderer2.ID()] = ports
   895  			network.Orderers = append(network.Orderers, orderer2)
   896  			network.GenerateOrdererConfig(orderer2)
   897  			extendNetwork(network)
   898  
   899  			secondOrdererCertificatePath := filepath.Join(network.OrdererLocalTLSDir(orderer2), "server.crt")
   900  			secondOrdererCertificate, err := ioutil.ReadFile(secondOrdererCertificatePath)
   901  			Expect(err).NotTo(HaveOccurred())
   902  
   903  			By("14) Adding the second orderer to system channel")
   904  			addConsenter(network, peer, orderer, syschannel, protosraft.Consenter{
   905  				ServerTlsCert: secondOrdererCertificate,
   906  				ClientTlsCert: secondOrdererCertificate,
   907  				Host:          "127.0.0.1",
   908  				Port:          uint32(network.OrdererPort(orderer2, nwo.ClusterPort)),
   909  			})
   910  
   911  			By("15) Obtaining the last config block from the orderer")
   912  			configBlock := nwo.GetConfigBlock(network, peer, orderer, syschannel)
   913  			err = ioutil.WriteFile(filepath.Join(testDir, "systemchannel_block.pb"), protoutil.MarshalOrPanic(configBlock), 0o644)
   914  			Expect(err).NotTo(HaveOccurred())
   915  
   916  			By("16) Waiting for the existing orderer to relinquish its leadership")
   917  			Eventually(o1Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("1 stepped down to follower since quorum is not active"))
   918  			Eventually(o1Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("No leader is present, cluster size is 2"))
   919  
   920  			By("17) Launching the second orderer")
   921  			o2Runner = network.OrdererRunner(orderer2)
   922  			o2Proc = ifrit.Invoke(o2Runner)
   923  			Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   924  			Eventually(o2Runner.Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say("Raft leader changed: 0 -> "))
   925  
   926  			By("18) Adding orderer2 to channel2")
   927  			addConsenter(network, peer, orderer, channel2, protosraft.Consenter{
   928  				ServerTlsCert: secondOrdererCertificate,
   929  				ClientTlsCert: secondOrdererCertificate,
   930  				Host:          "127.0.0.1",
   931  				Port:          uint32(network.OrdererPort(orderer2, nwo.ClusterPort)),
   932  			})
   933  
   934  			assertBlockReception(map[string]int{
   935  				syschannel: int(sysBlockNum + 2),
   936  				channel2:   int(nwo.CurrentConfigBlockNumber(network, peer, orderer, channel2)),
   937  			}, []*nwo.Orderer{orderer2}, peer, network)
   938  
   939  			By("19) Executing transaction against second orderer on channel2")
   940  			assertBlockCreation(network, orderer2, nil, channel2, 3)
   941  		})
   942  	})
   943  })
   944  
   945  func validateConsensusTypeValue(value *protosorderer.ConsensusType, cType string, state protosorderer.ConsensusType_State) {
   946  	Expect(value.Type).To(Equal(cType))
   947  	Expect(value.State).To(Equal(state))
   948  }
   949  
   950  func extractOrdererConsensusType(config *common.Config) *protosorderer.ConsensusType {
   951  	var consensusTypeValue protosorderer.ConsensusType
   952  	consensusTypeConfigValue := config.ChannelGroup.Groups["Orderer"].Values["ConsensusType"]
   953  	err := proto.Unmarshal(consensusTypeConfigValue.Value, &consensusTypeValue)
   954  	Expect(err).NotTo(HaveOccurred())
   955  	return &consensusTypeValue
   956  }
   957  
   958  func updateConfigWithConsensusType(
   959  	consensusType string,
   960  	consensusMetadata []byte,
   961  	migState protosorderer.ConsensusType_State,
   962  	updatedConfig *common.Config,
   963  	consensusTypeValue *protosorderer.ConsensusType,
   964  ) {
   965  	consensusTypeValue.Type = consensusType
   966  	consensusTypeValue.Metadata = consensusMetadata
   967  	consensusTypeValue.State = migState
   968  	updatedConfig.ChannelGroup.Groups["Orderer"].Values["ConsensusType"] = &common.ConfigValue{
   969  		ModPolicy: "Admins",
   970  		Value:     protoutil.MarshalOrPanic(consensusTypeValue),
   971  	}
   972  }
   973  
   974  func updateConfigWithBatchTimeout(updatedConfig *common.Config) {
   975  	batchTimeoutConfigValue := updatedConfig.ChannelGroup.Groups["Orderer"].Values["BatchTimeout"]
   976  	batchTimeoutValue := new(protosorderer.BatchTimeout)
   977  	err := proto.Unmarshal(batchTimeoutConfigValue.Value, batchTimeoutValue)
   978  	Expect(err).NotTo(HaveOccurred())
   979  	toDur, err := time.ParseDuration(batchTimeoutValue.Timeout)
   980  	Expect(err).NotTo(HaveOccurred())
   981  	toDur = toDur + time.Duration(100000000)
   982  	batchTimeoutValue.Timeout = toDur.String()
   983  	By(fmt.Sprintf("Increasing BatchTimeout to %s", batchTimeoutValue.Timeout))
   984  	updatedConfig.ChannelGroup.Groups["Orderer"].Values["BatchTimeout"] = &common.ConfigValue{
   985  		ModPolicy: "Admins",
   986  		Value:     protoutil.MarshalOrPanic(batchTimeoutValue),
   987  	}
   988  }
   989  
   990  func kafka2RaftMultiChannel() *nwo.Config {
   991  	config := nwo.BasicKafka()
   992  	config.Channels = []*nwo.Channel{
   993  		{Name: "testchannel1", Profile: "TwoOrgsChannel"},
   994  		{Name: "testchannel2", Profile: "TwoOrgsChannel"},
   995  		{Name: "testchannel3", Profile: "TwoOrgsChannel"},
   996  	}
   997  
   998  	for _, peer := range config.Peers {
   999  		peer.Channels = []*nwo.PeerChannel{
  1000  			{Name: "testchannel1", Anchor: true},
  1001  			{Name: "testchannel2", Anchor: true},
  1002  			{Name: "testchannel3", Anchor: true},
  1003  		}
  1004  	}
  1005  	return config
  1006  }
  1007  
  1008  func solo2RaftMultiChannel() *nwo.Config {
  1009  	config := nwo.BasicSolo()
  1010  	config.Channels = []*nwo.Channel{
  1011  		{Name: "testchannel1", Profile: "TwoOrgsChannel"},
  1012  		{Name: "testchannel2", Profile: "TwoOrgsChannel"},
  1013  	}
  1014  
  1015  	for _, peer := range config.Peers {
  1016  		peer.Channels = []*nwo.PeerChannel{
  1017  			{Name: "testchannel1", Anchor: true},
  1018  			{Name: "testchannel2", Anchor: true},
  1019  		}
  1020  	}
  1021  	return config
  1022  }
  1023  
  1024  func kafka2RaftMultiNode() *nwo.Config {
  1025  	config := nwo.BasicKafka()
  1026  	config.Orderers = []*nwo.Orderer{
  1027  		{Name: "orderer1", Organization: "OrdererOrg"},
  1028  		{Name: "orderer2", Organization: "OrdererOrg"},
  1029  		{Name: "orderer3", Organization: "OrdererOrg"},
  1030  	}
  1031  
  1032  	config.Profiles = []*nwo.Profile{{
  1033  		Name:     "TwoOrgsOrdererGenesis",
  1034  		Orderers: []string{"orderer1", "orderer2", "orderer3"},
  1035  	}, {
  1036  		Name:          "TwoOrgsChannel",
  1037  		Consortium:    "SampleConsortium",
  1038  		Organizations: []string{"Org1", "Org2"},
  1039  	}}
  1040  
  1041  	config.Channels = []*nwo.Channel{
  1042  		{Name: "testchannel1", Profile: "TwoOrgsChannel"},
  1043  		{Name: "testchannel2", Profile: "TwoOrgsChannel"},
  1044  		{Name: "testchannel3", Profile: "TwoOrgsChannel"},
  1045  	}
  1046  
  1047  	for _, peer := range config.Peers {
  1048  		peer.Channels = []*nwo.PeerChannel{
  1049  			{Name: "testchannel1", Anchor: true},
  1050  			{Name: "testchannel2", Anchor: true},
  1051  			{Name: "testchannel3", Anchor: true},
  1052  		}
  1053  	}
  1054  	return config
  1055  }
  1056  
  1057  func prepareRaftMetadata(network *nwo.Network) []byte {
  1058  	var consenters []*protosraft.Consenter
  1059  	for _, o := range network.Orderers {
  1060  		fullTlsPath := network.OrdererLocalTLSDir(o)
  1061  		certBytes, err := ioutil.ReadFile(filepath.Join(fullTlsPath, "server.crt"))
  1062  		Expect(err).NotTo(HaveOccurred())
  1063  		port := network.OrdererPort(o, nwo.ClusterPort)
  1064  
  1065  		consenter := &protosraft.Consenter{
  1066  			ClientTlsCert: certBytes,
  1067  			ServerTlsCert: certBytes,
  1068  			Host:          "127.0.0.1",
  1069  			Port:          uint32(port),
  1070  		}
  1071  		consenters = append(consenters, consenter)
  1072  	}
  1073  
  1074  	raftMetadata := &protosraft.ConfigMetadata{
  1075  		Consenters: consenters,
  1076  		Options: &protosraft.Options{
  1077  			TickInterval:         "500ms",
  1078  			ElectionTick:         10,
  1079  			HeartbeatTick:        1,
  1080  			MaxInflightBlocks:    5,
  1081  			SnapshotIntervalSize: 10 * 1024 * 1024,
  1082  		},
  1083  	}
  1084  
  1085  	raftMetadataBytes := protoutil.MarshalOrPanic(raftMetadata)
  1086  
  1087  	return raftMetadataBytes
  1088  }
  1089  
  1090  func checkPeerDeliverRequest(o *nwo.Orderer, submitter *nwo.Peer, network *nwo.Network, channelName string) error {
  1091  	c := commands.ChannelFetch{
  1092  		ChannelID:  channelName,
  1093  		Block:      "newest",
  1094  		OutputFile: "/dev/null",
  1095  		Orderer:    network.OrdererAddress(o, nwo.ListenPort),
  1096  	}
  1097  
  1098  	sess, err := network.PeerUserSession(submitter, "User1", c)
  1099  	Expect(err).NotTo(HaveOccurred())
  1100  	Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit())
  1101  	sessErr := string(sess.Err.Contents())
  1102  	sessExitCode := sess.ExitCode()
  1103  	if sessExitCode != 0 && strings.Contains(sessErr, "FORBIDDEN") {
  1104  		return errors.New("FORBIDDEN")
  1105  	}
  1106  	if sessExitCode == 0 && strings.Contains(sessErr, "Received block: ") {
  1107  		return nil
  1108  	}
  1109  
  1110  	return fmt.Errorf("Unexpected result: ExitCode=%d, Err=%s", sessExitCode, sessErr)
  1111  }
  1112  
  1113  func updateOrdererConfigFailed(n *nwo.Network, orderer *nwo.Orderer, channel string, current, updated *common.Config, peer *nwo.Peer, additionalSigners ...*nwo.Orderer) {
  1114  	sess := nwo.UpdateOrdererConfigSession(n, orderer, channel, current, updated, peer, additionalSigners...)
  1115  	Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(1))
  1116  	Expect(sess.Err).NotTo(gbytes.Say("Successfully submitted channel update"))
  1117  }
  1118  
  1119  func prepareTransition(
  1120  	network *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel string, // Auxiliary
  1121  	fromConsensusType string, fromMigState protosorderer.ConsensusType_State, // From
  1122  	toConsensusType string, toConsensusMetadata []byte, toMigState protosorderer.ConsensusType_State, // To
  1123  ) (current, updated *common.Config) {
  1124  	current = nwo.GetConfig(network, peer, orderer, channel)
  1125  	updated = proto.Clone(current).(*common.Config)
  1126  	consensusTypeValue := extractOrdererConsensusType(current)
  1127  	validateConsensusTypeValue(consensusTypeValue, fromConsensusType, fromMigState)
  1128  	updateConfigWithConsensusType(toConsensusType, toConsensusMetadata, toMigState, updated, consensusTypeValue)
  1129  	return current, updated
  1130  }
  1131  
  1132  func assertTransitionFailed(
  1133  	network *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel string, // Auxiliary
  1134  	fromConsensusType string, fromMigState protosorderer.ConsensusType_State, // From
  1135  	toConsensusType string, toConsensusMetadata []byte, toMigState protosorderer.ConsensusType_State, // To
  1136  ) {
  1137  	current, updated := prepareTransition(
  1138  		network, peer, orderer, channel,
  1139  		fromConsensusType, fromMigState,
  1140  		toConsensusType, toConsensusMetadata, toMigState)
  1141  	updateOrdererConfigFailed(network, orderer, channel, current, updated, peer, orderer)
  1142  }
  1143  
  1144  func assertBlockCreation(network *nwo.Network, orderer *nwo.Orderer, peer *nwo.Peer, channelID string, blkNum uint64) {
  1145  	var signer *nwo.SigningIdentity
  1146  	signer = network.OrdererUserSigner(orderer, "Admin")
  1147  	if peer != nil {
  1148  		signer = network.PeerUserSigner(peer, "Admin")
  1149  	}
  1150  	env := createBroadcastEnvelope(network, signer, channelID, []byte("hola"))
  1151  	resp, err := ordererclient.Broadcast(network, orderer, env)
  1152  	Expect(err).NotTo(HaveOccurred())
  1153  	Expect(resp.Status).To(Equal(common.Status_SUCCESS))
  1154  
  1155  	denv := createDeliverEnvelope(network, signer, blkNum, channelID)
  1156  	blk, err := ordererclient.Deliver(network, orderer, denv)
  1157  	Expect(err).NotTo(HaveOccurred())
  1158  	Expect(blk).ToNot(BeNil())
  1159  }
  1160  
  1161  func assertTxFailed(network *nwo.Network, orderer *nwo.Orderer, channelID string) {
  1162  	signer := network.OrdererUserSigner(orderer, "Admin")
  1163  	env := createBroadcastEnvelope(network, signer, channelID, []byte("hola"))
  1164  	resp, err := ordererclient.Broadcast(network, orderer, env)
  1165  	Expect(err).NotTo(HaveOccurred())
  1166  	Expect(resp.Status).To(Equal(common.Status_SERVICE_UNAVAILABLE))
  1167  	Expect(resp.Info).To(Equal("normal transactions are rejected: maintenance mode"))
  1168  }
  1169  
  1170  // assertBlockReception asserts that the given orderers have the expected
  1171  // newest block number for the specified channels
  1172  func assertBlockReception(expectedBlockNumPerChannel map[string]int, orderers []*nwo.Orderer, p *nwo.Peer, n *nwo.Network) {
  1173  	for channelName, blockNum := range expectedBlockNumPerChannel {
  1174  		for _, orderer := range orderers {
  1175  			waitForBlockReception(orderer, p, n, channelName, blockNum)
  1176  		}
  1177  	}
  1178  }
  1179  
  1180  func waitForBlockReception(o *nwo.Orderer, submitter *nwo.Peer, network *nwo.Network, channelName string, blockNum int) {
  1181  	c := commands.ChannelFetch{
  1182  		ChannelID:  channelName,
  1183  		Block:      "newest",
  1184  		OutputFile: "/dev/null",
  1185  		Orderer:    network.OrdererAddress(o, nwo.ListenPort),
  1186  	}
  1187  	Eventually(func() string {
  1188  		sess, err := network.OrdererAdminSession(o, submitter, c)
  1189  		Expect(err).NotTo(HaveOccurred())
  1190  		Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit())
  1191  		if sess.ExitCode() != 0 {
  1192  			return fmt.Sprintf("exit code is %d: %s", sess.ExitCode(), string(sess.Err.Contents()))
  1193  		}
  1194  		sessErr := string(sess.Err.Contents())
  1195  		expected := fmt.Sprintf("Received block: %d", blockNum)
  1196  		if strings.Contains(sessErr, expected) {
  1197  			return ""
  1198  		}
  1199  		return sessErr
  1200  	}, network.EventuallyTimeout, time.Second).Should(BeEmpty())
  1201  }
  1202  
  1203  func createBroadcastEnvelope(n *nwo.Network, signer *nwo.SigningIdentity, channel string, data []byte) *common.Envelope {
  1204  	env, err := protoutil.CreateSignedEnvelope(
  1205  		common.HeaderType_MESSAGE,
  1206  		channel,
  1207  		signer,
  1208  		&common.Envelope{Payload: data},
  1209  		0,
  1210  		0,
  1211  	)
  1212  	Expect(err).NotTo(HaveOccurred())
  1213  
  1214  	return env
  1215  }
  1216  
  1217  // CreateDeliverEnvelope creates a deliver env to seek for specified block.
  1218  func createDeliverEnvelope(n *nwo.Network, signer *nwo.SigningIdentity, blkNum uint64, channel string) *common.Envelope {
  1219  	specified := &protosorderer.SeekPosition{
  1220  		Type: &protosorderer.SeekPosition_Specified{
  1221  			Specified: &protosorderer.SeekSpecified{Number: blkNum},
  1222  		},
  1223  	}
  1224  	env, err := protoutil.CreateSignedEnvelope(
  1225  		common.HeaderType_DELIVER_SEEK_INFO,
  1226  		channel,
  1227  		signer,
  1228  		&protosorderer.SeekInfo{
  1229  			Start:    specified,
  1230  			Stop:     specified,
  1231  			Behavior: protosorderer.SeekInfo_BLOCK_UNTIL_READY,
  1232  		},
  1233  		0,
  1234  		0,
  1235  	)
  1236  	Expect(err).NotTo(HaveOccurred())
  1237  
  1238  	return env
  1239  }
  1240  
  1241  var extendedCryptoConfig = `---
  1242  OrdererOrgs:
  1243  - Name: OrdererOrg
  1244    Domain: example.com
  1245    EnableNodeOUs: false
  1246    CA:
  1247      Hostname: ca
  1248    Specs:
  1249    - Hostname: orderer1
  1250      SANS:
  1251      - localhost
  1252      - 127.0.0.1
  1253      - ::1
  1254    - Hostname: orderer1new
  1255      SANS:
  1256      - localhost
  1257      - 127.0.0.1
  1258      - ::1
  1259    - Hostname: orderer2
  1260      SANS:
  1261      - localhost
  1262      - 127.0.0.1
  1263      - ::1
  1264    - Hostname: orderer2new
  1265      SANS:
  1266      - localhost
  1267      - 127.0.0.1
  1268      - ::1
  1269    - Hostname: orderer3
  1270      SANS:
  1271      - localhost
  1272      - 127.0.0.1
  1273      - ::1
  1274    - Hostname: orderer3new
  1275      SANS:
  1276      - localhost
  1277      - 127.0.0.1
  1278      - ::1
  1279    - Hostname: orderer4
  1280      SANS:
  1281      - localhost
  1282      - 127.0.0.1
  1283      - ::1
  1284    - Hostname: orderer5
  1285      SANS:
  1286      - localhost
  1287      - 127.0.0.1
  1288      - ::1
  1289    - Hostname: orderer6
  1290      SANS:
  1291      - localhost
  1292      - 127.0.0.1
  1293      - ::1
  1294    - Hostname: orderer7
  1295      SANS:
  1296      - localhost
  1297      - 127.0.0.1
  1298      - ::1
  1299  `
  1300  
  1301  // extendNetwork rotates adds an additional orderer
  1302  func extendNetwork(n *nwo.Network) {
  1303  	// Overwrite the current crypto-config with additional orderers
  1304  	cryptoConfigYAML, err := ioutil.TempFile("", "crypto-config.yaml")
  1305  	Expect(err).NotTo(HaveOccurred())
  1306  	defer os.Remove(cryptoConfigYAML.Name())
  1307  
  1308  	err = ioutil.WriteFile(cryptoConfigYAML.Name(), []byte(extendedCryptoConfig), 0o644)
  1309  	Expect(err).NotTo(HaveOccurred())
  1310  
  1311  	// Invoke cryptogen extend to add new orderers
  1312  	sess, err := n.Cryptogen(commands.Extend{
  1313  		Config: cryptoConfigYAML.Name(),
  1314  		Input:  n.CryptoPath(),
  1315  	})
  1316  	Expect(err).NotTo(HaveOccurred())
  1317  	Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(0))
  1318  }
  1319  
  1320  // addConsenter adds a new consenter to the given channel.
  1321  func addConsenter(n *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel string, consenter protosraft.Consenter) {
  1322  	updateEtcdRaftMetadata(n, peer, orderer, channel, func(metadata *protosraft.ConfigMetadata) {
  1323  		metadata.Consenters = append(metadata.Consenters, &consenter)
  1324  	})
  1325  }
  1326  
  1327  // updateEtcdRaftMetadata executes a config update that updates the etcdraft
  1328  // metadata according to the given function f.
  1329  func updateEtcdRaftMetadata(network *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel string, f func(md *protosraft.ConfigMetadata)) {
  1330  	nwo.UpdateConsensusMetadata(network, peer, orderer, channel, func(originalMetadata []byte) []byte {
  1331  		metadata := &protosraft.ConfigMetadata{}
  1332  		err := proto.Unmarshal(originalMetadata, metadata)
  1333  		Expect(err).NotTo(HaveOccurred())
  1334  
  1335  		f(metadata)
  1336  
  1337  		newMetadata, err := proto.Marshal(metadata)
  1338  		Expect(err).NotTo(HaveOccurred())
  1339  		return newMetadata
  1340  	})
  1341  }