github.com/yacovm/fabric@v2.0.0-alpha.0.20191128145320-c5d4087dc723+incompatible/integration/raft/migration_test.go (about)

     1  /*
     2  Copyright IBM Corp All Rights Reserved.
     3  
     4  SPDX-License-Identifier: Apache-2.0
     5  */
     6  
     7  package raft
     8  
     9  import (
    10  	"errors"
    11  	"fmt"
    12  	"io/ioutil"
    13  	"os"
    14  	"path/filepath"
    15  	"strings"
    16  	"syscall"
    17  	"time"
    18  
    19  	docker "github.com/fsouza/go-dockerclient"
    20  	"github.com/golang/protobuf/proto"
    21  	"github.com/hyperledger/fabric-protos-go/common"
    22  	protosorderer "github.com/hyperledger/fabric-protos-go/orderer"
    23  	protosraft "github.com/hyperledger/fabric-protos-go/orderer/etcdraft"
    24  	"github.com/hyperledger/fabric/integration/nwo"
    25  	"github.com/hyperledger/fabric/integration/nwo/commands"
    26  	"github.com/hyperledger/fabric/protoutil"
    27  	. "github.com/onsi/ginkgo"
    28  	. "github.com/onsi/gomega"
    29  	"github.com/onsi/gomega/gbytes"
    30  	"github.com/onsi/gomega/gexec"
    31  	"github.com/tedsuo/ifrit"
    32  	"github.com/tedsuo/ifrit/ginkgomon"
    33  )
    34  
    35  var _ = Describe("Kafka2RaftMigration", func() {
    36  	var (
    37  		testDir   string
    38  		client    *docker.Client
    39  		network   *nwo.Network
    40  		chaincode nwo.Chaincode
    41  
    42  		process                ifrit.Process
    43  		peerProc, brokerProc   ifrit.Process
    44  		o1Proc, o2Proc, o3Proc ifrit.Process
    45  
    46  		o1Runner, o2Runner, o3Runner *ginkgomon.Runner
    47  	)
    48  
    49  	BeforeEach(func() {
    50  		var err error
    51  		testDir, err = ioutil.TempDir("", "kafka2raft-migration")
    52  		Expect(err).NotTo(HaveOccurred())
    53  
    54  		client, err = docker.NewClientFromEnv()
    55  		Expect(err).NotTo(HaveOccurred())
    56  
    57  		chaincode = nwo.Chaincode{
    58  			Name:            "mycc",
    59  			Version:         "0.0",
    60  			Path:            components.Build("github.com/hyperledger/fabric/integration/chaincode/module"),
    61  			Lang:            "binary",
    62  			PackageFile:     filepath.Join(testDir, "modulecc.tar.gz"),
    63  			Ctor:            `{"Args":["init","a","100","b","200"]}`,
    64  			SignaturePolicy: `AND ('Org1MSP.member','Org2MSP.member')`,
    65  			Sequence:        "1",
    66  			InitRequired:    true,
    67  			Label:           "my_prebuilt_chaincode",
    68  		}
    69  	})
    70  
    71  	AfterEach(func() {
    72  		if process != nil {
    73  			process.Signal(syscall.SIGTERM)
    74  			Eventually(process.Wait(), network.EventuallyTimeout).Should(Receive())
    75  		}
    76  
    77  		if peerProc != nil {
    78  			peerProc.Signal(syscall.SIGTERM)
    79  			Eventually(peerProc.Wait(), network.EventuallyTimeout).Should(Receive())
    80  		}
    81  
    82  		for _, oProc := range []ifrit.Process{o1Proc, o2Proc, o3Proc} {
    83  			if oProc != nil {
    84  				oProc.Signal(syscall.SIGTERM)
    85  				Eventually(oProc.Wait(), network.EventuallyTimeout).Should(Receive())
    86  			}
    87  		}
    88  
    89  		if brokerProc != nil {
    90  			brokerProc.Signal(syscall.SIGTERM)
    91  			Eventually(brokerProc.Wait(), network.EventuallyTimeout).Should(Receive())
    92  		}
    93  
    94  		if network != nil {
    95  			network.Cleanup()
    96  		}
    97  
    98  		_ = os.RemoveAll(testDir)
    99  	})
   100  
   101  	// These tests execute the migration config updates on Kafka based system, and verify that these config update
   102  	// have the desired effect. They focus on testing that filtering and the blocking of transitions into, in, and
   103  	// out-of maintenance mode are enforced. These tests use a single node.
   104  	Describe("Kafka to Raft migration kafka side", func() {
   105  		var (
   106  			orderer                                  *nwo.Orderer
   107  			peer                                     *nwo.Peer
   108  			syschannel, channel1, channel2, channel3 string
   109  			raftMetadata                             []byte
   110  		)
   111  
   112  		BeforeEach(func() {
   113  			network = nwo.New(kafka2RaftMultiChannel(), testDir, client, StartPort(), components)
   114  			network.GenerateConfigTree()
   115  			network.Bootstrap()
   116  
   117  			networkRunner := network.NetworkGroupRunner()
   118  			process = ifrit.Invoke(networkRunner)
   119  			Eventually(process.Ready(), network.EventuallyTimeout).Should(BeClosed())
   120  
   121  			orderer = network.Orderer("orderer")
   122  			peer = network.Peer("Org1", "peer1")
   123  
   124  			syschannel = network.SystemChannel.Name
   125  			channel1 = "testchannel1"
   126  			channel2 = "testchannel2"
   127  			channel3 = "testchannel3"
   128  
   129  			raftMetadata = prepareRaftMetadata(network)
   130  
   131  			By("Create & join first channel, deploy and invoke chaincode")
   132  			network.CreateAndJoinChannel(orderer, channel1)
   133  			nwo.EnableCapabilities(network, channel1, "Application", "V2_0", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0"))
   134  			nwo.DeployChaincode(network, channel1, orderer, chaincode)
   135  			RunExpectQueryInvokeQuery(network, orderer, peer, channel1, 100)
   136  			RunExpectQueryInvokeQuery(network, orderer, peer, channel1, 90)
   137  		})
   138  
   139  		// This test executes the "green path" migration config updates on Kafka based system with a system channel
   140  		// and a standard channel, and verifies that these config updates have the desired effect.
   141  		//
   142  		// The green path is entering maintenance mode, and then changing the consensus type.
   143  		// In maintenance mode we check that channel creation is blocked and that normal transactions are blocked.
   144  		//
   145  		// We also check that after entering maintenance mode, we can exit it without making any
   146  		// changes - the "abort path".
   147  		It("executes kafka2raft green path", func() {
   148  			//=== The abort path ======================================================================================
   149  			//=== Step 1: Config update on system channel, MAINTENANCE ===
   150  			By("1) Config update on system channel, State=MAINTENANCE, enter maintenance-mode")
   151  			config, updatedConfig := prepareTransition(network, peer, orderer, syschannel,
   152  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   153  				"kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   154  			nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   155  
   156  			By("1) Verify: system channel1 config changed")
   157  			sysStartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel)
   158  			Expect(sysStartBlockNum).ToNot(Equal(0))
   159  			config = nwo.GetConfig(network, peer, orderer, syschannel)
   160  			updatedConfig, consensusTypeValue, err := extractOrdererConsensusType(config)
   161  			Expect(err).NotTo(HaveOccurred())
   162  			validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE)
   163  
   164  			By("1) Verify: new channels cannot be created")
   165  			exitCode := network.CreateChannelExitCode(channel2, orderer, peer)
   166  			Expect(exitCode).ToNot(Equal(0))
   167  
   168  			//=== Step 2: Config update on standard channel, MAINTENANCE ===
   169  			By("2) Config update on standard channel, State=MAINTENANCE, enter maintenance-mode")
   170  			config, updatedConfig = prepareTransition(network, peer, orderer, channel1,
   171  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   172  				"kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   173  			nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer)
   174  
   175  			By("2) Verify: standard channel config changed")
   176  			std1EntryBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1)
   177  			Expect(std1EntryBlockNum).ToNot(Equal(0))
   178  			config = nwo.GetConfig(network, peer, orderer, channel1)
   179  			updatedConfig, consensusTypeValue, err = extractOrdererConsensusType(config)
   180  			Expect(err).NotTo(HaveOccurred())
   181  			validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE)
   182  
   183  			By("2) Verify: Normal TX's on standard channel are blocked")
   184  			RunExpectQueryInvokeFail(network, orderer, peer, channel1, 80)
   185  
   186  			//In maintenance mode deliver requests are open to those entities that satisfy the /Channel/Orderer/Readers policy
   187  			By("2) Verify: delivery request from peer is blocked")
   188  			err = checkPeerDeliverRequest(orderer, peer, network, channel1)
   189  			Expect(err).To(MatchError(errors.New("FORBIDDEN")))
   190  
   191  			//=== Step 3: config update on system channel, State=NORMAL, abort ===
   192  			By("3) Config update on system channel, State=NORMAL, exit maintenance-mode - abort path")
   193  			config, updatedConfig = prepareTransition(network, peer, orderer, syschannel,
   194  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   195  				"kafka", nil, protosorderer.ConsensusType_STATE_NORMAL)
   196  			nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   197  
   198  			By("3) Verify: system channel config changed")
   199  			sysBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel)
   200  			Expect(sysBlockNum).To(Equal(sysStartBlockNum + 1))
   201  
   202  			By("3) Verify: create new channel, executing transaction")
   203  			network.CreateAndJoinChannel(orderer, channel2)
   204  			nwo.EnableCapabilities(network, channel2, "Application", "V2_0", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0"))
   205  			nwo.DeployChaincode(network, channel2, orderer, chaincode, network.PeersWithChannel(channel2)...)
   206  			RunExpectQueryRetry(network, peer, channel2, 100)
   207  			RunExpectQueryInvokeQuery(network, orderer, peer, channel2, 100)
   208  			RunExpectQueryInvokeQuery(network, orderer, peer, channel2, 90)
   209  
   210  			By("3) Verify: delivery request from peer is not blocked on new channel")
   211  			err = checkPeerDeliverRequest(orderer, peer, network, channel2)
   212  			Expect(err).NotTo(HaveOccurred())
   213  
   214  			//=== Step 4: config update on standard channel, State=NORMAL, abort ===
   215  			By("4) Config update on standard channel, State=NORMAL, exit maintenance-mode - abort path")
   216  			config, updatedConfig = prepareTransition(network, peer, orderer, channel1,
   217  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   218  				"kafka", nil, protosorderer.ConsensusType_STATE_NORMAL)
   219  			nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer)
   220  
   221  			By("4) Verify: standard channel config changed")
   222  			std1BlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1)
   223  			Expect(std1BlockNum).To(Equal(std1EntryBlockNum + 1))
   224  
   225  			By("4) Verify: standard channel delivery requests from peer unblocked")
   226  			err = checkPeerDeliverRequest(orderer, peer, network, channel1)
   227  			Expect(err).NotTo(HaveOccurred())
   228  
   229  			By("4) Verify: Normal TX's on standard channel are permitted again")
   230  			RunExpectQueryInvokeQuery(network, orderer, peer, channel1, 80)
   231  
   232  			//=== The green path ======================================================================================
   233  			//=== Step 5: Config update on system channel, MAINTENANCE, again ===
   234  			By("5) Config update on system channel, State=MAINTENANCE, enter maintenance-mode again")
   235  			config, updatedConfig = prepareTransition(network, peer, orderer, syschannel,
   236  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   237  				"kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   238  			nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   239  
   240  			By("5) Verify: system channel config changed")
   241  			sysStartBlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel)
   242  			Expect(sysStartBlockNum).ToNot(Equal(0))
   243  			config = nwo.GetConfig(network, peer, orderer, syschannel)
   244  			updatedConfig, consensusTypeValue, err = extractOrdererConsensusType(config)
   245  			Expect(err).NotTo(HaveOccurred())
   246  			validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE)
   247  
   248  			//=== Step 6: Config update on standard channel1, MAINTENANCE, again ===
   249  			By("6) Config update on standard channel1, State=MAINTENANCE, enter maintenance-mode again")
   250  			config, updatedConfig = prepareTransition(network, peer, orderer, channel1,
   251  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   252  				"kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   253  			nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer)
   254  
   255  			By("6) Verify: standard channel config changed")
   256  			std1EntryBlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1)
   257  			Expect(std1EntryBlockNum).ToNot(Equal(0))
   258  
   259  			config = nwo.GetConfig(network, peer, orderer, channel1)
   260  			updatedConfig, consensusTypeValue, err = extractOrdererConsensusType(config)
   261  			Expect(err).NotTo(HaveOccurred())
   262  			validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE)
   263  
   264  			By("6) Verify: delivery request from peer is blocked")
   265  			err = checkPeerDeliverRequest(orderer, peer, network, channel1)
   266  			Expect(err).To(MatchError(errors.New("FORBIDDEN")))
   267  
   268  			By("6) Verify: Normal TX's on standard channel are blocked")
   269  			RunExpectQueryInvokeFail(network, orderer, peer, channel1, 70)
   270  
   271  			By("6) Verify: peers should not receive the config update on entry to maintenance, because deliver is blocked")
   272  			std1BlockNumFromPeer1 := nwo.CurrentConfigBlockNumber(network, peer, nil, channel1)
   273  			Expect(std1BlockNumFromPeer1).To(BeNumerically("<", std1EntryBlockNum))
   274  
   275  			//=== Step 7: Config update on standard channel2, MAINTENANCE ===
   276  			By("7) Config update on standard channel2, State=MAINTENANCE, enter maintenance-mode again")
   277  			config, updatedConfig = prepareTransition(network, peer, orderer, channel2,
   278  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   279  				"kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   280  			nwo.UpdateOrdererConfig(network, orderer, channel2, config, updatedConfig, peer, orderer)
   281  
   282  			By("7) Verify: standard channel config changed")
   283  			std2EntryBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel2)
   284  			Expect(std2EntryBlockNum).ToNot(Equal(0))
   285  
   286  			config = nwo.GetConfig(network, peer, orderer, channel2)
   287  			updatedConfig, consensusTypeValue, err = extractOrdererConsensusType(config)
   288  			Expect(err).NotTo(HaveOccurred())
   289  			validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE)
   290  
   291  			By("7) Verify: delivery request from peer is blocked")
   292  			err = checkPeerDeliverRequest(orderer, peer, network, channel2)
   293  			Expect(err).To(MatchError(errors.New("FORBIDDEN")))
   294  
   295  			By("7) Verify: Normal TX's on standard channel are blocked")
   296  			RunExpectQueryInvokeFail(network, orderer, peer, channel2, 80)
   297  
   298  			By("7) Verify: peers should not receive the config update on entry to maintenance, because deliver is blocked")
   299  			std2BlockNumFromPeer1 := nwo.CurrentConfigBlockNumber(network, peer, nil, channel2)
   300  			Expect(std2BlockNumFromPeer1).To(BeNumerically("<", std2EntryBlockNum))
   301  
   302  			//=== Step 8: config update on system channel, State=MAINTENANCE, type=etcdraft ===
   303  			By("8) Config update on system channel, State=MAINTENANCE, type=etcdraft")
   304  			config, updatedConfig = prepareTransition(network, peer, orderer, syschannel,
   305  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   306  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE)
   307  			nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   308  
   309  			By("8) Verify: system channel config changed")
   310  			sysBlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel)
   311  			Expect(sysBlockNum).To(Equal(sysStartBlockNum + 1))
   312  
   313  			By("8) Verify: new channels cannot be created")
   314  			exitCode = network.CreateChannelExitCode(channel3, orderer, peer)
   315  			Expect(exitCode).ToNot(Equal(0))
   316  
   317  			//=== Step 9: config update on standard channel1, State=MAINTENANCE, type=etcdraft ===
   318  			By("9) Config update on standard channel1, State=MAINTENANCE, type=etcdraft")
   319  			config, updatedConfig = prepareTransition(network, peer, orderer, channel1,
   320  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   321  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE)
   322  			nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer)
   323  
   324  			By("9) Verify: standard channel config changed")
   325  			std1BlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1)
   326  			Expect(std1BlockNum).To(Equal(std1EntryBlockNum + 1))
   327  
   328  			By("9) Verify: delivery request from peer is blocked")
   329  			err = checkPeerDeliverRequest(orderer, peer, network, channel1)
   330  			Expect(err).To(MatchError(errors.New("FORBIDDEN")))
   331  
   332  			By("9) Verify: Normal TX's on standard channel are blocked")
   333  			RunExpectQueryInvokeFail(network, orderer, peer, channel1, 70)
   334  
   335  			By("9) Verify: peers should not receive the config update on maintenance, because deliver is blocked")
   336  			std1BlockNumFromPeer2 := nwo.CurrentConfigBlockNumber(network, peer, nil, channel1)
   337  			Expect(std1BlockNumFromPeer2).To(BeNumerically("<", std1EntryBlockNum))
   338  			Expect(std1BlockNumFromPeer2).To(BeNumerically(">=", std1BlockNumFromPeer1))
   339  
   340  			//=== Step 10: config update on standard channel2, State=MAINTENANCE, type=etcdraft ===
   341  			By("10) Config update on standard channel2, State=MAINTENANCE, type=etcdraft")
   342  			config, updatedConfig = prepareTransition(network, peer, orderer, channel2,
   343  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   344  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE)
   345  			nwo.UpdateOrdererConfig(network, orderer, channel2, config, updatedConfig, peer, orderer)
   346  
   347  			By("10) Verify: standard channel config changed")
   348  			std2BlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel2)
   349  			Expect(std2BlockNum).To(Equal(std2EntryBlockNum + 1))
   350  
   351  			By("10) Verify: delivery request from peer is blocked")
   352  			err = checkPeerDeliverRequest(orderer, peer, network, channel2)
   353  			Expect(err).To(MatchError(errors.New("FORBIDDEN")))
   354  
   355  			By("10) Verify: Normal TX's on standard channel are blocked")
   356  			RunExpectQueryInvokeFail(network, orderer, peer, channel2, 80)
   357  
   358  			By("10) Verify: peers should not receive the config update on maintenance, because deliver is blocked")
   359  			std2BlockNumFromPeer2 := nwo.CurrentConfigBlockNumber(network, peer, nil, channel2)
   360  			Expect(std2BlockNumFromPeer2).To(BeNumerically("<", std2EntryBlockNum))
   361  			Expect(std2BlockNumFromPeer2).To(BeNumerically(">=", std2BlockNumFromPeer1))
   362  		})
   363  
   364  		// This test executes the migration flow and checks that forbidden transitions are rejected.
   365  		// These transitions are enforced by the maintenance filter:
   366  		// - Entry to & exit from maintenance mode can only change ConsensusType.State.
   367  		// - In maintenance mode one can only change ConsensusType.Type & ConsensusType.Metadata.
   368  		// - ConsensusType.Type can only change from "kafka" to "etcdraft", and only in maintenance mode.
   369  		It("executes kafka2raft forbidden transitions", func() {
   370  			//=== Step 1: ===
   371  			By("1) Config update on system channel, changing both ConsensusType State & Type is forbidden")
   372  			assertTransitionFailed(network, peer, orderer, syschannel,
   373  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   374  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE)
   375  
   376  			//=== Step 2: ===
   377  			By("2) Config update on standard channel, changing both ConsensusType State & Type is forbidden")
   378  			assertTransitionFailed(network, peer, orderer, channel1,
   379  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   380  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE)
   381  
   382  			//=== Step 3: ===
   383  			By("3) Config update on system channel, changing both ConsensusType State & some other value is forbidden")
   384  			config, updatedConfig := prepareTransition(network, peer, orderer, syschannel,
   385  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   386  				"kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   387  			updateConfigWithBatchTimeout(updatedConfig)
   388  			updateOrdererConfigFailed(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   389  
   390  			//=== Step 4: ===
   391  			By("4) Config update on standard channel, both ConsensusType State & some other value is forbidden")
   392  			config, updatedConfig = prepareTransition(network, peer, orderer, channel1,
   393  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   394  				"kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   395  			updateConfigWithBatchTimeout(updatedConfig)
   396  			updateOrdererConfigFailed(network, orderer, channel1, config, updatedConfig, peer, orderer)
   397  
   398  			//=== Step 5: ===
   399  			By("5) Config update on system channel, State=MAINTENANCE, enter maintenance-mode")
   400  			config, updatedConfig = prepareTransition(network, peer, orderer, syschannel,
   401  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   402  				"kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   403  			nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   404  
   405  			By("5) Verify: system channel config changed")
   406  			sysStartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel)
   407  			Expect(sysStartBlockNum).ToNot(Equal(0))
   408  			config = nwo.GetConfig(network, peer, orderer, syschannel)
   409  			updatedConfig, consensusTypeValue, err := extractOrdererConsensusType(config)
   410  			Expect(err).NotTo(HaveOccurred())
   411  			validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE)
   412  
   413  			//=== Step 6: ===
   414  			By("6) Config update on standard channel, State=MAINTENANCE, enter maintenance-mode")
   415  			config, updatedConfig = prepareTransition(network, peer, orderer, channel1,
   416  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   417  				"kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   418  			nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer)
   419  
   420  			By("6) Verify: standard channel config changed")
   421  			std1StartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1)
   422  			Expect(std1StartBlockNum).ToNot(Equal(0))
   423  			config = nwo.GetConfig(network, peer, orderer, channel1)
   424  			updatedConfig, consensusTypeValue, err = extractOrdererConsensusType(config)
   425  			Expect(err).NotTo(HaveOccurred())
   426  			validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE)
   427  
   428  			//=== Step 7: ===
   429  			By("7) Config update on system channel, change ConsensusType.Type to unsupported type, forbidden")
   430  			assertTransitionFailed(network, peer, orderer, syschannel,
   431  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   432  				"melville", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   433  
   434  			//=== Step 8: ===
   435  			By("8) Config update on standard channel, change ConsensusType.Type to unsupported type, forbidden")
   436  			assertTransitionFailed(network, peer, orderer, channel1,
   437  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   438  				"hesse", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   439  
   440  			//=== Step 9: ===
   441  			By("9) Config update on system channel, change ConsensusType.Type and State, forbidden")
   442  			assertTransitionFailed(network, peer, orderer, syschannel,
   443  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   444  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL)
   445  
   446  			//=== Step 10: ===
   447  			By("10) Config update on standard channel, change ConsensusType.Type and State, forbidden")
   448  			assertTransitionFailed(network, peer, orderer, channel1,
   449  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   450  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL)
   451  
   452  			//=== Step 11: ===
   453  			By("11) Config update on system channel, changing both ConsensusType.Type and other value is permitted")
   454  			config, updatedConfig = prepareTransition(network, peer, orderer, syschannel,
   455  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   456  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE)
   457  			updateConfigWithBatchTimeout(updatedConfig)
   458  			nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   459  
   460  			By("11) Verify: system channel config changed")
   461  			sysBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel)
   462  			Expect(sysBlockNum).To(Equal(sysStartBlockNum + 1))
   463  			config = nwo.GetConfig(network, peer, orderer, syschannel)
   464  			updatedConfig, consensusTypeValue, err = extractOrdererConsensusType(config)
   465  			Expect(err).NotTo(HaveOccurred())
   466  			validateConsensusTypeValue(consensusTypeValue, "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE)
   467  
   468  			//=== Step 12: ===
   469  			By("12) Config update on standard channel, changing both ConsensusType.Type and other value is permitted")
   470  			config, updatedConfig = prepareTransition(network, peer, orderer, channel1,
   471  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   472  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE)
   473  			updateConfigWithBatchTimeout(updatedConfig)
   474  			nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer)
   475  
   476  			By("12) Verify: standard channel config changed")
   477  			std1BlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1)
   478  			Expect(std1BlockNum).To(Equal(std1StartBlockNum + 1))
   479  			config = nwo.GetConfig(network, peer, orderer, channel1)
   480  			updatedConfig, consensusTypeValue, err = extractOrdererConsensusType(config)
   481  			Expect(err).NotTo(HaveOccurred())
   482  			validateConsensusTypeValue(consensusTypeValue, "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE)
   483  
   484  			//=== Step 13: ===
   485  			By("13) Config update on system channel, changing value other than ConsensusType.Type is permitted")
   486  			config = nwo.GetConfig(network, peer, orderer, syschannel)
   487  			updatedConfig, consensusTypeValue, err = extractOrdererConsensusType(config)
   488  			Expect(err).NotTo(HaveOccurred())
   489  			validateConsensusTypeValue(consensusTypeValue, "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE)
   490  			updateConfigWithBatchTimeout(updatedConfig)
   491  			nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   492  
   493  			By("13) Verify: system channel config changed")
   494  			sysBlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel)
   495  			Expect(sysBlockNum).To(Equal(sysStartBlockNum + 2))
   496  
   497  			//=== Step 14: ===
   498  			By("14) Config update on standard channel, changing value other than ConsensusType.Type is permitted")
   499  			config = nwo.GetConfig(network, peer, orderer, channel1)
   500  			updatedConfig, consensusTypeValue, err = extractOrdererConsensusType(config)
   501  			Expect(err).NotTo(HaveOccurred())
   502  			validateConsensusTypeValue(consensusTypeValue, "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE)
   503  			updateConfigWithBatchTimeout(updatedConfig)
   504  			nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer)
   505  
   506  			By("14) Verify: standard channel config changed")
   507  			std1BlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1)
   508  			Expect(std1BlockNum).To(Equal(std1StartBlockNum + 2))
   509  
   510  			//=== Step 15: ===
   511  			By("15) Config update on system channel, changing both ConsensusType State & some other value is forbidden")
   512  			config, updatedConfig = prepareTransition(network, peer, orderer, syschannel,
   513  				"etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE,
   514  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL)
   515  			updateConfigWithBatchTimeout(updatedConfig)
   516  			updateOrdererConfigFailed(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   517  
   518  			//=== Step 16: ===
   519  			By("16) Config update on standard channel, both ConsensusType State & some other value is forbidden")
   520  			config, updatedConfig = prepareTransition(network, peer, orderer, channel1,
   521  				"etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE,
   522  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL)
   523  			updateConfigWithBatchTimeout(updatedConfig)
   524  			updateOrdererConfigFailed(network, orderer, channel1, config, updatedConfig, peer, orderer)
   525  		})
   526  	})
   527  
   528  	// These tests execute the migration config updates on Kafka based system, restart the orderer onto a Raft-based
   529  	// system, and verifies that the newly restarted orderer cluster performs as expected.
   530  	Describe("Kafka to Raft migration raft side", func() {
   531  		var (
   532  			o1, o2, o3                     *nwo.Orderer
   533  			peer                           *nwo.Peer
   534  			syschannel, channel1, channel2 string
   535  			raftMetadata                   []byte
   536  		)
   537  
   538  		BeforeEach(func() {
   539  			network = nwo.New(kafka2RaftMultiNode(), testDir, client, StartPort(), components)
   540  			network.GenerateConfigTree()
   541  			network.Bootstrap()
   542  
   543  			o1, o2, o3 = network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3")
   544  			peer = network.Peer("Org1", "peer1")
   545  
   546  			brokerGroup := network.BrokerGroupRunner()
   547  			brokerProc = ifrit.Invoke(brokerGroup)
   548  			Eventually(brokerProc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   549  
   550  			o1Runner = network.OrdererRunner(o1)
   551  			o2Runner = network.OrdererRunner(o2)
   552  			o3Runner = network.OrdererRunner(o3)
   553  
   554  			peerGroup := network.PeerGroupRunner()
   555  
   556  			o1Proc = ifrit.Invoke(o1Runner)
   557  			o2Proc = ifrit.Invoke(o2Runner)
   558  			o3Proc = ifrit.Invoke(o3Runner)
   559  
   560  			Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   561  			Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   562  			Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   563  
   564  			peerProc = ifrit.Invoke(peerGroup)
   565  			Eventually(peerProc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   566  
   567  			raftMetadata = prepareRaftMetadata(network)
   568  
   569  			syschannel = network.SystemChannel.Name
   570  			channel1 = "testchannel1"
   571  			channel2 = "testchannel2"
   572  
   573  			By("Create & join first channel, deploy and invoke chaincode")
   574  			network.CreateAndJoinChannel(o1, channel1)
   575  			nwo.EnableCapabilities(network, channel1, "Application", "V2_0", o1, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0"))
   576  			nwo.DeployChaincode(network, channel1, o1, chaincode)
   577  			RunExpectQueryInvokeQuery(network, o1, peer, channel1, 100)
   578  			RunExpectQueryInvokeQuery(network, o1, peer, channel1, 90)
   579  		})
   580  
   581  		// This test executes the "green path" migration config updates on Kafka based system
   582  		// with a three orderers, a system channel and two standard channels.
   583  		// It then restarts the orderers onto a Raft-based system, and verifies that the
   584  		// newly restarted orderers perform as expected.
   585  		It("executes bootstrap to raft - multi node", func() {
   586  			//=== Step 1: Config update on system channel, MAINTENANCE ===
   587  			By("1) Config update on system channel, State=MAINTENANCE")
   588  			config, updatedConfig := prepareTransition(network, peer, o1, syschannel,
   589  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   590  				"kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   591  			nwo.UpdateOrdererConfig(network, o1, syschannel, config, updatedConfig, peer, o1)
   592  
   593  			By("1) Verify: system channel config changed")
   594  			sysStartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, o1, syschannel)
   595  			Expect(sysStartBlockNum).ToNot(Equal(0))
   596  
   597  			config = nwo.GetConfig(network, peer, o1, syschannel)
   598  			updatedConfig, consensusTypeValue, err := extractOrdererConsensusType(config)
   599  			Expect(err).NotTo(HaveOccurred())
   600  			validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE)
   601  
   602  			//=== Step 2: Config update on standard channel, MAINTENANCE ===
   603  			By("2) Config update on standard channel, State=MAINTENANCE")
   604  			config, updatedConfig = prepareTransition(network, peer, o1, channel1,
   605  				"kafka", protosorderer.ConsensusType_STATE_NORMAL,
   606  				"kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   607  			nwo.UpdateOrdererConfig(network, o1, channel1, config, updatedConfig, peer, o1)
   608  
   609  			By("2) Verify: standard channel config changed")
   610  			chan1StartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, o1, channel1)
   611  			Expect(chan1StartBlockNum).ToNot(Equal(0))
   612  
   613  			config = nwo.GetConfig(network, peer, o1, channel1)
   614  			updatedConfig, consensusTypeValue, err = extractOrdererConsensusType(config)
   615  			Expect(err).NotTo(HaveOccurred())
   616  			validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE)
   617  
   618  			//=== Step 3: config update on system channel, State=MAINTENANCE, type=etcdraft ===
   619  			By("3) Config update on system channel, State=MAINTENANCE, type=etcdraft")
   620  			config, updatedConfig = prepareTransition(network, peer, o1, syschannel,
   621  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   622  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE)
   623  			nwo.UpdateOrdererConfig(network, o1, syschannel, config, updatedConfig, peer, o1)
   624  
   625  			By("3) Verify: system channel config changed")
   626  			sysBlockNum := nwo.CurrentConfigBlockNumber(network, peer, o1, syschannel)
   627  			Expect(sysBlockNum).To(Equal(sysStartBlockNum + 1))
   628  
   629  			//=== Step 4: config update on standard channel, State=MAINTENANCE, type=etcdraft ===
   630  			By("4) Config update on standard channel, State=MAINTENANCE, type=etcdraft")
   631  			config, updatedConfig = prepareTransition(network, peer, o1, channel1,
   632  				"kafka", protosorderer.ConsensusType_STATE_MAINTENANCE,
   633  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE)
   634  			nwo.UpdateOrdererConfig(network, o1, channel1, config, updatedConfig, peer, o1)
   635  
   636  			By("4) Verify: standard channel config changed")
   637  			chan1BlockNum := nwo.CurrentConfigBlockNumber(network, peer, o1, channel1)
   638  			Expect(chan1BlockNum).To(Equal(chan1StartBlockNum + 1))
   639  
   640  			//=== Step 5: kill ===
   641  			By("5) killing orderer1,2,3")
   642  			for _, oProc := range []ifrit.Process{o1Proc, o2Proc, o3Proc} {
   643  				if oProc != nil {
   644  					oProc.Signal(syscall.SIGKILL)
   645  					Eventually(oProc.Wait(), network.EventuallyTimeout).Should(Receive(MatchError("exit status 137")))
   646  				}
   647  			}
   648  
   649  			//=== Step 6: restart ===
   650  			By("6) restarting orderer1,2,3")
   651  			network.Consensus.Type = "etcdraft"
   652  			o1Runner = network.OrdererRunner(o1)
   653  			o2Runner = network.OrdererRunner(o2)
   654  			o3Runner = network.OrdererRunner(o3)
   655  
   656  			o1Proc = ifrit.Invoke(o1Runner)
   657  			o2Proc = ifrit.Invoke(o2Runner)
   658  			o3Proc = ifrit.Invoke(o3Runner)
   659  
   660  			Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   661  			Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   662  			Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   663  
   664  			assertBlockReception(
   665  				map[string]int{
   666  					syschannel: int(sysBlockNum),
   667  					channel1:   int(chan1BlockNum),
   668  				},
   669  				[]*nwo.Orderer{o1, o2, o3},
   670  				peer,
   671  				network,
   672  			)
   673  
   674  			Eventually(o1Runner.Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say("Raft leader changed: 0 -> "))
   675  			Eventually(o2Runner.Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say("Raft leader changed: 0 -> "))
   676  			Eventually(o3Runner.Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say("Raft leader changed: 0 -> "))
   677  
   678  			By("7) System channel still in maintenance, State=MAINTENANCE, cannot create new channels")
   679  			exitCode := network.CreateChannelExitCode(channel2, o1, peer)
   680  			Expect(exitCode).ToNot(Equal(0))
   681  
   682  			By("8) Standard channel still in maintenance, State=MAINTENANCE, normal TX's blocked, delivery to peers blocked")
   683  			RunExpectQueryInvokeFail(network, o1, peer, channel1, 80)
   684  			err = checkPeerDeliverRequest(o1, peer, network, channel1)
   685  			Expect(err).To(MatchError(errors.New("FORBIDDEN")))
   686  
   687  			By("9) Release - executing config transaction on system channel with restarted orderer")
   688  			config, updatedConfig = prepareTransition(network, peer, o1, syschannel,
   689  				"etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE,
   690  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL)
   691  			nwo.UpdateOrdererConfig(network, o1, syschannel, config, updatedConfig, peer, o1)
   692  
   693  			By("9) Verify: system channel config changed")
   694  			sysBlockNum = nwo.CurrentConfigBlockNumber(network, peer, o1, syschannel)
   695  			Expect(sysBlockNum).To(Equal(sysStartBlockNum + 2))
   696  
   697  			By("10) Release - executing config transaction on standard channel with restarted orderer")
   698  			config, updatedConfig = prepareTransition(network, peer, o1, channel1,
   699  				"etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE,
   700  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL)
   701  			nwo.UpdateOrdererConfig(network, o1, channel1, config, updatedConfig, peer, o1)
   702  
   703  			By("10) Verify: standard channel config changed")
   704  			chan1BlockNum = nwo.CurrentConfigBlockNumber(network, peer, o1, channel1)
   705  			Expect(chan1BlockNum).To(Equal(chan1StartBlockNum + 2))
   706  			waitForBlockReceptionByPeer(peer, network, channel1, chan1BlockNum)
   707  
   708  			By("11) Executing transaction on standard channel with restarted orderer")
   709  			RunExpectQueryRetry(network, peer, channel1, 80)
   710  			RunExpectQueryInvokeQuery(network, o1, peer, channel1, 80)
   711  
   712  			By("12) Create new channel, executing transaction with restarted orderer")
   713  			network.CreateAndJoinChannel(o1, channel2)
   714  			nwo.EnableCapabilities(network, channel2, "Application", "V2_0", o1, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0"))
   715  			nwo.DeployChaincode(network, channel2, o1, chaincode)
   716  			RunExpectQueryRetry(network, peer, channel2, 100)
   717  			RunExpectQueryInvokeQuery(network, o1, peer, channel2, 100)
   718  			RunExpectQueryInvokeQuery(network, o1, peer, channel2, 90)
   719  		})
   720  	})
   721  
   722  	// These tests execute the migration config updates on a solo based system, restart the orderer onto a Raft-based
   723  	// system, and verifies that the newly restarted orderer (single node) cluster performs as expected.
   724  	Describe("Solo to Raft migration", func() {
   725  		var (
   726  			orderer                        *nwo.Orderer
   727  			peer                           *nwo.Peer
   728  			syschannel, channel1, channel2 string
   729  			raftMetadata                   []byte
   730  		)
   731  
   732  		BeforeEach(func() {
   733  			network = nwo.New(solo2RaftMultiChannel(), testDir, client, StartPort(), components)
   734  			network.GenerateConfigTree()
   735  			network.Bootstrap()
   736  
   737  			orderer = network.Orderer("orderer")
   738  			peer = network.Peer("Org1", "peer1")
   739  
   740  			brokerGroup := network.BrokerGroupRunner()
   741  			brokerProc = ifrit.Invoke(brokerGroup)
   742  			Eventually(brokerProc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   743  
   744  			o1Runner = network.OrdererRunner(orderer)
   745  			peerGroup := network.PeerGroupRunner()
   746  			o1Proc = ifrit.Invoke(o1Runner)
   747  			Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   748  			peerProc = ifrit.Invoke(peerGroup)
   749  			Eventually(peerProc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   750  
   751  			raftMetadata = prepareRaftMetadata(network)
   752  
   753  			syschannel = network.SystemChannel.Name
   754  			channel1 = "testchannel1"
   755  			channel2 = "testchannel2"
   756  
   757  			By("Create & join first channel, deploy and invoke chaincode")
   758  			network.CreateAndJoinChannel(orderer, channel1)
   759  			nwo.EnableCapabilities(network, channel1, "Application", "V2_0", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0"))
   760  			nwo.DeployChaincode(network, channel1, orderer, chaincode)
   761  			RunExpectQueryInvokeQuery(network, orderer, peer, channel1, 100)
   762  			RunExpectQueryInvokeQuery(network, orderer, peer, channel1, 90)
   763  		})
   764  
   765  		It("executes bootstrap to raft - single node", func() {
   766  			//=== Step 1: Config update on system channel, MAINTENANCE ===
   767  			By("1) Config update on system channel, State=MAINTENANCE")
   768  			config, updatedConfig := prepareTransition(network, peer, orderer, syschannel,
   769  				"solo", protosorderer.ConsensusType_STATE_NORMAL,
   770  				"solo", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   771  			nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   772  
   773  			By("1) Verify: system channel config changed")
   774  			sysStartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel)
   775  			Expect(sysStartBlockNum).ToNot(Equal(0))
   776  
   777  			config = nwo.GetConfig(network, peer, orderer, syschannel)
   778  			updatedConfig, consensusTypeValue, err := extractOrdererConsensusType(config)
   779  			Expect(err).NotTo(HaveOccurred())
   780  			validateConsensusTypeValue(consensusTypeValue, "solo", protosorderer.ConsensusType_STATE_MAINTENANCE)
   781  
   782  			//=== Step 2: Config update on standard channel, MAINTENANCE ===
   783  			By("2) Config update on standard channel, State=MAINTENANCE")
   784  			config, updatedConfig = prepareTransition(network, peer, orderer, channel1,
   785  				"solo", protosorderer.ConsensusType_STATE_NORMAL,
   786  				"solo", nil, protosorderer.ConsensusType_STATE_MAINTENANCE)
   787  			nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer)
   788  
   789  			By("2) Verify: standard channel config changed")
   790  			chan1StartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1)
   791  			Expect(chan1StartBlockNum).ToNot(Equal(0))
   792  
   793  			config = nwo.GetConfig(network, peer, orderer, channel1)
   794  			updatedConfig, consensusTypeValue, err = extractOrdererConsensusType(config)
   795  			Expect(err).NotTo(HaveOccurred())
   796  			validateConsensusTypeValue(consensusTypeValue, "solo", protosorderer.ConsensusType_STATE_MAINTENANCE)
   797  
   798  			//=== Step 3: config update on system channel, State=MAINTENANCE, type=etcdraft ===
   799  			By("3) Config update on system channel, State=MAINTENANCE, type=etcdraft")
   800  			config, updatedConfig = prepareTransition(network, peer, orderer, syschannel,
   801  				"solo", protosorderer.ConsensusType_STATE_MAINTENANCE,
   802  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE)
   803  			nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   804  
   805  			By("3) Verify: system channel config changed")
   806  			sysBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel)
   807  			Expect(sysBlockNum).To(Equal(sysStartBlockNum + 1))
   808  
   809  			//=== Step 4: config update on standard channel, State=MAINTENANCE, type=etcdraft ===
   810  			By("4) Config update on standard channel, State=MAINTENANCE, type=etcdraft")
   811  			config, updatedConfig = prepareTransition(network, peer, orderer, channel1,
   812  				"solo", protosorderer.ConsensusType_STATE_MAINTENANCE,
   813  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE)
   814  			nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer)
   815  
   816  			By("4) Verify: standard channel config changed")
   817  			chan1BlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1)
   818  			Expect(chan1BlockNum).To(Equal(chan1StartBlockNum + 1))
   819  
   820  			//=== Step 5: kill ===
   821  			By("5) killing orderer1")
   822  			o1Proc.Signal(syscall.SIGKILL)
   823  			Eventually(o1Proc.Wait(), network.EventuallyTimeout).Should(Receive(MatchError("exit status 137")))
   824  
   825  			//=== Step 6: restart ===
   826  			By("6) restarting orderer1")
   827  			network.Consensus.Type = "etcdraft"
   828  			o1Runner = network.OrdererRunner(orderer)
   829  			o1Proc = ifrit.Invoke(o1Runner)
   830  
   831  			Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   832  
   833  			assertBlockReception(
   834  				map[string]int{
   835  					syschannel: int(sysBlockNum),
   836  					channel1:   int(chan1BlockNum),
   837  				},
   838  				[]*nwo.Orderer{orderer},
   839  				peer,
   840  				network,
   841  			)
   842  
   843  			Eventually(o1Runner.Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say("Raft leader changed: 0 -> "))
   844  
   845  			By("7) System channel still in maintenance, State=MAINTENANCE, cannot create new channels")
   846  			exitCode := network.CreateChannelExitCode(channel2, orderer, peer)
   847  			Expect(exitCode).ToNot(Equal(0))
   848  
   849  			By("8) Standard channel still in maintenance, State=MAINTENANCE, normal TX's blocked, delivery to peers blocked")
   850  			RunExpectQueryInvokeFail(network, orderer, peer, channel1, 80)
   851  			err = checkPeerDeliverRequest(orderer, peer, network, channel1)
   852  			Expect(err).To(MatchError(errors.New("FORBIDDEN")))
   853  
   854  			By("9) Release - executing config transaction on system channel with restarted orderer")
   855  			config, updatedConfig = prepareTransition(network, peer, orderer, syschannel,
   856  				"etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE,
   857  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL)
   858  			nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer)
   859  
   860  			By("9) Verify: system channel config changed")
   861  			sysBlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel)
   862  			Expect(sysBlockNum).To(Equal(sysStartBlockNum + 2))
   863  
   864  			By("10) Release - executing config transaction on standard channel with restarted orderer")
   865  			config, updatedConfig = prepareTransition(network, peer, orderer, channel1,
   866  				"etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE,
   867  				"etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL)
   868  			nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer)
   869  
   870  			By("10) Verify: standard channel config changed")
   871  			chan1BlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1)
   872  			Expect(chan1BlockNum).To(Equal(chan1StartBlockNum + 2))
   873  			waitForBlockReceptionByPeer(peer, network, channel1, chan1BlockNum)
   874  
   875  			By("11) Executing transaction on standard channel with restarted orderer")
   876  			RunExpectQueryRetry(network, peer, channel1, 80)
   877  			RunExpectQueryInvokeQuery(network, orderer, peer, channel1, 80)
   878  
   879  			By("12) Create new channel, executing transaction with restarted orderer")
   880  			network.CreateAndJoinChannel(orderer, channel2)
   881  			nwo.EnableCapabilities(network, channel2, "Application", "V2_0", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0"))
   882  			nwo.DeployChaincode(network, channel2, orderer, chaincode)
   883  			RunExpectQueryRetry(network, peer, channel2, 100)
   884  			RunExpectQueryInvokeQuery(network, orderer, peer, channel2, 100)
   885  			RunExpectQueryInvokeQuery(network, orderer, peer, channel2, 90)
   886  		})
   887  	})
   888  })
   889  
   890  func RunExpectQueryInvokeQuery(n *nwo.Network, orderer *nwo.Orderer, peer *nwo.Peer, channel string, expect int) {
   891  	res := RunQuery(n, orderer, peer, channel)
   892  	Expect(res).To(Equal(expect))
   893  
   894  	RunInvoke(n, orderer, peer, channel)
   895  
   896  	res = RunQuery(n, orderer, peer, channel)
   897  	Expect(res).To(Equal(expect - 10))
   898  }
   899  
   900  func RunExpectQueryRetry(n *nwo.Network, peer *nwo.Peer, channel string, expect int) {
   901  	By("Querying the chaincode with retry")
   902  
   903  	Eventually(func() bool {
   904  		res := RunQuery(n, nil, peer, channel)
   905  		return res == expect
   906  	}, n.EventuallyTimeout, time.Second).Should(BeTrue())
   907  }
   908  
   909  func RunInvokeFail(n *nwo.Network, orderer *nwo.Orderer, peer *nwo.Peer, channel string) {
   910  	sess, err := n.PeerUserSession(peer, "User1", commands.ChaincodeInvoke{
   911  		ChannelID: channel,
   912  		Orderer:   n.OrdererAddress(orderer, nwo.ListenPort),
   913  		Name:      "mycc",
   914  		Ctor:      `{"Args":["invoke","a","b","10"]}`,
   915  		PeerAddresses: []string{
   916  			n.PeerAddress(n.Peer("Org1", "peer0"), nwo.ListenPort),
   917  			n.PeerAddress(n.Peer("Org2", "peer1"), nwo.ListenPort),
   918  		},
   919  		WaitForEvent: true,
   920  	})
   921  	Expect(err).NotTo(HaveOccurred())
   922  	Eventually(sess, n.EventuallyTimeout).ShouldNot(gexec.Exit(0))
   923  	Expect(sess.Err).NotTo(gbytes.Say("Chaincode invoke successful. result: status:200"))
   924  }
   925  
   926  func RunExpectQueryInvokeFail(n *nwo.Network, orderer *nwo.Orderer, peer *nwo.Peer, channel string, expect int) {
   927  	res := RunQuery(n, orderer, peer, channel)
   928  	Expect(res).To(Equal(expect))
   929  	RunInvokeFail(n, orderer, peer, channel)
   930  }
   931  
   932  func validateConsensusTypeValue(value *protosorderer.ConsensusType, cType string, state protosorderer.ConsensusType_State) {
   933  	Expect(value.Type).To(Equal(cType))
   934  	Expect(value.State).To(Equal(state))
   935  }
   936  
   937  func extractOrdererConsensusType(config *common.Config) (*common.Config, *protosorderer.ConsensusType, error) {
   938  	updatedConfig := proto.Clone(config).(*common.Config)
   939  	consensusTypeConfigValue := updatedConfig.ChannelGroup.Groups["Orderer"].Values["ConsensusType"]
   940  	consensusTypeValue := new(protosorderer.ConsensusType)
   941  	err := proto.Unmarshal(consensusTypeConfigValue.Value, consensusTypeValue)
   942  	return updatedConfig, consensusTypeValue, err
   943  }
   944  
   945  func updateConfigWithConsensusType(
   946  	consensusType string,
   947  	consensusMetadata []byte,
   948  	migState protosorderer.ConsensusType_State,
   949  	updatedConfig *common.Config,
   950  	consensusTypeValue *protosorderer.ConsensusType,
   951  ) {
   952  	consensusTypeValue.Type = consensusType
   953  	consensusTypeValue.Metadata = consensusMetadata
   954  	consensusTypeValue.State = migState
   955  	updatedConfig.ChannelGroup.Groups["Orderer"].Values["ConsensusType"] = &common.ConfigValue{
   956  		ModPolicy: "Admins",
   957  		Value:     protoutil.MarshalOrPanic(consensusTypeValue),
   958  	}
   959  }
   960  
   961  func updateConfigWithBatchTimeout(updatedConfig *common.Config) {
   962  	batchTimeoutConfigValue := updatedConfig.ChannelGroup.Groups["Orderer"].Values["BatchTimeout"]
   963  	batchTimeoutValue := new(protosorderer.BatchTimeout)
   964  	err := proto.Unmarshal(batchTimeoutConfigValue.Value, batchTimeoutValue)
   965  	Expect(err).NotTo(HaveOccurred())
   966  	toDur, err := time.ParseDuration(batchTimeoutValue.Timeout)
   967  	Expect(err).NotTo(HaveOccurred())
   968  	toDur = toDur + time.Duration(100000000)
   969  	batchTimeoutValue.Timeout = toDur.String()
   970  	By(fmt.Sprintf("Increasing BatchTimeout to %s", batchTimeoutValue.Timeout))
   971  	updatedConfig.ChannelGroup.Groups["Orderer"].Values["BatchTimeout"] = &common.ConfigValue{
   972  		ModPolicy: "Admins",
   973  		Value:     protoutil.MarshalOrPanic(batchTimeoutValue),
   974  	}
   975  }
   976  
   977  func kafka2RaftMultiChannel() *nwo.Config {
   978  	config := nwo.BasicKafka()
   979  	config.Channels = []*nwo.Channel{
   980  		{Name: "testchannel1", Profile: "TwoOrgsChannel"},
   981  		{Name: "testchannel2", Profile: "TwoOrgsChannel"},
   982  		{Name: "testchannel3", Profile: "TwoOrgsChannel"},
   983  	}
   984  
   985  	for _, peer := range config.Peers {
   986  		peer.Channels = []*nwo.PeerChannel{
   987  			{Name: "testchannel1", Anchor: true},
   988  			{Name: "testchannel2", Anchor: true},
   989  			{Name: "testchannel3", Anchor: true},
   990  		}
   991  	}
   992  	return config
   993  }
   994  
   995  func solo2RaftMultiChannel() *nwo.Config {
   996  	config := nwo.BasicSolo()
   997  	config.Channels = []*nwo.Channel{
   998  		{Name: "testchannel1", Profile: "TwoOrgsChannel"},
   999  		{Name: "testchannel2", Profile: "TwoOrgsChannel"},
  1000  	}
  1001  
  1002  	for _, peer := range config.Peers {
  1003  		peer.Channels = []*nwo.PeerChannel{
  1004  			{Name: "testchannel1", Anchor: true},
  1005  			{Name: "testchannel2", Anchor: true},
  1006  		}
  1007  	}
  1008  	return config
  1009  }
  1010  
  1011  func kafka2RaftMultiNode() *nwo.Config {
  1012  	config := nwo.BasicKafka()
  1013  	config.Orderers = []*nwo.Orderer{
  1014  		{Name: "orderer1", Organization: "OrdererOrg"},
  1015  		{Name: "orderer2", Organization: "OrdererOrg"},
  1016  		{Name: "orderer3", Organization: "OrdererOrg"},
  1017  	}
  1018  
  1019  	config.Profiles = []*nwo.Profile{{
  1020  		Name:     "TwoOrgsOrdererGenesis",
  1021  		Orderers: []string{"orderer1", "orderer2", "orderer3"},
  1022  	}, {
  1023  		Name:          "TwoOrgsChannel",
  1024  		Consortium:    "SampleConsortium",
  1025  		Organizations: []string{"Org1", "Org2"},
  1026  	}}
  1027  
  1028  	config.Channels = []*nwo.Channel{
  1029  		{Name: "testchannel1", Profile: "TwoOrgsChannel"},
  1030  		{Name: "testchannel2", Profile: "TwoOrgsChannel"},
  1031  		{Name: "testchannel3", Profile: "TwoOrgsChannel"},
  1032  	}
  1033  
  1034  	for _, peer := range config.Peers {
  1035  		peer.Channels = []*nwo.PeerChannel{
  1036  			{Name: "testchannel1", Anchor: true},
  1037  			{Name: "testchannel2", Anchor: true},
  1038  			{Name: "testchannel3", Anchor: true},
  1039  		}
  1040  	}
  1041  	return config
  1042  }
  1043  
  1044  func prepareRaftMetadata(network *nwo.Network) []byte {
  1045  	var consenters []*protosraft.Consenter
  1046  	for _, o := range network.Orderers {
  1047  		fullTlsPath := network.OrdererLocalTLSDir(o)
  1048  		certBytes, err := ioutil.ReadFile(filepath.Join(fullTlsPath, "server.crt"))
  1049  		Expect(err).NotTo(HaveOccurred())
  1050  		port := network.OrdererPort(o, nwo.ClusterPort)
  1051  
  1052  		consenter := &protosraft.Consenter{
  1053  			ClientTlsCert: certBytes,
  1054  			ServerTlsCert: certBytes,
  1055  			Host:          "127.0.0.1",
  1056  			Port:          uint32(port),
  1057  		}
  1058  		consenters = append(consenters, consenter)
  1059  	}
  1060  
  1061  	raftMetadata := &protosraft.ConfigMetadata{
  1062  		Consenters: consenters,
  1063  		Options: &protosraft.Options{
  1064  			TickInterval:         "500ms",
  1065  			ElectionTick:         10,
  1066  			HeartbeatTick:        1,
  1067  			MaxInflightBlocks:    5,
  1068  			SnapshotIntervalSize: 10 * 1024 * 1024,
  1069  		},
  1070  	}
  1071  
  1072  	raftMetadataBytes := protoutil.MarshalOrPanic(raftMetadata)
  1073  
  1074  	return raftMetadataBytes
  1075  }
  1076  
  1077  func waitForBlockReceptionByPeer(peer *nwo.Peer, network *nwo.Network, channelName string, blockSeq uint64) {
  1078  	Eventually(func() bool {
  1079  		blockNumFromPeer := nwo.CurrentConfigBlockNumber(network, peer, nil, channelName)
  1080  		return blockNumFromPeer == blockSeq
  1081  	}, network.EventuallyTimeout, time.Second).Should(BeTrue())
  1082  }
  1083  
  1084  func checkPeerDeliverRequest(o *nwo.Orderer, submitter *nwo.Peer, network *nwo.Network, channelName string) error {
  1085  	c := commands.ChannelFetch{
  1086  		ChannelID:  channelName,
  1087  		Block:      "newest",
  1088  		OutputFile: "/dev/null",
  1089  		Orderer:    network.OrdererAddress(o, nwo.ListenPort),
  1090  	}
  1091  
  1092  	sess, err := network.PeerUserSession(submitter, "User1", c)
  1093  	Expect(err).NotTo(HaveOccurred())
  1094  	Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit())
  1095  	sessErr := string(sess.Err.Contents())
  1096  	sessExitCode := sess.ExitCode()
  1097  	if sessExitCode != 0 && strings.Contains(sessErr, "FORBIDDEN") {
  1098  		return errors.New("FORBIDDEN")
  1099  	}
  1100  	if sessExitCode == 0 && strings.Contains(sessErr, "Received block: ") {
  1101  		return nil
  1102  	}
  1103  
  1104  	return fmt.Errorf("Unexpected result: ExitCode=%d, Err=%s", sessExitCode, sessErr)
  1105  }
  1106  
  1107  func updateOrdererConfigFailed(n *nwo.Network, orderer *nwo.Orderer, channel string, current, updated *common.Config, peer *nwo.Peer, additionalSigners ...*nwo.Orderer) {
  1108  	sess := nwo.UpdateOrdererConfigSession(n, orderer, channel, current, updated, peer, additionalSigners...)
  1109  	Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(1))
  1110  	Expect(sess.Err).NotTo(gbytes.Say("Successfully submitted channel update"))
  1111  }
  1112  
  1113  func prepareTransition(
  1114  	network *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel string, // Auxiliary
  1115  	fromConsensusType string, fromMigState protosorderer.ConsensusType_State, // From
  1116  	toConsensusType string, toConsensusMetadata []byte, toMigState protosorderer.ConsensusType_State, // To
  1117  ) (current, updated *common.Config) {
  1118  	current = nwo.GetConfig(network, peer, orderer, channel)
  1119  	updated, consensusTypeValue, err := extractOrdererConsensusType(current)
  1120  	Expect(err).NotTo(HaveOccurred())
  1121  	validateConsensusTypeValue(consensusTypeValue, fromConsensusType, fromMigState)
  1122  	updateConfigWithConsensusType(toConsensusType, toConsensusMetadata, toMigState, updated, consensusTypeValue)
  1123  	return current, updated
  1124  }
  1125  
  1126  func assertTransitionFailed(
  1127  	network *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel string, // Auxiliary
  1128  	fromConsensusType string, fromMigState protosorderer.ConsensusType_State, // From
  1129  	toConsensusType string, toConsensusMetadata []byte, toMigState protosorderer.ConsensusType_State, // To
  1130  ) {
  1131  	current, updated := prepareTransition(
  1132  		network, peer, orderer, channel,
  1133  		fromConsensusType, fromMigState,
  1134  		toConsensusType, toConsensusMetadata, toMigState)
  1135  	updateOrdererConfigFailed(network, orderer, channel, current, updated, peer, orderer)
  1136  }