github.com/osdi23p228/fabric@v0.0.0-20221218062954-77808885f5db/integration/raft/cft_test.go (about)

     1  /*
     2  Copyright IBM Corp All Rights Reserved.
     3  
     4  SPDX-License-Identifier: Apache-2.0
     5  */
     6  
     7  package raft
     8  
     9  import (
    10  	"crypto/ecdsa"
    11  	"crypto/rand"
    12  	"crypto/x509"
    13  	"encoding/pem"
    14  	"fmt"
    15  	"io/ioutil"
    16  	"os"
    17  	"path"
    18  	"path/filepath"
    19  	"strconv"
    20  	"strings"
    21  	"sync"
    22  	"syscall"
    23  	"time"
    24  
    25  	docker "github.com/fsouza/go-dockerclient"
    26  	"github.com/golang/protobuf/proto"
    27  	conftx "github.com/hyperledger/fabric-config/configtx"
    28  	"github.com/hyperledger/fabric-config/configtx/orderer"
    29  	"github.com/hyperledger/fabric-protos-go/common"
    30  	"github.com/hyperledger/fabric-protos-go/msp"
    31  	"github.com/osdi23p228/fabric/cmd/common/signer"
    32  	"github.com/osdi23p228/fabric/common/configtx"
    33  	"github.com/osdi23p228/fabric/common/util"
    34  	intconftx "github.com/osdi23p228/fabric/integration/configtx"
    35  	"github.com/osdi23p228/fabric/integration/nwo"
    36  	"github.com/osdi23p228/fabric/integration/nwo/commands"
    37  	"github.com/osdi23p228/fabric/protoutil"
    38  	. "github.com/onsi/ginkgo"
    39  	. "github.com/onsi/gomega"
    40  	"github.com/onsi/gomega/gbytes"
    41  	"github.com/onsi/gomega/gexec"
    42  	"github.com/tedsuo/ifrit"
    43  	"github.com/tedsuo/ifrit/ginkgomon"
    44  	"github.com/tedsuo/ifrit/grouper"
    45  )
    46  
    47  var _ = Describe("EndToEnd Crash Fault Tolerance", func() {
    48  	var (
    49  		testDir string
    50  		client  *docker.Client
    51  		network *nwo.Network
    52  		peer    *nwo.Peer
    53  
    54  		ordererProc, o1Proc, o2Proc, o3Proc ifrit.Process
    55  	)
    56  
    57  	BeforeEach(func() {
    58  		var err error
    59  		testDir, err = ioutil.TempDir("", "e2e")
    60  		Expect(err).NotTo(HaveOccurred())
    61  
    62  		client, err = docker.NewClientFromEnv()
    63  		Expect(err).NotTo(HaveOccurred())
    64  	})
    65  
    66  	AfterEach(func() {
    67  		for _, oProc := range []ifrit.Process{o1Proc, o2Proc, o3Proc} {
    68  			if oProc != nil {
    69  				oProc.Signal(syscall.SIGTERM)
    70  				Eventually(oProc.Wait(), network.EventuallyTimeout).Should(Receive())
    71  			}
    72  		}
    73  
    74  		if ordererProc != nil {
    75  			ordererProc.Signal(syscall.SIGTERM)
    76  			Eventually(ordererProc.Wait(), network.EventuallyTimeout).Should(Receive())
    77  		}
    78  
    79  		if network != nil {
    80  			network.Cleanup()
    81  		}
    82  		os.RemoveAll(testDir)
    83  	})
    84  
    85  	When("orderer stops and restarts", func() {
    86  		It("keeps network up and running", func() {
    87  			network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, StartPort(), components)
    88  
    89  			o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3")
    90  			peer = network.Peer("Org1", "peer0")
    91  
    92  			network.GenerateConfigTree()
    93  			network.Bootstrap()
    94  
    95  			o1Runner := network.OrdererRunner(o1)
    96  			// Enable debug log for orderer2 so we could assert its content later
    97  			o2Runner := network.OrdererRunner(o2, "FABRIC_LOGGING_SPEC=orderer.consensus.etcdraft=debug:info")
    98  			o3Runner := network.OrdererRunner(o3)
    99  			orderers := grouper.Members{
   100  				{Name: o2.ID(), Runner: o2Runner},
   101  				{Name: o3.ID(), Runner: o3Runner},
   102  			}
   103  			ordererGroup := grouper.NewParallel(syscall.SIGTERM, orderers)
   104  
   105  			o1Proc = ifrit.Invoke(o1Runner)
   106  			ordererProc = ifrit.Invoke(ordererGroup)
   107  			Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   108  			Eventually(ordererProc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   109  
   110  			findLeader([]*ginkgomon.Runner{o1Runner})
   111  
   112  			By("performing operation with orderer1")
   113  			env := CreateBroadcastEnvelope(network, o1, network.SystemChannel.Name, []byte("foo"))
   114  			resp, err := nwo.Broadcast(network, o1, env)
   115  			Expect(err).NotTo(HaveOccurred())
   116  			Expect(resp.Status).To(Equal(common.Status_SUCCESS))
   117  
   118  			block := FetchBlock(network, o1, 1, network.SystemChannel.Name)
   119  			Expect(block).NotTo(BeNil())
   120  
   121  			By("killing orderer1")
   122  			o1Proc.Signal(syscall.SIGKILL)
   123  			Eventually(o1Proc.Wait(), network.EventuallyTimeout).Should(Receive(MatchError("exit status 137")))
   124  
   125  			By("observing active nodes to shrink")
   126  			Eventually(o2Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("Current active nodes in cluster are: \\[2 3\\]"))
   127  
   128  			By("broadcasting envelope to running orderer")
   129  			resp, err = nwo.Broadcast(network, o2, env)
   130  			Expect(err).NotTo(HaveOccurred())
   131  			Expect(resp.Status).To(Equal(common.Status_SUCCESS))
   132  
   133  			block = FetchBlock(network, o2, 2, network.SystemChannel.Name)
   134  			Expect(block).NotTo(BeNil())
   135  
   136  			By("restarting orderer1")
   137  			o1Runner = network.OrdererRunner(o1)
   138  			o1Proc = ifrit.Invoke(o1Runner)
   139  			Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   140  			findLeader([]*ginkgomon.Runner{o1Runner})
   141  
   142  			By("broadcasting envelope to restarted orderer")
   143  			resp, err = nwo.Broadcast(network, o1, env)
   144  			Expect(err).NotTo(HaveOccurred())
   145  			Expect(resp.Status).To(Equal(common.Status_SUCCESS))
   146  
   147  			blko1 := FetchBlock(network, o1, 3, network.SystemChannel.Name)
   148  			blko2 := FetchBlock(network, o2, 3, network.SystemChannel.Name)
   149  
   150  			Expect(blko1.Header.DataHash).To(Equal(blko2.Header.DataHash))
   151  		})
   152  	})
   153  
   154  	When("an orderer is behind the latest snapshot on leader", func() {
   155  		It("catches up using the block stored in snapshot", func() {
   156  			// Steps:
   157  			// - start o2 & o3
   158  			// - send several transactions so snapshot is created
   159  			// - kill o2 & o3, so that entries prior to snapshot are not in memory upon restart
   160  			// - start o1 & o2
   161  			// - assert that o1 can catch up with o2 using snapshot
   162  			network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, StartPort(), components)
   163  			o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3")
   164  			peer = network.Peer("Org1", "peer0")
   165  
   166  			network.GenerateConfigTree()
   167  			network.Bootstrap()
   168  
   169  			orderers := grouper.Members{
   170  				{Name: o2.ID(), Runner: network.OrdererRunner(o2)},
   171  				{Name: o3.ID(), Runner: network.OrdererRunner(o3)},
   172  			}
   173  			ordererGroup := grouper.NewParallel(syscall.SIGTERM, orderers)
   174  
   175  			By("Starting 2/3 of cluster")
   176  			ordererProc = ifrit.Invoke(ordererGroup)
   177  			Eventually(ordererProc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   178  
   179  			By("Creating testchannel")
   180  			channelID := "testchannel"
   181  			network.CreateChannel(channelID, o2, peer)
   182  
   183  			By("Submitting several transactions to trigger snapshot")
   184  			o2SnapDir := path.Join(network.RootDir, "orderers", o2.ID(), "etcdraft", "snapshot")
   185  
   186  			env := CreateBroadcastEnvelope(network, o2, channelID, make([]byte, 2000))
   187  			for i := 1; i <= 4; i++ { // 4 < MaxSnapshotFiles(5), so that no snapshot is pruned
   188  				// Note that MaxMessageCount is 1 be default, so every tx results in a new block
   189  				resp, err := nwo.Broadcast(network, o2, env)
   190  				Expect(err).NotTo(HaveOccurred())
   191  				Expect(resp.Status).To(Equal(common.Status_SUCCESS))
   192  
   193  				// Assert that new snapshot file is created before broadcasting next tx,
   194  				// so that number of snapshots is deterministic. Otherwise, it is not
   195  				// guaranteed that every block triggers a snapshot file being created,
   196  				// due to the mechanism to prevent excessive snapshotting.
   197  				Eventually(func() int {
   198  					files, err := ioutil.ReadDir(path.Join(o2SnapDir, channelID))
   199  					Expect(err).NotTo(HaveOccurred())
   200  					return len(files)
   201  				}, network.EventuallyTimeout).Should(Equal(i)) // snapshot interval is 1 KB, every block triggers snapshot
   202  			}
   203  
   204  			By("Killing orderers so they don't have blocks prior to latest snapshot in the memory")
   205  			ordererProc.Signal(syscall.SIGKILL)
   206  			Eventually(ordererProc.Wait(), network.EventuallyTimeout).Should(Receive())
   207  
   208  			By("Starting lagged orderer and one of up-to-date orderers")
   209  			orderers = grouper.Members{
   210  				{Name: o1.ID(), Runner: network.OrdererRunner(o1)},
   211  				{Name: o2.ID(), Runner: network.OrdererRunner(o2)},
   212  			}
   213  			ordererGroup = grouper.NewParallel(syscall.SIGTERM, orderers)
   214  			ordererProc = ifrit.Invoke(ordererGroup)
   215  			Eventually(ordererProc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   216  
   217  			o1SnapDir := path.Join(network.RootDir, "orderers", o1.ID(), "etcdraft", "snapshot")
   218  
   219  			By("Asserting that orderer1 has snapshot dir for both system and application channel")
   220  			Eventually(func() int {
   221  				files, err := ioutil.ReadDir(o1SnapDir)
   222  				Expect(err).NotTo(HaveOccurred())
   223  				return len(files)
   224  			}, network.EventuallyTimeout).Should(Equal(2))
   225  
   226  			By("Asserting that orderer1 receives and persists snapshot")
   227  			Eventually(func() int {
   228  				files, err := ioutil.ReadDir(path.Join(o1SnapDir, channelID))
   229  				Expect(err).NotTo(HaveOccurred())
   230  				return len(files)
   231  			}, network.EventuallyTimeout).Should(Equal(1))
   232  
   233  			By("Asserting cluster is still functional")
   234  			env = CreateBroadcastEnvelope(network, o1, channelID, make([]byte, 1000))
   235  			resp, err := nwo.Broadcast(network, o1, env)
   236  			Expect(err).NotTo(HaveOccurred())
   237  			Expect(resp.Status).To(Equal(common.Status_SUCCESS))
   238  
   239  			blko1 := FetchBlock(network, o1, 5, channelID)
   240  			blko2 := FetchBlock(network, o2, 5, channelID)
   241  
   242  			Expect(blko1.Header.DataHash).To(Equal(blko2.Header.DataHash))
   243  		})
   244  	})
   245  
   246  	When("The leader dies", func() {
   247  		It("Elects a new leader", func() {
   248  			network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, StartPort(), components)
   249  
   250  			o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3")
   251  
   252  			network.GenerateConfigTree()
   253  			network.Bootstrap()
   254  
   255  			By("Running the orderer nodes")
   256  			o1Runner := network.OrdererRunner(o1)
   257  			o2Runner := network.OrdererRunner(o2)
   258  			o3Runner := network.OrdererRunner(o3)
   259  
   260  			o1Proc = ifrit.Invoke(o1Runner)
   261  			o2Proc = ifrit.Invoke(o2Runner)
   262  			o3Proc = ifrit.Invoke(o3Runner)
   263  
   264  			Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   265  			Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   266  			Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   267  
   268  			By("Waiting for them to elect a leader")
   269  			ordererProcesses := []ifrit.Process{o1Proc, o2Proc, o3Proc}
   270  			remainingAliveRunners := []*ginkgomon.Runner{o1Runner, o2Runner, o3Runner}
   271  			leader := findLeader(remainingAliveRunners)
   272  
   273  			leaderIndex := leader - 1
   274  			By(fmt.Sprintf("Killing the leader (%d)", leader))
   275  			ordererProcesses[leaderIndex].Signal(syscall.SIGTERM)
   276  			By("Waiting for it to die")
   277  			Eventually(ordererProcesses[leaderIndex].Wait(), network.EventuallyTimeout).Should(Receive())
   278  
   279  			// Remove the leader from the orderer runners
   280  			remainingAliveRunners = append(remainingAliveRunners[:leaderIndex], remainingAliveRunners[leaderIndex+1:]...)
   281  
   282  			By("Waiting for a new leader to be elected")
   283  			leader = findLeader(remainingAliveRunners)
   284  			By(fmt.Sprintf("Orderer %d took over as a leader", leader))
   285  		})
   286  	})
   287  
   288  	When("Leader cannot reach quorum", func() {
   289  		It("Steps down", func() {
   290  			network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, StartPort(), components)
   291  
   292  			o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3")
   293  			orderers := []*nwo.Orderer{o1, o2, o3}
   294  			peer = network.Peer("Org1", "peer0")
   295  			network.GenerateConfigTree()
   296  			network.Bootstrap()
   297  
   298  			By("Running the orderer nodes")
   299  			o1Runner := network.OrdererRunner(o1)
   300  			o2Runner := network.OrdererRunner(o2)
   301  			o3Runner := network.OrdererRunner(o3)
   302  			oRunners := []*ginkgomon.Runner{o1Runner, o2Runner, o3Runner}
   303  
   304  			o1Proc = ifrit.Invoke(o1Runner)
   305  			o2Proc = ifrit.Invoke(o2Runner)
   306  			o3Proc = ifrit.Invoke(o3Runner)
   307  
   308  			Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   309  			Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   310  			Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   311  
   312  			By("Waiting for them to elect a leader")
   313  			ordererProcesses := []ifrit.Process{o1Proc, o2Proc, o3Proc}
   314  			remainingAliveRunners := []*ginkgomon.Runner{o1Runner, o2Runner, o3Runner}
   315  			leaderID := findLeader(remainingAliveRunners)
   316  			leaderIndex := leaderID - 1
   317  			leader := orderers[leaderIndex]
   318  
   319  			followerIndices := func() []int {
   320  				var f []int
   321  				for i := range ordererProcesses {
   322  					if leaderIndex != i {
   323  						f = append(f, i)
   324  					}
   325  				}
   326  
   327  				return f
   328  			}()
   329  
   330  			By(fmt.Sprintf("Killing two followers (%d and %d)", followerIndices[0]+1, followerIndices[1]+1))
   331  			ordererProcesses[followerIndices[0]].Signal(syscall.SIGTERM)
   332  			ordererProcesses[followerIndices[1]].Signal(syscall.SIGTERM)
   333  
   334  			By("Waiting for followers to die")
   335  			// This significantly slows test (~10s). However, reducing ElectionTimeout
   336  			// would introduce some flakes when disk write is exceptionally slow.
   337  			Eventually(ordererProcesses[followerIndices[0]].Wait(), network.EventuallyTimeout).Should(Receive())
   338  			Eventually(ordererProcesses[followerIndices[1]].Wait(), network.EventuallyTimeout).Should(Receive())
   339  
   340  			By("Waiting for leader to step down")
   341  			Eventually(oRunners[leaderIndex].Err(), time.Minute, time.Second).Should(gbytes.Say(fmt.Sprintf("%d stepped down to follower since quorum is not active", leaderID)))
   342  
   343  			By("Submitting tx to leader")
   344  			// This should fail because current leader steps down
   345  			// and there is no leader at this point of time
   346  			env := CreateBroadcastEnvelope(network, leader, network.SystemChannel.Name, []byte("foo"))
   347  			resp, err := nwo.Broadcast(network, leader, env)
   348  			Expect(err).NotTo(HaveOccurred())
   349  			Expect(resp.Status).To(Equal(common.Status_SERVICE_UNAVAILABLE))
   350  		})
   351  	})
   352  
   353  	When("orderer TLS certificates expire", func() {
   354  		It("is still possible to recover", func() {
   355  			network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, StartPort(), components)
   356  
   357  			o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3")
   358  			peer = network.Peer("Org1", "peer0")
   359  
   360  			network.GenerateConfigTree()
   361  			network.Bootstrap()
   362  
   363  			ordererDomain := network.Organization(o1.Organization).Domain
   364  			ordererTLSCAKeyPath := filepath.Join(network.RootDir, "crypto", "ordererOrganizations",
   365  				ordererDomain, "tlsca", "priv_sk")
   366  
   367  			ordererTLSCAKey, err := ioutil.ReadFile(ordererTLSCAKeyPath)
   368  			Expect(err).NotTo(HaveOccurred())
   369  
   370  			ordererTLSCACertPath := filepath.Join(network.RootDir, "crypto", "ordererOrganizations",
   371  				ordererDomain, "tlsca", fmt.Sprintf("tlsca.%s-cert.pem", ordererDomain))
   372  			ordererTLSCACert, err := ioutil.ReadFile(ordererTLSCACertPath)
   373  			Expect(err).NotTo(HaveOccurred())
   374  
   375  			serverTLSCerts := make(map[string][]byte)
   376  			for _, orderer := range []*nwo.Orderer{o1, o2, o3} {
   377  				tlsCertPath := filepath.Join(network.OrdererLocalTLSDir(orderer), "server.crt")
   378  				serverTLSCerts[tlsCertPath], err = ioutil.ReadFile(tlsCertPath)
   379  				Expect(err).NotTo(HaveOccurred())
   380  			}
   381  
   382  			By("Expiring orderer TLS certificates")
   383  			for filePath, certPEM := range serverTLSCerts {
   384  				expiredCert, earlyMadeCACert := expireCertificate(certPEM, ordererTLSCACert, ordererTLSCAKey, time.Now())
   385  				err = ioutil.WriteFile(filePath, expiredCert, 0600)
   386  				Expect(err).NotTo(HaveOccurred())
   387  
   388  				err = ioutil.WriteFile(ordererTLSCACertPath, earlyMadeCACert, 0600)
   389  				Expect(err).NotTo(HaveOccurred())
   390  			}
   391  
   392  			By("Regenerating config")
   393  			sess, err := network.ConfigTxGen(commands.OutputBlock{
   394  				ChannelID:   network.SystemChannel.Name,
   395  				Profile:     network.SystemChannel.Profile,
   396  				ConfigPath:  network.RootDir,
   397  				OutputBlock: network.OutputBlockPath(network.SystemChannel.Name),
   398  			})
   399  			Expect(err).NotTo(HaveOccurred())
   400  			Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(0))
   401  
   402  			By("Running the orderer nodes")
   403  			o1Runner := network.OrdererRunner(o1)
   404  			o2Runner := network.OrdererRunner(o2)
   405  			o3Runner := network.OrdererRunner(o3)
   406  
   407  			o1Proc = ifrit.Invoke(o1Runner)
   408  			o2Proc = ifrit.Invoke(o2Runner)
   409  			o3Proc = ifrit.Invoke(o3Runner)
   410  
   411  			Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   412  			Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   413  			Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   414  
   415  			By("Waiting for TLS handshakes to fail")
   416  			Eventually(o1Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("tls: bad certificate"))
   417  			Eventually(o2Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("tls: bad certificate"))
   418  			Eventually(o3Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("tls: bad certificate"))
   419  
   420  			By("Killing orderers")
   421  			o1Proc.Signal(syscall.SIGTERM)
   422  			o2Proc.Signal(syscall.SIGTERM)
   423  			o3Proc.Signal(syscall.SIGTERM)
   424  			Eventually(o1Proc.Wait(), network.EventuallyTimeout).Should(Receive())
   425  			Eventually(o2Proc.Wait(), network.EventuallyTimeout).Should(Receive())
   426  			Eventually(o3Proc.Wait(), network.EventuallyTimeout).Should(Receive())
   427  
   428  			o1Runner = network.OrdererRunner(o1)
   429  			o2Runner = network.OrdererRunner(o2)
   430  			o3Runner = network.OrdererRunner(o3)
   431  
   432  			By("Launching orderers with a clustered timeshift")
   433  			orderers := []*nwo.Orderer{o1, o2, o3}
   434  			for _, orderer := range orderers {
   435  				ordererConfig := network.ReadOrdererConfig(orderer)
   436  				ordererConfig.General.Cluster.TLSHandshakeTimeShift = 5 * time.Minute
   437  				network.WriteOrdererConfig(orderer, ordererConfig)
   438  			}
   439  
   440  			o1Proc = ifrit.Invoke(o1Runner)
   441  			o2Proc = ifrit.Invoke(o2Runner)
   442  			o3Proc = ifrit.Invoke(o3Runner)
   443  
   444  			Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   445  			Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   446  			Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   447  
   448  			By("Waiting for a leader to be elected")
   449  			findLeader([]*ginkgomon.Runner{o1Runner, o2Runner, o3Runner})
   450  
   451  			By("Killing orderers")
   452  			o1Proc.Signal(syscall.SIGTERM)
   453  			o2Proc.Signal(syscall.SIGTERM)
   454  			o3Proc.Signal(syscall.SIGTERM)
   455  			Eventually(o1Proc.Wait(), network.EventuallyTimeout).Should(Receive())
   456  			Eventually(o2Proc.Wait(), network.EventuallyTimeout).Should(Receive())
   457  			Eventually(o3Proc.Wait(), network.EventuallyTimeout).Should(Receive())
   458  
   459  			o1Runner = network.OrdererRunner(o1)
   460  			o2Runner = network.OrdererRunner(o2)
   461  			o3Runner = network.OrdererRunner(o3)
   462  
   463  			By("Launching orderers again without a general timeshift re-using the cluster port")
   464  			for _, orderer := range orderers {
   465  				ordererConfig := network.ReadOrdererConfig(orderer)
   466  				ordererConfig.General.ListenPort = ordererConfig.General.Cluster.ListenPort
   467  				ordererConfig.General.TLS.Certificate = ordererConfig.General.Cluster.ServerCertificate
   468  				ordererConfig.General.TLS.PrivateKey = ordererConfig.General.Cluster.ServerPrivateKey
   469  				ordererConfig.General.Cluster.TLSHandshakeTimeShift = 0
   470  				ordererConfig.General.Cluster.ListenPort = 0
   471  				ordererConfig.General.Cluster.ListenAddress = ""
   472  				ordererConfig.General.Cluster.ServerCertificate = ""
   473  				ordererConfig.General.Cluster.ServerPrivateKey = ""
   474  				ordererConfig.General.Cluster.ClientCertificate = ""
   475  				ordererConfig.General.Cluster.ClientPrivateKey = ""
   476  				network.WriteOrdererConfig(orderer, ordererConfig)
   477  			}
   478  
   479  			o1Proc = ifrit.Invoke(o1Runner)
   480  			o2Proc = ifrit.Invoke(o2Runner)
   481  			o3Proc = ifrit.Invoke(o3Runner)
   482  
   483  			Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   484  			Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   485  			Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   486  
   487  			By("Waiting for TLS handshakes to fail")
   488  			Eventually(o1Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("tls: bad certificate"))
   489  			Eventually(o2Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("tls: bad certificate"))
   490  			Eventually(o3Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("tls: bad certificate"))
   491  
   492  			By("Killing orderers")
   493  			o1Proc.Signal(syscall.SIGTERM)
   494  			o2Proc.Signal(syscall.SIGTERM)
   495  			o3Proc.Signal(syscall.SIGTERM)
   496  			Eventually(o1Proc.Wait(), network.EventuallyTimeout).Should(Receive())
   497  			Eventually(o2Proc.Wait(), network.EventuallyTimeout).Should(Receive())
   498  			Eventually(o3Proc.Wait(), network.EventuallyTimeout).Should(Receive())
   499  
   500  			o1Runner = network.OrdererRunner(o1)
   501  			o2Runner = network.OrdererRunner(o2)
   502  			o3Runner = network.OrdererRunner(o3)
   503  
   504  			By("Launching orderers again with a general timeshift re-using the cluster port")
   505  			for _, orderer := range orderers {
   506  				ordererConfig := network.ReadOrdererConfig(orderer)
   507  				ordererConfig.General.TLS.TLSHandshakeTimeShift = 5 * time.Minute
   508  				network.WriteOrdererConfig(orderer, ordererConfig)
   509  			}
   510  
   511  			o1Proc = ifrit.Invoke(o1Runner)
   512  			o2Proc = ifrit.Invoke(o2Runner)
   513  			o3Proc = ifrit.Invoke(o3Runner)
   514  
   515  			Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   516  			Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   517  			Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   518  
   519  			By("Waiting for a leader to be elected")
   520  			findLeader([]*ginkgomon.Runner{o1Runner, o2Runner, o3Runner})
   521  
   522  			By("submitting config updates to orderers with expired TLS certs to replace the expired certs")
   523  			timeShift := 5 * time.Minute
   524  			for _, o := range orderers {
   525  				channelConfig := fetchConfig(network, peer, o, nwo.ClusterPort, network.SystemChannel.Name, timeShift)
   526  				c := conftx.New(channelConfig)
   527  				err = c.Orderer().RemoveConsenter(consenterChannelConfig(network, o))
   528  				Expect(err).NotTo(HaveOccurred())
   529  
   530  				By("renewing the orderer TLS certificates for " + o.Name)
   531  				renewOrdererCertificates(network, o)
   532  				err = c.Orderer().AddConsenter(consenterChannelConfig(network, o))
   533  				Expect(err).NotTo(HaveOccurred())
   534  
   535  				By("updating the config for " + o.Name)
   536  				updateOrdererConfig(network, o, nwo.ClusterPort, network.SystemChannel.Name, timeShift, c.OriginalConfig(), c.UpdatedConfig(), peer)
   537  			}
   538  
   539  			By("Killing orderers")
   540  			o1Proc.Signal(syscall.SIGTERM)
   541  			o2Proc.Signal(syscall.SIGTERM)
   542  			o3Proc.Signal(syscall.SIGTERM)
   543  			Eventually(o1Proc.Wait(), network.EventuallyTimeout).Should(Receive())
   544  			Eventually(o2Proc.Wait(), network.EventuallyTimeout).Should(Receive())
   545  			Eventually(o3Proc.Wait(), network.EventuallyTimeout).Should(Receive())
   546  
   547  			o1Runner = network.OrdererRunner(o1)
   548  			o2Runner = network.OrdererRunner(o2)
   549  			o3Runner = network.OrdererRunner(o3)
   550  
   551  			By("Launching orderers again without a general timeshift")
   552  			for _, o := range orderers {
   553  				ordererConfig := network.ReadOrdererConfig(o)
   554  				ordererConfig.General.TLS.TLSHandshakeTimeShift = 0
   555  				network.WriteOrdererConfig(o, ordererConfig)
   556  			}
   557  
   558  			o1Proc = ifrit.Invoke(o1Runner)
   559  			o2Proc = ifrit.Invoke(o2Runner)
   560  			o3Proc = ifrit.Invoke(o3Runner)
   561  
   562  			Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   563  			Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   564  			Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   565  
   566  			By("Waiting for a leader to be elected")
   567  			findLeader([]*ginkgomon.Runner{o1Runner, o2Runner, o3Runner})
   568  		})
   569  
   570  		It("disregards certificate renewal if only the validity period changed", func() {
   571  			config := nwo.MultiNodeEtcdRaft()
   572  			config.Channels = append(config.Channels, &nwo.Channel{Name: "foo", Profile: "TwoOrgsChannel"})
   573  			config.Channels = append(config.Channels, &nwo.Channel{Name: "bar", Profile: "TwoOrgsChannel"})
   574  			network = nwo.New(config, testDir, client, StartPort(), components)
   575  
   576  			network.GenerateConfigTree()
   577  			network.Bootstrap()
   578  
   579  			peer = network.Peer("Org1", "peer0")
   580  
   581  			o1 := network.Orderer("orderer1")
   582  			o2 := network.Orderer("orderer2")
   583  			o3 := network.Orderer("orderer3")
   584  
   585  			orderers := []*nwo.Orderer{o1, o2, o3}
   586  
   587  			o1Runner := network.OrdererRunner(o1)
   588  			o2Runner := network.OrdererRunner(o2)
   589  			o3Runner := network.OrdererRunner(o3)
   590  			ordererRunners := []*ginkgomon.Runner{o1Runner, o2Runner, o3Runner}
   591  
   592  			o1Proc = ifrit.Invoke(o1Runner)
   593  			o2Proc = ifrit.Invoke(o2Runner)
   594  			o3Proc = ifrit.Invoke(o3Runner)
   595  			ordererProcesses := []ifrit.Process{o1Proc, o2Proc, o3Proc}
   596  
   597  			Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   598  			Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   599  			Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
   600  
   601  			By("Waiting for them to elect a leader")
   602  			findLeader(ordererRunners)
   603  
   604  			By("Creating a channel")
   605  			network.CreateChannel("foo", o1, peer)
   606  
   607  			assertBlockReception(map[string]int{
   608  				"foo":           0,
   609  				"systemchannel": 1,
   610  			}, []*nwo.Orderer{o1, o2, o3}, peer, network)
   611  
   612  			By("Killing all orderers")
   613  			for i := range orderers {
   614  				ordererProcesses[i].Signal(syscall.SIGTERM)
   615  				Eventually(ordererProcesses[i].Wait(), network.EventuallyTimeout).Should(Receive())
   616  			}
   617  
   618  			By("Renewing the certificates for all orderers")
   619  			renewOrdererCertificates(network, o1, o2, o3)
   620  
   621  			By("Starting the orderers again")
   622  			for i := range orderers {
   623  				ordererRunner := network.OrdererRunner(orderers[i])
   624  				ordererRunners[i] = ordererRunner
   625  				ordererProcesses[i] = ifrit.Invoke(ordererRunner)
   626  				Eventually(ordererProcesses[0].Ready(), network.EventuallyTimeout).Should(BeClosed())
   627  			}
   628  
   629  			o1Proc = ordererProcesses[0]
   630  			o2Proc = ordererProcesses[1]
   631  			o3Proc = ordererProcesses[2]
   632  
   633  			By("Waiting for them to elect a leader once again")
   634  			findLeader(ordererRunners)
   635  
   636  			By("Creating a channel again")
   637  			network.CreateChannel("bar", o1, peer)
   638  
   639  			assertBlockReception(map[string]int{
   640  				"foo":           0,
   641  				"bar":           0,
   642  				"systemchannel": 2,
   643  			}, []*nwo.Orderer{o1, o2, o3}, peer, network)
   644  		})
   645  	})
   646  
   647  	When("admin certificate expires", func() {
   648  		It("is still possible to replace them", func() {
   649  			network = nwo.New(nwo.BasicEtcdRaft(), testDir, client, StartPort(), components)
   650  			network.GenerateConfigTree()
   651  			network.Bootstrap()
   652  
   653  			peer = network.Peer("Org1", "peer0")
   654  			orderer := network.Orderer("orderer")
   655  
   656  			ordererDomain := network.Organization(orderer.Organization).Domain
   657  			ordererCAKeyPath := filepath.Join(network.RootDir, "crypto", "ordererOrganizations", ordererDomain, "ca", "priv_sk")
   658  
   659  			ordererCAKey, err := ioutil.ReadFile(ordererCAKeyPath)
   660  			Expect(err).NotTo(HaveOccurred())
   661  
   662  			ordererCACertPath := filepath.Join(network.RootDir, "crypto", "ordererOrganizations", ordererDomain, "ca", fmt.Sprintf("ca.%s-cert.pem", ordererDomain))
   663  			ordererCACert, err := ioutil.ReadFile(ordererCACertPath)
   664  			Expect(err).NotTo(HaveOccurred())
   665  
   666  			adminCertPath := fmt.Sprintf("Admin@%s-cert.pem", ordererDomain)
   667  			adminCertPath = filepath.Join(network.OrdererUserMSPDir(orderer, "Admin"), "signcerts", adminCertPath)
   668  
   669  			originalAdminCert, err := ioutil.ReadFile(adminCertPath)
   670  			Expect(err).NotTo(HaveOccurred())
   671  
   672  			expiredAdminCert, earlyCACert := expireCertificate(originalAdminCert, ordererCACert, ordererCAKey, time.Now())
   673  			err = ioutil.WriteFile(adminCertPath, expiredAdminCert, 0600)
   674  			Expect(err).NotTo(HaveOccurred())
   675  
   676  			adminPath := filepath.Join(network.RootDir, "crypto", "ordererOrganizations",
   677  				ordererDomain, "msp", "admincerts", fmt.Sprintf("Admin@%s-cert.pem", ordererDomain))
   678  			err = ioutil.WriteFile(adminPath, expiredAdminCert, 0600)
   679  			Expect(err).NotTo(HaveOccurred())
   680  
   681  			err = ioutil.WriteFile(ordererCACertPath, earlyCACert, 0600)
   682  			Expect(err).NotTo(HaveOccurred())
   683  
   684  			ordererCACertPath = filepath.Join(network.RootDir, "crypto", "ordererOrganizations",
   685  				ordererDomain, "msp", "cacerts", fmt.Sprintf("ca.%s-cert.pem", ordererDomain))
   686  			err = ioutil.WriteFile(ordererCACertPath, earlyCACert, 0600)
   687  			Expect(err).NotTo(HaveOccurred())
   688  
   689  			By("Regenerating config")
   690  			sess, err := network.ConfigTxGen(commands.OutputBlock{
   691  				ChannelID:   network.SystemChannel.Name,
   692  				Profile:     network.SystemChannel.Profile,
   693  				ConfigPath:  network.RootDir,
   694  				OutputBlock: network.OutputBlockPath(network.SystemChannel.Name),
   695  			})
   696  			Expect(err).NotTo(HaveOccurred())
   697  			Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(0))
   698  
   699  			runner := network.OrdererRunner(orderer)
   700  			runner.Command.Env = append(runner.Command.Env, "FABRIC_LOGGING_SPEC=debug")
   701  			ordererProc = ifrit.Invoke(runner)
   702  
   703  			By("Waiting for orderer to elect a leader")
   704  			findLeader([]*ginkgomon.Runner{runner})
   705  
   706  			By("Creating config update that adds another orderer admin")
   707  			bootBlockPath := filepath.Join(network.RootDir, fmt.Sprintf("%s_block.pb", network.SystemChannel.Name))
   708  			bootBlock, err := ioutil.ReadFile(bootBlockPath)
   709  			Expect(err).NotTo(HaveOccurred())
   710  
   711  			current := configFromBootstrapBlock(bootBlock)
   712  			updatedConfig := addAdminCertToConfig(current, originalAdminCert)
   713  
   714  			tempDir, err := ioutil.TempDir("", "adminExpirationTest")
   715  			Expect(err).NotTo(HaveOccurred())
   716  
   717  			configBlockFile := filepath.Join(tempDir, "update.pb")
   718  			defer os.RemoveAll(tempDir)
   719  			nwo.ComputeUpdateOrdererConfig(configBlockFile, network, network.SystemChannel.Name, current, updatedConfig, peer)
   720  
   721  			updateTransaction, err := ioutil.ReadFile(configBlockFile)
   722  			Expect(err).NotTo(HaveOccurred())
   723  
   724  			By("Creating config update")
   725  			channelCreateTxn := createConfigTx(updateTransaction, network.SystemChannel.Name, network, orderer, peer)
   726  
   727  			By("Updating channel config and failing")
   728  			p, err := nwo.Broadcast(network, orderer, channelCreateTxn)
   729  			Expect(err).NotTo(HaveOccurred())
   730  			Expect(p.Status).To(Equal(common.Status_BAD_REQUEST))
   731  			Expect(p.Info).To(ContainSubstring("identity expired"))
   732  
   733  			By("Attempting to fetch a block from orderer and failing")
   734  			denv := CreateDeliverEnvelope(network, orderer, 0, network.SystemChannel.Name)
   735  			Expect(denv).NotTo(BeNil())
   736  
   737  			block, err := nwo.Deliver(network, orderer, denv)
   738  			Expect(err).To(HaveOccurred())
   739  			Expect(block).To(BeNil())
   740  			Eventually(runner.Err(), time.Minute, time.Second).Should(gbytes.Say("client identity expired"))
   741  
   742  			By("Killing orderer")
   743  			ordererProc.Signal(syscall.SIGTERM)
   744  			Eventually(ordererProc.Wait(), network.EventuallyTimeout).Should(Receive())
   745  
   746  			By("Launching orderers again")
   747  			runner = network.OrdererRunner(orderer)
   748  			runner.Command.Env = append(runner.Command.Env, "ORDERER_GENERAL_AUTHENTICATION_NOEXPIRATIONCHECKS=true")
   749  			ordererProc = ifrit.Invoke(runner)
   750  
   751  			By("Waiting for orderer to launch again")
   752  			findLeader([]*ginkgomon.Runner{runner})
   753  
   754  			By("Updating channel config and succeeding")
   755  			p, err = nwo.Broadcast(network, orderer, channelCreateTxn)
   756  			Expect(err).NotTo(HaveOccurred())
   757  			Expect(p.Status).To(Equal(common.Status_SUCCESS))
   758  
   759  			By("Fetching a block from the orderer and succeeding")
   760  			block = FetchBlock(network, orderer, 1, network.SystemChannel.Name)
   761  			Expect(block).NotTo(BeNil())
   762  
   763  			By("Restore the original admin cert")
   764  			err = ioutil.WriteFile(adminCertPath, originalAdminCert, 0600)
   765  			Expect(err).NotTo(HaveOccurred())
   766  
   767  			By("Ensure we can fetch the block using our original un-expired admin cert")
   768  			ccb := func() uint64 {
   769  				return nwo.GetConfigBlock(network, peer, orderer, network.SystemChannel.Name).Header.Number
   770  			}
   771  			Eventually(ccb, network.EventuallyTimeout).Should(Equal(uint64(1)))
   772  		})
   773  	})
   774  })
   775  
   776  func findLeader(ordererRunners []*ginkgomon.Runner) int {
   777  	var wg sync.WaitGroup
   778  	wg.Add(len(ordererRunners))
   779  
   780  	findLeader := func(runner *ginkgomon.Runner) int {
   781  		Eventually(runner.Err(), time.Minute, time.Second).Should(gbytes.Say("Raft leader changed: [0-9] -> "))
   782  
   783  		idBuff := make([]byte, 1)
   784  		_, err := runner.Err().Read(idBuff)
   785  		Expect(err).NotTo(HaveOccurred())
   786  
   787  		newLeader, err := strconv.ParseInt(string(idBuff), 10, 32)
   788  		Expect(err).To(BeNil())
   789  		return int(newLeader)
   790  	}
   791  
   792  	leaders := make(chan int, len(ordererRunners))
   793  
   794  	for _, runner := range ordererRunners {
   795  		go func(runner *ginkgomon.Runner) {
   796  			defer GinkgoRecover()
   797  			defer wg.Done()
   798  
   799  			for {
   800  				leader := findLeader(runner)
   801  				if leader != 0 {
   802  					leaders <- leader
   803  					break
   804  				}
   805  			}
   806  		}(runner)
   807  	}
   808  
   809  	wg.Wait()
   810  
   811  	close(leaders)
   812  	firstLeader := <-leaders
   813  	for leader := range leaders {
   814  		if firstLeader != leader {
   815  			Fail(fmt.Sprintf("First leader is %d but saw %d also as a leader", firstLeader, leader))
   816  		}
   817  	}
   818  
   819  	return firstLeader
   820  }
   821  
   822  func renewOrdererCertificates(network *nwo.Network, orderers ...*nwo.Orderer) {
   823  	if len(orderers) == 0 {
   824  		return
   825  	}
   826  	ordererDomain := network.Organization(orderers[0].Organization).Domain
   827  	ordererTLSCAKeyPath := filepath.Join(network.RootDir, "crypto", "ordererOrganizations",
   828  		ordererDomain, "tlsca", "priv_sk")
   829  
   830  	ordererTLSCAKey, err := ioutil.ReadFile(ordererTLSCAKeyPath)
   831  	Expect(err).NotTo(HaveOccurred())
   832  
   833  	ordererTLSCACertPath := filepath.Join(network.RootDir, "crypto", "ordererOrganizations",
   834  		ordererDomain, "tlsca", fmt.Sprintf("tlsca.%s-cert.pem", ordererDomain))
   835  	ordererTLSCACert, err := ioutil.ReadFile(ordererTLSCACertPath)
   836  	Expect(err).NotTo(HaveOccurred())
   837  
   838  	serverTLSCerts := map[string][]byte{}
   839  	for _, orderer := range orderers {
   840  		tlsCertPath := filepath.Join(network.OrdererLocalTLSDir(orderer), "server.crt")
   841  		serverTLSCerts[tlsCertPath], err = ioutil.ReadFile(tlsCertPath)
   842  		Expect(err).NotTo(HaveOccurred())
   843  	}
   844  
   845  	for filePath, certPEM := range serverTLSCerts {
   846  		renewedCert, _ := expireCertificate(certPEM, ordererTLSCACert, ordererTLSCAKey, time.Now().Add(time.Hour))
   847  		err = ioutil.WriteFile(filePath, renewedCert, 0600)
   848  		Expect(err).NotTo(HaveOccurred())
   849  	}
   850  }
   851  
   852  func expireCertificate(certPEM, caCertPEM, caKeyPEM []byte, expirationTime time.Time) (expiredcertPEM []byte, earlyMadeCACertPEM []byte) {
   853  	keyAsDER, _ := pem.Decode(caKeyPEM)
   854  	caKeyWithoutType, err := x509.ParsePKCS8PrivateKey(keyAsDER.Bytes)
   855  	Expect(err).NotTo(HaveOccurred())
   856  	caKey := caKeyWithoutType.(*ecdsa.PrivateKey)
   857  
   858  	caCertAsDER, _ := pem.Decode(caCertPEM)
   859  	caCert, err := x509.ParseCertificate(caCertAsDER.Bytes)
   860  	Expect(err).NotTo(HaveOccurred())
   861  
   862  	certAsDER, _ := pem.Decode(certPEM)
   863  	cert, err := x509.ParseCertificate(certAsDER.Bytes)
   864  	Expect(err).NotTo(HaveOccurred())
   865  
   866  	cert.Raw = nil
   867  	caCert.Raw = nil
   868  	// The certificate was made 1 hour ago
   869  	cert.NotBefore = time.Now().Add((-1) * time.Hour)
   870  	// As well as the CA certificate
   871  	caCert.NotBefore = time.Now().Add((-1) * time.Hour)
   872  	// The certificate expires now
   873  	cert.NotAfter = expirationTime
   874  
   875  	// The CA signs the certificate
   876  	certBytes, err := x509.CreateCertificate(rand.Reader, cert, caCert, cert.PublicKey, caKey)
   877  	Expect(err).NotTo(HaveOccurred())
   878  
   879  	// The CA signs its own certificate
   880  	caCertBytes, err := x509.CreateCertificate(rand.Reader, caCert, caCert, caCert.PublicKey, caKey)
   881  	Expect(err).NotTo(HaveOccurred())
   882  
   883  	expiredcertPEM = pem.EncodeToMemory(&pem.Block{Bytes: certBytes, Type: "CERTIFICATE"})
   884  	earlyMadeCACertPEM = pem.EncodeToMemory(&pem.Block{Bytes: caCertBytes, Type: "CERTIFICATE"})
   885  	return
   886  }
   887  
   888  func createConfigTx(txData []byte, channelName string, network *nwo.Network, orderer *nwo.Orderer, peer *nwo.Peer) *common.Envelope {
   889  	ctxEnv, err := protoutil.UnmarshalEnvelope(txData)
   890  	Expect(err).NotTo(HaveOccurred())
   891  
   892  	payload, err := protoutil.UnmarshalPayload(ctxEnv.Payload)
   893  	Expect(err).NotTo(HaveOccurred())
   894  
   895  	configUpdateEnv, err := configtx.UnmarshalConfigUpdateEnvelope(payload.Data)
   896  	Expect(err).NotTo(HaveOccurred())
   897  
   898  	conf := signer.Config{
   899  		MSPID:        network.Organization(orderer.Organization).MSPID,
   900  		IdentityPath: network.OrdererUserCert(orderer, "Admin"),
   901  		KeyPath:      network.OrdererUserKey(orderer, "Admin"),
   902  	}
   903  
   904  	s, err := signer.NewSigner(conf)
   905  	Expect(err).NotTo(HaveOccurred())
   906  
   907  	signConfigUpdate(conf, configUpdateEnv)
   908  
   909  	env, err := protoutil.CreateSignedEnvelope(common.HeaderType_CONFIG_UPDATE, channelName, s, configUpdateEnv, 0, 0)
   910  	Expect(err).NotTo(HaveOccurred())
   911  
   912  	return env
   913  }
   914  
   915  func signConfigUpdate(conf signer.Config, configUpdateEnv *common.ConfigUpdateEnvelope) *common.ConfigUpdateEnvelope {
   916  	s, err := signer.NewSigner(conf)
   917  	Expect(err).NotTo(HaveOccurred())
   918  
   919  	sigHeader, err := protoutil.NewSignatureHeader(s)
   920  	Expect(err).NotTo(HaveOccurred())
   921  
   922  	configSig := &common.ConfigSignature{
   923  		SignatureHeader: protoutil.MarshalOrPanic(sigHeader),
   924  	}
   925  
   926  	configSig.Signature, err = s.Sign(util.ConcatenateBytes(configSig.SignatureHeader, configUpdateEnv.ConfigUpdate))
   927  	Expect(err).NotTo(HaveOccurred())
   928  
   929  	configUpdateEnv.Signatures = append(configUpdateEnv.Signatures, configSig)
   930  	return configUpdateEnv
   931  }
   932  
   933  func addAdminCertToConfig(originalConfig *common.Config, additionalAdmin []byte) *common.Config {
   934  	updatedConfig := proto.Clone(originalConfig).(*common.Config)
   935  
   936  	rawMSPConfig := updatedConfig.ChannelGroup.Groups["Orderer"].Groups["OrdererOrg"].Values["MSP"]
   937  	mspConfig := &msp.MSPConfig{}
   938  	err := proto.Unmarshal(rawMSPConfig.Value, mspConfig)
   939  	Expect(err).NotTo(HaveOccurred())
   940  
   941  	fabricConfig := &msp.FabricMSPConfig{}
   942  	err = proto.Unmarshal(mspConfig.Config, fabricConfig)
   943  	Expect(err).NotTo(HaveOccurred())
   944  
   945  	fabricConfig.Admins = append(fabricConfig.Admins, additionalAdmin)
   946  	mspConfig.Config = protoutil.MarshalOrPanic(fabricConfig)
   947  
   948  	rawMSPConfig.Value = protoutil.MarshalOrPanic(mspConfig)
   949  	return updatedConfig
   950  }
   951  
   952  func configFromBootstrapBlock(bootstrapBlock []byte) *common.Config {
   953  	block := &common.Block{}
   954  	err := proto.Unmarshal(bootstrapBlock, block)
   955  	Expect(err).NotTo(HaveOccurred())
   956  	return configFromBlock(block)
   957  }
   958  
   959  func configFromBlock(block *common.Block) *common.Config {
   960  	envelope, err := protoutil.GetEnvelopeFromBlock(block.Data.Data[0])
   961  	Expect(err).NotTo(HaveOccurred())
   962  
   963  	payload, err := protoutil.UnmarshalPayload(envelope.Payload)
   964  	Expect(err).NotTo(HaveOccurred())
   965  
   966  	configEnv := &common.ConfigEnvelope{}
   967  	err = proto.Unmarshal(payload.Data, configEnv)
   968  	Expect(err).NotTo(HaveOccurred())
   969  
   970  	return configEnv.Config
   971  }
   972  
   973  func fetchConfig(n *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, port nwo.PortName, channel string, tlsHandshakeTimeShift time.Duration) *common.Config {
   974  	tempDir, err := ioutil.TempDir(n.RootDir, "fetchConfig")
   975  	Expect(err).NotTo(HaveOccurred())
   976  	defer os.RemoveAll(tempDir)
   977  
   978  	output := filepath.Join(tempDir, "config_block.pb")
   979  	fetchConfigBlock(n, peer, orderer, port, n.SystemChannel.Name, output, tlsHandshakeTimeShift)
   980  	configBlock := nwo.UnmarshalBlockFromFile(output)
   981  	return configFromBlock(configBlock)
   982  }
   983  
   984  func fetchConfigBlock(n *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, port nwo.PortName, channel, output string, tlsHandshakeTimeShift time.Duration) {
   985  	fetch := func() int {
   986  		sess, err := n.OrdererAdminSession(orderer, peer, commands.ChannelFetch{
   987  			ChannelID:             channel,
   988  			Block:                 "config",
   989  			Orderer:               n.OrdererAddress(orderer, port),
   990  			OutputFile:            output,
   991  			ClientAuth:            n.ClientAuthRequired,
   992  			TLSHandshakeTimeShift: tlsHandshakeTimeShift,
   993  		})
   994  		Expect(err).NotTo(HaveOccurred())
   995  		code := sess.Wait(n.EventuallyTimeout).ExitCode()
   996  		if code == 0 {
   997  			Expect(sess.Err).To(gbytes.Say("Received block: "))
   998  		}
   999  		return code
  1000  	}
  1001  	Eventually(fetch, n.EventuallyTimeout).Should(Equal(0))
  1002  }
  1003  
  1004  func currentConfigBlockNumber(n *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, port nwo.PortName, channel string, tlsHandshakeTimeShift time.Duration) uint64 {
  1005  	tempDir, err := ioutil.TempDir(n.RootDir, "currentConfigBlock")
  1006  	Expect(err).NotTo(HaveOccurred())
  1007  	defer os.RemoveAll(tempDir)
  1008  
  1009  	output := filepath.Join(tempDir, "config_block.pb")
  1010  	fetchConfigBlock(n, peer, orderer, port, channel, output, tlsHandshakeTimeShift)
  1011  	configBlock := nwo.UnmarshalBlockFromFile(output)
  1012  	return configBlock.Header.Number
  1013  }
  1014  
  1015  func updateOrdererConfig(n *nwo.Network, orderer *nwo.Orderer, port nwo.PortName, channel string, tlsHandshakeTimeShift time.Duration, current, updated *common.Config, submitter *nwo.Peer, additionalSigners ...*nwo.Orderer) {
  1016  	tempDir, err := ioutil.TempDir(n.RootDir, "updateConfig")
  1017  	Expect(err).NotTo(HaveOccurred())
  1018  	updateFile := filepath.Join(tempDir, "update.pb")
  1019  	defer os.RemoveAll(tempDir)
  1020  
  1021  	currentBlockNumber := currentConfigBlockNumber(n, submitter, orderer, port, channel, tlsHandshakeTimeShift)
  1022  	nwo.ComputeUpdateOrdererConfig(updateFile, n, channel, current, updated, submitter, additionalSigners...)
  1023  
  1024  	Eventually(func() bool {
  1025  		sess, err := n.OrdererAdminSession(orderer, submitter, commands.ChannelUpdate{
  1026  			ChannelID:             channel,
  1027  			Orderer:               n.OrdererAddress(orderer, port),
  1028  			File:                  updateFile,
  1029  			ClientAuth:            n.ClientAuthRequired,
  1030  			TLSHandshakeTimeShift: tlsHandshakeTimeShift,
  1031  		})
  1032  		Expect(err).NotTo(HaveOccurred())
  1033  
  1034  		sess.Wait(n.EventuallyTimeout)
  1035  		if sess.ExitCode() != 0 {
  1036  			return false
  1037  		}
  1038  
  1039  		return strings.Contains(string(sess.Err.Contents()), "Successfully submitted channel update")
  1040  	}, n.EventuallyTimeout).Should(BeTrue())
  1041  
  1042  	ccb := func() uint64 {
  1043  		return currentConfigBlockNumber(n, submitter, orderer, port, channel, tlsHandshakeTimeShift)
  1044  	}
  1045  	Eventually(ccb, n.EventuallyTimeout).Should(BeNumerically(">", currentBlockNumber))
  1046  }
  1047  
  1048  func consenterChannelConfig(n *nwo.Network, o *nwo.Orderer) orderer.Consenter {
  1049  	host, port := intconftx.OrdererClusterHostPort(n, o)
  1050  	tlsCert := parseCertificate(filepath.Join(n.OrdererLocalTLSDir(o), "server.crt"))
  1051  	return orderer.Consenter{
  1052  		Address: orderer.EtcdAddress{
  1053  			Host: host,
  1054  			Port: port,
  1055  		},
  1056  		ClientTLSCert: tlsCert,
  1057  		ServerTLSCert: tlsCert,
  1058  	}
  1059  }
  1060  
  1061  // parseCertificate loads the PEM-encoded x509 certificate at the specified
  1062  // path.
  1063  func parseCertificate(path string) *x509.Certificate {
  1064  	certBytes, err := ioutil.ReadFile(path)
  1065  	Expect(err).NotTo(HaveOccurred())
  1066  	pemBlock, _ := pem.Decode(certBytes)
  1067  	cert, err := x509.ParseCertificate(pemBlock.Bytes)
  1068  	Expect(err).NotTo(HaveOccurred())
  1069  	return cert
  1070  }