github.com/osdi23p228/fabric@v0.0.0-20221218062954-77808885f5db/integration/e2e/e2e_test.go (about)

     1  /*
     2  Copyright IBM Corp All Rights Reserved.
     3  
     4  SPDX-License-Identifier: Apache-2.0
     5  */
     6  
     7  package e2e
     8  
     9  import (
    10  	"bufio"
    11  	"bytes"
    12  	"context"
    13  	"crypto/sha256"
    14  	"crypto/tls"
    15  	"crypto/x509"
    16  	"encoding/json"
    17  	"fmt"
    18  	"io"
    19  	"io/ioutil"
    20  	"net/http"
    21  	"os"
    22  	"path"
    23  	"path/filepath"
    24  	"strings"
    25  	"syscall"
    26  	"time"
    27  
    28  	docker "github.com/fsouza/go-dockerclient"
    29  	"github.com/golang/protobuf/proto"
    30  	"github.com/hyperledger/fabric-lib-go/healthz"
    31  	"github.com/hyperledger/fabric-protos-go/orderer/etcdraft"
    32  	"github.com/osdi23p228/fabric/integration/nwo"
    33  	"github.com/osdi23p228/fabric/integration/nwo/commands"
    34  	"github.com/osdi23p228/fabric/integration/nwo/fabricconfig"
    35  	. "github.com/onsi/ginkgo"
    36  	. "github.com/onsi/gomega"
    37  	"github.com/onsi/gomega/gbytes"
    38  	"github.com/onsi/gomega/gexec"
    39  	"github.com/tedsuo/ifrit"
    40  	"github.com/tedsuo/ifrit/ginkgomon"
    41  )
    42  
    43  var _ = Describe("EndToEnd", func() {
    44  	var (
    45  		testDir   string
    46  		client    *docker.Client
    47  		network   *nwo.Network
    48  		chaincode nwo.Chaincode
    49  		process   ifrit.Process
    50  	)
    51  
    52  	BeforeEach(func() {
    53  		var err error
    54  		testDir, err = ioutil.TempDir("", "e2e")
    55  		Expect(err).NotTo(HaveOccurred())
    56  
    57  		client, err = docker.NewClientFromEnv()
    58  		Expect(err).NotTo(HaveOccurred())
    59  
    60  		chaincode = nwo.Chaincode{
    61  			Name:            "mycc",
    62  			Version:         "0.0",
    63  			Path:            components.Build("github.com/osdi23p228/fabric/integration/chaincode/simple/cmd"),
    64  			Lang:            "binary",
    65  			PackageFile:     filepath.Join(testDir, "simplecc.tar.gz"),
    66  			Ctor:            `{"Args":["init","a","100","b","200"]}`,
    67  			SignaturePolicy: `AND ('Org1MSP.member','Org2MSP.member')`,
    68  			Sequence:        "1",
    69  			InitRequired:    true,
    70  			Label:           "my_prebuilt_chaincode",
    71  		}
    72  	})
    73  
    74  	AfterEach(func() {
    75  		if process != nil {
    76  			process.Signal(syscall.SIGTERM)
    77  			Eventually(process.Wait(), network.EventuallyTimeout).Should(Receive())
    78  		}
    79  		if network != nil {
    80  			network.Cleanup()
    81  		}
    82  		os.RemoveAll(testDir)
    83  	})
    84  
    85  	Describe("basic solo network with 2 orgs and no docker", func() {
    86  		var (
    87  			metricsReader        *MetricsReader
    88  			runArtifactsFilePath string
    89  		)
    90  
    91  		BeforeEach(func() {
    92  			metricsReader = NewMetricsReader()
    93  			go metricsReader.Start()
    94  
    95  			network = nwo.New(nwo.BasicSolo(), testDir, nil, StartPort(), components)
    96  			network.MetricsProvider = "statsd"
    97  			network.StatsdEndpoint = metricsReader.Address()
    98  			network.ChannelParticipationEnabled = true
    99  			network.Profiles = append(network.Profiles, &nwo.Profile{
   100  				Name:          "TwoOrgsBaseProfileChannel",
   101  				Consortium:    "SampleConsortium",
   102  				Orderers:      []string{"orderer"},
   103  				Organizations: []string{"Org1", "Org2"},
   104  			})
   105  			network.Channels = append(network.Channels, &nwo.Channel{
   106  				Name:        "baseprofilechannel",
   107  				Profile:     "TwoOrgsBaseProfileChannel",
   108  				BaseProfile: "TwoOrgsOrdererGenesis",
   109  			})
   110  
   111  			runArtifactsFilePath = filepath.Join(testDir, "run-artifacts.txt")
   112  			os.Setenv("RUN_ARTIFACTS_FILE", runArtifactsFilePath)
   113  			for i, e := range network.ExternalBuilders {
   114  				e.PropagateEnvironment = append(e.PropagateEnvironment, "RUN_ARTIFACTS_FILE")
   115  				network.ExternalBuilders[i] = e
   116  			}
   117  
   118  			network.GenerateConfigTree()
   119  			for _, peer := range network.PeersWithChannel("testchannel") {
   120  				core := network.ReadPeerConfig(peer)
   121  				core.VM = nil
   122  				network.WritePeerConfig(peer, core)
   123  			}
   124  			network.Bootstrap()
   125  
   126  			networkRunner := network.NetworkGroupRunner()
   127  			process = ifrit.Invoke(networkRunner)
   128  			Eventually(process.Ready(), network.EventuallyTimeout).Should(BeClosed())
   129  		})
   130  
   131  		AfterEach(func() {
   132  			if metricsReader != nil {
   133  				metricsReader.Close()
   134  			}
   135  
   136  			// Terminate the processes but defer the network cleanup to the outer
   137  			// AfterEach.
   138  			if process != nil {
   139  				process.Signal(syscall.SIGTERM)
   140  				Eventually(process.Wait(), network.EventuallyTimeout).Should(Receive())
   141  				process = nil
   142  			}
   143  
   144  			// Ensure that the temporary directories generated by launched external
   145  			// chaincodes have been cleaned up. This must be done after the peers
   146  			// have been terminated.
   147  			contents, err := ioutil.ReadFile(runArtifactsFilePath)
   148  			Expect(err).NotTo(HaveOccurred())
   149  			scanner := bufio.NewScanner(bytes.NewBuffer(contents))
   150  			for scanner.Scan() {
   151  				Expect(scanner.Text()).NotTo(BeAnExistingFile())
   152  			}
   153  		})
   154  
   155  		It("executes a basic solo network with 2 orgs and no docker", func() {
   156  			By("getting the orderer by name")
   157  			orderer := network.Orderer("orderer")
   158  
   159  			By("setting up the channel")
   160  			network.CreateAndJoinChannel(orderer, "testchannel")
   161  			nwo.EnableCapabilities(network, "testchannel", "Application", "V2_0", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0"))
   162  
   163  			By("attempting to install unsupported chaincode without docker")
   164  			badCC := chaincode
   165  			badCC.Lang = "unsupported-type"
   166  			badCC.Label = "chaincode-label"
   167  			badCC.PackageFile = filepath.Join(testDir, "unsupported-type.tar.gz")
   168  			nwo.PackageChaincodeBinary(badCC)
   169  			badCC.SetPackageIDFromPackageFile()
   170  			sess, err := network.PeerAdminSession(
   171  				network.Peer("Org1", "peer0"),
   172  				commands.ChaincodeInstall{
   173  					PackageFile: badCC.PackageFile,
   174  					ClientAuth:  network.ClientAuthRequired,
   175  				},
   176  			)
   177  			Expect(err).NotTo(HaveOccurred())
   178  			Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(1))
   179  			Expect(sess.Err).To(gbytes.Say("docker build is disabled"))
   180  
   181  			By("deploying the chaincode")
   182  			nwo.DeployChaincode(network, "testchannel", orderer, chaincode)
   183  
   184  			By("ensuring external cc run artifacts exist after deploying")
   185  			contents, err := ioutil.ReadFile(runArtifactsFilePath)
   186  			Expect(err).NotTo(HaveOccurred())
   187  			scanner := bufio.NewScanner(bytes.NewBuffer(contents))
   188  			for scanner.Scan() {
   189  				Expect(scanner.Text()).To(BeADirectory())
   190  			}
   191  
   192  			By("getting the client peer by name")
   193  			peer := network.Peer("Org1", "peer0")
   194  
   195  			RunQueryInvokeQuery(network, orderer, peer, "testchannel")
   196  			RunRespondWith(network, orderer, peer, "testchannel")
   197  
   198  			By("evaluating statsd metrics")
   199  			metricsWriteInterval := 5 * time.Second
   200  			CheckPeerStatsdStreamMetrics(metricsReader, 2*metricsWriteInterval)
   201  			CheckPeerStatsdMetrics("org1_peer0", metricsReader, 2*metricsWriteInterval)
   202  			CheckPeerStatsdMetrics("org2_peer0", metricsReader, 2*metricsWriteInterval)
   203  
   204  			By("checking for orderer metrics")
   205  			CheckOrdererStatsdMetrics("ordererorg_orderer", metricsReader, 2*metricsWriteInterval)
   206  
   207  			By("setting up a channel from a base profile")
   208  			additionalPeer := network.Peer("Org2", "peer0")
   209  			network.CreateChannel("baseprofilechannel", orderer, peer, additionalPeer)
   210  		})
   211  	})
   212  
   213  	Describe("basic kafka network with 2 orgs", func() {
   214  		BeforeEach(func() {
   215  			network = nwo.New(nwo.BasicKafka(), testDir, client, StartPort(), components)
   216  			network.MetricsProvider = "prometheus"
   217  			network.ChannelParticipationEnabled = true
   218  			network.GenerateConfigTree()
   219  			network.Bootstrap()
   220  
   221  			networkRunner := network.NetworkGroupRunner()
   222  			process = ifrit.Invoke(networkRunner)
   223  			Eventually(process.Ready(), network.EventuallyTimeout).Should(BeClosed())
   224  		})
   225  
   226  		It("executes a basic kafka network with 2 orgs (using docker chaincode builds)", func() {
   227  			chaincodePath, err := filepath.Abs("../chaincode/module")
   228  			Expect(err).NotTo(HaveOccurred())
   229  
   230  			// use these two variants of the same chaincode to ensure we test
   231  			// the golang docker build for both module and gopath chaincode
   232  			chaincode = nwo.Chaincode{
   233  				Name:            "mycc",
   234  				Version:         "0.0",
   235  				Path:            chaincodePath,
   236  				Lang:            "golang",
   237  				PackageFile:     filepath.Join(testDir, "modulecc.tar.gz"),
   238  				Ctor:            `{"Args":["init","a","100","b","200"]}`,
   239  				SignaturePolicy: `AND ('Org1MSP.member','Org2MSP.member')`,
   240  				Sequence:        "1",
   241  				InitRequired:    true,
   242  				Label:           "my_module_chaincode",
   243  			}
   244  
   245  			gopathChaincode := nwo.Chaincode{
   246  				Name:            "mycc",
   247  				Version:         "0.0",
   248  				Path:            "github.com/osdi23p228/fabric/integration/chaincode/simple/cmd",
   249  				Lang:            "golang",
   250  				PackageFile:     filepath.Join(testDir, "simplecc.tar.gz"),
   251  				Ctor:            `{"Args":["init","a","100","b","200"]}`,
   252  				SignaturePolicy: `AND ('Org1MSP.member','Org2MSP.member')`,
   253  				Sequence:        "1",
   254  				InitRequired:    true,
   255  				Label:           "my_simple_chaincode",
   256  			}
   257  
   258  			orderer := network.Orderer("orderer")
   259  
   260  			network.CreateAndJoinChannel(orderer, "testchannel")
   261  			nwo.EnableCapabilities(network, "testchannel", "Application", "V2_0", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0"))
   262  
   263  			// package, install, and approve by org1 - module chaincode
   264  			packageInstallApproveChaincode(network, "testchannel", orderer, chaincode, network.Peer("Org1", "peer0"))
   265  
   266  			// package, install, and approve by org2 - gopath chaincode, same logic
   267  			packageInstallApproveChaincode(network, "testchannel", orderer, gopathChaincode, network.Peer("Org2", "peer0"))
   268  
   269  			testPeers := network.PeersWithChannel("testchannel")
   270  			nwo.CheckCommitReadinessUntilReady(network, "testchannel", chaincode, network.PeerOrgs(), testPeers...)
   271  			nwo.CommitChaincode(network, "testchannel", orderer, chaincode, testPeers[0], testPeers...)
   272  			nwo.InitChaincode(network, "testchannel", orderer, chaincode, testPeers...)
   273  
   274  			By("listing the containers after committing the chaincode definition")
   275  			initialContainerFilter := map[string][]string{
   276  				"name": {
   277  					chaincodeContainerNameFilter(network, chaincode),
   278  					chaincodeContainerNameFilter(network, gopathChaincode),
   279  				},
   280  			}
   281  
   282  			containers, err := client.ListContainers(docker.ListContainersOptions{Filters: initialContainerFilter})
   283  			Expect(err).NotTo(HaveOccurred())
   284  			Expect(containers).To(HaveLen(2))
   285  
   286  			RunQueryInvokeQuery(network, orderer, network.Peer("Org1", "peer0"), "testchannel")
   287  
   288  			CheckPeerOperationEndpoints(network, network.Peer("Org2", "peer0"))
   289  			CheckOrdererOperationEndpoints(network, orderer)
   290  
   291  			// upgrade chaincode to v2.0 with different label
   292  			chaincode.Version = "1.0"
   293  			chaincode.Sequence = "2"
   294  			chaincode.Label = "my_module_chaincode_updated"
   295  			gopathChaincode.Version = "1.0"
   296  			gopathChaincode.Sequence = "2"
   297  			gopathChaincode.Label = "my_simple_chaincode_updated"
   298  
   299  			// package, install, and approve by org1 - module chaincode
   300  			packageInstallApproveChaincode(network, "testchannel", orderer, chaincode, network.Peer("Org1", "peer0"))
   301  
   302  			// package, install, and approve by org2 - gopath chaincode, same logic
   303  			packageInstallApproveChaincode(network, "testchannel", orderer, gopathChaincode, network.Peer("Org2", "peer0"))
   304  
   305  			nwo.CheckCommitReadinessUntilReady(network, "testchannel", chaincode, network.PeerOrgs(), testPeers...)
   306  			nwo.CommitChaincode(network, "testchannel", orderer, chaincode, testPeers[0], testPeers...)
   307  			nwo.InitChaincode(network, "testchannel", orderer, chaincode, testPeers...)
   308  
   309  			By("listing the containers after updating the chaincode definition")
   310  			// expect the containers for the previous package id to be stopped
   311  			containers, err = client.ListContainers(docker.ListContainersOptions{Filters: initialContainerFilter})
   312  			Expect(err).NotTo(HaveOccurred())
   313  			Expect(containers).To(HaveLen(0))
   314  			updatedContainerFilter := map[string][]string{
   315  				"name": {
   316  					chaincodeContainerNameFilter(network, chaincode),
   317  					chaincodeContainerNameFilter(network, gopathChaincode),
   318  				},
   319  			}
   320  			containers, err = client.ListContainers(docker.ListContainersOptions{Filters: updatedContainerFilter})
   321  			Expect(err).NotTo(HaveOccurred())
   322  			Expect(containers).To(HaveLen(2))
   323  
   324  			RunQueryInvokeQuery(network, orderer, network.Peer("Org1", "peer0"), "testchannel")
   325  
   326  			By("retrieving the local mspid of the peer via simple chaincode")
   327  			sess, err := network.PeerUserSession(network.Peer("Org2", "peer0"), "User1", commands.ChaincodeQuery{
   328  				ChannelID: "testchannel",
   329  				Name:      "mycc",
   330  				Ctor:      `{"Args":["mspid"]}`,
   331  			})
   332  			Expect(err).NotTo(HaveOccurred())
   333  			Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(0))
   334  			Expect(sess).To(gbytes.Say("Org2MSP"))
   335  		})
   336  	})
   337  
   338  	Describe("basic single node etcdraft network", func() {
   339  		var (
   340  			peerRunners    []*ginkgomon.Runner
   341  			processes      map[string]ifrit.Process
   342  			ordererProcess ifrit.Process
   343  		)
   344  
   345  		BeforeEach(func() {
   346  			network = nwo.New(nwo.MultiChannelEtcdRaft(), testDir, client, StartPort(), components)
   347  			network.ChannelParticipationEnabled = true
   348  			network.GenerateConfigTree()
   349  			for _, peer := range network.Peers {
   350  				core := network.ReadPeerConfig(peer)
   351  				core.Peer.Gossip.UseLeaderElection = false
   352  				core.Peer.Gossip.OrgLeader = true
   353  				core.Peer.Deliveryclient.ReconnectTotalTimeThreshold = time.Duration(time.Second)
   354  				network.WritePeerConfig(peer, core)
   355  			}
   356  			network.Bootstrap()
   357  
   358  			ordererRunner := network.OrdererGroupRunner()
   359  			ordererProcess = ifrit.Invoke(ordererRunner)
   360  			Eventually(ordererProcess.Ready(), network.EventuallyTimeout).Should(BeClosed())
   361  
   362  			peerRunners = make([]*ginkgomon.Runner, len(network.Peers))
   363  			processes = map[string]ifrit.Process{}
   364  			for i, peer := range network.Peers {
   365  				pr := network.PeerRunner(peer)
   366  				peerRunners[i] = pr
   367  				p := ifrit.Invoke(pr)
   368  				processes[peer.ID()] = p
   369  				Eventually(p.Ready(), network.EventuallyTimeout).Should(BeClosed())
   370  			}
   371  		})
   372  
   373  		AfterEach(func() {
   374  			if ordererProcess != nil {
   375  				ordererProcess.Signal(syscall.SIGTERM)
   376  				Eventually(ordererProcess.Wait(), network.EventuallyTimeout).Should(Receive())
   377  			}
   378  			for _, p := range processes {
   379  				p.Signal(syscall.SIGTERM)
   380  				Eventually(p.Wait(), network.EventuallyTimeout).Should(Receive())
   381  			}
   382  		})
   383  
   384  		It("creates two channels with two orgs trying to reconfigure and update metadata", func() {
   385  			orderer := network.Orderer("orderer")
   386  			peer := network.Peer("Org1", "peer0")
   387  
   388  			By("Create first channel and deploy the chaincode")
   389  			network.CreateAndJoinChannel(orderer, "testchannel")
   390  			nwo.EnableCapabilities(network, "testchannel", "Application", "V2_0", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0"))
   391  			nwo.DeployChaincode(network, "testchannel", orderer, chaincode)
   392  			RunQueryInvokeQuery(network, orderer, peer, "testchannel")
   393  
   394  			By("Create second channel and deploy chaincode")
   395  			network.CreateAndJoinChannel(orderer, "testchannel2")
   396  			peers := network.PeersWithChannel("testchannel2")
   397  			nwo.EnableCapabilities(network, "testchannel2", "Application", "V2_0", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0"))
   398  			nwo.ApproveChaincodeForMyOrg(network, "testchannel2", orderer, chaincode, peers...)
   399  			nwo.CheckCommitReadinessUntilReady(network, "testchannel2", chaincode, network.PeerOrgs(), peers...)
   400  			nwo.CommitChaincode(network, "testchannel2", orderer, chaincode, peers[0], peers...)
   401  			nwo.InitChaincode(network, "testchannel2", orderer, chaincode, peers...)
   402  			RunQueryInvokeQuery(network, orderer, peer, "testchannel2")
   403  
   404  			By("Update consensus metadata to increase snapshot interval")
   405  			snapDir := path.Join(network.RootDir, "orderers", orderer.ID(), "etcdraft", "snapshot", "testchannel")
   406  			files, err := ioutil.ReadDir(snapDir)
   407  			Expect(err).NotTo(HaveOccurred())
   408  			numOfSnaps := len(files)
   409  
   410  			nwo.UpdateConsensusMetadata(network, peer, orderer, "testchannel", func(originalMetadata []byte) []byte {
   411  				metadata := &etcdraft.ConfigMetadata{}
   412  				err := proto.Unmarshal(originalMetadata, metadata)
   413  				Expect(err).NotTo(HaveOccurred())
   414  
   415  				// update max in flight messages
   416  				metadata.Options.MaxInflightBlocks = 1000
   417  				metadata.Options.SnapshotIntervalSize = 10 * 1024 * 1024 // 10 MB
   418  
   419  				// write metadata back
   420  				newMetadata, err := proto.Marshal(metadata)
   421  				Expect(err).NotTo(HaveOccurred())
   422  				return newMetadata
   423  			})
   424  
   425  			// assert that no new snapshot is taken because SnapshotIntervalSize has just enlarged
   426  			files, err = ioutil.ReadDir(snapDir)
   427  			Expect(err).NotTo(HaveOccurred())
   428  			Expect(len(files)).To(Equal(numOfSnaps))
   429  
   430  			By("ensuring that static leaders do not give up on retrieving blocks after the orderer goes down")
   431  			ordererProcess.Signal(syscall.SIGTERM)
   432  			Eventually(ordererProcess.Wait(), network.EventuallyTimeout).Should(Receive())
   433  			for _, peerRunner := range peerRunners {
   434  				Eventually(peerRunner.Err(), network.EventuallyTimeout).Should(gbytes.Say("peer is a static leader, ignoring peer.deliveryclient.reconnectTotalTimeThreshold"))
   435  			}
   436  		})
   437  	})
   438  
   439  	Describe("single node etcdraft network with remapped orderer endpoints", func() {
   440  		BeforeEach(func() {
   441  			network = nwo.New(nwo.MinimalRaft(), testDir, client, StartPort(), components)
   442  			network.GenerateConfigTree()
   443  
   444  			configtxConfig := network.ReadConfigTxConfig()
   445  			ordererEndpoints := configtxConfig.Profiles["SampleDevModeEtcdRaft"].Orderer.Organizations[0].OrdererEndpoints
   446  			correctOrdererEndpoint := ordererEndpoints[0]
   447  			ordererEndpoints[0] = "127.0.0.1:1"
   448  			network.WriteConfigTxConfig(configtxConfig)
   449  
   450  			peer := network.Peer("Org1", "peer0")
   451  			peerConfig := network.ReadPeerConfig(peer)
   452  			peerConfig.Peer.Deliveryclient.AddressOverrides = []*fabricconfig.AddressOverride{
   453  				{
   454  					From:        "127.0.0.1:1",
   455  					To:          correctOrdererEndpoint,
   456  					CACertsFile: network.CACertsBundlePath(),
   457  				},
   458  			}
   459  			network.WritePeerConfig(peer, peerConfig)
   460  
   461  			network.Bootstrap()
   462  
   463  			networkRunner := network.NetworkGroupRunner()
   464  			process = ifrit.Invoke(networkRunner)
   465  			Eventually(process.Ready(), network.EventuallyTimeout).Should(BeClosed())
   466  		})
   467  
   468  		It("creates and updates channel", func() {
   469  			orderer := network.Orderer("orderer")
   470  
   471  			network.CreateAndJoinChannel(orderer, "testchannel")
   472  
   473  			// The below call waits for the config update to commit on the peer, so
   474  			// it will fail if the orderer addresses are wrong.
   475  			nwo.EnableCapabilities(network, "testchannel", "Application", "V2_0", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0"))
   476  		})
   477  	})
   478  
   479  	Describe("basic etcdraft network without a system channel", func() {
   480  		var ordererProcess ifrit.Process
   481  		BeforeEach(func() {
   482  			raftConfig := nwo.BasicEtcdRaft()
   483  			network = nwo.New(raftConfig, testDir, client, StartPort(), components)
   484  			network.ChannelParticipationEnabled = true
   485  			network.GenerateConfigTree()
   486  
   487  			orderer := network.Orderer("orderer")
   488  			ordererConfig := network.ReadOrdererConfig(orderer)
   489  			ordererConfig.General.BootstrapMethod = "none"
   490  			network.WriteOrdererConfig(orderer, ordererConfig)
   491  			network.Bootstrap()
   492  
   493  			ordererRunner := network.OrdererRunner(orderer)
   494  			ordererProcess = ifrit.Invoke(ordererRunner)
   495  			Eventually(ordererProcess.Ready, network.EventuallyTimeout).Should(BeClosed())
   496  			Eventually(ordererRunner.Err(), network.EventuallyTimeout).Should(gbytes.Say("Registrar initializing without a system channel, number of application channels: 0"))
   497  		})
   498  
   499  		AfterEach(func() {
   500  			if ordererProcess != nil {
   501  				ordererProcess.Signal(syscall.SIGTERM)
   502  				Eventually(ordererProcess.Wait(), network.EventuallyTimeout).Should(Receive())
   503  			}
   504  		})
   505  
   506  		It("starts the orderer but rejects channel creation requests", func() {
   507  			By("attempting to create a channel without a system channel defined")
   508  			sess, err := network.PeerAdminSession(network.Peer("Org1", "peer0"), commands.ChannelCreate{
   509  				ChannelID:   "testchannel",
   510  				Orderer:     network.OrdererAddress(network.Orderer("orderer"), nwo.ListenPort),
   511  				File:        network.CreateChannelTxPath("testchannel"),
   512  				OutputBlock: "/dev/null",
   513  				ClientAuth:  network.ClientAuthRequired,
   514  			})
   515  			Expect(err).NotTo(HaveOccurred())
   516  			Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(1))
   517  			Eventually(sess.Err, network.EventuallyTimeout).Should(gbytes.Say("channel creation request not allowed because the orderer system channel is not defined"))
   518  		})
   519  	})
   520  
   521  	Describe("basic solo network with containers being interrupted", func() {
   522  		BeforeEach(func() {
   523  			network = nwo.New(nwo.FullSolo(), testDir, client, StartPort(), components)
   524  
   525  			network.GenerateConfigTree()
   526  			network.Bootstrap()
   527  
   528  			networkRunner := network.NetworkGroupRunner()
   529  			process = ifrit.Invoke(networkRunner)
   530  			Eventually(process.Ready(), network.EventuallyTimeout).Should(BeClosed())
   531  		})
   532  
   533  		It("recreates terminated chaincode containers", func() {
   534  			chaincode := nwo.Chaincode{
   535  				Name:            "mycc",
   536  				Version:         "0.0",
   537  				Path:            "github.com/osdi23p228/fabric/integration/chaincode/simple/cmd",
   538  				Lang:            "golang",
   539  				PackageFile:     filepath.Join(testDir, "simplecc.tar.gz"),
   540  				Ctor:            `{"Args":["init","a","100","b","200"]}`,
   541  				SignaturePolicy: `OR ('Org1MSP.peer', 'Org2MSP.peer')`,
   542  				Sequence:        "1",
   543  				InitRequired:    true,
   544  				Label:           "my_simple_chaincode",
   545  			}
   546  
   547  			peer := network.Peers[0]
   548  			orderer := network.Orderer("orderer")
   549  
   550  			By("creating and joining channels")
   551  			network.CreateAndJoinChannels(orderer)
   552  
   553  			By("enabling new lifecycle capabilities")
   554  			nwo.EnableCapabilities(network, "testchannel", "Application", "V2_0", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0"))
   555  			By("deploying the chaincode")
   556  			nwo.DeployChaincode(network, "testchannel", orderer, chaincode)
   557  
   558  			By("querying and invoking chaincode")
   559  			RunQueryInvokeQuery(network, orderer, peer, "testchannel")
   560  
   561  			By("removing chaincode containers from all peers")
   562  			listChaincodeContainers := docker.ListContainersOptions{
   563  				Filters: map[string][]string{
   564  					"name": {chaincodeContainerNameFilter(network, chaincode)},
   565  				},
   566  			}
   567  			ctx := context.Background()
   568  			containers, err := client.ListContainers(listChaincodeContainers)
   569  			Expect(err).NotTo(HaveOccurred())
   570  			Expect(containers).NotTo(BeEmpty())
   571  
   572  			var originalContainerIDs []string
   573  			for _, container := range containers {
   574  				originalContainerIDs = append(originalContainerIDs, container.ID)
   575  				err = client.RemoveContainer(docker.RemoveContainerOptions{
   576  					ID:            container.ID,
   577  					RemoveVolumes: true,
   578  					Force:         true,
   579  					Context:       ctx,
   580  				})
   581  				Expect(err).NotTo(HaveOccurred())
   582  			}
   583  
   584  			By("invoking chaincode against all peers in test channel")
   585  			for _, peer := range network.Peers {
   586  				sess, err := network.PeerUserSession(peer, "User1", commands.ChaincodeInvoke{
   587  					ChannelID: "testchannel",
   588  					Orderer:   network.OrdererAddress(orderer, nwo.ListenPort),
   589  					Name:      "mycc",
   590  					Ctor:      `{"Args":["invoke","a","b","10"]}`,
   591  					PeerAddresses: []string{
   592  						network.PeerAddress(peer, nwo.ListenPort),
   593  					},
   594  					WaitForEvent: true,
   595  				})
   596  				Expect(err).NotTo(HaveOccurred())
   597  				Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(0))
   598  				Expect(sess.Err).To(gbytes.Say("Chaincode invoke successful. result: status:200"))
   599  			}
   600  
   601  			By("checking successful removals of all old chaincode containers")
   602  			newContainers, err := client.ListContainers(listChaincodeContainers)
   603  			Expect(err).NotTo(HaveOccurred())
   604  			Expect(newContainers).To(HaveLen(len(containers)))
   605  
   606  			for _, container := range newContainers {
   607  				Expect(originalContainerIDs).NotTo(ContainElement(container.ID))
   608  			}
   609  		})
   610  	})
   611  })
   612  
   613  func RunQueryInvokeQuery(n *nwo.Network, orderer *nwo.Orderer, peer *nwo.Peer, channel string) {
   614  	By("querying the chaincode")
   615  	sess, err := n.PeerUserSession(peer, "User1", commands.ChaincodeQuery{
   616  		ChannelID: channel,
   617  		Name:      "mycc",
   618  		Ctor:      `{"Args":["query","a"]}`,
   619  	})
   620  	Expect(err).NotTo(HaveOccurred())
   621  	Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(0))
   622  	Expect(sess).To(gbytes.Say("100"))
   623  
   624  	sess, err = n.PeerUserSession(peer, "User1", commands.ChaincodeInvoke{
   625  		ChannelID: channel,
   626  		Orderer:   n.OrdererAddress(orderer, nwo.ListenPort),
   627  		Name:      "mycc",
   628  		Ctor:      `{"Args":["invoke","a","b","10"]}`,
   629  		PeerAddresses: []string{
   630  			n.PeerAddress(n.Peer("Org1", "peer0"), nwo.ListenPort),
   631  			n.PeerAddress(n.Peer("Org2", "peer0"), nwo.ListenPort),
   632  		},
   633  		WaitForEvent: true,
   634  	})
   635  	Expect(err).NotTo(HaveOccurred())
   636  	Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(0))
   637  	Expect(sess.Err).To(gbytes.Say("Chaincode invoke successful. result: status:200"))
   638  
   639  	sess, err = n.PeerUserSession(peer, "User1", commands.ChaincodeQuery{
   640  		ChannelID: channel,
   641  		Name:      "mycc",
   642  		Ctor:      `{"Args":["query","a"]}`,
   643  	})
   644  	Expect(err).NotTo(HaveOccurred())
   645  	Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(0))
   646  	Expect(sess).To(gbytes.Say("90"))
   647  }
   648  
   649  func RunRespondWith(n *nwo.Network, orderer *nwo.Orderer, peer *nwo.Peer, channel string) {
   650  	By("responding with a 300")
   651  	sess, err := n.PeerUserSession(peer, "User1", commands.ChaincodeInvoke{
   652  		ChannelID: channel,
   653  		Orderer:   n.OrdererAddress(orderer, nwo.ListenPort),
   654  		Name:      "mycc",
   655  		Ctor:      `{"Args":["respond","300","response-message","response-payload"]}`,
   656  		PeerAddresses: []string{
   657  			n.PeerAddress(n.Peer("Org1", "peer0"), nwo.ListenPort),
   658  			n.PeerAddress(n.Peer("Org2", "peer0"), nwo.ListenPort),
   659  		},
   660  		WaitForEvent: true,
   661  	})
   662  	Expect(err).NotTo(HaveOccurred())
   663  	Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(0))
   664  	Expect(sess.Err).To(gbytes.Say("Chaincode invoke successful. result: status:300"))
   665  
   666  	By("responding with a 400")
   667  	sess, err = n.PeerUserSession(peer, "User1", commands.ChaincodeInvoke{
   668  		ChannelID: channel,
   669  		Orderer:   n.OrdererAddress(orderer, nwo.ListenPort),
   670  		Name:      "mycc",
   671  		Ctor:      `{"Args":["respond","400","response-message","response-payload"]}`,
   672  		PeerAddresses: []string{
   673  			n.PeerAddress(n.Peer("Org1", "peer0"), nwo.ListenPort),
   674  			n.PeerAddress(n.Peer("Org2", "peer0"), nwo.ListenPort),
   675  		},
   676  		WaitForEvent: true,
   677  	})
   678  	Expect(err).NotTo(HaveOccurred())
   679  	Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(1))
   680  	Expect(sess.Err).To(gbytes.Say(`Error: endorsement failure during invoke.`))
   681  }
   682  
   683  func CheckPeerStatsdMetrics(prefix string, mr *MetricsReader, timeout time.Duration) {
   684  	By("checking for peer statsd metrics")
   685  	Eventually(mr.String, timeout).Should(SatisfyAll(
   686  		ContainSubstring(prefix+".logging.entries_checked.info:"),
   687  		ContainSubstring(prefix+".logging.entries_written.info:"),
   688  		ContainSubstring(prefix+".go.mem.gc_completed_count:"),
   689  		ContainSubstring(prefix+".grpc.server.unary_requests_received.protos_Endorser.ProcessProposal:"),
   690  		ContainSubstring(prefix+".grpc.server.unary_requests_completed.protos_Endorser.ProcessProposal.OK:"),
   691  		ContainSubstring(prefix+".grpc.server.unary_request_duration.protos_Endorser.ProcessProposal.OK:"),
   692  		ContainSubstring(prefix+".ledger.blockchain_height"),
   693  		ContainSubstring(prefix+".ledger.blockstorage_commit_time"),
   694  		ContainSubstring(prefix+".ledger.blockstorage_and_pvtdata_commit_time"),
   695  	))
   696  }
   697  
   698  func CheckPeerStatsdStreamMetrics(mr *MetricsReader, timeout time.Duration) {
   699  	By("checking for stream metrics")
   700  	Eventually(mr.String, timeout).Should(SatisfyAll(
   701  		ContainSubstring(".grpc.server.stream_requests_received.protos_Deliver.DeliverFiltered:"),
   702  		ContainSubstring(".grpc.server.stream_requests_completed.protos_Deliver.DeliverFiltered.Unknown:"),
   703  		ContainSubstring(".grpc.server.stream_request_duration.protos_Deliver.DeliverFiltered.Unknown:"),
   704  		ContainSubstring(".grpc.server.stream_messages_received.protos_Deliver.DeliverFiltered"),
   705  		ContainSubstring(".grpc.server.stream_messages_sent.protos_Deliver.DeliverFiltered"),
   706  	))
   707  }
   708  
   709  func CheckOrdererStatsdMetrics(prefix string, mr *MetricsReader, timeout time.Duration) {
   710  	Eventually(mr.String, timeout).Should(SatisfyAll(
   711  		ContainSubstring(prefix+".grpc.server.stream_request_duration.orderer_AtomicBroadcast.Broadcast.OK"),
   712  		ContainSubstring(prefix+".grpc.server.stream_request_duration.orderer_AtomicBroadcast.Deliver."),
   713  		ContainSubstring(prefix+".logging.entries_checked.info:"),
   714  		ContainSubstring(prefix+".logging.entries_written.info:"),
   715  		ContainSubstring(prefix+".go.mem.gc_completed_count:"),
   716  		ContainSubstring(prefix+".grpc.server.stream_requests_received.orderer_AtomicBroadcast.Deliver:"),
   717  		ContainSubstring(prefix+".grpc.server.stream_requests_completed.orderer_AtomicBroadcast.Deliver."),
   718  		ContainSubstring(prefix+".grpc.server.stream_messages_received.orderer_AtomicBroadcast.Deliver"),
   719  		ContainSubstring(prefix+".grpc.server.stream_messages_sent.orderer_AtomicBroadcast.Deliver"),
   720  		ContainSubstring(prefix+".ledger.blockchain_height"),
   721  		ContainSubstring(prefix+".ledger.blockstorage_commit_time"),
   722  	))
   723  }
   724  
   725  func OrdererOperationalClients(network *nwo.Network, orderer *nwo.Orderer) (authClient, unauthClient *http.Client) {
   726  	return operationalClients(network.OrdererLocalTLSDir(orderer))
   727  }
   728  
   729  func PeerOperationalClients(network *nwo.Network, peer *nwo.Peer) (authClient, unauthClient *http.Client) {
   730  	return operationalClients(network.PeerLocalTLSDir(peer))
   731  }
   732  
   733  func operationalClients(tlsDir string) (authClient, unauthClient *http.Client) {
   734  	clientCert, err := tls.LoadX509KeyPair(
   735  		filepath.Join(tlsDir, "server.crt"),
   736  		filepath.Join(tlsDir, "server.key"),
   737  	)
   738  	Expect(err).NotTo(HaveOccurred())
   739  
   740  	clientCertPool := x509.NewCertPool()
   741  	caCert, err := ioutil.ReadFile(filepath.Join(tlsDir, "ca.crt"))
   742  	Expect(err).NotTo(HaveOccurred())
   743  	clientCertPool.AppendCertsFromPEM(caCert)
   744  
   745  	authenticatedClient := &http.Client{
   746  		Transport: &http.Transport{
   747  			TLSClientConfig: &tls.Config{
   748  				Certificates: []tls.Certificate{clientCert},
   749  				RootCAs:      clientCertPool,
   750  			},
   751  		},
   752  	}
   753  	unauthenticatedClient := &http.Client{
   754  		Transport: &http.Transport{
   755  			TLSClientConfig: &tls.Config{RootCAs: clientCertPool},
   756  		},
   757  	}
   758  
   759  	return authenticatedClient, unauthenticatedClient
   760  }
   761  
   762  func CheckPeerOperationEndpoints(network *nwo.Network, peer *nwo.Peer) {
   763  	metricsURL := fmt.Sprintf("https://127.0.0.1:%d/metrics", network.PeerPort(peer, nwo.OperationsPort))
   764  	logspecURL := fmt.Sprintf("https://127.0.0.1:%d/logspec", network.PeerPort(peer, nwo.OperationsPort))
   765  	healthURL := fmt.Sprintf("https://127.0.0.1:%d/healthz", network.PeerPort(peer, nwo.OperationsPort))
   766  
   767  	authClient, unauthClient := PeerOperationalClients(network, peer)
   768  
   769  	CheckPeerPrometheusMetrics(authClient, metricsURL)
   770  	CheckLogspecOperations(authClient, logspecURL)
   771  	CheckHealthEndpoint(authClient, healthURL)
   772  
   773  	By("getting the logspec without a client cert")
   774  	resp, err := unauthClient.Get(logspecURL)
   775  	Expect(err).NotTo(HaveOccurred())
   776  	Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized))
   777  
   778  	By("ensuring health checks do not require a client cert")
   779  	CheckHealthEndpoint(unauthClient, healthURL)
   780  }
   781  
   782  func CheckOrdererOperationEndpoints(network *nwo.Network, orderer *nwo.Orderer) {
   783  	metricsURL := fmt.Sprintf("https://127.0.0.1:%d/metrics", network.OrdererPort(orderer, nwo.OperationsPort))
   784  	logspecURL := fmt.Sprintf("https://127.0.0.1:%d/logspec", network.OrdererPort(orderer, nwo.OperationsPort))
   785  	healthURL := fmt.Sprintf("https://127.0.0.1:%d/healthz", network.OrdererPort(orderer, nwo.OperationsPort))
   786  
   787  	authClient, unauthClient := OrdererOperationalClients(network, orderer)
   788  
   789  	CheckOrdererPrometheusMetrics(authClient, metricsURL)
   790  	CheckLogspecOperations(authClient, logspecURL)
   791  	CheckHealthEndpoint(authClient, healthURL)
   792  
   793  	By("getting the logspec without a client cert")
   794  	resp, err := unauthClient.Get(logspecURL)
   795  	Expect(err).NotTo(HaveOccurred())
   796  	Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized))
   797  
   798  	By("ensuring health checks do not require a client cert")
   799  	CheckHealthEndpoint(unauthClient, healthURL)
   800  }
   801  
   802  func CheckPeerPrometheusMetrics(client *http.Client, url string) {
   803  	By("hitting the prometheus metrics endpoint")
   804  	resp, err := client.Get(url)
   805  	Expect(err).NotTo(HaveOccurred())
   806  	Expect(resp.StatusCode).To(Equal(http.StatusOK))
   807  	resp.Body.Close()
   808  
   809  	Eventually(getBody(client, url)).Should(ContainSubstring(`# TYPE grpc_server_stream_request_duration histogram`))
   810  
   811  	By("checking for some expected metrics")
   812  	body := getBody(client, url)()
   813  	Expect(body).To(ContainSubstring(`# TYPE go_gc_duration_seconds summary`))
   814  	Expect(body).To(ContainSubstring(`# TYPE grpc_server_stream_request_duration histogram`))
   815  	Expect(body).To(ContainSubstring(`grpc_server_stream_request_duration_count{code="Unknown",method="DeliverFiltered",service="protos_Deliver"}`))
   816  	Expect(body).To(ContainSubstring(`grpc_server_stream_messages_received{method="DeliverFiltered",service="protos_Deliver"}`))
   817  	Expect(body).To(ContainSubstring(`grpc_server_stream_messages_sent{method="DeliverFiltered",service="protos_Deliver"}`))
   818  	Expect(body).To(ContainSubstring(`# TYPE grpc_comm_conn_closed counter`))
   819  	Expect(body).To(ContainSubstring(`# TYPE grpc_comm_conn_opened counter`))
   820  	Expect(body).To(ContainSubstring(`ledger_blockchain_height`))
   821  	Expect(body).To(ContainSubstring(`ledger_blockstorage_commit_time_bucket`))
   822  	Expect(body).To(ContainSubstring(`ledger_blockstorage_and_pvtdata_commit_time_bucket`))
   823  }
   824  
   825  func CheckOrdererPrometheusMetrics(client *http.Client, url string) {
   826  	By("hitting the prometheus metrics endpoint")
   827  	resp, err := client.Get(url)
   828  	Expect(err).NotTo(HaveOccurred())
   829  	Expect(resp.StatusCode).To(Equal(http.StatusOK))
   830  	resp.Body.Close()
   831  
   832  	Eventually(getBody(client, url)).Should(ContainSubstring(`# TYPE grpc_server_stream_request_duration histogram`))
   833  
   834  	By("checking for some expected metrics")
   835  	body := getBody(client, url)()
   836  	Expect(body).To(ContainSubstring(`# TYPE go_gc_duration_seconds summary`))
   837  	Expect(body).To(ContainSubstring(`# TYPE grpc_server_stream_request_duration histogram`))
   838  	Expect(body).To(ContainSubstring(`grpc_server_stream_request_duration_sum{code="OK",method="Deliver",service="orderer_AtomicBroadcast"`))
   839  	Expect(body).To(ContainSubstring(`grpc_server_stream_request_duration_sum{code="OK",method="Broadcast",service="orderer_AtomicBroadcast"`))
   840  	Expect(body).To(ContainSubstring(`# TYPE grpc_comm_conn_closed counter`))
   841  	Expect(body).To(ContainSubstring(`# TYPE grpc_comm_conn_opened counter`))
   842  	Expect(body).To(ContainSubstring(`ledger_blockchain_height`))
   843  	Expect(body).To(ContainSubstring(`ledger_blockstorage_commit_time_bucket`))
   844  }
   845  
   846  func CheckLogspecOperations(client *http.Client, logspecURL string) {
   847  	By("getting the logspec")
   848  	resp, err := client.Get(logspecURL)
   849  	Expect(err).NotTo(HaveOccurred())
   850  	Expect(resp.StatusCode).To(Equal(http.StatusOK))
   851  	bodyBytes, err := ioutil.ReadAll(resp.Body)
   852  	resp.Body.Close()
   853  	Expect(err).NotTo(HaveOccurred())
   854  	Expect(string(bodyBytes)).To(MatchJSON(`{"spec":"info"}`))
   855  
   856  	updateReq, err := http.NewRequest(http.MethodPut, logspecURL, strings.NewReader(`{"spec":"debug"}`))
   857  	Expect(err).NotTo(HaveOccurred())
   858  
   859  	By("setting the logspec")
   860  	resp, err = client.Do(updateReq)
   861  	Expect(err).NotTo(HaveOccurred())
   862  	Expect(resp.StatusCode).To(Equal(http.StatusNoContent))
   863  	resp.Body.Close()
   864  
   865  	resp, err = client.Get(logspecURL)
   866  	Expect(err).NotTo(HaveOccurred())
   867  	Expect(resp.StatusCode).To(Equal(http.StatusOK))
   868  	bodyBytes, err = ioutil.ReadAll(resp.Body)
   869  	resp.Body.Close()
   870  	Expect(err).NotTo(HaveOccurred())
   871  	Expect(string(bodyBytes)).To(MatchJSON(`{"spec":"debug"}`))
   872  
   873  	By("resetting the logspec")
   874  	updateReq, err = http.NewRequest(http.MethodPut, logspecURL, strings.NewReader(`{"spec":"info"}`))
   875  	Expect(err).NotTo(HaveOccurred())
   876  	resp, err = client.Do(updateReq)
   877  	Expect(err).NotTo(HaveOccurred())
   878  	Expect(resp.StatusCode).To(Equal(http.StatusNoContent))
   879  	resp.Body.Close()
   880  }
   881  
   882  func CheckHealthEndpoint(client *http.Client, url string) {
   883  	body := getBody(client, url)()
   884  
   885  	var healthStatus healthz.HealthStatus
   886  	err := json.Unmarshal([]byte(body), &healthStatus)
   887  	Expect(err).NotTo(HaveOccurred())
   888  	Expect(healthStatus.Status).To(Equal(healthz.StatusOK))
   889  }
   890  
   891  func getBody(client *http.Client, url string) func() string {
   892  	return func() string {
   893  		resp, err := client.Get(url)
   894  		Expect(err).NotTo(HaveOccurred())
   895  		Expect(resp.StatusCode).To(Equal(http.StatusOK))
   896  		bodyBytes, err := ioutil.ReadAll(resp.Body)
   897  		Expect(err).NotTo(HaveOccurred())
   898  		resp.Body.Close()
   899  		return string(bodyBytes)
   900  	}
   901  }
   902  
   903  func packageInstallApproveChaincode(network *nwo.Network, channel string, orderer *nwo.Orderer, chaincode nwo.Chaincode, peers ...*nwo.Peer) {
   904  	nwo.PackageChaincode(network, chaincode, peers[0])
   905  	nwo.InstallChaincode(network, chaincode, peers...)
   906  	nwo.ApproveChaincodeForMyOrg(network, channel, orderer, chaincode, peers...)
   907  }
   908  
   909  func hashFile(file string) string {
   910  	f, err := os.Open(file)
   911  	Expect(err).NotTo(HaveOccurred())
   912  	defer f.Close()
   913  
   914  	h := sha256.New()
   915  	_, err = io.Copy(h, f)
   916  	Expect(err).NotTo(HaveOccurred())
   917  
   918  	return fmt.Sprintf("%x", h.Sum(nil))
   919  }
   920  
   921  func chaincodeContainerNameFilter(n *nwo.Network, chaincode nwo.Chaincode) string {
   922  	return fmt.Sprintf("^/%s-.*-%s-%s$", n.NetworkID, chaincode.Label, hashFile(chaincode.PackageFile))
   923  }