github.com/osdi23p228/fabric@v0.0.0-20221218062954-77808885f5db/integration/e2e/e2e_test.go (about) 1 /* 2 Copyright IBM Corp All Rights Reserved. 3 4 SPDX-License-Identifier: Apache-2.0 5 */ 6 7 package e2e 8 9 import ( 10 "bufio" 11 "bytes" 12 "context" 13 "crypto/sha256" 14 "crypto/tls" 15 "crypto/x509" 16 "encoding/json" 17 "fmt" 18 "io" 19 "io/ioutil" 20 "net/http" 21 "os" 22 "path" 23 "path/filepath" 24 "strings" 25 "syscall" 26 "time" 27 28 docker "github.com/fsouza/go-dockerclient" 29 "github.com/golang/protobuf/proto" 30 "github.com/hyperledger/fabric-lib-go/healthz" 31 "github.com/hyperledger/fabric-protos-go/orderer/etcdraft" 32 "github.com/osdi23p228/fabric/integration/nwo" 33 "github.com/osdi23p228/fabric/integration/nwo/commands" 34 "github.com/osdi23p228/fabric/integration/nwo/fabricconfig" 35 . "github.com/onsi/ginkgo" 36 . "github.com/onsi/gomega" 37 "github.com/onsi/gomega/gbytes" 38 "github.com/onsi/gomega/gexec" 39 "github.com/tedsuo/ifrit" 40 "github.com/tedsuo/ifrit/ginkgomon" 41 ) 42 43 var _ = Describe("EndToEnd", func() { 44 var ( 45 testDir string 46 client *docker.Client 47 network *nwo.Network 48 chaincode nwo.Chaincode 49 process ifrit.Process 50 ) 51 52 BeforeEach(func() { 53 var err error 54 testDir, err = ioutil.TempDir("", "e2e") 55 Expect(err).NotTo(HaveOccurred()) 56 57 client, err = docker.NewClientFromEnv() 58 Expect(err).NotTo(HaveOccurred()) 59 60 chaincode = nwo.Chaincode{ 61 Name: "mycc", 62 Version: "0.0", 63 Path: components.Build("github.com/osdi23p228/fabric/integration/chaincode/simple/cmd"), 64 Lang: "binary", 65 PackageFile: filepath.Join(testDir, "simplecc.tar.gz"), 66 Ctor: `{"Args":["init","a","100","b","200"]}`, 67 SignaturePolicy: `AND ('Org1MSP.member','Org2MSP.member')`, 68 Sequence: "1", 69 InitRequired: true, 70 Label: "my_prebuilt_chaincode", 71 } 72 }) 73 74 AfterEach(func() { 75 if process != nil { 76 process.Signal(syscall.SIGTERM) 77 Eventually(process.Wait(), network.EventuallyTimeout).Should(Receive()) 78 } 79 if network != nil { 80 network.Cleanup() 81 } 82 os.RemoveAll(testDir) 83 }) 84 85 Describe("basic solo network with 2 orgs and no docker", func() { 86 var ( 87 metricsReader *MetricsReader 88 runArtifactsFilePath string 89 ) 90 91 BeforeEach(func() { 92 metricsReader = NewMetricsReader() 93 go metricsReader.Start() 94 95 network = nwo.New(nwo.BasicSolo(), testDir, nil, StartPort(), components) 96 network.MetricsProvider = "statsd" 97 network.StatsdEndpoint = metricsReader.Address() 98 network.ChannelParticipationEnabled = true 99 network.Profiles = append(network.Profiles, &nwo.Profile{ 100 Name: "TwoOrgsBaseProfileChannel", 101 Consortium: "SampleConsortium", 102 Orderers: []string{"orderer"}, 103 Organizations: []string{"Org1", "Org2"}, 104 }) 105 network.Channels = append(network.Channels, &nwo.Channel{ 106 Name: "baseprofilechannel", 107 Profile: "TwoOrgsBaseProfileChannel", 108 BaseProfile: "TwoOrgsOrdererGenesis", 109 }) 110 111 runArtifactsFilePath = filepath.Join(testDir, "run-artifacts.txt") 112 os.Setenv("RUN_ARTIFACTS_FILE", runArtifactsFilePath) 113 for i, e := range network.ExternalBuilders { 114 e.PropagateEnvironment = append(e.PropagateEnvironment, "RUN_ARTIFACTS_FILE") 115 network.ExternalBuilders[i] = e 116 } 117 118 network.GenerateConfigTree() 119 for _, peer := range network.PeersWithChannel("testchannel") { 120 core := network.ReadPeerConfig(peer) 121 core.VM = nil 122 network.WritePeerConfig(peer, core) 123 } 124 network.Bootstrap() 125 126 networkRunner := network.NetworkGroupRunner() 127 process = ifrit.Invoke(networkRunner) 128 Eventually(process.Ready(), network.EventuallyTimeout).Should(BeClosed()) 129 }) 130 131 AfterEach(func() { 132 if metricsReader != nil { 133 metricsReader.Close() 134 } 135 136 // Terminate the processes but defer the network cleanup to the outer 137 // AfterEach. 138 if process != nil { 139 process.Signal(syscall.SIGTERM) 140 Eventually(process.Wait(), network.EventuallyTimeout).Should(Receive()) 141 process = nil 142 } 143 144 // Ensure that the temporary directories generated by launched external 145 // chaincodes have been cleaned up. This must be done after the peers 146 // have been terminated. 147 contents, err := ioutil.ReadFile(runArtifactsFilePath) 148 Expect(err).NotTo(HaveOccurred()) 149 scanner := bufio.NewScanner(bytes.NewBuffer(contents)) 150 for scanner.Scan() { 151 Expect(scanner.Text()).NotTo(BeAnExistingFile()) 152 } 153 }) 154 155 It("executes a basic solo network with 2 orgs and no docker", func() { 156 By("getting the orderer by name") 157 orderer := network.Orderer("orderer") 158 159 By("setting up the channel") 160 network.CreateAndJoinChannel(orderer, "testchannel") 161 nwo.EnableCapabilities(network, "testchannel", "Application", "V2_0", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0")) 162 163 By("attempting to install unsupported chaincode without docker") 164 badCC := chaincode 165 badCC.Lang = "unsupported-type" 166 badCC.Label = "chaincode-label" 167 badCC.PackageFile = filepath.Join(testDir, "unsupported-type.tar.gz") 168 nwo.PackageChaincodeBinary(badCC) 169 badCC.SetPackageIDFromPackageFile() 170 sess, err := network.PeerAdminSession( 171 network.Peer("Org1", "peer0"), 172 commands.ChaincodeInstall{ 173 PackageFile: badCC.PackageFile, 174 ClientAuth: network.ClientAuthRequired, 175 }, 176 ) 177 Expect(err).NotTo(HaveOccurred()) 178 Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(1)) 179 Expect(sess.Err).To(gbytes.Say("docker build is disabled")) 180 181 By("deploying the chaincode") 182 nwo.DeployChaincode(network, "testchannel", orderer, chaincode) 183 184 By("ensuring external cc run artifacts exist after deploying") 185 contents, err := ioutil.ReadFile(runArtifactsFilePath) 186 Expect(err).NotTo(HaveOccurred()) 187 scanner := bufio.NewScanner(bytes.NewBuffer(contents)) 188 for scanner.Scan() { 189 Expect(scanner.Text()).To(BeADirectory()) 190 } 191 192 By("getting the client peer by name") 193 peer := network.Peer("Org1", "peer0") 194 195 RunQueryInvokeQuery(network, orderer, peer, "testchannel") 196 RunRespondWith(network, orderer, peer, "testchannel") 197 198 By("evaluating statsd metrics") 199 metricsWriteInterval := 5 * time.Second 200 CheckPeerStatsdStreamMetrics(metricsReader, 2*metricsWriteInterval) 201 CheckPeerStatsdMetrics("org1_peer0", metricsReader, 2*metricsWriteInterval) 202 CheckPeerStatsdMetrics("org2_peer0", metricsReader, 2*metricsWriteInterval) 203 204 By("checking for orderer metrics") 205 CheckOrdererStatsdMetrics("ordererorg_orderer", metricsReader, 2*metricsWriteInterval) 206 207 By("setting up a channel from a base profile") 208 additionalPeer := network.Peer("Org2", "peer0") 209 network.CreateChannel("baseprofilechannel", orderer, peer, additionalPeer) 210 }) 211 }) 212 213 Describe("basic kafka network with 2 orgs", func() { 214 BeforeEach(func() { 215 network = nwo.New(nwo.BasicKafka(), testDir, client, StartPort(), components) 216 network.MetricsProvider = "prometheus" 217 network.ChannelParticipationEnabled = true 218 network.GenerateConfigTree() 219 network.Bootstrap() 220 221 networkRunner := network.NetworkGroupRunner() 222 process = ifrit.Invoke(networkRunner) 223 Eventually(process.Ready(), network.EventuallyTimeout).Should(BeClosed()) 224 }) 225 226 It("executes a basic kafka network with 2 orgs (using docker chaincode builds)", func() { 227 chaincodePath, err := filepath.Abs("../chaincode/module") 228 Expect(err).NotTo(HaveOccurred()) 229 230 // use these two variants of the same chaincode to ensure we test 231 // the golang docker build for both module and gopath chaincode 232 chaincode = nwo.Chaincode{ 233 Name: "mycc", 234 Version: "0.0", 235 Path: chaincodePath, 236 Lang: "golang", 237 PackageFile: filepath.Join(testDir, "modulecc.tar.gz"), 238 Ctor: `{"Args":["init","a","100","b","200"]}`, 239 SignaturePolicy: `AND ('Org1MSP.member','Org2MSP.member')`, 240 Sequence: "1", 241 InitRequired: true, 242 Label: "my_module_chaincode", 243 } 244 245 gopathChaincode := nwo.Chaincode{ 246 Name: "mycc", 247 Version: "0.0", 248 Path: "github.com/osdi23p228/fabric/integration/chaincode/simple/cmd", 249 Lang: "golang", 250 PackageFile: filepath.Join(testDir, "simplecc.tar.gz"), 251 Ctor: `{"Args":["init","a","100","b","200"]}`, 252 SignaturePolicy: `AND ('Org1MSP.member','Org2MSP.member')`, 253 Sequence: "1", 254 InitRequired: true, 255 Label: "my_simple_chaincode", 256 } 257 258 orderer := network.Orderer("orderer") 259 260 network.CreateAndJoinChannel(orderer, "testchannel") 261 nwo.EnableCapabilities(network, "testchannel", "Application", "V2_0", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0")) 262 263 // package, install, and approve by org1 - module chaincode 264 packageInstallApproveChaincode(network, "testchannel", orderer, chaincode, network.Peer("Org1", "peer0")) 265 266 // package, install, and approve by org2 - gopath chaincode, same logic 267 packageInstallApproveChaincode(network, "testchannel", orderer, gopathChaincode, network.Peer("Org2", "peer0")) 268 269 testPeers := network.PeersWithChannel("testchannel") 270 nwo.CheckCommitReadinessUntilReady(network, "testchannel", chaincode, network.PeerOrgs(), testPeers...) 271 nwo.CommitChaincode(network, "testchannel", orderer, chaincode, testPeers[0], testPeers...) 272 nwo.InitChaincode(network, "testchannel", orderer, chaincode, testPeers...) 273 274 By("listing the containers after committing the chaincode definition") 275 initialContainerFilter := map[string][]string{ 276 "name": { 277 chaincodeContainerNameFilter(network, chaincode), 278 chaincodeContainerNameFilter(network, gopathChaincode), 279 }, 280 } 281 282 containers, err := client.ListContainers(docker.ListContainersOptions{Filters: initialContainerFilter}) 283 Expect(err).NotTo(HaveOccurred()) 284 Expect(containers).To(HaveLen(2)) 285 286 RunQueryInvokeQuery(network, orderer, network.Peer("Org1", "peer0"), "testchannel") 287 288 CheckPeerOperationEndpoints(network, network.Peer("Org2", "peer0")) 289 CheckOrdererOperationEndpoints(network, orderer) 290 291 // upgrade chaincode to v2.0 with different label 292 chaincode.Version = "1.0" 293 chaincode.Sequence = "2" 294 chaincode.Label = "my_module_chaincode_updated" 295 gopathChaincode.Version = "1.0" 296 gopathChaincode.Sequence = "2" 297 gopathChaincode.Label = "my_simple_chaincode_updated" 298 299 // package, install, and approve by org1 - module chaincode 300 packageInstallApproveChaincode(network, "testchannel", orderer, chaincode, network.Peer("Org1", "peer0")) 301 302 // package, install, and approve by org2 - gopath chaincode, same logic 303 packageInstallApproveChaincode(network, "testchannel", orderer, gopathChaincode, network.Peer("Org2", "peer0")) 304 305 nwo.CheckCommitReadinessUntilReady(network, "testchannel", chaincode, network.PeerOrgs(), testPeers...) 306 nwo.CommitChaincode(network, "testchannel", orderer, chaincode, testPeers[0], testPeers...) 307 nwo.InitChaincode(network, "testchannel", orderer, chaincode, testPeers...) 308 309 By("listing the containers after updating the chaincode definition") 310 // expect the containers for the previous package id to be stopped 311 containers, err = client.ListContainers(docker.ListContainersOptions{Filters: initialContainerFilter}) 312 Expect(err).NotTo(HaveOccurred()) 313 Expect(containers).To(HaveLen(0)) 314 updatedContainerFilter := map[string][]string{ 315 "name": { 316 chaincodeContainerNameFilter(network, chaincode), 317 chaincodeContainerNameFilter(network, gopathChaincode), 318 }, 319 } 320 containers, err = client.ListContainers(docker.ListContainersOptions{Filters: updatedContainerFilter}) 321 Expect(err).NotTo(HaveOccurred()) 322 Expect(containers).To(HaveLen(2)) 323 324 RunQueryInvokeQuery(network, orderer, network.Peer("Org1", "peer0"), "testchannel") 325 326 By("retrieving the local mspid of the peer via simple chaincode") 327 sess, err := network.PeerUserSession(network.Peer("Org2", "peer0"), "User1", commands.ChaincodeQuery{ 328 ChannelID: "testchannel", 329 Name: "mycc", 330 Ctor: `{"Args":["mspid"]}`, 331 }) 332 Expect(err).NotTo(HaveOccurred()) 333 Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(0)) 334 Expect(sess).To(gbytes.Say("Org2MSP")) 335 }) 336 }) 337 338 Describe("basic single node etcdraft network", func() { 339 var ( 340 peerRunners []*ginkgomon.Runner 341 processes map[string]ifrit.Process 342 ordererProcess ifrit.Process 343 ) 344 345 BeforeEach(func() { 346 network = nwo.New(nwo.MultiChannelEtcdRaft(), testDir, client, StartPort(), components) 347 network.ChannelParticipationEnabled = true 348 network.GenerateConfigTree() 349 for _, peer := range network.Peers { 350 core := network.ReadPeerConfig(peer) 351 core.Peer.Gossip.UseLeaderElection = false 352 core.Peer.Gossip.OrgLeader = true 353 core.Peer.Deliveryclient.ReconnectTotalTimeThreshold = time.Duration(time.Second) 354 network.WritePeerConfig(peer, core) 355 } 356 network.Bootstrap() 357 358 ordererRunner := network.OrdererGroupRunner() 359 ordererProcess = ifrit.Invoke(ordererRunner) 360 Eventually(ordererProcess.Ready(), network.EventuallyTimeout).Should(BeClosed()) 361 362 peerRunners = make([]*ginkgomon.Runner, len(network.Peers)) 363 processes = map[string]ifrit.Process{} 364 for i, peer := range network.Peers { 365 pr := network.PeerRunner(peer) 366 peerRunners[i] = pr 367 p := ifrit.Invoke(pr) 368 processes[peer.ID()] = p 369 Eventually(p.Ready(), network.EventuallyTimeout).Should(BeClosed()) 370 } 371 }) 372 373 AfterEach(func() { 374 if ordererProcess != nil { 375 ordererProcess.Signal(syscall.SIGTERM) 376 Eventually(ordererProcess.Wait(), network.EventuallyTimeout).Should(Receive()) 377 } 378 for _, p := range processes { 379 p.Signal(syscall.SIGTERM) 380 Eventually(p.Wait(), network.EventuallyTimeout).Should(Receive()) 381 } 382 }) 383 384 It("creates two channels with two orgs trying to reconfigure and update metadata", func() { 385 orderer := network.Orderer("orderer") 386 peer := network.Peer("Org1", "peer0") 387 388 By("Create first channel and deploy the chaincode") 389 network.CreateAndJoinChannel(orderer, "testchannel") 390 nwo.EnableCapabilities(network, "testchannel", "Application", "V2_0", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0")) 391 nwo.DeployChaincode(network, "testchannel", orderer, chaincode) 392 RunQueryInvokeQuery(network, orderer, peer, "testchannel") 393 394 By("Create second channel and deploy chaincode") 395 network.CreateAndJoinChannel(orderer, "testchannel2") 396 peers := network.PeersWithChannel("testchannel2") 397 nwo.EnableCapabilities(network, "testchannel2", "Application", "V2_0", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0")) 398 nwo.ApproveChaincodeForMyOrg(network, "testchannel2", orderer, chaincode, peers...) 399 nwo.CheckCommitReadinessUntilReady(network, "testchannel2", chaincode, network.PeerOrgs(), peers...) 400 nwo.CommitChaincode(network, "testchannel2", orderer, chaincode, peers[0], peers...) 401 nwo.InitChaincode(network, "testchannel2", orderer, chaincode, peers...) 402 RunQueryInvokeQuery(network, orderer, peer, "testchannel2") 403 404 By("Update consensus metadata to increase snapshot interval") 405 snapDir := path.Join(network.RootDir, "orderers", orderer.ID(), "etcdraft", "snapshot", "testchannel") 406 files, err := ioutil.ReadDir(snapDir) 407 Expect(err).NotTo(HaveOccurred()) 408 numOfSnaps := len(files) 409 410 nwo.UpdateConsensusMetadata(network, peer, orderer, "testchannel", func(originalMetadata []byte) []byte { 411 metadata := &etcdraft.ConfigMetadata{} 412 err := proto.Unmarshal(originalMetadata, metadata) 413 Expect(err).NotTo(HaveOccurred()) 414 415 // update max in flight messages 416 metadata.Options.MaxInflightBlocks = 1000 417 metadata.Options.SnapshotIntervalSize = 10 * 1024 * 1024 // 10 MB 418 419 // write metadata back 420 newMetadata, err := proto.Marshal(metadata) 421 Expect(err).NotTo(HaveOccurred()) 422 return newMetadata 423 }) 424 425 // assert that no new snapshot is taken because SnapshotIntervalSize has just enlarged 426 files, err = ioutil.ReadDir(snapDir) 427 Expect(err).NotTo(HaveOccurred()) 428 Expect(len(files)).To(Equal(numOfSnaps)) 429 430 By("ensuring that static leaders do not give up on retrieving blocks after the orderer goes down") 431 ordererProcess.Signal(syscall.SIGTERM) 432 Eventually(ordererProcess.Wait(), network.EventuallyTimeout).Should(Receive()) 433 for _, peerRunner := range peerRunners { 434 Eventually(peerRunner.Err(), network.EventuallyTimeout).Should(gbytes.Say("peer is a static leader, ignoring peer.deliveryclient.reconnectTotalTimeThreshold")) 435 } 436 }) 437 }) 438 439 Describe("single node etcdraft network with remapped orderer endpoints", func() { 440 BeforeEach(func() { 441 network = nwo.New(nwo.MinimalRaft(), testDir, client, StartPort(), components) 442 network.GenerateConfigTree() 443 444 configtxConfig := network.ReadConfigTxConfig() 445 ordererEndpoints := configtxConfig.Profiles["SampleDevModeEtcdRaft"].Orderer.Organizations[0].OrdererEndpoints 446 correctOrdererEndpoint := ordererEndpoints[0] 447 ordererEndpoints[0] = "127.0.0.1:1" 448 network.WriteConfigTxConfig(configtxConfig) 449 450 peer := network.Peer("Org1", "peer0") 451 peerConfig := network.ReadPeerConfig(peer) 452 peerConfig.Peer.Deliveryclient.AddressOverrides = []*fabricconfig.AddressOverride{ 453 { 454 From: "127.0.0.1:1", 455 To: correctOrdererEndpoint, 456 CACertsFile: network.CACertsBundlePath(), 457 }, 458 } 459 network.WritePeerConfig(peer, peerConfig) 460 461 network.Bootstrap() 462 463 networkRunner := network.NetworkGroupRunner() 464 process = ifrit.Invoke(networkRunner) 465 Eventually(process.Ready(), network.EventuallyTimeout).Should(BeClosed()) 466 }) 467 468 It("creates and updates channel", func() { 469 orderer := network.Orderer("orderer") 470 471 network.CreateAndJoinChannel(orderer, "testchannel") 472 473 // The below call waits for the config update to commit on the peer, so 474 // it will fail if the orderer addresses are wrong. 475 nwo.EnableCapabilities(network, "testchannel", "Application", "V2_0", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0")) 476 }) 477 }) 478 479 Describe("basic etcdraft network without a system channel", func() { 480 var ordererProcess ifrit.Process 481 BeforeEach(func() { 482 raftConfig := nwo.BasicEtcdRaft() 483 network = nwo.New(raftConfig, testDir, client, StartPort(), components) 484 network.ChannelParticipationEnabled = true 485 network.GenerateConfigTree() 486 487 orderer := network.Orderer("orderer") 488 ordererConfig := network.ReadOrdererConfig(orderer) 489 ordererConfig.General.BootstrapMethod = "none" 490 network.WriteOrdererConfig(orderer, ordererConfig) 491 network.Bootstrap() 492 493 ordererRunner := network.OrdererRunner(orderer) 494 ordererProcess = ifrit.Invoke(ordererRunner) 495 Eventually(ordererProcess.Ready, network.EventuallyTimeout).Should(BeClosed()) 496 Eventually(ordererRunner.Err(), network.EventuallyTimeout).Should(gbytes.Say("Registrar initializing without a system channel, number of application channels: 0")) 497 }) 498 499 AfterEach(func() { 500 if ordererProcess != nil { 501 ordererProcess.Signal(syscall.SIGTERM) 502 Eventually(ordererProcess.Wait(), network.EventuallyTimeout).Should(Receive()) 503 } 504 }) 505 506 It("starts the orderer but rejects channel creation requests", func() { 507 By("attempting to create a channel without a system channel defined") 508 sess, err := network.PeerAdminSession(network.Peer("Org1", "peer0"), commands.ChannelCreate{ 509 ChannelID: "testchannel", 510 Orderer: network.OrdererAddress(network.Orderer("orderer"), nwo.ListenPort), 511 File: network.CreateChannelTxPath("testchannel"), 512 OutputBlock: "/dev/null", 513 ClientAuth: network.ClientAuthRequired, 514 }) 515 Expect(err).NotTo(HaveOccurred()) 516 Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(1)) 517 Eventually(sess.Err, network.EventuallyTimeout).Should(gbytes.Say("channel creation request not allowed because the orderer system channel is not defined")) 518 }) 519 }) 520 521 Describe("basic solo network with containers being interrupted", func() { 522 BeforeEach(func() { 523 network = nwo.New(nwo.FullSolo(), testDir, client, StartPort(), components) 524 525 network.GenerateConfigTree() 526 network.Bootstrap() 527 528 networkRunner := network.NetworkGroupRunner() 529 process = ifrit.Invoke(networkRunner) 530 Eventually(process.Ready(), network.EventuallyTimeout).Should(BeClosed()) 531 }) 532 533 It("recreates terminated chaincode containers", func() { 534 chaincode := nwo.Chaincode{ 535 Name: "mycc", 536 Version: "0.0", 537 Path: "github.com/osdi23p228/fabric/integration/chaincode/simple/cmd", 538 Lang: "golang", 539 PackageFile: filepath.Join(testDir, "simplecc.tar.gz"), 540 Ctor: `{"Args":["init","a","100","b","200"]}`, 541 SignaturePolicy: `OR ('Org1MSP.peer', 'Org2MSP.peer')`, 542 Sequence: "1", 543 InitRequired: true, 544 Label: "my_simple_chaincode", 545 } 546 547 peer := network.Peers[0] 548 orderer := network.Orderer("orderer") 549 550 By("creating and joining channels") 551 network.CreateAndJoinChannels(orderer) 552 553 By("enabling new lifecycle capabilities") 554 nwo.EnableCapabilities(network, "testchannel", "Application", "V2_0", orderer, network.Peer("Org1", "peer0"), network.Peer("Org2", "peer0")) 555 By("deploying the chaincode") 556 nwo.DeployChaincode(network, "testchannel", orderer, chaincode) 557 558 By("querying and invoking chaincode") 559 RunQueryInvokeQuery(network, orderer, peer, "testchannel") 560 561 By("removing chaincode containers from all peers") 562 listChaincodeContainers := docker.ListContainersOptions{ 563 Filters: map[string][]string{ 564 "name": {chaincodeContainerNameFilter(network, chaincode)}, 565 }, 566 } 567 ctx := context.Background() 568 containers, err := client.ListContainers(listChaincodeContainers) 569 Expect(err).NotTo(HaveOccurred()) 570 Expect(containers).NotTo(BeEmpty()) 571 572 var originalContainerIDs []string 573 for _, container := range containers { 574 originalContainerIDs = append(originalContainerIDs, container.ID) 575 err = client.RemoveContainer(docker.RemoveContainerOptions{ 576 ID: container.ID, 577 RemoveVolumes: true, 578 Force: true, 579 Context: ctx, 580 }) 581 Expect(err).NotTo(HaveOccurred()) 582 } 583 584 By("invoking chaincode against all peers in test channel") 585 for _, peer := range network.Peers { 586 sess, err := network.PeerUserSession(peer, "User1", commands.ChaincodeInvoke{ 587 ChannelID: "testchannel", 588 Orderer: network.OrdererAddress(orderer, nwo.ListenPort), 589 Name: "mycc", 590 Ctor: `{"Args":["invoke","a","b","10"]}`, 591 PeerAddresses: []string{ 592 network.PeerAddress(peer, nwo.ListenPort), 593 }, 594 WaitForEvent: true, 595 }) 596 Expect(err).NotTo(HaveOccurred()) 597 Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(0)) 598 Expect(sess.Err).To(gbytes.Say("Chaincode invoke successful. result: status:200")) 599 } 600 601 By("checking successful removals of all old chaincode containers") 602 newContainers, err := client.ListContainers(listChaincodeContainers) 603 Expect(err).NotTo(HaveOccurred()) 604 Expect(newContainers).To(HaveLen(len(containers))) 605 606 for _, container := range newContainers { 607 Expect(originalContainerIDs).NotTo(ContainElement(container.ID)) 608 } 609 }) 610 }) 611 }) 612 613 func RunQueryInvokeQuery(n *nwo.Network, orderer *nwo.Orderer, peer *nwo.Peer, channel string) { 614 By("querying the chaincode") 615 sess, err := n.PeerUserSession(peer, "User1", commands.ChaincodeQuery{ 616 ChannelID: channel, 617 Name: "mycc", 618 Ctor: `{"Args":["query","a"]}`, 619 }) 620 Expect(err).NotTo(HaveOccurred()) 621 Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(0)) 622 Expect(sess).To(gbytes.Say("100")) 623 624 sess, err = n.PeerUserSession(peer, "User1", commands.ChaincodeInvoke{ 625 ChannelID: channel, 626 Orderer: n.OrdererAddress(orderer, nwo.ListenPort), 627 Name: "mycc", 628 Ctor: `{"Args":["invoke","a","b","10"]}`, 629 PeerAddresses: []string{ 630 n.PeerAddress(n.Peer("Org1", "peer0"), nwo.ListenPort), 631 n.PeerAddress(n.Peer("Org2", "peer0"), nwo.ListenPort), 632 }, 633 WaitForEvent: true, 634 }) 635 Expect(err).NotTo(HaveOccurred()) 636 Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(0)) 637 Expect(sess.Err).To(gbytes.Say("Chaincode invoke successful. result: status:200")) 638 639 sess, err = n.PeerUserSession(peer, "User1", commands.ChaincodeQuery{ 640 ChannelID: channel, 641 Name: "mycc", 642 Ctor: `{"Args":["query","a"]}`, 643 }) 644 Expect(err).NotTo(HaveOccurred()) 645 Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(0)) 646 Expect(sess).To(gbytes.Say("90")) 647 } 648 649 func RunRespondWith(n *nwo.Network, orderer *nwo.Orderer, peer *nwo.Peer, channel string) { 650 By("responding with a 300") 651 sess, err := n.PeerUserSession(peer, "User1", commands.ChaincodeInvoke{ 652 ChannelID: channel, 653 Orderer: n.OrdererAddress(orderer, nwo.ListenPort), 654 Name: "mycc", 655 Ctor: `{"Args":["respond","300","response-message","response-payload"]}`, 656 PeerAddresses: []string{ 657 n.PeerAddress(n.Peer("Org1", "peer0"), nwo.ListenPort), 658 n.PeerAddress(n.Peer("Org2", "peer0"), nwo.ListenPort), 659 }, 660 WaitForEvent: true, 661 }) 662 Expect(err).NotTo(HaveOccurred()) 663 Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(0)) 664 Expect(sess.Err).To(gbytes.Say("Chaincode invoke successful. result: status:300")) 665 666 By("responding with a 400") 667 sess, err = n.PeerUserSession(peer, "User1", commands.ChaincodeInvoke{ 668 ChannelID: channel, 669 Orderer: n.OrdererAddress(orderer, nwo.ListenPort), 670 Name: "mycc", 671 Ctor: `{"Args":["respond","400","response-message","response-payload"]}`, 672 PeerAddresses: []string{ 673 n.PeerAddress(n.Peer("Org1", "peer0"), nwo.ListenPort), 674 n.PeerAddress(n.Peer("Org2", "peer0"), nwo.ListenPort), 675 }, 676 WaitForEvent: true, 677 }) 678 Expect(err).NotTo(HaveOccurred()) 679 Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(1)) 680 Expect(sess.Err).To(gbytes.Say(`Error: endorsement failure during invoke.`)) 681 } 682 683 func CheckPeerStatsdMetrics(prefix string, mr *MetricsReader, timeout time.Duration) { 684 By("checking for peer statsd metrics") 685 Eventually(mr.String, timeout).Should(SatisfyAll( 686 ContainSubstring(prefix+".logging.entries_checked.info:"), 687 ContainSubstring(prefix+".logging.entries_written.info:"), 688 ContainSubstring(prefix+".go.mem.gc_completed_count:"), 689 ContainSubstring(prefix+".grpc.server.unary_requests_received.protos_Endorser.ProcessProposal:"), 690 ContainSubstring(prefix+".grpc.server.unary_requests_completed.protos_Endorser.ProcessProposal.OK:"), 691 ContainSubstring(prefix+".grpc.server.unary_request_duration.protos_Endorser.ProcessProposal.OK:"), 692 ContainSubstring(prefix+".ledger.blockchain_height"), 693 ContainSubstring(prefix+".ledger.blockstorage_commit_time"), 694 ContainSubstring(prefix+".ledger.blockstorage_and_pvtdata_commit_time"), 695 )) 696 } 697 698 func CheckPeerStatsdStreamMetrics(mr *MetricsReader, timeout time.Duration) { 699 By("checking for stream metrics") 700 Eventually(mr.String, timeout).Should(SatisfyAll( 701 ContainSubstring(".grpc.server.stream_requests_received.protos_Deliver.DeliverFiltered:"), 702 ContainSubstring(".grpc.server.stream_requests_completed.protos_Deliver.DeliverFiltered.Unknown:"), 703 ContainSubstring(".grpc.server.stream_request_duration.protos_Deliver.DeliverFiltered.Unknown:"), 704 ContainSubstring(".grpc.server.stream_messages_received.protos_Deliver.DeliverFiltered"), 705 ContainSubstring(".grpc.server.stream_messages_sent.protos_Deliver.DeliverFiltered"), 706 )) 707 } 708 709 func CheckOrdererStatsdMetrics(prefix string, mr *MetricsReader, timeout time.Duration) { 710 Eventually(mr.String, timeout).Should(SatisfyAll( 711 ContainSubstring(prefix+".grpc.server.stream_request_duration.orderer_AtomicBroadcast.Broadcast.OK"), 712 ContainSubstring(prefix+".grpc.server.stream_request_duration.orderer_AtomicBroadcast.Deliver."), 713 ContainSubstring(prefix+".logging.entries_checked.info:"), 714 ContainSubstring(prefix+".logging.entries_written.info:"), 715 ContainSubstring(prefix+".go.mem.gc_completed_count:"), 716 ContainSubstring(prefix+".grpc.server.stream_requests_received.orderer_AtomicBroadcast.Deliver:"), 717 ContainSubstring(prefix+".grpc.server.stream_requests_completed.orderer_AtomicBroadcast.Deliver."), 718 ContainSubstring(prefix+".grpc.server.stream_messages_received.orderer_AtomicBroadcast.Deliver"), 719 ContainSubstring(prefix+".grpc.server.stream_messages_sent.orderer_AtomicBroadcast.Deliver"), 720 ContainSubstring(prefix+".ledger.blockchain_height"), 721 ContainSubstring(prefix+".ledger.blockstorage_commit_time"), 722 )) 723 } 724 725 func OrdererOperationalClients(network *nwo.Network, orderer *nwo.Orderer) (authClient, unauthClient *http.Client) { 726 return operationalClients(network.OrdererLocalTLSDir(orderer)) 727 } 728 729 func PeerOperationalClients(network *nwo.Network, peer *nwo.Peer) (authClient, unauthClient *http.Client) { 730 return operationalClients(network.PeerLocalTLSDir(peer)) 731 } 732 733 func operationalClients(tlsDir string) (authClient, unauthClient *http.Client) { 734 clientCert, err := tls.LoadX509KeyPair( 735 filepath.Join(tlsDir, "server.crt"), 736 filepath.Join(tlsDir, "server.key"), 737 ) 738 Expect(err).NotTo(HaveOccurred()) 739 740 clientCertPool := x509.NewCertPool() 741 caCert, err := ioutil.ReadFile(filepath.Join(tlsDir, "ca.crt")) 742 Expect(err).NotTo(HaveOccurred()) 743 clientCertPool.AppendCertsFromPEM(caCert) 744 745 authenticatedClient := &http.Client{ 746 Transport: &http.Transport{ 747 TLSClientConfig: &tls.Config{ 748 Certificates: []tls.Certificate{clientCert}, 749 RootCAs: clientCertPool, 750 }, 751 }, 752 } 753 unauthenticatedClient := &http.Client{ 754 Transport: &http.Transport{ 755 TLSClientConfig: &tls.Config{RootCAs: clientCertPool}, 756 }, 757 } 758 759 return authenticatedClient, unauthenticatedClient 760 } 761 762 func CheckPeerOperationEndpoints(network *nwo.Network, peer *nwo.Peer) { 763 metricsURL := fmt.Sprintf("https://127.0.0.1:%d/metrics", network.PeerPort(peer, nwo.OperationsPort)) 764 logspecURL := fmt.Sprintf("https://127.0.0.1:%d/logspec", network.PeerPort(peer, nwo.OperationsPort)) 765 healthURL := fmt.Sprintf("https://127.0.0.1:%d/healthz", network.PeerPort(peer, nwo.OperationsPort)) 766 767 authClient, unauthClient := PeerOperationalClients(network, peer) 768 769 CheckPeerPrometheusMetrics(authClient, metricsURL) 770 CheckLogspecOperations(authClient, logspecURL) 771 CheckHealthEndpoint(authClient, healthURL) 772 773 By("getting the logspec without a client cert") 774 resp, err := unauthClient.Get(logspecURL) 775 Expect(err).NotTo(HaveOccurred()) 776 Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) 777 778 By("ensuring health checks do not require a client cert") 779 CheckHealthEndpoint(unauthClient, healthURL) 780 } 781 782 func CheckOrdererOperationEndpoints(network *nwo.Network, orderer *nwo.Orderer) { 783 metricsURL := fmt.Sprintf("https://127.0.0.1:%d/metrics", network.OrdererPort(orderer, nwo.OperationsPort)) 784 logspecURL := fmt.Sprintf("https://127.0.0.1:%d/logspec", network.OrdererPort(orderer, nwo.OperationsPort)) 785 healthURL := fmt.Sprintf("https://127.0.0.1:%d/healthz", network.OrdererPort(orderer, nwo.OperationsPort)) 786 787 authClient, unauthClient := OrdererOperationalClients(network, orderer) 788 789 CheckOrdererPrometheusMetrics(authClient, metricsURL) 790 CheckLogspecOperations(authClient, logspecURL) 791 CheckHealthEndpoint(authClient, healthURL) 792 793 By("getting the logspec without a client cert") 794 resp, err := unauthClient.Get(logspecURL) 795 Expect(err).NotTo(HaveOccurred()) 796 Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) 797 798 By("ensuring health checks do not require a client cert") 799 CheckHealthEndpoint(unauthClient, healthURL) 800 } 801 802 func CheckPeerPrometheusMetrics(client *http.Client, url string) { 803 By("hitting the prometheus metrics endpoint") 804 resp, err := client.Get(url) 805 Expect(err).NotTo(HaveOccurred()) 806 Expect(resp.StatusCode).To(Equal(http.StatusOK)) 807 resp.Body.Close() 808 809 Eventually(getBody(client, url)).Should(ContainSubstring(`# TYPE grpc_server_stream_request_duration histogram`)) 810 811 By("checking for some expected metrics") 812 body := getBody(client, url)() 813 Expect(body).To(ContainSubstring(`# TYPE go_gc_duration_seconds summary`)) 814 Expect(body).To(ContainSubstring(`# TYPE grpc_server_stream_request_duration histogram`)) 815 Expect(body).To(ContainSubstring(`grpc_server_stream_request_duration_count{code="Unknown",method="DeliverFiltered",service="protos_Deliver"}`)) 816 Expect(body).To(ContainSubstring(`grpc_server_stream_messages_received{method="DeliverFiltered",service="protos_Deliver"}`)) 817 Expect(body).To(ContainSubstring(`grpc_server_stream_messages_sent{method="DeliverFiltered",service="protos_Deliver"}`)) 818 Expect(body).To(ContainSubstring(`# TYPE grpc_comm_conn_closed counter`)) 819 Expect(body).To(ContainSubstring(`# TYPE grpc_comm_conn_opened counter`)) 820 Expect(body).To(ContainSubstring(`ledger_blockchain_height`)) 821 Expect(body).To(ContainSubstring(`ledger_blockstorage_commit_time_bucket`)) 822 Expect(body).To(ContainSubstring(`ledger_blockstorage_and_pvtdata_commit_time_bucket`)) 823 } 824 825 func CheckOrdererPrometheusMetrics(client *http.Client, url string) { 826 By("hitting the prometheus metrics endpoint") 827 resp, err := client.Get(url) 828 Expect(err).NotTo(HaveOccurred()) 829 Expect(resp.StatusCode).To(Equal(http.StatusOK)) 830 resp.Body.Close() 831 832 Eventually(getBody(client, url)).Should(ContainSubstring(`# TYPE grpc_server_stream_request_duration histogram`)) 833 834 By("checking for some expected metrics") 835 body := getBody(client, url)() 836 Expect(body).To(ContainSubstring(`# TYPE go_gc_duration_seconds summary`)) 837 Expect(body).To(ContainSubstring(`# TYPE grpc_server_stream_request_duration histogram`)) 838 Expect(body).To(ContainSubstring(`grpc_server_stream_request_duration_sum{code="OK",method="Deliver",service="orderer_AtomicBroadcast"`)) 839 Expect(body).To(ContainSubstring(`grpc_server_stream_request_duration_sum{code="OK",method="Broadcast",service="orderer_AtomicBroadcast"`)) 840 Expect(body).To(ContainSubstring(`# TYPE grpc_comm_conn_closed counter`)) 841 Expect(body).To(ContainSubstring(`# TYPE grpc_comm_conn_opened counter`)) 842 Expect(body).To(ContainSubstring(`ledger_blockchain_height`)) 843 Expect(body).To(ContainSubstring(`ledger_blockstorage_commit_time_bucket`)) 844 } 845 846 func CheckLogspecOperations(client *http.Client, logspecURL string) { 847 By("getting the logspec") 848 resp, err := client.Get(logspecURL) 849 Expect(err).NotTo(HaveOccurred()) 850 Expect(resp.StatusCode).To(Equal(http.StatusOK)) 851 bodyBytes, err := ioutil.ReadAll(resp.Body) 852 resp.Body.Close() 853 Expect(err).NotTo(HaveOccurred()) 854 Expect(string(bodyBytes)).To(MatchJSON(`{"spec":"info"}`)) 855 856 updateReq, err := http.NewRequest(http.MethodPut, logspecURL, strings.NewReader(`{"spec":"debug"}`)) 857 Expect(err).NotTo(HaveOccurred()) 858 859 By("setting the logspec") 860 resp, err = client.Do(updateReq) 861 Expect(err).NotTo(HaveOccurred()) 862 Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) 863 resp.Body.Close() 864 865 resp, err = client.Get(logspecURL) 866 Expect(err).NotTo(HaveOccurred()) 867 Expect(resp.StatusCode).To(Equal(http.StatusOK)) 868 bodyBytes, err = ioutil.ReadAll(resp.Body) 869 resp.Body.Close() 870 Expect(err).NotTo(HaveOccurred()) 871 Expect(string(bodyBytes)).To(MatchJSON(`{"spec":"debug"}`)) 872 873 By("resetting the logspec") 874 updateReq, err = http.NewRequest(http.MethodPut, logspecURL, strings.NewReader(`{"spec":"info"}`)) 875 Expect(err).NotTo(HaveOccurred()) 876 resp, err = client.Do(updateReq) 877 Expect(err).NotTo(HaveOccurred()) 878 Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) 879 resp.Body.Close() 880 } 881 882 func CheckHealthEndpoint(client *http.Client, url string) { 883 body := getBody(client, url)() 884 885 var healthStatus healthz.HealthStatus 886 err := json.Unmarshal([]byte(body), &healthStatus) 887 Expect(err).NotTo(HaveOccurred()) 888 Expect(healthStatus.Status).To(Equal(healthz.StatusOK)) 889 } 890 891 func getBody(client *http.Client, url string) func() string { 892 return func() string { 893 resp, err := client.Get(url) 894 Expect(err).NotTo(HaveOccurred()) 895 Expect(resp.StatusCode).To(Equal(http.StatusOK)) 896 bodyBytes, err := ioutil.ReadAll(resp.Body) 897 Expect(err).NotTo(HaveOccurred()) 898 resp.Body.Close() 899 return string(bodyBytes) 900 } 901 } 902 903 func packageInstallApproveChaincode(network *nwo.Network, channel string, orderer *nwo.Orderer, chaincode nwo.Chaincode, peers ...*nwo.Peer) { 904 nwo.PackageChaincode(network, chaincode, peers[0]) 905 nwo.InstallChaincode(network, chaincode, peers...) 906 nwo.ApproveChaincodeForMyOrg(network, channel, orderer, chaincode, peers...) 907 } 908 909 func hashFile(file string) string { 910 f, err := os.Open(file) 911 Expect(err).NotTo(HaveOccurred()) 912 defer f.Close() 913 914 h := sha256.New() 915 _, err = io.Copy(h, f) 916 Expect(err).NotTo(HaveOccurred()) 917 918 return fmt.Sprintf("%x", h.Sum(nil)) 919 } 920 921 func chaincodeContainerNameFilter(n *nwo.Network, chaincode nwo.Chaincode) string { 922 return fmt.Sprintf("^/%s-.*-%s-%s$", n.NetworkID, chaincode.Label, hashFile(chaincode.PackageFile)) 923 }