github.com/osdi23p228/fabric@v0.0.0-20221218062954-77808885f5db/integration/raft/cft_test.go (about) 1 /* 2 Copyright IBM Corp All Rights Reserved. 3 4 SPDX-License-Identifier: Apache-2.0 5 */ 6 7 package raft 8 9 import ( 10 "crypto/ecdsa" 11 "crypto/rand" 12 "crypto/x509" 13 "encoding/pem" 14 "fmt" 15 "io/ioutil" 16 "os" 17 "path" 18 "path/filepath" 19 "strconv" 20 "strings" 21 "sync" 22 "syscall" 23 "time" 24 25 docker "github.com/fsouza/go-dockerclient" 26 "github.com/golang/protobuf/proto" 27 conftx "github.com/hyperledger/fabric-config/configtx" 28 "github.com/hyperledger/fabric-config/configtx/orderer" 29 "github.com/hyperledger/fabric-protos-go/common" 30 "github.com/hyperledger/fabric-protos-go/msp" 31 "github.com/osdi23p228/fabric/cmd/common/signer" 32 "github.com/osdi23p228/fabric/common/configtx" 33 "github.com/osdi23p228/fabric/common/util" 34 intconftx "github.com/osdi23p228/fabric/integration/configtx" 35 "github.com/osdi23p228/fabric/integration/nwo" 36 "github.com/osdi23p228/fabric/integration/nwo/commands" 37 "github.com/osdi23p228/fabric/protoutil" 38 . "github.com/onsi/ginkgo" 39 . "github.com/onsi/gomega" 40 "github.com/onsi/gomega/gbytes" 41 "github.com/onsi/gomega/gexec" 42 "github.com/tedsuo/ifrit" 43 "github.com/tedsuo/ifrit/ginkgomon" 44 "github.com/tedsuo/ifrit/grouper" 45 ) 46 47 var _ = Describe("EndToEnd Crash Fault Tolerance", func() { 48 var ( 49 testDir string 50 client *docker.Client 51 network *nwo.Network 52 peer *nwo.Peer 53 54 ordererProc, o1Proc, o2Proc, o3Proc ifrit.Process 55 ) 56 57 BeforeEach(func() { 58 var err error 59 testDir, err = ioutil.TempDir("", "e2e") 60 Expect(err).NotTo(HaveOccurred()) 61 62 client, err = docker.NewClientFromEnv() 63 Expect(err).NotTo(HaveOccurred()) 64 }) 65 66 AfterEach(func() { 67 for _, oProc := range []ifrit.Process{o1Proc, o2Proc, o3Proc} { 68 if oProc != nil { 69 oProc.Signal(syscall.SIGTERM) 70 Eventually(oProc.Wait(), network.EventuallyTimeout).Should(Receive()) 71 } 72 } 73 74 if ordererProc != nil { 75 ordererProc.Signal(syscall.SIGTERM) 76 Eventually(ordererProc.Wait(), network.EventuallyTimeout).Should(Receive()) 77 } 78 79 if network != nil { 80 network.Cleanup() 81 } 82 os.RemoveAll(testDir) 83 }) 84 85 When("orderer stops and restarts", func() { 86 It("keeps network up and running", func() { 87 network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, StartPort(), components) 88 89 o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3") 90 peer = network.Peer("Org1", "peer0") 91 92 network.GenerateConfigTree() 93 network.Bootstrap() 94 95 o1Runner := network.OrdererRunner(o1) 96 // Enable debug log for orderer2 so we could assert its content later 97 o2Runner := network.OrdererRunner(o2, "FABRIC_LOGGING_SPEC=orderer.consensus.etcdraft=debug:info") 98 o3Runner := network.OrdererRunner(o3) 99 orderers := grouper.Members{ 100 {Name: o2.ID(), Runner: o2Runner}, 101 {Name: o3.ID(), Runner: o3Runner}, 102 } 103 ordererGroup := grouper.NewParallel(syscall.SIGTERM, orderers) 104 105 o1Proc = ifrit.Invoke(o1Runner) 106 ordererProc = ifrit.Invoke(ordererGroup) 107 Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 108 Eventually(ordererProc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 109 110 findLeader([]*ginkgomon.Runner{o1Runner}) 111 112 By("performing operation with orderer1") 113 env := CreateBroadcastEnvelope(network, o1, network.SystemChannel.Name, []byte("foo")) 114 resp, err := nwo.Broadcast(network, o1, env) 115 Expect(err).NotTo(HaveOccurred()) 116 Expect(resp.Status).To(Equal(common.Status_SUCCESS)) 117 118 block := FetchBlock(network, o1, 1, network.SystemChannel.Name) 119 Expect(block).NotTo(BeNil()) 120 121 By("killing orderer1") 122 o1Proc.Signal(syscall.SIGKILL) 123 Eventually(o1Proc.Wait(), network.EventuallyTimeout).Should(Receive(MatchError("exit status 137"))) 124 125 By("observing active nodes to shrink") 126 Eventually(o2Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("Current active nodes in cluster are: \\[2 3\\]")) 127 128 By("broadcasting envelope to running orderer") 129 resp, err = nwo.Broadcast(network, o2, env) 130 Expect(err).NotTo(HaveOccurred()) 131 Expect(resp.Status).To(Equal(common.Status_SUCCESS)) 132 133 block = FetchBlock(network, o2, 2, network.SystemChannel.Name) 134 Expect(block).NotTo(BeNil()) 135 136 By("restarting orderer1") 137 o1Runner = network.OrdererRunner(o1) 138 o1Proc = ifrit.Invoke(o1Runner) 139 Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 140 findLeader([]*ginkgomon.Runner{o1Runner}) 141 142 By("broadcasting envelope to restarted orderer") 143 resp, err = nwo.Broadcast(network, o1, env) 144 Expect(err).NotTo(HaveOccurred()) 145 Expect(resp.Status).To(Equal(common.Status_SUCCESS)) 146 147 blko1 := FetchBlock(network, o1, 3, network.SystemChannel.Name) 148 blko2 := FetchBlock(network, o2, 3, network.SystemChannel.Name) 149 150 Expect(blko1.Header.DataHash).To(Equal(blko2.Header.DataHash)) 151 }) 152 }) 153 154 When("an orderer is behind the latest snapshot on leader", func() { 155 It("catches up using the block stored in snapshot", func() { 156 // Steps: 157 // - start o2 & o3 158 // - send several transactions so snapshot is created 159 // - kill o2 & o3, so that entries prior to snapshot are not in memory upon restart 160 // - start o1 & o2 161 // - assert that o1 can catch up with o2 using snapshot 162 network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, StartPort(), components) 163 o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3") 164 peer = network.Peer("Org1", "peer0") 165 166 network.GenerateConfigTree() 167 network.Bootstrap() 168 169 orderers := grouper.Members{ 170 {Name: o2.ID(), Runner: network.OrdererRunner(o2)}, 171 {Name: o3.ID(), Runner: network.OrdererRunner(o3)}, 172 } 173 ordererGroup := grouper.NewParallel(syscall.SIGTERM, orderers) 174 175 By("Starting 2/3 of cluster") 176 ordererProc = ifrit.Invoke(ordererGroup) 177 Eventually(ordererProc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 178 179 By("Creating testchannel") 180 channelID := "testchannel" 181 network.CreateChannel(channelID, o2, peer) 182 183 By("Submitting several transactions to trigger snapshot") 184 o2SnapDir := path.Join(network.RootDir, "orderers", o2.ID(), "etcdraft", "snapshot") 185 186 env := CreateBroadcastEnvelope(network, o2, channelID, make([]byte, 2000)) 187 for i := 1; i <= 4; i++ { // 4 < MaxSnapshotFiles(5), so that no snapshot is pruned 188 // Note that MaxMessageCount is 1 be default, so every tx results in a new block 189 resp, err := nwo.Broadcast(network, o2, env) 190 Expect(err).NotTo(HaveOccurred()) 191 Expect(resp.Status).To(Equal(common.Status_SUCCESS)) 192 193 // Assert that new snapshot file is created before broadcasting next tx, 194 // so that number of snapshots is deterministic. Otherwise, it is not 195 // guaranteed that every block triggers a snapshot file being created, 196 // due to the mechanism to prevent excessive snapshotting. 197 Eventually(func() int { 198 files, err := ioutil.ReadDir(path.Join(o2SnapDir, channelID)) 199 Expect(err).NotTo(HaveOccurred()) 200 return len(files) 201 }, network.EventuallyTimeout).Should(Equal(i)) // snapshot interval is 1 KB, every block triggers snapshot 202 } 203 204 By("Killing orderers so they don't have blocks prior to latest snapshot in the memory") 205 ordererProc.Signal(syscall.SIGKILL) 206 Eventually(ordererProc.Wait(), network.EventuallyTimeout).Should(Receive()) 207 208 By("Starting lagged orderer and one of up-to-date orderers") 209 orderers = grouper.Members{ 210 {Name: o1.ID(), Runner: network.OrdererRunner(o1)}, 211 {Name: o2.ID(), Runner: network.OrdererRunner(o2)}, 212 } 213 ordererGroup = grouper.NewParallel(syscall.SIGTERM, orderers) 214 ordererProc = ifrit.Invoke(ordererGroup) 215 Eventually(ordererProc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 216 217 o1SnapDir := path.Join(network.RootDir, "orderers", o1.ID(), "etcdraft", "snapshot") 218 219 By("Asserting that orderer1 has snapshot dir for both system and application channel") 220 Eventually(func() int { 221 files, err := ioutil.ReadDir(o1SnapDir) 222 Expect(err).NotTo(HaveOccurred()) 223 return len(files) 224 }, network.EventuallyTimeout).Should(Equal(2)) 225 226 By("Asserting that orderer1 receives and persists snapshot") 227 Eventually(func() int { 228 files, err := ioutil.ReadDir(path.Join(o1SnapDir, channelID)) 229 Expect(err).NotTo(HaveOccurred()) 230 return len(files) 231 }, network.EventuallyTimeout).Should(Equal(1)) 232 233 By("Asserting cluster is still functional") 234 env = CreateBroadcastEnvelope(network, o1, channelID, make([]byte, 1000)) 235 resp, err := nwo.Broadcast(network, o1, env) 236 Expect(err).NotTo(HaveOccurred()) 237 Expect(resp.Status).To(Equal(common.Status_SUCCESS)) 238 239 blko1 := FetchBlock(network, o1, 5, channelID) 240 blko2 := FetchBlock(network, o2, 5, channelID) 241 242 Expect(blko1.Header.DataHash).To(Equal(blko2.Header.DataHash)) 243 }) 244 }) 245 246 When("The leader dies", func() { 247 It("Elects a new leader", func() { 248 network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, StartPort(), components) 249 250 o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3") 251 252 network.GenerateConfigTree() 253 network.Bootstrap() 254 255 By("Running the orderer nodes") 256 o1Runner := network.OrdererRunner(o1) 257 o2Runner := network.OrdererRunner(o2) 258 o3Runner := network.OrdererRunner(o3) 259 260 o1Proc = ifrit.Invoke(o1Runner) 261 o2Proc = ifrit.Invoke(o2Runner) 262 o3Proc = ifrit.Invoke(o3Runner) 263 264 Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 265 Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 266 Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 267 268 By("Waiting for them to elect a leader") 269 ordererProcesses := []ifrit.Process{o1Proc, o2Proc, o3Proc} 270 remainingAliveRunners := []*ginkgomon.Runner{o1Runner, o2Runner, o3Runner} 271 leader := findLeader(remainingAliveRunners) 272 273 leaderIndex := leader - 1 274 By(fmt.Sprintf("Killing the leader (%d)", leader)) 275 ordererProcesses[leaderIndex].Signal(syscall.SIGTERM) 276 By("Waiting for it to die") 277 Eventually(ordererProcesses[leaderIndex].Wait(), network.EventuallyTimeout).Should(Receive()) 278 279 // Remove the leader from the orderer runners 280 remainingAliveRunners = append(remainingAliveRunners[:leaderIndex], remainingAliveRunners[leaderIndex+1:]...) 281 282 By("Waiting for a new leader to be elected") 283 leader = findLeader(remainingAliveRunners) 284 By(fmt.Sprintf("Orderer %d took over as a leader", leader)) 285 }) 286 }) 287 288 When("Leader cannot reach quorum", func() { 289 It("Steps down", func() { 290 network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, StartPort(), components) 291 292 o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3") 293 orderers := []*nwo.Orderer{o1, o2, o3} 294 peer = network.Peer("Org1", "peer0") 295 network.GenerateConfigTree() 296 network.Bootstrap() 297 298 By("Running the orderer nodes") 299 o1Runner := network.OrdererRunner(o1) 300 o2Runner := network.OrdererRunner(o2) 301 o3Runner := network.OrdererRunner(o3) 302 oRunners := []*ginkgomon.Runner{o1Runner, o2Runner, o3Runner} 303 304 o1Proc = ifrit.Invoke(o1Runner) 305 o2Proc = ifrit.Invoke(o2Runner) 306 o3Proc = ifrit.Invoke(o3Runner) 307 308 Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 309 Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 310 Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 311 312 By("Waiting for them to elect a leader") 313 ordererProcesses := []ifrit.Process{o1Proc, o2Proc, o3Proc} 314 remainingAliveRunners := []*ginkgomon.Runner{o1Runner, o2Runner, o3Runner} 315 leaderID := findLeader(remainingAliveRunners) 316 leaderIndex := leaderID - 1 317 leader := orderers[leaderIndex] 318 319 followerIndices := func() []int { 320 var f []int 321 for i := range ordererProcesses { 322 if leaderIndex != i { 323 f = append(f, i) 324 } 325 } 326 327 return f 328 }() 329 330 By(fmt.Sprintf("Killing two followers (%d and %d)", followerIndices[0]+1, followerIndices[1]+1)) 331 ordererProcesses[followerIndices[0]].Signal(syscall.SIGTERM) 332 ordererProcesses[followerIndices[1]].Signal(syscall.SIGTERM) 333 334 By("Waiting for followers to die") 335 // This significantly slows test (~10s). However, reducing ElectionTimeout 336 // would introduce some flakes when disk write is exceptionally slow. 337 Eventually(ordererProcesses[followerIndices[0]].Wait(), network.EventuallyTimeout).Should(Receive()) 338 Eventually(ordererProcesses[followerIndices[1]].Wait(), network.EventuallyTimeout).Should(Receive()) 339 340 By("Waiting for leader to step down") 341 Eventually(oRunners[leaderIndex].Err(), time.Minute, time.Second).Should(gbytes.Say(fmt.Sprintf("%d stepped down to follower since quorum is not active", leaderID))) 342 343 By("Submitting tx to leader") 344 // This should fail because current leader steps down 345 // and there is no leader at this point of time 346 env := CreateBroadcastEnvelope(network, leader, network.SystemChannel.Name, []byte("foo")) 347 resp, err := nwo.Broadcast(network, leader, env) 348 Expect(err).NotTo(HaveOccurred()) 349 Expect(resp.Status).To(Equal(common.Status_SERVICE_UNAVAILABLE)) 350 }) 351 }) 352 353 When("orderer TLS certificates expire", func() { 354 It("is still possible to recover", func() { 355 network = nwo.New(nwo.MultiNodeEtcdRaft(), testDir, client, StartPort(), components) 356 357 o1, o2, o3 := network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3") 358 peer = network.Peer("Org1", "peer0") 359 360 network.GenerateConfigTree() 361 network.Bootstrap() 362 363 ordererDomain := network.Organization(o1.Organization).Domain 364 ordererTLSCAKeyPath := filepath.Join(network.RootDir, "crypto", "ordererOrganizations", 365 ordererDomain, "tlsca", "priv_sk") 366 367 ordererTLSCAKey, err := ioutil.ReadFile(ordererTLSCAKeyPath) 368 Expect(err).NotTo(HaveOccurred()) 369 370 ordererTLSCACertPath := filepath.Join(network.RootDir, "crypto", "ordererOrganizations", 371 ordererDomain, "tlsca", fmt.Sprintf("tlsca.%s-cert.pem", ordererDomain)) 372 ordererTLSCACert, err := ioutil.ReadFile(ordererTLSCACertPath) 373 Expect(err).NotTo(HaveOccurred()) 374 375 serverTLSCerts := make(map[string][]byte) 376 for _, orderer := range []*nwo.Orderer{o1, o2, o3} { 377 tlsCertPath := filepath.Join(network.OrdererLocalTLSDir(orderer), "server.crt") 378 serverTLSCerts[tlsCertPath], err = ioutil.ReadFile(tlsCertPath) 379 Expect(err).NotTo(HaveOccurred()) 380 } 381 382 By("Expiring orderer TLS certificates") 383 for filePath, certPEM := range serverTLSCerts { 384 expiredCert, earlyMadeCACert := expireCertificate(certPEM, ordererTLSCACert, ordererTLSCAKey, time.Now()) 385 err = ioutil.WriteFile(filePath, expiredCert, 0600) 386 Expect(err).NotTo(HaveOccurred()) 387 388 err = ioutil.WriteFile(ordererTLSCACertPath, earlyMadeCACert, 0600) 389 Expect(err).NotTo(HaveOccurred()) 390 } 391 392 By("Regenerating config") 393 sess, err := network.ConfigTxGen(commands.OutputBlock{ 394 ChannelID: network.SystemChannel.Name, 395 Profile: network.SystemChannel.Profile, 396 ConfigPath: network.RootDir, 397 OutputBlock: network.OutputBlockPath(network.SystemChannel.Name), 398 }) 399 Expect(err).NotTo(HaveOccurred()) 400 Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(0)) 401 402 By("Running the orderer nodes") 403 o1Runner := network.OrdererRunner(o1) 404 o2Runner := network.OrdererRunner(o2) 405 o3Runner := network.OrdererRunner(o3) 406 407 o1Proc = ifrit.Invoke(o1Runner) 408 o2Proc = ifrit.Invoke(o2Runner) 409 o3Proc = ifrit.Invoke(o3Runner) 410 411 Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 412 Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 413 Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 414 415 By("Waiting for TLS handshakes to fail") 416 Eventually(o1Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("tls: bad certificate")) 417 Eventually(o2Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("tls: bad certificate")) 418 Eventually(o3Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("tls: bad certificate")) 419 420 By("Killing orderers") 421 o1Proc.Signal(syscall.SIGTERM) 422 o2Proc.Signal(syscall.SIGTERM) 423 o3Proc.Signal(syscall.SIGTERM) 424 Eventually(o1Proc.Wait(), network.EventuallyTimeout).Should(Receive()) 425 Eventually(o2Proc.Wait(), network.EventuallyTimeout).Should(Receive()) 426 Eventually(o3Proc.Wait(), network.EventuallyTimeout).Should(Receive()) 427 428 o1Runner = network.OrdererRunner(o1) 429 o2Runner = network.OrdererRunner(o2) 430 o3Runner = network.OrdererRunner(o3) 431 432 By("Launching orderers with a clustered timeshift") 433 orderers := []*nwo.Orderer{o1, o2, o3} 434 for _, orderer := range orderers { 435 ordererConfig := network.ReadOrdererConfig(orderer) 436 ordererConfig.General.Cluster.TLSHandshakeTimeShift = 5 * time.Minute 437 network.WriteOrdererConfig(orderer, ordererConfig) 438 } 439 440 o1Proc = ifrit.Invoke(o1Runner) 441 o2Proc = ifrit.Invoke(o2Runner) 442 o3Proc = ifrit.Invoke(o3Runner) 443 444 Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 445 Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 446 Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 447 448 By("Waiting for a leader to be elected") 449 findLeader([]*ginkgomon.Runner{o1Runner, o2Runner, o3Runner}) 450 451 By("Killing orderers") 452 o1Proc.Signal(syscall.SIGTERM) 453 o2Proc.Signal(syscall.SIGTERM) 454 o3Proc.Signal(syscall.SIGTERM) 455 Eventually(o1Proc.Wait(), network.EventuallyTimeout).Should(Receive()) 456 Eventually(o2Proc.Wait(), network.EventuallyTimeout).Should(Receive()) 457 Eventually(o3Proc.Wait(), network.EventuallyTimeout).Should(Receive()) 458 459 o1Runner = network.OrdererRunner(o1) 460 o2Runner = network.OrdererRunner(o2) 461 o3Runner = network.OrdererRunner(o3) 462 463 By("Launching orderers again without a general timeshift re-using the cluster port") 464 for _, orderer := range orderers { 465 ordererConfig := network.ReadOrdererConfig(orderer) 466 ordererConfig.General.ListenPort = ordererConfig.General.Cluster.ListenPort 467 ordererConfig.General.TLS.Certificate = ordererConfig.General.Cluster.ServerCertificate 468 ordererConfig.General.TLS.PrivateKey = ordererConfig.General.Cluster.ServerPrivateKey 469 ordererConfig.General.Cluster.TLSHandshakeTimeShift = 0 470 ordererConfig.General.Cluster.ListenPort = 0 471 ordererConfig.General.Cluster.ListenAddress = "" 472 ordererConfig.General.Cluster.ServerCertificate = "" 473 ordererConfig.General.Cluster.ServerPrivateKey = "" 474 ordererConfig.General.Cluster.ClientCertificate = "" 475 ordererConfig.General.Cluster.ClientPrivateKey = "" 476 network.WriteOrdererConfig(orderer, ordererConfig) 477 } 478 479 o1Proc = ifrit.Invoke(o1Runner) 480 o2Proc = ifrit.Invoke(o2Runner) 481 o3Proc = ifrit.Invoke(o3Runner) 482 483 Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 484 Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 485 Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 486 487 By("Waiting for TLS handshakes to fail") 488 Eventually(o1Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("tls: bad certificate")) 489 Eventually(o2Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("tls: bad certificate")) 490 Eventually(o3Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("tls: bad certificate")) 491 492 By("Killing orderers") 493 o1Proc.Signal(syscall.SIGTERM) 494 o2Proc.Signal(syscall.SIGTERM) 495 o3Proc.Signal(syscall.SIGTERM) 496 Eventually(o1Proc.Wait(), network.EventuallyTimeout).Should(Receive()) 497 Eventually(o2Proc.Wait(), network.EventuallyTimeout).Should(Receive()) 498 Eventually(o3Proc.Wait(), network.EventuallyTimeout).Should(Receive()) 499 500 o1Runner = network.OrdererRunner(o1) 501 o2Runner = network.OrdererRunner(o2) 502 o3Runner = network.OrdererRunner(o3) 503 504 By("Launching orderers again with a general timeshift re-using the cluster port") 505 for _, orderer := range orderers { 506 ordererConfig := network.ReadOrdererConfig(orderer) 507 ordererConfig.General.TLS.TLSHandshakeTimeShift = 5 * time.Minute 508 network.WriteOrdererConfig(orderer, ordererConfig) 509 } 510 511 o1Proc = ifrit.Invoke(o1Runner) 512 o2Proc = ifrit.Invoke(o2Runner) 513 o3Proc = ifrit.Invoke(o3Runner) 514 515 Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 516 Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 517 Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 518 519 By("Waiting for a leader to be elected") 520 findLeader([]*ginkgomon.Runner{o1Runner, o2Runner, o3Runner}) 521 522 By("submitting config updates to orderers with expired TLS certs to replace the expired certs") 523 timeShift := 5 * time.Minute 524 for _, o := range orderers { 525 channelConfig := fetchConfig(network, peer, o, nwo.ClusterPort, network.SystemChannel.Name, timeShift) 526 c := conftx.New(channelConfig) 527 err = c.Orderer().RemoveConsenter(consenterChannelConfig(network, o)) 528 Expect(err).NotTo(HaveOccurred()) 529 530 By("renewing the orderer TLS certificates for " + o.Name) 531 renewOrdererCertificates(network, o) 532 err = c.Orderer().AddConsenter(consenterChannelConfig(network, o)) 533 Expect(err).NotTo(HaveOccurred()) 534 535 By("updating the config for " + o.Name) 536 updateOrdererConfig(network, o, nwo.ClusterPort, network.SystemChannel.Name, timeShift, c.OriginalConfig(), c.UpdatedConfig(), peer) 537 } 538 539 By("Killing orderers") 540 o1Proc.Signal(syscall.SIGTERM) 541 o2Proc.Signal(syscall.SIGTERM) 542 o3Proc.Signal(syscall.SIGTERM) 543 Eventually(o1Proc.Wait(), network.EventuallyTimeout).Should(Receive()) 544 Eventually(o2Proc.Wait(), network.EventuallyTimeout).Should(Receive()) 545 Eventually(o3Proc.Wait(), network.EventuallyTimeout).Should(Receive()) 546 547 o1Runner = network.OrdererRunner(o1) 548 o2Runner = network.OrdererRunner(o2) 549 o3Runner = network.OrdererRunner(o3) 550 551 By("Launching orderers again without a general timeshift") 552 for _, o := range orderers { 553 ordererConfig := network.ReadOrdererConfig(o) 554 ordererConfig.General.TLS.TLSHandshakeTimeShift = 0 555 network.WriteOrdererConfig(o, ordererConfig) 556 } 557 558 o1Proc = ifrit.Invoke(o1Runner) 559 o2Proc = ifrit.Invoke(o2Runner) 560 o3Proc = ifrit.Invoke(o3Runner) 561 562 Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 563 Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 564 Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 565 566 By("Waiting for a leader to be elected") 567 findLeader([]*ginkgomon.Runner{o1Runner, o2Runner, o3Runner}) 568 }) 569 570 It("disregards certificate renewal if only the validity period changed", func() { 571 config := nwo.MultiNodeEtcdRaft() 572 config.Channels = append(config.Channels, &nwo.Channel{Name: "foo", Profile: "TwoOrgsChannel"}) 573 config.Channels = append(config.Channels, &nwo.Channel{Name: "bar", Profile: "TwoOrgsChannel"}) 574 network = nwo.New(config, testDir, client, StartPort(), components) 575 576 network.GenerateConfigTree() 577 network.Bootstrap() 578 579 peer = network.Peer("Org1", "peer0") 580 581 o1 := network.Orderer("orderer1") 582 o2 := network.Orderer("orderer2") 583 o3 := network.Orderer("orderer3") 584 585 orderers := []*nwo.Orderer{o1, o2, o3} 586 587 o1Runner := network.OrdererRunner(o1) 588 o2Runner := network.OrdererRunner(o2) 589 o3Runner := network.OrdererRunner(o3) 590 ordererRunners := []*ginkgomon.Runner{o1Runner, o2Runner, o3Runner} 591 592 o1Proc = ifrit.Invoke(o1Runner) 593 o2Proc = ifrit.Invoke(o2Runner) 594 o3Proc = ifrit.Invoke(o3Runner) 595 ordererProcesses := []ifrit.Process{o1Proc, o2Proc, o3Proc} 596 597 Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 598 Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 599 Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 600 601 By("Waiting for them to elect a leader") 602 findLeader(ordererRunners) 603 604 By("Creating a channel") 605 network.CreateChannel("foo", o1, peer) 606 607 assertBlockReception(map[string]int{ 608 "foo": 0, 609 "systemchannel": 1, 610 }, []*nwo.Orderer{o1, o2, o3}, peer, network) 611 612 By("Killing all orderers") 613 for i := range orderers { 614 ordererProcesses[i].Signal(syscall.SIGTERM) 615 Eventually(ordererProcesses[i].Wait(), network.EventuallyTimeout).Should(Receive()) 616 } 617 618 By("Renewing the certificates for all orderers") 619 renewOrdererCertificates(network, o1, o2, o3) 620 621 By("Starting the orderers again") 622 for i := range orderers { 623 ordererRunner := network.OrdererRunner(orderers[i]) 624 ordererRunners[i] = ordererRunner 625 ordererProcesses[i] = ifrit.Invoke(ordererRunner) 626 Eventually(ordererProcesses[0].Ready(), network.EventuallyTimeout).Should(BeClosed()) 627 } 628 629 o1Proc = ordererProcesses[0] 630 o2Proc = ordererProcesses[1] 631 o3Proc = ordererProcesses[2] 632 633 By("Waiting for them to elect a leader once again") 634 findLeader(ordererRunners) 635 636 By("Creating a channel again") 637 network.CreateChannel("bar", o1, peer) 638 639 assertBlockReception(map[string]int{ 640 "foo": 0, 641 "bar": 0, 642 "systemchannel": 2, 643 }, []*nwo.Orderer{o1, o2, o3}, peer, network) 644 }) 645 }) 646 647 When("admin certificate expires", func() { 648 It("is still possible to replace them", func() { 649 network = nwo.New(nwo.BasicEtcdRaft(), testDir, client, StartPort(), components) 650 network.GenerateConfigTree() 651 network.Bootstrap() 652 653 peer = network.Peer("Org1", "peer0") 654 orderer := network.Orderer("orderer") 655 656 ordererDomain := network.Organization(orderer.Organization).Domain 657 ordererCAKeyPath := filepath.Join(network.RootDir, "crypto", "ordererOrganizations", ordererDomain, "ca", "priv_sk") 658 659 ordererCAKey, err := ioutil.ReadFile(ordererCAKeyPath) 660 Expect(err).NotTo(HaveOccurred()) 661 662 ordererCACertPath := filepath.Join(network.RootDir, "crypto", "ordererOrganizations", ordererDomain, "ca", fmt.Sprintf("ca.%s-cert.pem", ordererDomain)) 663 ordererCACert, err := ioutil.ReadFile(ordererCACertPath) 664 Expect(err).NotTo(HaveOccurred()) 665 666 adminCertPath := fmt.Sprintf("Admin@%s-cert.pem", ordererDomain) 667 adminCertPath = filepath.Join(network.OrdererUserMSPDir(orderer, "Admin"), "signcerts", adminCertPath) 668 669 originalAdminCert, err := ioutil.ReadFile(adminCertPath) 670 Expect(err).NotTo(HaveOccurred()) 671 672 expiredAdminCert, earlyCACert := expireCertificate(originalAdminCert, ordererCACert, ordererCAKey, time.Now()) 673 err = ioutil.WriteFile(adminCertPath, expiredAdminCert, 0600) 674 Expect(err).NotTo(HaveOccurred()) 675 676 adminPath := filepath.Join(network.RootDir, "crypto", "ordererOrganizations", 677 ordererDomain, "msp", "admincerts", fmt.Sprintf("Admin@%s-cert.pem", ordererDomain)) 678 err = ioutil.WriteFile(adminPath, expiredAdminCert, 0600) 679 Expect(err).NotTo(HaveOccurred()) 680 681 err = ioutil.WriteFile(ordererCACertPath, earlyCACert, 0600) 682 Expect(err).NotTo(HaveOccurred()) 683 684 ordererCACertPath = filepath.Join(network.RootDir, "crypto", "ordererOrganizations", 685 ordererDomain, "msp", "cacerts", fmt.Sprintf("ca.%s-cert.pem", ordererDomain)) 686 err = ioutil.WriteFile(ordererCACertPath, earlyCACert, 0600) 687 Expect(err).NotTo(HaveOccurred()) 688 689 By("Regenerating config") 690 sess, err := network.ConfigTxGen(commands.OutputBlock{ 691 ChannelID: network.SystemChannel.Name, 692 Profile: network.SystemChannel.Profile, 693 ConfigPath: network.RootDir, 694 OutputBlock: network.OutputBlockPath(network.SystemChannel.Name), 695 }) 696 Expect(err).NotTo(HaveOccurred()) 697 Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(0)) 698 699 runner := network.OrdererRunner(orderer) 700 runner.Command.Env = append(runner.Command.Env, "FABRIC_LOGGING_SPEC=debug") 701 ordererProc = ifrit.Invoke(runner) 702 703 By("Waiting for orderer to elect a leader") 704 findLeader([]*ginkgomon.Runner{runner}) 705 706 By("Creating config update that adds another orderer admin") 707 bootBlockPath := filepath.Join(network.RootDir, fmt.Sprintf("%s_block.pb", network.SystemChannel.Name)) 708 bootBlock, err := ioutil.ReadFile(bootBlockPath) 709 Expect(err).NotTo(HaveOccurred()) 710 711 current := configFromBootstrapBlock(bootBlock) 712 updatedConfig := addAdminCertToConfig(current, originalAdminCert) 713 714 tempDir, err := ioutil.TempDir("", "adminExpirationTest") 715 Expect(err).NotTo(HaveOccurred()) 716 717 configBlockFile := filepath.Join(tempDir, "update.pb") 718 defer os.RemoveAll(tempDir) 719 nwo.ComputeUpdateOrdererConfig(configBlockFile, network, network.SystemChannel.Name, current, updatedConfig, peer) 720 721 updateTransaction, err := ioutil.ReadFile(configBlockFile) 722 Expect(err).NotTo(HaveOccurred()) 723 724 By("Creating config update") 725 channelCreateTxn := createConfigTx(updateTransaction, network.SystemChannel.Name, network, orderer, peer) 726 727 By("Updating channel config and failing") 728 p, err := nwo.Broadcast(network, orderer, channelCreateTxn) 729 Expect(err).NotTo(HaveOccurred()) 730 Expect(p.Status).To(Equal(common.Status_BAD_REQUEST)) 731 Expect(p.Info).To(ContainSubstring("identity expired")) 732 733 By("Attempting to fetch a block from orderer and failing") 734 denv := CreateDeliverEnvelope(network, orderer, 0, network.SystemChannel.Name) 735 Expect(denv).NotTo(BeNil()) 736 737 block, err := nwo.Deliver(network, orderer, denv) 738 Expect(err).To(HaveOccurred()) 739 Expect(block).To(BeNil()) 740 Eventually(runner.Err(), time.Minute, time.Second).Should(gbytes.Say("client identity expired")) 741 742 By("Killing orderer") 743 ordererProc.Signal(syscall.SIGTERM) 744 Eventually(ordererProc.Wait(), network.EventuallyTimeout).Should(Receive()) 745 746 By("Launching orderers again") 747 runner = network.OrdererRunner(orderer) 748 runner.Command.Env = append(runner.Command.Env, "ORDERER_GENERAL_AUTHENTICATION_NOEXPIRATIONCHECKS=true") 749 ordererProc = ifrit.Invoke(runner) 750 751 By("Waiting for orderer to launch again") 752 findLeader([]*ginkgomon.Runner{runner}) 753 754 By("Updating channel config and succeeding") 755 p, err = nwo.Broadcast(network, orderer, channelCreateTxn) 756 Expect(err).NotTo(HaveOccurred()) 757 Expect(p.Status).To(Equal(common.Status_SUCCESS)) 758 759 By("Fetching a block from the orderer and succeeding") 760 block = FetchBlock(network, orderer, 1, network.SystemChannel.Name) 761 Expect(block).NotTo(BeNil()) 762 763 By("Restore the original admin cert") 764 err = ioutil.WriteFile(adminCertPath, originalAdminCert, 0600) 765 Expect(err).NotTo(HaveOccurred()) 766 767 By("Ensure we can fetch the block using our original un-expired admin cert") 768 ccb := func() uint64 { 769 return nwo.GetConfigBlock(network, peer, orderer, network.SystemChannel.Name).Header.Number 770 } 771 Eventually(ccb, network.EventuallyTimeout).Should(Equal(uint64(1))) 772 }) 773 }) 774 }) 775 776 func findLeader(ordererRunners []*ginkgomon.Runner) int { 777 var wg sync.WaitGroup 778 wg.Add(len(ordererRunners)) 779 780 findLeader := func(runner *ginkgomon.Runner) int { 781 Eventually(runner.Err(), time.Minute, time.Second).Should(gbytes.Say("Raft leader changed: [0-9] -> ")) 782 783 idBuff := make([]byte, 1) 784 _, err := runner.Err().Read(idBuff) 785 Expect(err).NotTo(HaveOccurred()) 786 787 newLeader, err := strconv.ParseInt(string(idBuff), 10, 32) 788 Expect(err).To(BeNil()) 789 return int(newLeader) 790 } 791 792 leaders := make(chan int, len(ordererRunners)) 793 794 for _, runner := range ordererRunners { 795 go func(runner *ginkgomon.Runner) { 796 defer GinkgoRecover() 797 defer wg.Done() 798 799 for { 800 leader := findLeader(runner) 801 if leader != 0 { 802 leaders <- leader 803 break 804 } 805 } 806 }(runner) 807 } 808 809 wg.Wait() 810 811 close(leaders) 812 firstLeader := <-leaders 813 for leader := range leaders { 814 if firstLeader != leader { 815 Fail(fmt.Sprintf("First leader is %d but saw %d also as a leader", firstLeader, leader)) 816 } 817 } 818 819 return firstLeader 820 } 821 822 func renewOrdererCertificates(network *nwo.Network, orderers ...*nwo.Orderer) { 823 if len(orderers) == 0 { 824 return 825 } 826 ordererDomain := network.Organization(orderers[0].Organization).Domain 827 ordererTLSCAKeyPath := filepath.Join(network.RootDir, "crypto", "ordererOrganizations", 828 ordererDomain, "tlsca", "priv_sk") 829 830 ordererTLSCAKey, err := ioutil.ReadFile(ordererTLSCAKeyPath) 831 Expect(err).NotTo(HaveOccurred()) 832 833 ordererTLSCACertPath := filepath.Join(network.RootDir, "crypto", "ordererOrganizations", 834 ordererDomain, "tlsca", fmt.Sprintf("tlsca.%s-cert.pem", ordererDomain)) 835 ordererTLSCACert, err := ioutil.ReadFile(ordererTLSCACertPath) 836 Expect(err).NotTo(HaveOccurred()) 837 838 serverTLSCerts := map[string][]byte{} 839 for _, orderer := range orderers { 840 tlsCertPath := filepath.Join(network.OrdererLocalTLSDir(orderer), "server.crt") 841 serverTLSCerts[tlsCertPath], err = ioutil.ReadFile(tlsCertPath) 842 Expect(err).NotTo(HaveOccurred()) 843 } 844 845 for filePath, certPEM := range serverTLSCerts { 846 renewedCert, _ := expireCertificate(certPEM, ordererTLSCACert, ordererTLSCAKey, time.Now().Add(time.Hour)) 847 err = ioutil.WriteFile(filePath, renewedCert, 0600) 848 Expect(err).NotTo(HaveOccurred()) 849 } 850 } 851 852 func expireCertificate(certPEM, caCertPEM, caKeyPEM []byte, expirationTime time.Time) (expiredcertPEM []byte, earlyMadeCACertPEM []byte) { 853 keyAsDER, _ := pem.Decode(caKeyPEM) 854 caKeyWithoutType, err := x509.ParsePKCS8PrivateKey(keyAsDER.Bytes) 855 Expect(err).NotTo(HaveOccurred()) 856 caKey := caKeyWithoutType.(*ecdsa.PrivateKey) 857 858 caCertAsDER, _ := pem.Decode(caCertPEM) 859 caCert, err := x509.ParseCertificate(caCertAsDER.Bytes) 860 Expect(err).NotTo(HaveOccurred()) 861 862 certAsDER, _ := pem.Decode(certPEM) 863 cert, err := x509.ParseCertificate(certAsDER.Bytes) 864 Expect(err).NotTo(HaveOccurred()) 865 866 cert.Raw = nil 867 caCert.Raw = nil 868 // The certificate was made 1 hour ago 869 cert.NotBefore = time.Now().Add((-1) * time.Hour) 870 // As well as the CA certificate 871 caCert.NotBefore = time.Now().Add((-1) * time.Hour) 872 // The certificate expires now 873 cert.NotAfter = expirationTime 874 875 // The CA signs the certificate 876 certBytes, err := x509.CreateCertificate(rand.Reader, cert, caCert, cert.PublicKey, caKey) 877 Expect(err).NotTo(HaveOccurred()) 878 879 // The CA signs its own certificate 880 caCertBytes, err := x509.CreateCertificate(rand.Reader, caCert, caCert, caCert.PublicKey, caKey) 881 Expect(err).NotTo(HaveOccurred()) 882 883 expiredcertPEM = pem.EncodeToMemory(&pem.Block{Bytes: certBytes, Type: "CERTIFICATE"}) 884 earlyMadeCACertPEM = pem.EncodeToMemory(&pem.Block{Bytes: caCertBytes, Type: "CERTIFICATE"}) 885 return 886 } 887 888 func createConfigTx(txData []byte, channelName string, network *nwo.Network, orderer *nwo.Orderer, peer *nwo.Peer) *common.Envelope { 889 ctxEnv, err := protoutil.UnmarshalEnvelope(txData) 890 Expect(err).NotTo(HaveOccurred()) 891 892 payload, err := protoutil.UnmarshalPayload(ctxEnv.Payload) 893 Expect(err).NotTo(HaveOccurred()) 894 895 configUpdateEnv, err := configtx.UnmarshalConfigUpdateEnvelope(payload.Data) 896 Expect(err).NotTo(HaveOccurred()) 897 898 conf := signer.Config{ 899 MSPID: network.Organization(orderer.Organization).MSPID, 900 IdentityPath: network.OrdererUserCert(orderer, "Admin"), 901 KeyPath: network.OrdererUserKey(orderer, "Admin"), 902 } 903 904 s, err := signer.NewSigner(conf) 905 Expect(err).NotTo(HaveOccurred()) 906 907 signConfigUpdate(conf, configUpdateEnv) 908 909 env, err := protoutil.CreateSignedEnvelope(common.HeaderType_CONFIG_UPDATE, channelName, s, configUpdateEnv, 0, 0) 910 Expect(err).NotTo(HaveOccurred()) 911 912 return env 913 } 914 915 func signConfigUpdate(conf signer.Config, configUpdateEnv *common.ConfigUpdateEnvelope) *common.ConfigUpdateEnvelope { 916 s, err := signer.NewSigner(conf) 917 Expect(err).NotTo(HaveOccurred()) 918 919 sigHeader, err := protoutil.NewSignatureHeader(s) 920 Expect(err).NotTo(HaveOccurred()) 921 922 configSig := &common.ConfigSignature{ 923 SignatureHeader: protoutil.MarshalOrPanic(sigHeader), 924 } 925 926 configSig.Signature, err = s.Sign(util.ConcatenateBytes(configSig.SignatureHeader, configUpdateEnv.ConfigUpdate)) 927 Expect(err).NotTo(HaveOccurred()) 928 929 configUpdateEnv.Signatures = append(configUpdateEnv.Signatures, configSig) 930 return configUpdateEnv 931 } 932 933 func addAdminCertToConfig(originalConfig *common.Config, additionalAdmin []byte) *common.Config { 934 updatedConfig := proto.Clone(originalConfig).(*common.Config) 935 936 rawMSPConfig := updatedConfig.ChannelGroup.Groups["Orderer"].Groups["OrdererOrg"].Values["MSP"] 937 mspConfig := &msp.MSPConfig{} 938 err := proto.Unmarshal(rawMSPConfig.Value, mspConfig) 939 Expect(err).NotTo(HaveOccurred()) 940 941 fabricConfig := &msp.FabricMSPConfig{} 942 err = proto.Unmarshal(mspConfig.Config, fabricConfig) 943 Expect(err).NotTo(HaveOccurred()) 944 945 fabricConfig.Admins = append(fabricConfig.Admins, additionalAdmin) 946 mspConfig.Config = protoutil.MarshalOrPanic(fabricConfig) 947 948 rawMSPConfig.Value = protoutil.MarshalOrPanic(mspConfig) 949 return updatedConfig 950 } 951 952 func configFromBootstrapBlock(bootstrapBlock []byte) *common.Config { 953 block := &common.Block{} 954 err := proto.Unmarshal(bootstrapBlock, block) 955 Expect(err).NotTo(HaveOccurred()) 956 return configFromBlock(block) 957 } 958 959 func configFromBlock(block *common.Block) *common.Config { 960 envelope, err := protoutil.GetEnvelopeFromBlock(block.Data.Data[0]) 961 Expect(err).NotTo(HaveOccurred()) 962 963 payload, err := protoutil.UnmarshalPayload(envelope.Payload) 964 Expect(err).NotTo(HaveOccurred()) 965 966 configEnv := &common.ConfigEnvelope{} 967 err = proto.Unmarshal(payload.Data, configEnv) 968 Expect(err).NotTo(HaveOccurred()) 969 970 return configEnv.Config 971 } 972 973 func fetchConfig(n *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, port nwo.PortName, channel string, tlsHandshakeTimeShift time.Duration) *common.Config { 974 tempDir, err := ioutil.TempDir(n.RootDir, "fetchConfig") 975 Expect(err).NotTo(HaveOccurred()) 976 defer os.RemoveAll(tempDir) 977 978 output := filepath.Join(tempDir, "config_block.pb") 979 fetchConfigBlock(n, peer, orderer, port, n.SystemChannel.Name, output, tlsHandshakeTimeShift) 980 configBlock := nwo.UnmarshalBlockFromFile(output) 981 return configFromBlock(configBlock) 982 } 983 984 func fetchConfigBlock(n *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, port nwo.PortName, channel, output string, tlsHandshakeTimeShift time.Duration) { 985 fetch := func() int { 986 sess, err := n.OrdererAdminSession(orderer, peer, commands.ChannelFetch{ 987 ChannelID: channel, 988 Block: "config", 989 Orderer: n.OrdererAddress(orderer, port), 990 OutputFile: output, 991 ClientAuth: n.ClientAuthRequired, 992 TLSHandshakeTimeShift: tlsHandshakeTimeShift, 993 }) 994 Expect(err).NotTo(HaveOccurred()) 995 code := sess.Wait(n.EventuallyTimeout).ExitCode() 996 if code == 0 { 997 Expect(sess.Err).To(gbytes.Say("Received block: ")) 998 } 999 return code 1000 } 1001 Eventually(fetch, n.EventuallyTimeout).Should(Equal(0)) 1002 } 1003 1004 func currentConfigBlockNumber(n *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, port nwo.PortName, channel string, tlsHandshakeTimeShift time.Duration) uint64 { 1005 tempDir, err := ioutil.TempDir(n.RootDir, "currentConfigBlock") 1006 Expect(err).NotTo(HaveOccurred()) 1007 defer os.RemoveAll(tempDir) 1008 1009 output := filepath.Join(tempDir, "config_block.pb") 1010 fetchConfigBlock(n, peer, orderer, port, channel, output, tlsHandshakeTimeShift) 1011 configBlock := nwo.UnmarshalBlockFromFile(output) 1012 return configBlock.Header.Number 1013 } 1014 1015 func updateOrdererConfig(n *nwo.Network, orderer *nwo.Orderer, port nwo.PortName, channel string, tlsHandshakeTimeShift time.Duration, current, updated *common.Config, submitter *nwo.Peer, additionalSigners ...*nwo.Orderer) { 1016 tempDir, err := ioutil.TempDir(n.RootDir, "updateConfig") 1017 Expect(err).NotTo(HaveOccurred()) 1018 updateFile := filepath.Join(tempDir, "update.pb") 1019 defer os.RemoveAll(tempDir) 1020 1021 currentBlockNumber := currentConfigBlockNumber(n, submitter, orderer, port, channel, tlsHandshakeTimeShift) 1022 nwo.ComputeUpdateOrdererConfig(updateFile, n, channel, current, updated, submitter, additionalSigners...) 1023 1024 Eventually(func() bool { 1025 sess, err := n.OrdererAdminSession(orderer, submitter, commands.ChannelUpdate{ 1026 ChannelID: channel, 1027 Orderer: n.OrdererAddress(orderer, port), 1028 File: updateFile, 1029 ClientAuth: n.ClientAuthRequired, 1030 TLSHandshakeTimeShift: tlsHandshakeTimeShift, 1031 }) 1032 Expect(err).NotTo(HaveOccurred()) 1033 1034 sess.Wait(n.EventuallyTimeout) 1035 if sess.ExitCode() != 0 { 1036 return false 1037 } 1038 1039 return strings.Contains(string(sess.Err.Contents()), "Successfully submitted channel update") 1040 }, n.EventuallyTimeout).Should(BeTrue()) 1041 1042 ccb := func() uint64 { 1043 return currentConfigBlockNumber(n, submitter, orderer, port, channel, tlsHandshakeTimeShift) 1044 } 1045 Eventually(ccb, n.EventuallyTimeout).Should(BeNumerically(">", currentBlockNumber)) 1046 } 1047 1048 func consenterChannelConfig(n *nwo.Network, o *nwo.Orderer) orderer.Consenter { 1049 host, port := intconftx.OrdererClusterHostPort(n, o) 1050 tlsCert := parseCertificate(filepath.Join(n.OrdererLocalTLSDir(o), "server.crt")) 1051 return orderer.Consenter{ 1052 Address: orderer.EtcdAddress{ 1053 Host: host, 1054 Port: port, 1055 }, 1056 ClientTLSCert: tlsCert, 1057 ServerTLSCert: tlsCert, 1058 } 1059 } 1060 1061 // parseCertificate loads the PEM-encoded x509 certificate at the specified 1062 // path. 1063 func parseCertificate(path string) *x509.Certificate { 1064 certBytes, err := ioutil.ReadFile(path) 1065 Expect(err).NotTo(HaveOccurred()) 1066 pemBlock, _ := pem.Decode(certBytes) 1067 cert, err := x509.ParseCertificate(pemBlock.Bytes) 1068 Expect(err).NotTo(HaveOccurred()) 1069 return cert 1070 }