github.com/hechain20/hechain@v0.0.0-20220316014945-b544036ba106/integration/kafka/migration_test.go (about) 1 /* 2 Copyright hechain All Rights Reserved. 3 4 SPDX-License-Identifier: Apache-2.0 5 */ 6 7 package kafka 8 9 import ( 10 "errors" 11 "fmt" 12 "io/ioutil" 13 "os" 14 "path/filepath" 15 "strings" 16 "syscall" 17 "time" 18 19 docker "github.com/fsouza/go-dockerclient" 20 "github.com/golang/protobuf/proto" 21 "github.com/hechain20/hechain/integration/nwo" 22 "github.com/hechain20/hechain/integration/nwo/commands" 23 "github.com/hechain20/hechain/integration/ordererclient" 24 "github.com/hechain20/hechain/protoutil" 25 "github.com/hyperledger/fabric-protos-go/common" 26 protosorderer "github.com/hyperledger/fabric-protos-go/orderer" 27 protosraft "github.com/hyperledger/fabric-protos-go/orderer/etcdraft" 28 . "github.com/onsi/ginkgo" 29 . "github.com/onsi/gomega" 30 "github.com/onsi/gomega/gbytes" 31 "github.com/onsi/gomega/gexec" 32 "github.com/tedsuo/ifrit" 33 "github.com/tedsuo/ifrit/ginkgomon" 34 ) 35 36 var _ = Describe("Kafka2RaftMigration", func() { 37 var ( 38 testDir string 39 client *docker.Client 40 network *nwo.Network 41 42 process ifrit.Process 43 brokerProc ifrit.Process 44 o1Proc, o2Proc, o3Proc ifrit.Process 45 46 o1Runner, o2Runner, o3Runner *ginkgomon.Runner 47 ) 48 49 BeforeEach(func() { 50 var err error 51 testDir, err = ioutil.TempDir("", "kafka2raft-migration") 52 Expect(err).NotTo(HaveOccurred()) 53 54 client, err = docker.NewClientFromEnv() 55 Expect(err).NotTo(HaveOccurred()) 56 }) 57 58 AfterEach(func() { 59 if process != nil { 60 process.Signal(syscall.SIGTERM) 61 Eventually(process.Wait(), network.EventuallyTimeout).Should(Receive()) 62 } 63 64 for _, oProc := range []ifrit.Process{o1Proc, o2Proc, o3Proc} { 65 if oProc != nil { 66 oProc.Signal(syscall.SIGTERM) 67 Eventually(oProc.Wait(), network.EventuallyTimeout).Should(Receive()) 68 } 69 } 70 71 if brokerProc != nil { 72 brokerProc.Signal(syscall.SIGTERM) 73 Eventually(brokerProc.Wait(), network.EventuallyTimeout).Should(Receive()) 74 } 75 76 if network != nil { 77 network.Cleanup() 78 } 79 80 _ = os.RemoveAll(testDir) 81 }) 82 83 // These tests execute the migration config updates on Kafka based system, and verify that these config update 84 // have the desired effect. They focus on testing that filtering and the blocking of transitions into, in, and 85 // out-of maintenance mode are enforced. These tests use a single node. 86 Describe("Kafka to Raft migration kafka side", func() { 87 var ( 88 orderer *nwo.Orderer 89 peer *nwo.Peer 90 syschannel, channel1, channel2, channel3 string 91 raftMetadata []byte 92 ) 93 94 BeforeEach(func() { 95 network = nwo.New(kafka2RaftMultiChannel(), testDir, client, StartPort(), components) 96 network.GenerateConfigTree() 97 network.Bootstrap() 98 99 networkRunner := network.NetworkGroupRunner() 100 process = ifrit.Invoke(networkRunner) 101 Eventually(process.Ready(), network.EventuallyTimeout).Should(BeClosed()) 102 103 orderer = network.Orderer("orderer") 104 peer = network.Peer("Org1", "peer0") 105 106 syschannel = network.SystemChannel.Name 107 channel1 = "testchannel1" 108 channel2 = "testchannel2" 109 channel3 = "testchannel3" 110 111 raftMetadata = prepareRaftMetadata(network) 112 113 By("Create & join first channel, deploy and invoke chaincode") 114 network.CreateChannel(channel1, orderer, peer) 115 }) 116 117 // This test executes the "green path" migration config updates on Kafka based system with a system channel 118 // and a standard channel, and verifies that these config updates have the desired effect. 119 // 120 // The green path is entering maintenance mode, and then changing the consensus type. 121 // In maintenance mode we check that channel creation is blocked and that normal transactions are blocked. 122 // 123 // We also check that after entering maintenance mode, we can exit it without making any 124 // changes - the "abort path". 125 It("executes kafka2raft green path", func() { 126 //=== The abort path ====================================================================================== 127 //=== Step 1: Config update on system channel, MAINTENANCE === 128 By("1) Config update on system channel, State=MAINTENANCE, enter maintenance-mode") 129 config, updatedConfig := prepareTransition(network, peer, orderer, syschannel, 130 "kafka", protosorderer.ConsensusType_STATE_NORMAL, 131 "kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) 132 nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer) 133 134 By("1) Verify: system channel1 config changed") 135 sysStartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel) 136 Expect(sysStartBlockNum).ToNot(Equal(0)) 137 config = nwo.GetConfig(network, peer, orderer, syschannel) 138 consensusTypeValue := extractOrdererConsensusType(config) 139 validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE) 140 141 By("1) Verify: new channels cannot be created") 142 exitCode := network.CreateChannelExitCode(channel2, orderer, peer) 143 Expect(exitCode).ToNot(Equal(0)) 144 145 //=== Step 2: Config update on standard channel, MAINTENANCE === 146 By("2) Config update on standard channel, State=MAINTENANCE, enter maintenance-mode") 147 config, updatedConfig = prepareTransition(network, peer, orderer, channel1, 148 "kafka", protosorderer.ConsensusType_STATE_NORMAL, 149 "kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) 150 nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer) 151 152 By("2) Verify: standard channel config changed") 153 std1EntryBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1) 154 Expect(std1EntryBlockNum).ToNot(Equal(0)) 155 config = nwo.GetConfig(network, peer, orderer, channel1) 156 consensusTypeValue = extractOrdererConsensusType(config) 157 validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE) 158 159 By("2) Verify: Normal TX's on standard channel are blocked") 160 assertTxFailed(network, orderer, channel1) 161 162 // In maintenance mode deliver requests are open to those entities that satisfy the /Channel/Orderer/Readers policy 163 By("2) Verify: delivery request from peer is blocked") 164 err := checkPeerDeliverRequest(orderer, peer, network, channel1) 165 Expect(err).To(MatchError(errors.New("FORBIDDEN"))) 166 167 //=== Step 3: config update on system channel, State=NORMAL, abort === 168 By("3) Config update on system channel, State=NORMAL, exit maintenance-mode - abort path") 169 config, updatedConfig = prepareTransition(network, peer, orderer, syschannel, 170 "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE, 171 "kafka", nil, protosorderer.ConsensusType_STATE_NORMAL) 172 nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer) 173 174 By("3) Verify: system channel config changed") 175 sysBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel) 176 Expect(sysBlockNum).To(Equal(sysStartBlockNum + 1)) 177 178 By("3) Verify: create new channel, executing transaction") 179 network.CreateChannel(channel2, orderer, peer) 180 181 assertBlockCreation(network, orderer, peer, channel2, 1) 182 183 By("3) Verify: delivery request from peer is not blocked on new channel") 184 err = checkPeerDeliverRequest(orderer, peer, network, channel2) 185 Expect(err).NotTo(HaveOccurred()) 186 187 //=== Step 4: config update on standard channel, State=NORMAL, abort === 188 By("4) Config update on standard channel, State=NORMAL, exit maintenance-mode - abort path") 189 config, updatedConfig = prepareTransition(network, peer, orderer, channel1, 190 "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE, 191 "kafka", nil, protosorderer.ConsensusType_STATE_NORMAL) 192 nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer) 193 194 By("4) Verify: standard channel config changed") 195 std1BlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1) 196 Expect(std1BlockNum).To(Equal(std1EntryBlockNum + 1)) 197 198 By("4) Verify: standard channel delivery requests from peer unblocked") 199 err = checkPeerDeliverRequest(orderer, peer, network, channel1) 200 Expect(err).NotTo(HaveOccurred()) 201 202 By("4) Verify: Normal TX's on standard channel are permitted again") 203 assertBlockCreation(network, orderer, nil, channel1, 3) 204 205 //=== The green path ====================================================================================== 206 //=== Step 5: Config update on system channel, MAINTENANCE, again === 207 By("5) Config update on system channel, State=MAINTENANCE, enter maintenance-mode again") 208 config, updatedConfig = prepareTransition(network, peer, orderer, syschannel, 209 "kafka", protosorderer.ConsensusType_STATE_NORMAL, 210 "kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) 211 nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer) 212 213 By("5) Verify: system channel config changed") 214 sysStartBlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel) 215 Expect(sysStartBlockNum).ToNot(Equal(0)) 216 config = nwo.GetConfig(network, peer, orderer, syschannel) 217 consensusTypeValue = extractOrdererConsensusType(config) 218 validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE) 219 220 //=== Step 6: Config update on standard channel1, MAINTENANCE, again === 221 By("6) Config update on standard channel1, State=MAINTENANCE, enter maintenance-mode again") 222 config, updatedConfig = prepareTransition(network, peer, orderer, channel1, 223 "kafka", protosorderer.ConsensusType_STATE_NORMAL, 224 "kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) 225 nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer) 226 227 By("6) Verify: standard channel config changed") 228 std1EntryBlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1) 229 Expect(std1EntryBlockNum).ToNot(Equal(0)) 230 231 config = nwo.GetConfig(network, peer, orderer, channel1) 232 consensusTypeValue = extractOrdererConsensusType(config) 233 validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE) 234 235 By("6) Verify: delivery request from peer is blocked") 236 err = checkPeerDeliverRequest(orderer, peer, network, channel1) 237 Expect(err).To(MatchError(errors.New("FORBIDDEN"))) 238 239 By("6) Verify: Normal TX's on standard channel are blocked") 240 assertTxFailed(network, orderer, channel1) 241 242 //=== Step 7: Config update on standard channel2, MAINTENANCE === 243 By("7) Config update on standard channel2, State=MAINTENANCE, enter maintenance-mode again") 244 config, updatedConfig = prepareTransition(network, peer, orderer, channel2, 245 "kafka", protosorderer.ConsensusType_STATE_NORMAL, 246 "kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) 247 nwo.UpdateOrdererConfig(network, orderer, channel2, config, updatedConfig, peer, orderer) 248 249 By("7) Verify: standard channel config changed") 250 std2EntryBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel2) 251 Expect(std2EntryBlockNum).ToNot(Equal(0)) 252 253 config = nwo.GetConfig(network, peer, orderer, channel2) 254 consensusTypeValue = extractOrdererConsensusType(config) 255 validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE) 256 257 By("7) Verify: delivery request from peer is blocked") 258 err = checkPeerDeliverRequest(orderer, peer, network, channel2) 259 Expect(err).To(MatchError(errors.New("FORBIDDEN"))) 260 261 By("7) Verify: Normal TX's on standard channel are blocked") 262 assertTxFailed(network, orderer, channel2) 263 264 //=== Step 8: config update on system channel, State=MAINTENANCE, type=etcdraft === 265 By("8) Config update on system channel, State=MAINTENANCE, type=etcdraft") 266 config, updatedConfig = prepareTransition(network, peer, orderer, syschannel, 267 "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE, 268 "etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE) 269 nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer) 270 271 By("8) Verify: system channel config changed") 272 sysBlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel) 273 Expect(sysBlockNum).To(Equal(sysStartBlockNum + 1)) 274 275 By("8) Verify: new channels cannot be created") 276 exitCode = network.CreateChannelExitCode(channel3, orderer, peer) 277 Expect(exitCode).ToNot(Equal(0)) 278 279 //=== Step 9: config update on standard channel1, State=MAINTENANCE, type=etcdraft === 280 By("9) Config update on standard channel1, State=MAINTENANCE, type=etcdraft") 281 config, updatedConfig = prepareTransition(network, peer, orderer, channel1, 282 "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE, 283 "etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE) 284 nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer) 285 286 By("9) Verify: standard channel config changed") 287 std1BlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1) 288 Expect(std1BlockNum).To(Equal(std1EntryBlockNum + 1)) 289 290 By("9) Verify: delivery request from peer is blocked") 291 err = checkPeerDeliverRequest(orderer, peer, network, channel1) 292 Expect(err).To(MatchError(errors.New("FORBIDDEN"))) 293 294 By("9) Verify: Normal TX's on standard channel are blocked") 295 assertTxFailed(network, orderer, channel1) 296 297 //=== Step 10: config update on standard channel2, State=MAINTENANCE, type=etcdraft === 298 By("10) Config update on standard channel2, State=MAINTENANCE, type=etcdraft") 299 config, updatedConfig = prepareTransition(network, peer, orderer, channel2, 300 "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE, 301 "etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE) 302 nwo.UpdateOrdererConfig(network, orderer, channel2, config, updatedConfig, peer, orderer) 303 304 By("10) Verify: standard channel config changed") 305 std2BlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel2) 306 Expect(std2BlockNum).To(Equal(std2EntryBlockNum + 1)) 307 308 By("10) Verify: delivery request from peer is blocked") 309 err = checkPeerDeliverRequest(orderer, peer, network, channel2) 310 Expect(err).To(MatchError(errors.New("FORBIDDEN"))) 311 312 By("10) Verify: Normal TX's on standard channel are blocked") 313 assertTxFailed(network, orderer, channel2) 314 }) 315 316 // This test executes the migration flow and checks that forbidden transitions are rejected. 317 // These transitions are enforced by the maintenance filter: 318 // - Entry to & exit from maintenance mode can only change ConsensusType.State. 319 // - In maintenance mode one can only change ConsensusType.Type & ConsensusType.Metadata. 320 // - ConsensusType.Type can only change from "kafka" to "etcdraft", and only in maintenance mode. 321 It("executes kafka2raft forbidden transitions", func() { 322 //=== Step 1: === 323 By("1) Config update on system channel, changing both ConsensusType State & Type is forbidden") 324 assertTransitionFailed(network, peer, orderer, syschannel, 325 "kafka", protosorderer.ConsensusType_STATE_NORMAL, 326 "etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE) 327 328 //=== Step 2: === 329 By("2) Config update on standard channel, changing both ConsensusType State & Type is forbidden") 330 assertTransitionFailed(network, peer, orderer, channel1, 331 "kafka", protosorderer.ConsensusType_STATE_NORMAL, 332 "etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE) 333 334 //=== Step 3: === 335 By("3) Config update on system channel, changing both ConsensusType State & some other value is forbidden") 336 config, updatedConfig := prepareTransition(network, peer, orderer, syschannel, 337 "kafka", protosorderer.ConsensusType_STATE_NORMAL, 338 "kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) 339 updateConfigWithBatchTimeout(updatedConfig) 340 updateOrdererConfigFailed(network, orderer, syschannel, config, updatedConfig, peer, orderer) 341 342 //=== Step 4: === 343 By("4) Config update on standard channel, both ConsensusType State & some other value is forbidden") 344 config, updatedConfig = prepareTransition(network, peer, orderer, channel1, 345 "kafka", protosorderer.ConsensusType_STATE_NORMAL, 346 "kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) 347 updateConfigWithBatchTimeout(updatedConfig) 348 updateOrdererConfigFailed(network, orderer, channel1, config, updatedConfig, peer, orderer) 349 350 //=== Step 5: === 351 By("5) Config update on system channel, State=MAINTENANCE, enter maintenance-mode") 352 config, updatedConfig = prepareTransition(network, peer, orderer, syschannel, 353 "kafka", protosorderer.ConsensusType_STATE_NORMAL, 354 "kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) 355 nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer) 356 357 By("5) Verify: system channel config changed") 358 sysStartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel) 359 Expect(sysStartBlockNum).ToNot(Equal(0)) 360 config = nwo.GetConfig(network, peer, orderer, syschannel) 361 consensusTypeValue := extractOrdererConsensusType(config) 362 validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE) 363 364 //=== Step 6: === 365 By("6) Config update on standard channel, State=MAINTENANCE, enter maintenance-mode") 366 config, updatedConfig = prepareTransition(network, peer, orderer, channel1, 367 "kafka", protosorderer.ConsensusType_STATE_NORMAL, 368 "kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) 369 nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer) 370 371 By("6) Verify: standard channel config changed") 372 std1StartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1) 373 Expect(std1StartBlockNum).ToNot(Equal(0)) 374 config = nwo.GetConfig(network, peer, orderer, channel1) 375 consensusTypeValue = extractOrdererConsensusType(config) 376 validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE) 377 378 //=== Step 7: === 379 By("7) Config update on system channel, change ConsensusType.Type to unsupported type, forbidden") 380 assertTransitionFailed(network, peer, orderer, syschannel, 381 "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE, 382 "melville", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) 383 384 //=== Step 8: === 385 By("8) Config update on standard channel, change ConsensusType.Type to unsupported type, forbidden") 386 assertTransitionFailed(network, peer, orderer, channel1, 387 "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE, 388 "hesse", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) 389 390 //=== Step 9: === 391 By("9) Config update on system channel, change ConsensusType.Type and State, forbidden") 392 assertTransitionFailed(network, peer, orderer, syschannel, 393 "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE, 394 "etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL) 395 396 //=== Step 10: === 397 By("10) Config update on standard channel, change ConsensusType.Type and State, forbidden") 398 assertTransitionFailed(network, peer, orderer, channel1, 399 "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE, 400 "etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL) 401 402 //=== Step 11: === 403 By("11) Config update on system channel, changing both ConsensusType.Type and other value is permitted") 404 config, updatedConfig = prepareTransition(network, peer, orderer, syschannel, 405 "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE, 406 "etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE) 407 updateConfigWithBatchTimeout(updatedConfig) 408 nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer) 409 410 By("11) Verify: system channel config changed") 411 sysBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel) 412 Expect(sysBlockNum).To(Equal(sysStartBlockNum + 1)) 413 config = nwo.GetConfig(network, peer, orderer, syschannel) 414 consensusTypeValue = extractOrdererConsensusType(config) 415 validateConsensusTypeValue(consensusTypeValue, "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE) 416 417 //=== Step 12: === 418 By("12) Config update on standard channel, changing both ConsensusType.Type and other value is permitted") 419 config, updatedConfig = prepareTransition(network, peer, orderer, channel1, 420 "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE, 421 "etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE) 422 updateConfigWithBatchTimeout(updatedConfig) 423 nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer) 424 425 By("12) Verify: standard channel config changed") 426 std1BlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1) 427 Expect(std1BlockNum).To(Equal(std1StartBlockNum + 1)) 428 config = nwo.GetConfig(network, peer, orderer, channel1) 429 consensusTypeValue = extractOrdererConsensusType(config) 430 validateConsensusTypeValue(consensusTypeValue, "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE) 431 432 //=== Step 13: === 433 By("13) Config update on system channel, changing value other than ConsensusType.Type is permitted") 434 config = nwo.GetConfig(network, peer, orderer, syschannel) 435 consensusTypeValue = extractOrdererConsensusType(config) 436 validateConsensusTypeValue(consensusTypeValue, "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE) 437 updatedConfig = proto.Clone(config).(*common.Config) 438 updateConfigWithBatchTimeout(updatedConfig) 439 nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer) 440 441 By("13) Verify: system channel config changed") 442 sysBlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel) 443 Expect(sysBlockNum).To(Equal(sysStartBlockNum + 2)) 444 445 //=== Step 14: === 446 By("14) Config update on standard channel, changing value other than ConsensusType.Type is permitted") 447 config = nwo.GetConfig(network, peer, orderer, channel1) 448 consensusTypeValue = extractOrdererConsensusType(config) 449 validateConsensusTypeValue(consensusTypeValue, "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE) 450 updatedConfig = proto.Clone(config).(*common.Config) 451 updateConfigWithBatchTimeout(updatedConfig) 452 nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer) 453 454 By("14) Verify: standard channel config changed") 455 std1BlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1) 456 Expect(std1BlockNum).To(Equal(std1StartBlockNum + 2)) 457 458 //=== Step 15: === 459 By("15) Config update on system channel, changing both ConsensusType State & some other value is forbidden") 460 config, updatedConfig = prepareTransition(network, peer, orderer, syschannel, 461 "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE, 462 "etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL) 463 updateConfigWithBatchTimeout(updatedConfig) 464 updateOrdererConfigFailed(network, orderer, syschannel, config, updatedConfig, peer, orderer) 465 466 //=== Step 16: === 467 By("16) Config update on standard channel, both ConsensusType State & some other value is forbidden") 468 config, updatedConfig = prepareTransition(network, peer, orderer, channel1, 469 "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE, 470 "etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL) 471 updateConfigWithBatchTimeout(updatedConfig) 472 updateOrdererConfigFailed(network, orderer, channel1, config, updatedConfig, peer, orderer) 473 }) 474 }) 475 476 // These tests execute the migration config updates on Kafka based system, restart the orderer onto a Raft-based 477 // system, and verifies that the newly restarted orderer cluster performs as expected. 478 Describe("Kafka to Raft migration raft side", func() { 479 var ( 480 o1, o2, o3 *nwo.Orderer 481 peer *nwo.Peer 482 syschannel, channel1, channel2 string 483 raftMetadata []byte 484 ) 485 486 BeforeEach(func() { 487 network = nwo.New(kafka2RaftMultiNode(), testDir, client, StartPort(), components) 488 network.GenerateConfigTree() 489 network.Bootstrap() 490 491 o1, o2, o3 = network.Orderer("orderer1"), network.Orderer("orderer2"), network.Orderer("orderer3") 492 peer = network.Peer("Org1", "peer0") 493 494 brokerGroup := network.BrokerGroupRunner() 495 brokerProc = ifrit.Invoke(brokerGroup) 496 Eventually(brokerProc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 497 498 o1Runner = network.OrdererRunner(o1) 499 o2Runner = network.OrdererRunner(o2) 500 o3Runner = network.OrdererRunner(o3) 501 502 o1Proc = ifrit.Invoke(o1Runner) 503 o2Proc = ifrit.Invoke(o2Runner) 504 o3Proc = ifrit.Invoke(o3Runner) 505 506 Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 507 Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 508 Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 509 510 raftMetadata = prepareRaftMetadata(network) 511 512 syschannel = network.SystemChannel.Name 513 channel1 = "testchannel1" 514 channel2 = "testchannel2" 515 516 By("Create & join first channel, deploy and invoke chaincode") 517 network.CreateChannel(channel1, o1, peer) 518 }) 519 520 // This test executes the "green path" migration config updates on Kafka based system 521 // with a three orderers, a system channel and two standard channels. 522 // It then restarts the orderers onto a Raft-based system, and verifies that the 523 // newly restarted orderers perform as expected. 524 It("executes bootstrap to raft - multi node", func() { 525 //=== Step 1: Config update on system channel, MAINTENANCE === 526 By("1) Config update on system channel, State=MAINTENANCE") 527 config, updatedConfig := prepareTransition(network, peer, o1, syschannel, 528 "kafka", protosorderer.ConsensusType_STATE_NORMAL, 529 "kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) 530 nwo.UpdateOrdererConfig(network, o1, syschannel, config, updatedConfig, peer, o1) 531 532 By("1) Verify: system channel config changed") 533 sysStartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, o1, syschannel) 534 Expect(sysStartBlockNum).ToNot(Equal(0)) 535 536 config = nwo.GetConfig(network, peer, o1, syschannel) 537 consensusTypeValue := extractOrdererConsensusType(config) 538 validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE) 539 540 //=== Step 2: Config update on standard channel, MAINTENANCE === 541 By("2) Config update on standard channel, State=MAINTENANCE") 542 config, updatedConfig = prepareTransition(network, peer, o1, channel1, 543 "kafka", protosorderer.ConsensusType_STATE_NORMAL, 544 "kafka", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) 545 nwo.UpdateOrdererConfig(network, o1, channel1, config, updatedConfig, peer, o1) 546 547 By("2) Verify: standard channel config changed") 548 chan1StartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, o1, channel1) 549 Expect(chan1StartBlockNum).ToNot(Equal(0)) 550 551 config = nwo.GetConfig(network, peer, o1, channel1) 552 consensusTypeValue = extractOrdererConsensusType(config) 553 validateConsensusTypeValue(consensusTypeValue, "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE) 554 555 //=== Step 3: config update on system channel, State=MAINTENANCE, type=etcdraft === 556 By("3) Config update on system channel, State=MAINTENANCE, type=etcdraft") 557 config, updatedConfig = prepareTransition(network, peer, o1, syschannel, 558 "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE, 559 "etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE) 560 nwo.UpdateOrdererConfig(network, o1, syschannel, config, updatedConfig, peer, o1) 561 562 By("3) Verify: system channel config changed") 563 sysBlockNum := nwo.CurrentConfigBlockNumber(network, peer, o1, syschannel) 564 Expect(sysBlockNum).To(Equal(sysStartBlockNum + 1)) 565 566 //=== Step 4: config update on standard channel, State=MAINTENANCE, type=etcdraft === 567 By("4) Config update on standard channel, State=MAINTENANCE, type=etcdraft") 568 config, updatedConfig = prepareTransition(network, peer, o1, channel1, 569 "kafka", protosorderer.ConsensusType_STATE_MAINTENANCE, 570 "etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE) 571 nwo.UpdateOrdererConfig(network, o1, channel1, config, updatedConfig, peer, o1) 572 573 By("4) Verify: standard channel config changed") 574 chan1BlockNum := nwo.CurrentConfigBlockNumber(network, peer, o1, channel1) 575 Expect(chan1BlockNum).To(Equal(chan1StartBlockNum + 1)) 576 577 //=== Step 5: kill === 578 By("5) killing orderer1,2,3") 579 for _, oProc := range []ifrit.Process{o1Proc, o2Proc, o3Proc} { 580 if oProc != nil { 581 oProc.Signal(syscall.SIGKILL) 582 Eventually(oProc.Wait(), network.EventuallyTimeout).Should(Receive(MatchError("exit status 137"))) 583 } 584 } 585 586 //=== Step 6: restart === 587 By("6) restarting orderer1,2,3") 588 network.Consensus.Type = "etcdraft" 589 o1Runner = network.OrdererRunner(o1) 590 o2Runner = network.OrdererRunner(o2) 591 o3Runner = network.OrdererRunner(o3) 592 593 o1Proc = ifrit.Invoke(o1Runner) 594 o2Proc = ifrit.Invoke(o2Runner) 595 o3Proc = ifrit.Invoke(o3Runner) 596 597 Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 598 Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 599 Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 600 601 assertBlockReception( 602 map[string]int{ 603 syschannel: int(sysBlockNum), 604 channel1: int(chan1BlockNum), 605 }, 606 []*nwo.Orderer{o1, o2, o3}, 607 peer, 608 network, 609 ) 610 611 Eventually(o1Runner.Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say("Raft leader changed: 0 -> ")) 612 Eventually(o2Runner.Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say("Raft leader changed: 0 -> ")) 613 Eventually(o3Runner.Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say("Raft leader changed: 0 -> ")) 614 615 By("7) System channel still in maintenance, State=MAINTENANCE, cannot create new channels") 616 exitCode := network.CreateChannelExitCode(channel2, o1, peer) 617 Expect(exitCode).ToNot(Equal(0)) 618 619 By("8) Standard channel still in maintenance, State=MAINTENANCE, normal TX's blocked, delivery to peers blocked") 620 assertTxFailed(network, o1, channel1) 621 622 err := checkPeerDeliverRequest(o1, peer, network, channel1) 623 Expect(err).To(MatchError(errors.New("FORBIDDEN"))) 624 625 By("9) Release - executing config transaction on system channel with restarted orderer") 626 config, updatedConfig = prepareTransition(network, peer, o1, syschannel, 627 "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE, 628 "etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL) 629 nwo.UpdateOrdererConfig(network, o1, syschannel, config, updatedConfig, peer, o1) 630 631 By("9) Verify: system channel config changed") 632 sysBlockNum = nwo.CurrentConfigBlockNumber(network, peer, o1, syschannel) 633 Expect(sysBlockNum).To(Equal(sysStartBlockNum + 2)) 634 635 By("10) Release - executing config transaction on standard channel with restarted orderer") 636 config, updatedConfig = prepareTransition(network, peer, o1, channel1, 637 "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE, 638 "etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL) 639 nwo.UpdateOrdererConfig(network, o1, channel1, config, updatedConfig, peer, o1) 640 641 By("10) Verify: standard channel config changed") 642 chan1BlockNum = nwo.CurrentConfigBlockNumber(network, peer, o1, channel1) 643 Expect(chan1BlockNum).To(Equal(chan1StartBlockNum + 2)) 644 645 By("11) Executing transaction on standard channel with restarted orderer") 646 assertBlockCreation(network, o1, peer, channel1, chan1StartBlockNum+3) 647 assertBlockCreation(network, o1, nil, channel1, chan1StartBlockNum+4) 648 649 By("12) Create new channel, executing transaction with restarted orderer") 650 network.CreateChannel(channel2, o1, peer) 651 652 chan2StartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, o1, channel2) 653 Expect(chan2StartBlockNum).ToNot(Equal(0)) 654 655 assertBlockCreation(network, o1, peer, channel2, chan2StartBlockNum+1) 656 assertBlockCreation(network, o1, nil, channel2, chan2StartBlockNum+2) 657 658 By("Extending the network configuration to add a new orderer") 659 o4 := &nwo.Orderer{ 660 Name: "orderer4", 661 Organization: "OrdererOrg", 662 } 663 ports := nwo.Ports{} 664 for _, portName := range nwo.OrdererPortNames() { 665 ports[portName] = network.ReservePort() 666 } 667 network.PortsByOrdererID[o4.ID()] = ports 668 network.Orderers = append(network.Orderers, o4) 669 network.GenerateOrdererConfig(o4) 670 extendNetwork(network) 671 672 fourthOrdererCertificatePath := filepath.Join(network.OrdererLocalTLSDir(o4), "server.crt") 673 fourthOrdererCertificate, err := ioutil.ReadFile(fourthOrdererCertificatePath) 674 Expect(err).NotTo(HaveOccurred()) 675 676 By("Adding the fourth orderer to the system channel") 677 addConsenter(network, peer, o1, "systemchannel", protosraft.Consenter{ 678 ServerTlsCert: fourthOrdererCertificate, 679 ClientTlsCert: fourthOrdererCertificate, 680 Host: "127.0.0.1", 681 Port: uint32(network.OrdererPort(o4, nwo.ClusterPort)), 682 }) 683 684 By("Obtaining the last config block from an orderer") 685 // Get the last config block of the system channel 686 configBlock := nwo.GetConfigBlock(network, peer, o1, "systemchannel") 687 // Plant it in the file system of orderer4, the new node to be onboarded. 688 err = ioutil.WriteFile(filepath.Join(testDir, "systemchannel_block.pb"), protoutil.MarshalOrPanic(configBlock), 0o644) 689 Expect(err).NotTo(HaveOccurred()) 690 691 By("Launching the fourth orderer") 692 o4Runner := network.OrdererRunner(o4) 693 o4Process := ifrit.Invoke(o4Runner) 694 695 defer func() { 696 o4Process.Signal(syscall.SIGTERM) 697 Eventually(o4Process.Wait(), network.EventuallyTimeout).Should(Receive()) 698 }() 699 700 Eventually(o4Process.Ready(), network.EventuallyTimeout).Should(BeClosed()) 701 702 By("Waiting for the orderer to figure out it was migrated") 703 Eventually(o4Runner.Err(), time.Minute, time.Second).Should(gbytes.Say("This node was migrated from Kafka to Raft, skipping activation of Kafka chain")) 704 705 By("Adding the fourth orderer to the application channel") 706 addConsenter(network, peer, o1, channel1, protosraft.Consenter{ 707 ServerTlsCert: fourthOrdererCertificate, 708 ClientTlsCert: fourthOrdererCertificate, 709 Host: "127.0.0.1", 710 Port: uint32(network.OrdererPort(o4, nwo.ClusterPort)), 711 }) 712 713 chan1BlockNum = nwo.CurrentConfigBlockNumber(network, peer, o1, channel1) 714 715 By("Ensuring the added orderer has synced the application channel") 716 assertBlockReception( 717 map[string]int{ 718 channel1: int(chan1BlockNum), 719 }, 720 []*nwo.Orderer{o4}, 721 peer, 722 network, 723 ) 724 }) 725 }) 726 727 // These tests execute the migration config updates on a solo based system, restart the orderer onto a Raft-based 728 // system, and verifies that the newly restarted orderer (single node) cluster performs as expected. 729 Describe("Solo to Raft migration", func() { 730 var ( 731 orderer *nwo.Orderer 732 peer *nwo.Peer 733 syschannel, channel1, channel2 string 734 raftMetadata []byte 735 ) 736 737 BeforeEach(func() { 738 network = nwo.New(solo2RaftMultiChannel(), testDir, client, StartPort(), components) 739 network.GenerateConfigTree() 740 network.Bootstrap() 741 742 orderer = network.Orderer("orderer") 743 peer = network.Peer("Org1", "peer0") 744 745 brokerGroup := network.BrokerGroupRunner() 746 brokerProc = ifrit.Invoke(brokerGroup) 747 Eventually(brokerProc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 748 749 o1Runner = network.OrdererRunner(orderer) 750 751 o1Proc = ifrit.Invoke(o1Runner) 752 Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 753 754 raftMetadata = prepareRaftMetadata(network) 755 756 syschannel = network.SystemChannel.Name 757 channel1 = "testchannel1" 758 channel2 = "testchannel2" 759 760 By("Create & join first channel, deploy and invoke chaincode") 761 network.CreateChannel(channel1, orderer, peer) 762 }) 763 764 It("executes bootstrap to raft - single node", func() { 765 //=== Step 1: Config update on system channel, MAINTENANCE === 766 By("1) Config update on system channel, State=MAINTENANCE") 767 config, updatedConfig := prepareTransition(network, peer, orderer, syschannel, 768 "solo", protosorderer.ConsensusType_STATE_NORMAL, 769 "solo", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) 770 nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer) 771 772 By("1) Verify: system channel config changed") 773 sysStartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel) 774 Expect(sysStartBlockNum).ToNot(Equal(0)) 775 776 config = nwo.GetConfig(network, peer, orderer, syschannel) 777 consensusTypeValue := extractOrdererConsensusType(config) 778 validateConsensusTypeValue(consensusTypeValue, "solo", protosorderer.ConsensusType_STATE_MAINTENANCE) 779 780 //=== Step 2: Config update on standard channel, MAINTENANCE === 781 By("2) Config update on standard channel, State=MAINTENANCE") 782 config, updatedConfig = prepareTransition(network, peer, orderer, channel1, 783 "solo", protosorderer.ConsensusType_STATE_NORMAL, 784 "solo", nil, protosorderer.ConsensusType_STATE_MAINTENANCE) 785 nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer) 786 787 By("2) Verify: standard channel config changed") 788 chan1StartBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1) 789 Expect(chan1StartBlockNum).ToNot(Equal(0)) 790 791 config = nwo.GetConfig(network, peer, orderer, channel1) 792 consensusTypeValue = extractOrdererConsensusType(config) 793 validateConsensusTypeValue(consensusTypeValue, "solo", protosorderer.ConsensusType_STATE_MAINTENANCE) 794 795 //=== Step 3: config update on system channel, State=MAINTENANCE, type=etcdraft === 796 By("3) Config update on system channel, State=MAINTENANCE, type=etcdraft") 797 config, updatedConfig = prepareTransition(network, peer, orderer, syschannel, 798 "solo", protosorderer.ConsensusType_STATE_MAINTENANCE, 799 "etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE) 800 nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer) 801 802 By("3) Verify: system channel config changed") 803 sysBlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel) 804 Expect(sysBlockNum).To(Equal(sysStartBlockNum + 1)) 805 806 //=== Step 4: config update on standard channel, State=MAINTENANCE, type=etcdraft === 807 By("4) Config update on standard channel, State=MAINTENANCE, type=etcdraft") 808 config, updatedConfig = prepareTransition(network, peer, orderer, channel1, 809 "solo", protosorderer.ConsensusType_STATE_MAINTENANCE, 810 "etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_MAINTENANCE) 811 nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer) 812 813 By("4) Verify: standard channel config changed") 814 chan1BlockNum := nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1) 815 Expect(chan1BlockNum).To(Equal(chan1StartBlockNum + 1)) 816 817 //=== Step 5: kill === 818 By("5) killing orderer1") 819 o1Proc.Signal(syscall.SIGKILL) 820 Eventually(o1Proc.Wait(), network.EventuallyTimeout).Should(Receive(MatchError("exit status 137"))) 821 822 //=== Step 6: restart === 823 By("6) restarting orderer1") 824 network.Consensus.Type = "etcdraft" 825 826 o1Runner = network.OrdererRunner(orderer) 827 o1Proc = ifrit.Invoke(o1Runner) 828 829 Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 830 831 assertBlockReception( 832 map[string]int{ 833 syschannel: int(sysBlockNum), 834 channel1: int(chan1BlockNum), 835 }, 836 []*nwo.Orderer{orderer}, 837 peer, 838 network, 839 ) 840 841 Eventually(o1Runner.Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say("Raft leader changed: 0 -> ")) 842 Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 843 844 By("7) System channel still in maintenance, State=MAINTENANCE, cannot create new channels") 845 exitCode := network.CreateChannelExitCode(channel2, orderer, peer) 846 Expect(exitCode).ToNot(Equal(0)) 847 848 By("8) Standard channel still in maintenance, State=MAINTENANCE, normal TX's blocked, delivery to peers blocked") 849 assertTxFailed(network, orderer, channel1) 850 851 err := checkPeerDeliverRequest(orderer, peer, network, channel1) 852 Expect(err).To(MatchError(errors.New("FORBIDDEN"))) 853 854 By("9) Release - executing config transaction on system channel with restarted orderer") 855 config, updatedConfig = prepareTransition(network, peer, orderer, syschannel, 856 "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE, 857 "etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL) 858 nwo.UpdateOrdererConfig(network, orderer, syschannel, config, updatedConfig, peer, orderer) 859 860 By("9) Verify: system channel config changed") 861 sysBlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, syschannel) 862 Expect(sysBlockNum).To(Equal(sysStartBlockNum + 2)) 863 864 By("10) Release - executing config transaction on standard channel with restarted orderer") 865 config, updatedConfig = prepareTransition(network, peer, orderer, channel1, 866 "etcdraft", protosorderer.ConsensusType_STATE_MAINTENANCE, 867 "etcdraft", raftMetadata, protosorderer.ConsensusType_STATE_NORMAL) 868 nwo.UpdateOrdererConfig(network, orderer, channel1, config, updatedConfig, peer, orderer) 869 870 By("10) Verify: standard channel config changed") 871 chan1BlockNum = nwo.CurrentConfigBlockNumber(network, peer, orderer, channel1) 872 Expect(chan1BlockNum).To(Equal(chan1StartBlockNum + 2)) 873 874 By("11) Executing transaction on standard channel with restarted orderer") 875 assertBlockCreation(network, orderer, peer, channel1, chan1StartBlockNum+3) 876 assertBlockCreation(network, orderer, nil, channel1, chan1StartBlockNum+4) 877 878 By("12) Create new channel, executing transaction with restarted orderer") 879 network.CreateChannel(channel2, orderer, peer) 880 881 assertBlockCreation(network, orderer, peer, channel2, 1) 882 assertBlockCreation(network, orderer, nil, channel2, 2) 883 884 By("13) Extending the network configuration to add a new orderer") 885 // Add another orderer 886 orderer2 := &nwo.Orderer{ 887 Name: "orderer2", 888 Organization: "OrdererOrg", 889 } 890 ports := nwo.Ports{} 891 for _, portName := range nwo.OrdererPortNames() { 892 ports[portName] = network.ReservePort() 893 } 894 network.PortsByOrdererID[orderer2.ID()] = ports 895 network.Orderers = append(network.Orderers, orderer2) 896 network.GenerateOrdererConfig(orderer2) 897 extendNetwork(network) 898 899 secondOrdererCertificatePath := filepath.Join(network.OrdererLocalTLSDir(orderer2), "server.crt") 900 secondOrdererCertificate, err := ioutil.ReadFile(secondOrdererCertificatePath) 901 Expect(err).NotTo(HaveOccurred()) 902 903 By("14) Adding the second orderer to system channel") 904 addConsenter(network, peer, orderer, syschannel, protosraft.Consenter{ 905 ServerTlsCert: secondOrdererCertificate, 906 ClientTlsCert: secondOrdererCertificate, 907 Host: "127.0.0.1", 908 Port: uint32(network.OrdererPort(orderer2, nwo.ClusterPort)), 909 }) 910 911 By("15) Obtaining the last config block from the orderer") 912 configBlock := nwo.GetConfigBlock(network, peer, orderer, syschannel) 913 err = ioutil.WriteFile(filepath.Join(testDir, "systemchannel_block.pb"), protoutil.MarshalOrPanic(configBlock), 0o644) 914 Expect(err).NotTo(HaveOccurred()) 915 916 By("16) Waiting for the existing orderer to relinquish its leadership") 917 Eventually(o1Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("1 stepped down to follower since quorum is not active")) 918 Eventually(o1Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("No leader is present, cluster size is 2")) 919 920 By("17) Launching the second orderer") 921 o2Runner = network.OrdererRunner(orderer2) 922 o2Proc = ifrit.Invoke(o2Runner) 923 Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed()) 924 Eventually(o2Runner.Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say("Raft leader changed: 0 -> ")) 925 926 By("18) Adding orderer2 to channel2") 927 addConsenter(network, peer, orderer, channel2, protosraft.Consenter{ 928 ServerTlsCert: secondOrdererCertificate, 929 ClientTlsCert: secondOrdererCertificate, 930 Host: "127.0.0.1", 931 Port: uint32(network.OrdererPort(orderer2, nwo.ClusterPort)), 932 }) 933 934 assertBlockReception(map[string]int{ 935 syschannel: int(sysBlockNum + 2), 936 channel2: int(nwo.CurrentConfigBlockNumber(network, peer, orderer, channel2)), 937 }, []*nwo.Orderer{orderer2}, peer, network) 938 939 By("19) Executing transaction against second orderer on channel2") 940 assertBlockCreation(network, orderer2, nil, channel2, 3) 941 }) 942 }) 943 }) 944 945 func validateConsensusTypeValue(value *protosorderer.ConsensusType, cType string, state protosorderer.ConsensusType_State) { 946 Expect(value.Type).To(Equal(cType)) 947 Expect(value.State).To(Equal(state)) 948 } 949 950 func extractOrdererConsensusType(config *common.Config) *protosorderer.ConsensusType { 951 var consensusTypeValue protosorderer.ConsensusType 952 consensusTypeConfigValue := config.ChannelGroup.Groups["Orderer"].Values["ConsensusType"] 953 err := proto.Unmarshal(consensusTypeConfigValue.Value, &consensusTypeValue) 954 Expect(err).NotTo(HaveOccurred()) 955 return &consensusTypeValue 956 } 957 958 func updateConfigWithConsensusType( 959 consensusType string, 960 consensusMetadata []byte, 961 migState protosorderer.ConsensusType_State, 962 updatedConfig *common.Config, 963 consensusTypeValue *protosorderer.ConsensusType, 964 ) { 965 consensusTypeValue.Type = consensusType 966 consensusTypeValue.Metadata = consensusMetadata 967 consensusTypeValue.State = migState 968 updatedConfig.ChannelGroup.Groups["Orderer"].Values["ConsensusType"] = &common.ConfigValue{ 969 ModPolicy: "Admins", 970 Value: protoutil.MarshalOrPanic(consensusTypeValue), 971 } 972 } 973 974 func updateConfigWithBatchTimeout(updatedConfig *common.Config) { 975 batchTimeoutConfigValue := updatedConfig.ChannelGroup.Groups["Orderer"].Values["BatchTimeout"] 976 batchTimeoutValue := new(protosorderer.BatchTimeout) 977 err := proto.Unmarshal(batchTimeoutConfigValue.Value, batchTimeoutValue) 978 Expect(err).NotTo(HaveOccurred()) 979 toDur, err := time.ParseDuration(batchTimeoutValue.Timeout) 980 Expect(err).NotTo(HaveOccurred()) 981 toDur = toDur + time.Duration(100000000) 982 batchTimeoutValue.Timeout = toDur.String() 983 By(fmt.Sprintf("Increasing BatchTimeout to %s", batchTimeoutValue.Timeout)) 984 updatedConfig.ChannelGroup.Groups["Orderer"].Values["BatchTimeout"] = &common.ConfigValue{ 985 ModPolicy: "Admins", 986 Value: protoutil.MarshalOrPanic(batchTimeoutValue), 987 } 988 } 989 990 func kafka2RaftMultiChannel() *nwo.Config { 991 config := nwo.BasicKafka() 992 config.Channels = []*nwo.Channel{ 993 {Name: "testchannel1", Profile: "TwoOrgsChannel"}, 994 {Name: "testchannel2", Profile: "TwoOrgsChannel"}, 995 {Name: "testchannel3", Profile: "TwoOrgsChannel"}, 996 } 997 998 for _, peer := range config.Peers { 999 peer.Channels = []*nwo.PeerChannel{ 1000 {Name: "testchannel1", Anchor: true}, 1001 {Name: "testchannel2", Anchor: true}, 1002 {Name: "testchannel3", Anchor: true}, 1003 } 1004 } 1005 return config 1006 } 1007 1008 func solo2RaftMultiChannel() *nwo.Config { 1009 config := nwo.BasicSolo() 1010 config.Channels = []*nwo.Channel{ 1011 {Name: "testchannel1", Profile: "TwoOrgsChannel"}, 1012 {Name: "testchannel2", Profile: "TwoOrgsChannel"}, 1013 } 1014 1015 for _, peer := range config.Peers { 1016 peer.Channels = []*nwo.PeerChannel{ 1017 {Name: "testchannel1", Anchor: true}, 1018 {Name: "testchannel2", Anchor: true}, 1019 } 1020 } 1021 return config 1022 } 1023 1024 func kafka2RaftMultiNode() *nwo.Config { 1025 config := nwo.BasicKafka() 1026 config.Orderers = []*nwo.Orderer{ 1027 {Name: "orderer1", Organization: "OrdererOrg"}, 1028 {Name: "orderer2", Organization: "OrdererOrg"}, 1029 {Name: "orderer3", Organization: "OrdererOrg"}, 1030 } 1031 1032 config.Profiles = []*nwo.Profile{{ 1033 Name: "TwoOrgsOrdererGenesis", 1034 Orderers: []string{"orderer1", "orderer2", "orderer3"}, 1035 }, { 1036 Name: "TwoOrgsChannel", 1037 Consortium: "SampleConsortium", 1038 Organizations: []string{"Org1", "Org2"}, 1039 }} 1040 1041 config.Channels = []*nwo.Channel{ 1042 {Name: "testchannel1", Profile: "TwoOrgsChannel"}, 1043 {Name: "testchannel2", Profile: "TwoOrgsChannel"}, 1044 {Name: "testchannel3", Profile: "TwoOrgsChannel"}, 1045 } 1046 1047 for _, peer := range config.Peers { 1048 peer.Channels = []*nwo.PeerChannel{ 1049 {Name: "testchannel1", Anchor: true}, 1050 {Name: "testchannel2", Anchor: true}, 1051 {Name: "testchannel3", Anchor: true}, 1052 } 1053 } 1054 return config 1055 } 1056 1057 func prepareRaftMetadata(network *nwo.Network) []byte { 1058 var consenters []*protosraft.Consenter 1059 for _, o := range network.Orderers { 1060 fullTlsPath := network.OrdererLocalTLSDir(o) 1061 certBytes, err := ioutil.ReadFile(filepath.Join(fullTlsPath, "server.crt")) 1062 Expect(err).NotTo(HaveOccurred()) 1063 port := network.OrdererPort(o, nwo.ClusterPort) 1064 1065 consenter := &protosraft.Consenter{ 1066 ClientTlsCert: certBytes, 1067 ServerTlsCert: certBytes, 1068 Host: "127.0.0.1", 1069 Port: uint32(port), 1070 } 1071 consenters = append(consenters, consenter) 1072 } 1073 1074 raftMetadata := &protosraft.ConfigMetadata{ 1075 Consenters: consenters, 1076 Options: &protosraft.Options{ 1077 TickInterval: "500ms", 1078 ElectionTick: 10, 1079 HeartbeatTick: 1, 1080 MaxInflightBlocks: 5, 1081 SnapshotIntervalSize: 10 * 1024 * 1024, 1082 }, 1083 } 1084 1085 raftMetadataBytes := protoutil.MarshalOrPanic(raftMetadata) 1086 1087 return raftMetadataBytes 1088 } 1089 1090 func checkPeerDeliverRequest(o *nwo.Orderer, submitter *nwo.Peer, network *nwo.Network, channelName string) error { 1091 c := commands.ChannelFetch{ 1092 ChannelID: channelName, 1093 Block: "newest", 1094 OutputFile: "/dev/null", 1095 Orderer: network.OrdererAddress(o, nwo.ListenPort), 1096 } 1097 1098 sess, err := network.PeerUserSession(submitter, "User1", c) 1099 Expect(err).NotTo(HaveOccurred()) 1100 Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit()) 1101 sessErr := string(sess.Err.Contents()) 1102 sessExitCode := sess.ExitCode() 1103 if sessExitCode != 0 && strings.Contains(sessErr, "FORBIDDEN") { 1104 return errors.New("FORBIDDEN") 1105 } 1106 if sessExitCode == 0 && strings.Contains(sessErr, "Received block: ") { 1107 return nil 1108 } 1109 1110 return fmt.Errorf("Unexpected result: ExitCode=%d, Err=%s", sessExitCode, sessErr) 1111 } 1112 1113 func updateOrdererConfigFailed(n *nwo.Network, orderer *nwo.Orderer, channel string, current, updated *common.Config, peer *nwo.Peer, additionalSigners ...*nwo.Orderer) { 1114 sess := nwo.UpdateOrdererConfigSession(n, orderer, channel, current, updated, peer, additionalSigners...) 1115 Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(1)) 1116 Expect(sess.Err).NotTo(gbytes.Say("Successfully submitted channel update")) 1117 } 1118 1119 func prepareTransition( 1120 network *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel string, // Auxiliary 1121 fromConsensusType string, fromMigState protosorderer.ConsensusType_State, // From 1122 toConsensusType string, toConsensusMetadata []byte, toMigState protosorderer.ConsensusType_State, // To 1123 ) (current, updated *common.Config) { 1124 current = nwo.GetConfig(network, peer, orderer, channel) 1125 updated = proto.Clone(current).(*common.Config) 1126 consensusTypeValue := extractOrdererConsensusType(current) 1127 validateConsensusTypeValue(consensusTypeValue, fromConsensusType, fromMigState) 1128 updateConfigWithConsensusType(toConsensusType, toConsensusMetadata, toMigState, updated, consensusTypeValue) 1129 return current, updated 1130 } 1131 1132 func assertTransitionFailed( 1133 network *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel string, // Auxiliary 1134 fromConsensusType string, fromMigState protosorderer.ConsensusType_State, // From 1135 toConsensusType string, toConsensusMetadata []byte, toMigState protosorderer.ConsensusType_State, // To 1136 ) { 1137 current, updated := prepareTransition( 1138 network, peer, orderer, channel, 1139 fromConsensusType, fromMigState, 1140 toConsensusType, toConsensusMetadata, toMigState) 1141 updateOrdererConfigFailed(network, orderer, channel, current, updated, peer, orderer) 1142 } 1143 1144 func assertBlockCreation(network *nwo.Network, orderer *nwo.Orderer, peer *nwo.Peer, channelID string, blkNum uint64) { 1145 var signer *nwo.SigningIdentity 1146 signer = network.OrdererUserSigner(orderer, "Admin") 1147 if peer != nil { 1148 signer = network.PeerUserSigner(peer, "Admin") 1149 } 1150 env := createBroadcastEnvelope(network, signer, channelID, []byte("hola")) 1151 resp, err := ordererclient.Broadcast(network, orderer, env) 1152 Expect(err).NotTo(HaveOccurred()) 1153 Expect(resp.Status).To(Equal(common.Status_SUCCESS)) 1154 1155 denv := createDeliverEnvelope(network, signer, blkNum, channelID) 1156 blk, err := ordererclient.Deliver(network, orderer, denv) 1157 Expect(err).NotTo(HaveOccurred()) 1158 Expect(blk).ToNot(BeNil()) 1159 } 1160 1161 func assertTxFailed(network *nwo.Network, orderer *nwo.Orderer, channelID string) { 1162 signer := network.OrdererUserSigner(orderer, "Admin") 1163 env := createBroadcastEnvelope(network, signer, channelID, []byte("hola")) 1164 resp, err := ordererclient.Broadcast(network, orderer, env) 1165 Expect(err).NotTo(HaveOccurred()) 1166 Expect(resp.Status).To(Equal(common.Status_SERVICE_UNAVAILABLE)) 1167 Expect(resp.Info).To(Equal("normal transactions are rejected: maintenance mode")) 1168 } 1169 1170 // assertBlockReception asserts that the given orderers have the expected 1171 // newest block number for the specified channels 1172 func assertBlockReception(expectedBlockNumPerChannel map[string]int, orderers []*nwo.Orderer, p *nwo.Peer, n *nwo.Network) { 1173 for channelName, blockNum := range expectedBlockNumPerChannel { 1174 for _, orderer := range orderers { 1175 waitForBlockReception(orderer, p, n, channelName, blockNum) 1176 } 1177 } 1178 } 1179 1180 func waitForBlockReception(o *nwo.Orderer, submitter *nwo.Peer, network *nwo.Network, channelName string, blockNum int) { 1181 c := commands.ChannelFetch{ 1182 ChannelID: channelName, 1183 Block: "newest", 1184 OutputFile: "/dev/null", 1185 Orderer: network.OrdererAddress(o, nwo.ListenPort), 1186 } 1187 Eventually(func() string { 1188 sess, err := network.OrdererAdminSession(o, submitter, c) 1189 Expect(err).NotTo(HaveOccurred()) 1190 Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit()) 1191 if sess.ExitCode() != 0 { 1192 return fmt.Sprintf("exit code is %d: %s", sess.ExitCode(), string(sess.Err.Contents())) 1193 } 1194 sessErr := string(sess.Err.Contents()) 1195 expected := fmt.Sprintf("Received block: %d", blockNum) 1196 if strings.Contains(sessErr, expected) { 1197 return "" 1198 } 1199 return sessErr 1200 }, network.EventuallyTimeout, time.Second).Should(BeEmpty()) 1201 } 1202 1203 func createBroadcastEnvelope(n *nwo.Network, signer *nwo.SigningIdentity, channel string, data []byte) *common.Envelope { 1204 env, err := protoutil.CreateSignedEnvelope( 1205 common.HeaderType_MESSAGE, 1206 channel, 1207 signer, 1208 &common.Envelope{Payload: data}, 1209 0, 1210 0, 1211 ) 1212 Expect(err).NotTo(HaveOccurred()) 1213 1214 return env 1215 } 1216 1217 // CreateDeliverEnvelope creates a deliver env to seek for specified block. 1218 func createDeliverEnvelope(n *nwo.Network, signer *nwo.SigningIdentity, blkNum uint64, channel string) *common.Envelope { 1219 specified := &protosorderer.SeekPosition{ 1220 Type: &protosorderer.SeekPosition_Specified{ 1221 Specified: &protosorderer.SeekSpecified{Number: blkNum}, 1222 }, 1223 } 1224 env, err := protoutil.CreateSignedEnvelope( 1225 common.HeaderType_DELIVER_SEEK_INFO, 1226 channel, 1227 signer, 1228 &protosorderer.SeekInfo{ 1229 Start: specified, 1230 Stop: specified, 1231 Behavior: protosorderer.SeekInfo_BLOCK_UNTIL_READY, 1232 }, 1233 0, 1234 0, 1235 ) 1236 Expect(err).NotTo(HaveOccurred()) 1237 1238 return env 1239 } 1240 1241 var extendedCryptoConfig = `--- 1242 OrdererOrgs: 1243 - Name: OrdererOrg 1244 Domain: example.com 1245 EnableNodeOUs: false 1246 CA: 1247 Hostname: ca 1248 Specs: 1249 - Hostname: orderer1 1250 SANS: 1251 - localhost 1252 - 127.0.0.1 1253 - ::1 1254 - Hostname: orderer1new 1255 SANS: 1256 - localhost 1257 - 127.0.0.1 1258 - ::1 1259 - Hostname: orderer2 1260 SANS: 1261 - localhost 1262 - 127.0.0.1 1263 - ::1 1264 - Hostname: orderer2new 1265 SANS: 1266 - localhost 1267 - 127.0.0.1 1268 - ::1 1269 - Hostname: orderer3 1270 SANS: 1271 - localhost 1272 - 127.0.0.1 1273 - ::1 1274 - Hostname: orderer3new 1275 SANS: 1276 - localhost 1277 - 127.0.0.1 1278 - ::1 1279 - Hostname: orderer4 1280 SANS: 1281 - localhost 1282 - 127.0.0.1 1283 - ::1 1284 - Hostname: orderer5 1285 SANS: 1286 - localhost 1287 - 127.0.0.1 1288 - ::1 1289 - Hostname: orderer6 1290 SANS: 1291 - localhost 1292 - 127.0.0.1 1293 - ::1 1294 - Hostname: orderer7 1295 SANS: 1296 - localhost 1297 - 127.0.0.1 1298 - ::1 1299 ` 1300 1301 // extendNetwork rotates adds an additional orderer 1302 func extendNetwork(n *nwo.Network) { 1303 // Overwrite the current crypto-config with additional orderers 1304 cryptoConfigYAML, err := ioutil.TempFile("", "crypto-config.yaml") 1305 Expect(err).NotTo(HaveOccurred()) 1306 defer os.Remove(cryptoConfigYAML.Name()) 1307 1308 err = ioutil.WriteFile(cryptoConfigYAML.Name(), []byte(extendedCryptoConfig), 0o644) 1309 Expect(err).NotTo(HaveOccurred()) 1310 1311 // Invoke cryptogen extend to add new orderers 1312 sess, err := n.Cryptogen(commands.Extend{ 1313 Config: cryptoConfigYAML.Name(), 1314 Input: n.CryptoPath(), 1315 }) 1316 Expect(err).NotTo(HaveOccurred()) 1317 Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(0)) 1318 } 1319 1320 // addConsenter adds a new consenter to the given channel. 1321 func addConsenter(n *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel string, consenter protosraft.Consenter) { 1322 updateEtcdRaftMetadata(n, peer, orderer, channel, func(metadata *protosraft.ConfigMetadata) { 1323 metadata.Consenters = append(metadata.Consenters, &consenter) 1324 }) 1325 } 1326 1327 // updateEtcdRaftMetadata executes a config update that updates the etcdraft 1328 // metadata according to the given function f. 1329 func updateEtcdRaftMetadata(network *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, channel string, f func(md *protosraft.ConfigMetadata)) { 1330 nwo.UpdateConsensusMetadata(network, peer, orderer, channel, func(originalMetadata []byte) []byte { 1331 metadata := &protosraft.ConfigMetadata{} 1332 err := proto.Unmarshal(originalMetadata, metadata) 1333 Expect(err).NotTo(HaveOccurred()) 1334 1335 f(metadata) 1336 1337 newMetadata, err := proto.Marshal(metadata) 1338 Expect(err).NotTo(HaveOccurred()) 1339 return newMetadata 1340 }) 1341 }