github.com/darrenli6/fabric-sdk-example@v0.0.0-20220109053535-94b13b56df8c/orderer/kafka/chain_test.go (about) 1 /* 2 Copyright IBM Corp. All Rights Reserved. 3 4 SPDX-License-Identifier: Apache-2.0 5 */ 6 7 package kafka 8 9 import ( 10 "fmt" 11 "testing" 12 "time" 13 14 "github.com/Shopify/sarama" 15 "github.com/Shopify/sarama/mocks" 16 mockconfig "github.com/hyperledger/fabric/common/mocks/config" 17 mockblockcutter "github.com/hyperledger/fabric/orderer/mocks/blockcutter" 18 mockmultichain "github.com/hyperledger/fabric/orderer/mocks/multichain" 19 cb "github.com/hyperledger/fabric/protos/common" 20 ab "github.com/hyperledger/fabric/protos/orderer" 21 "github.com/hyperledger/fabric/protos/utils" 22 "github.com/stretchr/testify/assert" 23 ) 24 25 var ( 26 extraShortTimeout = 1 * time.Millisecond 27 shortTimeout = 1 * time.Second 28 longTimeout = 1 * time.Hour 29 30 hitBranch = 50 * time.Millisecond 31 ) 32 33 func TestChain(t *testing.T) { 34 35 oldestOffset := int64(0) 36 newestOffset := int64(5) 37 38 message := sarama.StringEncoder("messageFoo") 39 40 newMocks := func(t *testing.T) (mockChannel channel, mockBroker *sarama.MockBroker, mockSupport *mockmultichain.ConsenterSupport) { 41 mockChannel = newChannel(channelNameForTest(t), defaultPartition) 42 mockBroker = sarama.NewMockBroker(t, 0) 43 mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{ 44 "MetadataRequest": sarama.NewMockMetadataResponse(t). 45 SetBroker(mockBroker.Addr(), mockBroker.BrokerID()). 46 SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()), 47 "ProduceRequest": sarama.NewMockProduceResponse(t). 48 SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError), 49 "OffsetRequest": sarama.NewMockOffsetResponse(t). 50 SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset). 51 SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset), 52 "FetchRequest": sarama.NewMockFetchResponse(t, 1). 53 SetMessage(mockChannel.topic(), mockChannel.partition(), newestOffset, message), 54 }) 55 mockSupport = &mockmultichain.ConsenterSupport{ 56 ChainIDVal: mockChannel.topic(), 57 HeightVal: uint64(3), 58 SharedConfigVal: &mockconfig.Orderer{KafkaBrokersVal: []string{mockBroker.Addr()}}, 59 } 60 return 61 } 62 63 t.Run("New", func(t *testing.T) { 64 _, mockBroker, mockSupport := newMocks(t) 65 defer func() { mockBroker.Close() }() 66 chain, err := newChain(mockConsenter, mockSupport, newestOffset-1) 67 68 assert.NoError(t, err, "Expected newChain to return without errors") 69 select { 70 case <-chain.errorChan: 71 logger.Debug("errorChan is closed as it should be") 72 default: 73 t.Fatal("errorChan should have been closed") 74 } 75 76 select { 77 case <-chain.haltChan: 78 t.Fatal("haltChan should have been open") 79 default: 80 logger.Debug("haltChan is open as it should be") 81 } 82 83 select { 84 case <-chain.startChan: 85 t.Fatal("startChan should have been open") 86 default: 87 logger.Debug("startChan is open as it should be") 88 } 89 }) 90 91 t.Run("Start", func(t *testing.T) { 92 _, mockBroker, mockSupport := newMocks(t) 93 defer func() { mockBroker.Close() }() 94 // Set to -1 because we haven't sent the CONNECT message yet 95 chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1) 96 97 chain.Start() 98 select { 99 case <-chain.startChan: 100 logger.Debug("startChan is closed as it should be") 101 case <-time.After(shortTimeout): 102 t.Fatal("startChan should have been closed by now") 103 } 104 105 // Trigger the haltChan clause in the processMessagesToBlocks goroutine 106 close(chain.haltChan) 107 }) 108 109 t.Run("Halt", func(t *testing.T) { 110 _, mockBroker, mockSupport := newMocks(t) 111 defer func() { mockBroker.Close() }() 112 chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1) 113 114 chain.Start() 115 select { 116 case <-chain.startChan: 117 logger.Debug("startChan is closed as it should be") 118 case <-time.After(shortTimeout): 119 t.Fatal("startChan should have been closed by now") 120 } 121 122 // Wait till the start phase has completed, then: 123 chain.Halt() 124 125 select { 126 case <-chain.haltChan: 127 logger.Debug("haltChan is closed as it should be") 128 case <-time.After(shortTimeout): 129 t.Fatal("haltChan should have been closed") 130 } 131 132 select { 133 case <-chain.errorChan: 134 logger.Debug("errorChan is closed as it should be") 135 case <-time.After(shortTimeout): 136 t.Fatal("errorChan should have been closed") 137 } 138 }) 139 140 t.Run("DoubleHalt", func(t *testing.T) { 141 _, mockBroker, mockSupport := newMocks(t) 142 defer func() { mockBroker.Close() }() 143 chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1) 144 145 chain.Start() 146 select { 147 case <-chain.startChan: 148 logger.Debug("startChan is closed as it should be") 149 case <-time.After(shortTimeout): 150 t.Fatal("startChan should have been closed by now") 151 } 152 153 chain.Halt() 154 155 assert.NotPanics(t, func() { chain.Halt() }, "Calling Halt() more than once shouldn't panic") 156 }) 157 158 t.Run("StartWithProducerForChannelError", func(t *testing.T) { 159 _, mockBroker, mockSupport := newMocks(t) 160 defer func() { mockBroker.Close() }() 161 // Point to an empty brokers list 162 mockSupportCopy := *mockSupport 163 mockSupportCopy.SharedConfigVal = &mockconfig.Orderer{KafkaBrokersVal: []string{}} 164 165 chain, _ := newChain(mockConsenter, &mockSupportCopy, newestOffset-1) 166 167 // The production path will actually call chain.Start(). This is 168 // functionally equivalent and allows us to run assertions on it. 169 assert.Panics(t, func() { startThread(chain) }, "Expected the Start() call to panic") 170 }) 171 172 t.Run("StartWithConnectMessageError", func(t *testing.T) { 173 // Note that this test is affected by the following parameters: 174 // - Net.ReadTimeout 175 // - Consumer.Retry.Backoff 176 // - Metadata.Retry.Max 177 mockChannel, mockBroker, mockSupport := newMocks(t) 178 defer func() { mockBroker.Close() }() 179 chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1) 180 181 // Have the broker return an ErrNotLeaderForPartition error 182 mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{ 183 "MetadataRequest": sarama.NewMockMetadataResponse(t). 184 SetBroker(mockBroker.Addr(), mockBroker.BrokerID()). 185 SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()), 186 "ProduceRequest": sarama.NewMockProduceResponse(t). 187 SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNotLeaderForPartition), 188 "OffsetRequest": sarama.NewMockOffsetResponse(t). 189 SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset). 190 SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset), 191 "FetchRequest": sarama.NewMockFetchResponse(t, 1). 192 SetMessage(mockChannel.topic(), mockChannel.partition(), newestOffset, message), 193 }) 194 195 assert.Panics(t, func() { startThread(chain) }, "Expected the Start() call to panic") 196 }) 197 198 t.Run("EnqueueIfNotStarted", func(t *testing.T) { 199 mockChannel, mockBroker, mockSupport := newMocks(t) 200 defer func() { mockBroker.Close() }() 201 chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1) 202 203 // As in StartWithConnectMessageError, have the broker return an 204 // ErrNotLeaderForPartition error, i.e. cause an error in the 205 // 'post connect message' step. 206 mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{ 207 "MetadataRequest": sarama.NewMockMetadataResponse(t). 208 SetBroker(mockBroker.Addr(), mockBroker.BrokerID()). 209 SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()), 210 "ProduceRequest": sarama.NewMockProduceResponse(t). 211 SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNotLeaderForPartition), 212 "OffsetRequest": sarama.NewMockOffsetResponse(t). 213 SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset). 214 SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset), 215 "FetchRequest": sarama.NewMockFetchResponse(t, 1). 216 SetMessage(mockChannel.topic(), mockChannel.partition(), newestOffset, message), 217 }) 218 219 assert.False(t, chain.Enqueue(newMockEnvelope("fooMessage")), "Expected Enqueue call to return false") 220 }) 221 222 t.Run("StartWithConsumerForChannelError", func(t *testing.T) { 223 // Note that this test is affected by the following parameters: 224 // - Net.ReadTimeout 225 // - Consumer.Retry.Backoff 226 // - Metadata.Retry.Max 227 228 mockChannel, mockBroker, mockSupport := newMocks(t) 229 defer func() { mockBroker.Close() }() 230 231 // Provide an out-of-range offset 232 chain, _ := newChain(mockConsenter, mockSupport, newestOffset) 233 234 mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{ 235 "MetadataRequest": sarama.NewMockMetadataResponse(t). 236 SetBroker(mockBroker.Addr(), mockBroker.BrokerID()). 237 SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()), 238 "ProduceRequest": sarama.NewMockProduceResponse(t). 239 SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError), 240 "OffsetRequest": sarama.NewMockOffsetResponse(t). 241 SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset). 242 SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset), 243 "FetchRequest": sarama.NewMockFetchResponse(t, 1). 244 SetMessage(mockChannel.topic(), mockChannel.partition(), newestOffset, message), 245 }) 246 247 assert.Panics(t, func() { startThread(chain) }, "Expected the Start() call to panic") 248 }) 249 250 t.Run("EnqueueProper", func(t *testing.T) { 251 mockChannel, mockBroker, mockSupport := newMocks(t) 252 defer func() { mockBroker.Close() }() 253 chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1) 254 255 mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{ 256 "MetadataRequest": sarama.NewMockMetadataResponse(t). 257 SetBroker(mockBroker.Addr(), mockBroker.BrokerID()). 258 SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()), 259 "ProduceRequest": sarama.NewMockProduceResponse(t). 260 SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError), 261 "OffsetRequest": sarama.NewMockOffsetResponse(t). 262 SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset). 263 SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset), 264 "FetchRequest": sarama.NewMockFetchResponse(t, 1). 265 SetMessage(mockChannel.topic(), mockChannel.partition(), newestOffset, message), 266 }) 267 268 chain.Start() 269 select { 270 case <-chain.startChan: 271 logger.Debug("startChan is closed as it should be") 272 case <-time.After(shortTimeout): 273 t.Fatal("startChan should have been closed by now") 274 } 275 276 // Enqueue should have access to the post path, and its ProduceRequest 277 // should go by without error 278 assert.True(t, chain.Enqueue(newMockEnvelope("fooMessage")), "Expected Enqueue call to return true") 279 280 chain.Halt() 281 }) 282 283 t.Run("EnqueueIfHalted", func(t *testing.T) { 284 mockChannel, mockBroker, mockSupport := newMocks(t) 285 defer func() { mockBroker.Close() }() 286 chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1) 287 288 mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{ 289 "MetadataRequest": sarama.NewMockMetadataResponse(t). 290 SetBroker(mockBroker.Addr(), mockBroker.BrokerID()). 291 SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()), 292 "ProduceRequest": sarama.NewMockProduceResponse(t). 293 SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError), 294 "OffsetRequest": sarama.NewMockOffsetResponse(t). 295 SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset). 296 SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset), 297 "FetchRequest": sarama.NewMockFetchResponse(t, 1). 298 SetMessage(mockChannel.topic(), mockChannel.partition(), newestOffset, message), 299 }) 300 301 chain.Start() 302 select { 303 case <-chain.startChan: 304 logger.Debug("startChan is closed as it should be") 305 case <-time.After(shortTimeout): 306 t.Fatal("startChan should have been closed by now") 307 } 308 chain.Halt() 309 310 // haltChan should close access to the post path 311 assert.False(t, chain.Enqueue(newMockEnvelope("fooMessage")), "Expected Enqueue call to return false") 312 }) 313 314 t.Run("EnqueueError", func(t *testing.T) { 315 mockChannel, mockBroker, mockSupport := newMocks(t) 316 defer func() { mockBroker.Close() }() 317 chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1) 318 319 // Use the "good" handler map that allows the Stage to complete without 320 // issues 321 mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{ 322 "MetadataRequest": sarama.NewMockMetadataResponse(t). 323 SetBroker(mockBroker.Addr(), mockBroker.BrokerID()). 324 SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()), 325 "ProduceRequest": sarama.NewMockProduceResponse(t). 326 SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError), 327 "OffsetRequest": sarama.NewMockOffsetResponse(t). 328 SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset). 329 SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset), 330 "FetchRequest": sarama.NewMockFetchResponse(t, 1). 331 SetMessage(mockChannel.topic(), mockChannel.partition(), newestOffset, message), 332 }) 333 334 chain.Start() 335 select { 336 case <-chain.startChan: 337 logger.Debug("startChan is closed as it should be") 338 case <-time.After(shortTimeout): 339 t.Fatal("startChan should have been closed by now") 340 } 341 342 // Now make it so that the next ProduceRequest is met with an error 343 mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{ 344 "ProduceRequest": sarama.NewMockProduceResponse(t). 345 SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNotLeaderForPartition), 346 }) 347 348 assert.False(t, chain.Enqueue(newMockEnvelope("fooMessage")), "Expected Enqueue call to return false") 349 }) 350 } 351 352 func TestSetupProducerForChannel(t *testing.T) { 353 if testing.Short() { 354 t.Skip("Skipping test in short mode") 355 } 356 357 mockBroker := sarama.NewMockBroker(t, 0) 358 defer mockBroker.Close() 359 360 mockChannel := newChannel(channelNameForTest(t), defaultPartition) 361 362 haltChan := make(chan struct{}) 363 364 t.Run("Proper", func(t *testing.T) { 365 metadataResponse := new(sarama.MetadataResponse) 366 metadataResponse.AddBroker(mockBroker.Addr(), mockBroker.BrokerID()) 367 metadataResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID(), nil, nil, sarama.ErrNoError) 368 mockBroker.Returns(metadataResponse) 369 370 producer, err := setupProducerForChannel(mockConsenter.retryOptions(), haltChan, []string{mockBroker.Addr()}, mockBrokerConfig, mockChannel) 371 assert.NoError(t, err, "Expected the setupProducerForChannel call to return without errors") 372 assert.NoError(t, producer.Close(), "Expected to close the producer without errors") 373 }) 374 375 t.Run("WithError", func(t *testing.T) { 376 _, err := setupProducerForChannel(mockConsenter.retryOptions(), haltChan, []string{}, mockBrokerConfig, mockChannel) 377 assert.Error(t, err, "Expected the setupProducerForChannel call to return an error") 378 }) 379 } 380 381 func TestSetupConsumerForChannel(t *testing.T) { 382 mockBroker := sarama.NewMockBroker(t, 0) 383 defer func() { mockBroker.Close() }() 384 385 mockChannel := newChannel(channelNameForTest(t), defaultPartition) 386 387 oldestOffset := int64(0) 388 newestOffset := int64(5) 389 390 startFrom := int64(3) 391 message := sarama.StringEncoder("messageFoo") 392 393 mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{ 394 "MetadataRequest": sarama.NewMockMetadataResponse(t). 395 SetBroker(mockBroker.Addr(), mockBroker.BrokerID()). 396 SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()), 397 "OffsetRequest": sarama.NewMockOffsetResponse(t). 398 SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset). 399 SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset), 400 "FetchRequest": sarama.NewMockFetchResponse(t, 1). 401 SetMessage(mockChannel.topic(), mockChannel.partition(), startFrom, message), 402 }) 403 404 haltChan := make(chan struct{}) 405 406 t.Run("ProperParent", func(t *testing.T) { 407 parentConsumer, err := setupParentConsumerForChannel(mockConsenter.retryOptions(), haltChan, []string{mockBroker.Addr()}, mockBrokerConfig, mockChannel) 408 assert.NoError(t, err, "Expected the setupParentConsumerForChannel call to return without errors") 409 assert.NoError(t, parentConsumer.Close(), "Expected to close the parentConsumer without errors") 410 }) 411 412 t.Run("ProperChannel", func(t *testing.T) { 413 parentConsumer, _ := setupParentConsumerForChannel(mockConsenter.retryOptions(), haltChan, []string{mockBroker.Addr()}, mockBrokerConfig, mockChannel) 414 defer func() { parentConsumer.Close() }() 415 channelConsumer, err := setupChannelConsumerForChannel(mockConsenter.retryOptions(), haltChan, parentConsumer, mockChannel, newestOffset) 416 assert.NoError(t, err, "Expected the setupChannelConsumerForChannel call to return without errors") 417 assert.NoError(t, channelConsumer.Close(), "Expected to close the channelConsumer without errors") 418 }) 419 420 t.Run("WithParentConsumerError", func(t *testing.T) { 421 // Provide an empty brokers list 422 _, err := setupParentConsumerForChannel(mockConsenter.retryOptions(), haltChan, []string{}, mockBrokerConfig, mockChannel) 423 assert.Error(t, err, "Expected the setupParentConsumerForChannel call to return an error") 424 }) 425 426 t.Run("WithChannelConsumerError", func(t *testing.T) { 427 // Provide an out-of-range offset 428 parentConsumer, _ := setupParentConsumerForChannel(mockConsenter.retryOptions(), haltChan, []string{mockBroker.Addr()}, mockBrokerConfig, mockChannel) 429 _, err := setupChannelConsumerForChannel(mockConsenter.retryOptions(), haltChan, parentConsumer, mockChannel, newestOffset+1) 430 defer func() { parentConsumer.Close() }() 431 assert.Error(t, err, "Expected the setupChannelConsumerForChannel call to return an error") 432 }) 433 } 434 435 func TestCloseKafkaObjects(t *testing.T) { 436 mockChannel := newChannel(channelNameForTest(t), defaultPartition) 437 438 mockSupport := &mockmultichain.ConsenterSupport{ 439 ChainIDVal: mockChannel.topic(), 440 } 441 442 oldestOffset := int64(0) 443 newestOffset := int64(5) 444 445 startFrom := int64(3) 446 message := sarama.StringEncoder("messageFoo") 447 448 mockBroker := sarama.NewMockBroker(t, 0) 449 defer func() { mockBroker.Close() }() 450 451 mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{ 452 "MetadataRequest": sarama.NewMockMetadataResponse(t). 453 SetBroker(mockBroker.Addr(), mockBroker.BrokerID()). 454 SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()), 455 "OffsetRequest": sarama.NewMockOffsetResponse(t). 456 SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset). 457 SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset), 458 "FetchRequest": sarama.NewMockFetchResponse(t, 1). 459 SetMessage(mockChannel.topic(), mockChannel.partition(), startFrom, message), 460 }) 461 462 haltChan := make(chan struct{}) 463 464 t.Run("Proper", func(t *testing.T) { 465 producer, _ := setupProducerForChannel(mockConsenter.retryOptions(), haltChan, []string{mockBroker.Addr()}, mockBrokerConfig, mockChannel) 466 parentConsumer, _ := setupParentConsumerForChannel(mockConsenter.retryOptions(), haltChan, []string{mockBroker.Addr()}, mockBrokerConfig, mockChannel) 467 channelConsumer, _ := setupChannelConsumerForChannel(mockConsenter.retryOptions(), haltChan, parentConsumer, mockChannel, startFrom) 468 469 // Set up a chain with just the minimum necessary fields instantiated so 470 // as to test the function 471 bareMinimumChain := &chainImpl{ 472 support: mockSupport, 473 producer: producer, 474 parentConsumer: parentConsumer, 475 channelConsumer: channelConsumer, 476 } 477 478 errs := bareMinimumChain.closeKafkaObjects() 479 480 assert.Len(t, errs, 0, "Expected zero errors") 481 482 assert.Panics(t, func() { 483 channelConsumer.Close() 484 }) 485 486 assert.NotPanics(t, func() { 487 parentConsumer.Close() 488 }) 489 490 // TODO For some reason this panic cannot be captured by the `assert` 491 // test framework. Not a dealbreaker but need to investigate further. 492 /* assert.Panics(t, func() { 493 producer.Close() 494 }) */ 495 }) 496 497 t.Run("ChannelConsumerError", func(t *testing.T) { 498 producer, _ := sarama.NewSyncProducer([]string{mockBroker.Addr()}, mockBrokerConfig) 499 500 // Unlike all other tests in this file, forcing an error on the 501 // channelConsumer.Close() call is more easily achieved using the mock 502 // Consumer. Thus we bypass the call to `setup*Consumer`. 503 504 // Have the consumer receive an ErrOutOfBrokers error. 505 mockParentConsumer := mocks.NewConsumer(t, nil) 506 mockParentConsumer.ExpectConsumePartition(mockChannel.topic(), mockChannel.partition(), startFrom).YieldError(sarama.ErrOutOfBrokers) 507 mockChannelConsumer, err := mockParentConsumer.ConsumePartition(mockChannel.topic(), mockChannel.partition(), startFrom) 508 assert.NoError(t, err, "Expected no error when setting up the mock partition consumer") 509 510 bareMinimumChain := &chainImpl{ 511 support: mockSupport, 512 producer: producer, 513 parentConsumer: mockParentConsumer, 514 channelConsumer: mockChannelConsumer, 515 } 516 517 errs := bareMinimumChain.closeKafkaObjects() 518 519 assert.Len(t, errs, 1, "Expected 1 error returned") 520 521 assert.NotPanics(t, func() { 522 mockChannelConsumer.Close() 523 }) 524 525 assert.NotPanics(t, func() { 526 mockParentConsumer.Close() 527 }) 528 }) 529 } 530 531 // Test helper functions here. 532 533 func TestGetLastCutBlockNumber(t *testing.T) { 534 testCases := []struct { 535 name string 536 input uint64 537 expected uint64 538 }{ 539 {"Proper", uint64(2), uint64(1)}, 540 {"Zero", uint64(1), uint64(0)}, 541 } 542 for _, tc := range testCases { 543 t.Run(tc.name, func(t *testing.T) { 544 assert.Equal(t, tc.expected, getLastCutBlockNumber(tc.input)) 545 }) 546 } 547 } 548 549 func TestGetLastOffsetPersisted(t *testing.T) { 550 mockChannel := newChannel(channelNameForTest(t), defaultPartition) 551 mockMetadata := &cb.Metadata{Value: utils.MarshalOrPanic(&ab.KafkaMetadata{LastOffsetPersisted: int64(5)})} 552 553 testCases := []struct { 554 name string 555 md []byte 556 expected int64 557 panics bool 558 }{ 559 {"Proper", mockMetadata.Value, int64(5), false}, 560 {"Empty", nil, sarama.OffsetOldest - 1, false}, 561 {"Panics", tamperBytes(mockMetadata.Value), sarama.OffsetOldest - 1, true}, 562 } 563 564 for _, tc := range testCases { 565 t.Run(tc.name, func(t *testing.T) { 566 if !tc.panics { 567 assert.Equal(t, tc.expected, getLastOffsetPersisted(tc.md, mockChannel.String())) 568 } else { 569 assert.Panics(t, func() { 570 getLastOffsetPersisted(tc.md, mockChannel.String()) 571 }, "Expected getLastOffsetPersisted call to panic") 572 } 573 }) 574 } 575 } 576 577 func TestSendConnectMessage(t *testing.T) { 578 mockBroker := sarama.NewMockBroker(t, 0) 579 defer func() { mockBroker.Close() }() 580 581 mockChannel := newChannel("mockChannelFoo", defaultPartition) 582 583 metadataResponse := new(sarama.MetadataResponse) 584 metadataResponse.AddBroker(mockBroker.Addr(), mockBroker.BrokerID()) 585 metadataResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID(), nil, nil, sarama.ErrNoError) 586 mockBroker.Returns(metadataResponse) 587 588 producer, _ := sarama.NewSyncProducer([]string{mockBroker.Addr()}, mockBrokerConfig) 589 defer func() { producer.Close() }() 590 591 haltChan := make(chan struct{}) 592 593 t.Run("Proper", func(t *testing.T) { 594 successResponse := new(sarama.ProduceResponse) 595 successResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError) 596 mockBroker.Returns(successResponse) 597 598 assert.NoError(t, sendConnectMessage(mockConsenter.retryOptions(), haltChan, producer, mockChannel), "Expected the sendConnectMessage call to return without errors") 599 }) 600 601 t.Run("WithError", func(t *testing.T) { 602 // Note that this test is affected by the following parameters: 603 // - Net.ReadTimeout 604 // - Consumer.Retry.Backoff 605 // - Metadata.Retry.Max 606 607 // Have the broker return an ErrNotEnoughReplicas error 608 failureResponse := new(sarama.ProduceResponse) 609 failureResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), sarama.ErrNotEnoughReplicas) 610 mockBroker.Returns(failureResponse) 611 612 assert.Error(t, sendConnectMessage(mockConsenter.retryOptions(), haltChan, producer, mockChannel), "Expected the sendConnectMessage call to return an error") 613 }) 614 } 615 616 func TestSendTimeToCut(t *testing.T) { 617 mockBroker := sarama.NewMockBroker(t, 0) 618 defer func() { mockBroker.Close() }() 619 620 mockChannel := newChannel("mockChannelFoo", defaultPartition) 621 622 metadataResponse := new(sarama.MetadataResponse) 623 metadataResponse.AddBroker(mockBroker.Addr(), mockBroker.BrokerID()) 624 metadataResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID(), nil, nil, sarama.ErrNoError) 625 mockBroker.Returns(metadataResponse) 626 627 producer, err := sarama.NewSyncProducer([]string{mockBroker.Addr()}, mockBrokerConfig) 628 assert.NoError(t, err, "Expected no error when setting up the sarama SyncProducer") 629 defer func() { producer.Close() }() 630 631 timeToCutBlockNumber := uint64(3) 632 var timer <-chan time.Time 633 634 t.Run("Proper", func(t *testing.T) { 635 successResponse := new(sarama.ProduceResponse) 636 successResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError) 637 mockBroker.Returns(successResponse) 638 639 timer = time.After(longTimeout) 640 641 assert.NoError(t, sendTimeToCut(producer, mockChannel, timeToCutBlockNumber, &timer), "Expected the sendTimeToCut call to return without errors") 642 assert.Nil(t, timer, "Expected the sendTimeToCut call to nil the timer") 643 }) 644 645 t.Run("WithError", func(t *testing.T) { 646 // Note that this test is affected by the following parameters: 647 // - Net.ReadTimeout 648 // - Consumer.Retry.Backoff 649 // - Metadata.Retry.Max 650 failureResponse := new(sarama.ProduceResponse) 651 failureResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), sarama.ErrNotEnoughReplicas) 652 mockBroker.Returns(failureResponse) 653 654 timer = time.After(longTimeout) 655 656 assert.Error(t, sendTimeToCut(producer, mockChannel, timeToCutBlockNumber, &timer), "Expected the sendTimeToCut call to return an error") 657 assert.Nil(t, timer, "Expected the sendTimeToCut call to nil the timer") 658 }) 659 } 660 661 func TestProcessMessagesToBlocks(t *testing.T) { 662 mockBroker := sarama.NewMockBroker(t, 0) 663 defer func() { mockBroker.Close() }() 664 665 mockChannel := newChannel("mockChannelFoo", defaultPartition) 666 667 metadataResponse := new(sarama.MetadataResponse) 668 metadataResponse.AddBroker(mockBroker.Addr(), mockBroker.BrokerID()) 669 metadataResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID(), nil, nil, sarama.ErrNoError) 670 mockBroker.Returns(metadataResponse) 671 672 producer, _ := sarama.NewSyncProducer([]string{mockBroker.Addr()}, mockBrokerConfig) 673 674 mockBrokerConfigCopy := *mockBrokerConfig 675 mockBrokerConfigCopy.ChannelBufferSize = 0 676 677 newestOffset := int64(0) 678 679 mockParentConsumer := mocks.NewConsumer(t, &mockBrokerConfigCopy) 680 mpc := mockParentConsumer.ExpectConsumePartition(mockChannel.topic(), mockChannel.partition(), newestOffset) 681 mockChannelConsumer, err := mockParentConsumer.ConsumePartition(mockChannel.topic(), mockChannel.partition(), newestOffset) 682 assert.NoError(t, err, "Expected no error when setting up the mock partition consumer") 683 684 t.Run("ReceiveConnect", func(t *testing.T) { 685 errorChan := make(chan struct{}) 686 close(errorChan) 687 haltChan := make(chan struct{}) 688 689 mockSupport := &mockmultichain.ConsenterSupport{ 690 ChainIDVal: mockChannel.topic(), 691 } 692 693 bareMinimumChain := &chainImpl{ 694 parentConsumer: mockParentConsumer, 695 channelConsumer: mockChannelConsumer, 696 697 channel: mockChannel, 698 support: mockSupport, 699 700 errorChan: errorChan, 701 haltChan: haltChan, 702 } 703 704 var counts []uint64 705 done := make(chan struct{}) 706 707 go func() { 708 counts, err = bareMinimumChain.processMessagesToBlocks() 709 done <- struct{}{} 710 }() 711 712 // This is the wrappedMessage that the for-loop will process 713 mpc.YieldMessage(newMockConsumerMessage(newConnectMessage())) 714 715 logger.Debug("Closing haltChan to exit the infinite for-loop") 716 close(haltChan) // Identical to chain.Halt() 717 logger.Debug("haltChan closed") 718 <-done 719 720 assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors") 721 assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled") 722 assert.Equal(t, uint64(1), counts[indexProcessConnectPass], "Expected 1 CONNECT message processed") 723 }) 724 725 t.Run("ReceiveRegularWithError", func(t *testing.T) { 726 errorChan := make(chan struct{}) 727 close(errorChan) 728 haltChan := make(chan struct{}) 729 730 mockSupport := &mockmultichain.ConsenterSupport{ 731 ChainIDVal: mockChannel.topic(), 732 } 733 734 bareMinimumChain := &chainImpl{ 735 parentConsumer: mockParentConsumer, 736 channelConsumer: mockChannelConsumer, 737 738 channel: mockChannel, 739 support: mockSupport, 740 741 errorChan: errorChan, 742 haltChan: haltChan, 743 } 744 745 var counts []uint64 746 done := make(chan struct{}) 747 748 go func() { 749 counts, err = bareMinimumChain.processMessagesToBlocks() 750 done <- struct{}{} 751 }() 752 753 // This is the wrappedMessage that the for-loop will process 754 mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(tamperBytes(utils.MarshalOrPanic(newMockEnvelope("fooMessage")))))) 755 756 logger.Debug("Closing haltChan to exit the infinite for-loop") 757 close(haltChan) // Identical to chain.Halt() 758 logger.Debug("haltChan closed") 759 <-done 760 761 assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors") 762 assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled") 763 assert.Equal(t, uint64(1), counts[indexProcessRegularError], "Expected 1 damaged REGULAR message processed") 764 }) 765 766 t.Run("ReceiveRegularAndQueue", func(t *testing.T) { 767 errorChan := make(chan struct{}) 768 close(errorChan) 769 haltChan := make(chan struct{}) 770 771 lastCutBlockNumber := uint64(3) 772 773 mockSupport := &mockmultichain.ConsenterSupport{ 774 Blocks: make(chan *cb.Block), // WriteBlock will post here 775 BlockCutterVal: mockblockcutter.NewReceiver(), 776 ChainIDVal: mockChannel.topic(), 777 HeightVal: lastCutBlockNumber, // Incremented during the WriteBlock call 778 SharedConfigVal: &mockconfig.Orderer{ 779 BatchTimeoutVal: longTimeout, 780 }, 781 } 782 defer close(mockSupport.BlockCutterVal.Block) 783 784 bareMinimumChain := &chainImpl{ 785 parentConsumer: mockParentConsumer, 786 channelConsumer: mockChannelConsumer, 787 788 channel: mockChannel, 789 support: mockSupport, 790 lastCutBlockNumber: lastCutBlockNumber, 791 792 errorChan: errorChan, 793 haltChan: haltChan, 794 } 795 796 var counts []uint64 797 done := make(chan struct{}) 798 799 go func() { 800 counts, err = bareMinimumChain.processMessagesToBlocks() 801 done <- struct{}{} 802 }() 803 804 // This is the wrappedMessage that the for-loop will process 805 mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage"))))) 806 807 mockSupport.BlockCutterVal.Block <- struct{}{} // Let the `mockblockcutter.Ordered` call return 808 logger.Debugf("Mock blockcutter's Ordered call has returned") 809 810 logger.Debug("Closing haltChan to exit the infinite for-loop") 811 // We are guaranteed to hit the haltChan branch after hitting the REGULAR branch at least once 812 close(haltChan) // Identical to chain.Halt() 813 logger.Debug("haltChan closed") 814 <-done 815 816 assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors") 817 assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled") 818 assert.Equal(t, uint64(1), counts[indexProcessRegularPass], "Expected 1 REGULAR message processed") 819 }) 820 821 t.Run("ReceiveRegularAndCutBlock", func(t *testing.T) { 822 errorChan := make(chan struct{}) 823 close(errorChan) 824 haltChan := make(chan struct{}) 825 826 lastCutBlockNumber := uint64(3) 827 828 mockSupport := &mockmultichain.ConsenterSupport{ 829 Blocks: make(chan *cb.Block), // WriteBlock will post here 830 BlockCutterVal: mockblockcutter.NewReceiver(), 831 ChainIDVal: mockChannel.topic(), 832 HeightVal: lastCutBlockNumber, // Incremented during the WriteBlock call 833 SharedConfigVal: &mockconfig.Orderer{ 834 BatchTimeoutVal: longTimeout, 835 }, 836 } 837 defer close(mockSupport.BlockCutterVal.Block) 838 839 bareMinimumChain := &chainImpl{ 840 parentConsumer: mockParentConsumer, 841 channelConsumer: mockChannelConsumer, 842 843 channel: mockChannel, 844 support: mockSupport, 845 lastCutBlockNumber: lastCutBlockNumber, 846 847 errorChan: errorChan, 848 haltChan: haltChan, 849 } 850 851 var counts []uint64 852 done := make(chan struct{}) 853 854 go func() { 855 counts, err = bareMinimumChain.processMessagesToBlocks() 856 done <- struct{}{} 857 }() 858 859 mockSupport.BlockCutterVal.CutNext = true 860 861 // This is the wrappedMessage that the for-loop will process 862 mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage"))))) 863 864 mockSupport.BlockCutterVal.Block <- struct{}{} // Let the `mockblockcutter.Ordered` call return 865 logger.Debugf("Mock blockcutter's Ordered call has returned") 866 <-mockSupport.Blocks // Let the `mockConsenterSupport.WriteBlock` proceed 867 868 logger.Debug("Closing haltChan to exit the infinite for-loop") 869 close(haltChan) // Identical to chain.Halt() 870 logger.Debug("haltChan closed") 871 <-done 872 873 assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors") 874 assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled") 875 assert.Equal(t, uint64(1), counts[indexProcessRegularPass], "Expected 1 REGULAR message processed") 876 assert.Equal(t, lastCutBlockNumber+1, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to be bumped up by one") 877 }) 878 879 t.Run("ReceiveTwoRegularAndCutTwoBlocks", func(t *testing.T) { 880 if testing.Short() { 881 t.Skip("Skipping test in short mode") 882 } 883 884 errorChan := make(chan struct{}) 885 close(errorChan) 886 haltChan := make(chan struct{}) 887 888 lastCutBlockNumber := uint64(3) 889 890 mockSupport := &mockmultichain.ConsenterSupport{ 891 Blocks: make(chan *cb.Block), // WriteBlock will post here 892 BlockCutterVal: mockblockcutter.NewReceiver(), 893 ChainIDVal: mockChannel.topic(), 894 HeightVal: lastCutBlockNumber, // Incremented during the WriteBlock call 895 SharedConfigVal: &mockconfig.Orderer{ 896 BatchTimeoutVal: longTimeout, 897 }, 898 } 899 defer close(mockSupport.BlockCutterVal.Block) 900 901 bareMinimumChain := &chainImpl{ 902 parentConsumer: mockParentConsumer, 903 channelConsumer: mockChannelConsumer, 904 905 channel: mockChannel, 906 support: mockSupport, 907 lastCutBlockNumber: lastCutBlockNumber, 908 909 errorChan: errorChan, 910 haltChan: haltChan, 911 } 912 913 var counts []uint64 914 done := make(chan struct{}) 915 916 go func() { 917 counts, err = bareMinimumChain.processMessagesToBlocks() 918 done <- struct{}{} 919 }() 920 921 var block1, block2 *cb.Block 922 923 // This is the first wrappedMessage that the for-loop will process 924 block1Offset := mpc.HighWaterMarkOffset() 925 mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage"))))) 926 mockSupport.BlockCutterVal.Block <- struct{}{} // Let the `mockblockcutter.Ordered` call return 927 logger.Debugf("Mock blockcutter's Ordered call has returned") 928 929 mockSupport.BlockCutterVal.IsolatedTx = true 930 931 // This is the first wrappedMessage that the for-loop will process 932 block2Offset := mpc.HighWaterMarkOffset() 933 mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage"))))) 934 mockSupport.BlockCutterVal.Block <- struct{}{} 935 logger.Debugf("Mock blockcutter's Ordered call has returned for the second time") 936 937 select { 938 case block1 = <-mockSupport.Blocks: // Let the `mockConsenterSupport.WriteBlock` proceed 939 case <-time.After(shortTimeout): 940 logger.Fatalf("Did not receive a block from the blockcutter as expected") 941 } 942 943 select { 944 case block2 = <-mockSupport.Blocks: 945 case <-time.After(shortTimeout): 946 logger.Fatalf("Did not receive a block from the blockcutter as expected") 947 } 948 949 logger.Debug("Closing haltChan to exit the infinite for-loop") 950 close(haltChan) // Identical to chain.Halt() 951 logger.Debug("haltChan closed") 952 <-done 953 954 assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors") 955 assert.Equal(t, uint64(2), counts[indexRecvPass], "Expected 2 messages received and unmarshaled") 956 assert.Equal(t, uint64(2), counts[indexProcessRegularPass], "Expected 2 REGULAR messages processed") 957 assert.Equal(t, lastCutBlockNumber+2, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to be bumped up by two") 958 assert.Equal(t, block1Offset, extractEncodedOffset(block1.GetMetadata().Metadata[cb.BlockMetadataIndex_ORDERER]), "Expected encoded offset in first block to be %d", block1Offset) 959 assert.Equal(t, block2Offset, extractEncodedOffset(block2.GetMetadata().Metadata[cb.BlockMetadataIndex_ORDERER]), "Expected encoded offset in first block to be %d", block2Offset) 960 }) 961 962 t.Run("SecondTxOverflows", func(t *testing.T) { 963 if testing.Short() { 964 t.Skip("Skipping test in short mode") 965 } 966 967 errorChan := make(chan struct{}) 968 close(errorChan) 969 haltChan := make(chan struct{}) 970 971 lastCutBlockNumber := uint64(3) 972 973 mockSupport := &mockmultichain.ConsenterSupport{ 974 Blocks: make(chan *cb.Block), // WriteBlock will post here 975 BlockCutterVal: mockblockcutter.NewReceiver(), 976 ChainIDVal: mockChannel.topic(), 977 HeightVal: lastCutBlockNumber, // Incremented during the WriteBlock call 978 SharedConfigVal: &mockconfig.Orderer{ 979 BatchTimeoutVal: longTimeout, 980 }, 981 } 982 defer close(mockSupport.BlockCutterVal.Block) 983 984 bareMinimumChain := &chainImpl{ 985 parentConsumer: mockParentConsumer, 986 channelConsumer: mockChannelConsumer, 987 988 channel: mockChannel, 989 support: mockSupport, 990 lastCutBlockNumber: lastCutBlockNumber, 991 992 errorChan: errorChan, 993 haltChan: haltChan, 994 } 995 996 var counts []uint64 997 done := make(chan struct{}) 998 999 go func() { 1000 counts, err = bareMinimumChain.processMessagesToBlocks() 1001 done <- struct{}{} 1002 }() 1003 1004 var block1, block2 *cb.Block 1005 1006 block1LastOffset := mpc.HighWaterMarkOffset() 1007 mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage"))))) 1008 mockSupport.BlockCutterVal.Block <- struct{}{} // Let the `mockblockcutter.Ordered` call return 1009 1010 // Set CutAncestors to true so that second message overflows receiver batch 1011 mockSupport.BlockCutterVal.CutAncestors = true 1012 mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage"))))) 1013 mockSupport.BlockCutterVal.Block <- struct{}{} 1014 1015 select { 1016 case block1 = <-mockSupport.Blocks: // Let the `mockConsenterSupport.WriteBlock` proceed 1017 case <-time.After(shortTimeout): 1018 logger.Fatalf("Did not receive a block from the blockcutter as expected") 1019 } 1020 1021 // Set CutNext to true to flush all pending messages 1022 mockSupport.BlockCutterVal.CutAncestors = false 1023 mockSupport.BlockCutterVal.CutNext = true 1024 block2LastOffset := mpc.HighWaterMarkOffset() 1025 mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage"))))) 1026 mockSupport.BlockCutterVal.Block <- struct{}{} 1027 1028 select { 1029 case block2 = <-mockSupport.Blocks: 1030 case <-time.After(shortTimeout): 1031 logger.Fatalf("Did not receive a block from the blockcutter as expected") 1032 } 1033 1034 logger.Debug("Closing haltChan to exit the infinite for-loop") 1035 close(haltChan) // Identical to chain.Halt() 1036 logger.Debug("haltChan closed") 1037 <-done 1038 1039 assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors") 1040 assert.Equal(t, uint64(3), counts[indexRecvPass], "Expected 2 messages received and unmarshaled") 1041 assert.Equal(t, uint64(3), counts[indexProcessRegularPass], "Expected 2 REGULAR messages processed") 1042 assert.Equal(t, lastCutBlockNumber+2, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to be bumped up by two") 1043 assert.Equal(t, block1LastOffset, extractEncodedOffset(block1.GetMetadata().Metadata[cb.BlockMetadataIndex_ORDERER]), "Expected encoded offset in first block to be %d", block1LastOffset) 1044 assert.Equal(t, block2LastOffset, extractEncodedOffset(block2.GetMetadata().Metadata[cb.BlockMetadataIndex_ORDERER]), "Expected encoded offset in second block to be %d", block2LastOffset) 1045 }) 1046 1047 t.Run("ReceiveRegularAndSendTimeToCut", func(t *testing.T) { 1048 t.Skip("Skipping test as it introduces a race condition") 1049 1050 // NB We haven't set a handlermap for the mock broker so we need to set 1051 // the ProduceResponse 1052 successResponse := new(sarama.ProduceResponse) 1053 successResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError) 1054 mockBroker.Returns(successResponse) 1055 1056 errorChan := make(chan struct{}) 1057 close(errorChan) 1058 haltChan := make(chan struct{}) 1059 1060 lastCutBlockNumber := uint64(3) 1061 1062 mockSupport := &mockmultichain.ConsenterSupport{ 1063 Blocks: make(chan *cb.Block), // WriteBlock will post here 1064 BlockCutterVal: mockblockcutter.NewReceiver(), 1065 ChainIDVal: mockChannel.topic(), 1066 HeightVal: lastCutBlockNumber, // Incremented during the WriteBlock call 1067 SharedConfigVal: &mockconfig.Orderer{ 1068 BatchTimeoutVal: extraShortTimeout, // ATTN 1069 }, 1070 } 1071 defer close(mockSupport.BlockCutterVal.Block) 1072 1073 bareMinimumChain := &chainImpl{ 1074 producer: producer, 1075 parentConsumer: mockParentConsumer, 1076 channelConsumer: mockChannelConsumer, 1077 1078 channel: mockChannel, 1079 support: mockSupport, 1080 lastCutBlockNumber: lastCutBlockNumber, 1081 1082 errorChan: errorChan, 1083 haltChan: haltChan, 1084 } 1085 1086 var counts []uint64 1087 done := make(chan struct{}) 1088 1089 go func() { 1090 counts, err = bareMinimumChain.processMessagesToBlocks() 1091 done <- struct{}{} 1092 }() 1093 1094 // This is the wrappedMessage that the for-loop will process 1095 mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage"))))) 1096 1097 mockSupport.BlockCutterVal.Block <- struct{}{} // Let the `mockblockcutter.Ordered` call return 1098 logger.Debugf("Mock blockcutter's Ordered call has returned") 1099 1100 // Sleep so that the timer branch is activated before the exitChan one. 1101 // TODO This is a race condition, will fix in follow-up changeset 1102 time.Sleep(hitBranch) 1103 1104 logger.Debug("Closing haltChan to exit the infinite for-loop") 1105 close(haltChan) // Identical to chain.Halt() 1106 logger.Debug("haltChan closed") 1107 <-done 1108 1109 assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors") 1110 assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled") 1111 assert.Equal(t, uint64(1), counts[indexProcessRegularPass], "Expected 1 REGULAR message processed") 1112 assert.Equal(t, uint64(1), counts[indexSendTimeToCutPass], "Expected 1 TIMER event processed") 1113 assert.Equal(t, lastCutBlockNumber, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to stay the same") 1114 }) 1115 1116 t.Run("ReceiveRegularAndSendTimeToCutError", func(t *testing.T) { 1117 // Note that this test is affected by the following parameters: 1118 // - Net.ReadTimeout 1119 // - Consumer.Retry.Backoff 1120 // - Metadata.Retry.Max 1121 1122 t.Skip("Skipping test as it introduces a race condition") 1123 1124 // Exact same test as ReceiveRegularAndSendTimeToCut. 1125 // Only difference is that the producer's attempt to send a TTC will 1126 // fail with an ErrNotEnoughReplicas error. 1127 failureResponse := new(sarama.ProduceResponse) 1128 failureResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), sarama.ErrNotEnoughReplicas) 1129 mockBroker.Returns(failureResponse) 1130 1131 errorChan := make(chan struct{}) 1132 close(errorChan) 1133 haltChan := make(chan struct{}) 1134 1135 lastCutBlockNumber := uint64(3) 1136 1137 mockSupport := &mockmultichain.ConsenterSupport{ 1138 Blocks: make(chan *cb.Block), // WriteBlock will post here 1139 BlockCutterVal: mockblockcutter.NewReceiver(), 1140 ChainIDVal: mockChannel.topic(), 1141 HeightVal: lastCutBlockNumber, // Incremented during the WriteBlock call 1142 SharedConfigVal: &mockconfig.Orderer{ 1143 BatchTimeoutVal: extraShortTimeout, // ATTN 1144 }, 1145 } 1146 defer close(mockSupport.BlockCutterVal.Block) 1147 1148 bareMinimumChain := &chainImpl{ 1149 producer: producer, 1150 parentConsumer: mockParentConsumer, 1151 channelConsumer: mockChannelConsumer, 1152 1153 channel: mockChannel, 1154 support: mockSupport, 1155 lastCutBlockNumber: lastCutBlockNumber, 1156 1157 errorChan: errorChan, 1158 haltChan: haltChan, 1159 } 1160 1161 var counts []uint64 1162 done := make(chan struct{}) 1163 1164 go func() { 1165 counts, err = bareMinimumChain.processMessagesToBlocks() 1166 done <- struct{}{} 1167 }() 1168 1169 // This is the wrappedMessage that the for-loop will process 1170 mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage"))))) 1171 1172 mockSupport.BlockCutterVal.Block <- struct{}{} // Let the `mockblockcutter.Ordered` call return 1173 logger.Debugf("Mock blockcutter's Ordered call has returned") 1174 1175 // Sleep so that the timer branch is activated before the exitChan one. 1176 // TODO This is a race condition, will fix in follow-up changeset 1177 time.Sleep(hitBranch) 1178 1179 logger.Debug("Closing haltChan to exit the infinite for-loop") 1180 close(haltChan) // Identical to chain.Halt() 1181 logger.Debug("haltChan closed") 1182 <-done 1183 1184 assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors") 1185 assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled") 1186 assert.Equal(t, uint64(1), counts[indexProcessRegularPass], "Expected 1 REGULAR message processed") 1187 assert.Equal(t, uint64(1), counts[indexSendTimeToCutError], "Expected 1 faulty TIMER event processed") 1188 assert.Equal(t, lastCutBlockNumber, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to stay the same") 1189 }) 1190 1191 t.Run("ReceiveTimeToCutProper", func(t *testing.T) { 1192 errorChan := make(chan struct{}) 1193 close(errorChan) 1194 haltChan := make(chan struct{}) 1195 1196 lastCutBlockNumber := uint64(3) 1197 1198 mockSupport := &mockmultichain.ConsenterSupport{ 1199 Blocks: make(chan *cb.Block), // WriteBlock will post here 1200 BlockCutterVal: mockblockcutter.NewReceiver(), 1201 ChainIDVal: mockChannel.topic(), 1202 HeightVal: lastCutBlockNumber, // Incremented during the WriteBlock call 1203 } 1204 defer close(mockSupport.BlockCutterVal.Block) 1205 1206 bareMinimumChain := &chainImpl{ 1207 parentConsumer: mockParentConsumer, 1208 channelConsumer: mockChannelConsumer, 1209 1210 channel: mockChannel, 1211 support: mockSupport, 1212 lastCutBlockNumber: lastCutBlockNumber, 1213 1214 errorChan: errorChan, 1215 haltChan: haltChan, 1216 } 1217 1218 // We need the mock blockcutter to deliver a non-empty batch 1219 go func() { 1220 mockSupport.BlockCutterVal.Block <- struct{}{} // Let the `mockblockcutter.Ordered` call below return 1221 logger.Debugf("Mock blockcutter's Ordered call has returned") 1222 }() 1223 // We are "planting" a message directly to the mock blockcutter 1224 mockSupport.BlockCutterVal.Ordered(newMockEnvelope("fooMessage")) 1225 1226 var counts []uint64 1227 done := make(chan struct{}) 1228 1229 go func() { 1230 counts, err = bareMinimumChain.processMessagesToBlocks() 1231 done <- struct{}{} 1232 }() 1233 1234 // This is the wrappedMessage that the for-loop will process 1235 mpc.YieldMessage(newMockConsumerMessage(newTimeToCutMessage(lastCutBlockNumber + 1))) 1236 1237 <-mockSupport.Blocks // Let the `mockConsenterSupport.WriteBlock` proceed 1238 1239 logger.Debug("Closing haltChan to exit the infinite for-loop") 1240 close(haltChan) // Identical to chain.Halt() 1241 logger.Debug("haltChan closed") 1242 <-done 1243 1244 assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors") 1245 assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled") 1246 assert.Equal(t, uint64(1), counts[indexProcessTimeToCutPass], "Expected 1 TIMETOCUT message processed") 1247 assert.Equal(t, lastCutBlockNumber+1, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to be bumped up by one") 1248 }) 1249 1250 t.Run("ReceiveTimeToCutZeroBatch", func(t *testing.T) { 1251 errorChan := make(chan struct{}) 1252 close(errorChan) 1253 haltChan := make(chan struct{}) 1254 1255 lastCutBlockNumber := uint64(3) 1256 1257 mockSupport := &mockmultichain.ConsenterSupport{ 1258 Blocks: make(chan *cb.Block), // WriteBlock will post here 1259 BlockCutterVal: mockblockcutter.NewReceiver(), 1260 ChainIDVal: mockChannel.topic(), 1261 HeightVal: lastCutBlockNumber, // Incremented during the WriteBlock call 1262 } 1263 defer close(mockSupport.BlockCutterVal.Block) 1264 1265 bareMinimumChain := &chainImpl{ 1266 parentConsumer: mockParentConsumer, 1267 channelConsumer: mockChannelConsumer, 1268 1269 channel: mockChannel, 1270 support: mockSupport, 1271 lastCutBlockNumber: lastCutBlockNumber, 1272 1273 errorChan: errorChan, 1274 haltChan: haltChan, 1275 } 1276 1277 var counts []uint64 1278 done := make(chan struct{}) 1279 1280 go func() { 1281 counts, err = bareMinimumChain.processMessagesToBlocks() 1282 done <- struct{}{} 1283 }() 1284 1285 // This is the wrappedMessage that the for-loop will process 1286 mpc.YieldMessage(newMockConsumerMessage(newTimeToCutMessage(lastCutBlockNumber + 1))) 1287 1288 logger.Debug("Closing haltChan to exit the infinite for-loop") 1289 close(haltChan) // Identical to chain.Halt() 1290 logger.Debug("haltChan closed") 1291 <-done 1292 1293 assert.Error(t, err, "Expected the processMessagesToBlocks call to return an error") 1294 assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled") 1295 assert.Equal(t, uint64(1), counts[indexProcessTimeToCutError], "Expected 1 faulty TIMETOCUT message processed") 1296 assert.Equal(t, lastCutBlockNumber, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to stay the same") 1297 }) 1298 1299 t.Run("ReceiveTimeToCutLargerThanExpected", func(t *testing.T) { 1300 errorChan := make(chan struct{}) 1301 close(errorChan) 1302 haltChan := make(chan struct{}) 1303 1304 lastCutBlockNumber := uint64(3) 1305 1306 mockSupport := &mockmultichain.ConsenterSupport{ 1307 Blocks: make(chan *cb.Block), // WriteBlock will post here 1308 BlockCutterVal: mockblockcutter.NewReceiver(), 1309 ChainIDVal: mockChannel.topic(), 1310 HeightVal: lastCutBlockNumber, // Incremented during the WriteBlock call 1311 } 1312 defer close(mockSupport.BlockCutterVal.Block) 1313 1314 bareMinimumChain := &chainImpl{ 1315 parentConsumer: mockParentConsumer, 1316 channelConsumer: mockChannelConsumer, 1317 1318 channel: mockChannel, 1319 support: mockSupport, 1320 lastCutBlockNumber: lastCutBlockNumber, 1321 1322 errorChan: errorChan, 1323 haltChan: haltChan, 1324 } 1325 1326 var counts []uint64 1327 done := make(chan struct{}) 1328 1329 go func() { 1330 counts, err = bareMinimumChain.processMessagesToBlocks() 1331 done <- struct{}{} 1332 }() 1333 1334 // This is the wrappedMessage that the for-loop will process 1335 mpc.YieldMessage(newMockConsumerMessage(newTimeToCutMessage(lastCutBlockNumber + 2))) 1336 1337 logger.Debug("Closing haltChan to exit the infinite for-loop") 1338 close(haltChan) // Identical to chain.Halt() 1339 logger.Debug("haltChan closed") 1340 <-done 1341 1342 assert.Error(t, err, "Expected the processMessagesToBlocks call to return an error") 1343 assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled") 1344 assert.Equal(t, uint64(1), counts[indexProcessTimeToCutError], "Expected 1 faulty TIMETOCUT message processed") 1345 assert.Equal(t, lastCutBlockNumber, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to stay the same") 1346 }) 1347 1348 t.Run("ReceiveTimeToCutStale", func(t *testing.T) { 1349 errorChan := make(chan struct{}) 1350 close(errorChan) 1351 haltChan := make(chan struct{}) 1352 1353 lastCutBlockNumber := uint64(3) 1354 1355 mockSupport := &mockmultichain.ConsenterSupport{ 1356 Blocks: make(chan *cb.Block), // WriteBlock will post here 1357 BlockCutterVal: mockblockcutter.NewReceiver(), 1358 ChainIDVal: mockChannel.topic(), 1359 HeightVal: lastCutBlockNumber, // Incremented during the WriteBlock call 1360 } 1361 defer close(mockSupport.BlockCutterVal.Block) 1362 1363 bareMinimumChain := &chainImpl{ 1364 parentConsumer: mockParentConsumer, 1365 channelConsumer: mockChannelConsumer, 1366 1367 channel: mockChannel, 1368 support: mockSupport, 1369 lastCutBlockNumber: lastCutBlockNumber, 1370 1371 errorChan: errorChan, 1372 haltChan: haltChan, 1373 } 1374 1375 var counts []uint64 1376 done := make(chan struct{}) 1377 1378 go func() { 1379 counts, err = bareMinimumChain.processMessagesToBlocks() 1380 done <- struct{}{} 1381 }() 1382 1383 // This is the wrappedMessage that the for-loop will process 1384 mpc.YieldMessage(newMockConsumerMessage(newTimeToCutMessage(lastCutBlockNumber))) 1385 1386 logger.Debug("Closing haltChan to exit the infinite for-loop") 1387 close(haltChan) // Identical to chain.Halt() 1388 logger.Debug("haltChan closed") 1389 <-done 1390 1391 assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors") 1392 assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled") 1393 assert.Equal(t, uint64(1), counts[indexProcessTimeToCutPass], "Expected 1 TIMETOCUT message processed") 1394 assert.Equal(t, lastCutBlockNumber, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to stay the same") 1395 }) 1396 1397 t.Run("ReceiveKafkaErrorAndCloseErrorChan", func(t *testing.T) { 1398 // If we set up the mock broker so that it returns a response, if the 1399 // test finishes before the sendConnectMessage goroutine has received 1400 // this response, we will get a failure ("not all expectations were 1401 // satisfied") from the mock broker. So we sabotage the producer. 1402 failedProducer, _ := sarama.NewSyncProducer([]string{}, mockBrokerConfig) 1403 1404 // We need to have the sendConnectMessage goroutine die instantaneously, 1405 // otherwise we'll get a nil pointer dereference panic. We are 1406 // exploiting the admittedly hacky shortcut where a retriable process 1407 // returns immediately when given the nil time.Duration value for its 1408 // ticker. 1409 zeroRetryConsenter := &consenterImpl{} 1410 1411 // Let's assume an open errorChan, i.e. a healthy link between the 1412 // consumer and the Kafka partition corresponding to the channel 1413 errorChan := make(chan struct{}) 1414 1415 haltChan := make(chan struct{}) 1416 1417 mockSupport := &mockmultichain.ConsenterSupport{ 1418 ChainIDVal: mockChannel.topic(), 1419 } 1420 1421 bareMinimumChain := &chainImpl{ 1422 consenter: zeroRetryConsenter, // For sendConnectMessage 1423 producer: failedProducer, // For sendConnectMessage 1424 parentConsumer: mockParentConsumer, 1425 channelConsumer: mockChannelConsumer, 1426 1427 channel: mockChannel, 1428 support: mockSupport, 1429 1430 errorChan: errorChan, 1431 haltChan: haltChan, 1432 } 1433 1434 var counts []uint64 1435 done := make(chan struct{}) 1436 1437 go func() { 1438 counts, err = bareMinimumChain.processMessagesToBlocks() 1439 done <- struct{}{} 1440 }() 1441 1442 // This is what the for-loop will process 1443 mpc.YieldError(fmt.Errorf("fooError")) 1444 1445 logger.Debug("Closing haltChan to exit the infinite for-loop") 1446 close(haltChan) // Identical to chain.Halt() 1447 logger.Debug("haltChan closed") 1448 <-done 1449 1450 assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors") 1451 assert.Equal(t, uint64(1), counts[indexRecvError], "Expected 1 Kafka error received") 1452 1453 select { 1454 case <-bareMinimumChain.errorChan: 1455 logger.Debug("errorChan is closed as it should be") 1456 default: 1457 t.Fatal("errorChan should have been closed") 1458 } 1459 }) 1460 1461 t.Run("ReceiveKafkaErrorAndThenReceiveRegularMessage", func(t *testing.T) { 1462 t.Skip("Skipping test as it introduces a race condition") 1463 1464 // If we set up the mock broker so that it returns a response, if the 1465 // test finishes before the sendConnectMessage goroutine has received 1466 // this response, we will get a failure ("not all expectations were 1467 // satisfied") from the mock broker. So we sabotage the producer. 1468 failedProducer, _ := sarama.NewSyncProducer([]string{}, mockBrokerConfig) 1469 1470 // We need to have the sendConnectMessage goroutine die instantaneously, 1471 // otherwise we'll get a nil pointer dereference panic. We are 1472 // exploiting the admittedly hacky shortcut where a retriable process 1473 // returns immediately when given the nil time.Duration value for its 1474 // ticker. 1475 zeroRetryConsenter := &consenterImpl{} 1476 1477 // If the errorChan is closed already, the kafkaErr branch shouldn't 1478 // touch it 1479 errorChan := make(chan struct{}) 1480 close(errorChan) 1481 1482 haltChan := make(chan struct{}) 1483 1484 mockSupport := &mockmultichain.ConsenterSupport{ 1485 ChainIDVal: mockChannel.topic(), 1486 } 1487 1488 bareMinimumChain := &chainImpl{ 1489 consenter: zeroRetryConsenter, // For sendConnectMessage 1490 producer: failedProducer, // For sendConnectMessage 1491 parentConsumer: mockParentConsumer, 1492 channelConsumer: mockChannelConsumer, 1493 1494 channel: mockChannel, 1495 support: mockSupport, 1496 1497 errorChan: errorChan, 1498 haltChan: haltChan, 1499 } 1500 1501 var counts []uint64 1502 done := make(chan struct{}) 1503 1504 go func() { 1505 counts, err = bareMinimumChain.processMessagesToBlocks() 1506 done <- struct{}{} 1507 }() 1508 1509 // This is what the for-loop will process 1510 mpc.YieldError(fmt.Errorf("foo")) 1511 1512 // We tested this in ReceiveKafkaErrorAndCloseErrorChan, so this check 1513 // is redundant in that regard. We use it however to ensure the 1514 // kafkaErrBranch has been activated before proceeding with pushing the 1515 // regular message. 1516 select { 1517 case <-bareMinimumChain.errorChan: 1518 logger.Debug("errorChan is closed as it should be") 1519 case <-time.After(shortTimeout): 1520 t.Fatal("errorChan should have been closed by now") 1521 } 1522 1523 // This is the wrappedMessage that the for-loop will process. We use 1524 // a broken regular message here on purpose since this is the shortest 1525 // path and it allows us to test what we want. 1526 mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(tamperBytes(utils.MarshalOrPanic(newMockEnvelope("fooMessage")))))) 1527 1528 // Sleep so that the Messages/errorChan branch is activated. 1529 // TODO Hacky approach, will need to revise eventually 1530 time.Sleep(hitBranch) 1531 1532 // Check that the errorChan was recreated 1533 select { 1534 case <-bareMinimumChain.errorChan: 1535 t.Fatal("errorChan should have been open") 1536 default: 1537 logger.Debug("errorChan is open as it should be") 1538 } 1539 1540 logger.Debug("Closing haltChan to exit the infinite for-loop") 1541 close(haltChan) // Identical to chain.Halt() 1542 logger.Debug("haltChan closed") 1543 <-done 1544 }) 1545 }