github.com/darrenli6/fabric-sdk-example@v0.0.0-20220109053535-94b13b56df8c/orderer/kafka/chain_test.go (about)

     1  /*
     2  Copyright IBM Corp. All Rights Reserved.
     3  
     4  SPDX-License-Identifier: Apache-2.0
     5  */
     6  
     7  package kafka
     8  
     9  import (
    10  	"fmt"
    11  	"testing"
    12  	"time"
    13  
    14  	"github.com/Shopify/sarama"
    15  	"github.com/Shopify/sarama/mocks"
    16  	mockconfig "github.com/hyperledger/fabric/common/mocks/config"
    17  	mockblockcutter "github.com/hyperledger/fabric/orderer/mocks/blockcutter"
    18  	mockmultichain "github.com/hyperledger/fabric/orderer/mocks/multichain"
    19  	cb "github.com/hyperledger/fabric/protos/common"
    20  	ab "github.com/hyperledger/fabric/protos/orderer"
    21  	"github.com/hyperledger/fabric/protos/utils"
    22  	"github.com/stretchr/testify/assert"
    23  )
    24  
    25  var (
    26  	extraShortTimeout = 1 * time.Millisecond
    27  	shortTimeout      = 1 * time.Second
    28  	longTimeout       = 1 * time.Hour
    29  
    30  	hitBranch = 50 * time.Millisecond
    31  )
    32  
    33  func TestChain(t *testing.T) {
    34  
    35  	oldestOffset := int64(0)
    36  	newestOffset := int64(5)
    37  
    38  	message := sarama.StringEncoder("messageFoo")
    39  
    40  	newMocks := func(t *testing.T) (mockChannel channel, mockBroker *sarama.MockBroker, mockSupport *mockmultichain.ConsenterSupport) {
    41  		mockChannel = newChannel(channelNameForTest(t), defaultPartition)
    42  		mockBroker = sarama.NewMockBroker(t, 0)
    43  		mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{
    44  			"MetadataRequest": sarama.NewMockMetadataResponse(t).
    45  				SetBroker(mockBroker.Addr(), mockBroker.BrokerID()).
    46  				SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()),
    47  			"ProduceRequest": sarama.NewMockProduceResponse(t).
    48  				SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError),
    49  			"OffsetRequest": sarama.NewMockOffsetResponse(t).
    50  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset).
    51  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset),
    52  			"FetchRequest": sarama.NewMockFetchResponse(t, 1).
    53  				SetMessage(mockChannel.topic(), mockChannel.partition(), newestOffset, message),
    54  		})
    55  		mockSupport = &mockmultichain.ConsenterSupport{
    56  			ChainIDVal:      mockChannel.topic(),
    57  			HeightVal:       uint64(3),
    58  			SharedConfigVal: &mockconfig.Orderer{KafkaBrokersVal: []string{mockBroker.Addr()}},
    59  		}
    60  		return
    61  	}
    62  
    63  	t.Run("New", func(t *testing.T) {
    64  		_, mockBroker, mockSupport := newMocks(t)
    65  		defer func() { mockBroker.Close() }()
    66  		chain, err := newChain(mockConsenter, mockSupport, newestOffset-1)
    67  
    68  		assert.NoError(t, err, "Expected newChain to return without errors")
    69  		select {
    70  		case <-chain.errorChan:
    71  			logger.Debug("errorChan is closed as it should be")
    72  		default:
    73  			t.Fatal("errorChan should have been closed")
    74  		}
    75  
    76  		select {
    77  		case <-chain.haltChan:
    78  			t.Fatal("haltChan should have been open")
    79  		default:
    80  			logger.Debug("haltChan is open as it should be")
    81  		}
    82  
    83  		select {
    84  		case <-chain.startChan:
    85  			t.Fatal("startChan should have been open")
    86  		default:
    87  			logger.Debug("startChan is open as it should be")
    88  		}
    89  	})
    90  
    91  	t.Run("Start", func(t *testing.T) {
    92  		_, mockBroker, mockSupport := newMocks(t)
    93  		defer func() { mockBroker.Close() }()
    94  		// Set to -1 because we haven't sent the CONNECT message yet
    95  		chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1)
    96  
    97  		chain.Start()
    98  		select {
    99  		case <-chain.startChan:
   100  			logger.Debug("startChan is closed as it should be")
   101  		case <-time.After(shortTimeout):
   102  			t.Fatal("startChan should have been closed by now")
   103  		}
   104  
   105  		// Trigger the haltChan clause in the processMessagesToBlocks goroutine
   106  		close(chain.haltChan)
   107  	})
   108  
   109  	t.Run("Halt", func(t *testing.T) {
   110  		_, mockBroker, mockSupport := newMocks(t)
   111  		defer func() { mockBroker.Close() }()
   112  		chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1)
   113  
   114  		chain.Start()
   115  		select {
   116  		case <-chain.startChan:
   117  			logger.Debug("startChan is closed as it should be")
   118  		case <-time.After(shortTimeout):
   119  			t.Fatal("startChan should have been closed by now")
   120  		}
   121  
   122  		// Wait till the start phase has completed, then:
   123  		chain.Halt()
   124  
   125  		select {
   126  		case <-chain.haltChan:
   127  			logger.Debug("haltChan is closed as it should be")
   128  		case <-time.After(shortTimeout):
   129  			t.Fatal("haltChan should have been closed")
   130  		}
   131  
   132  		select {
   133  		case <-chain.errorChan:
   134  			logger.Debug("errorChan is closed as it should be")
   135  		case <-time.After(shortTimeout):
   136  			t.Fatal("errorChan should have been closed")
   137  		}
   138  	})
   139  
   140  	t.Run("DoubleHalt", func(t *testing.T) {
   141  		_, mockBroker, mockSupport := newMocks(t)
   142  		defer func() { mockBroker.Close() }()
   143  		chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1)
   144  
   145  		chain.Start()
   146  		select {
   147  		case <-chain.startChan:
   148  			logger.Debug("startChan is closed as it should be")
   149  		case <-time.After(shortTimeout):
   150  			t.Fatal("startChan should have been closed by now")
   151  		}
   152  
   153  		chain.Halt()
   154  
   155  		assert.NotPanics(t, func() { chain.Halt() }, "Calling Halt() more than once shouldn't panic")
   156  	})
   157  
   158  	t.Run("StartWithProducerForChannelError", func(t *testing.T) {
   159  		_, mockBroker, mockSupport := newMocks(t)
   160  		defer func() { mockBroker.Close() }()
   161  		// Point to an empty brokers list
   162  		mockSupportCopy := *mockSupport
   163  		mockSupportCopy.SharedConfigVal = &mockconfig.Orderer{KafkaBrokersVal: []string{}}
   164  
   165  		chain, _ := newChain(mockConsenter, &mockSupportCopy, newestOffset-1)
   166  
   167  		// The production path will actually call chain.Start(). This is
   168  		// functionally equivalent and allows us to run assertions on it.
   169  		assert.Panics(t, func() { startThread(chain) }, "Expected the Start() call to panic")
   170  	})
   171  
   172  	t.Run("StartWithConnectMessageError", func(t *testing.T) {
   173  		// Note that this test is affected by the following parameters:
   174  		// - Net.ReadTimeout
   175  		// - Consumer.Retry.Backoff
   176  		// - Metadata.Retry.Max
   177  		mockChannel, mockBroker, mockSupport := newMocks(t)
   178  		defer func() { mockBroker.Close() }()
   179  		chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1)
   180  
   181  		// Have the broker return an ErrNotLeaderForPartition error
   182  		mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{
   183  			"MetadataRequest": sarama.NewMockMetadataResponse(t).
   184  				SetBroker(mockBroker.Addr(), mockBroker.BrokerID()).
   185  				SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()),
   186  			"ProduceRequest": sarama.NewMockProduceResponse(t).
   187  				SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNotLeaderForPartition),
   188  			"OffsetRequest": sarama.NewMockOffsetResponse(t).
   189  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset).
   190  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset),
   191  			"FetchRequest": sarama.NewMockFetchResponse(t, 1).
   192  				SetMessage(mockChannel.topic(), mockChannel.partition(), newestOffset, message),
   193  		})
   194  
   195  		assert.Panics(t, func() { startThread(chain) }, "Expected the Start() call to panic")
   196  	})
   197  
   198  	t.Run("EnqueueIfNotStarted", func(t *testing.T) {
   199  		mockChannel, mockBroker, mockSupport := newMocks(t)
   200  		defer func() { mockBroker.Close() }()
   201  		chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1)
   202  
   203  		// As in StartWithConnectMessageError, have the broker return an
   204  		// ErrNotLeaderForPartition error, i.e. cause an error in the
   205  		// 'post connect message' step.
   206  		mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{
   207  			"MetadataRequest": sarama.NewMockMetadataResponse(t).
   208  				SetBroker(mockBroker.Addr(), mockBroker.BrokerID()).
   209  				SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()),
   210  			"ProduceRequest": sarama.NewMockProduceResponse(t).
   211  				SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNotLeaderForPartition),
   212  			"OffsetRequest": sarama.NewMockOffsetResponse(t).
   213  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset).
   214  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset),
   215  			"FetchRequest": sarama.NewMockFetchResponse(t, 1).
   216  				SetMessage(mockChannel.topic(), mockChannel.partition(), newestOffset, message),
   217  		})
   218  
   219  		assert.False(t, chain.Enqueue(newMockEnvelope("fooMessage")), "Expected Enqueue call to return false")
   220  	})
   221  
   222  	t.Run("StartWithConsumerForChannelError", func(t *testing.T) {
   223  		// Note that this test is affected by the following parameters:
   224  		// - Net.ReadTimeout
   225  		// - Consumer.Retry.Backoff
   226  		// - Metadata.Retry.Max
   227  
   228  		mockChannel, mockBroker, mockSupport := newMocks(t)
   229  		defer func() { mockBroker.Close() }()
   230  
   231  		// Provide an out-of-range offset
   232  		chain, _ := newChain(mockConsenter, mockSupport, newestOffset)
   233  
   234  		mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{
   235  			"MetadataRequest": sarama.NewMockMetadataResponse(t).
   236  				SetBroker(mockBroker.Addr(), mockBroker.BrokerID()).
   237  				SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()),
   238  			"ProduceRequest": sarama.NewMockProduceResponse(t).
   239  				SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError),
   240  			"OffsetRequest": sarama.NewMockOffsetResponse(t).
   241  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset).
   242  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset),
   243  			"FetchRequest": sarama.NewMockFetchResponse(t, 1).
   244  				SetMessage(mockChannel.topic(), mockChannel.partition(), newestOffset, message),
   245  		})
   246  
   247  		assert.Panics(t, func() { startThread(chain) }, "Expected the Start() call to panic")
   248  	})
   249  
   250  	t.Run("EnqueueProper", func(t *testing.T) {
   251  		mockChannel, mockBroker, mockSupport := newMocks(t)
   252  		defer func() { mockBroker.Close() }()
   253  		chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1)
   254  
   255  		mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{
   256  			"MetadataRequest": sarama.NewMockMetadataResponse(t).
   257  				SetBroker(mockBroker.Addr(), mockBroker.BrokerID()).
   258  				SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()),
   259  			"ProduceRequest": sarama.NewMockProduceResponse(t).
   260  				SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError),
   261  			"OffsetRequest": sarama.NewMockOffsetResponse(t).
   262  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset).
   263  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset),
   264  			"FetchRequest": sarama.NewMockFetchResponse(t, 1).
   265  				SetMessage(mockChannel.topic(), mockChannel.partition(), newestOffset, message),
   266  		})
   267  
   268  		chain.Start()
   269  		select {
   270  		case <-chain.startChan:
   271  			logger.Debug("startChan is closed as it should be")
   272  		case <-time.After(shortTimeout):
   273  			t.Fatal("startChan should have been closed by now")
   274  		}
   275  
   276  		// Enqueue should have access to the post path, and its ProduceRequest
   277  		// should go by without error
   278  		assert.True(t, chain.Enqueue(newMockEnvelope("fooMessage")), "Expected Enqueue call to return true")
   279  
   280  		chain.Halt()
   281  	})
   282  
   283  	t.Run("EnqueueIfHalted", func(t *testing.T) {
   284  		mockChannel, mockBroker, mockSupport := newMocks(t)
   285  		defer func() { mockBroker.Close() }()
   286  		chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1)
   287  
   288  		mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{
   289  			"MetadataRequest": sarama.NewMockMetadataResponse(t).
   290  				SetBroker(mockBroker.Addr(), mockBroker.BrokerID()).
   291  				SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()),
   292  			"ProduceRequest": sarama.NewMockProduceResponse(t).
   293  				SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError),
   294  			"OffsetRequest": sarama.NewMockOffsetResponse(t).
   295  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset).
   296  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset),
   297  			"FetchRequest": sarama.NewMockFetchResponse(t, 1).
   298  				SetMessage(mockChannel.topic(), mockChannel.partition(), newestOffset, message),
   299  		})
   300  
   301  		chain.Start()
   302  		select {
   303  		case <-chain.startChan:
   304  			logger.Debug("startChan is closed as it should be")
   305  		case <-time.After(shortTimeout):
   306  			t.Fatal("startChan should have been closed by now")
   307  		}
   308  		chain.Halt()
   309  
   310  		// haltChan should close access to the post path
   311  		assert.False(t, chain.Enqueue(newMockEnvelope("fooMessage")), "Expected Enqueue call to return false")
   312  	})
   313  
   314  	t.Run("EnqueueError", func(t *testing.T) {
   315  		mockChannel, mockBroker, mockSupport := newMocks(t)
   316  		defer func() { mockBroker.Close() }()
   317  		chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1)
   318  
   319  		// Use the "good" handler map that allows the Stage to complete without
   320  		// issues
   321  		mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{
   322  			"MetadataRequest": sarama.NewMockMetadataResponse(t).
   323  				SetBroker(mockBroker.Addr(), mockBroker.BrokerID()).
   324  				SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()),
   325  			"ProduceRequest": sarama.NewMockProduceResponse(t).
   326  				SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError),
   327  			"OffsetRequest": sarama.NewMockOffsetResponse(t).
   328  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset).
   329  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset),
   330  			"FetchRequest": sarama.NewMockFetchResponse(t, 1).
   331  				SetMessage(mockChannel.topic(), mockChannel.partition(), newestOffset, message),
   332  		})
   333  
   334  		chain.Start()
   335  		select {
   336  		case <-chain.startChan:
   337  			logger.Debug("startChan is closed as it should be")
   338  		case <-time.After(shortTimeout):
   339  			t.Fatal("startChan should have been closed by now")
   340  		}
   341  
   342  		// Now make it so that the next ProduceRequest is met with an error
   343  		mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{
   344  			"ProduceRequest": sarama.NewMockProduceResponse(t).
   345  				SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNotLeaderForPartition),
   346  		})
   347  
   348  		assert.False(t, chain.Enqueue(newMockEnvelope("fooMessage")), "Expected Enqueue call to return false")
   349  	})
   350  }
   351  
   352  func TestSetupProducerForChannel(t *testing.T) {
   353  	if testing.Short() {
   354  		t.Skip("Skipping test in short mode")
   355  	}
   356  
   357  	mockBroker := sarama.NewMockBroker(t, 0)
   358  	defer mockBroker.Close()
   359  
   360  	mockChannel := newChannel(channelNameForTest(t), defaultPartition)
   361  
   362  	haltChan := make(chan struct{})
   363  
   364  	t.Run("Proper", func(t *testing.T) {
   365  		metadataResponse := new(sarama.MetadataResponse)
   366  		metadataResponse.AddBroker(mockBroker.Addr(), mockBroker.BrokerID())
   367  		metadataResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID(), nil, nil, sarama.ErrNoError)
   368  		mockBroker.Returns(metadataResponse)
   369  
   370  		producer, err := setupProducerForChannel(mockConsenter.retryOptions(), haltChan, []string{mockBroker.Addr()}, mockBrokerConfig, mockChannel)
   371  		assert.NoError(t, err, "Expected the setupProducerForChannel call to return without errors")
   372  		assert.NoError(t, producer.Close(), "Expected to close the producer without errors")
   373  	})
   374  
   375  	t.Run("WithError", func(t *testing.T) {
   376  		_, err := setupProducerForChannel(mockConsenter.retryOptions(), haltChan, []string{}, mockBrokerConfig, mockChannel)
   377  		assert.Error(t, err, "Expected the setupProducerForChannel call to return an error")
   378  	})
   379  }
   380  
   381  func TestSetupConsumerForChannel(t *testing.T) {
   382  	mockBroker := sarama.NewMockBroker(t, 0)
   383  	defer func() { mockBroker.Close() }()
   384  
   385  	mockChannel := newChannel(channelNameForTest(t), defaultPartition)
   386  
   387  	oldestOffset := int64(0)
   388  	newestOffset := int64(5)
   389  
   390  	startFrom := int64(3)
   391  	message := sarama.StringEncoder("messageFoo")
   392  
   393  	mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{
   394  		"MetadataRequest": sarama.NewMockMetadataResponse(t).
   395  			SetBroker(mockBroker.Addr(), mockBroker.BrokerID()).
   396  			SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()),
   397  		"OffsetRequest": sarama.NewMockOffsetResponse(t).
   398  			SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset).
   399  			SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset),
   400  		"FetchRequest": sarama.NewMockFetchResponse(t, 1).
   401  			SetMessage(mockChannel.topic(), mockChannel.partition(), startFrom, message),
   402  	})
   403  
   404  	haltChan := make(chan struct{})
   405  
   406  	t.Run("ProperParent", func(t *testing.T) {
   407  		parentConsumer, err := setupParentConsumerForChannel(mockConsenter.retryOptions(), haltChan, []string{mockBroker.Addr()}, mockBrokerConfig, mockChannel)
   408  		assert.NoError(t, err, "Expected the setupParentConsumerForChannel call to return without errors")
   409  		assert.NoError(t, parentConsumer.Close(), "Expected to close the parentConsumer without errors")
   410  	})
   411  
   412  	t.Run("ProperChannel", func(t *testing.T) {
   413  		parentConsumer, _ := setupParentConsumerForChannel(mockConsenter.retryOptions(), haltChan, []string{mockBroker.Addr()}, mockBrokerConfig, mockChannel)
   414  		defer func() { parentConsumer.Close() }()
   415  		channelConsumer, err := setupChannelConsumerForChannel(mockConsenter.retryOptions(), haltChan, parentConsumer, mockChannel, newestOffset)
   416  		assert.NoError(t, err, "Expected the setupChannelConsumerForChannel call to return without errors")
   417  		assert.NoError(t, channelConsumer.Close(), "Expected to close the channelConsumer without errors")
   418  	})
   419  
   420  	t.Run("WithParentConsumerError", func(t *testing.T) {
   421  		// Provide an empty brokers list
   422  		_, err := setupParentConsumerForChannel(mockConsenter.retryOptions(), haltChan, []string{}, mockBrokerConfig, mockChannel)
   423  		assert.Error(t, err, "Expected the setupParentConsumerForChannel call to return an error")
   424  	})
   425  
   426  	t.Run("WithChannelConsumerError", func(t *testing.T) {
   427  		// Provide an out-of-range offset
   428  		parentConsumer, _ := setupParentConsumerForChannel(mockConsenter.retryOptions(), haltChan, []string{mockBroker.Addr()}, mockBrokerConfig, mockChannel)
   429  		_, err := setupChannelConsumerForChannel(mockConsenter.retryOptions(), haltChan, parentConsumer, mockChannel, newestOffset+1)
   430  		defer func() { parentConsumer.Close() }()
   431  		assert.Error(t, err, "Expected the setupChannelConsumerForChannel call to return an error")
   432  	})
   433  }
   434  
   435  func TestCloseKafkaObjects(t *testing.T) {
   436  	mockChannel := newChannel(channelNameForTest(t), defaultPartition)
   437  
   438  	mockSupport := &mockmultichain.ConsenterSupport{
   439  		ChainIDVal: mockChannel.topic(),
   440  	}
   441  
   442  	oldestOffset := int64(0)
   443  	newestOffset := int64(5)
   444  
   445  	startFrom := int64(3)
   446  	message := sarama.StringEncoder("messageFoo")
   447  
   448  	mockBroker := sarama.NewMockBroker(t, 0)
   449  	defer func() { mockBroker.Close() }()
   450  
   451  	mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{
   452  		"MetadataRequest": sarama.NewMockMetadataResponse(t).
   453  			SetBroker(mockBroker.Addr(), mockBroker.BrokerID()).
   454  			SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()),
   455  		"OffsetRequest": sarama.NewMockOffsetResponse(t).
   456  			SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset).
   457  			SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset),
   458  		"FetchRequest": sarama.NewMockFetchResponse(t, 1).
   459  			SetMessage(mockChannel.topic(), mockChannel.partition(), startFrom, message),
   460  	})
   461  
   462  	haltChan := make(chan struct{})
   463  
   464  	t.Run("Proper", func(t *testing.T) {
   465  		producer, _ := setupProducerForChannel(mockConsenter.retryOptions(), haltChan, []string{mockBroker.Addr()}, mockBrokerConfig, mockChannel)
   466  		parentConsumer, _ := setupParentConsumerForChannel(mockConsenter.retryOptions(), haltChan, []string{mockBroker.Addr()}, mockBrokerConfig, mockChannel)
   467  		channelConsumer, _ := setupChannelConsumerForChannel(mockConsenter.retryOptions(), haltChan, parentConsumer, mockChannel, startFrom)
   468  
   469  		// Set up a chain with just the minimum necessary fields instantiated so
   470  		// as to test the function
   471  		bareMinimumChain := &chainImpl{
   472  			support:         mockSupport,
   473  			producer:        producer,
   474  			parentConsumer:  parentConsumer,
   475  			channelConsumer: channelConsumer,
   476  		}
   477  
   478  		errs := bareMinimumChain.closeKafkaObjects()
   479  
   480  		assert.Len(t, errs, 0, "Expected zero errors")
   481  
   482  		assert.Panics(t, func() {
   483  			channelConsumer.Close()
   484  		})
   485  
   486  		assert.NotPanics(t, func() {
   487  			parentConsumer.Close()
   488  		})
   489  
   490  		// TODO For some reason this panic cannot be captured by the `assert`
   491  		// test framework. Not a dealbreaker but need to investigate further.
   492  		/* assert.Panics(t, func() {
   493  			producer.Close()
   494  		}) */
   495  	})
   496  
   497  	t.Run("ChannelConsumerError", func(t *testing.T) {
   498  		producer, _ := sarama.NewSyncProducer([]string{mockBroker.Addr()}, mockBrokerConfig)
   499  
   500  		// Unlike all other tests in this file, forcing an error on the
   501  		// channelConsumer.Close() call is more easily achieved using the mock
   502  		// Consumer. Thus we bypass the call to `setup*Consumer`.
   503  
   504  		// Have the consumer receive an ErrOutOfBrokers error.
   505  		mockParentConsumer := mocks.NewConsumer(t, nil)
   506  		mockParentConsumer.ExpectConsumePartition(mockChannel.topic(), mockChannel.partition(), startFrom).YieldError(sarama.ErrOutOfBrokers)
   507  		mockChannelConsumer, err := mockParentConsumer.ConsumePartition(mockChannel.topic(), mockChannel.partition(), startFrom)
   508  		assert.NoError(t, err, "Expected no error when setting up the mock partition consumer")
   509  
   510  		bareMinimumChain := &chainImpl{
   511  			support:         mockSupport,
   512  			producer:        producer,
   513  			parentConsumer:  mockParentConsumer,
   514  			channelConsumer: mockChannelConsumer,
   515  		}
   516  
   517  		errs := bareMinimumChain.closeKafkaObjects()
   518  
   519  		assert.Len(t, errs, 1, "Expected 1 error returned")
   520  
   521  		assert.NotPanics(t, func() {
   522  			mockChannelConsumer.Close()
   523  		})
   524  
   525  		assert.NotPanics(t, func() {
   526  			mockParentConsumer.Close()
   527  		})
   528  	})
   529  }
   530  
   531  // Test helper functions here.
   532  
   533  func TestGetLastCutBlockNumber(t *testing.T) {
   534  	testCases := []struct {
   535  		name     string
   536  		input    uint64
   537  		expected uint64
   538  	}{
   539  		{"Proper", uint64(2), uint64(1)},
   540  		{"Zero", uint64(1), uint64(0)},
   541  	}
   542  	for _, tc := range testCases {
   543  		t.Run(tc.name, func(t *testing.T) {
   544  			assert.Equal(t, tc.expected, getLastCutBlockNumber(tc.input))
   545  		})
   546  	}
   547  }
   548  
   549  func TestGetLastOffsetPersisted(t *testing.T) {
   550  	mockChannel := newChannel(channelNameForTest(t), defaultPartition)
   551  	mockMetadata := &cb.Metadata{Value: utils.MarshalOrPanic(&ab.KafkaMetadata{LastOffsetPersisted: int64(5)})}
   552  
   553  	testCases := []struct {
   554  		name     string
   555  		md       []byte
   556  		expected int64
   557  		panics   bool
   558  	}{
   559  		{"Proper", mockMetadata.Value, int64(5), false},
   560  		{"Empty", nil, sarama.OffsetOldest - 1, false},
   561  		{"Panics", tamperBytes(mockMetadata.Value), sarama.OffsetOldest - 1, true},
   562  	}
   563  
   564  	for _, tc := range testCases {
   565  		t.Run(tc.name, func(t *testing.T) {
   566  			if !tc.panics {
   567  				assert.Equal(t, tc.expected, getLastOffsetPersisted(tc.md, mockChannel.String()))
   568  			} else {
   569  				assert.Panics(t, func() {
   570  					getLastOffsetPersisted(tc.md, mockChannel.String())
   571  				}, "Expected getLastOffsetPersisted call to panic")
   572  			}
   573  		})
   574  	}
   575  }
   576  
   577  func TestSendConnectMessage(t *testing.T) {
   578  	mockBroker := sarama.NewMockBroker(t, 0)
   579  	defer func() { mockBroker.Close() }()
   580  
   581  	mockChannel := newChannel("mockChannelFoo", defaultPartition)
   582  
   583  	metadataResponse := new(sarama.MetadataResponse)
   584  	metadataResponse.AddBroker(mockBroker.Addr(), mockBroker.BrokerID())
   585  	metadataResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID(), nil, nil, sarama.ErrNoError)
   586  	mockBroker.Returns(metadataResponse)
   587  
   588  	producer, _ := sarama.NewSyncProducer([]string{mockBroker.Addr()}, mockBrokerConfig)
   589  	defer func() { producer.Close() }()
   590  
   591  	haltChan := make(chan struct{})
   592  
   593  	t.Run("Proper", func(t *testing.T) {
   594  		successResponse := new(sarama.ProduceResponse)
   595  		successResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError)
   596  		mockBroker.Returns(successResponse)
   597  
   598  		assert.NoError(t, sendConnectMessage(mockConsenter.retryOptions(), haltChan, producer, mockChannel), "Expected the sendConnectMessage call to return without errors")
   599  	})
   600  
   601  	t.Run("WithError", func(t *testing.T) {
   602  		// Note that this test is affected by the following parameters:
   603  		// - Net.ReadTimeout
   604  		// - Consumer.Retry.Backoff
   605  		// - Metadata.Retry.Max
   606  
   607  		// Have the broker return an ErrNotEnoughReplicas error
   608  		failureResponse := new(sarama.ProduceResponse)
   609  		failureResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), sarama.ErrNotEnoughReplicas)
   610  		mockBroker.Returns(failureResponse)
   611  
   612  		assert.Error(t, sendConnectMessage(mockConsenter.retryOptions(), haltChan, producer, mockChannel), "Expected the sendConnectMessage call to return an error")
   613  	})
   614  }
   615  
   616  func TestSendTimeToCut(t *testing.T) {
   617  	mockBroker := sarama.NewMockBroker(t, 0)
   618  	defer func() { mockBroker.Close() }()
   619  
   620  	mockChannel := newChannel("mockChannelFoo", defaultPartition)
   621  
   622  	metadataResponse := new(sarama.MetadataResponse)
   623  	metadataResponse.AddBroker(mockBroker.Addr(), mockBroker.BrokerID())
   624  	metadataResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID(), nil, nil, sarama.ErrNoError)
   625  	mockBroker.Returns(metadataResponse)
   626  
   627  	producer, err := sarama.NewSyncProducer([]string{mockBroker.Addr()}, mockBrokerConfig)
   628  	assert.NoError(t, err, "Expected no error when setting up the sarama SyncProducer")
   629  	defer func() { producer.Close() }()
   630  
   631  	timeToCutBlockNumber := uint64(3)
   632  	var timer <-chan time.Time
   633  
   634  	t.Run("Proper", func(t *testing.T) {
   635  		successResponse := new(sarama.ProduceResponse)
   636  		successResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError)
   637  		mockBroker.Returns(successResponse)
   638  
   639  		timer = time.After(longTimeout)
   640  
   641  		assert.NoError(t, sendTimeToCut(producer, mockChannel, timeToCutBlockNumber, &timer), "Expected the sendTimeToCut call to return without errors")
   642  		assert.Nil(t, timer, "Expected the sendTimeToCut call to nil the timer")
   643  	})
   644  
   645  	t.Run("WithError", func(t *testing.T) {
   646  		// Note that this test is affected by the following parameters:
   647  		// - Net.ReadTimeout
   648  		// - Consumer.Retry.Backoff
   649  		// - Metadata.Retry.Max
   650  		failureResponse := new(sarama.ProduceResponse)
   651  		failureResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), sarama.ErrNotEnoughReplicas)
   652  		mockBroker.Returns(failureResponse)
   653  
   654  		timer = time.After(longTimeout)
   655  
   656  		assert.Error(t, sendTimeToCut(producer, mockChannel, timeToCutBlockNumber, &timer), "Expected the sendTimeToCut call to return an error")
   657  		assert.Nil(t, timer, "Expected the sendTimeToCut call to nil the timer")
   658  	})
   659  }
   660  
   661  func TestProcessMessagesToBlocks(t *testing.T) {
   662  	mockBroker := sarama.NewMockBroker(t, 0)
   663  	defer func() { mockBroker.Close() }()
   664  
   665  	mockChannel := newChannel("mockChannelFoo", defaultPartition)
   666  
   667  	metadataResponse := new(sarama.MetadataResponse)
   668  	metadataResponse.AddBroker(mockBroker.Addr(), mockBroker.BrokerID())
   669  	metadataResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID(), nil, nil, sarama.ErrNoError)
   670  	mockBroker.Returns(metadataResponse)
   671  
   672  	producer, _ := sarama.NewSyncProducer([]string{mockBroker.Addr()}, mockBrokerConfig)
   673  
   674  	mockBrokerConfigCopy := *mockBrokerConfig
   675  	mockBrokerConfigCopy.ChannelBufferSize = 0
   676  
   677  	newestOffset := int64(0)
   678  
   679  	mockParentConsumer := mocks.NewConsumer(t, &mockBrokerConfigCopy)
   680  	mpc := mockParentConsumer.ExpectConsumePartition(mockChannel.topic(), mockChannel.partition(), newestOffset)
   681  	mockChannelConsumer, err := mockParentConsumer.ConsumePartition(mockChannel.topic(), mockChannel.partition(), newestOffset)
   682  	assert.NoError(t, err, "Expected no error when setting up the mock partition consumer")
   683  
   684  	t.Run("ReceiveConnect", func(t *testing.T) {
   685  		errorChan := make(chan struct{})
   686  		close(errorChan)
   687  		haltChan := make(chan struct{})
   688  
   689  		mockSupport := &mockmultichain.ConsenterSupport{
   690  			ChainIDVal: mockChannel.topic(),
   691  		}
   692  
   693  		bareMinimumChain := &chainImpl{
   694  			parentConsumer:  mockParentConsumer,
   695  			channelConsumer: mockChannelConsumer,
   696  
   697  			channel: mockChannel,
   698  			support: mockSupport,
   699  
   700  			errorChan: errorChan,
   701  			haltChan:  haltChan,
   702  		}
   703  
   704  		var counts []uint64
   705  		done := make(chan struct{})
   706  
   707  		go func() {
   708  			counts, err = bareMinimumChain.processMessagesToBlocks()
   709  			done <- struct{}{}
   710  		}()
   711  
   712  		// This is the wrappedMessage that the for-loop will process
   713  		mpc.YieldMessage(newMockConsumerMessage(newConnectMessage()))
   714  
   715  		logger.Debug("Closing haltChan to exit the infinite for-loop")
   716  		close(haltChan) // Identical to chain.Halt()
   717  		logger.Debug("haltChan closed")
   718  		<-done
   719  
   720  		assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors")
   721  		assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled")
   722  		assert.Equal(t, uint64(1), counts[indexProcessConnectPass], "Expected 1 CONNECT message processed")
   723  	})
   724  
   725  	t.Run("ReceiveRegularWithError", func(t *testing.T) {
   726  		errorChan := make(chan struct{})
   727  		close(errorChan)
   728  		haltChan := make(chan struct{})
   729  
   730  		mockSupport := &mockmultichain.ConsenterSupport{
   731  			ChainIDVal: mockChannel.topic(),
   732  		}
   733  
   734  		bareMinimumChain := &chainImpl{
   735  			parentConsumer:  mockParentConsumer,
   736  			channelConsumer: mockChannelConsumer,
   737  
   738  			channel: mockChannel,
   739  			support: mockSupport,
   740  
   741  			errorChan: errorChan,
   742  			haltChan:  haltChan,
   743  		}
   744  
   745  		var counts []uint64
   746  		done := make(chan struct{})
   747  
   748  		go func() {
   749  			counts, err = bareMinimumChain.processMessagesToBlocks()
   750  			done <- struct{}{}
   751  		}()
   752  
   753  		// This is the wrappedMessage that the for-loop will process
   754  		mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(tamperBytes(utils.MarshalOrPanic(newMockEnvelope("fooMessage"))))))
   755  
   756  		logger.Debug("Closing haltChan to exit the infinite for-loop")
   757  		close(haltChan) // Identical to chain.Halt()
   758  		logger.Debug("haltChan closed")
   759  		<-done
   760  
   761  		assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors")
   762  		assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled")
   763  		assert.Equal(t, uint64(1), counts[indexProcessRegularError], "Expected 1 damaged REGULAR message processed")
   764  	})
   765  
   766  	t.Run("ReceiveRegularAndQueue", func(t *testing.T) {
   767  		errorChan := make(chan struct{})
   768  		close(errorChan)
   769  		haltChan := make(chan struct{})
   770  
   771  		lastCutBlockNumber := uint64(3)
   772  
   773  		mockSupport := &mockmultichain.ConsenterSupport{
   774  			Blocks:         make(chan *cb.Block), // WriteBlock will post here
   775  			BlockCutterVal: mockblockcutter.NewReceiver(),
   776  			ChainIDVal:     mockChannel.topic(),
   777  			HeightVal:      lastCutBlockNumber, // Incremented during the WriteBlock call
   778  			SharedConfigVal: &mockconfig.Orderer{
   779  				BatchTimeoutVal: longTimeout,
   780  			},
   781  		}
   782  		defer close(mockSupport.BlockCutterVal.Block)
   783  
   784  		bareMinimumChain := &chainImpl{
   785  			parentConsumer:  mockParentConsumer,
   786  			channelConsumer: mockChannelConsumer,
   787  
   788  			channel:            mockChannel,
   789  			support:            mockSupport,
   790  			lastCutBlockNumber: lastCutBlockNumber,
   791  
   792  			errorChan: errorChan,
   793  			haltChan:  haltChan,
   794  		}
   795  
   796  		var counts []uint64
   797  		done := make(chan struct{})
   798  
   799  		go func() {
   800  			counts, err = bareMinimumChain.processMessagesToBlocks()
   801  			done <- struct{}{}
   802  		}()
   803  
   804  		// This is the wrappedMessage that the for-loop will process
   805  		mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage")))))
   806  
   807  		mockSupport.BlockCutterVal.Block <- struct{}{} // Let the `mockblockcutter.Ordered` call return
   808  		logger.Debugf("Mock blockcutter's Ordered call has returned")
   809  
   810  		logger.Debug("Closing haltChan to exit the infinite for-loop")
   811  		// We are guaranteed to hit the haltChan branch after hitting the REGULAR branch at least once
   812  		close(haltChan) // Identical to chain.Halt()
   813  		logger.Debug("haltChan closed")
   814  		<-done
   815  
   816  		assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors")
   817  		assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled")
   818  		assert.Equal(t, uint64(1), counts[indexProcessRegularPass], "Expected 1 REGULAR message processed")
   819  	})
   820  
   821  	t.Run("ReceiveRegularAndCutBlock", func(t *testing.T) {
   822  		errorChan := make(chan struct{})
   823  		close(errorChan)
   824  		haltChan := make(chan struct{})
   825  
   826  		lastCutBlockNumber := uint64(3)
   827  
   828  		mockSupport := &mockmultichain.ConsenterSupport{
   829  			Blocks:         make(chan *cb.Block), // WriteBlock will post here
   830  			BlockCutterVal: mockblockcutter.NewReceiver(),
   831  			ChainIDVal:     mockChannel.topic(),
   832  			HeightVal:      lastCutBlockNumber, // Incremented during the WriteBlock call
   833  			SharedConfigVal: &mockconfig.Orderer{
   834  				BatchTimeoutVal: longTimeout,
   835  			},
   836  		}
   837  		defer close(mockSupport.BlockCutterVal.Block)
   838  
   839  		bareMinimumChain := &chainImpl{
   840  			parentConsumer:  mockParentConsumer,
   841  			channelConsumer: mockChannelConsumer,
   842  
   843  			channel:            mockChannel,
   844  			support:            mockSupport,
   845  			lastCutBlockNumber: lastCutBlockNumber,
   846  
   847  			errorChan: errorChan,
   848  			haltChan:  haltChan,
   849  		}
   850  
   851  		var counts []uint64
   852  		done := make(chan struct{})
   853  
   854  		go func() {
   855  			counts, err = bareMinimumChain.processMessagesToBlocks()
   856  			done <- struct{}{}
   857  		}()
   858  
   859  		mockSupport.BlockCutterVal.CutNext = true
   860  
   861  		// This is the wrappedMessage that the for-loop will process
   862  		mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage")))))
   863  
   864  		mockSupport.BlockCutterVal.Block <- struct{}{} // Let the `mockblockcutter.Ordered` call return
   865  		logger.Debugf("Mock blockcutter's Ordered call has returned")
   866  		<-mockSupport.Blocks // Let the `mockConsenterSupport.WriteBlock` proceed
   867  
   868  		logger.Debug("Closing haltChan to exit the infinite for-loop")
   869  		close(haltChan) // Identical to chain.Halt()
   870  		logger.Debug("haltChan closed")
   871  		<-done
   872  
   873  		assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors")
   874  		assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled")
   875  		assert.Equal(t, uint64(1), counts[indexProcessRegularPass], "Expected 1 REGULAR message processed")
   876  		assert.Equal(t, lastCutBlockNumber+1, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to be bumped up by one")
   877  	})
   878  
   879  	t.Run("ReceiveTwoRegularAndCutTwoBlocks", func(t *testing.T) {
   880  		if testing.Short() {
   881  			t.Skip("Skipping test in short mode")
   882  		}
   883  
   884  		errorChan := make(chan struct{})
   885  		close(errorChan)
   886  		haltChan := make(chan struct{})
   887  
   888  		lastCutBlockNumber := uint64(3)
   889  
   890  		mockSupport := &mockmultichain.ConsenterSupport{
   891  			Blocks:         make(chan *cb.Block), // WriteBlock will post here
   892  			BlockCutterVal: mockblockcutter.NewReceiver(),
   893  			ChainIDVal:     mockChannel.topic(),
   894  			HeightVal:      lastCutBlockNumber, // Incremented during the WriteBlock call
   895  			SharedConfigVal: &mockconfig.Orderer{
   896  				BatchTimeoutVal: longTimeout,
   897  			},
   898  		}
   899  		defer close(mockSupport.BlockCutterVal.Block)
   900  
   901  		bareMinimumChain := &chainImpl{
   902  			parentConsumer:  mockParentConsumer,
   903  			channelConsumer: mockChannelConsumer,
   904  
   905  			channel:            mockChannel,
   906  			support:            mockSupport,
   907  			lastCutBlockNumber: lastCutBlockNumber,
   908  
   909  			errorChan: errorChan,
   910  			haltChan:  haltChan,
   911  		}
   912  
   913  		var counts []uint64
   914  		done := make(chan struct{})
   915  
   916  		go func() {
   917  			counts, err = bareMinimumChain.processMessagesToBlocks()
   918  			done <- struct{}{}
   919  		}()
   920  
   921  		var block1, block2 *cb.Block
   922  
   923  		// This is the first wrappedMessage that the for-loop will process
   924  		block1Offset := mpc.HighWaterMarkOffset()
   925  		mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage")))))
   926  		mockSupport.BlockCutterVal.Block <- struct{}{} // Let the `mockblockcutter.Ordered` call return
   927  		logger.Debugf("Mock blockcutter's Ordered call has returned")
   928  
   929  		mockSupport.BlockCutterVal.IsolatedTx = true
   930  
   931  		// This is the first wrappedMessage that the for-loop will process
   932  		block2Offset := mpc.HighWaterMarkOffset()
   933  		mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage")))))
   934  		mockSupport.BlockCutterVal.Block <- struct{}{}
   935  		logger.Debugf("Mock blockcutter's Ordered call has returned for the second time")
   936  
   937  		select {
   938  		case block1 = <-mockSupport.Blocks: // Let the `mockConsenterSupport.WriteBlock` proceed
   939  		case <-time.After(shortTimeout):
   940  			logger.Fatalf("Did not receive a block from the blockcutter as expected")
   941  		}
   942  
   943  		select {
   944  		case block2 = <-mockSupport.Blocks:
   945  		case <-time.After(shortTimeout):
   946  			logger.Fatalf("Did not receive a block from the blockcutter as expected")
   947  		}
   948  
   949  		logger.Debug("Closing haltChan to exit the infinite for-loop")
   950  		close(haltChan) // Identical to chain.Halt()
   951  		logger.Debug("haltChan closed")
   952  		<-done
   953  
   954  		assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors")
   955  		assert.Equal(t, uint64(2), counts[indexRecvPass], "Expected 2 messages received and unmarshaled")
   956  		assert.Equal(t, uint64(2), counts[indexProcessRegularPass], "Expected 2 REGULAR messages processed")
   957  		assert.Equal(t, lastCutBlockNumber+2, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to be bumped up by two")
   958  		assert.Equal(t, block1Offset, extractEncodedOffset(block1.GetMetadata().Metadata[cb.BlockMetadataIndex_ORDERER]), "Expected encoded offset in first block to be %d", block1Offset)
   959  		assert.Equal(t, block2Offset, extractEncodedOffset(block2.GetMetadata().Metadata[cb.BlockMetadataIndex_ORDERER]), "Expected encoded offset in first block to be %d", block2Offset)
   960  	})
   961  
   962  	t.Run("SecondTxOverflows", func(t *testing.T) {
   963  		if testing.Short() {
   964  			t.Skip("Skipping test in short mode")
   965  		}
   966  
   967  		errorChan := make(chan struct{})
   968  		close(errorChan)
   969  		haltChan := make(chan struct{})
   970  
   971  		lastCutBlockNumber := uint64(3)
   972  
   973  		mockSupport := &mockmultichain.ConsenterSupport{
   974  			Blocks:         make(chan *cb.Block), // WriteBlock will post here
   975  			BlockCutterVal: mockblockcutter.NewReceiver(),
   976  			ChainIDVal:     mockChannel.topic(),
   977  			HeightVal:      lastCutBlockNumber, // Incremented during the WriteBlock call
   978  			SharedConfigVal: &mockconfig.Orderer{
   979  				BatchTimeoutVal: longTimeout,
   980  			},
   981  		}
   982  		defer close(mockSupport.BlockCutterVal.Block)
   983  
   984  		bareMinimumChain := &chainImpl{
   985  			parentConsumer:  mockParentConsumer,
   986  			channelConsumer: mockChannelConsumer,
   987  
   988  			channel:            mockChannel,
   989  			support:            mockSupport,
   990  			lastCutBlockNumber: lastCutBlockNumber,
   991  
   992  			errorChan: errorChan,
   993  			haltChan:  haltChan,
   994  		}
   995  
   996  		var counts []uint64
   997  		done := make(chan struct{})
   998  
   999  		go func() {
  1000  			counts, err = bareMinimumChain.processMessagesToBlocks()
  1001  			done <- struct{}{}
  1002  		}()
  1003  
  1004  		var block1, block2 *cb.Block
  1005  
  1006  		block1LastOffset := mpc.HighWaterMarkOffset()
  1007  		mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage")))))
  1008  		mockSupport.BlockCutterVal.Block <- struct{}{} // Let the `mockblockcutter.Ordered` call return
  1009  
  1010  		// Set CutAncestors to true so that second message overflows receiver batch
  1011  		mockSupport.BlockCutterVal.CutAncestors = true
  1012  		mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage")))))
  1013  		mockSupport.BlockCutterVal.Block <- struct{}{}
  1014  
  1015  		select {
  1016  		case block1 = <-mockSupport.Blocks: // Let the `mockConsenterSupport.WriteBlock` proceed
  1017  		case <-time.After(shortTimeout):
  1018  			logger.Fatalf("Did not receive a block from the blockcutter as expected")
  1019  		}
  1020  
  1021  		// Set CutNext to true to flush all pending messages
  1022  		mockSupport.BlockCutterVal.CutAncestors = false
  1023  		mockSupport.BlockCutterVal.CutNext = true
  1024  		block2LastOffset := mpc.HighWaterMarkOffset()
  1025  		mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage")))))
  1026  		mockSupport.BlockCutterVal.Block <- struct{}{}
  1027  
  1028  		select {
  1029  		case block2 = <-mockSupport.Blocks:
  1030  		case <-time.After(shortTimeout):
  1031  			logger.Fatalf("Did not receive a block from the blockcutter as expected")
  1032  		}
  1033  
  1034  		logger.Debug("Closing haltChan to exit the infinite for-loop")
  1035  		close(haltChan) // Identical to chain.Halt()
  1036  		logger.Debug("haltChan closed")
  1037  		<-done
  1038  
  1039  		assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors")
  1040  		assert.Equal(t, uint64(3), counts[indexRecvPass], "Expected 2 messages received and unmarshaled")
  1041  		assert.Equal(t, uint64(3), counts[indexProcessRegularPass], "Expected 2 REGULAR messages processed")
  1042  		assert.Equal(t, lastCutBlockNumber+2, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to be bumped up by two")
  1043  		assert.Equal(t, block1LastOffset, extractEncodedOffset(block1.GetMetadata().Metadata[cb.BlockMetadataIndex_ORDERER]), "Expected encoded offset in first block to be %d", block1LastOffset)
  1044  		assert.Equal(t, block2LastOffset, extractEncodedOffset(block2.GetMetadata().Metadata[cb.BlockMetadataIndex_ORDERER]), "Expected encoded offset in second block to be %d", block2LastOffset)
  1045  	})
  1046  
  1047  	t.Run("ReceiveRegularAndSendTimeToCut", func(t *testing.T) {
  1048  		t.Skip("Skipping test as it introduces a race condition")
  1049  
  1050  		// NB We haven't set a handlermap for the mock broker so we need to set
  1051  		// the ProduceResponse
  1052  		successResponse := new(sarama.ProduceResponse)
  1053  		successResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError)
  1054  		mockBroker.Returns(successResponse)
  1055  
  1056  		errorChan := make(chan struct{})
  1057  		close(errorChan)
  1058  		haltChan := make(chan struct{})
  1059  
  1060  		lastCutBlockNumber := uint64(3)
  1061  
  1062  		mockSupport := &mockmultichain.ConsenterSupport{
  1063  			Blocks:         make(chan *cb.Block), // WriteBlock will post here
  1064  			BlockCutterVal: mockblockcutter.NewReceiver(),
  1065  			ChainIDVal:     mockChannel.topic(),
  1066  			HeightVal:      lastCutBlockNumber, // Incremented during the WriteBlock call
  1067  			SharedConfigVal: &mockconfig.Orderer{
  1068  				BatchTimeoutVal: extraShortTimeout, // ATTN
  1069  			},
  1070  		}
  1071  		defer close(mockSupport.BlockCutterVal.Block)
  1072  
  1073  		bareMinimumChain := &chainImpl{
  1074  			producer:        producer,
  1075  			parentConsumer:  mockParentConsumer,
  1076  			channelConsumer: mockChannelConsumer,
  1077  
  1078  			channel:            mockChannel,
  1079  			support:            mockSupport,
  1080  			lastCutBlockNumber: lastCutBlockNumber,
  1081  
  1082  			errorChan: errorChan,
  1083  			haltChan:  haltChan,
  1084  		}
  1085  
  1086  		var counts []uint64
  1087  		done := make(chan struct{})
  1088  
  1089  		go func() {
  1090  			counts, err = bareMinimumChain.processMessagesToBlocks()
  1091  			done <- struct{}{}
  1092  		}()
  1093  
  1094  		// This is the wrappedMessage that the for-loop will process
  1095  		mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage")))))
  1096  
  1097  		mockSupport.BlockCutterVal.Block <- struct{}{} // Let the `mockblockcutter.Ordered` call return
  1098  		logger.Debugf("Mock blockcutter's Ordered call has returned")
  1099  
  1100  		// Sleep so that the timer branch is activated before the exitChan one.
  1101  		// TODO This is a race condition, will fix in follow-up changeset
  1102  		time.Sleep(hitBranch)
  1103  
  1104  		logger.Debug("Closing haltChan to exit the infinite for-loop")
  1105  		close(haltChan) // Identical to chain.Halt()
  1106  		logger.Debug("haltChan closed")
  1107  		<-done
  1108  
  1109  		assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors")
  1110  		assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled")
  1111  		assert.Equal(t, uint64(1), counts[indexProcessRegularPass], "Expected 1 REGULAR message processed")
  1112  		assert.Equal(t, uint64(1), counts[indexSendTimeToCutPass], "Expected 1 TIMER event processed")
  1113  		assert.Equal(t, lastCutBlockNumber, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to stay the same")
  1114  	})
  1115  
  1116  	t.Run("ReceiveRegularAndSendTimeToCutError", func(t *testing.T) {
  1117  		// Note that this test is affected by the following parameters:
  1118  		// - Net.ReadTimeout
  1119  		// - Consumer.Retry.Backoff
  1120  		// - Metadata.Retry.Max
  1121  
  1122  		t.Skip("Skipping test as it introduces a race condition")
  1123  
  1124  		// Exact same test as ReceiveRegularAndSendTimeToCut.
  1125  		// Only difference is that the producer's attempt to send a TTC will
  1126  		// fail with an ErrNotEnoughReplicas error.
  1127  		failureResponse := new(sarama.ProduceResponse)
  1128  		failureResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), sarama.ErrNotEnoughReplicas)
  1129  		mockBroker.Returns(failureResponse)
  1130  
  1131  		errorChan := make(chan struct{})
  1132  		close(errorChan)
  1133  		haltChan := make(chan struct{})
  1134  
  1135  		lastCutBlockNumber := uint64(3)
  1136  
  1137  		mockSupport := &mockmultichain.ConsenterSupport{
  1138  			Blocks:         make(chan *cb.Block), // WriteBlock will post here
  1139  			BlockCutterVal: mockblockcutter.NewReceiver(),
  1140  			ChainIDVal:     mockChannel.topic(),
  1141  			HeightVal:      lastCutBlockNumber, // Incremented during the WriteBlock call
  1142  			SharedConfigVal: &mockconfig.Orderer{
  1143  				BatchTimeoutVal: extraShortTimeout, // ATTN
  1144  			},
  1145  		}
  1146  		defer close(mockSupport.BlockCutterVal.Block)
  1147  
  1148  		bareMinimumChain := &chainImpl{
  1149  			producer:        producer,
  1150  			parentConsumer:  mockParentConsumer,
  1151  			channelConsumer: mockChannelConsumer,
  1152  
  1153  			channel:            mockChannel,
  1154  			support:            mockSupport,
  1155  			lastCutBlockNumber: lastCutBlockNumber,
  1156  
  1157  			errorChan: errorChan,
  1158  			haltChan:  haltChan,
  1159  		}
  1160  
  1161  		var counts []uint64
  1162  		done := make(chan struct{})
  1163  
  1164  		go func() {
  1165  			counts, err = bareMinimumChain.processMessagesToBlocks()
  1166  			done <- struct{}{}
  1167  		}()
  1168  
  1169  		// This is the wrappedMessage that the for-loop will process
  1170  		mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage")))))
  1171  
  1172  		mockSupport.BlockCutterVal.Block <- struct{}{} // Let the `mockblockcutter.Ordered` call return
  1173  		logger.Debugf("Mock blockcutter's Ordered call has returned")
  1174  
  1175  		// Sleep so that the timer branch is activated before the exitChan one.
  1176  		// TODO This is a race condition, will fix in follow-up changeset
  1177  		time.Sleep(hitBranch)
  1178  
  1179  		logger.Debug("Closing haltChan to exit the infinite for-loop")
  1180  		close(haltChan) // Identical to chain.Halt()
  1181  		logger.Debug("haltChan closed")
  1182  		<-done
  1183  
  1184  		assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors")
  1185  		assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled")
  1186  		assert.Equal(t, uint64(1), counts[indexProcessRegularPass], "Expected 1 REGULAR message processed")
  1187  		assert.Equal(t, uint64(1), counts[indexSendTimeToCutError], "Expected 1 faulty TIMER event processed")
  1188  		assert.Equal(t, lastCutBlockNumber, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to stay the same")
  1189  	})
  1190  
  1191  	t.Run("ReceiveTimeToCutProper", func(t *testing.T) {
  1192  		errorChan := make(chan struct{})
  1193  		close(errorChan)
  1194  		haltChan := make(chan struct{})
  1195  
  1196  		lastCutBlockNumber := uint64(3)
  1197  
  1198  		mockSupport := &mockmultichain.ConsenterSupport{
  1199  			Blocks:         make(chan *cb.Block), // WriteBlock will post here
  1200  			BlockCutterVal: mockblockcutter.NewReceiver(),
  1201  			ChainIDVal:     mockChannel.topic(),
  1202  			HeightVal:      lastCutBlockNumber, // Incremented during the WriteBlock call
  1203  		}
  1204  		defer close(mockSupport.BlockCutterVal.Block)
  1205  
  1206  		bareMinimumChain := &chainImpl{
  1207  			parentConsumer:  mockParentConsumer,
  1208  			channelConsumer: mockChannelConsumer,
  1209  
  1210  			channel:            mockChannel,
  1211  			support:            mockSupport,
  1212  			lastCutBlockNumber: lastCutBlockNumber,
  1213  
  1214  			errorChan: errorChan,
  1215  			haltChan:  haltChan,
  1216  		}
  1217  
  1218  		// We need the mock blockcutter to deliver a non-empty batch
  1219  		go func() {
  1220  			mockSupport.BlockCutterVal.Block <- struct{}{} // Let the `mockblockcutter.Ordered` call below return
  1221  			logger.Debugf("Mock blockcutter's Ordered call has returned")
  1222  		}()
  1223  		// We are "planting" a message directly to the mock blockcutter
  1224  		mockSupport.BlockCutterVal.Ordered(newMockEnvelope("fooMessage"))
  1225  
  1226  		var counts []uint64
  1227  		done := make(chan struct{})
  1228  
  1229  		go func() {
  1230  			counts, err = bareMinimumChain.processMessagesToBlocks()
  1231  			done <- struct{}{}
  1232  		}()
  1233  
  1234  		// This is the wrappedMessage that the for-loop will process
  1235  		mpc.YieldMessage(newMockConsumerMessage(newTimeToCutMessage(lastCutBlockNumber + 1)))
  1236  
  1237  		<-mockSupport.Blocks // Let the `mockConsenterSupport.WriteBlock` proceed
  1238  
  1239  		logger.Debug("Closing haltChan to exit the infinite for-loop")
  1240  		close(haltChan) // Identical to chain.Halt()
  1241  		logger.Debug("haltChan closed")
  1242  		<-done
  1243  
  1244  		assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors")
  1245  		assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled")
  1246  		assert.Equal(t, uint64(1), counts[indexProcessTimeToCutPass], "Expected 1 TIMETOCUT message processed")
  1247  		assert.Equal(t, lastCutBlockNumber+1, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to be bumped up by one")
  1248  	})
  1249  
  1250  	t.Run("ReceiveTimeToCutZeroBatch", func(t *testing.T) {
  1251  		errorChan := make(chan struct{})
  1252  		close(errorChan)
  1253  		haltChan := make(chan struct{})
  1254  
  1255  		lastCutBlockNumber := uint64(3)
  1256  
  1257  		mockSupport := &mockmultichain.ConsenterSupport{
  1258  			Blocks:         make(chan *cb.Block), // WriteBlock will post here
  1259  			BlockCutterVal: mockblockcutter.NewReceiver(),
  1260  			ChainIDVal:     mockChannel.topic(),
  1261  			HeightVal:      lastCutBlockNumber, // Incremented during the WriteBlock call
  1262  		}
  1263  		defer close(mockSupport.BlockCutterVal.Block)
  1264  
  1265  		bareMinimumChain := &chainImpl{
  1266  			parentConsumer:  mockParentConsumer,
  1267  			channelConsumer: mockChannelConsumer,
  1268  
  1269  			channel:            mockChannel,
  1270  			support:            mockSupport,
  1271  			lastCutBlockNumber: lastCutBlockNumber,
  1272  
  1273  			errorChan: errorChan,
  1274  			haltChan:  haltChan,
  1275  		}
  1276  
  1277  		var counts []uint64
  1278  		done := make(chan struct{})
  1279  
  1280  		go func() {
  1281  			counts, err = bareMinimumChain.processMessagesToBlocks()
  1282  			done <- struct{}{}
  1283  		}()
  1284  
  1285  		// This is the wrappedMessage that the for-loop will process
  1286  		mpc.YieldMessage(newMockConsumerMessage(newTimeToCutMessage(lastCutBlockNumber + 1)))
  1287  
  1288  		logger.Debug("Closing haltChan to exit the infinite for-loop")
  1289  		close(haltChan) // Identical to chain.Halt()
  1290  		logger.Debug("haltChan closed")
  1291  		<-done
  1292  
  1293  		assert.Error(t, err, "Expected the processMessagesToBlocks call to return an error")
  1294  		assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled")
  1295  		assert.Equal(t, uint64(1), counts[indexProcessTimeToCutError], "Expected 1 faulty TIMETOCUT message processed")
  1296  		assert.Equal(t, lastCutBlockNumber, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to stay the same")
  1297  	})
  1298  
  1299  	t.Run("ReceiveTimeToCutLargerThanExpected", func(t *testing.T) {
  1300  		errorChan := make(chan struct{})
  1301  		close(errorChan)
  1302  		haltChan := make(chan struct{})
  1303  
  1304  		lastCutBlockNumber := uint64(3)
  1305  
  1306  		mockSupport := &mockmultichain.ConsenterSupport{
  1307  			Blocks:         make(chan *cb.Block), // WriteBlock will post here
  1308  			BlockCutterVal: mockblockcutter.NewReceiver(),
  1309  			ChainIDVal:     mockChannel.topic(),
  1310  			HeightVal:      lastCutBlockNumber, // Incremented during the WriteBlock call
  1311  		}
  1312  		defer close(mockSupport.BlockCutterVal.Block)
  1313  
  1314  		bareMinimumChain := &chainImpl{
  1315  			parentConsumer:  mockParentConsumer,
  1316  			channelConsumer: mockChannelConsumer,
  1317  
  1318  			channel:            mockChannel,
  1319  			support:            mockSupport,
  1320  			lastCutBlockNumber: lastCutBlockNumber,
  1321  
  1322  			errorChan: errorChan,
  1323  			haltChan:  haltChan,
  1324  		}
  1325  
  1326  		var counts []uint64
  1327  		done := make(chan struct{})
  1328  
  1329  		go func() {
  1330  			counts, err = bareMinimumChain.processMessagesToBlocks()
  1331  			done <- struct{}{}
  1332  		}()
  1333  
  1334  		// This is the wrappedMessage that the for-loop will process
  1335  		mpc.YieldMessage(newMockConsumerMessage(newTimeToCutMessage(lastCutBlockNumber + 2)))
  1336  
  1337  		logger.Debug("Closing haltChan to exit the infinite for-loop")
  1338  		close(haltChan) // Identical to chain.Halt()
  1339  		logger.Debug("haltChan closed")
  1340  		<-done
  1341  
  1342  		assert.Error(t, err, "Expected the processMessagesToBlocks call to return an error")
  1343  		assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled")
  1344  		assert.Equal(t, uint64(1), counts[indexProcessTimeToCutError], "Expected 1 faulty TIMETOCUT message processed")
  1345  		assert.Equal(t, lastCutBlockNumber, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to stay the same")
  1346  	})
  1347  
  1348  	t.Run("ReceiveTimeToCutStale", func(t *testing.T) {
  1349  		errorChan := make(chan struct{})
  1350  		close(errorChan)
  1351  		haltChan := make(chan struct{})
  1352  
  1353  		lastCutBlockNumber := uint64(3)
  1354  
  1355  		mockSupport := &mockmultichain.ConsenterSupport{
  1356  			Blocks:         make(chan *cb.Block), // WriteBlock will post here
  1357  			BlockCutterVal: mockblockcutter.NewReceiver(),
  1358  			ChainIDVal:     mockChannel.topic(),
  1359  			HeightVal:      lastCutBlockNumber, // Incremented during the WriteBlock call
  1360  		}
  1361  		defer close(mockSupport.BlockCutterVal.Block)
  1362  
  1363  		bareMinimumChain := &chainImpl{
  1364  			parentConsumer:  mockParentConsumer,
  1365  			channelConsumer: mockChannelConsumer,
  1366  
  1367  			channel:            mockChannel,
  1368  			support:            mockSupport,
  1369  			lastCutBlockNumber: lastCutBlockNumber,
  1370  
  1371  			errorChan: errorChan,
  1372  			haltChan:  haltChan,
  1373  		}
  1374  
  1375  		var counts []uint64
  1376  		done := make(chan struct{})
  1377  
  1378  		go func() {
  1379  			counts, err = bareMinimumChain.processMessagesToBlocks()
  1380  			done <- struct{}{}
  1381  		}()
  1382  
  1383  		// This is the wrappedMessage that the for-loop will process
  1384  		mpc.YieldMessage(newMockConsumerMessage(newTimeToCutMessage(lastCutBlockNumber)))
  1385  
  1386  		logger.Debug("Closing haltChan to exit the infinite for-loop")
  1387  		close(haltChan) // Identical to chain.Halt()
  1388  		logger.Debug("haltChan closed")
  1389  		<-done
  1390  
  1391  		assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors")
  1392  		assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled")
  1393  		assert.Equal(t, uint64(1), counts[indexProcessTimeToCutPass], "Expected 1 TIMETOCUT message processed")
  1394  		assert.Equal(t, lastCutBlockNumber, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to stay the same")
  1395  	})
  1396  
  1397  	t.Run("ReceiveKafkaErrorAndCloseErrorChan", func(t *testing.T) {
  1398  		// If we set up the mock broker so that it returns a response, if the
  1399  		// test finishes before the sendConnectMessage goroutine has received
  1400  		// this response, we will get a failure ("not all expectations were
  1401  		// satisfied") from the mock broker. So we sabotage the producer.
  1402  		failedProducer, _ := sarama.NewSyncProducer([]string{}, mockBrokerConfig)
  1403  
  1404  		// We need to have the sendConnectMessage goroutine die instantaneously,
  1405  		// otherwise we'll get a nil pointer dereference panic. We are
  1406  		// exploiting the admittedly hacky shortcut where a retriable process
  1407  		// returns immediately when given the nil time.Duration value for its
  1408  		// ticker.
  1409  		zeroRetryConsenter := &consenterImpl{}
  1410  
  1411  		// Let's assume an open errorChan, i.e. a healthy link between the
  1412  		// consumer and the Kafka partition corresponding to the channel
  1413  		errorChan := make(chan struct{})
  1414  
  1415  		haltChan := make(chan struct{})
  1416  
  1417  		mockSupport := &mockmultichain.ConsenterSupport{
  1418  			ChainIDVal: mockChannel.topic(),
  1419  		}
  1420  
  1421  		bareMinimumChain := &chainImpl{
  1422  			consenter:       zeroRetryConsenter, // For sendConnectMessage
  1423  			producer:        failedProducer,     // For sendConnectMessage
  1424  			parentConsumer:  mockParentConsumer,
  1425  			channelConsumer: mockChannelConsumer,
  1426  
  1427  			channel: mockChannel,
  1428  			support: mockSupport,
  1429  
  1430  			errorChan: errorChan,
  1431  			haltChan:  haltChan,
  1432  		}
  1433  
  1434  		var counts []uint64
  1435  		done := make(chan struct{})
  1436  
  1437  		go func() {
  1438  			counts, err = bareMinimumChain.processMessagesToBlocks()
  1439  			done <- struct{}{}
  1440  		}()
  1441  
  1442  		// This is what the for-loop will process
  1443  		mpc.YieldError(fmt.Errorf("fooError"))
  1444  
  1445  		logger.Debug("Closing haltChan to exit the infinite for-loop")
  1446  		close(haltChan) // Identical to chain.Halt()
  1447  		logger.Debug("haltChan closed")
  1448  		<-done
  1449  
  1450  		assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors")
  1451  		assert.Equal(t, uint64(1), counts[indexRecvError], "Expected 1 Kafka error received")
  1452  
  1453  		select {
  1454  		case <-bareMinimumChain.errorChan:
  1455  			logger.Debug("errorChan is closed as it should be")
  1456  		default:
  1457  			t.Fatal("errorChan should have been closed")
  1458  		}
  1459  	})
  1460  
  1461  	t.Run("ReceiveKafkaErrorAndThenReceiveRegularMessage", func(t *testing.T) {
  1462  		t.Skip("Skipping test as it introduces a race condition")
  1463  
  1464  		// If we set up the mock broker so that it returns a response, if the
  1465  		// test finishes before the sendConnectMessage goroutine has received
  1466  		// this response, we will get a failure ("not all expectations were
  1467  		// satisfied") from the mock broker. So we sabotage the producer.
  1468  		failedProducer, _ := sarama.NewSyncProducer([]string{}, mockBrokerConfig)
  1469  
  1470  		// We need to have the sendConnectMessage goroutine die instantaneously,
  1471  		// otherwise we'll get a nil pointer dereference panic. We are
  1472  		// exploiting the admittedly hacky shortcut where a retriable process
  1473  		// returns immediately when given the nil time.Duration value for its
  1474  		// ticker.
  1475  		zeroRetryConsenter := &consenterImpl{}
  1476  
  1477  		// If the errorChan is closed already, the kafkaErr branch shouldn't
  1478  		// touch it
  1479  		errorChan := make(chan struct{})
  1480  		close(errorChan)
  1481  
  1482  		haltChan := make(chan struct{})
  1483  
  1484  		mockSupport := &mockmultichain.ConsenterSupport{
  1485  			ChainIDVal: mockChannel.topic(),
  1486  		}
  1487  
  1488  		bareMinimumChain := &chainImpl{
  1489  			consenter:       zeroRetryConsenter, // For sendConnectMessage
  1490  			producer:        failedProducer,     // For sendConnectMessage
  1491  			parentConsumer:  mockParentConsumer,
  1492  			channelConsumer: mockChannelConsumer,
  1493  
  1494  			channel: mockChannel,
  1495  			support: mockSupport,
  1496  
  1497  			errorChan: errorChan,
  1498  			haltChan:  haltChan,
  1499  		}
  1500  
  1501  		var counts []uint64
  1502  		done := make(chan struct{})
  1503  
  1504  		go func() {
  1505  			counts, err = bareMinimumChain.processMessagesToBlocks()
  1506  			done <- struct{}{}
  1507  		}()
  1508  
  1509  		// This is what the for-loop will process
  1510  		mpc.YieldError(fmt.Errorf("foo"))
  1511  
  1512  		// We tested this in ReceiveKafkaErrorAndCloseErrorChan, so this check
  1513  		// is redundant in that regard. We use it however to ensure the
  1514  		// kafkaErrBranch has been activated before proceeding with pushing the
  1515  		// regular message.
  1516  		select {
  1517  		case <-bareMinimumChain.errorChan:
  1518  			logger.Debug("errorChan is closed as it should be")
  1519  		case <-time.After(shortTimeout):
  1520  			t.Fatal("errorChan should have been closed by now")
  1521  		}
  1522  
  1523  		// This is the wrappedMessage that the for-loop will process. We use
  1524  		// a broken regular message here on purpose since this is the shortest
  1525  		// path and it allows us to test what we want.
  1526  		mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(tamperBytes(utils.MarshalOrPanic(newMockEnvelope("fooMessage"))))))
  1527  
  1528  		// Sleep so that the Messages/errorChan branch is activated.
  1529  		// TODO Hacky approach, will need to revise eventually
  1530  		time.Sleep(hitBranch)
  1531  
  1532  		// Check that the errorChan was recreated
  1533  		select {
  1534  		case <-bareMinimumChain.errorChan:
  1535  			t.Fatal("errorChan should have been open")
  1536  		default:
  1537  			logger.Debug("errorChan is open as it should be")
  1538  		}
  1539  
  1540  		logger.Debug("Closing haltChan to exit the infinite for-loop")
  1541  		close(haltChan) // Identical to chain.Halt()
  1542  		logger.Debug("haltChan closed")
  1543  		<-done
  1544  	})
  1545  }