github.com/tenywen/fabric@v1.0.0-beta.0.20170620030522-a5b1ed380643/orderer/kafka/chain_test.go (about)

     1  /*
     2  Copyright IBM Corp. All Rights Reserved.
     3  
     4  SPDX-License-Identifier: Apache-2.0
     5  */
     6  
     7  package kafka
     8  
     9  import (
    10  	"fmt"
    11  	"testing"
    12  	"time"
    13  
    14  	"github.com/Shopify/sarama"
    15  	"github.com/Shopify/sarama/mocks"
    16  	mockconfig "github.com/hyperledger/fabric/common/mocks/config"
    17  	mockblockcutter "github.com/hyperledger/fabric/orderer/mocks/blockcutter"
    18  	mockmultichain "github.com/hyperledger/fabric/orderer/mocks/multichain"
    19  	cb "github.com/hyperledger/fabric/protos/common"
    20  	ab "github.com/hyperledger/fabric/protos/orderer"
    21  	"github.com/hyperledger/fabric/protos/utils"
    22  	"github.com/stretchr/testify/assert"
    23  )
    24  
    25  var (
    26  	extraShortTimeout = 1 * time.Millisecond
    27  	shortTimeout      = 1 * time.Second
    28  	longTimeout       = 1 * time.Hour
    29  
    30  	hitBranch = 50 * time.Millisecond
    31  )
    32  
    33  func TestChain(t *testing.T) {
    34  
    35  	oldestOffset := int64(0)
    36  	newestOffset := int64(5)
    37  
    38  	message := sarama.StringEncoder("messageFoo")
    39  
    40  	newMocks := func(t *testing.T) (mockChannel channel, mockBroker *sarama.MockBroker, mockSupport *mockmultichain.ConsenterSupport) {
    41  		mockChannel = newChannel(channelNameForTest(t), defaultPartition)
    42  		mockBroker = sarama.NewMockBroker(t, 0)
    43  		mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{
    44  			"MetadataRequest": sarama.NewMockMetadataResponse(t).
    45  				SetBroker(mockBroker.Addr(), mockBroker.BrokerID()).
    46  				SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()),
    47  			"ProduceRequest": sarama.NewMockProduceResponse(t).
    48  				SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError),
    49  			"OffsetRequest": sarama.NewMockOffsetResponse(t).
    50  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset).
    51  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset),
    52  			"FetchRequest": sarama.NewMockFetchResponse(t, 1).
    53  				SetMessage(mockChannel.topic(), mockChannel.partition(), newestOffset, message),
    54  		})
    55  		mockSupport = &mockmultichain.ConsenterSupport{
    56  			ChainIDVal:      mockChannel.topic(),
    57  			HeightVal:       uint64(3),
    58  			SharedConfigVal: &mockconfig.Orderer{KafkaBrokersVal: []string{mockBroker.Addr()}},
    59  		}
    60  		return
    61  	}
    62  
    63  	t.Run("New", func(t *testing.T) {
    64  		_, mockBroker, mockSupport := newMocks(t)
    65  		defer func() { mockBroker.Close() }()
    66  		chain, err := newChain(mockConsenter, mockSupport, newestOffset-1)
    67  
    68  		assert.NoError(t, err, "Expected newChain to return without errors")
    69  		select {
    70  		case <-chain.errorChan:
    71  			logger.Debug("errorChan is closed as it should be")
    72  		default:
    73  			t.Fatal("errorChan should have been closed")
    74  		}
    75  
    76  		select {
    77  		case <-chain.haltChan:
    78  			t.Fatal("haltChan should have been open")
    79  		default:
    80  			logger.Debug("haltChan is open as it should be")
    81  		}
    82  
    83  		select {
    84  		case <-chain.startChan:
    85  			t.Fatal("startChan should have been open")
    86  		default:
    87  			logger.Debug("startChan is open as it should be")
    88  		}
    89  	})
    90  
    91  	t.Run("Start", func(t *testing.T) {
    92  		_, mockBroker, mockSupport := newMocks(t)
    93  		defer func() { mockBroker.Close() }()
    94  		// Set to -1 because we haven't sent the CONNECT message yet
    95  		chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1)
    96  
    97  		chain.Start()
    98  		select {
    99  		case <-chain.startChan:
   100  			logger.Debug("startChan is closed as it should be")
   101  		case <-time.After(shortTimeout):
   102  			t.Fatal("startChan should have been closed by now")
   103  		}
   104  
   105  		// Trigger the haltChan clause in the processMessagesToBlocks goroutine
   106  		close(chain.haltChan)
   107  	})
   108  
   109  	t.Run("Halt", func(t *testing.T) {
   110  		_, mockBroker, mockSupport := newMocks(t)
   111  		defer func() { mockBroker.Close() }()
   112  		chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1)
   113  
   114  		chain.Start()
   115  		select {
   116  		case <-chain.startChan:
   117  			logger.Debug("startChan is closed as it should be")
   118  		case <-time.After(shortTimeout):
   119  			t.Fatal("startChan should have been closed by now")
   120  		}
   121  
   122  		// Wait till the start phase has completed, then:
   123  		chain.Halt()
   124  
   125  		select {
   126  		case <-chain.haltChan:
   127  			logger.Debug("haltChan is closed as it should be")
   128  		case <-time.After(shortTimeout):
   129  			t.Fatal("haltChan should have been closed")
   130  		}
   131  
   132  		select {
   133  		case <-chain.errorChan:
   134  			logger.Debug("errorChan is closed as it should be")
   135  		case <-time.After(shortTimeout):
   136  			t.Fatal("errorChan should have been closed")
   137  		}
   138  	})
   139  
   140  	t.Run("DoubleHalt", func(t *testing.T) {
   141  		_, mockBroker, mockSupport := newMocks(t)
   142  		defer func() { mockBroker.Close() }()
   143  		chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1)
   144  
   145  		chain.Start()
   146  		select {
   147  		case <-chain.startChan:
   148  			logger.Debug("startChan is closed as it should be")
   149  		case <-time.After(shortTimeout):
   150  			t.Fatal("startChan should have been closed by now")
   151  		}
   152  
   153  		chain.Halt()
   154  
   155  		assert.NotPanics(t, func() { chain.Halt() }, "Calling Halt() more than once shouldn't panic")
   156  	})
   157  
   158  	t.Run("StartWithProducerForChannelError", func(t *testing.T) {
   159  		_, mockBroker, mockSupport := newMocks(t)
   160  		defer func() { mockBroker.Close() }()
   161  		// Point to an empty brokers list
   162  		mockSupportCopy := *mockSupport
   163  		mockSupportCopy.SharedConfigVal = &mockconfig.Orderer{KafkaBrokersVal: []string{}}
   164  
   165  		chain, _ := newChain(mockConsenter, &mockSupportCopy, newestOffset-1)
   166  
   167  		// The production path will actually call chain.Start(). This is
   168  		// functionally equivalent and allows us to run assertions on it.
   169  		assert.Panics(t, func() { startThread(chain) }, "Expected the Start() call to panic")
   170  	})
   171  
   172  	t.Run("StartWithConnectMessageError", func(t *testing.T) {
   173  		// Note that this test is affected by the following parameters:
   174  		// - Net.ReadTimeout
   175  		// - Consumer.Retry.Backoff
   176  		// - Metadata.Retry.Max
   177  		mockChannel, mockBroker, mockSupport := newMocks(t)
   178  		defer func() { mockBroker.Close() }()
   179  		chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1)
   180  
   181  		// Have the broker return an ErrNotLeaderForPartition error
   182  		mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{
   183  			"MetadataRequest": sarama.NewMockMetadataResponse(t).
   184  				SetBroker(mockBroker.Addr(), mockBroker.BrokerID()).
   185  				SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()),
   186  			"ProduceRequest": sarama.NewMockProduceResponse(t).
   187  				SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNotLeaderForPartition),
   188  			"OffsetRequest": sarama.NewMockOffsetResponse(t).
   189  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset).
   190  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset),
   191  			"FetchRequest": sarama.NewMockFetchResponse(t, 1).
   192  				SetMessage(mockChannel.topic(), mockChannel.partition(), newestOffset, message),
   193  		})
   194  
   195  		assert.Panics(t, func() { startThread(chain) }, "Expected the Start() call to panic")
   196  	})
   197  
   198  	t.Run("EnqueueIfNotStarted", func(t *testing.T) {
   199  		mockChannel, mockBroker, mockSupport := newMocks(t)
   200  		defer func() { mockBroker.Close() }()
   201  		chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1)
   202  
   203  		// As in StartWithConnectMessageError, have the broker return an
   204  		// ErrNotLeaderForPartition error, i.e. cause an error in the
   205  		// 'post connect message' step.
   206  		mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{
   207  			"MetadataRequest": sarama.NewMockMetadataResponse(t).
   208  				SetBroker(mockBroker.Addr(), mockBroker.BrokerID()).
   209  				SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()),
   210  			"ProduceRequest": sarama.NewMockProduceResponse(t).
   211  				SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNotLeaderForPartition),
   212  			"OffsetRequest": sarama.NewMockOffsetResponse(t).
   213  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset).
   214  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset),
   215  			"FetchRequest": sarama.NewMockFetchResponse(t, 1).
   216  				SetMessage(mockChannel.topic(), mockChannel.partition(), newestOffset, message),
   217  		})
   218  
   219  		assert.False(t, chain.Enqueue(newMockEnvelope("fooMessage")), "Expected Enqueue call to return false")
   220  	})
   221  
   222  	t.Run("StartWithConsumerForChannelError", func(t *testing.T) {
   223  		// Note that this test is affected by the following parameters:
   224  		// - Net.ReadTimeout
   225  		// - Consumer.Retry.Backoff
   226  		// - Metadata.Retry.Max
   227  
   228  		mockChannel, mockBroker, mockSupport := newMocks(t)
   229  		defer func() { mockBroker.Close() }()
   230  
   231  		// Provide an out-of-range offset
   232  		chain, _ := newChain(mockConsenter, mockSupport, newestOffset)
   233  
   234  		mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{
   235  			"MetadataRequest": sarama.NewMockMetadataResponse(t).
   236  				SetBroker(mockBroker.Addr(), mockBroker.BrokerID()).
   237  				SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()),
   238  			"ProduceRequest": sarama.NewMockProduceResponse(t).
   239  				SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError),
   240  			"OffsetRequest": sarama.NewMockOffsetResponse(t).
   241  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset).
   242  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset),
   243  			"FetchRequest": sarama.NewMockFetchResponse(t, 1).
   244  				SetMessage(mockChannel.topic(), mockChannel.partition(), newestOffset, message),
   245  		})
   246  
   247  		assert.Panics(t, func() { startThread(chain) }, "Expected the Start() call to panic")
   248  	})
   249  
   250  	t.Run("EnqueueProper", func(t *testing.T) {
   251  		mockChannel, mockBroker, mockSupport := newMocks(t)
   252  		defer func() { mockBroker.Close() }()
   253  		chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1)
   254  
   255  		mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{
   256  			"MetadataRequest": sarama.NewMockMetadataResponse(t).
   257  				SetBroker(mockBroker.Addr(), mockBroker.BrokerID()).
   258  				SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()),
   259  			"ProduceRequest": sarama.NewMockProduceResponse(t).
   260  				SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError),
   261  			"OffsetRequest": sarama.NewMockOffsetResponse(t).
   262  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset).
   263  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset),
   264  			"FetchRequest": sarama.NewMockFetchResponse(t, 1).
   265  				SetMessage(mockChannel.topic(), mockChannel.partition(), newestOffset, message),
   266  		})
   267  
   268  		chain.Start()
   269  		select {
   270  		case <-chain.startChan:
   271  			logger.Debug("startChan is closed as it should be")
   272  		case <-time.After(shortTimeout):
   273  			t.Fatal("startChan should have been closed by now")
   274  		}
   275  
   276  		// Enqueue should have access to the post path, and its ProduceRequest
   277  		// should go by without error
   278  		assert.True(t, chain.Enqueue(newMockEnvelope("fooMessage")), "Expected Enqueue call to return true")
   279  
   280  		chain.Halt()
   281  	})
   282  
   283  	t.Run("EnqueueIfHalted", func(t *testing.T) {
   284  		mockChannel, mockBroker, mockSupport := newMocks(t)
   285  		defer func() { mockBroker.Close() }()
   286  		chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1)
   287  
   288  		mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{
   289  			"MetadataRequest": sarama.NewMockMetadataResponse(t).
   290  				SetBroker(mockBroker.Addr(), mockBroker.BrokerID()).
   291  				SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()),
   292  			"ProduceRequest": sarama.NewMockProduceResponse(t).
   293  				SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError),
   294  			"OffsetRequest": sarama.NewMockOffsetResponse(t).
   295  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset).
   296  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset),
   297  			"FetchRequest": sarama.NewMockFetchResponse(t, 1).
   298  				SetMessage(mockChannel.topic(), mockChannel.partition(), newestOffset, message),
   299  		})
   300  
   301  		chain.Start()
   302  		select {
   303  		case <-chain.startChan:
   304  			logger.Debug("startChan is closed as it should be")
   305  		case <-time.After(shortTimeout):
   306  			t.Fatal("startChan should have been closed by now")
   307  		}
   308  		chain.Halt()
   309  
   310  		// haltChan should close access to the post path
   311  		assert.False(t, chain.Enqueue(newMockEnvelope("fooMessage")), "Expected Enqueue call to return false")
   312  	})
   313  
   314  	t.Run("EnqueueError", func(t *testing.T) {
   315  		mockChannel, mockBroker, mockSupport := newMocks(t)
   316  		defer func() { mockBroker.Close() }()
   317  		chain, _ := newChain(mockConsenter, mockSupport, newestOffset-1)
   318  
   319  		// Use the "good" handler map that allows the Stage to complete without
   320  		// issues
   321  		mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{
   322  			"MetadataRequest": sarama.NewMockMetadataResponse(t).
   323  				SetBroker(mockBroker.Addr(), mockBroker.BrokerID()).
   324  				SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()),
   325  			"ProduceRequest": sarama.NewMockProduceResponse(t).
   326  				SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError),
   327  			"OffsetRequest": sarama.NewMockOffsetResponse(t).
   328  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset).
   329  				SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset),
   330  			"FetchRequest": sarama.NewMockFetchResponse(t, 1).
   331  				SetMessage(mockChannel.topic(), mockChannel.partition(), newestOffset, message),
   332  		})
   333  
   334  		chain.Start()
   335  		select {
   336  		case <-chain.startChan:
   337  			logger.Debug("startChan is closed as it should be")
   338  		case <-time.After(shortTimeout):
   339  			t.Fatal("startChan should have been closed by now")
   340  		}
   341  
   342  		// Now make it so that the next ProduceRequest is met with an error
   343  		mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{
   344  			"ProduceRequest": sarama.NewMockProduceResponse(t).
   345  				SetError(mockChannel.topic(), mockChannel.partition(), sarama.ErrNotLeaderForPartition),
   346  		})
   347  
   348  		assert.False(t, chain.Enqueue(newMockEnvelope("fooMessage")), "Expected Enqueue call to return false")
   349  	})
   350  }
   351  
   352  func TestSetupProducerForChannel(t *testing.T) {
   353  	if testing.Short() {
   354  		t.Skip("Skipping test in short mode")
   355  	}
   356  
   357  	mockBroker := sarama.NewMockBroker(t, 0)
   358  	defer mockBroker.Close()
   359  
   360  	mockChannel := newChannel(channelNameForTest(t), defaultPartition)
   361  
   362  	haltChan := make(chan struct{})
   363  
   364  	t.Run("Proper", func(t *testing.T) {
   365  		metadataResponse := new(sarama.MetadataResponse)
   366  		metadataResponse.AddBroker(mockBroker.Addr(), mockBroker.BrokerID())
   367  		metadataResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID(), nil, nil, sarama.ErrNoError)
   368  		mockBroker.Returns(metadataResponse)
   369  
   370  		producer, err := setupProducerForChannel(mockConsenter.retryOptions(), haltChan, []string{mockBroker.Addr()}, mockBrokerConfig, mockChannel)
   371  		assert.NoError(t, err, "Expected the setupProducerForChannel call to return without errors")
   372  		assert.NoError(t, producer.Close(), "Expected to close the producer without errors")
   373  	})
   374  
   375  	t.Run("WithError", func(t *testing.T) {
   376  		_, err := setupProducerForChannel(mockConsenter.retryOptions(), haltChan, []string{}, mockBrokerConfig, mockChannel)
   377  		assert.Error(t, err, "Expected the setupProducerForChannel call to return an error")
   378  	})
   379  }
   380  
   381  func TestSetupConsumerForChannel(t *testing.T) {
   382  	mockBroker := sarama.NewMockBroker(t, 0)
   383  	defer func() { mockBroker.Close() }()
   384  
   385  	mockChannel := newChannel(channelNameForTest(t), defaultPartition)
   386  
   387  	oldestOffset := int64(0)
   388  	newestOffset := int64(5)
   389  
   390  	startFrom := int64(3)
   391  	message := sarama.StringEncoder("messageFoo")
   392  
   393  	mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{
   394  		"MetadataRequest": sarama.NewMockMetadataResponse(t).
   395  			SetBroker(mockBroker.Addr(), mockBroker.BrokerID()).
   396  			SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()),
   397  		"OffsetRequest": sarama.NewMockOffsetResponse(t).
   398  			SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset).
   399  			SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset),
   400  		"FetchRequest": sarama.NewMockFetchResponse(t, 1).
   401  			SetMessage(mockChannel.topic(), mockChannel.partition(), startFrom, message),
   402  	})
   403  
   404  	haltChan := make(chan struct{})
   405  
   406  	t.Run("ProperParent", func(t *testing.T) {
   407  		parentConsumer, err := setupParentConsumerForChannel(mockConsenter.retryOptions(), haltChan, []string{mockBroker.Addr()}, mockBrokerConfig, mockChannel)
   408  		assert.NoError(t, err, "Expected the setupParentConsumerForChannel call to return without errors")
   409  		assert.NoError(t, parentConsumer.Close(), "Expected to close the parentConsumer without errors")
   410  	})
   411  
   412  	t.Run("ProperChannel", func(t *testing.T) {
   413  		parentConsumer, _ := setupParentConsumerForChannel(mockConsenter.retryOptions(), haltChan, []string{mockBroker.Addr()}, mockBrokerConfig, mockChannel)
   414  		defer func() { parentConsumer.Close() }()
   415  		channelConsumer, err := setupChannelConsumerForChannel(mockConsenter.retryOptions(), haltChan, parentConsumer, mockChannel, newestOffset)
   416  		assert.NoError(t, err, "Expected the setupChannelConsumerForChannel call to return without errors")
   417  		assert.NoError(t, channelConsumer.Close(), "Expected to close the channelConsumer without errors")
   418  	})
   419  
   420  	t.Run("WithParentConsumerError", func(t *testing.T) {
   421  		// Provide an empty brokers list
   422  		_, err := setupParentConsumerForChannel(mockConsenter.retryOptions(), haltChan, []string{}, mockBrokerConfig, mockChannel)
   423  		assert.Error(t, err, "Expected the setupParentConsumerForChannel call to return an error")
   424  	})
   425  
   426  	t.Run("WithChannelConsumerError", func(t *testing.T) {
   427  		// Provide an out-of-range offset
   428  		parentConsumer, _ := setupParentConsumerForChannel(mockConsenter.retryOptions(), haltChan, []string{mockBroker.Addr()}, mockBrokerConfig, mockChannel)
   429  		_, err := setupChannelConsumerForChannel(mockConsenter.retryOptions(), haltChan, parentConsumer, mockChannel, newestOffset+1)
   430  		defer func() { parentConsumer.Close() }()
   431  		assert.Error(t, err, "Expected the setupChannelConsumerForChannel call to return an error")
   432  	})
   433  }
   434  
   435  func TestCloseKafkaObjects(t *testing.T) {
   436  	mockChannel := newChannel(channelNameForTest(t), defaultPartition)
   437  
   438  	mockSupport := &mockmultichain.ConsenterSupport{
   439  		ChainIDVal: mockChannel.topic(),
   440  	}
   441  
   442  	oldestOffset := int64(0)
   443  	newestOffset := int64(5)
   444  
   445  	startFrom := int64(3)
   446  	message := sarama.StringEncoder("messageFoo")
   447  
   448  	mockBroker := sarama.NewMockBroker(t, 0)
   449  	defer func() { mockBroker.Close() }()
   450  
   451  	mockBroker.SetHandlerByMap(map[string]sarama.MockResponse{
   452  		"MetadataRequest": sarama.NewMockMetadataResponse(t).
   453  			SetBroker(mockBroker.Addr(), mockBroker.BrokerID()).
   454  			SetLeader(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID()),
   455  		"OffsetRequest": sarama.NewMockOffsetResponse(t).
   456  			SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetOldest, oldestOffset).
   457  			SetOffset(mockChannel.topic(), mockChannel.partition(), sarama.OffsetNewest, newestOffset),
   458  		"FetchRequest": sarama.NewMockFetchResponse(t, 1).
   459  			SetMessage(mockChannel.topic(), mockChannel.partition(), startFrom, message),
   460  	})
   461  
   462  	haltChan := make(chan struct{})
   463  
   464  	t.Run("Proper", func(t *testing.T) {
   465  		producer, _ := setupProducerForChannel(mockConsenter.retryOptions(), haltChan, []string{mockBroker.Addr()}, mockBrokerConfig, mockChannel)
   466  		parentConsumer, _ := setupParentConsumerForChannel(mockConsenter.retryOptions(), haltChan, []string{mockBroker.Addr()}, mockBrokerConfig, mockChannel)
   467  		channelConsumer, _ := setupChannelConsumerForChannel(mockConsenter.retryOptions(), haltChan, parentConsumer, mockChannel, startFrom)
   468  
   469  		// Set up a chain with just the minimum necessary fields instantiated so
   470  		// as to test the function
   471  		bareMinimumChain := &chainImpl{
   472  			support:         mockSupport,
   473  			producer:        producer,
   474  			parentConsumer:  parentConsumer,
   475  			channelConsumer: channelConsumer,
   476  		}
   477  
   478  		errs := bareMinimumChain.closeKafkaObjects()
   479  
   480  		assert.Len(t, errs, 0, "Expected zero errors")
   481  
   482  		assert.Panics(t, func() {
   483  			channelConsumer.Close()
   484  		})
   485  
   486  		assert.NotPanics(t, func() {
   487  			parentConsumer.Close()
   488  		})
   489  
   490  		// TODO For some reason this panic cannot be captured by the `assert`
   491  		// test framework. Not a dealbreaker but need to investigate further.
   492  		/* assert.Panics(t, func() {
   493  			producer.Close()
   494  		}) */
   495  	})
   496  
   497  	t.Run("ChannelConsumerError", func(t *testing.T) {
   498  		producer, _ := sarama.NewSyncProducer([]string{mockBroker.Addr()}, mockBrokerConfig)
   499  
   500  		// Unlike all other tests in this file, forcing an error on the
   501  		// channelConsumer.Close() call is more easily achieved using the mock
   502  		// Consumer. Thus we bypass the call to `setup*Consumer`.
   503  
   504  		// Have the consumer receive an ErrOutOfBrokers error.
   505  		mockParentConsumer := mocks.NewConsumer(t, nil)
   506  		mockParentConsumer.ExpectConsumePartition(mockChannel.topic(), mockChannel.partition(), startFrom).YieldError(sarama.ErrOutOfBrokers)
   507  		mockChannelConsumer, err := mockParentConsumer.ConsumePartition(mockChannel.topic(), mockChannel.partition(), startFrom)
   508  		assert.NoError(t, err, "Expected no error when setting up the mock partition consumer")
   509  
   510  		bareMinimumChain := &chainImpl{
   511  			support:         mockSupport,
   512  			producer:        producer,
   513  			parentConsumer:  mockParentConsumer,
   514  			channelConsumer: mockChannelConsumer,
   515  		}
   516  
   517  		errs := bareMinimumChain.closeKafkaObjects()
   518  
   519  		assert.Len(t, errs, 1, "Expected 1 error returned")
   520  
   521  		assert.NotPanics(t, func() {
   522  			mockChannelConsumer.Close()
   523  		})
   524  
   525  		assert.NotPanics(t, func() {
   526  			mockParentConsumer.Close()
   527  		})
   528  	})
   529  }
   530  
   531  // Test helper functions here.
   532  
   533  func TestGetLastCutBlockNumber(t *testing.T) {
   534  	testCases := []struct {
   535  		name     string
   536  		input    uint64
   537  		expected uint64
   538  	}{
   539  		{"Proper", uint64(2), uint64(1)},
   540  		{"Zero", uint64(1), uint64(0)},
   541  	}
   542  	for _, tc := range testCases {
   543  		t.Run(tc.name, func(t *testing.T) {
   544  			assert.Equal(t, tc.expected, getLastCutBlockNumber(tc.input))
   545  		})
   546  	}
   547  }
   548  
   549  func TestGetLastOffsetPersisted(t *testing.T) {
   550  	mockChannel := newChannel(channelNameForTest(t), defaultPartition)
   551  	mockMetadata := &cb.Metadata{Value: utils.MarshalOrPanic(&ab.KafkaMetadata{LastOffsetPersisted: int64(5)})}
   552  
   553  	testCases := []struct {
   554  		name     string
   555  		md       []byte
   556  		expected int64
   557  		panics   bool
   558  	}{
   559  		{"Proper", mockMetadata.Value, int64(5), false},
   560  		{"Empty", nil, sarama.OffsetOldest - 1, false},
   561  		{"Panics", tamperBytes(mockMetadata.Value), sarama.OffsetOldest - 1, true},
   562  	}
   563  
   564  	for _, tc := range testCases {
   565  		t.Run(tc.name, func(t *testing.T) {
   566  			if !tc.panics {
   567  				assert.Equal(t, tc.expected, getLastOffsetPersisted(tc.md, mockChannel.String()))
   568  			} else {
   569  				assert.Panics(t, func() {
   570  					getLastOffsetPersisted(tc.md, mockChannel.String())
   571  				}, "Expected getLastOffsetPersisted call to panic")
   572  			}
   573  		})
   574  	}
   575  }
   576  
   577  func TestSendConnectMessage(t *testing.T) {
   578  	mockBroker := sarama.NewMockBroker(t, 0)
   579  	defer func() { mockBroker.Close() }()
   580  
   581  	mockChannel := newChannel("mockChannelFoo", defaultPartition)
   582  
   583  	metadataResponse := new(sarama.MetadataResponse)
   584  	metadataResponse.AddBroker(mockBroker.Addr(), mockBroker.BrokerID())
   585  	metadataResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID(), nil, nil, sarama.ErrNoError)
   586  	mockBroker.Returns(metadataResponse)
   587  
   588  	producer, _ := sarama.NewSyncProducer([]string{mockBroker.Addr()}, mockBrokerConfig)
   589  	defer func() { producer.Close() }()
   590  
   591  	haltChan := make(chan struct{})
   592  
   593  	t.Run("Proper", func(t *testing.T) {
   594  		successResponse := new(sarama.ProduceResponse)
   595  		successResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError)
   596  		mockBroker.Returns(successResponse)
   597  
   598  		assert.NoError(t, sendConnectMessage(mockConsenter.retryOptions(), haltChan, producer, mockChannel), "Expected the sendConnectMessage call to return without errors")
   599  	})
   600  
   601  	t.Run("WithError", func(t *testing.T) {
   602  		// Note that this test is affected by the following parameters:
   603  		// - Net.ReadTimeout
   604  		// - Consumer.Retry.Backoff
   605  		// - Metadata.Retry.Max
   606  
   607  		// Have the broker return an ErrNotEnoughReplicas error
   608  		failureResponse := new(sarama.ProduceResponse)
   609  		failureResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), sarama.ErrNotEnoughReplicas)
   610  		mockBroker.Returns(failureResponse)
   611  
   612  		assert.Error(t, sendConnectMessage(mockConsenter.retryOptions(), haltChan, producer, mockChannel), "Expected the sendConnectMessage call to return an error")
   613  	})
   614  }
   615  
   616  func TestSendTimeToCut(t *testing.T) {
   617  	mockBroker := sarama.NewMockBroker(t, 0)
   618  	defer func() { mockBroker.Close() }()
   619  
   620  	mockChannel := newChannel("mockChannelFoo", defaultPartition)
   621  
   622  	metadataResponse := new(sarama.MetadataResponse)
   623  	metadataResponse.AddBroker(mockBroker.Addr(), mockBroker.BrokerID())
   624  	metadataResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID(), nil, nil, sarama.ErrNoError)
   625  	mockBroker.Returns(metadataResponse)
   626  
   627  	producer, err := sarama.NewSyncProducer([]string{mockBroker.Addr()}, mockBrokerConfig)
   628  	assert.NoError(t, err, "Expected no error when setting up the sarama SyncProducer")
   629  	defer func() { producer.Close() }()
   630  
   631  	timeToCutBlockNumber := uint64(3)
   632  	var timer <-chan time.Time
   633  
   634  	t.Run("Proper", func(t *testing.T) {
   635  		successResponse := new(sarama.ProduceResponse)
   636  		successResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError)
   637  		mockBroker.Returns(successResponse)
   638  
   639  		timer = time.After(longTimeout)
   640  
   641  		assert.NoError(t, sendTimeToCut(producer, mockChannel, timeToCutBlockNumber, &timer), "Expected the sendTimeToCut call to return without errors")
   642  		assert.Nil(t, timer, "Expected the sendTimeToCut call to nil the timer")
   643  	})
   644  
   645  	t.Run("WithError", func(t *testing.T) {
   646  		// Note that this test is affected by the following parameters:
   647  		// - Net.ReadTimeout
   648  		// - Consumer.Retry.Backoff
   649  		// - Metadata.Retry.Max
   650  		failureResponse := new(sarama.ProduceResponse)
   651  		failureResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), sarama.ErrNotEnoughReplicas)
   652  		mockBroker.Returns(failureResponse)
   653  
   654  		timer = time.After(longTimeout)
   655  
   656  		assert.Error(t, sendTimeToCut(producer, mockChannel, timeToCutBlockNumber, &timer), "Expected the sendTimeToCut call to return an error")
   657  		assert.Nil(t, timer, "Expected the sendTimeToCut call to nil the timer")
   658  	})
   659  }
   660  
   661  func TestProcessMessagesToBlocks(t *testing.T) {
   662  	subtestIndex := -1 // Used to calculate the right offset at each subtest
   663  
   664  	mockBroker := sarama.NewMockBroker(t, 0)
   665  	defer func() { mockBroker.Close() }()
   666  
   667  	mockChannel := newChannel("mockChannelFoo", defaultPartition)
   668  
   669  	metadataResponse := new(sarama.MetadataResponse)
   670  	metadataResponse.AddBroker(mockBroker.Addr(), mockBroker.BrokerID())
   671  	metadataResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), mockBroker.BrokerID(), nil, nil, sarama.ErrNoError)
   672  	mockBroker.Returns(metadataResponse)
   673  
   674  	producer, _ := sarama.NewSyncProducer([]string{mockBroker.Addr()}, mockBrokerConfig)
   675  
   676  	mockBrokerConfigCopy := *mockBrokerConfig
   677  	mockBrokerConfigCopy.ChannelBufferSize = 0
   678  
   679  	newestOffset := int64(0)
   680  
   681  	mockParentConsumer := mocks.NewConsumer(t, &mockBrokerConfigCopy)
   682  	mpc := mockParentConsumer.ExpectConsumePartition(mockChannel.topic(), mockChannel.partition(), newestOffset)
   683  	mockChannelConsumer, err := mockParentConsumer.ConsumePartition(mockChannel.topic(), mockChannel.partition(), newestOffset)
   684  	assert.NoError(t, err, "Expected no error when setting up the mock partition consumer")
   685  
   686  	t.Run("ReceiveConnect", func(t *testing.T) {
   687  		subtestIndex++
   688  
   689  		errorChan := make(chan struct{})
   690  		close(errorChan)
   691  		haltChan := make(chan struct{})
   692  
   693  		mockSupport := &mockmultichain.ConsenterSupport{
   694  			ChainIDVal: mockChannel.topic(),
   695  		}
   696  
   697  		bareMinimumChain := &chainImpl{
   698  			parentConsumer:  mockParentConsumer,
   699  			channelConsumer: mockChannelConsumer,
   700  
   701  			channel: mockChannel,
   702  			support: mockSupport,
   703  
   704  			errorChan: errorChan,
   705  			haltChan:  haltChan,
   706  		}
   707  
   708  		var counts []uint64
   709  		done := make(chan struct{})
   710  
   711  		go func() {
   712  			counts, err = bareMinimumChain.processMessagesToBlocks()
   713  			done <- struct{}{}
   714  		}()
   715  
   716  		// This is the wrappedMessage that the for-loop will process
   717  		mpc.YieldMessage(newMockConsumerMessage(newConnectMessage()))
   718  
   719  		logger.Debug("Closing haltChan to exit the infinite for-loop")
   720  		close(haltChan) // Identical to chain.Halt()
   721  		logger.Debug("haltChan closed")
   722  		<-done
   723  
   724  		assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors")
   725  		assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled")
   726  		assert.Equal(t, uint64(1), counts[indexProcessConnectPass], "Expected 1 CONNECT message processed")
   727  	})
   728  
   729  	t.Run("ReceiveRegularWithError", func(t *testing.T) {
   730  		subtestIndex++
   731  
   732  		errorChan := make(chan struct{})
   733  		close(errorChan)
   734  		haltChan := make(chan struct{})
   735  
   736  		mockSupport := &mockmultichain.ConsenterSupport{
   737  			ChainIDVal: mockChannel.topic(),
   738  		}
   739  
   740  		bareMinimumChain := &chainImpl{
   741  			parentConsumer:  mockParentConsumer,
   742  			channelConsumer: mockChannelConsumer,
   743  
   744  			channel: mockChannel,
   745  			support: mockSupport,
   746  
   747  			errorChan: errorChan,
   748  			haltChan:  haltChan,
   749  		}
   750  
   751  		var counts []uint64
   752  		done := make(chan struct{})
   753  
   754  		go func() {
   755  			counts, err = bareMinimumChain.processMessagesToBlocks()
   756  			done <- struct{}{}
   757  		}()
   758  
   759  		// This is the wrappedMessage that the for-loop will process
   760  		mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(tamperBytes(utils.MarshalOrPanic(newMockEnvelope("fooMessage"))))))
   761  
   762  		logger.Debug("Closing haltChan to exit the infinite for-loop")
   763  		close(haltChan) // Identical to chain.Halt()
   764  		logger.Debug("haltChan closed")
   765  		<-done
   766  
   767  		assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors")
   768  		assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled")
   769  		assert.Equal(t, uint64(1), counts[indexProcessRegularError], "Expected 1 damaged REGULAR message processed")
   770  	})
   771  
   772  	t.Run("ReceiveRegularAndQueue", func(t *testing.T) {
   773  		subtestIndex++
   774  
   775  		errorChan := make(chan struct{})
   776  		close(errorChan)
   777  		haltChan := make(chan struct{})
   778  
   779  		lastCutBlockNumber := uint64(3)
   780  
   781  		mockSupport := &mockmultichain.ConsenterSupport{
   782  			Blocks:         make(chan *cb.Block), // WriteBlock will post here
   783  			BlockCutterVal: mockblockcutter.NewReceiver(),
   784  			ChainIDVal:     mockChannel.topic(),
   785  			HeightVal:      lastCutBlockNumber, // Incremented during the WriteBlock call
   786  			SharedConfigVal: &mockconfig.Orderer{
   787  				BatchTimeoutVal: longTimeout,
   788  			},
   789  		}
   790  		defer close(mockSupport.BlockCutterVal.Block)
   791  
   792  		bareMinimumChain := &chainImpl{
   793  			parentConsumer:  mockParentConsumer,
   794  			channelConsumer: mockChannelConsumer,
   795  
   796  			channel:            mockChannel,
   797  			support:            mockSupport,
   798  			lastCutBlockNumber: lastCutBlockNumber,
   799  
   800  			errorChan: errorChan,
   801  			haltChan:  haltChan,
   802  		}
   803  
   804  		var counts []uint64
   805  		done := make(chan struct{})
   806  
   807  		go func() {
   808  			counts, err = bareMinimumChain.processMessagesToBlocks()
   809  			done <- struct{}{}
   810  		}()
   811  
   812  		// This is the wrappedMessage that the for-loop will process
   813  		mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage")))))
   814  
   815  		mockSupport.BlockCutterVal.Block <- struct{}{} // Let the `mockblockcutter.Ordered` call return
   816  		logger.Debugf("Mock blockcutter's Ordered call has returned")
   817  
   818  		logger.Debug("Closing haltChan to exit the infinite for-loop")
   819  		// We are guaranteed to hit the haltChan branch after hitting the REGULAR branch at least once
   820  		close(haltChan) // Identical to chain.Halt()
   821  		logger.Debug("haltChan closed")
   822  		<-done
   823  
   824  		assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors")
   825  		assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled")
   826  		assert.Equal(t, uint64(1), counts[indexProcessRegularPass], "Expected 1 REGULAR message processed")
   827  	})
   828  
   829  	t.Run("ReceiveRegularAndCutBlock", func(t *testing.T) {
   830  		subtestIndex++
   831  
   832  		errorChan := make(chan struct{})
   833  		close(errorChan)
   834  		haltChan := make(chan struct{})
   835  
   836  		lastCutBlockNumber := uint64(3)
   837  
   838  		mockSupport := &mockmultichain.ConsenterSupport{
   839  			Blocks:         make(chan *cb.Block), // WriteBlock will post here
   840  			BlockCutterVal: mockblockcutter.NewReceiver(),
   841  			ChainIDVal:     mockChannel.topic(),
   842  			HeightVal:      lastCutBlockNumber, // Incremented during the WriteBlock call
   843  			SharedConfigVal: &mockconfig.Orderer{
   844  				BatchTimeoutVal: longTimeout,
   845  			},
   846  		}
   847  		defer close(mockSupport.BlockCutterVal.Block)
   848  
   849  		bareMinimumChain := &chainImpl{
   850  			parentConsumer:  mockParentConsumer,
   851  			channelConsumer: mockChannelConsumer,
   852  
   853  			channel:            mockChannel,
   854  			support:            mockSupport,
   855  			lastCutBlockNumber: lastCutBlockNumber,
   856  
   857  			errorChan: errorChan,
   858  			haltChan:  haltChan,
   859  		}
   860  
   861  		var counts []uint64
   862  		done := make(chan struct{})
   863  
   864  		go func() {
   865  			counts, err = bareMinimumChain.processMessagesToBlocks()
   866  			done <- struct{}{}
   867  		}()
   868  
   869  		mockSupport.BlockCutterVal.CutNext = true
   870  
   871  		// This is the wrappedMessage that the for-loop will process
   872  		mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage")))))
   873  
   874  		mockSupport.BlockCutterVal.Block <- struct{}{} // Let the `mockblockcutter.Ordered` call return
   875  		logger.Debugf("Mock blockcutter's Ordered call has returned")
   876  		<-mockSupport.Blocks // Let the `mockConsenterSupport.WriteBlock` proceed
   877  
   878  		logger.Debug("Closing haltChan to exit the infinite for-loop")
   879  		close(haltChan) // Identical to chain.Halt()
   880  		logger.Debug("haltChan closed")
   881  		<-done
   882  
   883  		assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors")
   884  		assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled")
   885  		assert.Equal(t, uint64(1), counts[indexProcessRegularPass], "Expected 1 REGULAR message processed")
   886  		assert.Equal(t, lastCutBlockNumber+1, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to be bumped up by one")
   887  	})
   888  
   889  	t.Run("ReceiveTwoRegularAndCutTwoBlocks", func(t *testing.T) {
   890  		subtestIndex++
   891  
   892  		if testing.Short() {
   893  			t.Skip("Skipping test in short mode")
   894  		}
   895  
   896  		errorChan := make(chan struct{})
   897  		close(errorChan)
   898  		haltChan := make(chan struct{})
   899  
   900  		lastCutBlockNumber := uint64(3)
   901  
   902  		mockSupport := &mockmultichain.ConsenterSupport{
   903  			Blocks:         make(chan *cb.Block), // WriteBlock will post here
   904  			BlockCutterVal: mockblockcutter.NewReceiver(),
   905  			ChainIDVal:     mockChannel.topic(),
   906  			HeightVal:      lastCutBlockNumber, // Incremented during the WriteBlock call
   907  			SharedConfigVal: &mockconfig.Orderer{
   908  				BatchTimeoutVal: longTimeout,
   909  			},
   910  		}
   911  		defer close(mockSupport.BlockCutterVal.Block)
   912  
   913  		bareMinimumChain := &chainImpl{
   914  			parentConsumer:  mockParentConsumer,
   915  			channelConsumer: mockChannelConsumer,
   916  
   917  			channel:            mockChannel,
   918  			support:            mockSupport,
   919  			lastCutBlockNumber: lastCutBlockNumber,
   920  
   921  			errorChan: errorChan,
   922  			haltChan:  haltChan,
   923  		}
   924  
   925  		var counts []uint64
   926  		done := make(chan struct{})
   927  
   928  		go func() {
   929  			counts, err = bareMinimumChain.processMessagesToBlocks()
   930  			done <- struct{}{}
   931  		}()
   932  
   933  		var block1, block2 *cb.Block
   934  
   935  		// This is the first wrappedMessage that the for-loop will process
   936  		mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage")))))
   937  		mockSupport.BlockCutterVal.Block <- struct{}{} // Let the `mockblockcutter.Ordered` call return
   938  		logger.Debugf("Mock blockcutter's Ordered call has returned")
   939  
   940  		mockSupport.BlockCutterVal.IsolatedTx = true
   941  
   942  		// This is the first wrappedMessage that the for-loop will process
   943  		mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage")))))
   944  		mockSupport.BlockCutterVal.Block <- struct{}{}
   945  		logger.Debugf("Mock blockcutter's Ordered call has returned for the second time")
   946  
   947  		select {
   948  		case block1 = <-mockSupport.Blocks: // Let the `mockConsenterSupport.WriteBlock` proceed
   949  		case <-time.After(shortTimeout):
   950  			logger.Fatalf("Did not receive a block from the blockcutter as expected")
   951  		}
   952  
   953  		select {
   954  		case block2 = <-mockSupport.Blocks:
   955  		case <-time.After(shortTimeout):
   956  			logger.Fatalf("Did not receive a block from the blockcutter as expected")
   957  		}
   958  
   959  		logger.Debug("Closing haltChan to exit the infinite for-loop")
   960  		close(haltChan) // Identical to chain.Halt()
   961  		logger.Debug("haltChan closed")
   962  		<-done
   963  
   964  		expectedOffset := newestOffset + int64(subtestIndex) // TODO Hacky, revise eventually
   965  
   966  		assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors")
   967  		assert.Equal(t, uint64(2), counts[indexRecvPass], "Expected 2 messages received and unmarshaled")
   968  		assert.Equal(t, uint64(2), counts[indexProcessRegularPass], "Expected 2 REGULAR messages processed")
   969  		assert.Equal(t, lastCutBlockNumber+2, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to be bumped up by two")
   970  		assert.Equal(t, expectedOffset+1, extractEncodedOffset(block1.GetMetadata().Metadata[cb.BlockMetadataIndex_ORDERER]), "Expected encoded offset in first block to be %d", newestOffset+1)
   971  		assert.Equal(t, expectedOffset+2, extractEncodedOffset(block2.GetMetadata().Metadata[cb.BlockMetadataIndex_ORDERER]), "Expected encoded offset in first block to be %d", newestOffset+2)
   972  	})
   973  
   974  	t.Run("ReceiveRegularAndSendTimeToCut", func(t *testing.T) {
   975  		subtestIndex++
   976  
   977  		t.Skip("Skipping test as it introduces a race condition")
   978  
   979  		// NB We haven't set a handlermap for the mock broker so we need to set
   980  		// the ProduceResponse
   981  		successResponse := new(sarama.ProduceResponse)
   982  		successResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), sarama.ErrNoError)
   983  		mockBroker.Returns(successResponse)
   984  
   985  		errorChan := make(chan struct{})
   986  		close(errorChan)
   987  		haltChan := make(chan struct{})
   988  
   989  		lastCutBlockNumber := uint64(3)
   990  
   991  		mockSupport := &mockmultichain.ConsenterSupport{
   992  			Blocks:         make(chan *cb.Block), // WriteBlock will post here
   993  			BlockCutterVal: mockblockcutter.NewReceiver(),
   994  			ChainIDVal:     mockChannel.topic(),
   995  			HeightVal:      lastCutBlockNumber, // Incremented during the WriteBlock call
   996  			SharedConfigVal: &mockconfig.Orderer{
   997  				BatchTimeoutVal: extraShortTimeout, // ATTN
   998  			},
   999  		}
  1000  		defer close(mockSupport.BlockCutterVal.Block)
  1001  
  1002  		bareMinimumChain := &chainImpl{
  1003  			producer:        producer,
  1004  			parentConsumer:  mockParentConsumer,
  1005  			channelConsumer: mockChannelConsumer,
  1006  
  1007  			channel:            mockChannel,
  1008  			support:            mockSupport,
  1009  			lastCutBlockNumber: lastCutBlockNumber,
  1010  
  1011  			errorChan: errorChan,
  1012  			haltChan:  haltChan,
  1013  		}
  1014  
  1015  		var counts []uint64
  1016  		done := make(chan struct{})
  1017  
  1018  		go func() {
  1019  			counts, err = bareMinimumChain.processMessagesToBlocks()
  1020  			done <- struct{}{}
  1021  		}()
  1022  
  1023  		// This is the wrappedMessage that the for-loop will process
  1024  		mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage")))))
  1025  
  1026  		mockSupport.BlockCutterVal.Block <- struct{}{} // Let the `mockblockcutter.Ordered` call return
  1027  		logger.Debugf("Mock blockcutter's Ordered call has returned")
  1028  
  1029  		// Sleep so that the timer branch is activated before the exitChan one.
  1030  		// TODO This is a race condition, will fix in follow-up changeset
  1031  		time.Sleep(hitBranch)
  1032  
  1033  		logger.Debug("Closing haltChan to exit the infinite for-loop")
  1034  		close(haltChan) // Identical to chain.Halt()
  1035  		logger.Debug("haltChan closed")
  1036  		<-done
  1037  
  1038  		assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors")
  1039  		assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled")
  1040  		assert.Equal(t, uint64(1), counts[indexProcessRegularPass], "Expected 1 REGULAR message processed")
  1041  		assert.Equal(t, uint64(1), counts[indexSendTimeToCutPass], "Expected 1 TIMER event processed")
  1042  		assert.Equal(t, lastCutBlockNumber, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to stay the same")
  1043  	})
  1044  
  1045  	t.Run("ReceiveRegularAndSendTimeToCutError", func(t *testing.T) {
  1046  		// Note that this test is affected by the following parameters:
  1047  		// - Net.ReadTimeout
  1048  		// - Consumer.Retry.Backoff
  1049  		// - Metadata.Retry.Max
  1050  
  1051  		subtestIndex++
  1052  
  1053  		t.Skip("Skipping test as it introduces a race condition")
  1054  
  1055  		// Exact same test as ReceiveRegularAndSendTimeToCut.
  1056  		// Only difference is that the producer's attempt to send a TTC will
  1057  		// fail with an ErrNotEnoughReplicas error.
  1058  		failureResponse := new(sarama.ProduceResponse)
  1059  		failureResponse.AddTopicPartition(mockChannel.topic(), mockChannel.partition(), sarama.ErrNotEnoughReplicas)
  1060  		mockBroker.Returns(failureResponse)
  1061  
  1062  		errorChan := make(chan struct{})
  1063  		close(errorChan)
  1064  		haltChan := make(chan struct{})
  1065  
  1066  		lastCutBlockNumber := uint64(3)
  1067  
  1068  		mockSupport := &mockmultichain.ConsenterSupport{
  1069  			Blocks:         make(chan *cb.Block), // WriteBlock will post here
  1070  			BlockCutterVal: mockblockcutter.NewReceiver(),
  1071  			ChainIDVal:     mockChannel.topic(),
  1072  			HeightVal:      lastCutBlockNumber, // Incremented during the WriteBlock call
  1073  			SharedConfigVal: &mockconfig.Orderer{
  1074  				BatchTimeoutVal: extraShortTimeout, // ATTN
  1075  			},
  1076  		}
  1077  		defer close(mockSupport.BlockCutterVal.Block)
  1078  
  1079  		bareMinimumChain := &chainImpl{
  1080  			producer:        producer,
  1081  			parentConsumer:  mockParentConsumer,
  1082  			channelConsumer: mockChannelConsumer,
  1083  
  1084  			channel:            mockChannel,
  1085  			support:            mockSupport,
  1086  			lastCutBlockNumber: lastCutBlockNumber,
  1087  
  1088  			errorChan: errorChan,
  1089  			haltChan:  haltChan,
  1090  		}
  1091  
  1092  		var counts []uint64
  1093  		done := make(chan struct{})
  1094  
  1095  		go func() {
  1096  			counts, err = bareMinimumChain.processMessagesToBlocks()
  1097  			done <- struct{}{}
  1098  		}()
  1099  
  1100  		// This is the wrappedMessage that the for-loop will process
  1101  		mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(utils.MarshalOrPanic(newMockEnvelope("fooMessage")))))
  1102  
  1103  		mockSupport.BlockCutterVal.Block <- struct{}{} // Let the `mockblockcutter.Ordered` call return
  1104  		logger.Debugf("Mock blockcutter's Ordered call has returned")
  1105  
  1106  		// Sleep so that the timer branch is activated before the exitChan one.
  1107  		// TODO This is a race condition, will fix in follow-up changeset
  1108  		time.Sleep(hitBranch)
  1109  
  1110  		logger.Debug("Closing haltChan to exit the infinite for-loop")
  1111  		close(haltChan) // Identical to chain.Halt()
  1112  		logger.Debug("haltChan closed")
  1113  		<-done
  1114  
  1115  		assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors")
  1116  		assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled")
  1117  		assert.Equal(t, uint64(1), counts[indexProcessRegularPass], "Expected 1 REGULAR message processed")
  1118  		assert.Equal(t, uint64(1), counts[indexSendTimeToCutError], "Expected 1 faulty TIMER event processed")
  1119  		assert.Equal(t, lastCutBlockNumber, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to stay the same")
  1120  	})
  1121  
  1122  	t.Run("ReceiveTimeToCutProper", func(t *testing.T) {
  1123  		subtestIndex++
  1124  
  1125  		errorChan := make(chan struct{})
  1126  		close(errorChan)
  1127  		haltChan := make(chan struct{})
  1128  
  1129  		lastCutBlockNumber := uint64(3)
  1130  
  1131  		mockSupport := &mockmultichain.ConsenterSupport{
  1132  			Blocks:         make(chan *cb.Block), // WriteBlock will post here
  1133  			BlockCutterVal: mockblockcutter.NewReceiver(),
  1134  			ChainIDVal:     mockChannel.topic(),
  1135  			HeightVal:      lastCutBlockNumber, // Incremented during the WriteBlock call
  1136  		}
  1137  		defer close(mockSupport.BlockCutterVal.Block)
  1138  
  1139  		bareMinimumChain := &chainImpl{
  1140  			parentConsumer:  mockParentConsumer,
  1141  			channelConsumer: mockChannelConsumer,
  1142  
  1143  			channel:            mockChannel,
  1144  			support:            mockSupport,
  1145  			lastCutBlockNumber: lastCutBlockNumber,
  1146  
  1147  			errorChan: errorChan,
  1148  			haltChan:  haltChan,
  1149  		}
  1150  
  1151  		// We need the mock blockcutter to deliver a non-empty batch
  1152  		go func() {
  1153  			mockSupport.BlockCutterVal.Block <- struct{}{} // Let the `mockblockcutter.Ordered` call below return
  1154  			logger.Debugf("Mock blockcutter's Ordered call has returned")
  1155  		}()
  1156  		// We are "planting" a message directly to the mock blockcutter
  1157  		mockSupport.BlockCutterVal.Ordered(newMockEnvelope("fooMessage"))
  1158  
  1159  		var counts []uint64
  1160  		done := make(chan struct{})
  1161  
  1162  		go func() {
  1163  			counts, err = bareMinimumChain.processMessagesToBlocks()
  1164  			done <- struct{}{}
  1165  		}()
  1166  
  1167  		// This is the wrappedMessage that the for-loop will process
  1168  		mpc.YieldMessage(newMockConsumerMessage(newTimeToCutMessage(lastCutBlockNumber + 1)))
  1169  
  1170  		<-mockSupport.Blocks // Let the `mockConsenterSupport.WriteBlock` proceed
  1171  
  1172  		logger.Debug("Closing haltChan to exit the infinite for-loop")
  1173  		close(haltChan) // Identical to chain.Halt()
  1174  		logger.Debug("haltChan closed")
  1175  		<-done
  1176  
  1177  		assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors")
  1178  		assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled")
  1179  		assert.Equal(t, uint64(1), counts[indexProcessTimeToCutPass], "Expected 1 TIMETOCUT message processed")
  1180  		assert.Equal(t, lastCutBlockNumber+1, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to be bumped up by one")
  1181  	})
  1182  
  1183  	t.Run("ReceiveTimeToCutZeroBatch", func(t *testing.T) {
  1184  		subtestIndex++
  1185  
  1186  		errorChan := make(chan struct{})
  1187  		close(errorChan)
  1188  		haltChan := make(chan struct{})
  1189  
  1190  		lastCutBlockNumber := uint64(3)
  1191  
  1192  		mockSupport := &mockmultichain.ConsenterSupport{
  1193  			Blocks:         make(chan *cb.Block), // WriteBlock will post here
  1194  			BlockCutterVal: mockblockcutter.NewReceiver(),
  1195  			ChainIDVal:     mockChannel.topic(),
  1196  			HeightVal:      lastCutBlockNumber, // Incremented during the WriteBlock call
  1197  		}
  1198  		defer close(mockSupport.BlockCutterVal.Block)
  1199  
  1200  		bareMinimumChain := &chainImpl{
  1201  			parentConsumer:  mockParentConsumer,
  1202  			channelConsumer: mockChannelConsumer,
  1203  
  1204  			channel:            mockChannel,
  1205  			support:            mockSupport,
  1206  			lastCutBlockNumber: lastCutBlockNumber,
  1207  
  1208  			errorChan: errorChan,
  1209  			haltChan:  haltChan,
  1210  		}
  1211  
  1212  		var counts []uint64
  1213  		done := make(chan struct{})
  1214  
  1215  		go func() {
  1216  			counts, err = bareMinimumChain.processMessagesToBlocks()
  1217  			done <- struct{}{}
  1218  		}()
  1219  
  1220  		// This is the wrappedMessage that the for-loop will process
  1221  		mpc.YieldMessage(newMockConsumerMessage(newTimeToCutMessage(lastCutBlockNumber + 1)))
  1222  
  1223  		logger.Debug("Closing haltChan to exit the infinite for-loop")
  1224  		close(haltChan) // Identical to chain.Halt()
  1225  		logger.Debug("haltChan closed")
  1226  		<-done
  1227  
  1228  		assert.Error(t, err, "Expected the processMessagesToBlocks call to return an error")
  1229  		assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled")
  1230  		assert.Equal(t, uint64(1), counts[indexProcessTimeToCutError], "Expected 1 faulty TIMETOCUT message processed")
  1231  		assert.Equal(t, lastCutBlockNumber, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to stay the same")
  1232  	})
  1233  
  1234  	t.Run("ReceiveTimeToCutLargerThanExpected", func(t *testing.T) {
  1235  		subtestIndex++
  1236  
  1237  		errorChan := make(chan struct{})
  1238  		close(errorChan)
  1239  		haltChan := make(chan struct{})
  1240  
  1241  		lastCutBlockNumber := uint64(3)
  1242  
  1243  		mockSupport := &mockmultichain.ConsenterSupport{
  1244  			Blocks:         make(chan *cb.Block), // WriteBlock will post here
  1245  			BlockCutterVal: mockblockcutter.NewReceiver(),
  1246  			ChainIDVal:     mockChannel.topic(),
  1247  			HeightVal:      lastCutBlockNumber, // Incremented during the WriteBlock call
  1248  		}
  1249  		defer close(mockSupport.BlockCutterVal.Block)
  1250  
  1251  		bareMinimumChain := &chainImpl{
  1252  			parentConsumer:  mockParentConsumer,
  1253  			channelConsumer: mockChannelConsumer,
  1254  
  1255  			channel:            mockChannel,
  1256  			support:            mockSupport,
  1257  			lastCutBlockNumber: lastCutBlockNumber,
  1258  
  1259  			errorChan: errorChan,
  1260  			haltChan:  haltChan,
  1261  		}
  1262  
  1263  		var counts []uint64
  1264  		done := make(chan struct{})
  1265  
  1266  		go func() {
  1267  			counts, err = bareMinimumChain.processMessagesToBlocks()
  1268  			done <- struct{}{}
  1269  		}()
  1270  
  1271  		// This is the wrappedMessage that the for-loop will process
  1272  		mpc.YieldMessage(newMockConsumerMessage(newTimeToCutMessage(lastCutBlockNumber + 2)))
  1273  
  1274  		logger.Debug("Closing haltChan to exit the infinite for-loop")
  1275  		close(haltChan) // Identical to chain.Halt()
  1276  		logger.Debug("haltChan closed")
  1277  		<-done
  1278  
  1279  		assert.Error(t, err, "Expected the processMessagesToBlocks call to return an error")
  1280  		assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled")
  1281  		assert.Equal(t, uint64(1), counts[indexProcessTimeToCutError], "Expected 1 faulty TIMETOCUT message processed")
  1282  		assert.Equal(t, lastCutBlockNumber, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to stay the same")
  1283  	})
  1284  
  1285  	t.Run("ReceiveTimeToCutStale", func(t *testing.T) {
  1286  		subtestIndex++
  1287  
  1288  		errorChan := make(chan struct{})
  1289  		close(errorChan)
  1290  		haltChan := make(chan struct{})
  1291  
  1292  		lastCutBlockNumber := uint64(3)
  1293  
  1294  		mockSupport := &mockmultichain.ConsenterSupport{
  1295  			Blocks:         make(chan *cb.Block), // WriteBlock will post here
  1296  			BlockCutterVal: mockblockcutter.NewReceiver(),
  1297  			ChainIDVal:     mockChannel.topic(),
  1298  			HeightVal:      lastCutBlockNumber, // Incremented during the WriteBlock call
  1299  		}
  1300  		defer close(mockSupport.BlockCutterVal.Block)
  1301  
  1302  		bareMinimumChain := &chainImpl{
  1303  			parentConsumer:  mockParentConsumer,
  1304  			channelConsumer: mockChannelConsumer,
  1305  
  1306  			channel:            mockChannel,
  1307  			support:            mockSupport,
  1308  			lastCutBlockNumber: lastCutBlockNumber,
  1309  
  1310  			errorChan: errorChan,
  1311  			haltChan:  haltChan,
  1312  		}
  1313  
  1314  		var counts []uint64
  1315  		done := make(chan struct{})
  1316  
  1317  		go func() {
  1318  			counts, err = bareMinimumChain.processMessagesToBlocks()
  1319  			done <- struct{}{}
  1320  		}()
  1321  
  1322  		// This is the wrappedMessage that the for-loop will process
  1323  		mpc.YieldMessage(newMockConsumerMessage(newTimeToCutMessage(lastCutBlockNumber)))
  1324  
  1325  		logger.Debug("Closing haltChan to exit the infinite for-loop")
  1326  		close(haltChan) // Identical to chain.Halt()
  1327  		logger.Debug("haltChan closed")
  1328  		<-done
  1329  
  1330  		assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors")
  1331  		assert.Equal(t, uint64(1), counts[indexRecvPass], "Expected 1 message received and unmarshaled")
  1332  		assert.Equal(t, uint64(1), counts[indexProcessTimeToCutPass], "Expected 1 TIMETOCUT message processed")
  1333  		assert.Equal(t, lastCutBlockNumber, bareMinimumChain.lastCutBlockNumber, "Expected lastCutBlockNumber to stay the same")
  1334  	})
  1335  
  1336  	t.Run("ReceiveKafkaErrorAndCloseErrorChan", func(t *testing.T) {
  1337  		subtestIndex++
  1338  
  1339  		// If we set up the mock broker so that it returns a response, if the
  1340  		// test finishes before the sendConnectMessage goroutine has received
  1341  		// this response, we will get a failure ("not all expectations were
  1342  		// satisfied") from the mock broker. So we sabotage the producer.
  1343  		failedProducer, _ := sarama.NewSyncProducer([]string{}, mockBrokerConfig)
  1344  
  1345  		// We need to have the sendConnectMessage goroutine die instantaneously,
  1346  		// otherwise we'll get a nil pointer dereference panic. We are
  1347  		// exploiting the admittedly hacky shortcut where a retriable process
  1348  		// returns immediately when given the nil time.Duration value for its
  1349  		// ticker.
  1350  		zeroRetryConsenter := &consenterImpl{}
  1351  
  1352  		// Let's assume an open errorChan, i.e. a healthy link between the
  1353  		// consumer and the Kafka partition corresponding to the channel
  1354  		errorChan := make(chan struct{})
  1355  
  1356  		haltChan := make(chan struct{})
  1357  
  1358  		mockSupport := &mockmultichain.ConsenterSupport{
  1359  			ChainIDVal: mockChannel.topic(),
  1360  		}
  1361  
  1362  		bareMinimumChain := &chainImpl{
  1363  			consenter:       zeroRetryConsenter, // For sendConnectMessage
  1364  			producer:        failedProducer,     // For sendConnectMessage
  1365  			parentConsumer:  mockParentConsumer,
  1366  			channelConsumer: mockChannelConsumer,
  1367  
  1368  			channel: mockChannel,
  1369  			support: mockSupport,
  1370  
  1371  			errorChan: errorChan,
  1372  			haltChan:  haltChan,
  1373  		}
  1374  
  1375  		var counts []uint64
  1376  		done := make(chan struct{})
  1377  
  1378  		go func() {
  1379  			counts, err = bareMinimumChain.processMessagesToBlocks()
  1380  			done <- struct{}{}
  1381  		}()
  1382  
  1383  		// This is what the for-loop will process
  1384  		mpc.YieldError(fmt.Errorf("fooError"))
  1385  
  1386  		logger.Debug("Closing haltChan to exit the infinite for-loop")
  1387  		close(haltChan) // Identical to chain.Halt()
  1388  		logger.Debug("haltChan closed")
  1389  		<-done
  1390  
  1391  		assert.NoError(t, err, "Expected the processMessagesToBlocks call to return without errors")
  1392  		assert.Equal(t, uint64(1), counts[indexRecvError], "Expected 1 Kafka error received")
  1393  
  1394  		select {
  1395  		case <-bareMinimumChain.errorChan:
  1396  			logger.Debug("errorChan is closed as it should be")
  1397  		default:
  1398  			t.Fatal("errorChan should have been closed")
  1399  		}
  1400  	})
  1401  
  1402  	t.Run("ReceiveKafkaErrorAndThenReceiveRegularMessage", func(t *testing.T) {
  1403  		subtestIndex++
  1404  
  1405  		t.Skip("Skipping test as it introduces a race condition")
  1406  
  1407  		// If we set up the mock broker so that it returns a response, if the
  1408  		// test finishes before the sendConnectMessage goroutine has received
  1409  		// this response, we will get a failure ("not all expectations were
  1410  		// satisfied") from the mock broker. So we sabotage the producer.
  1411  		failedProducer, _ := sarama.NewSyncProducer([]string{}, mockBrokerConfig)
  1412  
  1413  		// We need to have the sendConnectMessage goroutine die instantaneously,
  1414  		// otherwise we'll get a nil pointer dereference panic. We are
  1415  		// exploiting the admittedly hacky shortcut where a retriable process
  1416  		// returns immediately when given the nil time.Duration value for its
  1417  		// ticker.
  1418  		zeroRetryConsenter := &consenterImpl{}
  1419  
  1420  		// If the errorChan is closed already, the kafkaErr branch shouldn't
  1421  		// touch it
  1422  		errorChan := make(chan struct{})
  1423  		close(errorChan)
  1424  
  1425  		haltChan := make(chan struct{})
  1426  
  1427  		mockSupport := &mockmultichain.ConsenterSupport{
  1428  			ChainIDVal: mockChannel.topic(),
  1429  		}
  1430  
  1431  		bareMinimumChain := &chainImpl{
  1432  			consenter:       zeroRetryConsenter, // For sendConnectMessage
  1433  			producer:        failedProducer,     // For sendConnectMessage
  1434  			parentConsumer:  mockParentConsumer,
  1435  			channelConsumer: mockChannelConsumer,
  1436  
  1437  			channel: mockChannel,
  1438  			support: mockSupport,
  1439  
  1440  			errorChan: errorChan,
  1441  			haltChan:  haltChan,
  1442  		}
  1443  
  1444  		var counts []uint64
  1445  		done := make(chan struct{})
  1446  
  1447  		go func() {
  1448  			counts, err = bareMinimumChain.processMessagesToBlocks()
  1449  			done <- struct{}{}
  1450  		}()
  1451  
  1452  		// This is what the for-loop will process
  1453  		mpc.YieldError(fmt.Errorf("foo"))
  1454  
  1455  		// We tested this in ReceiveKafkaErrorAndCloseErrorChan, so this check
  1456  		// is redundant in that regard. We use it however to ensure the
  1457  		// kafkaErrBranch has been activated before proceeding with pushing the
  1458  		// regular message.
  1459  		select {
  1460  		case <-bareMinimumChain.errorChan:
  1461  			logger.Debug("errorChan is closed as it should be")
  1462  		case <-time.After(shortTimeout):
  1463  			t.Fatal("errorChan should have been closed by now")
  1464  		}
  1465  
  1466  		// This is the wrappedMessage that the for-loop will process. We use
  1467  		// a broken regular message here on purpose since this is the shortest
  1468  		// path and it allows us to test what we want.
  1469  		mpc.YieldMessage(newMockConsumerMessage(newRegularMessage(tamperBytes(utils.MarshalOrPanic(newMockEnvelope("fooMessage"))))))
  1470  
  1471  		// Sleep so that the Messages/errorChan branch is activated.
  1472  		// TODO Hacky approach, will need to revise eventually
  1473  		time.Sleep(hitBranch)
  1474  
  1475  		// Check that the errorChan was recreated
  1476  		select {
  1477  		case <-bareMinimumChain.errorChan:
  1478  			t.Fatal("errorChan should have been open")
  1479  		default:
  1480  			logger.Debug("errorChan is open as it should be")
  1481  		}
  1482  
  1483  		logger.Debug("Closing haltChan to exit the infinite for-loop")
  1484  		close(haltChan) // Identical to chain.Halt()
  1485  		logger.Debug("haltChan closed")
  1486  		<-done
  1487  	})
  1488  }