github.com/onflow/flow-go@v0.35.7-crescendo-preview.23-atree-inlining/engine/execution/execution_test.go (about)

     1  package execution_test
     2  
     3  import (
     4  	"context"
     5  	"sync"
     6  	"testing"
     7  	"time"
     8  
     9  	"github.com/rs/zerolog/log"
    10  	"github.com/stretchr/testify/assert"
    11  	"github.com/stretchr/testify/mock"
    12  	"github.com/stretchr/testify/require"
    13  	"github.com/vmihailenco/msgpack"
    14  	"go.uber.org/atomic"
    15  
    16  	execTestutil "github.com/onflow/flow-go/engine/execution/testutil"
    17  	"github.com/onflow/flow-go/engine/testutil"
    18  	testmock "github.com/onflow/flow-go/engine/testutil/mock"
    19  	"github.com/onflow/flow-go/model/flow"
    20  	"github.com/onflow/flow-go/model/messages"
    21  	"github.com/onflow/flow-go/module/signature"
    22  	"github.com/onflow/flow-go/network/channels"
    23  	"github.com/onflow/flow-go/network/mocknetwork"
    24  	"github.com/onflow/flow-go/network/stub"
    25  	"github.com/onflow/flow-go/state/cluster"
    26  	"github.com/onflow/flow-go/utils/unittest"
    27  )
    28  
    29  func sendBlock(exeNode *testmock.ExecutionNode, from flow.Identifier, proposal *messages.BlockProposal) error {
    30  	return exeNode.FollowerEngine.Process(channels.ReceiveBlocks, from, proposal)
    31  }
    32  
    33  // Test when the ingestion engine receives a block, it will
    34  // request collections from collection node, and send ER to
    35  // verification node and consensus node.
    36  // create a block that has two collections: col1 and col2;
    37  // col1 has tx1 and tx2, col2 has tx3 and tx4.
    38  // create another child block which will trigger the parent
    39  // block to be incorporated and be passed to the ingestion engine
    40  func TestExecutionFlow(t *testing.T) {
    41  	hub := stub.NewNetworkHub()
    42  
    43  	chainID := flow.Testnet
    44  
    45  	colID := unittest.PrivateNodeInfoFixture(
    46  		unittest.WithRole(flow.RoleCollection),
    47  		unittest.WithKeys,
    48  	)
    49  	conID := unittest.PrivateNodeInfoFixture(
    50  		unittest.WithRole(flow.RoleConsensus),
    51  		unittest.WithKeys,
    52  	)
    53  	exeID := unittest.PrivateNodeInfoFixture(
    54  		unittest.WithRole(flow.RoleExecution),
    55  		unittest.WithKeys,
    56  	)
    57  	verID := unittest.PrivateNodeInfoFixture(
    58  		unittest.WithRole(flow.RoleVerification),
    59  		unittest.WithKeys,
    60  	)
    61  
    62  	identities := unittest.CompleteIdentitySet(colID.Identity(), conID.Identity(), exeID.Identity(), verID.Identity()).
    63  		Sort(flow.Canonical[flow.Identity])
    64  
    65  	// create execution node
    66  	exeNode := testutil.ExecutionNode(t, hub, exeID, identities, 21, chainID)
    67  
    68  	ctx, cancel := context.WithCancel(context.Background())
    69  	unittest.RequireReturnsBefore(t, func() {
    70  		exeNode.Ready(ctx)
    71  	}, 1*time.Second, "could not start execution node on time")
    72  	defer exeNode.Done(cancel)
    73  
    74  	genesis, err := exeNode.Blocks.ByHeight(0)
    75  	require.NoError(t, err)
    76  
    77  	tx1 := flow.TransactionBody{
    78  		Script: []byte("transaction { execute { log(1) } }"),
    79  	}
    80  
    81  	tx2 := flow.TransactionBody{
    82  		Script: []byte("transaction { execute { log(2) } }"),
    83  	}
    84  
    85  	tx3 := flow.TransactionBody{
    86  		Script: []byte("transaction { execute { log(3) } }"),
    87  	}
    88  
    89  	tx4 := flow.TransactionBody{
    90  		Script: []byte("transaction { execute { log(4) } }"),
    91  	}
    92  
    93  	col1 := flow.Collection{Transactions: []*flow.TransactionBody{&tx1, &tx2}}
    94  	col2 := flow.Collection{Transactions: []*flow.TransactionBody{&tx3, &tx4}}
    95  
    96  	collections := map[flow.Identifier]*flow.Collection{
    97  		col1.ID(): &col1,
    98  		col2.ID(): &col2,
    99  	}
   100  
   101  	clusterChainID := cluster.CanonicalClusterID(1, flow.IdentityList{colID.Identity()}.NodeIDs())
   102  
   103  	// signed by the only collector
   104  	block := unittest.BlockWithParentAndProposerFixture(t, genesis.Header, conID.NodeID)
   105  	voterIndices, err := signature.EncodeSignersToIndices(
   106  		[]flow.Identifier{conID.NodeID}, []flow.Identifier{conID.NodeID})
   107  	require.NoError(t, err)
   108  	block.Header.ParentVoterIndices = voterIndices
   109  	signerIndices, err := signature.EncodeSignersToIndices(
   110  		[]flow.Identifier{colID.NodeID}, []flow.Identifier{colID.NodeID})
   111  	require.NoError(t, err)
   112  	block.SetPayload(flow.Payload{
   113  		Guarantees: []*flow.CollectionGuarantee{
   114  			{
   115  				CollectionID:     col1.ID(),
   116  				SignerIndices:    signerIndices,
   117  				ChainID:          clusterChainID,
   118  				ReferenceBlockID: genesis.ID(),
   119  			},
   120  			{
   121  				CollectionID:     col2.ID(),
   122  				SignerIndices:    signerIndices,
   123  				ChainID:          clusterChainID,
   124  				ReferenceBlockID: genesis.ID(),
   125  			},
   126  		},
   127  		ProtocolStateID: genesis.Payload.ProtocolStateID,
   128  	})
   129  
   130  	child := unittest.BlockWithParentAndProposerFixture(t, block.Header, conID.NodeID)
   131  	// the default signer indices is 2 bytes, but in this test cases
   132  	// we need 1 byte
   133  	child.Header.ParentVoterIndices = voterIndices
   134  	child.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(block.Payload.ProtocolStateID)))
   135  
   136  	log.Info().Msgf("child block ID: %v, indices: %x", child.Header.ID(), child.Header.ParentVoterIndices)
   137  
   138  	collectionNode := testutil.GenericNodeFromParticipants(t, hub, colID, identities, chainID)
   139  	defer collectionNode.Done()
   140  	verificationNode := testutil.GenericNodeFromParticipants(t, hub, verID, identities, chainID)
   141  	defer verificationNode.Done()
   142  	consensusNode := testutil.GenericNodeFromParticipants(t, hub, conID, identities, chainID)
   143  	defer consensusNode.Done()
   144  
   145  	// create collection node that can respond collections to execution node
   146  	// check collection node received the collection request from execution node
   147  	providerEngine := new(mocknetwork.Engine)
   148  	provConduit, _ := collectionNode.Net.Register(channels.ProvideCollections, providerEngine)
   149  	providerEngine.On("Process", mock.AnythingOfType("channels.Channel"), exeID.NodeID, mock.Anything).
   150  		Run(func(args mock.Arguments) {
   151  			originID := args.Get(1).(flow.Identifier)
   152  			req := args.Get(2).(*messages.EntityRequest)
   153  
   154  			var entities []flow.Entity
   155  			for _, entityID := range req.EntityIDs {
   156  				coll, exists := collections[entityID]
   157  				require.True(t, exists)
   158  				entities = append(entities, coll)
   159  			}
   160  
   161  			var blobs [][]byte
   162  			for _, entity := range entities {
   163  				blob, _ := msgpack.Marshal(entity)
   164  				blobs = append(blobs, blob)
   165  			}
   166  
   167  			res := &messages.EntityResponse{
   168  				Nonce:     req.Nonce,
   169  				EntityIDs: req.EntityIDs,
   170  				Blobs:     blobs,
   171  			}
   172  
   173  			err := provConduit.Publish(res, originID)
   174  			assert.NoError(t, err)
   175  		}).
   176  		Once().
   177  		Return(nil)
   178  
   179  	var lock sync.Mutex
   180  	var receipt *flow.ExecutionReceipt
   181  
   182  	// create verification engine that can create approvals and send to consensus nodes
   183  	// check the verification engine received the ER from execution node
   184  	verificationEngine := new(mocknetwork.Engine)
   185  	_, _ = verificationNode.Net.Register(channels.ReceiveReceipts, verificationEngine)
   186  	verificationEngine.On("Process", mock.AnythingOfType("channels.Channel"), exeID.NodeID, mock.Anything).
   187  		Run(func(args mock.Arguments) {
   188  			lock.Lock()
   189  			defer lock.Unlock()
   190  			receipt, _ = args[2].(*flow.ExecutionReceipt)
   191  
   192  			assert.Equal(t, block.ID(), receipt.ExecutionResult.BlockID)
   193  		}).
   194  		Return(nil).
   195  		Once()
   196  
   197  	// create consensus engine that accepts the result
   198  	// check the consensus engine has received the result from execution node
   199  	consensusEngine := new(mocknetwork.Engine)
   200  	_, _ = consensusNode.Net.Register(channels.ReceiveReceipts, consensusEngine)
   201  	consensusEngine.On("Process", mock.AnythingOfType("channels.Channel"), exeID.NodeID, mock.Anything).
   202  		Run(func(args mock.Arguments) {
   203  			lock.Lock()
   204  			defer lock.Unlock()
   205  
   206  			receipt, _ = args[2].(*flow.ExecutionReceipt)
   207  
   208  			assert.Equal(t, block.ID(), receipt.ExecutionResult.BlockID)
   209  			assert.Equal(t, len(collections), len(receipt.ExecutionResult.Chunks)-1) // don't count system chunk
   210  
   211  			for i, chunk := range receipt.ExecutionResult.Chunks {
   212  				assert.EqualValues(t, i, chunk.CollectionIndex)
   213  			}
   214  		}).
   215  		Return(nil).
   216  		Once()
   217  
   218  	// submit block from consensus node
   219  	err = sendBlock(&exeNode, conID.NodeID, unittest.ProposalFromBlock(&block))
   220  	require.NoError(t, err)
   221  
   222  	// submit the child block from consensus node, which trigger the parent block
   223  	// to be passed to BlockProcessable
   224  	err = sendBlock(&exeNode, conID.NodeID, unittest.ProposalFromBlock(&child))
   225  	require.NoError(t, err)
   226  
   227  	require.Eventually(t, func() bool {
   228  		// when sendBlock returned, ingestion engine might not have processed
   229  		// the block yet, because the process is async. we have to wait
   230  		hub.DeliverAll()
   231  
   232  		lock.Lock()
   233  		defer lock.Unlock()
   234  		return receipt != nil
   235  	}, time.Second*10, time.Millisecond*500)
   236  
   237  	// check that the block has been executed.
   238  	exeNode.AssertBlockIsExecuted(t, block.Header)
   239  
   240  	if exeNode.StorehouseEnabled {
   241  		exeNode.AssertHighestExecutedBlock(t, genesis.Header)
   242  	} else {
   243  		exeNode.AssertHighestExecutedBlock(t, block.Header)
   244  	}
   245  
   246  	myReceipt, err := exeNode.MyExecutionReceipts.MyReceipt(block.ID())
   247  	require.NoError(t, err)
   248  	require.NotNil(t, myReceipt)
   249  	require.Equal(t, exeNode.Me.NodeID(), myReceipt.ExecutorID)
   250  
   251  	providerEngine.AssertExpectations(t)
   252  	verificationEngine.AssertExpectations(t)
   253  	consensusEngine.AssertExpectations(t)
   254  }
   255  
   256  func deployContractBlock(
   257  	t *testing.T,
   258  	conID *flow.Identity,
   259  	colID *flow.Identity,
   260  	chain flow.Chain,
   261  	seq uint64,
   262  	parent *flow.Block,
   263  	ref *flow.Header,
   264  ) (
   265  	*flow.TransactionBody, *flow.Collection, *flow.Block, *messages.BlockProposal, uint64) {
   266  	// make tx
   267  	tx := execTestutil.DeployCounterContractTransaction(chain.ServiceAddress(), chain)
   268  	err := execTestutil.SignTransactionAsServiceAccount(tx, seq, chain)
   269  	require.NoError(t, err)
   270  
   271  	// make collection
   272  	col := &flow.Collection{Transactions: []*flow.TransactionBody{tx}}
   273  
   274  	signerIndices, err := signature.EncodeSignersToIndices(
   275  		[]flow.Identifier{colID.NodeID}, []flow.Identifier{colID.NodeID})
   276  	require.NoError(t, err)
   277  
   278  	clusterChainID := cluster.CanonicalClusterID(1, flow.IdentityList{colID}.NodeIDs())
   279  
   280  	// make block
   281  	block := unittest.BlockWithParentAndProposerFixture(t, parent.Header, conID.NodeID)
   282  	voterIndices, err := signature.EncodeSignersToIndices(
   283  		[]flow.Identifier{conID.NodeID}, []flow.Identifier{conID.NodeID})
   284  	require.NoError(t, err)
   285  	block.Header.ParentVoterIndices = voterIndices
   286  	block.SetPayload(flow.Payload{
   287  		Guarantees: []*flow.CollectionGuarantee{
   288  			{
   289  				CollectionID:     col.ID(),
   290  				SignerIndices:    signerIndices,
   291  				ChainID:          clusterChainID,
   292  				ReferenceBlockID: ref.ID(),
   293  			},
   294  		},
   295  		ProtocolStateID: parent.Payload.ProtocolStateID,
   296  	})
   297  
   298  	// make proposal
   299  	proposal := unittest.ProposalFromBlock(&block)
   300  
   301  	return tx, col, &block, proposal, seq + 1
   302  }
   303  
   304  func makePanicBlock(t *testing.T, conID *flow.Identity, colID *flow.Identity, chain flow.Chain, seq uint64, parent *flow.Block, ref *flow.Header) (
   305  	*flow.TransactionBody, *flow.Collection, *flow.Block, *messages.BlockProposal, uint64) {
   306  	// make tx
   307  	tx := execTestutil.CreateCounterPanicTransaction(chain.ServiceAddress(), chain.ServiceAddress())
   308  	err := execTestutil.SignTransactionAsServiceAccount(tx, seq, chain)
   309  	require.NoError(t, err)
   310  
   311  	// make collection
   312  	col := &flow.Collection{Transactions: []*flow.TransactionBody{tx}}
   313  
   314  	clusterChainID := cluster.CanonicalClusterID(1, flow.IdentityList{colID}.NodeIDs())
   315  	// make block
   316  	block := unittest.BlockWithParentAndProposerFixture(t, parent.Header, conID.NodeID)
   317  	voterIndices, err := signature.EncodeSignersToIndices(
   318  		[]flow.Identifier{conID.NodeID}, []flow.Identifier{conID.NodeID})
   319  	require.NoError(t, err)
   320  	block.Header.ParentVoterIndices = voterIndices
   321  
   322  	signerIndices, err := signature.EncodeSignersToIndices(
   323  		[]flow.Identifier{colID.NodeID}, []flow.Identifier{colID.NodeID})
   324  	require.NoError(t, err)
   325  
   326  	block.SetPayload(flow.Payload{
   327  		Guarantees: []*flow.CollectionGuarantee{
   328  			{CollectionID: col.ID(), SignerIndices: signerIndices, ChainID: clusterChainID, ReferenceBlockID: ref.ID()},
   329  		},
   330  		ProtocolStateID: parent.Payload.ProtocolStateID,
   331  	})
   332  
   333  	proposal := unittest.ProposalFromBlock(&block)
   334  
   335  	return tx, col, &block, proposal, seq + 1
   336  }
   337  
   338  func makeSuccessBlock(t *testing.T, conID *flow.Identity, colID *flow.Identity, chain flow.Chain, seq uint64, parent *flow.Block, ref *flow.Header) (
   339  	*flow.TransactionBody, *flow.Collection, *flow.Block, *messages.BlockProposal, uint64) {
   340  	tx := execTestutil.AddToCounterTransaction(chain.ServiceAddress(), chain.ServiceAddress())
   341  	err := execTestutil.SignTransactionAsServiceAccount(tx, seq, chain)
   342  	require.NoError(t, err)
   343  
   344  	signerIndices, err := signature.EncodeSignersToIndices(
   345  		[]flow.Identifier{colID.NodeID}, []flow.Identifier{colID.NodeID})
   346  	require.NoError(t, err)
   347  	clusterChainID := cluster.CanonicalClusterID(1, flow.IdentityList{colID}.NodeIDs())
   348  
   349  	col := &flow.Collection{Transactions: []*flow.TransactionBody{tx}}
   350  	block := unittest.BlockWithParentAndProposerFixture(t, parent.Header, conID.NodeID)
   351  	voterIndices, err := signature.EncodeSignersToIndices(
   352  		[]flow.Identifier{conID.NodeID}, []flow.Identifier{conID.NodeID})
   353  	require.NoError(t, err)
   354  	block.Header.ParentVoterIndices = voterIndices
   355  	block.SetPayload(flow.Payload{
   356  		Guarantees: []*flow.CollectionGuarantee{
   357  			{CollectionID: col.ID(), SignerIndices: signerIndices, ChainID: clusterChainID, ReferenceBlockID: ref.ID()},
   358  		},
   359  		ProtocolStateID: parent.Payload.ProtocolStateID,
   360  	})
   361  
   362  	proposal := unittest.ProposalFromBlock(&block)
   363  
   364  	return tx, col, &block, proposal, seq + 1
   365  }
   366  
   367  // Test a successful tx should change the statecommitment,
   368  // but a failed Tx should not change the statecommitment.
   369  func TestFailedTxWillNotChangeStateCommitment(t *testing.T) {
   370  	hub := stub.NewNetworkHub()
   371  
   372  	chainID := flow.Emulator
   373  
   374  	colNodeInfo := unittest.PrivateNodeInfoFixture(
   375  		unittest.WithRole(flow.RoleCollection),
   376  		unittest.WithKeys,
   377  	)
   378  	conNodeInfo := unittest.PrivateNodeInfoFixture(
   379  		unittest.WithRole(flow.RoleConsensus),
   380  		unittest.WithKeys,
   381  	)
   382  	exe1NodeInfo := unittest.PrivateNodeInfoFixture(
   383  		unittest.WithRole(flow.RoleExecution),
   384  		unittest.WithKeys,
   385  	)
   386  
   387  	colID := colNodeInfo.Identity()
   388  	conID := conNodeInfo.Identity()
   389  	exe1ID := exe1NodeInfo.Identity()
   390  
   391  	identities := unittest.CompleteIdentitySet(colID, conID, exe1ID)
   392  	key := unittest.NetworkingPrivKeyFixture()
   393  	identities[3].NetworkPubKey = key.PublicKey()
   394  
   395  	collectionNode := testutil.GenericNodeFromParticipants(t, hub, colNodeInfo, identities, chainID)
   396  	defer collectionNode.Done()
   397  	consensusNode := testutil.GenericNodeFromParticipants(t, hub, conNodeInfo, identities, chainID)
   398  	defer consensusNode.Done()
   399  	exe1Node := testutil.ExecutionNode(t, hub, exe1NodeInfo, identities, 27, chainID)
   400  
   401  	ctx, cancel := context.WithCancel(context.Background())
   402  	unittest.RequireReturnsBefore(t, func() {
   403  		exe1Node.Ready(ctx)
   404  	}, 1*time.Second, "could not start execution node on time")
   405  	defer exe1Node.Done(cancel)
   406  
   407  	genesis, err := exe1Node.Blocks.ByHeight(0)
   408  	require.NoError(t, err)
   409  
   410  	seq := uint64(0)
   411  
   412  	chain := exe1Node.ChainID.Chain()
   413  
   414  	// transaction that will change state and succeed, used to test that state commitment changes
   415  	// genesis <- block1 [tx1] <- block2 [tx2] <- block3 [tx3] <- child
   416  	_, col1, block1, proposal1, seq := deployContractBlock(t, conID, colID, chain, seq, genesis, genesis.Header)
   417  
   418  	// we don't set the proper sequence number of this one
   419  	_, col2, block2, proposal2, _ := makePanicBlock(t, conID, colID, chain, uint64(0), block1, genesis.Header)
   420  
   421  	_, col3, block3, proposal3, seq := makeSuccessBlock(t, conID, colID, chain, seq, block2, genesis.Header)
   422  
   423  	_, _, _, proposal4, _ := makeSuccessBlock(t, conID, colID, chain, seq, block3, genesis.Header)
   424  	// seq++
   425  
   426  	// setup mocks and assertions
   427  	collectionEngine := mockCollectionEngineToReturnCollections(
   428  		t,
   429  		&collectionNode,
   430  		[]*flow.Collection{col1, col2, col3},
   431  	)
   432  
   433  	receiptsReceived := atomic.Uint64{}
   434  
   435  	consensusEngine := new(mocknetwork.Engine)
   436  	_, _ = consensusNode.Net.Register(channels.ReceiveReceipts, consensusEngine)
   437  	consensusEngine.On("Process", mock.AnythingOfType("channels.Channel"), mock.Anything, mock.Anything).
   438  		Run(func(args mock.Arguments) {
   439  			receiptsReceived.Inc()
   440  			originID := args[1].(flow.Identifier)
   441  			receipt := args[2].(*flow.ExecutionReceipt)
   442  			finalState, _ := receipt.ExecutionResult.FinalStateCommitment()
   443  			consensusNode.Log.Debug().
   444  				Hex("origin", originID[:]).
   445  				Hex("block", receipt.ExecutionResult.BlockID[:]).
   446  				Hex("final_state_commit", finalState[:]).
   447  				Msg("execution receipt delivered")
   448  		}).Return(nil)
   449  
   450  	// submit block2 from consensus node to execution node 1
   451  	err = sendBlock(&exe1Node, conID.NodeID, proposal1)
   452  	require.NoError(t, err)
   453  
   454  	err = sendBlock(&exe1Node, conID.NodeID, proposal2)
   455  	assert.NoError(t, err)
   456  
   457  	// ensure block 1 has been executed
   458  	hub.DeliverAllEventually(t, func() bool {
   459  		return receiptsReceived.Load() == 1
   460  	})
   461  
   462  	if exe1Node.StorehouseEnabled {
   463  		exe1Node.AssertHighestExecutedBlock(t, genesis.Header)
   464  	} else {
   465  		exe1Node.AssertHighestExecutedBlock(t, block1.Header)
   466  	}
   467  
   468  	exe1Node.AssertBlockIsExecuted(t, block1.Header)
   469  	exe1Node.AssertBlockNotExecuted(t, block2.Header)
   470  
   471  	scExe1Genesis, err := exe1Node.ExecutionState.StateCommitmentByBlockID(genesis.ID())
   472  	assert.NoError(t, err)
   473  
   474  	scExe1Block1, err := exe1Node.ExecutionState.StateCommitmentByBlockID(block1.ID())
   475  	assert.NoError(t, err)
   476  	assert.NotEqual(t, scExe1Genesis, scExe1Block1)
   477  
   478  	// submit block 3 and block 4 from consensus node to execution node 1 (who have block1),
   479  	err = sendBlock(&exe1Node, conID.NodeID, proposal3)
   480  	assert.NoError(t, err)
   481  
   482  	err = sendBlock(&exe1Node, conID.NodeID, proposal4)
   483  	assert.NoError(t, err)
   484  
   485  	// ensure block 1, 2 and 3 have been executed
   486  	hub.DeliverAllEventually(t, func() bool {
   487  		return receiptsReceived.Load() == 3
   488  	})
   489  
   490  	// ensure state has been synced across both nodes
   491  	exe1Node.AssertBlockIsExecuted(t, block2.Header)
   492  	exe1Node.AssertBlockIsExecuted(t, block3.Header)
   493  
   494  	// verify state commitment of block 2 is the same as block 1, since tx failed on seq number verification
   495  	scExe1Block2, err := exe1Node.ExecutionState.StateCommitmentByBlockID(block2.ID())
   496  	assert.NoError(t, err)
   497  	// TODO this is no longer valid because the system chunk can change the state
   498  	//assert.Equal(t, scExe1Block1, scExe1Block2)
   499  	_ = scExe1Block2
   500  
   501  	collectionEngine.AssertExpectations(t)
   502  	consensusEngine.AssertExpectations(t)
   503  }
   504  
   505  func mockCollectionEngineToReturnCollections(t *testing.T, collectionNode *testmock.GenericNode, cols []*flow.Collection) *mocknetwork.Engine {
   506  	collectionEngine := new(mocknetwork.Engine)
   507  	colConduit, _ := collectionNode.Net.Register(channels.RequestCollections, collectionEngine)
   508  
   509  	// make lookup
   510  	colMap := make(map[flow.Identifier][]byte)
   511  	for _, col := range cols {
   512  		blob, _ := msgpack.Marshal(col)
   513  		colMap[col.ID()] = blob
   514  	}
   515  	collectionEngine.On("Process", mock.AnythingOfType("channels.Channel"), mock.Anything, mock.Anything).
   516  		Run(func(args mock.Arguments) {
   517  			originID := args[1].(flow.Identifier)
   518  			req := args[2].(*messages.EntityRequest)
   519  			blob, ok := colMap[req.EntityIDs[0]]
   520  			if !ok {
   521  				assert.FailNow(t, "requesting unexpected collection", req.EntityIDs[0])
   522  			}
   523  			res := &messages.EntityResponse{Blobs: [][]byte{blob}, EntityIDs: req.EntityIDs[:1]}
   524  			err := colConduit.Publish(res, originID)
   525  			assert.NoError(t, err)
   526  		}).
   527  		Return(nil).
   528  		Times(len(cols))
   529  	return collectionEngine
   530  }
   531  
   532  // Test the receipt will be sent to multiple verification nodes
   533  func TestBroadcastToMultipleVerificationNodes(t *testing.T) {
   534  	hub := stub.NewNetworkHub()
   535  
   536  	chainID := flow.Emulator
   537  
   538  	colID := unittest.PrivateNodeInfoFixture(
   539  		unittest.WithRole(flow.RoleCollection),
   540  		unittest.WithKeys,
   541  	)
   542  	conID := unittest.PrivateNodeInfoFixture(
   543  		unittest.WithRole(flow.RoleConsensus),
   544  		unittest.WithKeys,
   545  	)
   546  	exeID := unittest.PrivateNodeInfoFixture(
   547  		unittest.WithRole(flow.RoleExecution),
   548  		unittest.WithKeys,
   549  	)
   550  	ver1ID := unittest.PrivateNodeInfoFixture(
   551  		unittest.WithRole(flow.RoleVerification),
   552  		unittest.WithKeys,
   553  	)
   554  	ver2ID := unittest.PrivateNodeInfoFixture(
   555  		unittest.WithRole(flow.RoleVerification),
   556  		unittest.WithKeys,
   557  	)
   558  
   559  	identities := unittest.CompleteIdentitySet(colID.Identity(),
   560  		conID.Identity(),
   561  		exeID.Identity(),
   562  		ver1ID.Identity(),
   563  		ver2ID.Identity(),
   564  	)
   565  
   566  	exeNode := testutil.ExecutionNode(t, hub, exeID, identities, 21, chainID)
   567  	ctx, cancel := context.WithCancel(context.Background())
   568  
   569  	unittest.RequireReturnsBefore(t, func() {
   570  		exeNode.Ready(ctx)
   571  	}, 1*time.Second, "could not start execution node on time")
   572  	defer exeNode.Done(cancel)
   573  
   574  	verification1Node := testutil.GenericNodeFromParticipants(t, hub, ver1ID, identities, chainID)
   575  	defer verification1Node.Done()
   576  	verification2Node := testutil.GenericNodeFromParticipants(t, hub, ver2ID, identities, chainID)
   577  	defer verification2Node.Done()
   578  
   579  	genesis, err := exeNode.Blocks.ByHeight(0)
   580  	require.NoError(t, err)
   581  
   582  	block := unittest.BlockWithParentAndProposerFixture(t, genesis.Header, conID.NodeID)
   583  	voterIndices, err := signature.EncodeSignersToIndices([]flow.Identifier{conID.NodeID}, []flow.Identifier{conID.NodeID})
   584  	require.NoError(t, err)
   585  	block.Header.ParentVoterIndices = voterIndices
   586  	block.SetPayload(unittest.PayloadFixture(unittest.WithProtocolStateID(genesis.Payload.ProtocolStateID)))
   587  	proposal := unittest.ProposalFromBlock(&block)
   588  
   589  	child := unittest.BlockWithParentAndProposerFixture(t, block.Header, conID.NodeID)
   590  	child.Header.ParentVoterIndices = voterIndices
   591  
   592  	actualCalls := atomic.Uint64{}
   593  
   594  	verificationEngine := new(mocknetwork.Engine)
   595  	_, _ = verification1Node.Net.Register(channels.ReceiveReceipts, verificationEngine)
   596  	_, _ = verification2Node.Net.Register(channels.ReceiveReceipts, verificationEngine)
   597  	verificationEngine.On("Process", mock.AnythingOfType("channels.Channel"), exeID.NodeID, mock.Anything).
   598  		Run(func(args mock.Arguments) {
   599  			actualCalls.Inc()
   600  
   601  			var receipt *flow.ExecutionReceipt
   602  			receipt, _ = args[2].(*flow.ExecutionReceipt)
   603  
   604  			assert.Equal(t, block.ID(), receipt.ExecutionResult.BlockID)
   605  			for i, chunk := range receipt.ExecutionResult.Chunks {
   606  				assert.EqualValues(t, i, chunk.CollectionIndex)
   607  				assert.Greater(t, chunk.TotalComputationUsed, uint64(0))
   608  			}
   609  		}).
   610  		Return(nil)
   611  
   612  	err = sendBlock(&exeNode, exeID.NodeID, proposal)
   613  	require.NoError(t, err)
   614  
   615  	err = sendBlock(&exeNode, conID.NodeID, unittest.ProposalFromBlock(&child))
   616  	require.NoError(t, err)
   617  
   618  	hub.DeliverAllEventually(t, func() bool {
   619  		return actualCalls.Load() == 2
   620  	})
   621  
   622  	verificationEngine.AssertExpectations(t)
   623  }