github.com/onflow/flow-go@v0.33.17/engine/consensus/compliance/core_test.go (about)

     1  package compliance
     2  
     3  import (
     4  	"errors"
     5  	"testing"
     6  
     7  	"github.com/stretchr/testify/assert"
     8  	"github.com/stretchr/testify/mock"
     9  	"github.com/stretchr/testify/require"
    10  	"github.com/stretchr/testify/suite"
    11  
    12  	hotstuff "github.com/onflow/flow-go/consensus/hotstuff/mocks"
    13  	"github.com/onflow/flow-go/consensus/hotstuff/model"
    14  	consensus "github.com/onflow/flow-go/engine/consensus/mock"
    15  	"github.com/onflow/flow-go/model/flow"
    16  	"github.com/onflow/flow-go/model/messages"
    17  	realModule "github.com/onflow/flow-go/module"
    18  	real "github.com/onflow/flow-go/module/buffer"
    19  	"github.com/onflow/flow-go/module/compliance"
    20  	"github.com/onflow/flow-go/module/metrics"
    21  	module "github.com/onflow/flow-go/module/mock"
    22  	"github.com/onflow/flow-go/module/trace"
    23  	netint "github.com/onflow/flow-go/network"
    24  	"github.com/onflow/flow-go/network/channels"
    25  	"github.com/onflow/flow-go/network/mocknetwork"
    26  	"github.com/onflow/flow-go/state"
    27  	protint "github.com/onflow/flow-go/state/protocol"
    28  	protocol "github.com/onflow/flow-go/state/protocol/mock"
    29  	storerr "github.com/onflow/flow-go/storage"
    30  	storage "github.com/onflow/flow-go/storage/mock"
    31  	"github.com/onflow/flow-go/utils/unittest"
    32  )
    33  
    34  func TestComplianceCore(t *testing.T) {
    35  	suite.Run(t, new(CoreSuite))
    36  }
    37  
    38  // CoreSuite tests the compliance core logic.
    39  type CoreSuite struct {
    40  	CommonSuite
    41  }
    42  
    43  // CommonSuite is shared between compliance core and engine testing.
    44  type CommonSuite struct {
    45  	suite.Suite
    46  
    47  	// engine parameters
    48  	participants flow.IdentityList
    49  	myID         flow.Identifier
    50  	head         *flow.Header
    51  
    52  	// storage data
    53  	headerDB   map[flow.Identifier]*flow.Header
    54  	payloadDB  map[flow.Identifier]*flow.Payload
    55  	pendingDB  map[flow.Identifier]flow.Slashable[*flow.Block]
    56  	childrenDB map[flow.Identifier][]flow.Slashable[*flow.Block]
    57  
    58  	// mocked dependencies
    59  	me                        *module.Local
    60  	metrics                   *metrics.NoopCollector
    61  	tracer                    realModule.Tracer
    62  	headers                   *storage.Headers
    63  	payloads                  *storage.Payloads
    64  	state                     *protocol.ParticipantState
    65  	snapshot                  *protocol.Snapshot
    66  	con                       *mocknetwork.Conduit
    67  	net                       *mocknetwork.Network
    68  	prov                      *consensus.ProposalProvider
    69  	pending                   *module.PendingBlockBuffer
    70  	hotstuff                  *module.HotStuff
    71  	sync                      *module.BlockRequester
    72  	proposalViolationNotifier *hotstuff.ProposalViolationConsumer
    73  	validator                 *hotstuff.Validator
    74  	voteAggregator            *hotstuff.VoteAggregator
    75  	timeoutAggregator         *hotstuff.TimeoutAggregator
    76  
    77  	// engine under test
    78  	core *Core
    79  }
    80  
    81  func (cs *CommonSuite) SetupTest() {
    82  	// initialize the paramaters
    83  	cs.participants = unittest.IdentityListFixture(3,
    84  		unittest.WithRole(flow.RoleConsensus),
    85  		unittest.WithWeight(1000),
    86  	)
    87  	cs.myID = cs.participants[0].NodeID
    88  	block := unittest.BlockFixture()
    89  	cs.head = block.Header
    90  
    91  	// initialize the storage data
    92  	cs.headerDB = make(map[flow.Identifier]*flow.Header)
    93  	cs.payloadDB = make(map[flow.Identifier]*flow.Payload)
    94  	cs.pendingDB = make(map[flow.Identifier]flow.Slashable[*flow.Block])
    95  	cs.childrenDB = make(map[flow.Identifier][]flow.Slashable[*flow.Block])
    96  
    97  	// store the head header and payload
    98  	cs.headerDB[block.ID()] = block.Header
    99  	cs.payloadDB[block.ID()] = block.Payload
   100  
   101  	// set up local module mock
   102  	cs.me = &module.Local{}
   103  	cs.me.On("NodeID").Return(
   104  		func() flow.Identifier {
   105  			return cs.myID
   106  		},
   107  	)
   108  
   109  	// set up header storage mock
   110  	cs.headers = &storage.Headers{}
   111  	cs.headers.On("Store", mock.Anything).Return(
   112  		func(header *flow.Header) error {
   113  			cs.headerDB[header.ID()] = header
   114  			return nil
   115  		},
   116  	)
   117  	cs.headers.On("ByBlockID", mock.Anything).Return(
   118  		func(blockID flow.Identifier) *flow.Header {
   119  			return cs.headerDB[blockID]
   120  		},
   121  		func(blockID flow.Identifier) error {
   122  			_, exists := cs.headerDB[blockID]
   123  			if !exists {
   124  				return storerr.ErrNotFound
   125  			}
   126  			return nil
   127  		},
   128  	)
   129  	cs.headers.On("Exists", mock.Anything).Return(
   130  		func(blockID flow.Identifier) bool {
   131  			_, exists := cs.headerDB[blockID]
   132  			return exists
   133  		}, func(blockID flow.Identifier) error {
   134  			return nil
   135  		})
   136  
   137  	// set up payload storage mock
   138  	cs.payloads = &storage.Payloads{}
   139  	cs.payloads.On("Store", mock.Anything, mock.Anything).Return(
   140  		func(header *flow.Header, payload *flow.Payload) error {
   141  			cs.payloadDB[header.ID()] = payload
   142  			return nil
   143  		},
   144  	)
   145  	cs.payloads.On("ByBlockID", mock.Anything).Return(
   146  		func(blockID flow.Identifier) *flow.Payload {
   147  			return cs.payloadDB[blockID]
   148  		},
   149  		func(blockID flow.Identifier) error {
   150  			_, exists := cs.payloadDB[blockID]
   151  			if !exists {
   152  				return storerr.ErrNotFound
   153  			}
   154  			return nil
   155  		},
   156  	)
   157  
   158  	// set up protocol state mock
   159  	cs.state = &protocol.ParticipantState{}
   160  	cs.state.On("Final").Return(
   161  		func() protint.Snapshot {
   162  			return cs.snapshot
   163  		},
   164  	)
   165  	cs.state.On("AtBlockID", mock.Anything).Return(
   166  		func(blockID flow.Identifier) protint.Snapshot {
   167  			return cs.snapshot
   168  		},
   169  	)
   170  	cs.state.On("Extend", mock.Anything, mock.Anything).Return(nil)
   171  
   172  	// set up protocol snapshot mock
   173  	cs.snapshot = &protocol.Snapshot{}
   174  	cs.snapshot.On("Identities", mock.Anything).Return(
   175  		func(filter flow.IdentityFilter) flow.IdentityList {
   176  			return cs.participants.Filter(filter)
   177  		},
   178  		nil,
   179  	)
   180  	cs.snapshot.On("Head").Return(
   181  		func() *flow.Header {
   182  			return cs.head
   183  		},
   184  		nil,
   185  	)
   186  
   187  	// set up network conduit mock
   188  	cs.con = &mocknetwork.Conduit{}
   189  	cs.con.On("Publish", mock.Anything, mock.Anything).Return(nil)
   190  	cs.con.On("Publish", mock.Anything, mock.Anything, mock.Anything).Return(nil)
   191  	cs.con.On("Publish", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
   192  	cs.con.On("Unicast", mock.Anything, mock.Anything).Return(nil)
   193  
   194  	// set up network module mock
   195  	cs.net = &mocknetwork.Network{}
   196  	cs.net.On("Register", mock.Anything, mock.Anything).Return(
   197  		func(channel channels.Channel, engine netint.MessageProcessor) netint.Conduit {
   198  			return cs.con
   199  		},
   200  		nil,
   201  	)
   202  
   203  	// set up the provider engine
   204  	cs.prov = &consensus.ProposalProvider{}
   205  	cs.prov.On("ProvideProposal", mock.Anything).Return()
   206  
   207  	// set up pending module mock
   208  	cs.pending = &module.PendingBlockBuffer{}
   209  	cs.pending.On("Add", mock.Anything, mock.Anything).Return(true)
   210  	cs.pending.On("ByID", mock.Anything).Return(
   211  		func(blockID flow.Identifier) flow.Slashable[*flow.Block] {
   212  			return cs.pendingDB[blockID]
   213  		},
   214  		func(blockID flow.Identifier) bool {
   215  			_, ok := cs.pendingDB[blockID]
   216  			return ok
   217  		},
   218  	)
   219  	cs.pending.On("ByParentID", mock.Anything).Return(
   220  		func(blockID flow.Identifier) []flow.Slashable[*flow.Block] {
   221  			return cs.childrenDB[blockID]
   222  		},
   223  		func(blockID flow.Identifier) bool {
   224  			_, ok := cs.childrenDB[blockID]
   225  			return ok
   226  		},
   227  	)
   228  	cs.pending.On("DropForParent", mock.Anything).Return()
   229  	cs.pending.On("Size").Return(uint(0))
   230  	cs.pending.On("PruneByView", mock.Anything).Return()
   231  
   232  	// set up hotstuff module mock
   233  	cs.hotstuff = module.NewHotStuff(cs.T())
   234  
   235  	cs.validator = hotstuff.NewValidator(cs.T())
   236  	cs.voteAggregator = hotstuff.NewVoteAggregator(cs.T())
   237  	cs.timeoutAggregator = hotstuff.NewTimeoutAggregator(cs.T())
   238  
   239  	// set up synchronization module mock
   240  	cs.sync = &module.BlockRequester{}
   241  	cs.sync.On("RequestBlock", mock.Anything, mock.Anything).Return(nil)
   242  	cs.sync.On("Done", mock.Anything).Return(unittest.ClosedChannel())
   243  
   244  	// set up no-op metrics mock
   245  	cs.metrics = metrics.NewNoopCollector()
   246  
   247  	// set up no-op tracer
   248  	cs.tracer = trace.NewNoopTracer()
   249  
   250  	// set up notifier for reporting protocol violations
   251  	cs.proposalViolationNotifier = hotstuff.NewProposalViolationConsumer(cs.T())
   252  
   253  	// initialize the engine
   254  	e, err := NewCore(
   255  		unittest.Logger(),
   256  		cs.metrics,
   257  		cs.metrics,
   258  		cs.metrics,
   259  		cs.metrics,
   260  		cs.proposalViolationNotifier,
   261  		cs.tracer,
   262  		cs.headers,
   263  		cs.payloads,
   264  		cs.state,
   265  		cs.pending,
   266  		cs.sync,
   267  		cs.validator,
   268  		cs.hotstuff,
   269  		cs.voteAggregator,
   270  		cs.timeoutAggregator,
   271  		compliance.DefaultConfig(),
   272  	)
   273  	require.NoError(cs.T(), err, "engine initialization should pass")
   274  
   275  	cs.core = e
   276  }
   277  
   278  func (cs *CoreSuite) TestOnBlockProposalValidParent() {
   279  
   280  	// create a proposal that directly descends from the latest finalized header
   281  	originID := cs.participants[1].NodeID
   282  	block := unittest.BlockWithParentFixture(cs.head)
   283  	proposal := unittest.ProposalFromBlock(block)
   284  
   285  	// store the data for retrieval
   286  	cs.headerDB[block.Header.ParentID] = cs.head
   287  
   288  	hotstuffProposal := model.ProposalFromFlow(block.Header)
   289  	cs.validator.On("ValidateProposal", hotstuffProposal).Return(nil)
   290  	cs.voteAggregator.On("AddBlock", hotstuffProposal).Once()
   291  	cs.hotstuff.On("SubmitProposal", hotstuffProposal)
   292  
   293  	// it should be processed without error
   294  	err := cs.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{
   295  		OriginID: originID,
   296  		Message:  proposal,
   297  	})
   298  	require.NoError(cs.T(), err, "valid block proposal should pass")
   299  
   300  	// we should extend the state with the header
   301  	cs.state.AssertCalled(cs.T(), "Extend", mock.Anything, block)
   302  }
   303  
   304  func (cs *CoreSuite) TestOnBlockProposalValidAncestor() {
   305  
   306  	// create a proposal that has two ancestors in the cache
   307  	originID := cs.participants[1].NodeID
   308  	ancestor := unittest.BlockWithParentFixture(cs.head)
   309  	parent := unittest.BlockWithParentFixture(ancestor.Header)
   310  	block := unittest.BlockWithParentFixture(parent.Header)
   311  	proposal := unittest.ProposalFromBlock(block)
   312  
   313  	// store the data for retrieval
   314  	cs.headerDB[parent.ID()] = parent.Header
   315  	cs.headerDB[ancestor.ID()] = ancestor.Header
   316  
   317  	hotstuffProposal := model.ProposalFromFlow(block.Header)
   318  	cs.validator.On("ValidateProposal", hotstuffProposal).Return(nil)
   319  	cs.voteAggregator.On("AddBlock", hotstuffProposal).Once()
   320  	cs.hotstuff.On("SubmitProposal", hotstuffProposal)
   321  
   322  	// it should be processed without error
   323  	err := cs.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{
   324  		OriginID: originID,
   325  		Message:  proposal,
   326  	})
   327  	require.NoError(cs.T(), err, "valid block proposal should pass")
   328  
   329  	// we should extend the state with the header
   330  	cs.state.AssertCalled(cs.T(), "Extend", mock.Anything, block)
   331  }
   332  
   333  func (cs *CoreSuite) TestOnBlockProposalSkipProposalThreshold() {
   334  
   335  	// create a proposal which is far enough ahead to be dropped
   336  	originID := cs.participants[1].NodeID
   337  	block := unittest.BlockFixture()
   338  	block.Header.View = cs.head.View + compliance.DefaultConfig().SkipNewProposalsThreshold + 1
   339  	proposal := unittest.ProposalFromBlock(&block)
   340  
   341  	err := cs.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{
   342  		OriginID: originID,
   343  		Message:  proposal,
   344  	})
   345  	require.NoError(cs.T(), err)
   346  
   347  	// block should be dropped - not added to state or cache
   348  	cs.state.AssertNotCalled(cs.T(), "Extend", mock.Anything)
   349  	cs.validator.AssertNotCalled(cs.T(), "ValidateProposal", mock.Anything)
   350  	cs.pending.AssertNotCalled(cs.T(), "Add", originID, mock.Anything)
   351  }
   352  
   353  // TestOnBlockProposal_FailsHotStuffValidation tests that a proposal which fails HotStuff validation.
   354  //   - should not go through protocol state validation
   355  //   - should not be added to the state
   356  //   - we should not attempt to process its children
   357  //   - we should notify VoteAggregator, for known errors
   358  func (cs *CoreSuite) TestOnBlockProposal_FailsHotStuffValidation() {
   359  
   360  	// create a proposal that has two ancestors in the cache
   361  	originID := cs.participants[1].NodeID
   362  	ancestor := unittest.BlockWithParentFixture(cs.head)
   363  	parent := unittest.BlockWithParentFixture(ancestor.Header)
   364  	block := unittest.BlockWithParentFixture(parent.Header)
   365  	proposal := unittest.ProposalFromBlock(block)
   366  	hotstuffProposal := model.ProposalFromFlow(block.Header)
   367  
   368  	// store the data for retrieval
   369  	cs.headerDB[parent.ID()] = parent.Header
   370  	cs.headerDB[ancestor.ID()] = ancestor.Header
   371  
   372  	cs.Run("invalid block error", func() {
   373  		// the block fails HotStuff validation
   374  		*cs.validator = *hotstuff.NewValidator(cs.T())
   375  		sentinelError := model.NewInvalidProposalErrorf(hotstuffProposal, "")
   376  		cs.validator.On("ValidateProposal", hotstuffProposal).Return(sentinelError)
   377  		cs.proposalViolationNotifier.On("OnInvalidBlockDetected", flow.Slashable[model.InvalidProposalError]{
   378  			OriginID: originID,
   379  			Message:  sentinelError.(model.InvalidProposalError),
   380  		}).Return().Once()
   381  		// we should notify VoteAggregator about the invalid block
   382  		cs.voteAggregator.On("InvalidBlock", hotstuffProposal).Return(nil)
   383  
   384  		// the expected error should be handled within the Core
   385  		err := cs.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{
   386  			OriginID: originID,
   387  			Message:  proposal,
   388  		})
   389  		require.NoError(cs.T(), err, "proposal with invalid extension should fail")
   390  
   391  		// we should not extend the state with the header
   392  		cs.state.AssertNotCalled(cs.T(), "Extend", mock.Anything, block)
   393  		// we should not attempt to process the children
   394  		cs.pending.AssertNotCalled(cs.T(), "ByParentID", mock.Anything)
   395  	})
   396  
   397  	cs.Run("view for unknown epoch error", func() {
   398  		// the block fails HotStuff validation
   399  		*cs.validator = *hotstuff.NewValidator(cs.T())
   400  		cs.validator.On("ValidateProposal", hotstuffProposal).Return(model.ErrViewForUnknownEpoch)
   401  
   402  		// the expected error should be handled within the Core
   403  		err := cs.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{
   404  			OriginID: originID,
   405  			Message:  proposal,
   406  		})
   407  		require.NoError(cs.T(), err, "proposal with invalid extension should fail")
   408  
   409  		// we should not extend the state with the header
   410  		cs.state.AssertNotCalled(cs.T(), "Extend", mock.Anything, block)
   411  		// we should not attempt to process the children
   412  		cs.pending.AssertNotCalled(cs.T(), "ByParentID", mock.Anything)
   413  	})
   414  
   415  	cs.Run("unexpected error", func() {
   416  		// the block fails HotStuff validation
   417  		unexpectedErr := errors.New("generic unexpected error")
   418  		*cs.validator = *hotstuff.NewValidator(cs.T())
   419  		cs.validator.On("ValidateProposal", hotstuffProposal).Return(unexpectedErr)
   420  
   421  		// the error should be propagated
   422  		err := cs.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{
   423  			OriginID: originID,
   424  			Message:  proposal,
   425  		})
   426  		require.ErrorIs(cs.T(), err, unexpectedErr)
   427  
   428  		// we should not extend the state with the header
   429  		cs.state.AssertNotCalled(cs.T(), "Extend", mock.Anything, block)
   430  		// we should not attempt to process the children
   431  		cs.pending.AssertNotCalled(cs.T(), "ByParentID", mock.Anything)
   432  	})
   433  }
   434  
   435  // TestOnBlockProposal_FailsProtocolStateValidation tests processing a proposal which passes HotStuff validation,
   436  // but fails protocol state validation
   437  //   - should not be added to the state
   438  //   - we should not attempt to process its children
   439  //   - we should notify VoteAggregator, for known errors
   440  func (cs *CoreSuite) TestOnBlockProposal_FailsProtocolStateValidation() {
   441  
   442  	// create a proposal that has two ancestors in the cache
   443  	originID := cs.participants[1].NodeID
   444  	ancestor := unittest.BlockWithParentFixture(cs.head)
   445  	parent := unittest.BlockWithParentFixture(ancestor.Header)
   446  	block := unittest.BlockWithParentFixture(parent.Header)
   447  	proposal := unittest.ProposalFromBlock(block)
   448  	hotstuffProposal := model.ProposalFromFlow(block.Header)
   449  
   450  	// store the data for retrieval
   451  	cs.headerDB[parent.ID()] = parent.Header
   452  	cs.headerDB[ancestor.ID()] = ancestor.Header
   453  
   454  	// the block passes HotStuff validation
   455  	cs.validator.On("ValidateProposal", hotstuffProposal).Return(nil)
   456  
   457  	cs.Run("invalid block", func() {
   458  		// make sure we fail to extend the state
   459  		*cs.state = protocol.ParticipantState{}
   460  		cs.state.On("Final").Return(func() protint.Snapshot { return cs.snapshot })
   461  		sentinelErr := state.NewInvalidExtensionError("")
   462  		cs.state.On("Extend", mock.Anything, mock.Anything).Return(sentinelErr)
   463  		cs.proposalViolationNotifier.On("OnInvalidBlockDetected", mock.Anything).Run(func(args mock.Arguments) {
   464  			err := args.Get(0).(flow.Slashable[model.InvalidProposalError])
   465  			require.ErrorIs(cs.T(), err.Message, sentinelErr)
   466  			require.Equal(cs.T(), err.Message.InvalidProposal, hotstuffProposal)
   467  			require.Equal(cs.T(), err.OriginID, originID)
   468  		}).Return().Once()
   469  		// we should notify VoteAggregator about the invalid block
   470  		cs.voteAggregator.On("InvalidBlock", hotstuffProposal).Return(nil)
   471  
   472  		// the expected error should be handled within the Core
   473  		err := cs.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{
   474  			OriginID: originID,
   475  			Message:  proposal,
   476  		})
   477  		require.NoError(cs.T(), err, "proposal with invalid extension should fail")
   478  
   479  		// we should extend the state with the header
   480  		cs.state.AssertCalled(cs.T(), "Extend", mock.Anything, block)
   481  		// we should not pass the block to hotstuff
   482  		cs.hotstuff.AssertNotCalled(cs.T(), "SubmitProposal", mock.Anything)
   483  		// we should not attempt to process the children
   484  		cs.pending.AssertNotCalled(cs.T(), "ByParentID", mock.Anything)
   485  	})
   486  
   487  	cs.Run("outdated block", func() {
   488  		// make sure we fail to extend the state
   489  		*cs.state = protocol.ParticipantState{}
   490  		cs.state.On("Final").Return(func() protint.Snapshot { return cs.snapshot })
   491  		cs.state.On("Extend", mock.Anything, mock.Anything).Return(state.NewOutdatedExtensionError(""))
   492  
   493  		// the expected error should be handled within the Core
   494  		err := cs.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{
   495  			OriginID: originID,
   496  			Message:  proposal,
   497  		})
   498  		require.NoError(cs.T(), err, "proposal with invalid extension should fail")
   499  
   500  		// we should extend the state with the header
   501  		cs.state.AssertCalled(cs.T(), "Extend", mock.Anything, block)
   502  		// we should not pass the block to hotstuff
   503  		cs.hotstuff.AssertNotCalled(cs.T(), "SubmitProposal", mock.Anything)
   504  		// we should not attempt to process the children
   505  		cs.pending.AssertNotCalled(cs.T(), "ByParentID", mock.Anything)
   506  	})
   507  
   508  	cs.Run("unexpected error", func() {
   509  		// make sure we fail to extend the state
   510  		*cs.state = protocol.ParticipantState{}
   511  		cs.state.On("Final").Return(func() protint.Snapshot { return cs.snapshot })
   512  		unexpectedErr := errors.New("unexpected generic error")
   513  		cs.state.On("Extend", mock.Anything, mock.Anything).Return(unexpectedErr)
   514  
   515  		// it should be processed without error
   516  		err := cs.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{
   517  			OriginID: originID,
   518  			Message:  proposal,
   519  		})
   520  		require.ErrorIs(cs.T(), err, unexpectedErr)
   521  
   522  		// we should extend the state with the header
   523  		cs.state.AssertCalled(cs.T(), "Extend", mock.Anything, block)
   524  		// we should not pass the block to hotstuff
   525  		cs.hotstuff.AssertNotCalled(cs.T(), "SubmitProposal", mock.Anything)
   526  		// we should not attempt to process the children
   527  		cs.pending.AssertNotCalled(cs.T(), "ByParentID", mock.Anything)
   528  	})
   529  }
   530  
   531  func (cs *CoreSuite) TestProcessBlockAndDescendants() {
   532  
   533  	// create three children blocks
   534  	parent := unittest.BlockWithParentFixture(cs.head)
   535  	block1 := unittest.BlockWithParentFixture(parent.Header)
   536  	block2 := unittest.BlockWithParentFixture(parent.Header)
   537  	block3 := unittest.BlockWithParentFixture(parent.Header)
   538  
   539  	// create the pending blocks
   540  	pending1 := unittest.AsSlashable(block1)
   541  	pending2 := unittest.AsSlashable(block2)
   542  	pending3 := unittest.AsSlashable(block3)
   543  
   544  	// store the parent on disk
   545  	parentID := parent.ID()
   546  	cs.headerDB[parentID] = parent.Header
   547  
   548  	// store the pending children in the cache
   549  	cs.childrenDB[parentID] = append(cs.childrenDB[parentID], pending1)
   550  	cs.childrenDB[parentID] = append(cs.childrenDB[parentID], pending2)
   551  	cs.childrenDB[parentID] = append(cs.childrenDB[parentID], pending3)
   552  
   553  	for _, block := range []*flow.Block{parent, block1, block2, block3} {
   554  		hotstuffProposal := model.ProposalFromFlow(block.Header)
   555  		cs.validator.On("ValidateProposal", hotstuffProposal).Return(nil)
   556  		cs.voteAggregator.On("AddBlock", hotstuffProposal).Once()
   557  		cs.hotstuff.On("SubmitProposal", hotstuffProposal).Once()
   558  	}
   559  
   560  	// execute the connected children handling
   561  	err := cs.core.processBlockAndDescendants(flow.Slashable[*flow.Block]{
   562  		OriginID: unittest.IdentifierFixture(),
   563  		Message:  parent,
   564  	})
   565  	require.NoError(cs.T(), err, "should pass handling children")
   566  
   567  	// make sure we drop the cache after trying to process
   568  	cs.pending.AssertCalled(cs.T(), "DropForParent", parent.Header.ID())
   569  }
   570  
   571  func (cs *CoreSuite) TestProposalBufferingOrder() {
   572  
   573  	// create a proposal that we will not submit until the end
   574  	originID := cs.participants[1].NodeID
   575  	missingBlock := unittest.BlockWithParentFixture(cs.head)
   576  	missingProposal := unittest.ProposalFromBlock(missingBlock)
   577  
   578  	// create a chain of descendants
   579  	var proposals []*messages.BlockProposal
   580  	parent := missingProposal
   581  	for i := 0; i < 3; i++ {
   582  		descendant := unittest.BlockWithParentFixture(&parent.Block.Header)
   583  		proposal := unittest.ProposalFromBlock(descendant)
   584  		proposals = append(proposals, proposal)
   585  		parent = proposal
   586  	}
   587  
   588  	// replace the engine buffer with the real one
   589  	cs.core.pending = real.NewPendingBlocks()
   590  
   591  	// check that we request the ancestor block each time
   592  	cs.sync.On("RequestBlock", missingBlock.Header.ID(), missingBlock.Header.Height).Times(len(proposals))
   593  
   594  	// process all the descendants
   595  	for _, proposal := range proposals {
   596  		// process and make sure no error occurs (as they are unverifiable)
   597  		err := cs.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{
   598  			OriginID: originID,
   599  			Message:  proposal,
   600  		})
   601  		require.NoError(cs.T(), err, "proposal buffering should pass")
   602  
   603  		// make sure no block is forwarded to hotstuff
   604  		cs.hotstuff.AssertNotCalled(cs.T(), "SubmitProposal", model.ProposalFromFlow(&proposal.Block.Header))
   605  	}
   606  
   607  	// check that we submit each proposal in a valid order
   608  	//  - we must process the missingProposal parent first
   609  	//  - we can process the children next, in any order
   610  	cs.validator.On("ValidateProposal", mock.Anything).Return(nil).Times(4)
   611  
   612  	calls := 0                                   // track # of calls to SubmitProposal
   613  	unprocessed := map[flow.Identifier]struct{}{ // track un-processed proposals
   614  		missingProposal.Block.Header.ID(): {},
   615  		proposals[0].Block.Header.ID():    {},
   616  		proposals[1].Block.Header.ID():    {},
   617  		proposals[2].Block.Header.ID():    {},
   618  	}
   619  	cs.hotstuff.On("SubmitProposal", mock.Anything).Times(4).Run(
   620  		func(args mock.Arguments) {
   621  			proposal := args.Get(0).(*model.Proposal)
   622  			header := proposal.Block
   623  			if calls == 0 {
   624  				// first header processed must be the common parent
   625  				assert.Equal(cs.T(), missingProposal.Block.Header.ID(), header.BlockID)
   626  			}
   627  			// mark the proposal as processed
   628  			delete(unprocessed, header.BlockID)
   629  			cs.headerDB[header.BlockID] = model.ProposalToFlow(proposal)
   630  			calls++
   631  		},
   632  	)
   633  	cs.voteAggregator.On("AddBlock", mock.Anything).Times(4)
   634  
   635  	// process the root proposal
   636  	err := cs.core.OnBlockProposal(flow.Slashable[*messages.BlockProposal]{
   637  		OriginID: originID,
   638  		Message:  missingProposal,
   639  	})
   640  	require.NoError(cs.T(), err, "root proposal should pass")
   641  
   642  	// all proposals should be processed
   643  	assert.Len(cs.T(), unprocessed, 0)
   644  }