github.com/koko1123/flow-go-1@v0.29.6/engine/consensus/sealing/core_test.go (about)

     1  package sealing
     2  
     3  import (
     4  	"fmt"
     5  	"testing"
     6  	"time"
     7  
     8  	"github.com/stretchr/testify/mock"
     9  	"github.com/stretchr/testify/require"
    10  	"github.com/stretchr/testify/suite"
    11  
    12  	"github.com/koko1123/flow-go-1/engine"
    13  	"github.com/koko1123/flow-go-1/engine/consensus/approvals"
    14  	"github.com/koko1123/flow-go-1/engine/consensus/approvals/tracker"
    15  	"github.com/koko1123/flow-go-1/model/chunks"
    16  	"github.com/koko1123/flow-go-1/model/flow"
    17  	realmodule "github.com/koko1123/flow-go-1/module"
    18  	"github.com/koko1123/flow-go-1/module/metrics"
    19  	module "github.com/koko1123/flow-go-1/module/mock"
    20  	"github.com/koko1123/flow-go-1/module/trace"
    21  	"github.com/koko1123/flow-go-1/module/updatable_configs"
    22  	mockstate "github.com/koko1123/flow-go-1/state/protocol/mock"
    23  	storage "github.com/koko1123/flow-go-1/storage/mock"
    24  	"github.com/koko1123/flow-go-1/utils/unittest"
    25  )
    26  
    27  // TestApprovalProcessingCore performs testing of approval processing core
    28  // Core is responsible for delegating processing to assignment collectorTree for each separate execution result
    29  // Core performs height based checks and decides if approval or incorporated result has to be processed at all
    30  // or rejected as outdated or unverifiable.
    31  // Core maintains a LRU cache of known approvals that cannot be verified at the moment/
    32  func TestApprovalProcessingCore(t *testing.T) {
    33  	suite.Run(t, new(ApprovalProcessingCoreTestSuite))
    34  }
    35  
    36  // RequiredApprovalsForSealConstructionTestingValue defines the number of approvals that are
    37  // required to construct a seal for testing purposes. Thereby, the default production value
    38  // can be set independently without changing test behaviour.
    39  const RequiredApprovalsForSealConstructionTestingValue = 1
    40  
    41  type ApprovalProcessingCoreTestSuite struct {
    42  	approvals.BaseAssignmentCollectorTestSuite
    43  
    44  	sealsDB    *storage.Seals
    45  	rootHeader *flow.Header
    46  	core       *Core
    47  	setter     realmodule.SealingConfigsSetter
    48  }
    49  
    50  func (s *ApprovalProcessingCoreTestSuite) TearDownTest() {
    51  	s.BaseAssignmentCollectorTestSuite.TearDownTest()
    52  }
    53  
    54  func (s *ApprovalProcessingCoreTestSuite) SetupTest() {
    55  	s.BaseAssignmentCollectorTestSuite.SetupTest()
    56  
    57  	s.sealsDB = &storage.Seals{}
    58  
    59  	s.rootHeader = unittest.GenesisFixture().Header
    60  	params := new(mockstate.Params)
    61  	s.State.On("Sealed").Return(unittest.StateSnapshotForKnownBlock(s.ParentBlock, nil)).Maybe()
    62  	s.State.On("Params").Return(params)
    63  	params.On("Root").Return(
    64  		func() *flow.Header { return s.rootHeader },
    65  		func() error { return nil },
    66  	)
    67  
    68  	metrics := metrics.NewNoopCollector()
    69  	tracer := trace.NewNoopTracer()
    70  
    71  	setter := unittest.NewSealingConfigs(flow.DefaultChunkAssignmentAlpha)
    72  	var err error
    73  	s.core, err = NewCore(unittest.Logger(), s.WorkerPool, tracer, metrics, &tracker.NoopSealingTracker{}, engine.NewUnit(), s.Headers, s.State, s.sealsDB, s.Assigner, s.SigHasher, s.SealsPL, s.Conduit, setter)
    74  	require.NoError(s.T(), err)
    75  	s.setter = setter
    76  }
    77  
    78  // TestOnBlockFinalized_RejectOutdatedApprovals tests that approvals will be rejected as outdated
    79  // for block that is already sealed
    80  func (s *ApprovalProcessingCoreTestSuite) TestOnBlockFinalized_RejectOutdatedApprovals() {
    81  	approval := unittest.ResultApprovalFixture(unittest.WithApproverID(s.VerID),
    82  		unittest.WithChunk(s.Chunks[0].Index),
    83  		unittest.WithBlockID(s.Block.ID()))
    84  	err := s.core.processApproval(approval)
    85  	require.NoError(s.T(), err)
    86  
    87  	seal := unittest.Seal.Fixture(unittest.Seal.WithBlock(s.Block))
    88  	s.sealsDB.On("HighestInFork", mock.Anything).Return(seal, nil).Once()
    89  
    90  	err = s.core.ProcessFinalizedBlock(s.Block.ID())
    91  	require.NoError(s.T(), err)
    92  
    93  	err = s.core.processApproval(approval)
    94  	require.Error(s.T(), err)
    95  	require.True(s.T(), engine.IsOutdatedInputError(err))
    96  }
    97  
    98  // TestOnBlockFinalized_RejectOutdatedExecutionResult tests that incorporated result will be rejected as outdated
    99  // if the block which is targeted by execution result is already sealed.
   100  func (s *ApprovalProcessingCoreTestSuite) TestOnBlockFinalized_RejectOutdatedExecutionResult() {
   101  	seal := unittest.Seal.Fixture(unittest.Seal.WithBlock(s.Block))
   102  	s.sealsDB.On("HighestInFork", mock.Anything).Return(seal, nil).Once()
   103  
   104  	err := s.core.ProcessFinalizedBlock(s.Block.ID())
   105  	require.NoError(s.T(), err)
   106  
   107  	err = s.core.processIncorporatedResult(s.IncorporatedResult)
   108  	require.Error(s.T(), err)
   109  	require.True(s.T(), engine.IsOutdatedInputError(err))
   110  }
   111  
   112  // TestOnBlockFinalized_RejectUnverifiableEntries tests that core will reject both execution results
   113  // and approvals for blocks that we have no information about.
   114  func (s *ApprovalProcessingCoreTestSuite) TestOnBlockFinalized_RejectUnverifiableEntries() {
   115  	s.IncorporatedResult.Result.BlockID = unittest.IdentifierFixture() // replace blockID with random one
   116  	err := s.core.processIncorporatedResult(s.IncorporatedResult)
   117  	require.Error(s.T(), err)
   118  	require.True(s.T(), engine.IsUnverifiableInputError(err))
   119  
   120  	approval := unittest.ResultApprovalFixture(unittest.WithApproverID(s.VerID),
   121  		unittest.WithChunk(s.Chunks[0].Index))
   122  
   123  	err = s.core.processApproval(approval)
   124  	require.Error(s.T(), err)
   125  	require.True(s.T(), engine.IsUnverifiableInputError(err))
   126  }
   127  
   128  // TestOnBlockFinalized_RejectOrphanIncorporatedResults tests that execution results incorporated in orphan blocks
   129  // are rejected as outdated in next situation
   130  //
   131  //	A <- B_1
   132  //		 <- B_2
   133  //
   134  // B_1 is finalized rendering B_2 as orphan, submitting IR[ER[A], B_1] is a success, submitting IR[ER[A], B_2] is an outdated incorporated result
   135  func (s *ApprovalProcessingCoreTestSuite) TestOnBlockFinalized_RejectOrphanIncorporatedResults() {
   136  	blockB1 := unittest.BlockHeaderWithParentFixture(s.Block)
   137  	blockB2 := unittest.BlockHeaderWithParentFixture(s.Block)
   138  
   139  	s.Blocks[blockB1.ID()] = blockB1
   140  	s.Blocks[blockB2.ID()] = blockB2
   141  
   142  	IR1 := unittest.IncorporatedResult.Fixture(
   143  		unittest.IncorporatedResult.WithIncorporatedBlockID(blockB1.ID()),
   144  		unittest.IncorporatedResult.WithResult(s.IncorporatedResult.Result))
   145  
   146  	IR2 := unittest.IncorporatedResult.Fixture(
   147  		unittest.IncorporatedResult.WithIncorporatedBlockID(blockB2.ID()),
   148  		unittest.IncorporatedResult.WithResult(s.IncorporatedResult.Result))
   149  
   150  	s.MarkFinalized(blockB1)
   151  
   152  	seal := unittest.Seal.Fixture(unittest.Seal.WithBlock(s.ParentBlock))
   153  	s.sealsDB.On("HighestInFork", mock.Anything).Return(seal, nil).Once()
   154  
   155  	// blockB1 becomes finalized
   156  	err := s.core.ProcessFinalizedBlock(blockB1.ID())
   157  	require.NoError(s.T(), err)
   158  
   159  	err = s.core.processIncorporatedResult(IR1)
   160  	require.NoError(s.T(), err)
   161  
   162  	err = s.core.processIncorporatedResult(IR2)
   163  	require.Error(s.T(), err)
   164  	require.True(s.T(), engine.IsOutdatedInputError(err))
   165  }
   166  
   167  func (s *ApprovalProcessingCoreTestSuite) TestOnBlockFinalized_RejectOldFinalizedBlock() {
   168  	blockB1 := unittest.BlockHeaderWithParentFixture(s.Block)
   169  	blockB2 := unittest.BlockHeaderWithParentFixture(blockB1)
   170  
   171  	s.Blocks[blockB1.ID()] = blockB1
   172  	s.Blocks[blockB2.ID()] = blockB2
   173  
   174  	seal := unittest.Seal.Fixture(unittest.Seal.WithBlock(s.Block))
   175  	// should only call it once
   176  	s.sealsDB.On("HighestInFork", mock.Anything).Return(seal, nil).Once()
   177  	s.MarkFinalized(blockB1)
   178  	s.MarkFinalized(blockB2)
   179  
   180  	// blockB1 becomes finalized
   181  	err := s.core.ProcessFinalizedBlock(blockB2.ID())
   182  	require.NoError(s.T(), err)
   183  
   184  	err = s.core.ProcessFinalizedBlock(blockB1.ID())
   185  	require.NoError(s.T(), err)
   186  }
   187  
   188  // TestProcessFinalizedBlock_CollectorsCleanup tests that stale collectorTree are cleaned up for
   189  // already sealed blocks.
   190  func (s *ApprovalProcessingCoreTestSuite) TestProcessFinalizedBlock_CollectorsCleanup() {
   191  	blockID := s.Block.ID()
   192  	numResults := uint(10)
   193  	for i := uint(0); i < numResults; i++ {
   194  		// all results incorporated in different blocks
   195  		incorporatedBlock := unittest.BlockHeaderWithParentFixture(s.IncorporatedBlock)
   196  		s.Blocks[incorporatedBlock.ID()] = incorporatedBlock
   197  		// create different incorporated results for same block ID
   198  		result := unittest.ExecutionResultFixture()
   199  		result.BlockID = blockID
   200  		result.PreviousResultID = s.IncorporatedResult.Result.ID()
   201  		incorporatedResult := unittest.IncorporatedResult.Fixture(
   202  			unittest.IncorporatedResult.WithResult(result),
   203  			unittest.IncorporatedResult.WithIncorporatedBlockID(incorporatedBlock.ID()))
   204  		err := s.core.processIncorporatedResult(incorporatedResult)
   205  		require.NoError(s.T(), err)
   206  	}
   207  	require.Equal(s.T(), uint64(numResults), s.core.collectorTree.GetSize())
   208  
   209  	candidate := unittest.BlockHeaderWithParentFixture(s.Block)
   210  	s.Blocks[candidate.ID()] = candidate
   211  
   212  	// candidate becomes new sealed and finalized block, it means that
   213  	// we will need to cleanup our tree till new height, removing all outdated collectors
   214  	seal := unittest.Seal.Fixture(unittest.Seal.WithBlock(candidate))
   215  	s.sealsDB.On("HighestInFork", mock.Anything).Return(seal, nil).Once()
   216  
   217  	s.MarkFinalized(candidate)
   218  	err := s.core.ProcessFinalizedBlock(candidate.ID())
   219  	require.NoError(s.T(), err)
   220  	require.Equal(s.T(), uint64(0), s.core.collectorTree.GetSize())
   221  }
   222  
   223  // TestProcessIncorporated_ApprovalsBeforeResult tests a scenario when first we have received approvals for unknown
   224  // execution result and after that we discovered execution result. In this scenario we should be able
   225  // to create a seal right after discovering execution result since all approvals should be cached.(if cache capacity is big enough)
   226  func (s *ApprovalProcessingCoreTestSuite) TestProcessIncorporated_ApprovalsBeforeResult() {
   227  	s.PublicKey.On("Verify", mock.Anything, mock.Anything, mock.Anything).Return(true, nil)
   228  
   229  	for _, chunk := range s.Chunks {
   230  		for verID := range s.AuthorizedVerifiers {
   231  			approval := unittest.ResultApprovalFixture(unittest.WithChunk(chunk.Index),
   232  				unittest.WithApproverID(verID),
   233  				unittest.WithBlockID(s.Block.ID()),
   234  				unittest.WithExecutionResultID(s.IncorporatedResult.Result.ID()))
   235  			err := s.core.processApproval(approval)
   236  			require.NoError(s.T(), err)
   237  		}
   238  	}
   239  
   240  	s.SealsPL.On("Add", mock.Anything).Return(true, nil).Once()
   241  
   242  	err := s.core.processIncorporatedResult(s.IncorporatedResult)
   243  	require.NoError(s.T(), err)
   244  
   245  	s.SealsPL.AssertCalled(s.T(), "Add", mock.Anything)
   246  }
   247  
   248  // TestProcessIncorporated_ApprovalsAfterResult tests a scenario when first we have discovered execution result
   249  // and after that we started receiving approvals. In this scenario we should be able to create a seal right
   250  // after processing last needed approval to meet `RequiredApprovalsForSealConstruction` threshold.
   251  func (s *ApprovalProcessingCoreTestSuite) TestProcessIncorporated_ApprovalsAfterResult() {
   252  	s.PublicKey.On("Verify", mock.Anything, mock.Anything, mock.Anything).Return(true, nil)
   253  
   254  	s.SealsPL.On("Add", mock.Anything).Return(true, nil).Once()
   255  
   256  	err := s.core.processIncorporatedResult(s.IncorporatedResult)
   257  	require.NoError(s.T(), err)
   258  
   259  	for _, chunk := range s.Chunks {
   260  		for verID := range s.AuthorizedVerifiers {
   261  			approval := unittest.ResultApprovalFixture(unittest.WithChunk(chunk.Index),
   262  				unittest.WithApproverID(verID),
   263  				unittest.WithBlockID(s.Block.ID()),
   264  				unittest.WithExecutionResultID(s.IncorporatedResult.Result.ID()))
   265  			err := s.core.processApproval(approval)
   266  			require.NoError(s.T(), err)
   267  		}
   268  	}
   269  
   270  	s.SealsPL.AssertCalled(s.T(), "Add", mock.Anything)
   271  }
   272  
   273  // TestProcessIncorporated_ProcessingInvalidApproval tests that processing invalid approval when result is discovered
   274  // is correctly handled in case of sentinel error
   275  func (s *ApprovalProcessingCoreTestSuite) TestProcessIncorporated_ProcessingInvalidApproval() {
   276  	// fail signature verification for first approval
   277  	s.PublicKey.On("Verify", mock.Anything, mock.Anything, mock.Anything).Return(false, nil).Once()
   278  
   279  	// generate approvals for first chunk
   280  	approval := unittest.ResultApprovalFixture(unittest.WithChunk(s.Chunks[0].Index),
   281  		unittest.WithApproverID(s.VerID),
   282  		unittest.WithBlockID(s.Block.ID()),
   283  		unittest.WithExecutionResultID(s.IncorporatedResult.Result.ID()))
   284  
   285  	// this approval has to be cached since execution result is not known yet
   286  	err := s.core.processApproval(approval)
   287  	require.NoError(s.T(), err)
   288  
   289  	// at this point approval has to be processed, even if it's invalid
   290  	// if it's an expected sentinel error, it has to be handled internally
   291  	err = s.core.processIncorporatedResult(s.IncorporatedResult)
   292  	require.NoError(s.T(), err)
   293  }
   294  
   295  // TestProcessIncorporated_ApprovalVerificationException tests that processing invalid approval when result is discovered
   296  // is correctly handled in case of exception
   297  func (s *ApprovalProcessingCoreTestSuite) TestProcessIncorporated_ApprovalVerificationException() {
   298  	// fail signature verification with exception
   299  	s.PublicKey.On("Verify", mock.Anything, mock.Anything, mock.Anything).Return(false, fmt.Errorf("exception")).Once()
   300  
   301  	// generate approvals for first chunk
   302  	approval := unittest.ResultApprovalFixture(unittest.WithChunk(s.Chunks[0].Index),
   303  		unittest.WithApproverID(s.VerID),
   304  		unittest.WithBlockID(s.Block.ID()),
   305  		unittest.WithExecutionResultID(s.IncorporatedResult.Result.ID()))
   306  
   307  	// this approval has to be cached since execution result is not known yet
   308  	err := s.core.processApproval(approval)
   309  	require.NoError(s.T(), err)
   310  
   311  	// at this point approval has to be processed, even if it's invalid
   312  	// if it's an expected sentinel error, it has to be handled internally
   313  	err = s.core.processIncorporatedResult(s.IncorporatedResult)
   314  	require.Error(s.T(), err)
   315  }
   316  
   317  // TestOnBlockFinalized_EmergencySealing tests that emergency sealing kicks in to resolve sealing halt
   318  func (s *ApprovalProcessingCoreTestSuite) TestOnBlockFinalized_EmergencySealing() {
   319  
   320  	metrics := metrics.NewNoopCollector()
   321  	tracer := trace.NewNoopTracer()
   322  
   323  	setter, err := updatable_configs.NewSealingConfigs(
   324  		flow.DefaultRequiredApprovalsForSealConstruction,
   325  		flow.DefaultRequiredApprovalsForSealValidation,
   326  		flow.DefaultChunkAssignmentAlpha,
   327  		true, // enable emergency sealing
   328  	)
   329  	require.NoError(s.T(), err)
   330  	s.core, err = NewCore(unittest.Logger(), s.WorkerPool, tracer, metrics, &tracker.NoopSealingTracker{}, engine.NewUnit(), s.Headers, s.State, s.sealsDB, s.Assigner, s.SigHasher, s.SealsPL, s.Conduit, setter)
   331  	require.NoError(s.T(), err)
   332  	s.setter = setter
   333  
   334  	s.SealsPL.On("ByID", mock.Anything).Return(nil, false).Maybe()
   335  	s.SealsPL.On("Add", mock.Anything).Run(
   336  		func(args mock.Arguments) {
   337  			seal := args.Get(0).(*flow.IncorporatedResultSeal)
   338  			require.Equal(s.T(), s.Block.ID(), seal.Seal.BlockID)
   339  			require.Equal(s.T(), s.IncorporatedResult.Result.ID(), seal.Seal.ResultID)
   340  		},
   341  	).Return(true, nil).Once()
   342  
   343  	seal := unittest.Seal.Fixture(unittest.Seal.WithBlock(s.ParentBlock))
   344  	s.sealsDB.On("HighestInFork", mock.Anything).Return(seal, nil).Times(approvals.DefaultEmergencySealingThresholdForFinalization)
   345  	s.State.On("Sealed").Return(unittest.StateSnapshotForKnownBlock(s.ParentBlock, nil))
   346  
   347  	err = s.core.ProcessIncorporatedResult(s.IncorporatedResult)
   348  	require.NoError(s.T(), err)
   349  
   350  	lastFinalizedBlock := s.IncorporatedBlock
   351  	s.MarkFinalized(lastFinalizedBlock)
   352  	for i := 0; i < approvals.DefaultEmergencySealingThresholdForFinalization; i++ {
   353  		finalizedBlock := unittest.BlockHeaderWithParentFixture(lastFinalizedBlock)
   354  		s.Blocks[finalizedBlock.ID()] = finalizedBlock
   355  		s.MarkFinalized(finalizedBlock)
   356  		err := s.core.ProcessFinalizedBlock(finalizedBlock.ID())
   357  		require.NoError(s.T(), err)
   358  		lastFinalizedBlock = finalizedBlock
   359  	}
   360  
   361  	s.SealsPL.AssertExpectations(s.T())
   362  }
   363  
   364  // TestOnBlockFinalized_ProcessingOrphanApprovals tests that approvals for orphan forks are rejected as outdated entries without processing
   365  //
   366  //	 A <- B_1 <- C_1{ IER[B_1] }
   367  //		 <- B_2 <- C_2{ IER[B_2] } <- D_2{ IER[C_2] }
   368  //	 	 <- B_3 <- C_3{ IER[B_3] } <- D_3{ IER[C_3] } <- E_3{ IER[D_3] }
   369  //
   370  // B_1 becomes finalized rendering forks starting at B_2 and B_3 as orphans
   371  func (s *ApprovalProcessingCoreTestSuite) TestOnBlockFinalized_ProcessingOrphanApprovals() {
   372  	forks := make([][]*flow.Block, 3)
   373  	forkResults := make([][]*flow.ExecutionResult, len(forks))
   374  
   375  	for forkIndex := range forks {
   376  		forks[forkIndex] = unittest.ChainFixtureFrom(forkIndex+2, s.ParentBlock)
   377  		fork := forks[forkIndex]
   378  
   379  		previousResult := s.IncorporatedResult.Result
   380  		for blockIndex, block := range fork {
   381  			s.Blocks[block.ID()] = block.Header
   382  			s.IdentitiesCache[block.ID()] = s.AuthorizedVerifiers
   383  
   384  			// create and incorporate result for every block in fork except first one
   385  			if blockIndex > 0 {
   386  				// create a result
   387  				result := unittest.ExecutionResultFixture(unittest.WithPreviousResult(*previousResult))
   388  				result.BlockID = block.Header.ParentID
   389  				result.Chunks = s.Chunks
   390  				forkResults[forkIndex] = append(forkResults[forkIndex], result)
   391  				previousResult = result
   392  
   393  				// incorporate in fork
   394  				IR := unittest.IncorporatedResult.Fixture(
   395  					unittest.IncorporatedResult.WithIncorporatedBlockID(block.ID()),
   396  					unittest.IncorporatedResult.WithResult(result))
   397  
   398  				err := s.core.processIncorporatedResult(IR)
   399  				require.NoError(s.T(), err)
   400  			}
   401  		}
   402  	}
   403  
   404  	// same block sealed
   405  	seal := unittest.Seal.Fixture(unittest.Seal.WithBlock(s.ParentBlock))
   406  	s.sealsDB.On("HighestInFork", mock.Anything).Return(seal, nil).Once()
   407  
   408  	// block B_1 becomes finalized
   409  	finalized := forks[0][0].Header
   410  	s.MarkFinalized(finalized)
   411  	err := s.core.ProcessFinalizedBlock(finalized.ID())
   412  	require.NoError(s.T(), err)
   413  
   414  	// verify will be called twice for every approval in first fork
   415  	s.PublicKey.On("Verify", mock.Anything, mock.Anything, mock.Anything).Return(true, nil).Times(len(forkResults[0]) * 2)
   416  
   417  	// try submitting approvals for each result
   418  	for _, results := range forkResults {
   419  		for _, result := range results {
   420  			executedBlockID := result.BlockID
   421  			resultID := result.ID()
   422  
   423  			approval := unittest.ResultApprovalFixture(unittest.WithChunk(0),
   424  				unittest.WithApproverID(s.VerID),
   425  				unittest.WithBlockID(executedBlockID),
   426  				unittest.WithExecutionResultID(resultID))
   427  
   428  			err := s.core.processApproval(approval)
   429  			require.NoError(s.T(), err)
   430  		}
   431  	}
   432  }
   433  
   434  // TestOnBlockFinalized_ExtendingUnprocessableFork tests that extending orphan fork results in non processable collectors
   435  //
   436  //	.       - X <- Y <- Z
   437  //	.    /
   438  //	. <- A <- B <- C <- D <- E
   439  //	.           |
   440  //	.       finalized
   441  func (s *ApprovalProcessingCoreTestSuite) TestOnBlockFinalized_ExtendingUnprocessableFork() {
   442  	forks := make([][]*flow.Block, 2)
   443  
   444  	for forkIndex := range forks {
   445  		forks[forkIndex] = unittest.ChainFixtureFrom(forkIndex+3, s.Block)
   446  		fork := forks[forkIndex]
   447  		for _, block := range fork {
   448  			s.Blocks[block.ID()] = block.Header
   449  			s.IdentitiesCache[block.ID()] = s.AuthorizedVerifiers
   450  		}
   451  	}
   452  
   453  	finalized := forks[1][0].Header
   454  
   455  	s.MarkFinalized(finalized)
   456  	seal := unittest.Seal.Fixture(unittest.Seal.WithBlock(s.ParentBlock))
   457  	s.sealsDB.On("HighestInFork", mock.Anything).Return(seal, nil).Once()
   458  
   459  	// finalize block B
   460  	err := s.core.ProcessFinalizedBlock(finalized.ID())
   461  	require.NoError(s.T(), err)
   462  
   463  	// create incorporated result for each block in main fork
   464  	for forkIndex, fork := range forks {
   465  		previousResult := s.IncorporatedResult.Result
   466  		for blockIndex, block := range fork {
   467  			result := unittest.ExecutionResultFixture(unittest.WithPreviousResult(*previousResult))
   468  			result.BlockID = block.Header.ParentID
   469  			result.Chunks = s.Chunks
   470  			previousResult = result
   471  
   472  			// incorporate in fork
   473  			IR := unittest.IncorporatedResult.Fixture(
   474  				unittest.IncorporatedResult.WithIncorporatedBlockID(block.ID()),
   475  				unittest.IncorporatedResult.WithResult(result))
   476  			err := s.core.processIncorporatedResult(IR)
   477  			collector := s.core.collectorTree.GetCollector(result.ID())
   478  			if forkIndex > 0 {
   479  				require.NoError(s.T(), err)
   480  				require.Equal(s.T(), approvals.VerifyingApprovals, collector.ProcessingStatus())
   481  			} else {
   482  				if blockIndex == 0 {
   483  					require.Error(s.T(), err)
   484  					require.True(s.T(), engine.IsOutdatedInputError(err))
   485  				} else {
   486  					require.Equal(s.T(), approvals.CachingApprovals, collector.ProcessingStatus())
   487  				}
   488  			}
   489  		}
   490  	}
   491  }
   492  
   493  // TestOnBlockFinalized_ExtendingSealedResult tests if assignment collector tree accepts collector which extends sealed result
   494  func (s *ApprovalProcessingCoreTestSuite) TestOnBlockFinalized_ExtendingSealedResult() {
   495  	seal := unittest.Seal.Fixture(unittest.Seal.WithBlock(s.Block))
   496  	s.sealsDB.On("HighestInFork", mock.Anything).Return(seal, nil).Once()
   497  
   498  	unsealedBlock := unittest.BlockHeaderWithParentFixture(s.Block)
   499  	s.Blocks[unsealedBlock.ID()] = unsealedBlock
   500  	s.IdentitiesCache[unsealedBlock.ID()] = s.AuthorizedVerifiers
   501  	result := unittest.ExecutionResultFixture(unittest.WithPreviousResult(*s.IncorporatedResult.Result))
   502  	result.BlockID = unsealedBlock.ID()
   503  
   504  	s.MarkFinalized(unsealedBlock)
   505  	err := s.core.ProcessFinalizedBlock(unsealedBlock.ID())
   506  	require.NoError(s.T(), err)
   507  
   508  	incorporatedBlock := unittest.BlockHeaderWithParentFixture(unsealedBlock)
   509  	s.Blocks[incorporatedBlock.ID()] = incorporatedBlock
   510  	s.IdentitiesCache[incorporatedBlock.ID()] = s.AuthorizedVerifiers
   511  	IR := unittest.IncorporatedResult.Fixture(
   512  		unittest.IncorporatedResult.WithIncorporatedBlockID(incorporatedBlock.ID()),
   513  		unittest.IncorporatedResult.WithResult(result))
   514  	err = s.core.processIncorporatedResult(IR)
   515  	require.NoError(s.T(), err)
   516  
   517  	s.sealsDB.AssertExpectations(s.T())
   518  }
   519  
   520  // TestRequestPendingApprovals checks that requests are sent only for chunks
   521  // that have not collected enough approvals yet, and are sent only to the
   522  // verifiers assigned to those chunks. It also checks that the threshold and
   523  // rate limiting is respected.
   524  func (s *ApprovalProcessingCoreTestSuite) TestRequestPendingApprovals() {
   525  	s.core.requestTracker = approvals.NewRequestTracker(s.core.headers, 1, 3)
   526  	s.SealsPL.On("ByID", mock.Anything).Return(nil, false)
   527  
   528  	// n is the total number of blocks and incorporated-results we add to the
   529  	// chain and mempool
   530  	n := 100
   531  
   532  	// create blocks
   533  	unsealedFinalizedBlocks := make([]flow.Block, 0, n)
   534  	parentBlock := s.ParentBlock
   535  	for i := 0; i < n; i++ {
   536  		block := unittest.BlockWithParentFixture(parentBlock)
   537  		s.Blocks[block.ID()] = block.Header
   538  		s.IdentitiesCache[block.ID()] = s.AuthorizedVerifiers
   539  		unsealedFinalizedBlocks = append(unsealedFinalizedBlocks, *block)
   540  		parentBlock = block.Header
   541  	}
   542  
   543  	// progress latest sealed and latest finalized:
   544  	//s.LatestSealedBlock = unsealedFinalizedBlocks[0]
   545  	//s.LatestFinalizedBlock = &unsealedFinalizedBlocks[n-1]
   546  
   547  	// add an unfinalized block; it shouldn't require an approval request
   548  	unfinalizedBlock := unittest.BlockWithParentFixture(parentBlock)
   549  	s.Blocks[unfinalizedBlock.ID()] = unfinalizedBlock.Header
   550  
   551  	// we will assume that all chunks are assigned to the same two verifiers.
   552  	verifiers := make([]flow.Identifier, 0)
   553  	for nodeID := range s.AuthorizedVerifiers {
   554  		if len(verifiers) > 2 {
   555  			break
   556  		}
   557  		verifiers = append(verifiers, nodeID)
   558  	}
   559  
   560  	// the sealing Core requires approvals from both verifiers for each chunk
   561  	err := s.setter.SetRequiredApprovalsForSealingConstruction(2)
   562  	require.NoError(s.T(), err)
   563  
   564  	// populate the incorporated-results tree with:
   565  	// - 50 that have collected two signatures per chunk
   566  	// - 25 that have collected only one signature
   567  	// - 25 that have collected no signatures
   568  	//
   569  	//
   570  	//     sealed          unsealed/finalized
   571  	// |              ||                        |
   572  	// 1 <- 2 <- .. <- s <- s+1 <- .. <- n-t <- n
   573  	//                 |                  |
   574  	//                    expected reqs
   575  	prevResult := s.IncorporatedResult.Result
   576  	resultIDs := make([]flow.Identifier, 0, n)
   577  	chunkCount := 2
   578  	for i := 0; i < n-1; i++ {
   579  
   580  		// Create an incorporated result for unsealedFinalizedBlocks[i].
   581  		// By default the result will contain 17 chunks.
   582  		ir := unittest.IncorporatedResult.Fixture(
   583  			unittest.IncorporatedResult.WithResult(
   584  				unittest.ExecutionResultFixture(
   585  					unittest.WithBlock(&unsealedFinalizedBlocks[i]),
   586  					unittest.WithPreviousResult(*prevResult),
   587  					unittest.WithChunks(uint(chunkCount)),
   588  				),
   589  			),
   590  			unittest.IncorporatedResult.WithIncorporatedBlockID(
   591  				unsealedFinalizedBlocks[i+1].ID(),
   592  			),
   593  		)
   594  
   595  		prevResult = ir.Result
   596  
   597  		s.ChunksAssignment = chunks.NewAssignment()
   598  
   599  		for _, chunk := range ir.Result.Chunks {
   600  			// assign the verifier to this chunk
   601  			s.ChunksAssignment.Add(chunk, verifiers)
   602  		}
   603  
   604  		err := s.core.processIncorporatedResult(ir)
   605  		require.NoError(s.T(), err)
   606  
   607  		resultIDs = append(resultIDs, ir.Result.ID())
   608  	}
   609  
   610  	// sealed block doesn't change
   611  	seal := unittest.Seal.Fixture(unittest.Seal.WithBlock(s.ParentBlock))
   612  	s.sealsDB.On("HighestInFork", mock.Anything).Return(seal, nil)
   613  
   614  	s.State.On("Sealed").Return(unittest.StateSnapshotForKnownBlock(s.ParentBlock, nil))
   615  
   616  	// start delivering finalization events
   617  	lastProcessedIndex := 0
   618  	for ; lastProcessedIndex < int(s.core.sealingConfigsGetter.ApprovalRequestsThresholdConst()); lastProcessedIndex++ {
   619  		finalized := unsealedFinalizedBlocks[lastProcessedIndex].Header
   620  		s.MarkFinalized(finalized)
   621  		err := s.core.ProcessFinalizedBlock(finalized.ID())
   622  		require.NoError(s.T(), err)
   623  	}
   624  
   625  	require.Empty(s.T(), s.core.requestTracker.GetAllIds())
   626  
   627  	// process two more blocks, this will trigger requesting approvals for lastSealed + 1 height
   628  	// but they will be in blackout period
   629  	for i := 0; i < 2; i++ {
   630  		finalized := unsealedFinalizedBlocks[lastProcessedIndex].Header
   631  		s.MarkFinalized(finalized)
   632  		err := s.core.ProcessFinalizedBlock(finalized.ID())
   633  		require.NoError(s.T(), err)
   634  		lastProcessedIndex += 1
   635  	}
   636  
   637  	require.ElementsMatch(s.T(), s.core.requestTracker.GetAllIds(), resultIDs[:1])
   638  
   639  	// wait for the max blackout period to elapse
   640  	time.Sleep(3 * time.Second)
   641  
   642  	// our setup is for 5 verification nodes
   643  	s.Conduit.On("Publish", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
   644  		Return(nil).Times(chunkCount)
   645  
   646  	// process next block
   647  	finalized := unsealedFinalizedBlocks[lastProcessedIndex].Header
   648  	s.MarkFinalized(finalized)
   649  	err = s.core.ProcessFinalizedBlock(finalized.ID())
   650  	require.NoError(s.T(), err)
   651  
   652  	// now 2 results should be pending
   653  	require.ElementsMatch(s.T(), s.core.requestTracker.GetAllIds(), resultIDs[:2])
   654  
   655  	s.Conduit.AssertExpectations(s.T())
   656  }
   657  
   658  // TestRepopulateAssignmentCollectorTree tests that the
   659  // collectors tree will contain execution results and assignment collectors will be created.
   660  //
   661  //	P <- A[ER{P}] <- B[ER{A}] <- C[ER{B}] <- D[ER{C}] <- E[ER{D}]
   662  //	        |     <- F[ER{A}] <- G[ER{B}] <- H[ER{G}]
   663  //	     finalized
   664  //
   665  // collectors tree has to be repopulated with incorporated results from blocks [A, B, C, D, F, G]
   666  // E, H shouldn't be considered since
   667  func (s *ApprovalProcessingCoreTestSuite) TestRepopulateAssignmentCollectorTree() {
   668  	metrics := metrics.NewNoopCollector()
   669  	tracer := trace.NewNoopTracer()
   670  	assigner := &module.ChunkAssigner{}
   671  
   672  	// setup mocks
   673  	payloads := &storage.Payloads{}
   674  	expectedResults := []*flow.IncorporatedResult{s.IncorporatedResult}
   675  	blockChildren := make([]flow.Identifier, 0)
   676  
   677  	rootSnapshot := unittest.StateSnapshotForKnownBlock(s.rootHeader, nil)
   678  	s.Snapshots[s.rootHeader.ID()] = rootSnapshot
   679  	rootSnapshot.On("SealingSegment").Return(
   680  		&flow.SealingSegment{
   681  			Blocks: []*flow.Block{{
   682  				Header:  s.rootHeader,
   683  				Payload: &flow.Payload{},
   684  			}},
   685  		}, nil)
   686  
   687  	s.sealsDB.On("HighestInFork", s.IncorporatedBlock.ID()).Return(
   688  		unittest.Seal.Fixture(
   689  			unittest.Seal.WithBlock(s.ParentBlock)), nil)
   690  
   691  	// the incorporated block contains the result for the sealing candidate block
   692  	incorporatedBlockPayload := unittest.PayloadFixture(
   693  		unittest.WithReceipts(
   694  			unittest.ExecutionReceiptFixture(
   695  				unittest.WithResult(s.IncorporatedResult.Result))))
   696  	payloads.On("ByBlockID", s.IncorporatedBlock.ID()).Return(&incorporatedBlockPayload, nil)
   697  
   698  	emptyPayload := flow.EmptyPayload()
   699  	payloads.On("ByBlockID", s.Block.ID()).Return(&emptyPayload, nil)
   700  
   701  	s.IdentitiesCache[s.IncorporatedBlock.ID()] = s.AuthorizedVerifiers
   702  
   703  	assigner.On("Assign", s.IncorporatedResult.Result, mock.Anything).Return(s.ChunksAssignment, nil)
   704  
   705  	// two forks
   706  	for i := 0; i < 2; i++ {
   707  		fork := unittest.ChainFixtureFrom(i+3, s.IncorporatedBlock)
   708  		prevResult := s.IncorporatedResult.Result
   709  		// create execution results for all blocks except last one, since it won't be valid by definition
   710  		for blockIndex, block := range fork {
   711  			blockID := block.ID()
   712  
   713  			// create execution result for previous block in chain
   714  			// this result will be incorporated in current block.
   715  			result := unittest.ExecutionResultFixture(
   716  				unittest.WithPreviousResult(*prevResult),
   717  			)
   718  			result.BlockID = block.Header.ParentID
   719  
   720  			// update caches
   721  			s.Blocks[blockID] = block.Header
   722  			s.IdentitiesCache[blockID] = s.AuthorizedVerifiers
   723  			blockChildren = append(blockChildren, blockID)
   724  
   725  			IR := unittest.IncorporatedResult.Fixture(
   726  				unittest.IncorporatedResult.WithResult(result),
   727  				unittest.IncorporatedResult.WithIncorporatedBlockID(blockID))
   728  
   729  			// TODO: change this test for phase 3, assigner should expect incorporated block ID, not executed
   730  			if blockIndex < len(fork)-1 {
   731  				assigner.On("Assign", result, blockID).Return(s.ChunksAssignment, nil)
   732  				expectedResults = append(expectedResults, IR)
   733  			} else {
   734  				assigner.On("Assign", result, blockID).Return(nil, fmt.Errorf("no assignment for block without valid child"))
   735  			}
   736  
   737  			payload := unittest.PayloadFixture()
   738  			payload.Results = append(payload.Results, result)
   739  			payloads.On("ByBlockID", blockID).Return(&payload, nil)
   740  
   741  			prevResult = result
   742  		}
   743  	}
   744  
   745  	// ValidDescendants has to return all valid descendants from finalized block
   746  	finalSnapShot := unittest.StateSnapshotForKnownBlock(s.IncorporatedBlock, nil)
   747  	finalSnapShot.On("ValidDescendants").Return(blockChildren, nil)
   748  	s.State.On("Final").Return(finalSnapShot)
   749  
   750  	core, err := NewCore(unittest.Logger(), s.WorkerPool, tracer, metrics, &tracker.NoopSealingTracker{}, engine.NewUnit(),
   751  		s.Headers, s.State, s.sealsDB, assigner, s.SigHasher, s.SealsPL, s.Conduit, s.setter)
   752  	require.NoError(s.T(), err)
   753  
   754  	err = core.RepopulateAssignmentCollectorTree(payloads)
   755  	require.NoError(s.T(), err)
   756  
   757  	// check collector tree, after repopulating we should have all collectors for execution results that we have
   758  	// traversed and they have to be processable.
   759  	for _, incorporatedResult := range expectedResults {
   760  		collector, err := core.collectorTree.GetOrCreateCollector(incorporatedResult.Result)
   761  		require.NoError(s.T(), err)
   762  		require.False(s.T(), collector.Created)
   763  		require.Equal(s.T(), approvals.VerifyingApprovals, collector.Collector.ProcessingStatus())
   764  	}
   765  }
   766  
   767  // TestRepopulateAssignmentCollectorTree_RootSealingSegment tests that the sealing
   768  // engine will be initialized correctly when bootstrapping with a root sealing
   769  // segment with multiple blocks, as is the case when joining the network at an epoch
   770  // boundary.
   771  //
   772  // In particular, the assignment collector tree population step should ignore
   773  // unknown block references below the root height.
   774  func (s *ApprovalProcessingCoreTestSuite) TestRepopulateAssignmentCollectorTree_RootSealingSegment() {
   775  	metrics := metrics.NewNoopCollector()
   776  	tracer := trace.NewNoopTracer()
   777  	assigner := &module.ChunkAssigner{}
   778  	payloads := &storage.Payloads{}
   779  
   780  	// setup mocks
   781  	s.rootHeader = s.IncorporatedBlock
   782  	expectedResults := []*flow.IncorporatedResult{s.IncorporatedResult}
   783  
   784  	s.sealsDB.On("HighestInFork", s.IncorporatedBlock.ID()).Return(
   785  		unittest.Seal.Fixture(
   786  			unittest.Seal.WithBlock(s.ParentBlock)), nil)
   787  
   788  	// the incorporated block contains the result for the sealing candidate block
   789  	incorporatedBlockPayload := unittest.PayloadFixture(
   790  		unittest.WithReceipts(
   791  			unittest.ExecutionReceiptFixture(
   792  				unittest.WithResult(s.IncorporatedResult.Result))))
   793  	payloads.On("ByBlockID", s.IncorporatedBlock.ID()).Return(&incorporatedBlockPayload, nil)
   794  
   795  	// the sealing candidate block (S) is the lowest block in the segment under consideration here
   796  	// initially, this block would represent the lowest block in a node's root sealing segment,
   797  	// meaning that all earlier blocks are not known. In this case we should ignore results and seals
   798  	// referencing unknown blocks (tested here by adding such a result+seal to the candidate payload).
   799  	candidatePayload := unittest.PayloadFixture(
   800  		unittest.WithReceipts(unittest.ExecutionReceiptFixture()), // receipt referencing pre-root block
   801  		unittest.WithSeals(unittest.Seal.Fixture()),               // seal referencing pre-root block
   802  	)
   803  	payloads.On("ByBlockID", s.Block.ID()).Return(&candidatePayload, nil)
   804  
   805  	s.IdentitiesCache[s.IncorporatedBlock.ID()] = s.AuthorizedVerifiers
   806  
   807  	assigner.On("Assign", s.IncorporatedResult.Result, mock.Anything).Return(s.ChunksAssignment, nil)
   808  
   809  	finalSnapShot := unittest.StateSnapshotForKnownBlock(s.rootHeader, nil)
   810  	s.Snapshots[s.rootHeader.ID()] = finalSnapShot
   811  	// root snapshot has no pending children
   812  	finalSnapShot.On("ValidDescendants").Return(nil, nil)
   813  	// set up sealing segment
   814  	finalSnapShot.On("SealingSegment").Return(
   815  		&flow.SealingSegment{
   816  			Blocks: []*flow.Block{{
   817  				Header:  s.Block,
   818  				Payload: &candidatePayload,
   819  			}, {
   820  				Header:  s.ParentBlock,
   821  				Payload: &flow.Payload{},
   822  			}, {
   823  				Header:  s.IncorporatedBlock,
   824  				Payload: &incorporatedBlockPayload,
   825  			}},
   826  		}, nil)
   827  	s.State.On("Final").Return(finalSnapShot)
   828  
   829  	core, err := NewCore(unittest.Logger(), s.WorkerPool, tracer, metrics, &tracker.NoopSealingTracker{}, engine.NewUnit(),
   830  		s.Headers, s.State, s.sealsDB, assigner, s.SigHasher, s.SealsPL, s.Conduit, s.setter)
   831  	require.NoError(s.T(), err)
   832  
   833  	err = core.RepopulateAssignmentCollectorTree(payloads)
   834  	require.NoError(s.T(), err)
   835  
   836  	// check collector tree, after repopulating we should have all collectors for execution results that we have
   837  	// traversed and they have to be processable.
   838  	for _, incorporatedResult := range expectedResults {
   839  		collector, err := core.collectorTree.GetOrCreateCollector(incorporatedResult.Result)
   840  		require.NoError(s.T(), err)
   841  		require.False(s.T(), collector.Created)
   842  		require.Equal(s.T(), approvals.VerifyingApprovals, collector.Collector.ProcessingStatus())
   843  	}
   844  }