github.com/onflow/flow-go@v0.33.17/state/protocol/badger/mutator_test.go (about)

     1  // (c) 2019 Dapper Labs - ALL RIGHTS RESERVED
     2  
     3  package badger_test
     4  
     5  import (
     6  	"context"
     7  	"errors"
     8  	"math/rand"
     9  	"sync"
    10  	"testing"
    11  	"time"
    12  
    13  	"github.com/dgraph-io/badger/v2"
    14  	"github.com/rs/zerolog"
    15  	"github.com/stretchr/testify/assert"
    16  	"github.com/stretchr/testify/mock"
    17  	"github.com/stretchr/testify/require"
    18  
    19  	"github.com/onflow/flow-go/crypto"
    20  	"github.com/onflow/flow-go/engine"
    21  	"github.com/onflow/flow-go/model/flow"
    22  	"github.com/onflow/flow-go/model/flow/filter"
    23  	"github.com/onflow/flow-go/module/metrics"
    24  	mockmodule "github.com/onflow/flow-go/module/mock"
    25  	"github.com/onflow/flow-go/module/signature"
    26  	"github.com/onflow/flow-go/module/trace"
    27  	st "github.com/onflow/flow-go/state"
    28  	realprotocol "github.com/onflow/flow-go/state/protocol"
    29  	protocol "github.com/onflow/flow-go/state/protocol/badger"
    30  	"github.com/onflow/flow-go/state/protocol/events"
    31  	"github.com/onflow/flow-go/state/protocol/inmem"
    32  	mockprotocol "github.com/onflow/flow-go/state/protocol/mock"
    33  	"github.com/onflow/flow-go/state/protocol/util"
    34  	"github.com/onflow/flow-go/storage"
    35  	stoerr "github.com/onflow/flow-go/storage"
    36  	bstorage "github.com/onflow/flow-go/storage/badger"
    37  	"github.com/onflow/flow-go/storage/badger/operation"
    38  	storeutil "github.com/onflow/flow-go/storage/util"
    39  	"github.com/onflow/flow-go/utils/unittest"
    40  )
    41  
    42  var participants = unittest.IdentityListFixture(5, unittest.WithAllRoles())
    43  
    44  func TestBootstrapValid(t *testing.T) {
    45  	rootSnapshot := unittest.RootSnapshotFixture(participants)
    46  	util.RunWithBootstrapState(t, rootSnapshot, func(db *badger.DB, state *protocol.State) {
    47  		var finalized uint64
    48  		err := db.View(operation.RetrieveFinalizedHeight(&finalized))
    49  		require.NoError(t, err)
    50  
    51  		var sealed uint64
    52  		err = db.View(operation.RetrieveSealedHeight(&sealed))
    53  		require.NoError(t, err)
    54  
    55  		var genesisID flow.Identifier
    56  		err = db.View(operation.LookupBlockHeight(0, &genesisID))
    57  		require.NoError(t, err)
    58  
    59  		var header flow.Header
    60  		err = db.View(operation.RetrieveHeader(genesisID, &header))
    61  		require.NoError(t, err)
    62  
    63  		var sealID flow.Identifier
    64  		err = db.View(operation.LookupLatestSealAtBlock(genesisID, &sealID))
    65  		require.NoError(t, err)
    66  
    67  		_, seal, err := rootSnapshot.SealedResult()
    68  		require.NoError(t, err)
    69  		err = db.View(operation.RetrieveSeal(sealID, seal))
    70  		require.NoError(t, err)
    71  
    72  		block, err := rootSnapshot.Head()
    73  		require.NoError(t, err)
    74  		require.Equal(t, block.Height, finalized)
    75  		require.Equal(t, block.Height, sealed)
    76  		require.Equal(t, block.ID(), genesisID)
    77  		require.Equal(t, block.ID(), seal.BlockID)
    78  		require.Equal(t, block, &header)
    79  	})
    80  }
    81  
    82  // TestExtendValid tests the happy path of extending the state with a single block.
    83  // * BlockFinalized is emitted when the block is finalized
    84  // * BlockProcessable is emitted when a block's child is inserted
    85  func TestExtendValid(t *testing.T) {
    86  	unittest.RunWithBadgerDB(t, func(db *badger.DB) {
    87  		metrics := metrics.NewNoopCollector()
    88  		tracer := trace.NewNoopTracer()
    89  		log := zerolog.Nop()
    90  		all := storeutil.StorageLayer(t, db)
    91  
    92  		distributor := events.NewDistributor()
    93  		consumer := mockprotocol.NewConsumer(t)
    94  		distributor.AddConsumer(consumer)
    95  
    96  		block, result, seal := unittest.BootstrapFixture(participants)
    97  		qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(block.ID()))
    98  		rootSnapshot, err := inmem.SnapshotFromBootstrapState(block, result, seal, qc)
    99  		require.NoError(t, err)
   100  
   101  		state, err := protocol.Bootstrap(
   102  			metrics,
   103  			db,
   104  			all.Headers,
   105  			all.Seals,
   106  			all.Results,
   107  			all.Blocks,
   108  			all.QuorumCertificates,
   109  			all.Setups,
   110  			all.EpochCommits,
   111  			all.Statuses,
   112  			all.VersionBeacons,
   113  			rootSnapshot,
   114  		)
   115  		require.NoError(t, err)
   116  
   117  		fullState, err := protocol.NewFullConsensusState(
   118  			log,
   119  			tracer,
   120  			consumer,
   121  			state,
   122  			all.Index,
   123  			all.Payloads,
   124  			util.MockBlockTimer(),
   125  			util.MockReceiptValidator(),
   126  			util.MockSealValidator(all.Seals),
   127  		)
   128  		require.NoError(t, err)
   129  
   130  		// insert block1 on top of the root block
   131  		block1 := unittest.BlockWithParentFixture(block.Header)
   132  		err = fullState.Extend(context.Background(), block1)
   133  		require.NoError(t, err)
   134  
   135  		// we should not emit BlockProcessable for the root block
   136  		consumer.AssertNotCalled(t, "BlockProcessable", block.Header)
   137  
   138  		t.Run("BlockFinalized event should be emitted when block1 is finalized", func(t *testing.T) {
   139  			consumer.On("BlockFinalized", block1.Header).Once()
   140  			err := fullState.Finalize(context.Background(), block1.ID())
   141  			require.NoError(t, err)
   142  		})
   143  
   144  		t.Run("BlockProcessable event should be emitted when any child of block1 is inserted", func(t *testing.T) {
   145  			block2 := unittest.BlockWithParentFixture(block1.Header)
   146  			consumer.On("BlockProcessable", block1.Header, mock.Anything).Once()
   147  			err := fullState.Extend(context.Background(), block2)
   148  			require.NoError(t, err)
   149  		})
   150  	})
   151  }
   152  
   153  func TestSealedIndex(t *testing.T) {
   154  	rootSnapshot := unittest.RootSnapshotFixture(participants)
   155  	util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
   156  		rootHeader, err := rootSnapshot.Head()
   157  		require.NoError(t, err)
   158  
   159  		// build a chain:
   160  		// G <- B1 <- B2 (resultB1) <- B3 <- B4 (resultB2, resultB3) <- B5 (sealB1) <- B6 (sealB2, sealB3) <- B7
   161  		// test that when B4 is finalized, can only find seal for G
   162  		// 					 when B5 is finalized, can find seal for B1
   163  		//					 when B7 is finalized, can find seals for B2, B3
   164  
   165  		// block 1
   166  		b1 := unittest.BlockWithParentFixture(rootHeader)
   167  		b1.SetPayload(flow.EmptyPayload())
   168  		err = state.Extend(context.Background(), b1)
   169  		require.NoError(t, err)
   170  
   171  		// block 2(result B1)
   172  		b1Receipt := unittest.ReceiptForBlockFixture(b1)
   173  		b2 := unittest.BlockWithParentFixture(b1.Header)
   174  		b2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(b1Receipt)))
   175  		err = state.Extend(context.Background(), b2)
   176  		require.NoError(t, err)
   177  
   178  		// block 3
   179  		b3 := unittest.BlockWithParentFixture(b2.Header)
   180  		b3.SetPayload(flow.EmptyPayload())
   181  		err = state.Extend(context.Background(), b3)
   182  		require.NoError(t, err)
   183  
   184  		// block 4 (resultB2, resultB3)
   185  		b2Receipt := unittest.ReceiptForBlockFixture(b2)
   186  		b3Receipt := unittest.ReceiptForBlockFixture(b3)
   187  		b4 := unittest.BlockWithParentFixture(b3.Header)
   188  		b4.SetPayload(flow.Payload{
   189  			Receipts: []*flow.ExecutionReceiptMeta{b2Receipt.Meta(), b3Receipt.Meta()},
   190  			Results:  []*flow.ExecutionResult{&b2Receipt.ExecutionResult, &b3Receipt.ExecutionResult},
   191  		})
   192  		err = state.Extend(context.Background(), b4)
   193  		require.NoError(t, err)
   194  
   195  		// block 5 (sealB1)
   196  		b1Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&b1Receipt.ExecutionResult))
   197  		b5 := unittest.BlockWithParentFixture(b4.Header)
   198  		b5.SetPayload(flow.Payload{
   199  			Seals: []*flow.Seal{b1Seal},
   200  		})
   201  		err = state.Extend(context.Background(), b5)
   202  		require.NoError(t, err)
   203  
   204  		// block 6 (sealB2, sealB3)
   205  		b2Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&b2Receipt.ExecutionResult))
   206  		b3Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&b3Receipt.ExecutionResult))
   207  		b6 := unittest.BlockWithParentFixture(b5.Header)
   208  		b6.SetPayload(flow.Payload{
   209  			Seals: []*flow.Seal{b2Seal, b3Seal},
   210  		})
   211  		err = state.Extend(context.Background(), b6)
   212  		require.NoError(t, err)
   213  
   214  		// block 7
   215  		b7 := unittest.BlockWithParentFixture(b6.Header)
   216  		b7.SetPayload(flow.EmptyPayload())
   217  		err = state.Extend(context.Background(), b7)
   218  		require.NoError(t, err)
   219  
   220  		// finalizing b1 - b4
   221  		// when B4 is finalized, can only find seal for G
   222  		err = state.Finalize(context.Background(), b1.ID())
   223  		require.NoError(t, err)
   224  		err = state.Finalize(context.Background(), b2.ID())
   225  		require.NoError(t, err)
   226  		err = state.Finalize(context.Background(), b3.ID())
   227  		require.NoError(t, err)
   228  		err = state.Finalize(context.Background(), b4.ID())
   229  		require.NoError(t, err)
   230  
   231  		metrics := metrics.NewNoopCollector()
   232  		seals := bstorage.NewSeals(metrics, db)
   233  
   234  		// can only find seal for G
   235  		_, err = seals.FinalizedSealForBlock(rootHeader.ID())
   236  		require.NoError(t, err)
   237  
   238  		_, err = seals.FinalizedSealForBlock(b1.ID())
   239  		require.Error(t, err)
   240  		require.ErrorIs(t, err, storage.ErrNotFound)
   241  
   242  		// when B5 is finalized, can find seal for B1
   243  		err = state.Finalize(context.Background(), b5.ID())
   244  		require.NoError(t, err)
   245  
   246  		s1, err := seals.FinalizedSealForBlock(b1.ID())
   247  		require.NoError(t, err)
   248  		require.Equal(t, b1Seal, s1)
   249  
   250  		_, err = seals.FinalizedSealForBlock(b2.ID())
   251  		require.Error(t, err)
   252  		require.ErrorIs(t, err, storage.ErrNotFound)
   253  
   254  		// when B7 is finalized, can find seals for B2, B3
   255  		err = state.Finalize(context.Background(), b6.ID())
   256  		require.NoError(t, err)
   257  
   258  		err = state.Finalize(context.Background(), b7.ID())
   259  		require.NoError(t, err)
   260  
   261  		s2, err := seals.FinalizedSealForBlock(b2.ID())
   262  		require.NoError(t, err)
   263  		require.Equal(t, b2Seal, s2)
   264  
   265  		s3, err := seals.FinalizedSealForBlock(b3.ID())
   266  		require.NoError(t, err)
   267  		require.Equal(t, b3Seal, s3)
   268  	})
   269  
   270  }
   271  
   272  func TestVersionBeaconIndex(t *testing.T) {
   273  	rootSnapshot := unittest.RootSnapshotFixture(participants)
   274  	util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
   275  		rootHeader, err := rootSnapshot.Head()
   276  		require.NoError(t, err)
   277  
   278  		// build a chain:
   279  		// G <- B1 <- B2 (resultB1(vb1)) <- B3 <- B4 (resultB2(vb2), resultB3(vb3)) <- B5 (sealB1) <- B6 (sealB2, sealB3)
   280  		// up until and including finalization of B5 there should be no VBs indexed
   281  		//    when B5 is finalized, index VB1
   282  		//    when B6 is finalized, we can index VB2 and VB3, but (only) the last one should be indexed by seal height
   283  
   284  		// block 1
   285  		b1 := unittest.BlockWithParentFixture(rootHeader)
   286  		b1.SetPayload(flow.EmptyPayload())
   287  		err = state.Extend(context.Background(), b1)
   288  		require.NoError(t, err)
   289  
   290  		vb1 := unittest.VersionBeaconFixture(
   291  			unittest.WithBoundaries(
   292  				flow.VersionBoundary{
   293  					BlockHeight: rootHeader.Height,
   294  					Version:     "0.21.37",
   295  				},
   296  				flow.VersionBoundary{
   297  					BlockHeight: rootHeader.Height + 100,
   298  					Version:     "0.21.38",
   299  				},
   300  			),
   301  		)
   302  		vb2 := unittest.VersionBeaconFixture(
   303  			unittest.WithBoundaries(
   304  				flow.VersionBoundary{
   305  					BlockHeight: rootHeader.Height,
   306  					Version:     "0.21.37",
   307  				},
   308  				flow.VersionBoundary{
   309  					BlockHeight: rootHeader.Height + 101,
   310  					Version:     "0.21.38",
   311  				},
   312  				flow.VersionBoundary{
   313  					BlockHeight: rootHeader.Height + 201,
   314  					Version:     "0.21.39",
   315  				},
   316  			),
   317  		)
   318  		vb3 := unittest.VersionBeaconFixture(
   319  			unittest.WithBoundaries(
   320  				flow.VersionBoundary{
   321  					BlockHeight: rootHeader.Height,
   322  					Version:     "0.21.37",
   323  				},
   324  				flow.VersionBoundary{
   325  					BlockHeight: rootHeader.Height + 99,
   326  					Version:     "0.21.38",
   327  				},
   328  				flow.VersionBoundary{
   329  					BlockHeight: rootHeader.Height + 199,
   330  					Version:     "0.21.39",
   331  				},
   332  				flow.VersionBoundary{
   333  					BlockHeight: rootHeader.Height + 299,
   334  					Version:     "0.21.40",
   335  				},
   336  			),
   337  		)
   338  
   339  		b1Receipt := unittest.ReceiptForBlockFixture(b1)
   340  		b1Receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{vb1.ServiceEvent()}
   341  		b2 := unittest.BlockWithParentFixture(b1.Header)
   342  		b2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(b1Receipt)))
   343  		err = state.Extend(context.Background(), b2)
   344  		require.NoError(t, err)
   345  
   346  		// block 3
   347  		b3 := unittest.BlockWithParentFixture(b2.Header)
   348  		b3.SetPayload(flow.EmptyPayload())
   349  		err = state.Extend(context.Background(), b3)
   350  		require.NoError(t, err)
   351  
   352  		// block 4 (resultB2, resultB3)
   353  		b2Receipt := unittest.ReceiptForBlockFixture(b2)
   354  		b2Receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{vb2.ServiceEvent()}
   355  
   356  		b3Receipt := unittest.ReceiptForBlockFixture(b3)
   357  		b3Receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{vb3.ServiceEvent()}
   358  
   359  		b4 := unittest.BlockWithParentFixture(b3.Header)
   360  		b4.SetPayload(flow.Payload{
   361  			Receipts: []*flow.ExecutionReceiptMeta{b2Receipt.Meta(), b3Receipt.Meta()},
   362  			Results:  []*flow.ExecutionResult{&b2Receipt.ExecutionResult, &b3Receipt.ExecutionResult},
   363  		})
   364  		err = state.Extend(context.Background(), b4)
   365  		require.NoError(t, err)
   366  
   367  		// block 5 (sealB1)
   368  		b1Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&b1Receipt.ExecutionResult))
   369  		b5 := unittest.BlockWithParentFixture(b4.Header)
   370  		b5.SetPayload(flow.Payload{
   371  			Seals: []*flow.Seal{b1Seal},
   372  		})
   373  		err = state.Extend(context.Background(), b5)
   374  		require.NoError(t, err)
   375  
   376  		// block 6 (sealB2, sealB3)
   377  		b2Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&b2Receipt.ExecutionResult))
   378  		b3Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&b3Receipt.ExecutionResult))
   379  		b6 := unittest.BlockWithParentFixture(b5.Header)
   380  		b6.SetPayload(flow.Payload{
   381  			Seals: []*flow.Seal{b2Seal, b3Seal},
   382  		})
   383  		err = state.Extend(context.Background(), b6)
   384  		require.NoError(t, err)
   385  
   386  		versionBeacons := bstorage.NewVersionBeacons(db)
   387  
   388  		// No VB can be found before finalizing anything
   389  		vb, err := versionBeacons.Highest(b6.Header.Height)
   390  		require.NoError(t, err)
   391  		require.Nil(t, vb)
   392  
   393  		// finalizing b1 - b5
   394  		err = state.Finalize(context.Background(), b1.ID())
   395  		require.NoError(t, err)
   396  		err = state.Finalize(context.Background(), b2.ID())
   397  		require.NoError(t, err)
   398  		err = state.Finalize(context.Background(), b3.ID())
   399  		require.NoError(t, err)
   400  		err = state.Finalize(context.Background(), b4.ID())
   401  		require.NoError(t, err)
   402  
   403  		// No VB can be found after finalizing B4
   404  		vb, err = versionBeacons.Highest(b6.Header.Height)
   405  		require.NoError(t, err)
   406  		require.Nil(t, vb)
   407  
   408  		// once B5 is finalized, B1 and VB1 are sealed, hence index should now find it
   409  		err = state.Finalize(context.Background(), b5.ID())
   410  		require.NoError(t, err)
   411  
   412  		versionBeacon, err := versionBeacons.Highest(b6.Header.Height)
   413  		require.NoError(t, err)
   414  		require.Equal(t,
   415  			&flow.SealedVersionBeacon{
   416  				VersionBeacon: vb1,
   417  				SealHeight:    b5.Header.Height,
   418  			},
   419  			versionBeacon,
   420  		)
   421  
   422  		// finalizing B6 should index events sealed by B6, so VB2 and VB3
   423  		// while we don't expect multiple VBs in one block, we index newest, so last one emitted - VB3
   424  		err = state.Finalize(context.Background(), b6.ID())
   425  		require.NoError(t, err)
   426  
   427  		versionBeacon, err = versionBeacons.Highest(b6.Header.Height)
   428  		require.NoError(t, err)
   429  		require.Equal(t,
   430  			&flow.SealedVersionBeacon{
   431  				VersionBeacon: vb3,
   432  				SealHeight:    b6.Header.Height,
   433  			},
   434  			versionBeacon,
   435  		)
   436  	})
   437  }
   438  
   439  func TestExtendSealedBoundary(t *testing.T) {
   440  	rootSnapshot := unittest.RootSnapshotFixture(participants)
   441  	util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
   442  		head, err := rootSnapshot.Head()
   443  		require.NoError(t, err)
   444  		_, seal, err := rootSnapshot.SealedResult()
   445  		require.NoError(t, err)
   446  		finalCommit, err := state.Final().Commit()
   447  		require.NoError(t, err)
   448  		require.Equal(t, seal.FinalState, finalCommit, "original commit should be root commit")
   449  
   450  		// Create a first block on top of the snapshot
   451  		block1 := unittest.BlockWithParentFixture(head)
   452  		block1.SetPayload(flow.EmptyPayload())
   453  		err = state.Extend(context.Background(), block1)
   454  		require.NoError(t, err)
   455  
   456  		// Add a second block containing a receipt committing to the first block
   457  		block1Receipt := unittest.ReceiptForBlockFixture(block1)
   458  		block2 := unittest.BlockWithParentFixture(block1.Header)
   459  		block2.SetPayload(flow.Payload{
   460  			Receipts: []*flow.ExecutionReceiptMeta{block1Receipt.Meta()},
   461  			Results:  []*flow.ExecutionResult{&block1Receipt.ExecutionResult},
   462  		})
   463  		err = state.Extend(context.Background(), block2)
   464  		require.NoError(t, err)
   465  
   466  		// Add a third block containing a seal for the first block
   467  		block1Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&block1Receipt.ExecutionResult))
   468  		block3 := unittest.BlockWithParentFixture(block2.Header)
   469  		block3.SetPayload(flow.Payload{
   470  			Seals: []*flow.Seal{block1Seal},
   471  		})
   472  		err = state.Extend(context.Background(), block3)
   473  		require.NoError(t, err)
   474  
   475  		finalCommit, err = state.Final().Commit()
   476  		require.NoError(t, err)
   477  		require.Equal(t, seal.FinalState, finalCommit, "commit should not change before finalizing")
   478  
   479  		err = state.Finalize(context.Background(), block1.ID())
   480  		require.NoError(t, err)
   481  
   482  		finalCommit, err = state.Final().Commit()
   483  		require.NoError(t, err)
   484  		require.Equal(t, seal.FinalState, finalCommit, "commit should not change after finalizing non-sealing block")
   485  
   486  		err = state.Finalize(context.Background(), block2.ID())
   487  		require.NoError(t, err)
   488  
   489  		finalCommit, err = state.Final().Commit()
   490  		require.NoError(t, err)
   491  		require.Equal(t, seal.FinalState, finalCommit, "commit should not change after finalizing non-sealing block")
   492  
   493  		err = state.Finalize(context.Background(), block3.ID())
   494  		require.NoError(t, err)
   495  
   496  		finalCommit, err = state.Final().Commit()
   497  		require.NoError(t, err)
   498  		require.Equal(t, block1Seal.FinalState, finalCommit, "commit should change after finalizing sealing block")
   499  	})
   500  }
   501  
   502  func TestExtendMissingParent(t *testing.T) {
   503  	rootSnapshot := unittest.RootSnapshotFixture(participants)
   504  	util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
   505  		extend := unittest.BlockFixture()
   506  		extend.Payload.Guarantees = nil
   507  		extend.Payload.Seals = nil
   508  		extend.Header.Height = 2
   509  		extend.Header.View = 2
   510  		extend.Header.ParentID = unittest.BlockFixture().ID()
   511  		extend.Header.PayloadHash = extend.Payload.Hash()
   512  
   513  		err := state.Extend(context.Background(), &extend)
   514  		require.Error(t, err)
   515  		require.True(t, st.IsInvalidExtensionError(err), err)
   516  
   517  		// verify seal not indexed
   518  		var sealID flow.Identifier
   519  		err = db.View(operation.LookupLatestSealAtBlock(extend.ID(), &sealID))
   520  		require.Error(t, err)
   521  		require.ErrorIs(t, err, stoerr.ErrNotFound)
   522  	})
   523  }
   524  
   525  func TestExtendHeightTooSmall(t *testing.T) {
   526  	rootSnapshot := unittest.RootSnapshotFixture(participants)
   527  	util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
   528  		head, err := rootSnapshot.Head()
   529  		require.NoError(t, err)
   530  
   531  		extend := unittest.BlockFixture()
   532  		extend.SetPayload(flow.EmptyPayload())
   533  		extend.Header.Height = 1
   534  		extend.Header.View = 1
   535  		extend.Header.ParentID = head.ID()
   536  		extend.Header.ParentView = head.View
   537  
   538  		err = state.Extend(context.Background(), &extend)
   539  		require.NoError(t, err)
   540  
   541  		// create another block with the same height and view, that is coming after
   542  		extend.Header.ParentID = extend.Header.ID()
   543  		extend.Header.Height = 1
   544  		extend.Header.View = 2
   545  
   546  		err = state.Extend(context.Background(), &extend)
   547  		require.Error(t, err)
   548  
   549  		// verify seal not indexed
   550  		var sealID flow.Identifier
   551  		err = db.View(operation.LookupLatestSealAtBlock(extend.ID(), &sealID))
   552  		require.Error(t, err)
   553  		require.ErrorIs(t, err, stoerr.ErrNotFound)
   554  	})
   555  }
   556  
   557  func TestExtendHeightTooLarge(t *testing.T) {
   558  	rootSnapshot := unittest.RootSnapshotFixture(participants)
   559  	util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
   560  
   561  		head, err := rootSnapshot.Head()
   562  		require.NoError(t, err)
   563  
   564  		block := unittest.BlockWithParentFixture(head)
   565  		block.SetPayload(flow.EmptyPayload())
   566  		// set an invalid height
   567  		block.Header.Height = head.Height + 2
   568  
   569  		err = state.Extend(context.Background(), block)
   570  		require.Error(t, err)
   571  	})
   572  }
   573  
   574  // TestExtendInconsistentParentView tests if mutator rejects block with invalid ParentView. ParentView must be consistent
   575  // with view of block referred by ParentID.
   576  func TestExtendInconsistentParentView(t *testing.T) {
   577  	rootSnapshot := unittest.RootSnapshotFixture(participants)
   578  	util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
   579  
   580  		head, err := rootSnapshot.Head()
   581  		require.NoError(t, err)
   582  
   583  		block := unittest.BlockWithParentFixture(head)
   584  		block.SetPayload(flow.EmptyPayload())
   585  		// set an invalid parent view
   586  		block.Header.ParentView++
   587  
   588  		err = state.Extend(context.Background(), block)
   589  		require.Error(t, err)
   590  		require.True(t, st.IsInvalidExtensionError(err))
   591  	})
   592  }
   593  
   594  func TestExtendBlockNotConnected(t *testing.T) {
   595  	rootSnapshot := unittest.RootSnapshotFixture(participants)
   596  	util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
   597  
   598  		head, err := rootSnapshot.Head()
   599  		require.NoError(t, err)
   600  
   601  		// add 2 blocks, the second finalizing/sealing the state of the first
   602  		extend := unittest.BlockWithParentFixture(head)
   603  		extend.SetPayload(flow.EmptyPayload())
   604  
   605  		err = state.Extend(context.Background(), extend)
   606  		require.NoError(t, err)
   607  
   608  		err = state.Finalize(context.Background(), extend.ID())
   609  		require.NoError(t, err)
   610  
   611  		// create a fork at view/height 1 and try to connect it to root
   612  		extend.Header.Timestamp = extend.Header.Timestamp.Add(time.Second)
   613  		extend.Header.ParentID = head.ID()
   614  
   615  		err = state.Extend(context.Background(), extend)
   616  		require.Error(t, err)
   617  
   618  		// verify seal not indexed
   619  		var sealID flow.Identifier
   620  		err = db.View(operation.LookupLatestSealAtBlock(extend.ID(), &sealID))
   621  		require.Error(t, err)
   622  		require.ErrorIs(t, err, stoerr.ErrNotFound)
   623  	})
   624  }
   625  
   626  func TestExtendInvalidChainID(t *testing.T) {
   627  	rootSnapshot := unittest.RootSnapshotFixture(participants)
   628  	util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
   629  		head, err := rootSnapshot.Head()
   630  		require.NoError(t, err)
   631  
   632  		block := unittest.BlockWithParentFixture(head)
   633  		block.SetPayload(flow.EmptyPayload())
   634  		// use an invalid chain ID
   635  		block.Header.ChainID = head.ChainID + "-invalid"
   636  
   637  		err = state.Extend(context.Background(), block)
   638  		require.Error(t, err)
   639  		require.True(t, st.IsInvalidExtensionError(err), err)
   640  	})
   641  }
   642  
   643  func TestExtendReceiptsNotSorted(t *testing.T) {
   644  	// TODO: this test needs to be updated:
   645  	// We don't require the receipts to be sorted by height anymore
   646  	// We could require an "parent first" ordering, which is less strict than
   647  	// a full ordering by height
   648  	unittest.SkipUnless(t, unittest.TEST_TODO, "needs update")
   649  
   650  	rootSnapshot := unittest.RootSnapshotFixture(participants)
   651  	head, err := rootSnapshot.Head()
   652  	require.NoError(t, err)
   653  	util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
   654  		// create block2 and block3
   655  		block2 := unittest.BlockWithParentFixture(head)
   656  		block2.Payload.Guarantees = nil
   657  		block2.Header.PayloadHash = block2.Payload.Hash()
   658  		err := state.Extend(context.Background(), block2)
   659  		require.NoError(t, err)
   660  
   661  		block3 := unittest.BlockWithParentFixture(block2.Header)
   662  		block3.Payload.Guarantees = nil
   663  		block3.Header.PayloadHash = block3.Payload.Hash()
   664  		err = state.Extend(context.Background(), block3)
   665  		require.NoError(t, err)
   666  
   667  		receiptA := unittest.ReceiptForBlockFixture(block3)
   668  		receiptB := unittest.ReceiptForBlockFixture(block2)
   669  
   670  		// insert a block with payload receipts not sorted by block height.
   671  		block4 := unittest.BlockWithParentFixture(block3.Header)
   672  		block4.Payload = &flow.Payload{
   673  			Receipts: []*flow.ExecutionReceiptMeta{receiptA.Meta(), receiptB.Meta()},
   674  			Results:  []*flow.ExecutionResult{&receiptA.ExecutionResult, &receiptB.ExecutionResult},
   675  		}
   676  		block4.Header.PayloadHash = block4.Payload.Hash()
   677  		err = state.Extend(context.Background(), block4)
   678  		require.Error(t, err)
   679  		require.True(t, st.IsInvalidExtensionError(err), err)
   680  	})
   681  }
   682  
   683  func TestExtendReceiptsInvalid(t *testing.T) {
   684  	validator := mockmodule.NewReceiptValidator(t)
   685  
   686  	rootSnapshot := unittest.RootSnapshotFixture(participants)
   687  	util.RunWithFullProtocolStateAndValidator(t, rootSnapshot, validator, func(db *badger.DB, state *protocol.ParticipantState) {
   688  		head, err := rootSnapshot.Head()
   689  		require.NoError(t, err)
   690  
   691  		validator.On("ValidatePayload", mock.Anything).Return(nil).Once()
   692  
   693  		// create block2 and block3
   694  		block2 := unittest.BlockWithParentFixture(head)
   695  		block2.SetPayload(flow.EmptyPayload())
   696  		err = state.Extend(context.Background(), block2)
   697  		require.NoError(t, err)
   698  
   699  		// Add a receipt for block 2
   700  		receipt := unittest.ExecutionReceiptFixture()
   701  
   702  		block3 := unittest.BlockWithParentFixture(block2.Header)
   703  		block3.SetPayload(flow.Payload{
   704  			Receipts: []*flow.ExecutionReceiptMeta{receipt.Meta()},
   705  			Results:  []*flow.ExecutionResult{&receipt.ExecutionResult},
   706  		})
   707  
   708  		// force the receipt validator to refuse this payload
   709  		validator.On("ValidatePayload", block3).Return(engine.NewInvalidInputError("")).Once()
   710  
   711  		err = state.Extend(context.Background(), block3)
   712  		require.Error(t, err)
   713  		require.True(t, st.IsInvalidExtensionError(err), err)
   714  	})
   715  }
   716  
   717  func TestExtendReceiptsValid(t *testing.T) {
   718  	rootSnapshot := unittest.RootSnapshotFixture(participants)
   719  	util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
   720  		head, err := rootSnapshot.Head()
   721  		require.NoError(t, err)
   722  		block2 := unittest.BlockWithParentFixture(head)
   723  		block2.SetPayload(flow.EmptyPayload())
   724  		err = state.Extend(context.Background(), block2)
   725  		require.NoError(t, err)
   726  
   727  		block3 := unittest.BlockWithParentFixture(block2.Header)
   728  		block3.SetPayload(flow.EmptyPayload())
   729  		err = state.Extend(context.Background(), block3)
   730  		require.NoError(t, err)
   731  
   732  		block4 := unittest.BlockWithParentFixture(block3.Header)
   733  		block4.SetPayload(flow.EmptyPayload())
   734  		err = state.Extend(context.Background(), block4)
   735  		require.NoError(t, err)
   736  
   737  		receipt3a := unittest.ReceiptForBlockFixture(block3)
   738  		receipt3b := unittest.ReceiptForBlockFixture(block3)
   739  		receipt3c := unittest.ReceiptForBlockFixture(block4)
   740  
   741  		block5 := unittest.BlockWithParentFixture(block4.Header)
   742  		block5.SetPayload(flow.Payload{
   743  			Receipts: []*flow.ExecutionReceiptMeta{
   744  				receipt3a.Meta(),
   745  				receipt3b.Meta(),
   746  				receipt3c.Meta(),
   747  			},
   748  			Results: []*flow.ExecutionResult{
   749  				&receipt3a.ExecutionResult,
   750  				&receipt3b.ExecutionResult,
   751  				&receipt3c.ExecutionResult,
   752  			},
   753  		})
   754  		err = state.Extend(context.Background(), block5)
   755  		require.NoError(t, err)
   756  	})
   757  }
   758  
   759  // Tests the full flow of transitioning between epochs by finalizing a setup
   760  // event, then a commit event, then finalizing the first block of the next epoch.
   761  // Also tests that appropriate epoch transition events are fired.
   762  //
   763  // Epoch information becomes available in the protocol state in the block containing the seal
   764  // for the block whose execution emitted the service event.
   765  //
   766  // ROOT <- B1 <- B2(R1) <- B3(S1) <- B4 <- B5(R2) <- B6(S2) <- B7 <-|- B8
   767  //
   768  // B3 seals B1, in which EpochSetup is emitted.
   769  //   - we can query the EpochSetup beginning with B3
   770  //   - EpochSetupPhaseStarted triggered when B3 is finalized
   771  //
   772  // B6 seals B2, in which EpochCommitted is emitted.
   773  //   - we can query the EpochCommit beginning with B6
   774  //   - EpochCommittedPhaseStarted triggered when B6 is finalized
   775  //
   776  // B7 is the final block of the epoch.
   777  // B8 is the first block of the NEXT epoch.
   778  func TestExtendEpochTransitionValid(t *testing.T) {
   779  	// create an event consumer to test epoch transition events
   780  	consumer := mockprotocol.NewConsumer(t)
   781  	consumer.On("BlockFinalized", mock.Anything)
   782  	consumer.On("BlockProcessable", mock.Anything, mock.Anything)
   783  	rootSnapshot := unittest.RootSnapshotFixture(participants)
   784  
   785  	unittest.RunWithBadgerDB(t, func(db *badger.DB) {
   786  
   787  		// set up state and mock ComplianceMetrics object
   788  		metrics := mockmodule.NewComplianceMetrics(t)
   789  		metrics.On("BlockSealed", mock.Anything)
   790  		metrics.On("SealedHeight", mock.Anything)
   791  		metrics.On("FinalizedHeight", mock.Anything)
   792  		metrics.On("BlockFinalized", mock.Anything)
   793  
   794  		// expect epoch metric calls on bootstrap
   795  		initialCurrentEpoch := rootSnapshot.Epochs().Current()
   796  		counter, err := initialCurrentEpoch.Counter()
   797  		require.NoError(t, err)
   798  		finalView, err := initialCurrentEpoch.FinalView()
   799  		require.NoError(t, err)
   800  		initialPhase, err := rootSnapshot.Phase()
   801  		require.NoError(t, err)
   802  		metrics.On("CurrentEpochCounter", counter).Once()
   803  		metrics.On("CurrentEpochPhase", initialPhase).Once()
   804  		metrics.On("CommittedEpochFinalView", finalView).Once()
   805  
   806  		metrics.On("CurrentEpochFinalView", finalView).Once()
   807  
   808  		dkgPhase1FinalView, dkgPhase2FinalView, dkgPhase3FinalView, err := realprotocol.DKGPhaseViews(initialCurrentEpoch)
   809  		require.NoError(t, err)
   810  		metrics.On("CurrentDKGPhase1FinalView", dkgPhase1FinalView).Once()
   811  		metrics.On("CurrentDKGPhase2FinalView", dkgPhase2FinalView).Once()
   812  		metrics.On("CurrentDKGPhase3FinalView", dkgPhase3FinalView).Once()
   813  
   814  		tracer := trace.NewNoopTracer()
   815  		log := zerolog.Nop()
   816  		all := storeutil.StorageLayer(t, db)
   817  		protoState, err := protocol.Bootstrap(
   818  			metrics,
   819  			db,
   820  			all.Headers,
   821  			all.Seals,
   822  			all.Results,
   823  			all.Blocks,
   824  			all.QuorumCertificates,
   825  			all.Setups,
   826  			all.EpochCommits,
   827  			all.Statuses,
   828  			all.VersionBeacons,
   829  			rootSnapshot,
   830  		)
   831  		require.NoError(t, err)
   832  		receiptValidator := util.MockReceiptValidator()
   833  		sealValidator := util.MockSealValidator(all.Seals)
   834  		state, err := protocol.NewFullConsensusState(
   835  			log,
   836  			tracer,
   837  			consumer,
   838  			protoState,
   839  			all.Index,
   840  			all.Payloads,
   841  			util.MockBlockTimer(),
   842  			receiptValidator,
   843  			sealValidator,
   844  		)
   845  		require.NoError(t, err)
   846  
   847  		head, err := rootSnapshot.Head()
   848  		require.NoError(t, err)
   849  		result, _, err := rootSnapshot.SealedResult()
   850  		require.NoError(t, err)
   851  
   852  		// we should begin the epoch in the staking phase
   853  		phase, err := state.AtBlockID(head.ID()).Phase()
   854  		assert.NoError(t, err)
   855  		require.Equal(t, flow.EpochPhaseStaking, phase)
   856  
   857  		// add a block for the first seal to reference
   858  		block1 := unittest.BlockWithParentFixture(head)
   859  		block1.SetPayload(flow.EmptyPayload())
   860  		err = state.Extend(context.Background(), block1)
   861  		require.NoError(t, err)
   862  		err = state.Finalize(context.Background(), block1.ID())
   863  		require.NoError(t, err)
   864  
   865  		epoch1Setup := result.ServiceEvents[0].Event.(*flow.EpochSetup)
   866  		epoch1FinalView := epoch1Setup.FinalView
   867  
   868  		// add a participant for the next epoch
   869  		epoch2NewParticipant := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification))
   870  		epoch2Participants := append(participants, epoch2NewParticipant).Sort(flow.Canonical)
   871  
   872  		// create the epoch setup event for the second epoch
   873  		epoch2Setup := unittest.EpochSetupFixture(
   874  			unittest.WithParticipants(epoch2Participants),
   875  			unittest.SetupWithCounter(epoch1Setup.Counter+1),
   876  			unittest.WithFinalView(epoch1FinalView+1000),
   877  			unittest.WithFirstView(epoch1FinalView+1),
   878  		)
   879  
   880  		// create a receipt for block 1 containing the EpochSetup event
   881  		receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1)
   882  		receipt1.ExecutionResult.ServiceEvents = []flow.ServiceEvent{epoch2Setup.ServiceEvent()}
   883  		seal1.ResultID = receipt1.ExecutionResult.ID()
   884  
   885  		// add a second block with the receipt for block 1
   886  		block2 := unittest.BlockWithParentFixture(block1.Header)
   887  		block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1)))
   888  
   889  		err = state.Extend(context.Background(), block2)
   890  		require.NoError(t, err)
   891  		err = state.Finalize(context.Background(), block2.ID())
   892  		require.NoError(t, err)
   893  
   894  		// block 3 contains the seal for block 1
   895  		block3 := unittest.BlockWithParentFixture(block2.Header)
   896  		block3.SetPayload(flow.Payload{
   897  			Seals: []*flow.Seal{seal1},
   898  		})
   899  
   900  		// insert the block sealing the EpochSetup event
   901  		err = state.Extend(context.Background(), block3)
   902  		require.NoError(t, err)
   903  
   904  		// now that the setup event has been emitted, we should be in the setup phase
   905  		phase, err = state.AtBlockID(block3.ID()).Phase()
   906  		assert.NoError(t, err)
   907  		require.Equal(t, flow.EpochPhaseSetup, phase)
   908  
   909  		// we should NOT be able to query epoch 2 wrt blocks before 3
   910  		for _, blockID := range []flow.Identifier{block1.ID(), block2.ID()} {
   911  			_, err = state.AtBlockID(blockID).Epochs().Next().InitialIdentities()
   912  			require.Error(t, err)
   913  			_, err = state.AtBlockID(blockID).Epochs().Next().Clustering()
   914  			require.Error(t, err)
   915  		}
   916  
   917  		// we should be able to query epoch 2 wrt block 3
   918  		_, err = state.AtBlockID(block3.ID()).Epochs().Next().InitialIdentities()
   919  		assert.NoError(t, err)
   920  		_, err = state.AtBlockID(block3.ID()).Epochs().Next().Clustering()
   921  		assert.NoError(t, err)
   922  
   923  		// only setup event is finalized, not commit, so shouldn't be able to get certain info
   924  		_, err = state.AtBlockID(block3.ID()).Epochs().Next().DKG()
   925  		require.Error(t, err)
   926  
   927  		// insert B4
   928  		block4 := unittest.BlockWithParentFixture(block3.Header)
   929  		err = state.Extend(context.Background(), block4)
   930  		require.NoError(t, err)
   931  
   932  		consumer.On("EpochSetupPhaseStarted", epoch2Setup.Counter-1, block3.Header).Once()
   933  		metrics.On("CurrentEpochPhase", flow.EpochPhaseSetup).Once()
   934  		// finalize block 3, so we can finalize subsequent blocks
   935  		// ensure an epoch phase transition when we finalize block 3
   936  		err = state.Finalize(context.Background(), block3.ID())
   937  		require.NoError(t, err)
   938  		consumer.AssertCalled(t, "EpochSetupPhaseStarted", epoch2Setup.Counter-1, block3.Header)
   939  		metrics.AssertCalled(t, "CurrentEpochPhase", flow.EpochPhaseSetup)
   940  
   941  		// now that the setup event has been emitted, we should be in the setup phase
   942  		phase, err = state.AtBlockID(block3.ID()).Phase()
   943  		require.NoError(t, err)
   944  		require.Equal(t, flow.EpochPhaseSetup, phase)
   945  
   946  		// finalize block 4
   947  		err = state.Finalize(context.Background(), block4.ID())
   948  		require.NoError(t, err)
   949  
   950  		epoch2Commit := unittest.EpochCommitFixture(
   951  			unittest.CommitWithCounter(epoch2Setup.Counter),
   952  			unittest.WithClusterQCsFromAssignments(epoch2Setup.Assignments),
   953  			unittest.WithDKGFromParticipants(epoch2Participants),
   954  		)
   955  
   956  		// create receipt and seal for block 2
   957  		// the receipt for block 2 contains the EpochCommit event
   958  		receipt2, seal2 := unittest.ReceiptAndSealForBlock(block2)
   959  		receipt2.ExecutionResult.ServiceEvents = []flow.ServiceEvent{epoch2Commit.ServiceEvent()}
   960  		seal2.ResultID = receipt2.ExecutionResult.ID()
   961  
   962  		// block 5 contains the receipt for block 2
   963  		block5 := unittest.BlockWithParentFixture(block4.Header)
   964  		block5.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt2)))
   965  
   966  		err = state.Extend(context.Background(), block5)
   967  		require.NoError(t, err)
   968  		err = state.Finalize(context.Background(), block5.ID())
   969  		require.NoError(t, err)
   970  
   971  		// block 6 contains the seal for block 2
   972  		block6 := unittest.BlockWithParentFixture(block5.Header)
   973  		block6.SetPayload(flow.Payload{
   974  			Seals: []*flow.Seal{seal2},
   975  		})
   976  
   977  		err = state.Extend(context.Background(), block6)
   978  		require.NoError(t, err)
   979  
   980  		// we should NOT be able to query epoch 2 commit info wrt blocks before 6
   981  		for _, blockID := range []flow.Identifier{block4.ID(), block5.ID()} {
   982  			_, err = state.AtBlockID(blockID).Epochs().Next().DKG()
   983  			require.Error(t, err)
   984  		}
   985  
   986  		// now epoch 2 is fully ready, we can query anything we want about it wrt block 6 (or later)
   987  		_, err = state.AtBlockID(block6.ID()).Epochs().Next().InitialIdentities()
   988  		require.NoError(t, err)
   989  		_, err = state.AtBlockID(block6.ID()).Epochs().Next().Clustering()
   990  		require.NoError(t, err)
   991  		_, err = state.AtBlockID(block6.ID()).Epochs().Next().DKG()
   992  		assert.NoError(t, err)
   993  
   994  		// now that the commit event has been emitted, we should be in the committed phase
   995  		phase, err = state.AtBlockID(block6.ID()).Phase()
   996  		assert.NoError(t, err)
   997  		require.Equal(t, flow.EpochPhaseCommitted, phase)
   998  
   999  		// block 7 has the final view of the epoch, insert it, finalized after finalizing block 6
  1000  		block7 := unittest.BlockWithParentFixture(block6.Header)
  1001  		block7.SetPayload(flow.EmptyPayload())
  1002  		block7.Header.View = epoch1FinalView
  1003  		err = state.Extend(context.Background(), block7)
  1004  		require.NoError(t, err)
  1005  
  1006  		// expect epoch phase transition once we finalize block 6
  1007  		consumer.On("EpochCommittedPhaseStarted", epoch2Setup.Counter-1, block6.Header).Once()
  1008  		// expect committed final view to be updated, since we are committing epoch 2
  1009  		metrics.On("CommittedEpochFinalView", epoch2Setup.FinalView).Once()
  1010  		metrics.On("CurrentEpochPhase", flow.EpochPhaseCommitted).Once()
  1011  
  1012  		err = state.Finalize(context.Background(), block6.ID())
  1013  		require.NoError(t, err)
  1014  
  1015  		consumer.AssertCalled(t, "EpochCommittedPhaseStarted", epoch2Setup.Counter-1, block6.Header)
  1016  		metrics.AssertCalled(t, "CommittedEpochFinalView", epoch2Setup.FinalView)
  1017  		metrics.AssertCalled(t, "CurrentEpochPhase", flow.EpochPhaseCommitted)
  1018  
  1019  		// we should still be in epoch 1
  1020  		epochCounter, err := state.AtBlockID(block4.ID()).Epochs().Current().Counter()
  1021  		require.NoError(t, err)
  1022  		require.Equal(t, epoch1Setup.Counter, epochCounter)
  1023  
  1024  		err = state.Finalize(context.Background(), block7.ID())
  1025  		require.NoError(t, err)
  1026  
  1027  		// we should still be in epoch 1, since epochs are inclusive of final view
  1028  		epochCounter, err = state.AtBlockID(block7.ID()).Epochs().Current().Counter()
  1029  		require.NoError(t, err)
  1030  		require.Equal(t, epoch1Setup.Counter, epochCounter)
  1031  
  1032  		// block 8 has a view > final view of epoch 1, it will be considered the first block of epoch 2
  1033  		block8 := unittest.BlockWithParentFixture(block7.Header)
  1034  		block8.SetPayload(flow.EmptyPayload())
  1035  		// we should handle views that aren't exactly the first valid view of the epoch
  1036  		block8.Header.View = epoch1FinalView + uint64(1+rand.Intn(10))
  1037  
  1038  		err = state.Extend(context.Background(), block8)
  1039  		require.NoError(t, err)
  1040  
  1041  		// now, at long last, we are in epoch 2
  1042  		epochCounter, err = state.AtBlockID(block8.ID()).Epochs().Current().Counter()
  1043  		require.NoError(t, err)
  1044  		require.Equal(t, epoch2Setup.Counter, epochCounter)
  1045  
  1046  		// we should begin epoch 2 in staking phase
  1047  		// how that the commit event has been emitted, we should be in the committed phase
  1048  		phase, err = state.AtBlockID(block8.ID()).Phase()
  1049  		assert.NoError(t, err)
  1050  		require.Equal(t, flow.EpochPhaseStaking, phase)
  1051  
  1052  		// expect epoch transition once we finalize block 9
  1053  		consumer.On("EpochTransition", epoch2Setup.Counter, block8.Header).Once()
  1054  		metrics.On("EpochTransitionHeight", block8.Header.Height).Once()
  1055  		metrics.On("CurrentEpochCounter", epoch2Setup.Counter).Once()
  1056  		metrics.On("CurrentEpochPhase", flow.EpochPhaseStaking).Once()
  1057  		metrics.On("CurrentEpochFinalView", epoch2Setup.FinalView).Once()
  1058  		metrics.On("CurrentDKGPhase1FinalView", epoch2Setup.DKGPhase1FinalView).Once()
  1059  		metrics.On("CurrentDKGPhase2FinalView", epoch2Setup.DKGPhase2FinalView).Once()
  1060  		metrics.On("CurrentDKGPhase3FinalView", epoch2Setup.DKGPhase3FinalView).Once()
  1061  
  1062  		// before block 9 is finalized, the epoch 1-2 boundary is unknown
  1063  		_, err = state.AtBlockID(block8.ID()).Epochs().Current().FinalHeight()
  1064  		assert.ErrorIs(t, err, realprotocol.ErrEpochTransitionNotFinalized)
  1065  		_, err = state.AtBlockID(block8.ID()).Epochs().Current().FirstHeight()
  1066  		assert.ErrorIs(t, err, realprotocol.ErrEpochTransitionNotFinalized)
  1067  
  1068  		err = state.Finalize(context.Background(), block8.ID())
  1069  		require.NoError(t, err)
  1070  
  1071  		// once block 8 is finalized, epoch 2 has unambiguously begun - the epoch 1-2 boundary is known
  1072  		epoch1FinalHeight, err := state.AtBlockID(block8.ID()).Epochs().Previous().FinalHeight()
  1073  		require.NoError(t, err)
  1074  		assert.Equal(t, block7.Header.Height, epoch1FinalHeight)
  1075  		epoch2FirstHeight, err := state.AtBlockID(block8.ID()).Epochs().Current().FirstHeight()
  1076  		require.NoError(t, err)
  1077  		assert.Equal(t, block8.Header.Height, epoch2FirstHeight)
  1078  	})
  1079  }
  1080  
  1081  // we should be able to have conflicting forks with two different instances of
  1082  // the same service event for the same epoch
  1083  //
  1084  //	         /--B1<--B3(R1)<--B5(S1)<--B7
  1085  //	ROOT <--+
  1086  //	         \--B2<--B4(R2)<--B6(S2)<--B8
  1087  func TestExtendConflictingEpochEvents(t *testing.T) {
  1088  	rootSnapshot := unittest.RootSnapshotFixture(participants)
  1089  	util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
  1090  
  1091  		head, err := rootSnapshot.Head()
  1092  		require.NoError(t, err)
  1093  		result, _, err := rootSnapshot.SealedResult()
  1094  		require.NoError(t, err)
  1095  
  1096  		// add two conflicting blocks for each service event to reference
  1097  		block1 := unittest.BlockWithParentFixture(head)
  1098  		block1.SetPayload(flow.EmptyPayload())
  1099  		err = state.Extend(context.Background(), block1)
  1100  		require.NoError(t, err)
  1101  
  1102  		block2 := unittest.BlockWithParentFixture(head)
  1103  		block2.SetPayload(flow.EmptyPayload())
  1104  		err = state.Extend(context.Background(), block2)
  1105  		require.NoError(t, err)
  1106  
  1107  		rootSetup := result.ServiceEvents[0].Event.(*flow.EpochSetup)
  1108  
  1109  		// create two conflicting epoch setup events for the next epoch (final view differs)
  1110  		nextEpochSetup1 := unittest.EpochSetupFixture(
  1111  			unittest.WithParticipants(rootSetup.Participants),
  1112  			unittest.SetupWithCounter(rootSetup.Counter+1),
  1113  			unittest.WithFinalView(rootSetup.FinalView+1000),
  1114  			unittest.WithFirstView(rootSetup.FinalView+1),
  1115  		)
  1116  		nextEpochSetup2 := unittest.EpochSetupFixture(
  1117  			unittest.WithParticipants(rootSetup.Participants),
  1118  			unittest.SetupWithCounter(rootSetup.Counter+1),
  1119  			unittest.WithFinalView(rootSetup.FinalView+2000), // final view differs
  1120  			unittest.WithFirstView(rootSetup.FinalView+1),
  1121  		)
  1122  
  1123  		// add blocks containing receipts for block1 and block2 (necessary for sealing)
  1124  		// block 1 receipt contains nextEpochSetup1
  1125  		block1Receipt := unittest.ReceiptForBlockFixture(block1)
  1126  		block1Receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{nextEpochSetup1.ServiceEvent()}
  1127  
  1128  		// add block 1 receipt to block 3 payload
  1129  		block3 := unittest.BlockWithParentFixture(block1.Header)
  1130  		block3.SetPayload(flow.Payload{
  1131  			Receipts: []*flow.ExecutionReceiptMeta{block1Receipt.Meta()},
  1132  			Results:  []*flow.ExecutionResult{&block1Receipt.ExecutionResult},
  1133  		})
  1134  		err = state.Extend(context.Background(), block3)
  1135  		require.NoError(t, err)
  1136  
  1137  		// block 2 receipt contains nextEpochSetup2
  1138  		block2Receipt := unittest.ReceiptForBlockFixture(block2)
  1139  		block2Receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{nextEpochSetup2.ServiceEvent()}
  1140  
  1141  		// add block 2 receipt to block 4 payload
  1142  		block4 := unittest.BlockWithParentFixture(block2.Header)
  1143  		block4.SetPayload(flow.Payload{
  1144  			Receipts: []*flow.ExecutionReceiptMeta{block2Receipt.Meta()},
  1145  			Results:  []*flow.ExecutionResult{&block2Receipt.ExecutionResult},
  1146  		})
  1147  		err = state.Extend(context.Background(), block4)
  1148  		require.NoError(t, err)
  1149  
  1150  		// seal for block 1
  1151  		seal1 := unittest.Seal.Fixture(unittest.Seal.WithResult(&block1Receipt.ExecutionResult))
  1152  
  1153  		// seal for block 2
  1154  		seal2 := unittest.Seal.Fixture(unittest.Seal.WithResult(&block2Receipt.ExecutionResult))
  1155  
  1156  		// block 5 builds on block 3, contains seal for block 1
  1157  		block5 := unittest.BlockWithParentFixture(block3.Header)
  1158  		block5.SetPayload(flow.Payload{
  1159  			Seals: []*flow.Seal{seal1},
  1160  		})
  1161  		err = state.Extend(context.Background(), block5)
  1162  		require.NoError(t, err)
  1163  
  1164  		// block 6 builds on block 4, contains seal for block 2
  1165  		block6 := unittest.BlockWithParentFixture(block4.Header)
  1166  		block6.SetPayload(flow.Payload{
  1167  			Seals: []*flow.Seal{seal2},
  1168  		})
  1169  		err = state.Extend(context.Background(), block6)
  1170  		require.NoError(t, err)
  1171  
  1172  		// block 7 builds on block 5, contains QC for block 7
  1173  		block7 := unittest.BlockWithParentFixture(block5.Header)
  1174  		err = state.Extend(context.Background(), block7)
  1175  		require.NoError(t, err)
  1176  
  1177  		// block 8 builds on block 6, contains QC for block 6
  1178  		block8 := unittest.BlockWithParentFixture(block6.Header)
  1179  		err = state.Extend(context.Background(), block8)
  1180  		require.NoError(t, err)
  1181  
  1182  		// should be able query each epoch from the appropriate reference block
  1183  		setup1FinalView, err := state.AtBlockID(block7.ID()).Epochs().Next().FinalView()
  1184  		assert.NoError(t, err)
  1185  		require.Equal(t, nextEpochSetup1.FinalView, setup1FinalView)
  1186  
  1187  		setup2FinalView, err := state.AtBlockID(block8.ID()).Epochs().Next().FinalView()
  1188  		assert.NoError(t, err)
  1189  		require.Equal(t, nextEpochSetup2.FinalView, setup2FinalView)
  1190  	})
  1191  }
  1192  
  1193  // we should be able to have conflicting forks with two DUPLICATE instances of
  1194  // the same service event for the same epoch
  1195  //
  1196  //	        /--B1<--B3(R1)<--B5(S1)<--B7
  1197  //	ROOT <--+
  1198  //	        \--B2<--B4(R2)<--B6(S2)<--B8
  1199  func TestExtendDuplicateEpochEvents(t *testing.T) {
  1200  	rootSnapshot := unittest.RootSnapshotFixture(participants)
  1201  	util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
  1202  
  1203  		head, err := rootSnapshot.Head()
  1204  		require.NoError(t, err)
  1205  		result, _, err := rootSnapshot.SealedResult()
  1206  		require.NoError(t, err)
  1207  
  1208  		// add two conflicting blocks for each service event to reference
  1209  		block1 := unittest.BlockWithParentFixture(head)
  1210  		block1.SetPayload(flow.EmptyPayload())
  1211  		err = state.Extend(context.Background(), block1)
  1212  		require.NoError(t, err)
  1213  
  1214  		block2 := unittest.BlockWithParentFixture(head)
  1215  		block2.SetPayload(flow.EmptyPayload())
  1216  		err = state.Extend(context.Background(), block2)
  1217  		require.NoError(t, err)
  1218  
  1219  		rootSetup := result.ServiceEvents[0].Event.(*flow.EpochSetup)
  1220  
  1221  		// create an epoch setup event to insert to BOTH forks
  1222  		nextEpochSetup := unittest.EpochSetupFixture(
  1223  			unittest.WithParticipants(rootSetup.Participants),
  1224  			unittest.SetupWithCounter(rootSetup.Counter+1),
  1225  			unittest.WithFinalView(rootSetup.FinalView+1000),
  1226  			unittest.WithFirstView(rootSetup.FinalView+1),
  1227  		)
  1228  
  1229  		// add blocks containing receipts for block1 and block2 (necessary for sealing)
  1230  		// block 1 receipt contains nextEpochSetup1
  1231  		block1Receipt := unittest.ReceiptForBlockFixture(block1)
  1232  		block1Receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{nextEpochSetup.ServiceEvent()}
  1233  
  1234  		// add block 1 receipt to block 3 payload
  1235  		block3 := unittest.BlockWithParentFixture(block1.Header)
  1236  		block3.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(block1Receipt)))
  1237  		err = state.Extend(context.Background(), block3)
  1238  		require.NoError(t, err)
  1239  
  1240  		// block 2 receipt contains nextEpochSetup2
  1241  		block2Receipt := unittest.ReceiptForBlockFixture(block2)
  1242  		block2Receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{nextEpochSetup.ServiceEvent()}
  1243  
  1244  		// add block 2 receipt to block 4 payload
  1245  		block4 := unittest.BlockWithParentFixture(block2.Header)
  1246  		block4.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(block2Receipt)))
  1247  		err = state.Extend(context.Background(), block4)
  1248  		require.NoError(t, err)
  1249  
  1250  		// seal for block 1
  1251  		seal1 := unittest.Seal.Fixture(unittest.Seal.WithResult(&block1Receipt.ExecutionResult))
  1252  
  1253  		// seal for block 2
  1254  		seal2 := unittest.Seal.Fixture(unittest.Seal.WithResult(&block2Receipt.ExecutionResult))
  1255  
  1256  		// block 5 builds on block 3, contains seal for block 1
  1257  		block5 := unittest.BlockWithParentFixture(block3.Header)
  1258  		block5.SetPayload(flow.Payload{
  1259  			Seals: []*flow.Seal{seal1},
  1260  		})
  1261  		err = state.Extend(context.Background(), block5)
  1262  		require.NoError(t, err)
  1263  
  1264  		// block 6 builds on block 4, contains seal for block 2
  1265  		block6 := unittest.BlockWithParentFixture(block4.Header)
  1266  		block6.SetPayload(flow.Payload{
  1267  			Seals: []*flow.Seal{seal2},
  1268  		})
  1269  		err = state.Extend(context.Background(), block6)
  1270  		require.NoError(t, err)
  1271  
  1272  		// block 7 builds on block 5, contains QC for block 7
  1273  		block7 := unittest.BlockWithParentFixture(block5.Header)
  1274  		err = state.Extend(context.Background(), block7)
  1275  		require.NoError(t, err)
  1276  
  1277  		// block 8 builds on block 6, contains QC for block 6
  1278  		// at this point we are inserting the duplicate EpochSetup, should not error
  1279  		block8 := unittest.BlockWithParentFixture(block6.Header)
  1280  		err = state.Extend(context.Background(), block8)
  1281  		require.NoError(t, err)
  1282  
  1283  		// should be able query each epoch from the appropriate reference block
  1284  		finalView, err := state.AtBlockID(block7.ID()).Epochs().Next().FinalView()
  1285  		assert.NoError(t, err)
  1286  		require.Equal(t, nextEpochSetup.FinalView, finalView)
  1287  
  1288  		finalView, err = state.AtBlockID(block8.ID()).Epochs().Next().FinalView()
  1289  		assert.NoError(t, err)
  1290  		require.Equal(t, nextEpochSetup.FinalView, finalView)
  1291  	})
  1292  }
  1293  
  1294  // TestExtendEpochSetupInvalid tests that incorporating an invalid EpochSetup
  1295  // service event should trigger epoch fallback when the fork is finalized.
  1296  func TestExtendEpochSetupInvalid(t *testing.T) {
  1297  	rootSnapshot := unittest.RootSnapshotFixture(participants)
  1298  
  1299  	// setupState initializes the protocol state for a test case
  1300  	// * creates and finalizes a new block for the first seal to reference
  1301  	// * creates a factory method for test cases to generated valid EpochSetup events
  1302  	setupState := func(t *testing.T, db *badger.DB, state *protocol.ParticipantState) (
  1303  		*flow.Block,
  1304  		func(...func(*flow.EpochSetup)) (*flow.EpochSetup, *flow.ExecutionReceipt, *flow.Seal),
  1305  	) {
  1306  
  1307  		head, err := rootSnapshot.Head()
  1308  		require.NoError(t, err)
  1309  		result, _, err := rootSnapshot.SealedResult()
  1310  		require.NoError(t, err)
  1311  
  1312  		// add a block for the first seal to reference
  1313  		block1 := unittest.BlockWithParentFixture(head)
  1314  		block1.SetPayload(flow.EmptyPayload())
  1315  		unittest.InsertAndFinalize(t, state, block1)
  1316  
  1317  		epoch1Setup := result.ServiceEvents[0].Event.(*flow.EpochSetup)
  1318  
  1319  		// add a participant for the next epoch
  1320  		epoch2NewParticipant := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification))
  1321  		epoch2Participants := append(participants, epoch2NewParticipant).Sort(flow.Canonical)
  1322  
  1323  		// this function will return a VALID setup event and seal, we will modify
  1324  		// in different ways in each test case
  1325  		createSetupEvent := func(opts ...func(*flow.EpochSetup)) (*flow.EpochSetup, *flow.ExecutionReceipt, *flow.Seal) {
  1326  			setup := unittest.EpochSetupFixture(
  1327  				unittest.WithParticipants(epoch2Participants),
  1328  				unittest.SetupWithCounter(epoch1Setup.Counter+1),
  1329  				unittest.WithFinalView(epoch1Setup.FinalView+1000),
  1330  				unittest.WithFirstView(epoch1Setup.FinalView+1),
  1331  			)
  1332  			for _, apply := range opts {
  1333  				apply(setup)
  1334  			}
  1335  			receipt, seal := unittest.ReceiptAndSealForBlock(block1)
  1336  			receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{setup.ServiceEvent()}
  1337  			seal.ResultID = receipt.ExecutionResult.ID()
  1338  			return setup, receipt, seal
  1339  		}
  1340  
  1341  		return block1, createSetupEvent
  1342  	}
  1343  
  1344  	// expect a setup event with wrong counter to trigger EECC without error
  1345  	t.Run("wrong counter (EECC)", func(t *testing.T) {
  1346  		util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
  1347  			block1, createSetup := setupState(t, db, state)
  1348  
  1349  			_, receipt, seal := createSetup(func(setup *flow.EpochSetup) {
  1350  				setup.Counter = rand.Uint64()
  1351  			})
  1352  
  1353  			receiptBlock, sealingBlock := unittest.SealBlock(t, state, block1, receipt, seal)
  1354  			err := state.Finalize(context.Background(), receiptBlock.ID())
  1355  			require.NoError(t, err)
  1356  			// epoch fallback not triggered before finalization
  1357  			assertEpochEmergencyFallbackTriggered(t, state, false)
  1358  			err = state.Finalize(context.Background(), sealingBlock.ID())
  1359  			require.NoError(t, err)
  1360  			// epoch fallback triggered after finalization
  1361  			assertEpochEmergencyFallbackTriggered(t, state, true)
  1362  		})
  1363  	})
  1364  
  1365  	// expect a setup event with wrong final view to trigger EECC without error
  1366  	t.Run("invalid final view (EECC)", func(t *testing.T) {
  1367  		util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
  1368  			block1, createSetup := setupState(t, db, state)
  1369  
  1370  			_, receipt, seal := createSetup(func(setup *flow.EpochSetup) {
  1371  				setup.FinalView = block1.Header.View
  1372  			})
  1373  
  1374  			receiptBlock, sealingBlock := unittest.SealBlock(t, state, block1, receipt, seal)
  1375  			err := state.Finalize(context.Background(), receiptBlock.ID())
  1376  			require.NoError(t, err)
  1377  			// epoch fallback not triggered before finalization
  1378  			assertEpochEmergencyFallbackTriggered(t, state, false)
  1379  			err = state.Finalize(context.Background(), sealingBlock.ID())
  1380  			require.NoError(t, err)
  1381  			// epoch fallback triggered after finalization
  1382  			assertEpochEmergencyFallbackTriggered(t, state, true)
  1383  		})
  1384  	})
  1385  
  1386  	// expect a setup event with empty seed to trigger EECC without error
  1387  	t.Run("empty seed (EECC)", func(t *testing.T) {
  1388  		util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
  1389  			block1, createSetup := setupState(t, db, state)
  1390  
  1391  			_, receipt, seal := createSetup(func(setup *flow.EpochSetup) {
  1392  				setup.RandomSource = nil
  1393  			})
  1394  
  1395  			receiptBlock, sealingBlock := unittest.SealBlock(t, state, block1, receipt, seal)
  1396  			err := state.Finalize(context.Background(), receiptBlock.ID())
  1397  			require.NoError(t, err)
  1398  			// epoch fallback not triggered before finalization
  1399  			assertEpochEmergencyFallbackTriggered(t, state, false)
  1400  			err = state.Finalize(context.Background(), sealingBlock.ID())
  1401  			require.NoError(t, err)
  1402  			// epoch fallback triggered after finalization
  1403  			assertEpochEmergencyFallbackTriggered(t, state, true)
  1404  		})
  1405  	})
  1406  }
  1407  
  1408  // TestExtendEpochCommitInvalid tests that incorporating an invalid EpochCommit
  1409  // service event should trigger epoch fallback when the fork is finalized.
  1410  func TestExtendEpochCommitInvalid(t *testing.T) {
  1411  	rootSnapshot := unittest.RootSnapshotFixture(participants)
  1412  
  1413  	// setupState initializes the protocol state for a test case
  1414  	// * creates and finalizes a new block for the first seal to reference
  1415  	// * creates a factory method for test cases to generated valid EpochSetup events
  1416  	// * creates a factory method for test cases to generated valid EpochCommit events
  1417  	setupState := func(t *testing.T, state *protocol.ParticipantState) (
  1418  		*flow.Block,
  1419  		func(*flow.Block) (*flow.EpochSetup, *flow.ExecutionReceipt, *flow.Seal),
  1420  		func(*flow.Block, ...func(*flow.EpochCommit)) (*flow.EpochCommit, *flow.ExecutionReceipt, *flow.Seal),
  1421  	) {
  1422  		head, err := rootSnapshot.Head()
  1423  		require.NoError(t, err)
  1424  		result, _, err := rootSnapshot.SealedResult()
  1425  		require.NoError(t, err)
  1426  
  1427  		// add a block for the first seal to reference
  1428  		block1 := unittest.BlockWithParentFixture(head)
  1429  		block1.SetPayload(flow.EmptyPayload())
  1430  		unittest.InsertAndFinalize(t, state, block1)
  1431  
  1432  		epoch1Setup := result.ServiceEvents[0].Event.(*flow.EpochSetup)
  1433  
  1434  		// swap consensus node for a new one for epoch 2
  1435  		epoch2NewParticipant := unittest.IdentityFixture(unittest.WithRole(flow.RoleConsensus))
  1436  		epoch2Participants := append(
  1437  			participants.Filter(filter.Not(filter.HasRole(flow.RoleConsensus))),
  1438  			epoch2NewParticipant,
  1439  		).Sort(flow.Canonical)
  1440  
  1441  		// factory method to create a valid EpochSetup method w.r.t. the generated state
  1442  		createSetup := func(block *flow.Block) (*flow.EpochSetup, *flow.ExecutionReceipt, *flow.Seal) {
  1443  			setup := unittest.EpochSetupFixture(
  1444  				unittest.WithParticipants(epoch2Participants),
  1445  				unittest.SetupWithCounter(epoch1Setup.Counter+1),
  1446  				unittest.WithFinalView(epoch1Setup.FinalView+1000),
  1447  				unittest.WithFirstView(epoch1Setup.FinalView+1),
  1448  			)
  1449  
  1450  			receipt, seal := unittest.ReceiptAndSealForBlock(block)
  1451  			receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{setup.ServiceEvent()}
  1452  			seal.ResultID = receipt.ExecutionResult.ID()
  1453  			return setup, receipt, seal
  1454  		}
  1455  
  1456  		// factory method to create a valid EpochCommit method w.r.t. the generated state
  1457  		createCommit := func(block *flow.Block, opts ...func(*flow.EpochCommit)) (*flow.EpochCommit, *flow.ExecutionReceipt, *flow.Seal) {
  1458  			commit := unittest.EpochCommitFixture(
  1459  				unittest.CommitWithCounter(epoch1Setup.Counter+1),
  1460  				unittest.WithDKGFromParticipants(epoch2Participants),
  1461  			)
  1462  			for _, apply := range opts {
  1463  				apply(commit)
  1464  			}
  1465  			receipt, seal := unittest.ReceiptAndSealForBlock(block)
  1466  			receipt.ExecutionResult.ServiceEvents = []flow.ServiceEvent{commit.ServiceEvent()}
  1467  			seal.ResultID = receipt.ExecutionResult.ID()
  1468  			return commit, receipt, seal
  1469  		}
  1470  
  1471  		return block1, createSetup, createCommit
  1472  	}
  1473  
  1474  	t.Run("without setup (EECC)", func(t *testing.T) {
  1475  		util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
  1476  			block1, _, createCommit := setupState(t, state)
  1477  
  1478  			_, receipt, seal := createCommit(block1)
  1479  
  1480  			receiptBlock, sealingBlock := unittest.SealBlock(t, state, block1, receipt, seal)
  1481  			err := state.Finalize(context.Background(), receiptBlock.ID())
  1482  			require.NoError(t, err)
  1483  			// epoch fallback not triggered before finalization
  1484  			assertEpochEmergencyFallbackTriggered(t, state, false)
  1485  			err = state.Finalize(context.Background(), sealingBlock.ID())
  1486  			require.NoError(t, err)
  1487  			// epoch fallback triggered after finalization
  1488  			assertEpochEmergencyFallbackTriggered(t, state, true)
  1489  		})
  1490  	})
  1491  
  1492  	// expect a commit event with wrong counter to trigger EECC without error
  1493  	t.Run("inconsistent counter (EECC)", func(t *testing.T) {
  1494  		util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
  1495  			block1, createSetup, createCommit := setupState(t, state)
  1496  
  1497  			// seal block 1, in which EpochSetup was emitted
  1498  			epoch2Setup, setupReceipt, setupSeal := createSetup(block1)
  1499  			epochSetupReceiptBlock, epochSetupSealingBlock := unittest.SealBlock(t, state, block1, setupReceipt, setupSeal)
  1500  			err := state.Finalize(context.Background(), epochSetupReceiptBlock.ID())
  1501  			require.NoError(t, err)
  1502  			err = state.Finalize(context.Background(), epochSetupSealingBlock.ID())
  1503  			require.NoError(t, err)
  1504  
  1505  			// insert a block with a QC for block 2
  1506  			block3 := unittest.BlockWithParentFixture(epochSetupSealingBlock)
  1507  			unittest.InsertAndFinalize(t, state, block3)
  1508  
  1509  			_, receipt, seal := createCommit(block3, func(commit *flow.EpochCommit) {
  1510  				commit.Counter = epoch2Setup.Counter + 1
  1511  			})
  1512  
  1513  			receiptBlock, sealingBlock := unittest.SealBlock(t, state, block3, receipt, seal)
  1514  			err = state.Finalize(context.Background(), receiptBlock.ID())
  1515  			require.NoError(t, err)
  1516  			// epoch fallback not triggered before finalization
  1517  			assertEpochEmergencyFallbackTriggered(t, state, false)
  1518  			err = state.Finalize(context.Background(), sealingBlock.ID())
  1519  			require.NoError(t, err)
  1520  			// epoch fallback triggered after finalization
  1521  			assertEpochEmergencyFallbackTriggered(t, state, true)
  1522  		})
  1523  	})
  1524  
  1525  	// expect a commit event with wrong cluster QCs to trigger EECC without error
  1526  	t.Run("inconsistent cluster QCs (EECC)", func(t *testing.T) {
  1527  		util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
  1528  			block1, createSetup, createCommit := setupState(t, state)
  1529  
  1530  			// seal block 1, in which EpochSetup was emitted
  1531  			_, setupReceipt, setupSeal := createSetup(block1)
  1532  			epochSetupReceiptBlock, epochSetupSealingBlock := unittest.SealBlock(t, state, block1, setupReceipt, setupSeal)
  1533  			err := state.Finalize(context.Background(), epochSetupReceiptBlock.ID())
  1534  			require.NoError(t, err)
  1535  			err = state.Finalize(context.Background(), epochSetupSealingBlock.ID())
  1536  			require.NoError(t, err)
  1537  
  1538  			// insert a block with a QC for block 2
  1539  			block3 := unittest.BlockWithParentFixture(epochSetupSealingBlock)
  1540  			unittest.InsertAndFinalize(t, state, block3)
  1541  
  1542  			_, receipt, seal := createCommit(block3, func(commit *flow.EpochCommit) {
  1543  				commit.ClusterQCs = append(commit.ClusterQCs, flow.ClusterQCVoteDataFromQC(unittest.QuorumCertificateWithSignerIDsFixture()))
  1544  			})
  1545  
  1546  			receiptBlock, sealingBlock := unittest.SealBlock(t, state, block3, receipt, seal)
  1547  			err = state.Finalize(context.Background(), receiptBlock.ID())
  1548  			require.NoError(t, err)
  1549  			// epoch fallback not triggered before finalization
  1550  			assertEpochEmergencyFallbackTriggered(t, state, false)
  1551  			err = state.Finalize(context.Background(), sealingBlock.ID())
  1552  			require.NoError(t, err)
  1553  			// epoch fallback triggered after finalization
  1554  			assertEpochEmergencyFallbackTriggered(t, state, true)
  1555  		})
  1556  	})
  1557  
  1558  	// expect a commit event with wrong dkg participants to trigger EECC without error
  1559  	t.Run("inconsistent DKG participants (EECC)", func(t *testing.T) {
  1560  		util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
  1561  			block1, createSetup, createCommit := setupState(t, state)
  1562  
  1563  			// seal block 1, in which EpochSetup was emitted
  1564  			_, setupReceipt, setupSeal := createSetup(block1)
  1565  			epochSetupReceiptBlock, epochSetupSealingBlock := unittest.SealBlock(t, state, block1, setupReceipt, setupSeal)
  1566  			err := state.Finalize(context.Background(), epochSetupReceiptBlock.ID())
  1567  			require.NoError(t, err)
  1568  			err = state.Finalize(context.Background(), epochSetupSealingBlock.ID())
  1569  			require.NoError(t, err)
  1570  
  1571  			// insert a block with a QC for block 2
  1572  			block3 := unittest.BlockWithParentFixture(epochSetupSealingBlock)
  1573  			unittest.InsertAndFinalize(t, state, block3)
  1574  
  1575  			_, receipt, seal := createCommit(block3, func(commit *flow.EpochCommit) {
  1576  				// add an extra dkg key
  1577  				commit.DKGParticipantKeys = append(commit.DKGParticipantKeys, unittest.KeyFixture(crypto.BLSBLS12381).PublicKey())
  1578  			})
  1579  
  1580  			receiptBlock, sealingBlock := unittest.SealBlock(t, state, block3, receipt, seal)
  1581  			err = state.Finalize(context.Background(), receiptBlock.ID())
  1582  			require.NoError(t, err)
  1583  			// epoch fallback not triggered before finalization
  1584  			assertEpochEmergencyFallbackTriggered(t, state, false)
  1585  			err = state.Finalize(context.Background(), sealingBlock.ID())
  1586  			require.NoError(t, err)
  1587  			// epoch fallback triggered after finalization
  1588  			assertEpochEmergencyFallbackTriggered(t, state, true)
  1589  		})
  1590  	})
  1591  }
  1592  
  1593  // if we reach the first block of the next epoch before both setup and commit
  1594  // service events are finalized, the chain should halt
  1595  //
  1596  // ROOT <- B1 <- B2(R1) <- B3(S1) <- B4
  1597  func TestExtendEpochTransitionWithoutCommit(t *testing.T) {
  1598  
  1599  	// skipping because this case will now result in emergency epoch continuation kicking in
  1600  	unittest.SkipUnless(t, unittest.TEST_TODO, "disabled as the current implementation uses a temporary fallback measure in this case (triggers EECC), rather than returning an error")
  1601  
  1602  	rootSnapshot := unittest.RootSnapshotFixture(participants)
  1603  	util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
  1604  		head, err := rootSnapshot.Head()
  1605  		require.NoError(t, err)
  1606  		result, _, err := rootSnapshot.SealedResult()
  1607  		require.NoError(t, err)
  1608  
  1609  		// add a block for the first seal to reference
  1610  		block1 := unittest.BlockWithParentFixture(head)
  1611  		block1.SetPayload(flow.EmptyPayload())
  1612  		err = state.Extend(context.Background(), block1)
  1613  		require.NoError(t, err)
  1614  		err = state.Finalize(context.Background(), block1.ID())
  1615  		require.NoError(t, err)
  1616  
  1617  		epoch1Setup := result.ServiceEvents[0].Event.(*flow.EpochSetup)
  1618  		epoch1FinalView := epoch1Setup.FinalView
  1619  
  1620  		// add a participant for the next epoch
  1621  		epoch2NewParticipant := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification))
  1622  		epoch2Participants := append(participants, epoch2NewParticipant).Sort(flow.Canonical)
  1623  
  1624  		// create the epoch setup event for the second epoch
  1625  		epoch2Setup := unittest.EpochSetupFixture(
  1626  			unittest.WithParticipants(epoch2Participants),
  1627  			unittest.SetupWithCounter(epoch1Setup.Counter+1),
  1628  			unittest.WithFinalView(epoch1FinalView+1000),
  1629  			unittest.WithFirstView(epoch1FinalView+1),
  1630  		)
  1631  
  1632  		receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1)
  1633  		receipt1.ExecutionResult.ServiceEvents = []flow.ServiceEvent{epoch2Setup.ServiceEvent()}
  1634  
  1635  		// add a block containing a receipt for block 1
  1636  		block2 := unittest.BlockWithParentFixture(block1.Header)
  1637  		block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1)))
  1638  		err = state.Extend(context.Background(), block2)
  1639  		require.NoError(t, err)
  1640  		err = state.Finalize(context.Background(), block2.ID())
  1641  		require.NoError(t, err)
  1642  
  1643  		// block 3 seals block 1
  1644  		block3 := unittest.BlockWithParentFixture(block2.Header)
  1645  		block3.SetPayload(flow.Payload{
  1646  			Seals: []*flow.Seal{seal1},
  1647  		})
  1648  		err = state.Extend(context.Background(), block3)
  1649  		require.NoError(t, err)
  1650  
  1651  		// block 4 will be the first block for epoch 2
  1652  		block4 := unittest.BlockWithParentFixture(block3.Header)
  1653  		block4.Header.View = epoch1Setup.FinalView + 1
  1654  
  1655  		err = state.Extend(context.Background(), block4)
  1656  		require.Error(t, err)
  1657  	})
  1658  }
  1659  
  1660  // TestEmergencyEpochFallback tests that epoch emergency fallback is triggered
  1661  // when an epoch fails to be committed before the epoch commitment deadline,
  1662  // or when an invalid service event (indicating service account smart contract bug)
  1663  // is sealed.
  1664  func TestEmergencyEpochFallback(t *testing.T) {
  1665  
  1666  	// if we finalize the first block past the epoch commitment deadline while
  1667  	// in the EpochStaking phase, EECC should be triggered
  1668  	//
  1669  	//       Epoch Commitment Deadline
  1670  	//       |     Epoch Boundary
  1671  	//       |     |
  1672  	//       v     v
  1673  	// ROOT <- B1 <- B2
  1674  	t.Run("passed epoch commitment deadline in EpochStaking phase - should trigger EECC", func(t *testing.T) {
  1675  
  1676  		rootSnapshot := unittest.RootSnapshotFixture(participants)
  1677  		metricsMock := mockmodule.NewComplianceMetrics(t)
  1678  		mockMetricsForRootSnapshot(metricsMock, rootSnapshot)
  1679  		protoEventsMock := mockprotocol.NewConsumer(t)
  1680  		protoEventsMock.On("BlockFinalized", mock.Anything)
  1681  		protoEventsMock.On("BlockProcessable", mock.Anything, mock.Anything)
  1682  
  1683  		util.RunWithFullProtocolStateAndMetricsAndConsumer(t, rootSnapshot, metricsMock, protoEventsMock, func(db *badger.DB, state *protocol.ParticipantState) {
  1684  			head, err := rootSnapshot.Head()
  1685  			require.NoError(t, err)
  1686  			result, _, err := rootSnapshot.SealedResult()
  1687  			require.NoError(t, err)
  1688  			safetyThreshold, err := rootSnapshot.Params().EpochCommitSafetyThreshold()
  1689  			require.NoError(t, err)
  1690  
  1691  			epoch1Setup := result.ServiceEvents[0].Event.(*flow.EpochSetup)
  1692  			epoch1FinalView := epoch1Setup.FinalView
  1693  			epoch1CommitmentDeadline := epoch1FinalView - safetyThreshold
  1694  
  1695  			// finalizing block 1 should trigger EECC
  1696  			metricsMock.On("EpochEmergencyFallbackTriggered").Once()
  1697  			protoEventsMock.On("EpochEmergencyFallbackTriggered").Once()
  1698  
  1699  			// we begin the epoch in the EpochStaking phase and
  1700  			// block 1 will be the first block on or past the epoch commitment deadline
  1701  			block1 := unittest.BlockWithParentFixture(head)
  1702  			block1.Header.View = epoch1CommitmentDeadline + rand.Uint64()%2
  1703  			err = state.Extend(context.Background(), block1)
  1704  			require.NoError(t, err)
  1705  			assertEpochEmergencyFallbackTriggered(t, state, false) // not triggered before finalization
  1706  			err = state.Finalize(context.Background(), block1.ID())
  1707  			require.NoError(t, err)
  1708  			assertEpochEmergencyFallbackTriggered(t, state, true) // triggered after finalization
  1709  
  1710  			// block 2 will be the first block past the first epoch boundary
  1711  			block2 := unittest.BlockWithParentFixture(block1.Header)
  1712  			block2.Header.View = epoch1FinalView + 1
  1713  			err = state.Extend(context.Background(), block2)
  1714  			require.NoError(t, err)
  1715  			err = state.Finalize(context.Background(), block2.ID())
  1716  			require.NoError(t, err)
  1717  
  1718  			// since EECC has been triggered, epoch transition metrics should not be updated
  1719  			metricsMock.AssertNotCalled(t, "EpochTransition", mock.Anything, mock.Anything)
  1720  			metricsMock.AssertNotCalled(t, "CurrentEpochCounter", epoch1Setup.Counter+1)
  1721  		})
  1722  	})
  1723  
  1724  	// if we finalize the first block past the epoch commitment deadline while
  1725  	// in the EpochSetup phase, EECC should be triggered
  1726  	//
  1727  	//                       Epoch Commitment Deadline
  1728  	//                       |         Epoch Boundary
  1729  	//                       |         |
  1730  	//                       v         v
  1731  	// ROOT <- B1 <- B2(R1) <- B3(S1) <- B4
  1732  	t.Run("passed epoch commitment deadline in EpochSetup phase - should trigger EECC", func(t *testing.T) {
  1733  
  1734  		rootSnapshot := unittest.RootSnapshotFixture(participants)
  1735  		metricsMock := mockmodule.NewComplianceMetrics(t)
  1736  		mockMetricsForRootSnapshot(metricsMock, rootSnapshot)
  1737  		protoEventsMock := mockprotocol.NewConsumer(t)
  1738  		protoEventsMock.On("BlockFinalized", mock.Anything)
  1739  		protoEventsMock.On("BlockProcessable", mock.Anything, mock.Anything)
  1740  
  1741  		util.RunWithFullProtocolStateAndMetricsAndConsumer(t, rootSnapshot, metricsMock, protoEventsMock, func(db *badger.DB, state *protocol.ParticipantState) {
  1742  			head, err := rootSnapshot.Head()
  1743  			require.NoError(t, err)
  1744  			result, _, err := rootSnapshot.SealedResult()
  1745  			require.NoError(t, err)
  1746  			safetyThreshold, err := rootSnapshot.Params().EpochCommitSafetyThreshold()
  1747  			require.NoError(t, err)
  1748  
  1749  			// add a block for the first seal to reference
  1750  			block1 := unittest.BlockWithParentFixture(head)
  1751  			block1.SetPayload(flow.EmptyPayload())
  1752  			err = state.Extend(context.Background(), block1)
  1753  			require.NoError(t, err)
  1754  			err = state.Finalize(context.Background(), block1.ID())
  1755  			require.NoError(t, err)
  1756  
  1757  			epoch1Setup := result.ServiceEvents[0].Event.(*flow.EpochSetup)
  1758  			epoch1FinalView := epoch1Setup.FinalView
  1759  			epoch1CommitmentDeadline := epoch1FinalView - safetyThreshold
  1760  
  1761  			// add a participant for the next epoch
  1762  			epoch2NewParticipant := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification))
  1763  			epoch2Participants := append(participants, epoch2NewParticipant).Sort(flow.Canonical)
  1764  
  1765  			// create the epoch setup event for the second epoch
  1766  			epoch2Setup := unittest.EpochSetupFixture(
  1767  				unittest.WithParticipants(epoch2Participants),
  1768  				unittest.SetupWithCounter(epoch1Setup.Counter+1),
  1769  				unittest.WithFinalView(epoch1FinalView+1000),
  1770  				unittest.WithFirstView(epoch1FinalView+1),
  1771  			)
  1772  
  1773  			receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1)
  1774  			receipt1.ExecutionResult.ServiceEvents = []flow.ServiceEvent{epoch2Setup.ServiceEvent()}
  1775  			seal1.ResultID = receipt1.ExecutionResult.ID()
  1776  
  1777  			// add a block containing a receipt for block 1
  1778  			block2 := unittest.BlockWithParentFixture(block1.Header)
  1779  			block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1)))
  1780  			err = state.Extend(context.Background(), block2)
  1781  			require.NoError(t, err)
  1782  			err = state.Finalize(context.Background(), block2.ID())
  1783  			require.NoError(t, err)
  1784  
  1785  			// block 3 seals block 1 and will be the first block on or past the epoch commitment deadline
  1786  			block3 := unittest.BlockWithParentFixture(block2.Header)
  1787  			block3.Header.View = epoch1CommitmentDeadline + rand.Uint64()%2
  1788  			block3.SetPayload(flow.Payload{
  1789  				Seals: []*flow.Seal{seal1},
  1790  			})
  1791  			err = state.Extend(context.Background(), block3)
  1792  			require.NoError(t, err)
  1793  
  1794  			// finalizing block 3 should trigger EECC
  1795  			metricsMock.On("EpochEmergencyFallbackTriggered").Once()
  1796  			protoEventsMock.On("EpochEmergencyFallbackTriggered").Once()
  1797  
  1798  			assertEpochEmergencyFallbackTriggered(t, state, false) // not triggered before finalization
  1799  			err = state.Finalize(context.Background(), block3.ID())
  1800  			require.NoError(t, err)
  1801  			assertEpochEmergencyFallbackTriggered(t, state, true) // triggered after finalization
  1802  
  1803  			// block 4 will be the first block past the first epoch boundary
  1804  			block4 := unittest.BlockWithParentFixture(block3.Header)
  1805  			block4.Header.View = epoch1FinalView + 1
  1806  			err = state.Extend(context.Background(), block4)
  1807  			require.NoError(t, err)
  1808  			err = state.Finalize(context.Background(), block4.ID())
  1809  			require.NoError(t, err)
  1810  
  1811  			// since EECC has been triggered, epoch transition metrics should not be updated
  1812  			metricsMock.AssertNotCalled(t, "EpochTransition", epoch2Setup.Counter, mock.Anything)
  1813  			metricsMock.AssertNotCalled(t, "CurrentEpochCounter", epoch2Setup.Counter)
  1814  		})
  1815  	})
  1816  
  1817  	// if an invalid epoch service event is incorporated, we should:
  1818  	//   - not apply the phase transition corresponding to the invalid service event
  1819  	//   - immediately trigger EECC
  1820  	//
  1821  	//                            Epoch Boundary
  1822  	//                                 |
  1823  	//                                 v
  1824  	// ROOT <- B1 <- B2(R1) <- B3(S1) <- B4
  1825  	t.Run("epoch transition with invalid service event - should trigger EECC", func(t *testing.T) {
  1826  
  1827  		rootSnapshot := unittest.RootSnapshotFixture(participants)
  1828  		metricsMock := mockmodule.NewComplianceMetrics(t)
  1829  		mockMetricsForRootSnapshot(metricsMock, rootSnapshot)
  1830  		protoEventsMock := mockprotocol.NewConsumer(t)
  1831  		protoEventsMock.On("BlockFinalized", mock.Anything)
  1832  		protoEventsMock.On("BlockProcessable", mock.Anything, mock.Anything)
  1833  
  1834  		util.RunWithFullProtocolStateAndMetricsAndConsumer(t, rootSnapshot, metricsMock, protoEventsMock, func(db *badger.DB, state *protocol.ParticipantState) {
  1835  			head, err := rootSnapshot.Head()
  1836  			require.NoError(t, err)
  1837  			result, _, err := rootSnapshot.SealedResult()
  1838  			require.NoError(t, err)
  1839  
  1840  			// add a block for the first seal to reference
  1841  			block1 := unittest.BlockWithParentFixture(head)
  1842  			block1.SetPayload(flow.EmptyPayload())
  1843  			err = state.Extend(context.Background(), block1)
  1844  			require.NoError(t, err)
  1845  			err = state.Finalize(context.Background(), block1.ID())
  1846  			require.NoError(t, err)
  1847  
  1848  			epoch1Setup := result.ServiceEvents[0].Event.(*flow.EpochSetup)
  1849  			epoch1FinalView := epoch1Setup.FinalView
  1850  
  1851  			// add a participant for the next epoch
  1852  			epoch2NewParticipant := unittest.IdentityFixture(unittest.WithRole(flow.RoleVerification))
  1853  			epoch2Participants := append(participants, epoch2NewParticipant).Sort(flow.Canonical)
  1854  
  1855  			// create the epoch setup event for the second epoch
  1856  			// this event is invalid because it used a non-contiguous first view
  1857  			epoch2Setup := unittest.EpochSetupFixture(
  1858  				unittest.WithParticipants(epoch2Participants),
  1859  				unittest.SetupWithCounter(epoch1Setup.Counter+1),
  1860  				unittest.WithFinalView(epoch1FinalView+1000),
  1861  				unittest.WithFirstView(epoch1FinalView+10), // invalid first view
  1862  			)
  1863  
  1864  			receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1)
  1865  			receipt1.ExecutionResult.ServiceEvents = []flow.ServiceEvent{epoch2Setup.ServiceEvent()}
  1866  			seal1.ResultID = receipt1.ExecutionResult.ID()
  1867  
  1868  			// add a block containing a receipt for block 1
  1869  			block2 := unittest.BlockWithParentFixture(block1.Header)
  1870  			block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1)))
  1871  			err = state.Extend(context.Background(), block2)
  1872  			require.NoError(t, err)
  1873  			err = state.Finalize(context.Background(), block2.ID())
  1874  			require.NoError(t, err)
  1875  
  1876  			// block 3 is where the service event state change comes into effect
  1877  			block3 := unittest.BlockWithParentFixture(block2.Header)
  1878  			block3.SetPayload(flow.Payload{
  1879  				Seals: []*flow.Seal{seal1},
  1880  			})
  1881  			err = state.Extend(context.Background(), block3)
  1882  			require.NoError(t, err)
  1883  
  1884  			// incorporating the service event should trigger EECC
  1885  			metricsMock.On("EpochEmergencyFallbackTriggered").Once()
  1886  			protoEventsMock.On("EpochEmergencyFallbackTriggered").Once()
  1887  
  1888  			assertEpochEmergencyFallbackTriggered(t, state, false) // not triggered before finalization
  1889  			err = state.Finalize(context.Background(), block3.ID())
  1890  			require.NoError(t, err)
  1891  			assertEpochEmergencyFallbackTriggered(t, state, true) // triggered after finalization
  1892  
  1893  			// block 5 is the first block past the current epoch boundary
  1894  			block4 := unittest.BlockWithParentFixture(block3.Header)
  1895  			block4.Header.View = epoch1Setup.FinalView + 1
  1896  			err = state.Extend(context.Background(), block4)
  1897  			require.NoError(t, err)
  1898  			err = state.Finalize(context.Background(), block4.ID())
  1899  			require.NoError(t, err)
  1900  
  1901  			// since EECC has been triggered, epoch transition metrics should not be updated
  1902  			metricsMock.AssertNotCalled(t, "EpochTransition", epoch2Setup.Counter, mock.Anything)
  1903  			metricsMock.AssertNotCalled(t, "CurrentEpochCounter", epoch2Setup.Counter)
  1904  		})
  1905  	})
  1906  }
  1907  
  1908  func TestExtendInvalidSealsInBlock(t *testing.T) {
  1909  	unittest.RunWithBadgerDB(t, func(db *badger.DB) {
  1910  		metrics := metrics.NewNoopCollector()
  1911  		tracer := trace.NewNoopTracer()
  1912  		log := zerolog.Nop()
  1913  		all := storeutil.StorageLayer(t, db)
  1914  
  1915  		// create a event consumer to test epoch transition events
  1916  		distributor := events.NewDistributor()
  1917  		consumer := mockprotocol.NewConsumer(t)
  1918  		distributor.AddConsumer(consumer)
  1919  		consumer.On("BlockProcessable", mock.Anything, mock.Anything)
  1920  
  1921  		rootSnapshot := unittest.RootSnapshotFixture(participants)
  1922  
  1923  		state, err := protocol.Bootstrap(
  1924  			metrics,
  1925  			db,
  1926  			all.Headers,
  1927  			all.Seals,
  1928  			all.Results,
  1929  			all.Blocks,
  1930  			all.QuorumCertificates,
  1931  			all.Setups,
  1932  			all.EpochCommits,
  1933  			all.Statuses,
  1934  			all.VersionBeacons,
  1935  			rootSnapshot,
  1936  		)
  1937  		require.NoError(t, err)
  1938  
  1939  		head, err := rootSnapshot.Head()
  1940  		require.NoError(t, err)
  1941  
  1942  		block1 := unittest.BlockWithParentFixture(head)
  1943  		block1.Payload.Guarantees = nil
  1944  		block1.Header.PayloadHash = block1.Payload.Hash()
  1945  
  1946  		block1Receipt := unittest.ReceiptForBlockFixture(block1)
  1947  		block2 := unittest.BlockWithParentFixture(block1.Header)
  1948  		block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(block1Receipt)))
  1949  
  1950  		block1Seal := unittest.Seal.Fixture(unittest.Seal.WithResult(&block1Receipt.ExecutionResult))
  1951  		block3 := unittest.BlockWithParentFixture(block2.Header)
  1952  		block3.SetPayload(flow.Payload{
  1953  			Seals: []*flow.Seal{block1Seal},
  1954  		})
  1955  
  1956  		sealValidator := mockmodule.NewSealValidator(t)
  1957  		sealValidator.On("Validate", mock.Anything).
  1958  			Return(func(candidate *flow.Block) *flow.Seal {
  1959  				if candidate.ID() == block3.ID() {
  1960  					return nil
  1961  				}
  1962  				seal, _ := all.Seals.HighestInFork(candidate.Header.ParentID)
  1963  				return seal
  1964  			}, func(candidate *flow.Block) error {
  1965  				if candidate.ID() == block3.ID() {
  1966  					return engine.NewInvalidInputError("")
  1967  				}
  1968  				_, err := all.Seals.HighestInFork(candidate.Header.ParentID)
  1969  				return err
  1970  			}).
  1971  			Times(3)
  1972  
  1973  		fullState, err := protocol.NewFullConsensusState(
  1974  			log,
  1975  			tracer,
  1976  			consumer,
  1977  			state,
  1978  			all.Index,
  1979  			all.Payloads,
  1980  			util.MockBlockTimer(),
  1981  			util.MockReceiptValidator(),
  1982  			sealValidator,
  1983  		)
  1984  		require.NoError(t, err)
  1985  
  1986  		err = fullState.Extend(context.Background(), block1)
  1987  		require.NoError(t, err)
  1988  		err = fullState.Extend(context.Background(), block2)
  1989  		require.NoError(t, err)
  1990  		err = fullState.Extend(context.Background(), block3)
  1991  		require.Error(t, err)
  1992  		require.True(t, st.IsInvalidExtensionError(err))
  1993  	})
  1994  }
  1995  
  1996  func TestHeaderExtendValid(t *testing.T) {
  1997  	rootSnapshot := unittest.RootSnapshotFixture(participants)
  1998  	util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState) {
  1999  		head, err := rootSnapshot.Head()
  2000  		require.NoError(t, err)
  2001  		_, seal, err := rootSnapshot.SealedResult()
  2002  		require.NoError(t, err)
  2003  
  2004  		extend := unittest.BlockWithParentFixture(head)
  2005  		extend.SetPayload(flow.EmptyPayload())
  2006  
  2007  		err = state.ExtendCertified(context.Background(), extend, unittest.CertifyBlock(extend.Header))
  2008  		require.NoError(t, err)
  2009  
  2010  		finalCommit, err := state.Final().Commit()
  2011  		require.NoError(t, err)
  2012  		require.Equal(t, seal.FinalState, finalCommit)
  2013  	})
  2014  }
  2015  
  2016  func TestHeaderExtendMissingParent(t *testing.T) {
  2017  	rootSnapshot := unittest.RootSnapshotFixture(participants)
  2018  	util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState) {
  2019  		extend := unittest.BlockFixture()
  2020  		extend.Payload.Guarantees = nil
  2021  		extend.Payload.Seals = nil
  2022  		extend.Header.Height = 2
  2023  		extend.Header.View = 2
  2024  		extend.Header.ParentID = unittest.BlockFixture().ID()
  2025  		extend.Header.PayloadHash = extend.Payload.Hash()
  2026  
  2027  		err := state.ExtendCertified(context.Background(), &extend, unittest.CertifyBlock(extend.Header))
  2028  		require.Error(t, err)
  2029  		require.False(t, st.IsInvalidExtensionError(err), err)
  2030  
  2031  		// verify seal not indexed
  2032  		var sealID flow.Identifier
  2033  		err = db.View(operation.LookupLatestSealAtBlock(extend.ID(), &sealID))
  2034  		require.Error(t, err)
  2035  		require.ErrorIs(t, err, stoerr.ErrNotFound)
  2036  	})
  2037  }
  2038  
  2039  func TestHeaderExtendHeightTooSmall(t *testing.T) {
  2040  	rootSnapshot := unittest.RootSnapshotFixture(participants)
  2041  	util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState) {
  2042  		head, err := rootSnapshot.Head()
  2043  		require.NoError(t, err)
  2044  
  2045  		block1 := unittest.BlockWithParentFixture(head)
  2046  
  2047  		// create another block that points to the previous block `extend` as parent
  2048  		// but has _same_ height as parent. This violates the condition that a child's
  2049  		// height must increment the parent's height by one, i.e. it should be rejected
  2050  		// by the follower right away
  2051  		block2 := unittest.BlockWithParentFixture(block1.Header)
  2052  		block2.Header.Height = block1.Header.Height
  2053  
  2054  		err = state.ExtendCertified(context.Background(), block1, block2.Header.QuorumCertificate())
  2055  		require.NoError(t, err)
  2056  
  2057  		err = state.ExtendCertified(context.Background(), block2, unittest.CertifyBlock(block2.Header))
  2058  		require.False(t, st.IsInvalidExtensionError(err))
  2059  
  2060  		// verify seal not indexed
  2061  		var sealID flow.Identifier
  2062  		err = db.View(operation.LookupLatestSealAtBlock(block2.ID(), &sealID))
  2063  		require.ErrorIs(t, err, stoerr.ErrNotFound)
  2064  	})
  2065  }
  2066  
  2067  func TestHeaderExtendHeightTooLarge(t *testing.T) {
  2068  	rootSnapshot := unittest.RootSnapshotFixture(participants)
  2069  	util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState) {
  2070  		head, err := rootSnapshot.Head()
  2071  		require.NoError(t, err)
  2072  
  2073  		block := unittest.BlockWithParentFixture(head)
  2074  		block.SetPayload(flow.EmptyPayload())
  2075  		// set an invalid height
  2076  		block.Header.Height = head.Height + 2
  2077  
  2078  		err = state.ExtendCertified(context.Background(), block, unittest.CertifyBlock(block.Header))
  2079  		require.False(t, st.IsInvalidExtensionError(err))
  2080  	})
  2081  }
  2082  
  2083  // TestExtendBlockProcessable tests that BlockProcessable is called correctly and doesn't produce duplicates of same notifications
  2084  // when extending blocks with and without certifying QCs.
  2085  func TestExtendBlockProcessable(t *testing.T) {
  2086  	rootSnapshot := unittest.RootSnapshotFixture(participants)
  2087  	head, err := rootSnapshot.Head()
  2088  	require.NoError(t, err)
  2089  	consumer := mockprotocol.NewConsumer(t)
  2090  	util.RunWithFullProtocolStateAndConsumer(t, rootSnapshot, consumer, func(db *badger.DB, state *protocol.ParticipantState) {
  2091  		block := unittest.BlockWithParentFixture(head)
  2092  		child := unittest.BlockWithParentFixture(block.Header)
  2093  		grandChild := unittest.BlockWithParentFixture(child.Header)
  2094  
  2095  		// extend block using certifying QC, expect that BlockProcessable will be emitted once
  2096  		consumer.On("BlockProcessable", block.Header, child.Header.QuorumCertificate()).Once()
  2097  		err := state.ExtendCertified(context.Background(), block, child.Header.QuorumCertificate())
  2098  		require.NoError(t, err)
  2099  
  2100  		// extend block without certifying QC, expect that BlockProcessable won't be called
  2101  		err = state.Extend(context.Background(), child)
  2102  		require.NoError(t, err)
  2103  		consumer.AssertNumberOfCalls(t, "BlockProcessable", 1)
  2104  
  2105  		// extend block using certifying QC, expect that BlockProcessable will be emitted twice.
  2106  		// One for parent block and second for current block.
  2107  		grandChildCertifyingQC := unittest.CertifyBlock(grandChild.Header)
  2108  		consumer.On("BlockProcessable", child.Header, grandChild.Header.QuorumCertificate()).Once()
  2109  		consumer.On("BlockProcessable", grandChild.Header, grandChildCertifyingQC).Once()
  2110  		err = state.ExtendCertified(context.Background(), grandChild, grandChildCertifyingQC)
  2111  		require.NoError(t, err)
  2112  	})
  2113  }
  2114  
  2115  // TestFollowerHeaderExtendBlockNotConnected tests adding an orphaned block to the follower state.
  2116  // Specifically, we add 2 blocks, where:
  2117  // first block is added and then finalized;
  2118  // second block is a sibling to the finalized block
  2119  // The Follower should accept this block since tracking of orphan blocks is implemented by another component.
  2120  func TestFollowerHeaderExtendBlockNotConnected(t *testing.T) {
  2121  	rootSnapshot := unittest.RootSnapshotFixture(participants)
  2122  	util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState) {
  2123  		head, err := rootSnapshot.Head()
  2124  		require.NoError(t, err)
  2125  
  2126  		block1 := unittest.BlockWithParentFixture(head)
  2127  		err = state.ExtendCertified(context.Background(), block1, unittest.CertifyBlock(block1.Header))
  2128  		require.NoError(t, err)
  2129  
  2130  		err = state.Finalize(context.Background(), block1.ID())
  2131  		require.NoError(t, err)
  2132  
  2133  		// create a fork at view/height 1 and try to connect it to root
  2134  		block2 := unittest.BlockWithParentFixture(head)
  2135  		err = state.ExtendCertified(context.Background(), block2, unittest.CertifyBlock(block2.Header))
  2136  		require.NoError(t, err)
  2137  
  2138  		// verify seal not indexed
  2139  		var sealID flow.Identifier
  2140  		err = db.View(operation.LookupLatestSealAtBlock(block2.ID(), &sealID))
  2141  		require.NoError(t, err)
  2142  	})
  2143  }
  2144  
  2145  // TestParticipantHeaderExtendBlockNotConnected tests adding an orphaned block to the consensus participant state.
  2146  // Specifically, we add 2 blocks, where:
  2147  // first block is added and then finalized;
  2148  // second block is a sibling to the finalized block
  2149  // The Participant should reject this block as an outdated chain extension
  2150  func TestParticipantHeaderExtendBlockNotConnected(t *testing.T) {
  2151  	rootSnapshot := unittest.RootSnapshotFixture(participants)
  2152  	util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
  2153  		head, err := rootSnapshot.Head()
  2154  		require.NoError(t, err)
  2155  
  2156  		block1 := unittest.BlockWithParentFixture(head)
  2157  		err = state.Extend(context.Background(), block1)
  2158  		require.NoError(t, err)
  2159  
  2160  		err = state.Finalize(context.Background(), block1.ID())
  2161  		require.NoError(t, err)
  2162  
  2163  		// create a fork at view/height 1 and try to connect it to root
  2164  		block2 := unittest.BlockWithParentFixture(head)
  2165  		err = state.Extend(context.Background(), block2)
  2166  		require.True(t, st.IsOutdatedExtensionError(err), err)
  2167  
  2168  		// verify seal not indexed
  2169  		var sealID flow.Identifier
  2170  		err = db.View(operation.LookupLatestSealAtBlock(block2.ID(), &sealID))
  2171  		require.ErrorIs(t, err, stoerr.ErrNotFound)
  2172  	})
  2173  }
  2174  
  2175  func TestHeaderExtendHighestSeal(t *testing.T) {
  2176  	rootSnapshot := unittest.RootSnapshotFixture(participants)
  2177  	head, err := rootSnapshot.Head()
  2178  	require.NoError(t, err)
  2179  	util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState) {
  2180  		// create block2 and block3
  2181  		block2 := unittest.BlockWithParentFixture(head)
  2182  		block2.SetPayload(flow.EmptyPayload())
  2183  
  2184  		block3 := unittest.BlockWithParentFixture(block2.Header)
  2185  		block3.SetPayload(flow.EmptyPayload())
  2186  
  2187  		err := state.ExtendCertified(context.Background(), block2, block3.Header.QuorumCertificate())
  2188  		require.NoError(t, err)
  2189  
  2190  		// create receipts and seals for block2 and block3
  2191  		receipt2, seal2 := unittest.ReceiptAndSealForBlock(block2)
  2192  		receipt3, seal3 := unittest.ReceiptAndSealForBlock(block3)
  2193  
  2194  		// include the seals in block4
  2195  		block4 := unittest.BlockWithParentFixture(block3.Header)
  2196  		// include receipts and results
  2197  		block4.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt3, receipt2)))
  2198  
  2199  		// include the seals in block4
  2200  		block5 := unittest.BlockWithParentFixture(block4.Header)
  2201  		// placing seals in the reversed order to test
  2202  		// Extend will pick the highest sealed block
  2203  		block5.SetPayload(unittest.PayloadFixture(unittest.WithSeals(seal3, seal2)))
  2204  
  2205  		err = state.ExtendCertified(context.Background(), block3, block4.Header.QuorumCertificate())
  2206  		require.NoError(t, err)
  2207  
  2208  		err = state.ExtendCertified(context.Background(), block4, block5.Header.QuorumCertificate())
  2209  		require.NoError(t, err)
  2210  
  2211  		err = state.ExtendCertified(context.Background(), block5, unittest.CertifyBlock(block5.Header))
  2212  		require.NoError(t, err)
  2213  
  2214  		finalCommit, err := state.AtBlockID(block5.ID()).Commit()
  2215  		require.NoError(t, err)
  2216  		require.Equal(t, seal3.FinalState, finalCommit)
  2217  	})
  2218  }
  2219  
  2220  // TestExtendCertifiedInvalidQC checks if ExtendCertified performs a sanity check of certifying QC.
  2221  func TestExtendCertifiedInvalidQC(t *testing.T) {
  2222  	rootSnapshot := unittest.RootSnapshotFixture(participants)
  2223  	head, err := rootSnapshot.Head()
  2224  	require.NoError(t, err)
  2225  	util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
  2226  		// create child block
  2227  		block := unittest.BlockWithParentFixture(head)
  2228  		block.SetPayload(flow.EmptyPayload())
  2229  
  2230  		t.Run("qc-invalid-view", func(t *testing.T) {
  2231  			certifyingQC := unittest.CertifyBlock(block.Header)
  2232  			certifyingQC.View++ // invalidate block view
  2233  			err = state.ExtendCertified(context.Background(), block, certifyingQC)
  2234  			require.Error(t, err)
  2235  			require.False(t, st.IsOutdatedExtensionError(err))
  2236  		})
  2237  		t.Run("qc-invalid-block-id", func(t *testing.T) {
  2238  			certifyingQC := unittest.CertifyBlock(block.Header)
  2239  			certifyingQC.BlockID = unittest.IdentifierFixture() // invalidate blockID
  2240  			err = state.ExtendCertified(context.Background(), block, certifyingQC)
  2241  			require.Error(t, err)
  2242  			require.False(t, st.IsOutdatedExtensionError(err))
  2243  		})
  2244  	})
  2245  }
  2246  
  2247  // TestExtendInvalidGuarantee checks if Extend method will reject invalid blocks that contain
  2248  // guarantees with invalid guarantors
  2249  func TestExtendInvalidGuarantee(t *testing.T) {
  2250  	rootSnapshot := unittest.RootSnapshotFixture(participants)
  2251  	util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
  2252  		// create a valid block
  2253  		head, err := rootSnapshot.Head()
  2254  		require.NoError(t, err)
  2255  
  2256  		cluster, err := unittest.SnapshotClusterByIndex(rootSnapshot, 0)
  2257  		require.NoError(t, err)
  2258  
  2259  		// prepare for a valid guarantor signer indices to be used in the valid block
  2260  		all := cluster.Members().NodeIDs()
  2261  		validSignerIndices, err := signature.EncodeSignersToIndices(all, all)
  2262  		require.NoError(t, err)
  2263  
  2264  		block := unittest.BlockWithParentFixture(head)
  2265  		payload := flow.EmptyPayload()
  2266  		payload.Guarantees = []*flow.CollectionGuarantee{
  2267  			{
  2268  				ChainID:          cluster.ChainID(),
  2269  				ReferenceBlockID: head.ID(),
  2270  				SignerIndices:    validSignerIndices,
  2271  			},
  2272  		}
  2273  
  2274  		// now the valid block has a guarantee in the payload with valid signer indices.
  2275  		block.SetPayload(payload)
  2276  
  2277  		// check Extend should accept this valid block
  2278  		err = state.Extend(context.Background(), block)
  2279  		require.NoError(t, err)
  2280  
  2281  		// now the guarantee has invalid signer indices: the checksum should have 4 bytes, but it only has 1
  2282  		payload.Guarantees[0].SignerIndices = []byte{byte(1)}
  2283  
  2284  		// create new block that has invalid collection guarantee
  2285  		block = unittest.BlockWithParentFixture(head)
  2286  		block.SetPayload(payload)
  2287  
  2288  		err = state.Extend(context.Background(), block)
  2289  		require.True(t, signature.IsInvalidSignerIndicesError(err), err)
  2290  		require.ErrorIs(t, err, signature.ErrInvalidChecksum)
  2291  		require.True(t, st.IsInvalidExtensionError(err), err)
  2292  
  2293  		// now the guarantee has invalid signer indices: the checksum should have 4 bytes, but it only has 1
  2294  		checksumMismatch := make([]byte, len(validSignerIndices))
  2295  		copy(checksumMismatch, validSignerIndices)
  2296  		checksumMismatch[0] = byte(1)
  2297  		if checksumMismatch[0] == validSignerIndices[0] {
  2298  			checksumMismatch[0] = byte(2)
  2299  		}
  2300  		payload.Guarantees[0].SignerIndices = checksumMismatch
  2301  		err = state.Extend(context.Background(), block)
  2302  		require.True(t, signature.IsInvalidSignerIndicesError(err), err)
  2303  		require.ErrorIs(t, err, signature.ErrInvalidChecksum)
  2304  		require.True(t, st.IsInvalidExtensionError(err), err)
  2305  
  2306  		// let's test even if the checksum is correct, but signer indices is still wrong because the tailing are not 0,
  2307  		// then the block should still be rejected.
  2308  		wrongTailing := make([]byte, len(validSignerIndices))
  2309  		copy(wrongTailing, validSignerIndices)
  2310  		wrongTailing[len(wrongTailing)-1] = byte(255)
  2311  
  2312  		payload.Guarantees[0].SignerIndices = wrongTailing
  2313  		err = state.Extend(context.Background(), block)
  2314  		require.Error(t, err)
  2315  		require.True(t, signature.IsInvalidSignerIndicesError(err), err)
  2316  		require.ErrorIs(t, err, signature.ErrIllegallyPaddedBitVector)
  2317  		require.True(t, st.IsInvalidExtensionError(err), err)
  2318  
  2319  		// test imcompatible bit vector length
  2320  		wrongbitVectorLength := validSignerIndices[0 : len(validSignerIndices)-1]
  2321  		payload.Guarantees[0].SignerIndices = wrongbitVectorLength
  2322  		err = state.Extend(context.Background(), block)
  2323  		require.True(t, signature.IsInvalidSignerIndicesError(err), err)
  2324  		require.ErrorIs(t, err, signature.ErrIncompatibleBitVectorLength)
  2325  		require.True(t, st.IsInvalidExtensionError(err), err)
  2326  
  2327  		// revert back to good value
  2328  		payload.Guarantees[0].SignerIndices = validSignerIndices
  2329  
  2330  		// test the ReferenceBlockID is not found
  2331  		payload.Guarantees[0].ReferenceBlockID = flow.ZeroID
  2332  		err = state.Extend(context.Background(), block)
  2333  		require.ErrorIs(t, err, storage.ErrNotFound)
  2334  		require.True(t, st.IsInvalidExtensionError(err), err)
  2335  
  2336  		// revert back to good value
  2337  		payload.Guarantees[0].ReferenceBlockID = head.ID()
  2338  
  2339  		// TODO: test the guarantee has bad reference block ID that would return protocol.ErrNextEpochNotCommitted
  2340  		// this case is not easy to create, since the test case has no such block yet.
  2341  		// we need to refactor the ParticipantState to add a guaranteeValidator, so that we can mock it and
  2342  		// return the protocol.ErrNextEpochNotCommitted for testing
  2343  
  2344  		// test the guarantee has wrong chain ID, and should return ErrClusterNotFound
  2345  		payload.Guarantees[0].ChainID = flow.ChainID("some_bad_chain_ID")
  2346  		err = state.Extend(context.Background(), block)
  2347  		require.Error(t, err)
  2348  		require.ErrorIs(t, err, realprotocol.ErrClusterNotFound)
  2349  		require.True(t, st.IsInvalidExtensionError(err), err)
  2350  	})
  2351  }
  2352  
  2353  // If block B is finalized and contains a seal for block A, then A is the last sealed block
  2354  func TestSealed(t *testing.T) {
  2355  	rootSnapshot := unittest.RootSnapshotFixture(participants)
  2356  	util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState) {
  2357  		head, err := rootSnapshot.Head()
  2358  		require.NoError(t, err)
  2359  
  2360  		// block 1 will be sealed
  2361  		block1 := unittest.BlockWithParentFixture(head)
  2362  
  2363  		receipt1, seal1 := unittest.ReceiptAndSealForBlock(block1)
  2364  
  2365  		// block 2 contains receipt for block 1
  2366  		block2 := unittest.BlockWithParentFixture(block1.Header)
  2367  		block2.SetPayload(unittest.PayloadFixture(unittest.WithReceipts(receipt1)))
  2368  
  2369  		err = state.ExtendCertified(context.Background(), block1, block2.Header.QuorumCertificate())
  2370  		require.NoError(t, err)
  2371  		err = state.Finalize(context.Background(), block1.ID())
  2372  		require.NoError(t, err)
  2373  
  2374  		// block 3 contains seal for block 1
  2375  		block3 := unittest.BlockWithParentFixture(block2.Header)
  2376  		block3.SetPayload(flow.Payload{
  2377  			Seals: []*flow.Seal{seal1},
  2378  		})
  2379  
  2380  		err = state.ExtendCertified(context.Background(), block2, block3.Header.QuorumCertificate())
  2381  		require.NoError(t, err)
  2382  		err = state.Finalize(context.Background(), block2.ID())
  2383  		require.NoError(t, err)
  2384  
  2385  		err = state.ExtendCertified(context.Background(), block3, unittest.CertifyBlock(block3.Header))
  2386  		require.NoError(t, err)
  2387  		err = state.Finalize(context.Background(), block3.ID())
  2388  		require.NoError(t, err)
  2389  
  2390  		sealed, err := state.Sealed().Head()
  2391  		require.NoError(t, err)
  2392  		require.Equal(t, block1.ID(), sealed.ID())
  2393  	})
  2394  }
  2395  
  2396  // Test that when adding a block to database, there are only two cases at any point of time:
  2397  // 1) neither the block header, nor the payload index exist in database
  2398  // 2) both the block header and the payload index can be found in database
  2399  // A non atomic bug would be: header is found in DB, but payload index is not found
  2400  func TestCacheAtomicity(t *testing.T) {
  2401  	rootSnapshot := unittest.RootSnapshotFixture(participants)
  2402  	util.RunWithFollowerProtocolStateAndHeaders(t, rootSnapshot,
  2403  		func(db *badger.DB, state *protocol.FollowerState, headers storage.Headers, index storage.Index) {
  2404  			head, err := rootSnapshot.Head()
  2405  			require.NoError(t, err)
  2406  
  2407  			block := unittest.BlockWithParentFixture(head)
  2408  			blockID := block.ID()
  2409  
  2410  			// check 100 times to see if either 1) or 2) satisfies
  2411  			var wg sync.WaitGroup
  2412  			wg.Add(1)
  2413  			go func(blockID flow.Identifier) {
  2414  				for i := 0; i < 100; i++ {
  2415  					_, err := headers.ByBlockID(blockID)
  2416  					if errors.Is(err, stoerr.ErrNotFound) {
  2417  						continue
  2418  					}
  2419  					require.NoError(t, err)
  2420  
  2421  					_, err = index.ByBlockID(blockID)
  2422  					require.NoError(t, err, "found block ID, but index is missing, DB updates is non-atomic")
  2423  				}
  2424  				wg.Done()
  2425  			}(blockID)
  2426  
  2427  			// storing the block to database, which supposed to be atomic updates to headers and index,
  2428  			// both to badger database and the cache.
  2429  			err = state.ExtendCertified(context.Background(), block, unittest.CertifyBlock(block.Header))
  2430  			require.NoError(t, err)
  2431  			wg.Wait()
  2432  		})
  2433  }
  2434  
  2435  // TestHeaderInvalidTimestamp tests that extending header with invalid timestamp results in sentinel error
  2436  func TestHeaderInvalidTimestamp(t *testing.T) {
  2437  	unittest.RunWithBadgerDB(t, func(db *badger.DB) {
  2438  		metrics := metrics.NewNoopCollector()
  2439  		tracer := trace.NewNoopTracer()
  2440  		log := zerolog.Nop()
  2441  		all := storeutil.StorageLayer(t, db)
  2442  
  2443  		// create a event consumer to test epoch transition events
  2444  		distributor := events.NewDistributor()
  2445  		consumer := mockprotocol.NewConsumer(t)
  2446  		distributor.AddConsumer(consumer)
  2447  
  2448  		block, result, seal := unittest.BootstrapFixture(participants)
  2449  		qc := unittest.QuorumCertificateFixture(unittest.QCWithRootBlockID(block.ID()))
  2450  		rootSnapshot, err := inmem.SnapshotFromBootstrapState(block, result, seal, qc)
  2451  		require.NoError(t, err)
  2452  
  2453  		state, err := protocol.Bootstrap(
  2454  			metrics,
  2455  			db,
  2456  			all.Headers,
  2457  			all.Seals,
  2458  			all.Results,
  2459  			all.Blocks,
  2460  			all.QuorumCertificates,
  2461  			all.Setups,
  2462  			all.EpochCommits,
  2463  			all.Statuses,
  2464  			all.VersionBeacons,
  2465  			rootSnapshot,
  2466  		)
  2467  		require.NoError(t, err)
  2468  
  2469  		blockTimer := &mockprotocol.BlockTimer{}
  2470  		blockTimer.On("Validate", mock.Anything, mock.Anything).Return(realprotocol.NewInvalidBlockTimestamp(""))
  2471  
  2472  		fullState, err := protocol.NewFullConsensusState(
  2473  			log,
  2474  			tracer,
  2475  			consumer,
  2476  			state,
  2477  			all.Index,
  2478  			all.Payloads,
  2479  			blockTimer,
  2480  			util.MockReceiptValidator(),
  2481  			util.MockSealValidator(all.Seals),
  2482  		)
  2483  		require.NoError(t, err)
  2484  
  2485  		extend := unittest.BlockWithParentFixture(block.Header)
  2486  		extend.Payload.Guarantees = nil
  2487  		extend.Header.PayloadHash = extend.Payload.Hash()
  2488  
  2489  		err = fullState.Extend(context.Background(), extend)
  2490  		assert.Error(t, err, "a proposal with invalid timestamp has to be rejected")
  2491  		assert.True(t, st.IsInvalidExtensionError(err), "if timestamp is invalid it should return invalid block error")
  2492  	})
  2493  }
  2494  
  2495  // TestProtocolStateIdempotent tests that both participant and follower states correctly process adding same block twice
  2496  // where second extend doesn't result in an error and effectively is no-op.
  2497  func TestProtocolStateIdempotent(t *testing.T) {
  2498  	rootSnapshot := unittest.RootSnapshotFixture(participants)
  2499  	head, err := rootSnapshot.Head()
  2500  	require.NoError(t, err)
  2501  	t.Run("follower", func(t *testing.T) {
  2502  		util.RunWithFollowerProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.FollowerState) {
  2503  			block := unittest.BlockWithParentFixture(head)
  2504  			err := state.ExtendCertified(context.Background(), block, unittest.CertifyBlock(block.Header))
  2505  			require.NoError(t, err)
  2506  
  2507  			// same operation should be no-op
  2508  			err = state.ExtendCertified(context.Background(), block, unittest.CertifyBlock(block.Header))
  2509  			require.NoError(t, err)
  2510  		})
  2511  	})
  2512  	t.Run("participant", func(t *testing.T) {
  2513  		util.RunWithFullProtocolState(t, rootSnapshot, func(db *badger.DB, state *protocol.ParticipantState) {
  2514  			block := unittest.BlockWithParentFixture(head)
  2515  			err := state.Extend(context.Background(), block)
  2516  			require.NoError(t, err)
  2517  
  2518  			// same operation should be no-op
  2519  			err = state.Extend(context.Background(), block)
  2520  			require.NoError(t, err)
  2521  
  2522  			err = state.ExtendCertified(context.Background(), block, unittest.CertifyBlock(block.Header))
  2523  			require.NoError(t, err)
  2524  		})
  2525  	})
  2526  }
  2527  
  2528  func assertEpochEmergencyFallbackTriggered(t *testing.T, state realprotocol.State, expected bool) {
  2529  	triggered, err := state.Params().EpochFallbackTriggered()
  2530  	require.NoError(t, err)
  2531  	assert.Equal(t, expected, triggered)
  2532  }
  2533  
  2534  // mockMetricsForRootSnapshot mocks the given metrics mock object to expect all
  2535  // metrics which are set during bootstrapping and building blocks.
  2536  func mockMetricsForRootSnapshot(metricsMock *mockmodule.ComplianceMetrics, rootSnapshot *inmem.Snapshot) {
  2537  	metricsMock.On("CurrentEpochCounter", rootSnapshot.Encodable().Epochs.Current.Counter)
  2538  	metricsMock.On("CurrentEpochPhase", rootSnapshot.Encodable().Phase)
  2539  	metricsMock.On("CurrentEpochFinalView", rootSnapshot.Encodable().Epochs.Current.FinalView)
  2540  	metricsMock.On("CommittedEpochFinalView", rootSnapshot.Encodable().Epochs.Current.FinalView)
  2541  	metricsMock.On("CurrentDKGPhase1FinalView", rootSnapshot.Encodable().Epochs.Current.DKGPhase1FinalView)
  2542  	metricsMock.On("CurrentDKGPhase2FinalView", rootSnapshot.Encodable().Epochs.Current.DKGPhase2FinalView)
  2543  	metricsMock.On("CurrentDKGPhase3FinalView", rootSnapshot.Encodable().Epochs.Current.DKGPhase3FinalView)
  2544  	metricsMock.On("BlockSealed", mock.Anything)
  2545  	metricsMock.On("BlockFinalized", mock.Anything)
  2546  	metricsMock.On("FinalizedHeight", mock.Anything)
  2547  	metricsMock.On("SealedHeight", mock.Anything)
  2548  }