github.com/onflow/flow-go@v0.35.7-crescendo-preview.23-atree-inlining/engine/execution/ingestion/stop/stop_control_test.go (about)

     1  package stop
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"testing"
     7  	"time"
     8  
     9  	"github.com/coreos/go-semver/semver"
    10  	testifyMock "github.com/stretchr/testify/mock"
    11  	"github.com/stretchr/testify/require"
    12  
    13  	"github.com/onflow/flow-go/engine"
    14  	"github.com/onflow/flow-go/engine/execution/state/mock"
    15  	"github.com/onflow/flow-go/model/flow"
    16  	"github.com/onflow/flow-go/module/irrecoverable"
    17  	storageMock "github.com/onflow/flow-go/storage/mock"
    18  	"github.com/onflow/flow-go/utils/unittest"
    19  )
    20  
    21  // If stopping mechanism has caused any changes to execution flow
    22  // (skipping execution of blocks) we disallow setting new values
    23  func TestCannotSetNewValuesAfterStoppingCommenced(t *testing.T) {
    24  
    25  	t.Run("when processing block at stop height", func(t *testing.T) {
    26  		sc := NewStopControl(
    27  			engine.NewUnit(),
    28  			time.Second,
    29  			unittest.Logger(),
    30  			nil,
    31  			nil,
    32  			nil,
    33  			nil,
    34  			&flow.Header{Height: 1},
    35  			false,
    36  			false,
    37  		)
    38  
    39  		require.False(t, sc.GetStopParameters().Set())
    40  
    41  		// first update is always successful
    42  		stop := StopParameters{StopBeforeHeight: 21}
    43  		err := sc.SetStopParameters(stop)
    44  		require.NoError(t, err)
    45  
    46  		require.Equal(t, stop, sc.GetStopParameters())
    47  
    48  		// no stopping has started yet, block below stop height
    49  		header := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20))
    50  		require.True(t, sc.ShouldExecuteBlock(header.ID(), header.Height))
    51  
    52  		stop2 := StopParameters{StopBeforeHeight: 37}
    53  		err = sc.SetStopParameters(stop2)
    54  		require.NoError(t, err)
    55  
    56  		// block at stop height, it should be skipped
    57  		header = unittest.BlockHeaderFixture(unittest.WithHeaderHeight(37))
    58  		require.False(t, sc.ShouldExecuteBlock(header.ID(), header.Height))
    59  
    60  		// cannot set new stop height after stopping has started
    61  		err = sc.SetStopParameters(StopParameters{StopBeforeHeight: 2137})
    62  		require.ErrorIs(t, err, ErrCannotChangeStop)
    63  
    64  		// state did not change
    65  		require.Equal(t, stop2, sc.GetStopParameters())
    66  	})
    67  
    68  	t.Run("when processing finalized blocks", func(t *testing.T) {
    69  
    70  		execState := mock.NewExecutionState(t)
    71  
    72  		sc := NewStopControl(
    73  			engine.NewUnit(),
    74  			time.Second,
    75  			unittest.Logger(),
    76  			execState,
    77  			nil,
    78  			nil,
    79  			nil,
    80  			&flow.Header{Height: 1},
    81  			false,
    82  			false,
    83  		)
    84  
    85  		require.False(t, sc.GetStopParameters().Set())
    86  
    87  		// first update is always successful
    88  		stop := StopParameters{StopBeforeHeight: 21}
    89  		err := sc.SetStopParameters(stop)
    90  		require.NoError(t, err)
    91  		require.Equal(t, stop, sc.GetStopParameters())
    92  
    93  		// make execution check pretends block has been executed
    94  		execState.On("IsBlockExecuted", testifyMock.Anything, testifyMock.Anything).Return(true, nil)
    95  
    96  		// no stopping has started yet, block below stop height
    97  		header := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20))
    98  		sc.BlockFinalizedForTesting(header)
    99  
   100  		stop2 := StopParameters{StopBeforeHeight: 37}
   101  		err = sc.SetStopParameters(stop2)
   102  		require.NoError(t, err)
   103  		require.Equal(t, stop2, sc.GetStopParameters())
   104  
   105  		// block at stop height, it should be triggered stop
   106  		header = unittest.BlockHeaderFixture(unittest.WithHeaderHeight(37))
   107  		sc.BlockFinalizedForTesting(header)
   108  
   109  		// since we set shouldCrash to false, execution should be stopped
   110  		require.True(t, sc.IsExecutionStopped())
   111  
   112  		err = sc.SetStopParameters(StopParameters{StopBeforeHeight: 2137})
   113  		require.ErrorIs(t, err, ErrCannotChangeStop)
   114  	})
   115  }
   116  
   117  // TestExecutionFallingBehind check if StopControl behaves properly even if EN runs behind
   118  // and blocks are finalized before they are executed
   119  func TestExecutionFallingBehind(t *testing.T) {
   120  
   121  	execState := mock.NewExecutionState(t)
   122  
   123  	headerA := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20))
   124  	headerB := unittest.BlockHeaderWithParentFixture(headerA) // 21
   125  	headerC := unittest.BlockHeaderWithParentFixture(headerB) // 22
   126  	headerD := unittest.BlockHeaderWithParentFixture(headerC) // 23
   127  
   128  	sc := NewStopControl(
   129  		engine.NewUnit(),
   130  		time.Second,
   131  		unittest.Logger(),
   132  		execState,
   133  		nil,
   134  		nil,
   135  		nil,
   136  		&flow.Header{Height: 1},
   137  		false,
   138  		false,
   139  	)
   140  
   141  	// set stop at 22, so 21 is the last height which should be processed
   142  	stop := StopParameters{StopBeforeHeight: 22}
   143  	err := sc.SetStopParameters(stop)
   144  	require.NoError(t, err)
   145  	require.Equal(t, stop, sc.GetStopParameters())
   146  
   147  	execState.On("IsBlockExecuted", headerC.Height-1, headerC.ParentID).Return(false, nil)
   148  
   149  	// finalize blocks first
   150  	sc.BlockFinalizedForTesting(headerA)
   151  	sc.BlockFinalizedForTesting(headerB)
   152  	sc.BlockFinalizedForTesting(headerC)
   153  	sc.BlockFinalizedForTesting(headerD)
   154  
   155  	// simulate execution
   156  	sc.OnBlockExecuted(headerA)
   157  	sc.OnBlockExecuted(headerB)
   158  	require.True(t, sc.IsExecutionStopped())
   159  }
   160  
   161  type stopControlMockHeaders struct {
   162  	headers map[uint64]*flow.Header
   163  }
   164  
   165  func (m *stopControlMockHeaders) BlockIDByHeight(height uint64) (flow.Identifier, error) {
   166  	h, ok := m.headers[height]
   167  	if !ok {
   168  		return flow.ZeroID, fmt.Errorf("header not found")
   169  	}
   170  	return h.ID(), nil
   171  }
   172  
   173  func TestAddStopForPastBlocks(t *testing.T) {
   174  	execState := mock.NewExecutionState(t)
   175  
   176  	headerA := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20))
   177  	headerB := unittest.BlockHeaderWithParentFixture(headerA) // 21
   178  	headerC := unittest.BlockHeaderWithParentFixture(headerB) // 22
   179  	headerD := unittest.BlockHeaderWithParentFixture(headerC) // 23
   180  
   181  	headers := &stopControlMockHeaders{
   182  		headers: map[uint64]*flow.Header{
   183  			headerA.Height: headerA,
   184  			headerB.Height: headerB,
   185  			headerC.Height: headerC,
   186  			headerD.Height: headerD,
   187  		},
   188  	}
   189  
   190  	sc := NewStopControl(
   191  		engine.NewUnit(),
   192  		time.Second,
   193  		unittest.Logger(),
   194  		execState,
   195  		headers,
   196  		nil,
   197  		nil,
   198  		&flow.Header{Height: 1},
   199  		false,
   200  		false,
   201  	)
   202  
   203  	// finalize blocks first
   204  	sc.BlockFinalizedForTesting(headerA)
   205  	sc.BlockFinalizedForTesting(headerB)
   206  	sc.BlockFinalizedForTesting(headerC)
   207  
   208  	// simulate execution
   209  	sc.OnBlockExecuted(headerA)
   210  	sc.OnBlockExecuted(headerB)
   211  	sc.OnBlockExecuted(headerC)
   212  
   213  	// block is executed
   214  	execState.On("IsBlockExecuted", headerD.Height-1, headerD.ParentID).Return(true, nil)
   215  
   216  	// set stop at 22, but finalization and execution is at 23
   217  	// so stop right away
   218  	stop := StopParameters{StopBeforeHeight: 22}
   219  	err := sc.SetStopParameters(stop)
   220  	require.NoError(t, err)
   221  	require.Equal(t, stop, sc.GetStopParameters())
   222  
   223  	// finalize one more block after stop is set
   224  	sc.BlockFinalizedForTesting(headerD)
   225  
   226  	require.True(t, sc.IsExecutionStopped())
   227  }
   228  
   229  func TestAddStopForPastBlocksExecutionFallingBehind(t *testing.T) {
   230  	execState := mock.NewExecutionState(t)
   231  
   232  	headerA := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20))
   233  	headerB := unittest.BlockHeaderWithParentFixture(headerA) // 21
   234  	headerC := unittest.BlockHeaderWithParentFixture(headerB) // 22
   235  	headerD := unittest.BlockHeaderWithParentFixture(headerC) // 23
   236  
   237  	headers := &stopControlMockHeaders{
   238  		headers: map[uint64]*flow.Header{
   239  			headerA.Height: headerA,
   240  			headerB.Height: headerB,
   241  			headerC.Height: headerC,
   242  			headerD.Height: headerD,
   243  		},
   244  	}
   245  
   246  	sc := NewStopControl(
   247  		engine.NewUnit(),
   248  		time.Second,
   249  		unittest.Logger(),
   250  		execState,
   251  		headers,
   252  		nil,
   253  		nil,
   254  		&flow.Header{Height: 1},
   255  		false,
   256  		false,
   257  	)
   258  
   259  	execState.On("IsBlockExecuted", headerD.Height-1, headerD.ParentID).Return(false, nil)
   260  
   261  	// finalize blocks first
   262  	sc.BlockFinalizedForTesting(headerA)
   263  	sc.BlockFinalizedForTesting(headerB)
   264  	sc.BlockFinalizedForTesting(headerC)
   265  
   266  	// set stop at 22, but finalization is at 23 so 21
   267  	// is the last height which wil be executed
   268  	stop := StopParameters{StopBeforeHeight: 22}
   269  	err := sc.SetStopParameters(stop)
   270  	require.NoError(t, err)
   271  	require.Equal(t, stop, sc.GetStopParameters())
   272  
   273  	// finalize one more block after stop is set
   274  	sc.BlockFinalizedForTesting(headerD)
   275  
   276  	// simulate execution
   277  	sc.OnBlockExecuted(headerA)
   278  	sc.OnBlockExecuted(headerB)
   279  	require.True(t, sc.IsExecutionStopped())
   280  }
   281  
   282  func TestStopControlWithVersionControl(t *testing.T) {
   283  	t.Run("normal case", func(t *testing.T) {
   284  		execState := mock.NewExecutionState(t)
   285  		versionBeacons := new(storageMock.VersionBeacons)
   286  
   287  		headerA := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20))
   288  		headerB := unittest.BlockHeaderWithParentFixture(headerA) // 21
   289  		headerC := unittest.BlockHeaderWithParentFixture(headerB) // 22
   290  
   291  		headers := &stopControlMockHeaders{
   292  			headers: map[uint64]*flow.Header{
   293  				headerA.Height: headerA,
   294  				headerB.Height: headerB,
   295  				headerC.Height: headerC,
   296  			},
   297  		}
   298  
   299  		sc := NewStopControl(
   300  			engine.NewUnit(),
   301  			time.Second,
   302  			unittest.Logger(),
   303  			execState,
   304  			headers,
   305  			versionBeacons,
   306  			semver.New("1.0.0"),
   307  			&flow.Header{Height: 1},
   308  			false,
   309  			false,
   310  		)
   311  
   312  		// setting this means all finalized blocks are considered already executed
   313  		execState.On("IsBlockExecuted", headerC.Height-1, headerC.ParentID).Return(true, nil)
   314  
   315  		versionBeacons.
   316  			On("Highest", testifyMock.Anything).
   317  			Return(&flow.SealedVersionBeacon{
   318  				VersionBeacon: unittest.VersionBeaconFixture(
   319  					unittest.WithBoundaries(
   320  						// zero boundary is expected if there
   321  						// is no boundary set by the contract yet
   322  						flow.VersionBoundary{
   323  							BlockHeight: 0,
   324  							Version:     "0.0.0",
   325  						}),
   326  				),
   327  				SealHeight: headerA.Height,
   328  			}, nil).Once()
   329  
   330  		// finalize first block
   331  		sc.BlockFinalizedForTesting(headerA)
   332  		require.False(t, sc.IsExecutionStopped())
   333  		require.False(t, sc.GetStopParameters().Set())
   334  
   335  		// new version beacon
   336  		versionBeacons.
   337  			On("Highest", testifyMock.Anything).
   338  			Return(&flow.SealedVersionBeacon{
   339  				VersionBeacon: unittest.VersionBeaconFixture(
   340  					unittest.WithBoundaries(
   341  						// zero boundary is expected if there
   342  						// is no boundary set by the contract yet
   343  						flow.VersionBoundary{
   344  							BlockHeight: 0,
   345  							Version:     "0.0.0",
   346  						}, flow.VersionBoundary{
   347  							BlockHeight: 21,
   348  							Version:     "1.0.0",
   349  						}),
   350  				),
   351  				SealHeight: headerB.Height,
   352  			}, nil).Once()
   353  
   354  		// finalize second block. we are still ok as the node version
   355  		// is the same as the version beacon one
   356  		sc.BlockFinalizedForTesting(headerB)
   357  		require.False(t, sc.IsExecutionStopped())
   358  		require.False(t, sc.GetStopParameters().Set())
   359  
   360  		// new version beacon
   361  		versionBeacons.
   362  			On("Highest", testifyMock.Anything).
   363  			Return(&flow.SealedVersionBeacon{
   364  				VersionBeacon: unittest.VersionBeaconFixture(
   365  					unittest.WithBoundaries(
   366  						// The previous version is included in the new version beacon
   367  						flow.VersionBoundary{
   368  							BlockHeight: 21,
   369  							Version:     "1.0.0",
   370  						}, flow.VersionBoundary{
   371  							BlockHeight: 22,
   372  							Version:     "2.0.0",
   373  						}),
   374  				),
   375  				SealHeight: headerC.Height,
   376  			}, nil).Once()
   377  		sc.BlockFinalizedForTesting(headerC)
   378  		// should be stopped as this is height 22 and height 21 is already considered executed
   379  		require.True(t, sc.IsExecutionStopped())
   380  	})
   381  
   382  	t.Run("version boundary removed", func(t *testing.T) {
   383  
   384  		// future version boundaries can be removed
   385  		// in which case they will be missing from the version beacon
   386  		execState := mock.NewExecutionState(t)
   387  		versionBeacons := storageMock.NewVersionBeacons(t)
   388  
   389  		headerA := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20))
   390  		headerB := unittest.BlockHeaderWithParentFixture(headerA) // 21
   391  		headerC := unittest.BlockHeaderWithParentFixture(headerB) // 22
   392  
   393  		headers := &stopControlMockHeaders{
   394  			headers: map[uint64]*flow.Header{
   395  				headerA.Height: headerA,
   396  				headerB.Height: headerB,
   397  				headerC.Height: headerC,
   398  			},
   399  		}
   400  
   401  		sc := NewStopControl(
   402  			engine.NewUnit(),
   403  			time.Second,
   404  			unittest.Logger(),
   405  			execState,
   406  			headers,
   407  			versionBeacons,
   408  			semver.New("1.0.0"),
   409  			&flow.Header{Height: 1},
   410  			false,
   411  			false,
   412  		)
   413  
   414  		versionBeacons.
   415  			On("Highest", testifyMock.Anything).
   416  			Return(&flow.SealedVersionBeacon{
   417  				VersionBeacon: unittest.VersionBeaconFixture(
   418  					unittest.WithBoundaries(
   419  						// set to stop at height 21
   420  						flow.VersionBoundary{
   421  							BlockHeight: 0,
   422  							Version:     "0.0.0",
   423  						}, flow.VersionBoundary{
   424  							BlockHeight: 21,
   425  							Version:     "2.0.0",
   426  						}),
   427  				),
   428  				SealHeight: headerA.Height,
   429  			}, nil).Once()
   430  
   431  		// finalize first block
   432  		sc.BlockFinalizedForTesting(headerA)
   433  		require.False(t, sc.IsExecutionStopped())
   434  		require.Equal(t, StopParameters{
   435  			StopBeforeHeight: 21,
   436  			ShouldCrash:      false,
   437  		}, sc.GetStopParameters())
   438  
   439  		// new version beacon
   440  		versionBeacons.
   441  			On("Highest", testifyMock.Anything).
   442  			Return(&flow.SealedVersionBeacon{
   443  				VersionBeacon: unittest.VersionBeaconFixture(
   444  					unittest.WithBoundaries(
   445  						// stop removed
   446  						flow.VersionBoundary{
   447  							BlockHeight: 0,
   448  							Version:     "0.0.0",
   449  						}),
   450  				),
   451  				SealHeight: headerB.Height,
   452  			}, nil).Once()
   453  
   454  		// finalize second block. we are still ok as the node version
   455  		// is the same as the version beacon one
   456  		sc.BlockFinalizedForTesting(headerB)
   457  		require.False(t, sc.IsExecutionStopped())
   458  		require.False(t, sc.GetStopParameters().Set())
   459  	})
   460  
   461  	t.Run("manual not cleared by version beacon", func(t *testing.T) {
   462  		// future version boundaries can be removed
   463  		// in which case they will be missing from the version beacon
   464  		execState := mock.NewExecutionState(t)
   465  		versionBeacons := storageMock.NewVersionBeacons(t)
   466  
   467  		headerA := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20))
   468  		headerB := unittest.BlockHeaderWithParentFixture(headerA) // 21
   469  		headerC := unittest.BlockHeaderWithParentFixture(headerB) // 22
   470  
   471  		headers := &stopControlMockHeaders{
   472  			headers: map[uint64]*flow.Header{
   473  				headerA.Height: headerA,
   474  				headerB.Height: headerB,
   475  				headerC.Height: headerC,
   476  			},
   477  		}
   478  
   479  		sc := NewStopControl(
   480  			engine.NewUnit(),
   481  			time.Second,
   482  			unittest.Logger(),
   483  			execState,
   484  			headers,
   485  			versionBeacons,
   486  			semver.New("1.0.0"),
   487  			&flow.Header{Height: 1},
   488  			false,
   489  			false,
   490  		)
   491  
   492  		versionBeacons.
   493  			On("Highest", testifyMock.Anything).
   494  			Return(&flow.SealedVersionBeacon{
   495  				VersionBeacon: unittest.VersionBeaconFixture(
   496  					unittest.WithBoundaries(
   497  						// set to stop at height 21
   498  						flow.VersionBoundary{
   499  							BlockHeight: 0,
   500  							Version:     "0.0.0",
   501  						}),
   502  				),
   503  				SealHeight: headerA.Height,
   504  			}, nil).Once()
   505  
   506  		// finalize first block
   507  		sc.BlockFinalizedForTesting(headerA)
   508  		require.False(t, sc.IsExecutionStopped())
   509  		require.False(t, sc.GetStopParameters().Set())
   510  
   511  		// set manual stop
   512  		stop := StopParameters{
   513  			StopBeforeHeight: 22,
   514  			ShouldCrash:      false,
   515  		}
   516  		err := sc.SetStopParameters(stop)
   517  		require.NoError(t, err)
   518  		require.Equal(t, stop, sc.GetStopParameters())
   519  
   520  		// new version beacon
   521  		versionBeacons.
   522  			On("Highest", testifyMock.Anything).
   523  			Return(&flow.SealedVersionBeacon{
   524  				VersionBeacon: unittest.VersionBeaconFixture(
   525  					unittest.WithBoundaries(
   526  						// stop removed
   527  						flow.VersionBoundary{
   528  							BlockHeight: 0,
   529  							Version:     "0.0.0",
   530  						}),
   531  				),
   532  				SealHeight: headerB.Height,
   533  			}, nil).Once()
   534  
   535  		sc.BlockFinalizedForTesting(headerB)
   536  		require.False(t, sc.IsExecutionStopped())
   537  		// stop is not cleared due to being set manually
   538  		require.Equal(t, stop, sc.GetStopParameters())
   539  	})
   540  
   541  	t.Run("version beacon not cleared by manual", func(t *testing.T) {
   542  		// future version boundaries can be removed
   543  		// in which case they will be missing from the version beacon
   544  		execState := mock.NewExecutionState(t)
   545  		versionBeacons := storageMock.NewVersionBeacons(t)
   546  
   547  		headerA := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20))
   548  		headerB := unittest.BlockHeaderWithParentFixture(headerA) // 21
   549  
   550  		headers := &stopControlMockHeaders{
   551  			headers: map[uint64]*flow.Header{
   552  				headerA.Height: headerA,
   553  				headerB.Height: headerB,
   554  			},
   555  		}
   556  
   557  		sc := NewStopControl(
   558  			engine.NewUnit(),
   559  			time.Second,
   560  			unittest.Logger(),
   561  			execState,
   562  			headers,
   563  			versionBeacons,
   564  			semver.New("1.0.0"),
   565  			&flow.Header{Height: 1},
   566  			false,
   567  			false,
   568  		)
   569  
   570  		vbStop := StopParameters{
   571  			StopBeforeHeight: 22,
   572  			ShouldCrash:      false,
   573  		}
   574  		versionBeacons.
   575  			On("Highest", testifyMock.Anything).
   576  			Return(&flow.SealedVersionBeacon{
   577  				VersionBeacon: unittest.VersionBeaconFixture(
   578  					unittest.WithBoundaries(
   579  						// set to stop at height 21
   580  						flow.VersionBoundary{
   581  							BlockHeight: 0,
   582  							Version:     "0.0.0",
   583  						}, flow.VersionBoundary{
   584  							BlockHeight: vbStop.StopBeforeHeight,
   585  							Version:     "2.0.0",
   586  						}),
   587  				),
   588  				SealHeight: headerA.Height,
   589  			}, nil).Once()
   590  
   591  		// finalize first block
   592  		sc.BlockFinalizedForTesting(headerA)
   593  		require.False(t, sc.IsExecutionStopped())
   594  		require.Equal(t, vbStop, sc.GetStopParameters())
   595  
   596  		// set manual stop
   597  		stop := StopParameters{
   598  			StopBeforeHeight: 23,
   599  			ShouldCrash:      false,
   600  		}
   601  		err := sc.SetStopParameters(stop)
   602  		require.ErrorIs(t, err, ErrCannotChangeStop)
   603  		// stop is not cleared due to being set earlier by a version beacon
   604  		require.Equal(t, vbStop, sc.GetStopParameters())
   605  	})
   606  }
   607  
   608  // StopControl created as stopped will keep the state
   609  func TestStartingStopped(t *testing.T) {
   610  
   611  	sc := NewStopControl(
   612  		engine.NewUnit(),
   613  		time.Second,
   614  		unittest.Logger(),
   615  		nil,
   616  		nil,
   617  		nil,
   618  		nil,
   619  		&flow.Header{Height: 1},
   620  		true,
   621  		false,
   622  	)
   623  	require.True(t, sc.IsExecutionStopped())
   624  }
   625  
   626  func TestStoppedStateRejectsAllBlocksAndChanged(t *testing.T) {
   627  
   628  	// make sure we don't even query executed status if stopped
   629  	// mock should fail test on any method call
   630  	execState := mock.NewExecutionState(t)
   631  
   632  	sc := NewStopControl(
   633  		engine.NewUnit(),
   634  		time.Second,
   635  		unittest.Logger(),
   636  		execState,
   637  		nil,
   638  		nil,
   639  		nil,
   640  		&flow.Header{Height: 1},
   641  		true,
   642  		false,
   643  	)
   644  	require.True(t, sc.IsExecutionStopped())
   645  
   646  	err := sc.SetStopParameters(StopParameters{
   647  		StopBeforeHeight: 2137,
   648  		ShouldCrash:      true,
   649  	})
   650  	require.ErrorIs(t, err, ErrCannotChangeStop)
   651  
   652  	header := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20))
   653  
   654  	sc.BlockFinalizedForTesting(header)
   655  	require.True(t, sc.IsExecutionStopped())
   656  }
   657  
   658  func Test_StopControlWorkers(t *testing.T) {
   659  
   660  	t.Run("start and stop, stopped = true", func(t *testing.T) {
   661  
   662  		sc := NewStopControl(
   663  			engine.NewUnit(),
   664  			time.Second,
   665  			unittest.Logger(),
   666  			nil,
   667  			nil,
   668  			nil,
   669  			nil,
   670  			&flow.Header{Height: 1},
   671  			true,
   672  			false,
   673  		)
   674  
   675  		ctx, cancel := context.WithCancel(context.Background())
   676  		ictx := irrecoverable.NewMockSignalerContext(t, ctx)
   677  
   678  		sc.Start(ictx)
   679  
   680  		unittest.AssertClosesBefore(t, sc.Ready(), 10*time.Second)
   681  
   682  		cancel()
   683  
   684  		unittest.AssertClosesBefore(t, sc.Done(), 10*time.Second)
   685  	})
   686  
   687  	t.Run("start and stop, stopped = false", func(t *testing.T) {
   688  
   689  		sc := NewStopControl(
   690  			engine.NewUnit(),
   691  			time.Second,
   692  			unittest.Logger(),
   693  			nil,
   694  			nil,
   695  			nil,
   696  			nil,
   697  			&flow.Header{Height: 1},
   698  			false,
   699  			false,
   700  		)
   701  
   702  		ctx, cancel := context.WithCancel(context.Background())
   703  		ictx := irrecoverable.NewMockSignalerContext(t, ctx)
   704  
   705  		sc.Start(ictx)
   706  
   707  		unittest.AssertClosesBefore(t, sc.Ready(), 10*time.Second)
   708  
   709  		cancel()
   710  
   711  		unittest.AssertClosesBefore(t, sc.Done(), 10*time.Second)
   712  	})
   713  
   714  	t.Run("start as stopped if execution is at version boundary", func(t *testing.T) {
   715  
   716  		headerA := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20))
   717  		headerB := unittest.BlockHeaderWithParentFixture(headerA) // 21
   718  
   719  		versionBeacons := storageMock.NewVersionBeacons(t)
   720  		versionBeacons.On("Highest", headerB.Height).
   721  			Return(&flow.SealedVersionBeacon{
   722  				VersionBeacon: unittest.VersionBeaconFixture(
   723  					unittest.WithBoundaries(
   724  						flow.VersionBoundary{
   725  							BlockHeight: headerB.Height,
   726  							Version:     "2.0.0",
   727  						},
   728  					),
   729  				),
   730  				SealHeight: 1, // sealed in the past
   731  			}, nil).
   732  			Once()
   733  
   734  		execState := mock.NewExecutionState(t)
   735  
   736  		execState.On("IsBlockExecuted", headerA.Height, headerA.ID()).Return(true, nil).Once()
   737  
   738  		headers := &stopControlMockHeaders{
   739  			headers: map[uint64]*flow.Header{
   740  				headerA.Height: headerA,
   741  				headerB.Height: headerB,
   742  			},
   743  		}
   744  
   745  		// This is a likely scenario where the node stopped because of a version
   746  		// boundary but was restarted without being upgraded to the new version.
   747  		// In this case, the node should start as stopped.
   748  		sc := NewStopControl(
   749  			engine.NewUnit(),
   750  			time.Second,
   751  			unittest.Logger(),
   752  			execState,
   753  			headers,
   754  			versionBeacons,
   755  			semver.New("1.0.0"),
   756  			headerB,
   757  			false,
   758  			false,
   759  		)
   760  
   761  		ctx, cancel := context.WithCancel(context.Background())
   762  		ictx := irrecoverable.NewMockSignalerContext(t, ctx)
   763  
   764  		sc.Start(ictx)
   765  
   766  		unittest.AssertClosesBefore(t, sc.Ready(), 10*time.Second)
   767  
   768  		// should start as stopped
   769  		require.True(t, sc.IsExecutionStopped())
   770  		require.Equal(t, StopParameters{
   771  			StopBeforeHeight: headerB.Height,
   772  			ShouldCrash:      false,
   773  		}, sc.GetStopParameters())
   774  
   775  		cancel()
   776  
   777  		unittest.AssertClosesBefore(t, sc.Done(), 10*time.Second)
   778  	})
   779  
   780  	t.Run("test stopping with block finalized events", func(t *testing.T) {
   781  
   782  		headerA := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(20))
   783  		headerB := unittest.BlockHeaderWithParentFixture(headerA) // 21
   784  		headerC := unittest.BlockHeaderWithParentFixture(headerB) // 22
   785  
   786  		vb := &flow.SealedVersionBeacon{
   787  			VersionBeacon: unittest.VersionBeaconFixture(
   788  				unittest.WithBoundaries(
   789  					flow.VersionBoundary{
   790  						BlockHeight: headerC.Height,
   791  						Version:     "2.0.0",
   792  					},
   793  				),
   794  			),
   795  			SealHeight: 1, // sealed in the past
   796  		}
   797  
   798  		versionBeacons := storageMock.NewVersionBeacons(t)
   799  		versionBeacons.On("Highest", headerB.Height).
   800  			Return(vb, nil).
   801  			Once()
   802  		versionBeacons.On("Highest", headerC.Height).
   803  			Return(vb, nil).
   804  			Once()
   805  
   806  		execState := mock.NewExecutionState(t)
   807  		execState.On("IsBlockExecuted", headerB.Height, headerB.ID()).Return(true, nil).Once()
   808  
   809  		headers := &stopControlMockHeaders{
   810  			headers: map[uint64]*flow.Header{
   811  				headerA.Height: headerA,
   812  				headerB.Height: headerB,
   813  				headerC.Height: headerC,
   814  			},
   815  		}
   816  
   817  		// The stop is set by a previous version beacon and is in one blocks time.
   818  		sc := NewStopControl(
   819  			engine.NewUnit(),
   820  			time.Second,
   821  			unittest.Logger(),
   822  			execState,
   823  			headers,
   824  			versionBeacons,
   825  			semver.New("1.0.0"),
   826  			headerB,
   827  			false,
   828  			false,
   829  		)
   830  
   831  		ctx, cancel := context.WithCancel(context.Background())
   832  		ictx := irrecoverable.NewMockSignalerContext(t, ctx)
   833  
   834  		sc.Start(ictx)
   835  
   836  		unittest.AssertClosesBefore(t, sc.Ready(), 10*time.Second)
   837  
   838  		require.False(t, sc.IsExecutionStopped())
   839  		require.Equal(t, StopParameters{
   840  			StopBeforeHeight: headerC.Height,
   841  			ShouldCrash:      false,
   842  		}, sc.GetStopParameters())
   843  
   844  		sc.BlockFinalized(headerC)
   845  
   846  		done := make(chan struct{})
   847  		go func() {
   848  			for !sc.IsExecutionStopped() {
   849  				<-time.After(100 * time.Millisecond)
   850  			}
   851  			close(done)
   852  		}()
   853  
   854  		select {
   855  		case <-done:
   856  		case <-time.After(2 * time.Second):
   857  			t.Fatal("timed out waiting for stop control to stop execution")
   858  		}
   859  
   860  		cancel()
   861  		unittest.AssertClosesBefore(t, sc.Done(), 10*time.Second)
   862  	})
   863  }
   864  
   865  func TestPatchedVersion(t *testing.T) {
   866  	require.True(t, semver.New("0.31.20").LessThan(*semver.New("0.31.21")))
   867  	require.True(t, semver.New("0.31.20-patch.1").LessThan(*semver.New("0.31.20"))) // be careful with this one
   868  	require.True(t, semver.New("0.31.20-without-adx").LessThan(*semver.New("0.31.20")))
   869  
   870  	// a special build created with "+" would not change the version priority for standard and pre-release versions
   871  	require.True(t, semver.New("0.31.20+without-adx").Equal(*semver.New("0.31.20")))
   872  	require.True(t, semver.New("0.31.20-patch.1+without-adx").Equal(*semver.New("0.31.20-patch.1")))
   873  	require.True(t, semver.New("0.31.20+without-netgo-without-adx").Equal(*semver.New("0.31.20")))
   874  	require.True(t, semver.New("0.31.20+arm").Equal(*semver.New("0.31.20")))
   875  }