github.com/ethereum-optimism/optimism@v1.7.2/op-node/rollup/driver/sequencer_test.go (about) 1 package driver 2 3 import ( 4 "context" 5 crand "crypto/rand" 6 "encoding/binary" 7 "errors" 8 "fmt" 9 "math/big" 10 "math/rand" 11 "testing" 12 "time" 13 14 "github.com/ethereum/go-ethereum/common" 15 "github.com/ethereum/go-ethereum/core/types" 16 "github.com/ethereum/go-ethereum/log" 17 "github.com/stretchr/testify/require" 18 19 "github.com/ethereum-optimism/optimism/op-node/metrics" 20 "github.com/ethereum-optimism/optimism/op-node/rollup" 21 "github.com/ethereum-optimism/optimism/op-node/rollup/async" 22 "github.com/ethereum-optimism/optimism/op-node/rollup/conductor" 23 "github.com/ethereum-optimism/optimism/op-node/rollup/derive" 24 "github.com/ethereum-optimism/optimism/op-service/eth" 25 "github.com/ethereum-optimism/optimism/op-service/testlog" 26 "github.com/ethereum-optimism/optimism/op-service/testutils" 27 ) 28 29 var mockResetErr = fmt.Errorf("mock reset err: %w", derive.ErrReset) 30 31 type FakeEngineControl struct { 32 finalized eth.L2BlockRef 33 safe eth.L2BlockRef 34 unsafe eth.L2BlockRef 35 36 buildingOnto eth.L2BlockRef 37 buildingID eth.PayloadID 38 buildingSafe bool 39 40 buildingAttrs *eth.PayloadAttributes 41 buildingStart time.Time 42 43 cfg *rollup.Config 44 45 timeNow func() time.Time 46 47 makePayload func(onto eth.L2BlockRef, attrs *eth.PayloadAttributes) *eth.ExecutionPayload 48 49 errTyp derive.BlockInsertionErrType 50 err error 51 52 totalBuildingTime time.Duration 53 totalBuiltBlocks int 54 totalTxs int 55 } 56 57 func (m *FakeEngineControl) avgBuildingTime() time.Duration { 58 return m.totalBuildingTime / time.Duration(m.totalBuiltBlocks) 59 } 60 61 func (m *FakeEngineControl) avgTxsPerBlock() float64 { 62 return float64(m.totalTxs) / float64(m.totalBuiltBlocks) 63 } 64 65 func (m *FakeEngineControl) StartPayload(ctx context.Context, parent eth.L2BlockRef, attrs *derive.AttributesWithParent, updateSafe bool) (errType derive.BlockInsertionErrType, err error) { 66 if m.err != nil { 67 return m.errTyp, m.err 68 } 69 m.buildingID = eth.PayloadID{} 70 _, _ = crand.Read(m.buildingID[:]) 71 m.buildingOnto = parent 72 m.buildingSafe = updateSafe 73 m.buildingAttrs = attrs.Attributes() 74 m.buildingStart = m.timeNow() 75 return derive.BlockInsertOK, nil 76 } 77 78 func (m *FakeEngineControl) ConfirmPayload(ctx context.Context, agossip async.AsyncGossiper, sequencerConductor conductor.SequencerConductor) (out *eth.ExecutionPayloadEnvelope, errTyp derive.BlockInsertionErrType, err error) { 79 if m.err != nil { 80 return nil, m.errTyp, m.err 81 } 82 buildTime := m.timeNow().Sub(m.buildingStart) 83 m.totalBuildingTime += buildTime 84 m.totalBuiltBlocks += 1 85 payload := m.makePayload(m.buildingOnto, m.buildingAttrs) 86 ref, err := derive.PayloadToBlockRef(m.cfg, payload) 87 if err != nil { 88 panic(err) 89 } 90 m.unsafe = ref 91 if m.buildingSafe { 92 m.safe = ref 93 } 94 95 m.resetBuildingState() 96 m.totalTxs += len(payload.Transactions) 97 return ð.ExecutionPayloadEnvelope{ExecutionPayload: payload}, derive.BlockInsertOK, nil 98 } 99 100 func (m *FakeEngineControl) CancelPayload(ctx context.Context, force bool) error { 101 if force { 102 m.resetBuildingState() 103 } 104 return m.err 105 } 106 107 func (m *FakeEngineControl) BuildingPayload() (onto eth.L2BlockRef, id eth.PayloadID, safe bool) { 108 return m.buildingOnto, m.buildingID, m.buildingSafe 109 } 110 111 func (m *FakeEngineControl) Finalized() eth.L2BlockRef { 112 return m.finalized 113 } 114 115 func (m *FakeEngineControl) UnsafeL2Head() eth.L2BlockRef { 116 return m.unsafe 117 } 118 119 func (m *FakeEngineControl) SafeL2Head() eth.L2BlockRef { 120 return m.safe 121 } 122 123 func (m *FakeEngineControl) resetBuildingState() { 124 m.buildingID = eth.PayloadID{} 125 m.buildingOnto = eth.L2BlockRef{} 126 m.buildingSafe = false 127 m.buildingAttrs = nil 128 } 129 130 var _ derive.EngineControl = (*FakeEngineControl)(nil) 131 132 type testAttrBuilderFn func(ctx context.Context, l2Parent eth.L2BlockRef, epoch eth.BlockID) (attrs *eth.PayloadAttributes, err error) 133 134 func (fn testAttrBuilderFn) PreparePayloadAttributes(ctx context.Context, l2Parent eth.L2BlockRef, epoch eth.BlockID) (attrs *eth.PayloadAttributes, err error) { 135 return fn(ctx, l2Parent, epoch) 136 } 137 138 var _ derive.AttributesBuilder = (testAttrBuilderFn)(nil) 139 140 type testOriginSelectorFn func(ctx context.Context, l2Head eth.L2BlockRef) (eth.L1BlockRef, error) 141 142 func (fn testOriginSelectorFn) FindL1Origin(ctx context.Context, l2Head eth.L2BlockRef) (eth.L1BlockRef, error) { 143 return fn(ctx, l2Head) 144 } 145 146 var _ L1OriginSelectorIface = (testOriginSelectorFn)(nil) 147 148 // TestSequencerChaosMonkey runs the sequencer in a mocked adversarial environment with 149 // repeated random errors in dependencies and poor clock timing. 150 // At the end the health of the chain is checked to show that the sequencer kept the chain in shape. 151 func TestSequencerChaosMonkey(t *testing.T) { 152 mockL1Hash := func(num uint64) (out common.Hash) { 153 out[31] = 1 154 binary.BigEndian.PutUint64(out[:], num) 155 return 156 } 157 mockL2Hash := func(num uint64) (out common.Hash) { 158 out[31] = 2 159 binary.BigEndian.PutUint64(out[:], num) 160 return 161 } 162 mockL1ID := func(num uint64) eth.BlockID { 163 return eth.BlockID{Hash: mockL1Hash(num), Number: num} 164 } 165 mockL2ID := func(num uint64) eth.BlockID { 166 return eth.BlockID{Hash: mockL2Hash(num), Number: num} 167 } 168 169 rng := rand.New(rand.NewSource(12345)) 170 171 l1Time := uint64(100000) 172 173 // mute errors. We expect a lot of the mocked errors to cause error-logs. We check chain health at the end of the test. 174 log := testlog.Logger(t, log.LevelCrit) 175 176 cfg := &rollup.Config{ 177 Genesis: rollup.Genesis{ 178 L1: mockL1ID(100000), 179 L2: mockL2ID(200000), 180 L2Time: l1Time + 300, // L2 may start with a relative old L1 origin and will have to catch it up 181 SystemConfig: eth.SystemConfig{}, 182 }, 183 BlockTime: 2, 184 MaxSequencerDrift: 30, 185 } 186 // keep track of the L1 timestamps we mock because sometimes we only have the L1 hash/num handy 187 l1Times := map[eth.BlockID]uint64{cfg.Genesis.L1: l1Time} 188 189 genesisL2 := eth.L2BlockRef{ 190 Hash: cfg.Genesis.L2.Hash, 191 Number: cfg.Genesis.L2.Number, 192 ParentHash: mockL2Hash(cfg.Genesis.L2.Number - 1), 193 Time: cfg.Genesis.L2Time, 194 L1Origin: cfg.Genesis.L1, 195 SequenceNumber: 0, 196 } 197 // initialize our engine state 198 engControl := &FakeEngineControl{ 199 finalized: genesisL2, 200 safe: genesisL2, 201 unsafe: genesisL2, 202 cfg: cfg, 203 } 204 205 // start wallclock at 5 minutes after the current L2 head. The sequencer has some catching up to do! 206 clockTime := time.Unix(int64(engControl.unsafe.Time)+5*60, 0) 207 clockFn := func() time.Time { 208 return clockTime 209 } 210 engControl.timeNow = clockFn 211 212 // mock payload building, we don't need to process any real txs. 213 engControl.makePayload = func(onto eth.L2BlockRef, attrs *eth.PayloadAttributes) *eth.ExecutionPayload { 214 txs := make([]eth.Data, 0) 215 txs = append(txs, attrs.Transactions...) // include deposits 216 if !attrs.NoTxPool { // if we are allowed to sequence from tx pool, mock some txs 217 n := rng.Intn(20) 218 for i := 0; i < n; i++ { 219 txs = append(txs, []byte(fmt.Sprintf("mock sequenced tx %d", i))) 220 } 221 } 222 return ð.ExecutionPayload{ 223 ParentHash: onto.Hash, 224 BlockNumber: eth.Uint64Quantity(onto.Number) + 1, 225 Timestamp: attrs.Timestamp, 226 BlockHash: mockL2Hash(onto.Number), 227 Transactions: txs, 228 } 229 } 230 231 // We keep attribute building simple, we don't talk to a real execution engine in this test. 232 // Sometimes we fake an error in the attributes preparation. 233 var attrsErr error 234 attrBuilder := testAttrBuilderFn(func(ctx context.Context, l2Parent eth.L2BlockRef, epoch eth.BlockID) (attrs *eth.PayloadAttributes, err error) { 235 if attrsErr != nil { 236 return nil, attrsErr 237 } 238 seqNr := l2Parent.SequenceNumber + 1 239 if epoch != l2Parent.L1Origin { 240 seqNr = 0 241 } 242 l1Info := &testutils.MockBlockInfo{ 243 InfoHash: epoch.Hash, 244 InfoParentHash: mockL1Hash(epoch.Number - 1), 245 InfoCoinbase: common.Address{}, 246 InfoRoot: common.Hash{}, 247 InfoNum: epoch.Number, 248 InfoTime: l1Times[epoch], 249 InfoMixDigest: [32]byte{}, 250 InfoBaseFee: big.NewInt(1234), 251 InfoReceiptRoot: common.Hash{}, 252 } 253 infoDep, err := derive.L1InfoDepositBytes(cfg, cfg.Genesis.SystemConfig, seqNr, l1Info, 0) 254 require.NoError(t, err) 255 256 testGasLimit := eth.Uint64Quantity(10_000_000) 257 return ð.PayloadAttributes{ 258 Timestamp: eth.Uint64Quantity(l2Parent.Time + cfg.BlockTime), 259 PrevRandao: eth.Bytes32{}, 260 SuggestedFeeRecipient: common.Address{}, 261 Transactions: []eth.Data{infoDep}, 262 NoTxPool: false, 263 GasLimit: &testGasLimit, 264 }, nil 265 }) 266 267 maxL1BlockTimeGap := uint64(100) 268 // The origin selector just generates random L1 blocks based on RNG 269 var originErr error 270 originSelector := testOriginSelectorFn(func(ctx context.Context, l2Head eth.L2BlockRef) (eth.L1BlockRef, error) { 271 if originErr != nil { 272 return eth.L1BlockRef{}, originErr 273 } 274 origin := eth.L1BlockRef{ 275 Hash: mockL1Hash(l2Head.L1Origin.Number), 276 Number: l2Head.L1Origin.Number, 277 ParentHash: mockL1Hash(l2Head.L1Origin.Number), 278 Time: l1Times[l2Head.L1Origin], 279 } 280 // randomly make a L1 origin appear, if we can even select it 281 nextL2Time := l2Head.Time + cfg.BlockTime 282 if nextL2Time <= origin.Time { 283 return origin, nil 284 } 285 maxTimeIncrement := nextL2Time - origin.Time 286 if maxTimeIncrement > maxL1BlockTimeGap { 287 maxTimeIncrement = maxL1BlockTimeGap 288 } 289 if rng.Intn(10) == 0 { 290 nextOrigin := eth.L1BlockRef{ 291 Hash: mockL1Hash(origin.Number + 1), 292 Number: origin.Number + 1, 293 ParentHash: origin.Hash, 294 Time: origin.Time + 1 + uint64(rng.Int63n(int64(maxTimeIncrement))), 295 } 296 l1Times[nextOrigin.ID()] = nextOrigin.Time 297 return nextOrigin, nil 298 } else { 299 return origin, nil 300 } 301 }) 302 303 seq := NewSequencer(log, cfg, engControl, attrBuilder, originSelector, metrics.NoopMetrics) 304 seq.timeNow = clockFn 305 306 // try to build 1000 blocks, with 5x as many planning attempts, to handle errors and clock problems 307 desiredBlocks := 1000 308 for i := 0; i < 5*desiredBlocks && engControl.totalBuiltBlocks < desiredBlocks; i++ { 309 delta := seq.PlanNextSequencerAction() 310 311 x := rng.Float32() 312 if x < 0.01 { // 1%: mess a lot with the clock: simulate a hang of up to 30 seconds 313 if i < desiredBlocks/2 { // only in first 50% of blocks to let it heal, hangs take time 314 delta = time.Duration(rng.Float64() * float64(time.Second*30)) 315 } 316 } else if x < 0.1 { // 9%: mess with the timing, -50% to 50% off 317 delta = time.Duration((0.5 + rng.Float64()) * float64(delta)) 318 } else if x < 0.5 { 319 // 40%: mess slightly with the timing, -10% to 10% off 320 delta = time.Duration((0.9 + rng.Float64()*0.2) * float64(delta)) 321 } 322 clockTime = clockTime.Add(delta) 323 324 // reset errors 325 originErr = nil 326 attrsErr = nil 327 if engControl.err != mockResetErr { // the mockResetErr requires the sequencer to Reset() to recover. 328 engControl.err = nil 329 } 330 engControl.errTyp = derive.BlockInsertOK 331 332 // maybe make something maybe fail, or try a new L1 origin 333 switch rng.Intn(20) { // 9/20 = 45% chance to fail sequencer action (!!!) 334 case 0, 1: 335 originErr = errors.New("mock origin error") 336 case 2, 3: 337 attrsErr = errors.New("mock attributes error") 338 case 4, 5: 339 engControl.err = errors.New("mock temporary engine error") 340 engControl.errTyp = derive.BlockInsertTemporaryErr 341 case 6, 7: 342 engControl.err = errors.New("mock prestate engine error") 343 engControl.errTyp = derive.BlockInsertPrestateErr 344 case 8: 345 engControl.err = mockResetErr 346 default: 347 // no error 348 } 349 payload, err := seq.RunNextSequencerAction(context.Background(), async.NoOpGossiper{}, &conductor.NoOpConductor{}) 350 // RunNextSequencerAction passes ErrReset & ErrCritical through. 351 // Only suppress ErrReset, not ErrCritical 352 if !errors.Is(err, derive.ErrReset) { 353 require.NoError(t, err) 354 } 355 if payload != nil { 356 require.Equal(t, engControl.UnsafeL2Head().ID(), payload.ExecutionPayload.ID(), "head must stay in sync with emitted payloads") 357 var tx types.Transaction 358 require.NoError(t, tx.UnmarshalBinary(payload.ExecutionPayload.Transactions[0])) 359 info, err := derive.L1BlockInfoFromBytes(cfg, uint64(payload.ExecutionPayload.Timestamp), tx.Data()) 360 require.NoError(t, err) 361 require.GreaterOrEqual(t, uint64(payload.ExecutionPayload.Timestamp), info.Time, "ensure L2 time >= L1 time") 362 } 363 } 364 365 // Now, even though: 366 // - the start state was behind the wallclock 367 // - the L1 origin was far behind the L2 368 // - we made all components fail at random 369 // - messed with the clock 370 // the L2 chain was still built and stats are healthy on average! 371 l2Head := engControl.UnsafeL2Head() 372 t.Logf("avg build time: %s, clock timestamp: %d, L2 head time: %d, L1 origin time: %d, avg txs per block: %f", engControl.avgBuildingTime(), clockFn().Unix(), l2Head.Time, l1Times[l2Head.L1Origin], engControl.avgTxsPerBlock()) 373 require.Equal(t, engControl.totalBuiltBlocks, desiredBlocks, "persist through random errors and build the desired blocks") 374 require.Equal(t, l2Head.Time, cfg.Genesis.L2Time+uint64(desiredBlocks)*cfg.BlockTime, "reached desired L2 block timestamp") 375 require.GreaterOrEqual(t, l2Head.Time, l1Times[l2Head.L1Origin], "the L2 time >= the L1 time") 376 require.Less(t, l2Head.Time-l1Times[l2Head.L1Origin], uint64(100), "The L1 origin time is close to the L2 time") 377 require.Less(t, clockTime.Sub(time.Unix(int64(l2Head.Time), 0)).Abs(), 2*time.Second, "L2 time is accurate, within 2 seconds of wallclock") 378 require.Greater(t, engControl.avgBuildingTime(), time.Second, "With 2 second block time and 1 second error backoff and healthy-on-average errors, building time should at least be a second") 379 require.Greater(t, engControl.avgTxsPerBlock(), 3.0, "We expect at least 1 system tx per block, but with a mocked 0-10 txs we expect an higher avg") 380 }