github.com/palisadeinc/bor@v0.0.0-20230615125219-ab7196213d15/core/tests/blockchain_snapshot_test.go (about)

     1  // Copyright 2020 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  // Tests that abnormal program termination (i.e.crash) and restart can recovery
    18  // the snapshot properly if the snapshot is enabled.
    19  
    20  package tests
    21  
    22  import (
    23  	"bytes"
    24  	"fmt"
    25  	"io/ioutil"
    26  	"math/big"
    27  	"os"
    28  	"strings"
    29  	"testing"
    30  	"time"
    31  
    32  	"github.com/ethereum/go-ethereum/consensus"
    33  	"github.com/ethereum/go-ethereum/consensus/ethash"
    34  	"github.com/ethereum/go-ethereum/core"
    35  	"github.com/ethereum/go-ethereum/core/rawdb"
    36  	"github.com/ethereum/go-ethereum/core/types"
    37  	"github.com/ethereum/go-ethereum/core/vm"
    38  	"github.com/ethereum/go-ethereum/ethdb"
    39  	"github.com/ethereum/go-ethereum/params"
    40  )
    41  
    42  // snapshotTestBasic wraps the common testing fields in the snapshot tests.
    43  type snapshotTestBasic struct {
    44  	chainBlocks   int    // Number of blocks to generate for the canonical chain
    45  	snapshotBlock uint64 // Block number of the relevant snapshot disk layer
    46  	commitBlock   uint64 // Block number for which to commit the state to disk
    47  
    48  	expCanonicalBlocks int    // Number of canonical blocks expected to remain in the database (excl. genesis)
    49  	expHeadHeader      uint64 // Block number of the expected head header
    50  	expHeadFastBlock   uint64 // Block number of the expected head fast sync block
    51  	expHeadBlock       uint64 // Block number of the expected head full block
    52  	expSnapshotBottom  uint64 // The block height corresponding to the snapshot disk layer
    53  
    54  	// share fields, set in runtime
    55  	datadir string
    56  	db      ethdb.Database
    57  	gendb   ethdb.Database
    58  	engine  consensus.Engine
    59  }
    60  
    61  func (basic *snapshotTestBasic) prepare(t *testing.T) (*core.BlockChain, []*types.Block) {
    62  	t.Helper()
    63  
    64  	// Create a temporary persistent database
    65  	datadir, err := ioutil.TempDir("", "")
    66  	if err != nil {
    67  		t.Fatalf("Failed to create temporary datadir: %v", err)
    68  	}
    69  	os.RemoveAll(datadir)
    70  
    71  	db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false)
    72  	if err != nil {
    73  		t.Fatalf("Failed to create persistent database: %v", err)
    74  	}
    75  	// Initialize a fresh chain
    76  	var (
    77  		genesis = (&core.Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db)
    78  		engine  = ethash.NewFullFaker()
    79  		gendb   = rawdb.NewMemoryDatabase()
    80  
    81  		// Snapshot is enabled, the first snapshot is created from the Genesis.
    82  		// The snapshot memory allowance is 256MB, it means no snapshot flush
    83  		// will happen during the block insertion.
    84  		cacheConfig = core.DefaultCacheConfig
    85  	)
    86  
    87  	chain, err := core.NewBlockChain(db, cacheConfig, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil, nil)
    88  	if err != nil {
    89  		t.Fatalf("Failed to create chain: %v", err)
    90  	}
    91  
    92  	blocks, _ := core.GenerateChain(params.TestChainConfig, genesis, engine, gendb, basic.chainBlocks, func(i int, b *core.BlockGen) {})
    93  
    94  	// Insert the blocks with configured settings.
    95  	var breakpoints []uint64
    96  	if basic.commitBlock > basic.snapshotBlock {
    97  		breakpoints = append(breakpoints, basic.snapshotBlock, basic.commitBlock)
    98  	} else {
    99  		breakpoints = append(breakpoints, basic.commitBlock, basic.snapshotBlock)
   100  	}
   101  
   102  	var startPoint uint64
   103  
   104  	for _, point := range breakpoints {
   105  		if _, err := chain.InsertChain(blocks[startPoint:point]); err != nil {
   106  			t.Fatalf("Failed to import canonical chain start: %v", err)
   107  		}
   108  
   109  		startPoint = point
   110  
   111  		if basic.commitBlock > 0 && basic.commitBlock == point {
   112  			err = chain.StateCache().TrieDB().Commit(blocks[point-1].Root(), true, nil)
   113  			if err != nil {
   114  				t.Fatal("on trieDB.Commit", err)
   115  			}
   116  		}
   117  
   118  		if basic.snapshotBlock > 0 && basic.snapshotBlock == point {
   119  			// Flushing the entire snap tree into the disk, the
   120  			// relevant (a) snapshot root and (b) snapshot generator
   121  			// will be persisted atomically.
   122  			err = chain.Snaps().Cap(blocks[point-1].Root(), 0)
   123  			if err != nil {
   124  				t.Fatal("on Snaps.Cap", err)
   125  			}
   126  
   127  			diskRoot, blockRoot := chain.Snaps().DiskRoot(), blocks[point-1].Root()
   128  			if !bytes.Equal(diskRoot.Bytes(), blockRoot.Bytes()) {
   129  				t.Fatalf("Failed to flush disk layer change, want %x, got %x", blockRoot, diskRoot)
   130  			}
   131  		}
   132  	}
   133  
   134  	if _, err := chain.InsertChain(blocks[startPoint:]); err != nil {
   135  		t.Fatalf("Failed to import canonical chain tail: %v", err)
   136  	}
   137  
   138  	// Set runtime fields
   139  	basic.datadir = datadir
   140  	basic.db = db
   141  	basic.gendb = gendb
   142  	basic.engine = engine
   143  
   144  	return chain, blocks
   145  }
   146  
   147  func (basic *snapshotTestBasic) verify(t *testing.T, chain *core.BlockChain, blocks []*types.Block) {
   148  	t.Helper()
   149  
   150  	// Iterate over all the remaining blocks and ensure there are no gaps
   151  	verifyNoGaps(t, chain, true, blocks)
   152  	verifyCutoff(t, chain, true, blocks, basic.expCanonicalBlocks)
   153  
   154  	if head := chain.CurrentHeader(); head.Number.Uint64() != basic.expHeadHeader {
   155  		t.Errorf("Head header mismatch: have %d, want %d", head.Number, basic.expHeadHeader)
   156  	}
   157  
   158  	if head := chain.CurrentFastBlock(); head.NumberU64() != basic.expHeadFastBlock {
   159  		t.Errorf("Head fast block mismatch: have %d, want %d", head.NumberU64(), basic.expHeadFastBlock)
   160  	}
   161  
   162  	if head := chain.CurrentBlock(); head.NumberU64() != basic.expHeadBlock {
   163  		t.Errorf("Head block mismatch: have %d, want %d", head.NumberU64(), basic.expHeadBlock)
   164  	}
   165  
   166  	// Check the disk layer, ensure they are matched
   167  	block := chain.GetBlockByNumber(basic.expSnapshotBottom)
   168  	if block == nil {
   169  		t.Errorf("The correspnding block[%d] of snapshot disk layer is missing", basic.expSnapshotBottom)
   170  	} else if !bytes.Equal(chain.Snaps().DiskRoot().Bytes(), block.Root().Bytes()) {
   171  		t.Errorf("The snapshot disk layer root is incorrect, want %x, get %x", block.Root(), chain.Snaps().DiskRoot())
   172  	}
   173  
   174  	// Check the snapshot, ensure it's integrated
   175  	if err := chain.Snaps().Verify(block.Root()); err != nil {
   176  		t.Errorf("The disk layer is not integrated %v", err)
   177  	}
   178  }
   179  
   180  func (basic *snapshotTestBasic) dump() string {
   181  	buffer := new(strings.Builder)
   182  
   183  	fmt.Fprint(buffer, "Chain:\n  G")
   184  	for i := 0; i < basic.chainBlocks; i++ {
   185  		fmt.Fprintf(buffer, "->C%d", i+1)
   186  	}
   187  	fmt.Fprint(buffer, " (HEAD)\n\n")
   188  
   189  	fmt.Fprintf(buffer, "Commit:   G")
   190  	if basic.commitBlock > 0 {
   191  		fmt.Fprintf(buffer, ", C%d", basic.commitBlock)
   192  	}
   193  	fmt.Fprint(buffer, "\n")
   194  
   195  	fmt.Fprintf(buffer, "Snapshot: G")
   196  	if basic.snapshotBlock > 0 {
   197  		fmt.Fprintf(buffer, ", C%d", basic.snapshotBlock)
   198  	}
   199  	fmt.Fprint(buffer, "\n")
   200  
   201  	//if crash {
   202  	//	fmt.Fprintf(buffer, "\nCRASH\n\n")
   203  	//} else {
   204  	//	fmt.Fprintf(buffer, "\nSetHead(%d)\n\n", basic.setHead)
   205  	//}
   206  	fmt.Fprintf(buffer, "------------------------------\n\n")
   207  
   208  	fmt.Fprint(buffer, "Expected in leveldb:\n  G")
   209  	for i := 0; i < basic.expCanonicalBlocks; i++ {
   210  		fmt.Fprintf(buffer, "->C%d", i+1)
   211  	}
   212  	fmt.Fprintf(buffer, "\n\n")
   213  	fmt.Fprintf(buffer, "Expected head header    : C%d\n", basic.expHeadHeader)
   214  	fmt.Fprintf(buffer, "Expected head fast block: C%d\n", basic.expHeadFastBlock)
   215  	if basic.expHeadBlock == 0 {
   216  		fmt.Fprintf(buffer, "Expected head block     : G\n")
   217  	} else {
   218  		fmt.Fprintf(buffer, "Expected head block     : C%d\n", basic.expHeadBlock)
   219  	}
   220  	if basic.expSnapshotBottom == 0 {
   221  		fmt.Fprintf(buffer, "Expected snapshot disk  : G\n")
   222  	} else {
   223  		fmt.Fprintf(buffer, "Expected snapshot disk  : C%d\n", basic.expSnapshotBottom)
   224  	}
   225  	return buffer.String()
   226  }
   227  
   228  func (basic *snapshotTestBasic) teardown() {
   229  	basic.db.Close()
   230  	basic.gendb.Close()
   231  	os.RemoveAll(basic.datadir)
   232  }
   233  
   234  // snapshotTest is a test case type for normal snapshot recovery.
   235  // It can be used for testing that restart Geth normally.
   236  type snapshotTest struct {
   237  	snapshotTestBasic
   238  }
   239  
   240  func (snaptest *snapshotTest) test(t *testing.T) {
   241  	// It's hard to follow the test case, visualize the input
   242  	// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
   243  	// fmt.Println(tt.dump())
   244  	chain, blocks := snaptest.prepare(t)
   245  
   246  	// Restart the chain normally
   247  	chain.Stop()
   248  
   249  	newchain, err := core.NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, nil)
   250  	if err != nil {
   251  		t.Fatalf("Failed to recreate chain: %v", err)
   252  	}
   253  
   254  	defer newchain.Stop()
   255  
   256  	snaptest.verify(t, newchain, blocks)
   257  }
   258  
   259  // crashSnapshotTest is a test case type for innormal snapshot recovery.
   260  // It can be used for testing that restart Geth after the crash.
   261  type crashSnapshotTest struct {
   262  	snapshotTestBasic
   263  }
   264  
   265  func (snaptest *crashSnapshotTest) test(t *testing.T) {
   266  	// It's hard to follow the test case, visualize the input
   267  	// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
   268  	// fmt.Println(tt.dump())
   269  	chain, blocks := snaptest.prepare(t)
   270  
   271  	// Pull the plug on the database, simulating a hard crash
   272  	db := chain.DB()
   273  	db.Close()
   274  
   275  	// Start a new blockchain back up and see where the repair leads us
   276  	newdb, err := rawdb.NewLevelDBDatabaseWithFreezer(snaptest.datadir, 0, 0, snaptest.datadir, "", false)
   277  	if err != nil {
   278  		t.Fatalf("Failed to reopen persistent database: %v", err)
   279  	}
   280  	defer newdb.Close()
   281  
   282  	// The interesting thing is: instead of starting the blockchain after
   283  	// the crash, we do restart twice here: one after the crash and one
   284  	// after the normal stop. It's used to ensure the broken snapshot
   285  	// can be detected all the time.
   286  	newchain, err := core.NewBlockChain(newdb, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, nil)
   287  	if err != nil {
   288  		t.Fatalf("Failed to recreate chain: %v", err)
   289  	}
   290  	newchain.Stop()
   291  
   292  	newchain, err = core.NewBlockChain(newdb, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, nil)
   293  	if err != nil {
   294  		t.Fatalf("Failed to recreate chain: %v", err)
   295  	}
   296  	defer newchain.Stop()
   297  
   298  	snaptest.verify(t, newchain, blocks)
   299  }
   300  
   301  // gappedSnapshotTest is a test type used to test this scenario:
   302  // - have a complete snapshot
   303  // - restart without enabling the snapshot
   304  // - insert a few blocks
   305  // - restart with enabling the snapshot again
   306  type gappedSnapshotTest struct {
   307  	snapshotTestBasic
   308  	gapped int // Number of blocks to insert without enabling snapshot
   309  }
   310  
   311  func (snaptest *gappedSnapshotTest) test(t *testing.T) {
   312  	// It's hard to follow the test case, visualize the input
   313  	// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
   314  	// fmt.Println(tt.dump())
   315  	chain, blocks := snaptest.prepare(t)
   316  
   317  	// Insert blocks without enabling snapshot if gapping is required.
   318  	chain.Stop()
   319  
   320  	gappedBlocks, _ := core.GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.gendb, snaptest.gapped, func(i int, b *core.BlockGen) {})
   321  
   322  	// Insert a few more blocks without enabling snapshot
   323  	var cacheConfig = &core.CacheConfig{
   324  		TrieCleanLimit: 256,
   325  		TrieDirtyLimit: 256,
   326  		TrieTimeLimit:  5 * time.Minute,
   327  		SnapshotLimit:  0,
   328  	}
   329  
   330  	newchain, err := core.NewBlockChain(snaptest.db, cacheConfig, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, nil)
   331  	if err != nil {
   332  		t.Fatalf("Failed to recreate chain: %v", err)
   333  	}
   334  
   335  	newchain.InsertChain(gappedBlocks)
   336  	newchain.Stop()
   337  
   338  	// Restart the chain with enabling the snapshot
   339  	newchain, err = core.NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, nil)
   340  	if err != nil {
   341  		t.Fatalf("Failed to recreate chain: %v", err)
   342  	}
   343  
   344  	defer newchain.Stop()
   345  
   346  	snaptest.verify(t, newchain, blocks)
   347  }
   348  
   349  // setHeadSnapshotTest is the test type used to test this scenario:
   350  // - have a complete snapshot
   351  // - set the head to a lower point
   352  // - restart
   353  type setHeadSnapshotTest struct {
   354  	snapshotTestBasic
   355  	setHead uint64 // Block number to set head back to
   356  }
   357  
   358  func (snaptest *setHeadSnapshotTest) test(t *testing.T) {
   359  	// It's hard to follow the test case, visualize the input
   360  	// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
   361  	// fmt.Println(tt.dump())
   362  	chain, blocks := snaptest.prepare(t)
   363  
   364  	// Rewind the chain if setHead operation is required.
   365  	chain.SetHead(snaptest.setHead)
   366  	chain.Stop()
   367  
   368  	newchain, err := core.NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, nil)
   369  	if err != nil {
   370  		t.Fatalf("Failed to recreate chain: %v", err)
   371  	}
   372  	defer newchain.Stop()
   373  
   374  	snaptest.verify(t, newchain, blocks)
   375  }
   376  
   377  // restartCrashSnapshotTest is the test type used to test this scenario:
   378  // - have a complete snapshot
   379  // - restart chain
   380  // - insert more blocks with enabling the snapshot
   381  // - commit the snapshot
   382  // - crash
   383  // - restart again
   384  type restartCrashSnapshotTest struct {
   385  	snapshotTestBasic
   386  	newBlocks int
   387  }
   388  
   389  func (snaptest *restartCrashSnapshotTest) test(t *testing.T) {
   390  	// It's hard to follow the test case, visualize the input
   391  	// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
   392  	// fmt.Println(tt.dump())
   393  	chain, blocks := snaptest.prepare(t)
   394  
   395  	// Firstly, stop the chain properly, with all snapshot journal
   396  	// and state committed.
   397  	chain.Stop()
   398  
   399  	newchain, err := core.NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, nil)
   400  	if err != nil {
   401  		t.Fatalf("Failed to recreate chain: %v", err)
   402  	}
   403  
   404  	newBlocks, _ := core.GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.gendb, snaptest.newBlocks, func(i int, b *core.BlockGen) {})
   405  	newchain.InsertChain(newBlocks)
   406  
   407  	// Commit the entire snapshot into the disk if requested. Note only
   408  	// (a) snapshot root and (b) snapshot generator will be committed,
   409  	// the diff journal is not.
   410  	newchain.Snapshots().Cap(newBlocks[len(newBlocks)-1].Root(), 0)
   411  
   412  	// Simulate the blockchain crash
   413  	// Don't call chain.Stop here, so that no snapshot
   414  	// journal and latest state will be committed
   415  
   416  	// Restart the chain after the crash
   417  	newchain, err = core.NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, nil)
   418  	if err != nil {
   419  		t.Fatalf("Failed to recreate chain: %v", err)
   420  	}
   421  	defer newchain.Stop()
   422  
   423  	snaptest.verify(t, newchain, blocks)
   424  }
   425  
   426  // wipeCrashSnapshotTest is the test type used to test this scenario:
   427  // - have a complete snapshot
   428  // - restart, insert more blocks without enabling the snapshot
   429  // - restart again with enabling the snapshot
   430  // - crash
   431  type wipeCrashSnapshotTest struct {
   432  	snapshotTestBasic
   433  	newBlocks int
   434  }
   435  
   436  func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) {
   437  	// It's hard to follow the test case, visualize the input
   438  	// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
   439  	// fmt.Println(tt.dump())
   440  	chain, blocks := snaptest.prepare(t)
   441  
   442  	// Firstly, stop the chain properly, with all snapshot journal
   443  	// and state committed.
   444  	chain.Stop()
   445  
   446  	config := &core.CacheConfig{
   447  		TrieCleanLimit: 256,
   448  		TrieDirtyLimit: 256,
   449  		TrieTimeLimit:  5 * time.Minute,
   450  		SnapshotLimit:  0,
   451  	}
   452  	newchain, err := core.NewBlockChain(snaptest.db, config, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, nil)
   453  	if err != nil {
   454  		t.Fatalf("Failed to recreate chain: %v", err)
   455  	}
   456  
   457  	newBlocks, _ := core.GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.gendb, snaptest.newBlocks, func(i int, b *core.BlockGen) {})
   458  	newchain.InsertChain(newBlocks)
   459  	newchain.Stop()
   460  
   461  	// Restart the chain, the wiper should starts working
   462  	config = &core.CacheConfig{
   463  		TrieCleanLimit: 256,
   464  		TrieDirtyLimit: 256,
   465  		TrieTimeLimit:  5 * time.Minute,
   466  		SnapshotLimit:  256,
   467  		SnapshotWait:   false, // Don't wait rebuild
   468  	}
   469  	_, err = core.NewBlockChain(snaptest.db, config, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, nil)
   470  	if err != nil {
   471  		t.Fatalf("Failed to recreate chain: %v", err)
   472  	}
   473  	// Simulate the blockchain crash.
   474  
   475  	newchain, err = core.NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil, nil)
   476  	if err != nil {
   477  		t.Fatalf("Failed to recreate chain: %v", err)
   478  	}
   479  
   480  	snaptest.verify(t, newchain, blocks)
   481  }
   482  
   483  // Tests a Geth restart with valid snapshot. Before the shutdown, all snapshot
   484  // journal will be persisted correctly. In this case no snapshot recovery is
   485  // required.
   486  func TestRestartWithNewSnapshot(t *testing.T) {
   487  	// Chain:
   488  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   489  	//
   490  	// Commit:   G
   491  	// Snapshot: G
   492  	//
   493  	// SetHead(0)
   494  	//
   495  	// ------------------------------
   496  	//
   497  	// Expected in leveldb:
   498  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   499  	//
   500  	// Expected head header    : C8
   501  	// Expected head fast block: C8
   502  	// Expected head block     : C8
   503  	// Expected snapshot disk  : G
   504  	test := &snapshotTest{
   505  		snapshotTestBasic{
   506  			chainBlocks:        8,
   507  			snapshotBlock:      0,
   508  			commitBlock:        0,
   509  			expCanonicalBlocks: 8,
   510  			expHeadHeader:      8,
   511  			expHeadFastBlock:   8,
   512  			expHeadBlock:       8,
   513  			expSnapshotBottom:  0, // Initial disk layer built from genesis
   514  		},
   515  	}
   516  	test.test(t)
   517  	test.teardown()
   518  }
   519  
   520  // Tests a Geth was crashed and restarts with a broken snapshot. In this case the
   521  // chain head should be rewound to the point with available state. And also the
   522  // new head should must be lower than disk layer. But there is no committed point
   523  // so the chain should be rewound to genesis and the disk layer should be left
   524  // for recovery.
   525  func TestNoCommitCrashWithNewSnapshot(t *testing.T) {
   526  	// Chain:
   527  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   528  	//
   529  	// Commit:   G
   530  	// Snapshot: G, C4
   531  	//
   532  	// CRASH
   533  	//
   534  	// ------------------------------
   535  	//
   536  	// Expected in leveldb:
   537  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   538  	//
   539  	// Expected head header    : C8
   540  	// Expected head fast block: C8
   541  	// Expected head block     : G
   542  	// Expected snapshot disk  : C4
   543  	test := &crashSnapshotTest{
   544  		snapshotTestBasic{
   545  			chainBlocks:        8,
   546  			snapshotBlock:      4,
   547  			commitBlock:        0,
   548  			expCanonicalBlocks: 8,
   549  			expHeadHeader:      8,
   550  			expHeadFastBlock:   8,
   551  			expHeadBlock:       0,
   552  			expSnapshotBottom:  4, // Last committed disk layer, wait recovery
   553  		},
   554  	}
   555  	test.test(t)
   556  	test.teardown()
   557  }
   558  
   559  // Tests a Geth was crashed and restarts with a broken snapshot. In this case the
   560  // chain head should be rewound to the point with available state. And also the
   561  // new head should must be lower than disk layer. But there is only a low committed
   562  // point so the chain should be rewound to committed point and the disk layer
   563  // should be left for recovery.
   564  func TestLowCommitCrashWithNewSnapshot(t *testing.T) {
   565  	// Chain:
   566  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   567  	//
   568  	// Commit:   G, C2
   569  	// Snapshot: G, C4
   570  	//
   571  	// CRASH
   572  	//
   573  	// ------------------------------
   574  	//
   575  	// Expected in leveldb:
   576  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   577  	//
   578  	// Expected head header    : C8
   579  	// Expected head fast block: C8
   580  	// Expected head block     : C2
   581  	// Expected snapshot disk  : C4
   582  	test := &crashSnapshotTest{
   583  		snapshotTestBasic{
   584  			chainBlocks:        8,
   585  			snapshotBlock:      4,
   586  			commitBlock:        2,
   587  			expCanonicalBlocks: 8,
   588  			expHeadHeader:      8,
   589  			expHeadFastBlock:   8,
   590  			expHeadBlock:       2,
   591  			expSnapshotBottom:  4, // Last committed disk layer, wait recovery
   592  		},
   593  	}
   594  	test.test(t)
   595  	test.teardown()
   596  }
   597  
   598  // Tests a Geth was crashed and restarts with a broken snapshot. In this case
   599  // the chain head should be rewound to the point with available state. And also
   600  // the new head should must be lower than disk layer. But there is only a high
   601  // committed point so the chain should be rewound to genesis and the disk layer
   602  // should be left for recovery.
   603  func TestHighCommitCrashWithNewSnapshot(t *testing.T) {
   604  	// Chain:
   605  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   606  	//
   607  	// Commit:   G, C6
   608  	// Snapshot: G, C4
   609  	//
   610  	// CRASH
   611  	//
   612  	// ------------------------------
   613  	//
   614  	// Expected in leveldb:
   615  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   616  	//
   617  	// Expected head header    : C8
   618  	// Expected head fast block: C8
   619  	// Expected head block     : G
   620  	// Expected snapshot disk  : C4
   621  	test := &crashSnapshotTest{
   622  		snapshotTestBasic{
   623  			chainBlocks:        8,
   624  			snapshotBlock:      4,
   625  			commitBlock:        6,
   626  			expCanonicalBlocks: 8,
   627  			expHeadHeader:      8,
   628  			expHeadFastBlock:   8,
   629  			expHeadBlock:       0,
   630  			expSnapshotBottom:  4, // Last committed disk layer, wait recovery
   631  		},
   632  	}
   633  	test.test(t)
   634  	test.teardown()
   635  }
   636  
   637  // Tests a Geth was running with snapshot enabled. Then restarts without
   638  // enabling snapshot and after that re-enable the snapshot again. In this
   639  // case the snapshot should be rebuilt with latest chain head.
   640  func TestGappedNewSnapshot(t *testing.T) {
   641  	// Chain:
   642  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   643  	//
   644  	// Commit:   G
   645  	// Snapshot: G
   646  	//
   647  	// SetHead(0)
   648  	//
   649  	// ------------------------------
   650  	//
   651  	// Expected in leveldb:
   652  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10
   653  	//
   654  	// Expected head header    : C10
   655  	// Expected head fast block: C10
   656  	// Expected head block     : C10
   657  	// Expected snapshot disk  : C10
   658  	test := &gappedSnapshotTest{
   659  		snapshotTestBasic: snapshotTestBasic{
   660  			chainBlocks:        8,
   661  			snapshotBlock:      0,
   662  			commitBlock:        0,
   663  			expCanonicalBlocks: 10,
   664  			expHeadHeader:      10,
   665  			expHeadFastBlock:   10,
   666  			expHeadBlock:       10,
   667  			expSnapshotBottom:  10, // Rebuilt snapshot from the latest HEAD
   668  		},
   669  		gapped: 2,
   670  	}
   671  	test.test(t)
   672  	test.teardown()
   673  }
   674  
   675  // Tests the Geth was running with snapshot enabled and resetHead is applied.
   676  // In this case the head is rewound to the target(with state available). After
   677  // that the chain is restarted and the original disk layer is kept.
   678  func TestSetHeadWithNewSnapshot(t *testing.T) {
   679  	// Chain:
   680  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   681  	//
   682  	// Commit:   G
   683  	// Snapshot: G
   684  	//
   685  	// SetHead(4)
   686  	//
   687  	// ------------------------------
   688  	//
   689  	// Expected in leveldb:
   690  	//   G->C1->C2->C3->C4
   691  	//
   692  	// Expected head header    : C4
   693  	// Expected head fast block: C4
   694  	// Expected head block     : C4
   695  	// Expected snapshot disk  : G
   696  	test := &setHeadSnapshotTest{
   697  		snapshotTestBasic: snapshotTestBasic{
   698  			chainBlocks:        8,
   699  			snapshotBlock:      0,
   700  			commitBlock:        0,
   701  			expCanonicalBlocks: 4,
   702  			expHeadHeader:      4,
   703  			expHeadFastBlock:   4,
   704  			expHeadBlock:       4,
   705  			expSnapshotBottom:  0, // The initial disk layer is built from the genesis
   706  		},
   707  		setHead: 4,
   708  	}
   709  	test.test(t)
   710  	test.teardown()
   711  }
   712  
   713  // Tests the Geth was running with a complete snapshot and then imports a few
   714  // more new blocks on top without enabling the snapshot. After the restart,
   715  // crash happens. Check everything is ok after the restart.
   716  func TestRecoverSnapshotFromWipingCrash(t *testing.T) {
   717  	// Chain:
   718  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   719  	//
   720  	// Commit:   G
   721  	// Snapshot: G
   722  	//
   723  	// SetHead(0)
   724  	//
   725  	// ------------------------------
   726  	//
   727  	// Expected in leveldb:
   728  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10
   729  	//
   730  	// Expected head header    : C10
   731  	// Expected head fast block: C10
   732  	// Expected head block     : C8
   733  	// Expected snapshot disk  : C10
   734  	test := &wipeCrashSnapshotTest{
   735  		snapshotTestBasic: snapshotTestBasic{
   736  			chainBlocks:        8,
   737  			snapshotBlock:      4,
   738  			commitBlock:        0,
   739  			expCanonicalBlocks: 10,
   740  			expHeadHeader:      10,
   741  			expHeadFastBlock:   10,
   742  			expHeadBlock:       10,
   743  			expSnapshotBottom:  10,
   744  		},
   745  		newBlocks: 2,
   746  	}
   747  	test.test(t)
   748  	test.teardown()
   749  }