github.com/haliliceylan/bsc@v1.1.10-0.20220501224556-eb78d644ebcb/core/blockchain_snapshot_test.go (about)

     1  // Copyright 2020 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  // Tests that abnormal program termination (i.e.crash) and restart can recovery
    18  // the snapshot properly if the snapshot is enabled.
    19  
    20  package core
    21  
    22  import (
    23  	"bytes"
    24  	"fmt"
    25  	"io/ioutil"
    26  	"os"
    27  	"strings"
    28  	"testing"
    29  	"time"
    30  
    31  	"github.com/ethereum/go-ethereum/consensus"
    32  	"github.com/ethereum/go-ethereum/consensus/ethash"
    33  	"github.com/ethereum/go-ethereum/core/rawdb"
    34  	"github.com/ethereum/go-ethereum/core/types"
    35  	"github.com/ethereum/go-ethereum/core/vm"
    36  	"github.com/ethereum/go-ethereum/ethdb"
    37  	"github.com/ethereum/go-ethereum/params"
    38  )
    39  
    40  // snapshotTestBasic wraps the common testing fields in the snapshot tests.
    41  type snapshotTestBasic struct {
    42  	chainBlocks   int    // Number of blocks to generate for the canonical chain
    43  	snapshotBlock uint64 // Block number of the relevant snapshot disk layer
    44  	commitBlock   uint64 // Block number for which to commit the state to disk
    45  
    46  	expCanonicalBlocks int    // Number of canonical blocks expected to remain in the database (excl. genesis)
    47  	expHeadHeader      uint64 // Block number of the expected head header
    48  	expHeadFastBlock   uint64 // Block number of the expected head fast sync block
    49  	expHeadBlock       uint64 // Block number of the expected head full block
    50  	expSnapshotBottom  uint64 // The block height corresponding to the snapshot disk layer
    51  
    52  	// share fields, set in runtime
    53  	datadir string
    54  	db      ethdb.Database
    55  	gendb   ethdb.Database
    56  	engine  consensus.Engine
    57  }
    58  
    59  func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Block) {
    60  	// Create a temporary persistent database
    61  	datadir, err := ioutil.TempDir("", "")
    62  	if err != nil {
    63  		t.Fatalf("Failed to create temporary datadir: %v", err)
    64  	}
    65  	os.RemoveAll(datadir)
    66  
    67  	db, err := rawdb.NewLevelDBDatabaseWithFreezer(datadir, 0, 0, datadir, "", false, false, false)
    68  	if err != nil {
    69  		t.Fatalf("Failed to create persistent database: %v", err)
    70  	}
    71  	// Initialize a fresh chain
    72  	var (
    73  		genesis = new(Genesis).MustCommit(db)
    74  		engine  = ethash.NewFullFaker()
    75  		gendb   = rawdb.NewMemoryDatabase()
    76  
    77  		// Snapshot is enabled, the first snapshot is created from the Genesis.
    78  		// The snapshot memory allowance is 256MB, it means no snapshot flush
    79  		// will happen during the block insertion.
    80  		cacheConfig = defaultCacheConfig
    81  	)
    82  	chain, err := NewBlockChain(db, cacheConfig, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil)
    83  	if err != nil {
    84  		t.Fatalf("Failed to create chain: %v", err)
    85  	}
    86  	blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, gendb, basic.chainBlocks, func(i int, b *BlockGen) {})
    87  
    88  	// Insert the blocks with configured settings.
    89  	var breakpoints []uint64
    90  	if basic.commitBlock > basic.snapshotBlock {
    91  		breakpoints = append(breakpoints, basic.snapshotBlock, basic.commitBlock)
    92  	} else {
    93  		breakpoints = append(breakpoints, basic.commitBlock, basic.snapshotBlock)
    94  	}
    95  	var startPoint uint64
    96  	for _, point := range breakpoints {
    97  		if _, err := chain.InsertChain(blocks[startPoint:point]); err != nil {
    98  			t.Fatalf("Failed to import canonical chain start: %v", err)
    99  		}
   100  		startPoint = point
   101  
   102  		if basic.commitBlock > 0 && basic.commitBlock == point {
   103  			chain.stateCache.TrieDB().Commit(blocks[point-1].Root(), true, nil)
   104  		}
   105  		if basic.snapshotBlock > 0 && basic.snapshotBlock == point {
   106  			// Flushing the entire snap tree into the disk, the
   107  			// relavant (a) snapshot root and (b) snapshot generator
   108  			// will be persisted atomically.
   109  			chain.snaps.Cap(blocks[point-1].Root(), 0)
   110  			diskRoot, blockRoot := chain.snaps.DiskRoot(), blocks[point-1].Root()
   111  			if !bytes.Equal(diskRoot.Bytes(), blockRoot.Bytes()) {
   112  				t.Fatalf("Failed to flush disk layer change, want %x, got %x", blockRoot, diskRoot)
   113  			}
   114  		}
   115  	}
   116  	if _, err := chain.InsertChain(blocks[startPoint:]); err != nil {
   117  		t.Fatalf("Failed to import canonical chain tail: %v", err)
   118  	}
   119  
   120  	// Set runtime fields
   121  	basic.datadir = datadir
   122  	basic.db = db
   123  	basic.gendb = gendb
   124  	basic.engine = engine
   125  	return chain, blocks
   126  }
   127  
   128  func (basic *snapshotTestBasic) verify(t *testing.T, chain *BlockChain, blocks []*types.Block) {
   129  	// Iterate over all the remaining blocks and ensure there are no gaps
   130  	verifyNoGaps(t, chain, true, blocks)
   131  	verifyCutoff(t, chain, true, blocks, basic.expCanonicalBlocks)
   132  
   133  	if head := chain.CurrentHeader(); head.Number.Uint64() != basic.expHeadHeader {
   134  		t.Errorf("Head header mismatch: have %d, want %d", head.Number, basic.expHeadHeader)
   135  	}
   136  	if head := chain.CurrentFastBlock(); head.NumberU64() != basic.expHeadFastBlock {
   137  		t.Errorf("Head fast block mismatch: have %d, want %d", head.NumberU64(), basic.expHeadFastBlock)
   138  	}
   139  	if head := chain.CurrentBlock(); head.NumberU64() != basic.expHeadBlock {
   140  		t.Errorf("Head block mismatch: have %d, want %d", head.NumberU64(), basic.expHeadBlock)
   141  	}
   142  
   143  	// Check the disk layer, ensure they are matched
   144  	block := chain.GetBlockByNumber(basic.expSnapshotBottom)
   145  	if block == nil {
   146  		t.Errorf("The correspnding block[%d] of snapshot disk layer is missing", basic.expSnapshotBottom)
   147  	} else if !bytes.Equal(chain.snaps.DiskRoot().Bytes(), block.Root().Bytes()) {
   148  		t.Errorf("The snapshot disk layer root is incorrect, want %x, get %x", block.Root(), chain.snaps.DiskRoot())
   149  	}
   150  
   151  	// Check the snapshot, ensure it's integrated
   152  	if err := chain.snaps.Verify(block.Root()); err != nil {
   153  		t.Errorf("The disk layer is not integrated %v", err)
   154  	}
   155  }
   156  
   157  func (basic *snapshotTestBasic) dump() string {
   158  	buffer := new(strings.Builder)
   159  
   160  	fmt.Fprint(buffer, "Chain:\n  G")
   161  	for i := 0; i < basic.chainBlocks; i++ {
   162  		fmt.Fprintf(buffer, "->C%d", i+1)
   163  	}
   164  	fmt.Fprint(buffer, " (HEAD)\n\n")
   165  
   166  	fmt.Fprintf(buffer, "Commit:   G")
   167  	if basic.commitBlock > 0 {
   168  		fmt.Fprintf(buffer, ", C%d", basic.commitBlock)
   169  	}
   170  	fmt.Fprint(buffer, "\n")
   171  
   172  	fmt.Fprintf(buffer, "Snapshot: G")
   173  	if basic.snapshotBlock > 0 {
   174  		fmt.Fprintf(buffer, ", C%d", basic.snapshotBlock)
   175  	}
   176  	fmt.Fprint(buffer, "\n")
   177  
   178  	//if crash {
   179  	//	fmt.Fprintf(buffer, "\nCRASH\n\n")
   180  	//} else {
   181  	//	fmt.Fprintf(buffer, "\nSetHead(%d)\n\n", basic.setHead)
   182  	//}
   183  	fmt.Fprintf(buffer, "------------------------------\n\n")
   184  
   185  	fmt.Fprint(buffer, "Expected in leveldb:\n  G")
   186  	for i := 0; i < basic.expCanonicalBlocks; i++ {
   187  		fmt.Fprintf(buffer, "->C%d", i+1)
   188  	}
   189  	fmt.Fprintf(buffer, "\n\n")
   190  	fmt.Fprintf(buffer, "Expected head header    : C%d\n", basic.expHeadHeader)
   191  	fmt.Fprintf(buffer, "Expected head fast block: C%d\n", basic.expHeadFastBlock)
   192  	if basic.expHeadBlock == 0 {
   193  		fmt.Fprintf(buffer, "Expected head block     : G\n")
   194  	} else {
   195  		fmt.Fprintf(buffer, "Expected head block     : C%d\n", basic.expHeadBlock)
   196  	}
   197  	if basic.expSnapshotBottom == 0 {
   198  		fmt.Fprintf(buffer, "Expected snapshot disk  : G\n")
   199  	} else {
   200  		fmt.Fprintf(buffer, "Expected snapshot disk  : C%d\n", basic.expSnapshotBottom)
   201  	}
   202  	return buffer.String()
   203  }
   204  
   205  func (basic *snapshotTestBasic) teardown() {
   206  	basic.db.Close()
   207  	basic.gendb.Close()
   208  	os.RemoveAll(basic.datadir)
   209  }
   210  
   211  // snapshotTest is a test case type for normal snapshot recovery.
   212  // It can be used for testing that restart Geth normally.
   213  type snapshotTest struct {
   214  	snapshotTestBasic
   215  }
   216  
   217  func (snaptest *snapshotTest) test(t *testing.T) {
   218  	// It's hard to follow the test case, visualize the input
   219  	// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
   220  	// fmt.Println(tt.dump())
   221  	chain, blocks := snaptest.prepare(t)
   222  
   223  	// Restart the chain normally
   224  	chain.Stop()
   225  	newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil)
   226  	if err != nil {
   227  		t.Fatalf("Failed to recreate chain: %v", err)
   228  	}
   229  	defer newchain.Stop()
   230  
   231  	snaptest.verify(t, newchain, blocks)
   232  }
   233  
   234  // crashSnapshotTest is a test case type for innormal snapshot recovery.
   235  // It can be used for testing that restart Geth after the crash.
   236  type crashSnapshotTest struct {
   237  	snapshotTestBasic
   238  }
   239  
   240  func (snaptest *crashSnapshotTest) test(t *testing.T) {
   241  	// It's hard to follow the test case, visualize the input
   242  	// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
   243  	// fmt.Println(tt.dump())
   244  	chain, blocks := snaptest.prepare(t)
   245  
   246  	// Pull the plug on the database, simulating a hard crash
   247  	db := chain.db
   248  	db.Close()
   249  
   250  	// Start a new blockchain back up and see where the repair leads us
   251  	newdb, err := rawdb.NewLevelDBDatabaseWithFreezer(snaptest.datadir, 0, 0, snaptest.datadir, "", false, false, false)
   252  	if err != nil {
   253  		t.Fatalf("Failed to reopen persistent database: %v", err)
   254  	}
   255  	defer newdb.Close()
   256  
   257  	// The interesting thing is: instead of starting the blockchain after
   258  	// the crash, we do restart twice here: one after the crash and one
   259  	// after the normal stop. It's used to ensure the broken snapshot
   260  	// can be detected all the time.
   261  	newchain, err := NewBlockChain(newdb, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil)
   262  	if err != nil {
   263  		t.Fatalf("Failed to recreate chain: %v", err)
   264  	}
   265  	newchain.Stop()
   266  
   267  	newchain, err = NewBlockChain(newdb, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil)
   268  	if err != nil {
   269  		t.Fatalf("Failed to recreate chain: %v", err)
   270  	}
   271  	defer newchain.Stop()
   272  
   273  	snaptest.verify(t, newchain, blocks)
   274  }
   275  
   276  // gappedSnapshotTest is a test type used to test this scenario:
   277  // - have a complete snapshot
   278  // - restart without enabling the snapshot
   279  // - insert a few blocks
   280  // - restart with enabling the snapshot again
   281  type gappedSnapshotTest struct {
   282  	snapshotTestBasic
   283  	gapped int // Number of blocks to insert without enabling snapshot
   284  }
   285  
   286  func (snaptest *gappedSnapshotTest) test(t *testing.T) {
   287  	// It's hard to follow the test case, visualize the input
   288  	// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
   289  	// fmt.Println(tt.dump())
   290  	chain, blocks := snaptest.prepare(t)
   291  
   292  	// Insert blocks without enabling snapshot if gapping is required.
   293  	chain.Stop()
   294  	gappedBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.gendb, snaptest.gapped, func(i int, b *BlockGen) {})
   295  
   296  	// Insert a few more blocks without enabling snapshot
   297  	var cacheConfig = &CacheConfig{
   298  		TrieCleanLimit: 256,
   299  		TrieDirtyLimit: 256,
   300  		TrieTimeLimit:  5 * time.Minute,
   301  		TriesInMemory:  128,
   302  		SnapshotLimit:  0,
   303  	}
   304  	newchain, err := NewBlockChain(snaptest.db, cacheConfig, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil)
   305  	if err != nil {
   306  		t.Fatalf("Failed to recreate chain: %v", err)
   307  	}
   308  	newchain.InsertChain(gappedBlocks)
   309  	newchain.Stop()
   310  
   311  	// Restart the chain with enabling the snapshot
   312  	newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil)
   313  	if err != nil {
   314  		t.Fatalf("Failed to recreate chain: %v", err)
   315  	}
   316  	defer newchain.Stop()
   317  
   318  	snaptest.verify(t, newchain, blocks)
   319  }
   320  
   321  // setHeadSnapshotTest is the test type used to test this scenario:
   322  // - have a complete snapshot
   323  // - set the head to a lower point
   324  // - restart
   325  type setHeadSnapshotTest struct {
   326  	snapshotTestBasic
   327  	setHead uint64 // Block number to set head back to
   328  }
   329  
   330  func (snaptest *setHeadSnapshotTest) test(t *testing.T) {
   331  	// It's hard to follow the test case, visualize the input
   332  	// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
   333  	// fmt.Println(tt.dump())
   334  	chain, blocks := snaptest.prepare(t)
   335  
   336  	// Rewind the chain if setHead operation is required.
   337  	chain.SetHead(snaptest.setHead)
   338  	chain.Stop()
   339  
   340  	newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil)
   341  	if err != nil {
   342  		t.Fatalf("Failed to recreate chain: %v", err)
   343  	}
   344  	defer newchain.Stop()
   345  
   346  	snaptest.verify(t, newchain, blocks)
   347  }
   348  
   349  // restartCrashSnapshotTest is the test type used to test this scenario:
   350  // - have a complete snapshot
   351  // - restart chain
   352  // - insert more blocks with enabling the snapshot
   353  // - commit the snapshot
   354  // - crash
   355  // - restart again
   356  type restartCrashSnapshotTest struct {
   357  	snapshotTestBasic
   358  	newBlocks int
   359  }
   360  
   361  func (snaptest *restartCrashSnapshotTest) test(t *testing.T) {
   362  	// It's hard to follow the test case, visualize the input
   363  	// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
   364  	// fmt.Println(tt.dump())
   365  	chain, blocks := snaptest.prepare(t)
   366  
   367  	// Firstly, stop the chain properly, with all snapshot journal
   368  	// and state committed.
   369  	chain.Stop()
   370  
   371  	newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil)
   372  	if err != nil {
   373  		t.Fatalf("Failed to recreate chain: %v", err)
   374  	}
   375  	newBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.gendb, snaptest.newBlocks, func(i int, b *BlockGen) {})
   376  	newchain.InsertChain(newBlocks)
   377  
   378  	// Commit the entire snapshot into the disk if requested. Note only
   379  	// (a) snapshot root and (b) snapshot generator will be committed,
   380  	// the diff journal is not.
   381  	newchain.Snapshots().Cap(newBlocks[len(newBlocks)-1].Root(), 0)
   382  
   383  	// Simulate the blockchain crash
   384  	// Don't call chain.Stop here, so that no snapshot
   385  	// journal and latest state will be committed
   386  
   387  	// Restart the chain after the crash
   388  	newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil)
   389  	if err != nil {
   390  		t.Fatalf("Failed to recreate chain: %v", err)
   391  	}
   392  	defer newchain.Stop()
   393  
   394  	snaptest.verify(t, newchain, blocks)
   395  }
   396  
   397  // wipeCrashSnapshotTest is the test type used to test this scenario:
   398  // - have a complete snapshot
   399  // - restart, insert more blocks without enabling the snapshot
   400  // - restart again with enabling the snapshot
   401  // - crash
   402  type wipeCrashSnapshotTest struct {
   403  	snapshotTestBasic
   404  	newBlocks int
   405  }
   406  
   407  func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) {
   408  	// It's hard to follow the test case, visualize the input
   409  	// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
   410  	// fmt.Println(tt.dump())
   411  	chain, blocks := snaptest.prepare(t)
   412  
   413  	// Firstly, stop the chain properly, with all snapshot journal
   414  	// and state committed.
   415  	chain.Stop()
   416  
   417  	config := &CacheConfig{
   418  		TrieCleanLimit: 256,
   419  		TrieDirtyLimit: 256,
   420  		TrieTimeLimit:  5 * time.Minute,
   421  		SnapshotLimit:  0,
   422  		TriesInMemory:  128,
   423  	}
   424  	newchain, err := NewBlockChain(snaptest.db, config, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil)
   425  	if err != nil {
   426  		t.Fatalf("Failed to recreate chain: %v", err)
   427  	}
   428  	newBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.gendb, snaptest.newBlocks, func(i int, b *BlockGen) {})
   429  	newchain.InsertChain(newBlocks)
   430  	newchain.Stop()
   431  
   432  	// Restart the chain, the wiper should starts working
   433  	config = &CacheConfig{
   434  		TrieCleanLimit: 256,
   435  		TrieDirtyLimit: 256,
   436  		TrieTimeLimit:  5 * time.Minute,
   437  		SnapshotLimit:  256,
   438  		SnapshotWait:   false, // Don't wait rebuild
   439  		TriesInMemory:  128,
   440  	}
   441  	newchain, err = NewBlockChain(snaptest.db, config, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil)
   442  	if err != nil {
   443  		t.Fatalf("Failed to recreate chain: %v", err)
   444  	}
   445  	// Simulate the blockchain crash.
   446  
   447  	newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil)
   448  	if err != nil {
   449  		t.Fatalf("Failed to recreate chain: %v", err)
   450  	}
   451  	snaptest.verify(t, newchain, blocks)
   452  }
   453  
   454  // Tests a Geth restart with valid snapshot. Before the shutdown, all snapshot
   455  // journal will be persisted correctly. In this case no snapshot recovery is
   456  // required.
   457  func TestRestartWithNewSnapshot(t *testing.T) {
   458  	// Chain:
   459  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   460  	//
   461  	// Commit:   G
   462  	// Snapshot: G
   463  	//
   464  	// SetHead(0)
   465  	//
   466  	// ------------------------------
   467  	//
   468  	// Expected in leveldb:
   469  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   470  	//
   471  	// Expected head header    : C8
   472  	// Expected head fast block: C8
   473  	// Expected head block     : C8
   474  	// Expected snapshot disk  : G
   475  	test := &snapshotTest{
   476  		snapshotTestBasic{
   477  			chainBlocks:        8,
   478  			snapshotBlock:      0,
   479  			commitBlock:        0,
   480  			expCanonicalBlocks: 8,
   481  			expHeadHeader:      8,
   482  			expHeadFastBlock:   8,
   483  			expHeadBlock:       8,
   484  			expSnapshotBottom:  0, // Initial disk layer built from genesis
   485  		},
   486  	}
   487  	test.test(t)
   488  	test.teardown()
   489  }
   490  
   491  // Tests a Geth was crashed and restarts with a broken snapshot. In this case the
   492  // chain head should be rewound to the point with available state. And also the
   493  // new head should must be lower than disk layer. But there is no committed point
   494  // so the chain should be rewound to genesis and the disk layer should be left
   495  // for recovery.
   496  func TestNoCommitCrashWithNewSnapshot(t *testing.T) {
   497  	// Chain:
   498  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   499  	//
   500  	// Commit:   G
   501  	// Snapshot: G, C4
   502  	//
   503  	// CRASH
   504  	//
   505  	// ------------------------------
   506  	//
   507  	// Expected in leveldb:
   508  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   509  	//
   510  	// Expected head header    : C8
   511  	// Expected head fast block: C8
   512  	// Expected head block     : G
   513  	// Expected snapshot disk  : C4
   514  	test := &crashSnapshotTest{
   515  		snapshotTestBasic{
   516  			chainBlocks:        8,
   517  			snapshotBlock:      4,
   518  			commitBlock:        0,
   519  			expCanonicalBlocks: 8,
   520  			expHeadHeader:      8,
   521  			expHeadFastBlock:   8,
   522  			expHeadBlock:       0,
   523  			expSnapshotBottom:  4, // Last committed disk layer, wait recovery
   524  		},
   525  	}
   526  	test.test(t)
   527  	test.teardown()
   528  }
   529  
   530  // Tests a Geth was crashed and restarts with a broken snapshot. In this case the
   531  // chain head should be rewound to the point with available state. And also the
   532  // new head should must be lower than disk layer. But there is only a low committed
   533  // point so the chain should be rewound to committed point and the disk layer
   534  // should be left for recovery.
   535  func TestLowCommitCrashWithNewSnapshot(t *testing.T) {
   536  	// Chain:
   537  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   538  	//
   539  	// Commit:   G, C2
   540  	// Snapshot: G, C4
   541  	//
   542  	// CRASH
   543  	//
   544  	// ------------------------------
   545  	//
   546  	// Expected in leveldb:
   547  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   548  	//
   549  	// Expected head header    : C8
   550  	// Expected head fast block: C8
   551  	// Expected head block     : C2
   552  	// Expected snapshot disk  : C4
   553  	test := &crashSnapshotTest{
   554  		snapshotTestBasic{
   555  			chainBlocks:        8,
   556  			snapshotBlock:      4,
   557  			commitBlock:        2,
   558  			expCanonicalBlocks: 8,
   559  			expHeadHeader:      8,
   560  			expHeadFastBlock:   8,
   561  			expHeadBlock:       2,
   562  			expSnapshotBottom:  4, // Last committed disk layer, wait recovery
   563  		},
   564  	}
   565  	test.test(t)
   566  	test.teardown()
   567  }
   568  
   569  // Tests a Geth was crashed and restarts with a broken snapshot. In this case
   570  // the chain head should be rewound to the point with available state. And also
   571  // the new head should must be lower than disk layer. But there is only a high
   572  // committed point so the chain should be rewound to genesis and the disk layer
   573  // should be left for recovery.
   574  func TestHighCommitCrashWithNewSnapshot(t *testing.T) {
   575  	// Chain:
   576  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   577  	//
   578  	// Commit:   G, C6
   579  	// Snapshot: G, C4
   580  	//
   581  	// CRASH
   582  	//
   583  	// ------------------------------
   584  	//
   585  	// Expected in leveldb:
   586  	//   G->C1->C2->C3->C4->C5->C6->C7->C8
   587  	//
   588  	// Expected head header    : C8
   589  	// Expected head fast block: C8
   590  	// Expected head block     : G
   591  	// Expected snapshot disk  : C4
   592  	test := &crashSnapshotTest{
   593  		snapshotTestBasic{
   594  			chainBlocks:        8,
   595  			snapshotBlock:      4,
   596  			commitBlock:        6,
   597  			expCanonicalBlocks: 8,
   598  			expHeadHeader:      8,
   599  			expHeadFastBlock:   8,
   600  			expHeadBlock:       0,
   601  			expSnapshotBottom:  4, // Last committed disk layer, wait recovery
   602  		},
   603  	}
   604  	test.test(t)
   605  	test.teardown()
   606  }
   607  
   608  // Tests a Geth was running with snapshot enabled. Then restarts without
   609  // enabling snapshot and after that re-enable the snapshot again. In this
   610  // case the snapshot should be rebuilt with latest chain head.
   611  func TestGappedNewSnapshot(t *testing.T) {
   612  	// Chain:
   613  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   614  	//
   615  	// Commit:   G
   616  	// Snapshot: G
   617  	//
   618  	// SetHead(0)
   619  	//
   620  	// ------------------------------
   621  	//
   622  	// Expected in leveldb:
   623  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10
   624  	//
   625  	// Expected head header    : C10
   626  	// Expected head fast block: C10
   627  	// Expected head block     : C10
   628  	// Expected snapshot disk  : C10
   629  	test := &gappedSnapshotTest{
   630  		snapshotTestBasic: snapshotTestBasic{
   631  			chainBlocks:        8,
   632  			snapshotBlock:      0,
   633  			commitBlock:        0,
   634  			expCanonicalBlocks: 10,
   635  			expHeadHeader:      10,
   636  			expHeadFastBlock:   10,
   637  			expHeadBlock:       10,
   638  			expSnapshotBottom:  10, // Rebuilt snapshot from the latest HEAD
   639  		},
   640  		gapped: 2,
   641  	}
   642  	test.test(t)
   643  	test.teardown()
   644  }
   645  
   646  // Tests the Geth was running with snapshot enabled and resetHead is applied.
   647  // In this case the head is rewound to the target(with state available). After
   648  // that the chain is restarted and the original disk layer is kept.
   649  func TestSetHeadWithNewSnapshot(t *testing.T) {
   650  	// Chain:
   651  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   652  	//
   653  	// Commit:   G
   654  	// Snapshot: G
   655  	//
   656  	// SetHead(4)
   657  	//
   658  	// ------------------------------
   659  	//
   660  	// Expected in leveldb:
   661  	//   G->C1->C2->C3->C4
   662  	//
   663  	// Expected head header    : C4
   664  	// Expected head fast block: C4
   665  	// Expected head block     : C4
   666  	// Expected snapshot disk  : G
   667  	test := &setHeadSnapshotTest{
   668  		snapshotTestBasic: snapshotTestBasic{
   669  			chainBlocks:        8,
   670  			snapshotBlock:      0,
   671  			commitBlock:        0,
   672  			expCanonicalBlocks: 4,
   673  			expHeadHeader:      4,
   674  			expHeadFastBlock:   4,
   675  			expHeadBlock:       4,
   676  			expSnapshotBottom:  0, // The initial disk layer is built from the genesis
   677  		},
   678  		setHead: 4,
   679  	}
   680  	test.test(t)
   681  	test.teardown()
   682  }
   683  
   684  // Tests the Geth was running with a complete snapshot and then imports a few
   685  // more new blocks on top without enabling the snapshot. After the restart,
   686  // crash happens. Check everything is ok after the restart.
   687  func TestRecoverSnapshotFromWipingCrash(t *testing.T) {
   688  	// Chain:
   689  	//   G->C1->C2->C3->C4->C5->C6->C7->C8 (HEAD)
   690  	//
   691  	// Commit:   G
   692  	// Snapshot: G
   693  	//
   694  	// SetHead(0)
   695  	//
   696  	// ------------------------------
   697  	//
   698  	// Expected in leveldb:
   699  	//   G->C1->C2->C3->C4->C5->C6->C7->C8->C9->C10
   700  	//
   701  	// Expected head header    : C10
   702  	// Expected head fast block: C10
   703  	// Expected head block     : C8
   704  	// Expected snapshot disk  : C10
   705  	test := &wipeCrashSnapshotTest{
   706  		snapshotTestBasic: snapshotTestBasic{
   707  			chainBlocks:        8,
   708  			snapshotBlock:      4,
   709  			commitBlock:        0,
   710  			expCanonicalBlocks: 10,
   711  			expHeadHeader:      10,
   712  			expHeadFastBlock:   10,
   713  			expHeadBlock:       10,
   714  			expSnapshotBottom:  10,
   715  		},
   716  		newBlocks: 2,
   717  	}
   718  	test.test(t)
   719  	test.teardown()
   720  }