github.com/tacshi/go-ethereum@v0.0.0-20230616113857-84a434e20921/core/state/snapshot/snapshot_test.go (about)

     1  // Copyright 2017 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package snapshot
    18  
    19  import (
    20  	crand "crypto/rand"
    21  	"encoding/binary"
    22  	"fmt"
    23  	"math/big"
    24  	"math/rand"
    25  	"testing"
    26  	"time"
    27  
    28  	"github.com/VictoriaMetrics/fastcache"
    29  	"github.com/tacshi/go-ethereum/common"
    30  	"github.com/tacshi/go-ethereum/core/rawdb"
    31  	"github.com/tacshi/go-ethereum/core/types"
    32  	"github.com/tacshi/go-ethereum/rlp"
    33  )
    34  
    35  // randomHash generates a random blob of data and returns it as a hash.
    36  func randomHash() common.Hash {
    37  	var hash common.Hash
    38  	if n, err := crand.Read(hash[:]); n != common.HashLength || err != nil {
    39  		panic(err)
    40  	}
    41  	return hash
    42  }
    43  
    44  // randomAccount generates a random account and returns it RLP encoded.
    45  func randomAccount() []byte {
    46  	root := randomHash()
    47  	a := Account{
    48  		Balance:  big.NewInt(rand.Int63()),
    49  		Nonce:    rand.Uint64(),
    50  		Root:     root[:],
    51  		CodeHash: types.EmptyCodeHash[:],
    52  	}
    53  	data, _ := rlp.EncodeToBytes(a)
    54  	return data
    55  }
    56  
    57  // randomAccountSet generates a set of random accounts with the given strings as
    58  // the account address hashes.
    59  func randomAccountSet(hashes ...string) map[common.Hash][]byte {
    60  	accounts := make(map[common.Hash][]byte)
    61  	for _, hash := range hashes {
    62  		accounts[common.HexToHash(hash)] = randomAccount()
    63  	}
    64  	return accounts
    65  }
    66  
    67  // randomStorageSet generates a set of random slots with the given strings as
    68  // the slot addresses.
    69  func randomStorageSet(accounts []string, hashes [][]string, nilStorage [][]string) map[common.Hash]map[common.Hash][]byte {
    70  	storages := make(map[common.Hash]map[common.Hash][]byte)
    71  	for index, account := range accounts {
    72  		storages[common.HexToHash(account)] = make(map[common.Hash][]byte)
    73  
    74  		if index < len(hashes) {
    75  			hashes := hashes[index]
    76  			for _, hash := range hashes {
    77  				storages[common.HexToHash(account)][common.HexToHash(hash)] = randomHash().Bytes()
    78  			}
    79  		}
    80  		if index < len(nilStorage) {
    81  			nils := nilStorage[index]
    82  			for _, hash := range nils {
    83  				storages[common.HexToHash(account)][common.HexToHash(hash)] = nil
    84  			}
    85  		}
    86  	}
    87  	return storages
    88  }
    89  
    90  // Tests that if a disk layer becomes stale, no active external references will
    91  // be returned with junk data. This version of the test flattens every diff layer
    92  // to check internal corner case around the bottom-most memory accumulator.
    93  func TestDiskLayerExternalInvalidationFullFlatten(t *testing.T) {
    94  	// Create an empty base layer and a snapshot tree out of it
    95  	base := &diskLayer{
    96  		diskdb: rawdb.NewMemoryDatabase(),
    97  		root:   common.HexToHash("0x01"),
    98  		cache:  fastcache.New(1024 * 500),
    99  	}
   100  	snaps := &Tree{
   101  		layers: map[common.Hash]snapshot{
   102  			base.root: base,
   103  		},
   104  	}
   105  	// Retrieve a reference to the base and commit a diff on top
   106  	ref := snaps.Snapshot(base.root)
   107  
   108  	accounts := map[common.Hash][]byte{
   109  		common.HexToHash("0xa1"): randomAccount(),
   110  	}
   111  	if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil {
   112  		t.Fatalf("failed to create a diff layer: %v", err)
   113  	}
   114  	if n := len(snaps.layers); n != 2 {
   115  		t.Errorf("pre-cap layer count mismatch: have %d, want %d", n, 2)
   116  	}
   117  	// Commit the diff layer onto the disk and ensure it's persisted
   118  	if err := snaps.Cap(common.HexToHash("0x02"), 0); err != nil {
   119  		t.Fatalf("failed to merge diff layer onto disk: %v", err)
   120  	}
   121  	// Since the base layer was modified, ensure that data retrieval on the external reference fail
   122  	if acc, err := ref.Account(common.HexToHash("0x01")); err != ErrSnapshotStale {
   123  		t.Errorf("stale reference returned account: %#x (err: %v)", acc, err)
   124  	}
   125  	if slot, err := ref.Storage(common.HexToHash("0xa1"), common.HexToHash("0xb1")); err != ErrSnapshotStale {
   126  		t.Errorf("stale reference returned storage slot: %#x (err: %v)", slot, err)
   127  	}
   128  	if n := len(snaps.layers); n != 1 {
   129  		t.Errorf("post-cap layer count mismatch: have %d, want %d", n, 1)
   130  		fmt.Println(snaps.layers)
   131  	}
   132  }
   133  
   134  // Tests that if a disk layer becomes stale, no active external references will
   135  // be returned with junk data. This version of the test retains the bottom diff
   136  // layer to check the usual mode of operation where the accumulator is retained.
   137  func TestDiskLayerExternalInvalidationPartialFlatten(t *testing.T) {
   138  	// Create an empty base layer and a snapshot tree out of it
   139  	base := &diskLayer{
   140  		diskdb: rawdb.NewMemoryDatabase(),
   141  		root:   common.HexToHash("0x01"),
   142  		cache:  fastcache.New(1024 * 500),
   143  	}
   144  	snaps := &Tree{
   145  		layers: map[common.Hash]snapshot{
   146  			base.root: base,
   147  		},
   148  	}
   149  	// Retrieve a reference to the base and commit two diffs on top
   150  	ref := snaps.Snapshot(base.root)
   151  
   152  	accounts := map[common.Hash][]byte{
   153  		common.HexToHash("0xa1"): randomAccount(),
   154  	}
   155  	if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil {
   156  		t.Fatalf("failed to create a diff layer: %v", err)
   157  	}
   158  	if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil {
   159  		t.Fatalf("failed to create a diff layer: %v", err)
   160  	}
   161  	if n := len(snaps.layers); n != 3 {
   162  		t.Errorf("pre-cap layer count mismatch: have %d, want %d", n, 3)
   163  	}
   164  	// Commit the diff layer onto the disk and ensure it's persisted
   165  	defer func(memcap uint64) { aggregatorMemoryLimit = memcap }(aggregatorMemoryLimit)
   166  	aggregatorMemoryLimit = 0
   167  
   168  	if err := snaps.Cap(common.HexToHash("0x03"), 1); err != nil {
   169  		t.Fatalf("failed to merge accumulator onto disk: %v", err)
   170  	}
   171  	// Since the base layer was modified, ensure that data retrievals on the external reference fail
   172  	if acc, err := ref.Account(common.HexToHash("0x01")); err != ErrSnapshotStale {
   173  		t.Errorf("stale reference returned account: %#x (err: %v)", acc, err)
   174  	}
   175  	if slot, err := ref.Storage(common.HexToHash("0xa1"), common.HexToHash("0xb1")); err != ErrSnapshotStale {
   176  		t.Errorf("stale reference returned storage slot: %#x (err: %v)", slot, err)
   177  	}
   178  	if n := len(snaps.layers); n != 2 {
   179  		t.Errorf("post-cap layer count mismatch: have %d, want %d", n, 2)
   180  		fmt.Println(snaps.layers)
   181  	}
   182  }
   183  
   184  // Tests that if a diff layer becomes stale, no active external references will
   185  // be returned with junk data. This version of the test retains the bottom diff
   186  // layer to check the usual mode of operation where the accumulator is retained.
   187  func TestDiffLayerExternalInvalidationPartialFlatten(t *testing.T) {
   188  	// Create an empty base layer and a snapshot tree out of it
   189  	base := &diskLayer{
   190  		diskdb: rawdb.NewMemoryDatabase(),
   191  		root:   common.HexToHash("0x01"),
   192  		cache:  fastcache.New(1024 * 500),
   193  	}
   194  	snaps := &Tree{
   195  		layers: map[common.Hash]snapshot{
   196  			base.root: base,
   197  		},
   198  	}
   199  	// Commit three diffs on top and retrieve a reference to the bottommost
   200  	accounts := map[common.Hash][]byte{
   201  		common.HexToHash("0xa1"): randomAccount(),
   202  	}
   203  	if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil {
   204  		t.Fatalf("failed to create a diff layer: %v", err)
   205  	}
   206  	if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil {
   207  		t.Fatalf("failed to create a diff layer: %v", err)
   208  	}
   209  	if err := snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, accounts, nil); err != nil {
   210  		t.Fatalf("failed to create a diff layer: %v", err)
   211  	}
   212  	if n := len(snaps.layers); n != 4 {
   213  		t.Errorf("pre-cap layer count mismatch: have %d, want %d", n, 4)
   214  	}
   215  	ref := snaps.Snapshot(common.HexToHash("0x02"))
   216  
   217  	// Doing a Cap operation with many allowed layers should be a no-op
   218  	exp := len(snaps.layers)
   219  	if err := snaps.Cap(common.HexToHash("0x04"), 2000); err != nil {
   220  		t.Fatalf("failed to flatten diff layer into accumulator: %v", err)
   221  	}
   222  	if got := len(snaps.layers); got != exp {
   223  		t.Errorf("layers modified, got %d exp %d", got, exp)
   224  	}
   225  	// Flatten the diff layer into the bottom accumulator
   226  	if err := snaps.Cap(common.HexToHash("0x04"), 1); err != nil {
   227  		t.Fatalf("failed to flatten diff layer into accumulator: %v", err)
   228  	}
   229  	// Since the accumulator diff layer was modified, ensure that data retrievals on the external reference fail
   230  	if acc, err := ref.Account(common.HexToHash("0x01")); err != ErrSnapshotStale {
   231  		t.Errorf("stale reference returned account: %#x (err: %v)", acc, err)
   232  	}
   233  	if slot, err := ref.Storage(common.HexToHash("0xa1"), common.HexToHash("0xb1")); err != ErrSnapshotStale {
   234  		t.Errorf("stale reference returned storage slot: %#x (err: %v)", slot, err)
   235  	}
   236  	if n := len(snaps.layers); n != 3 {
   237  		t.Errorf("post-cap layer count mismatch: have %d, want %d", n, 3)
   238  		fmt.Println(snaps.layers)
   239  	}
   240  }
   241  
   242  // TestPostCapBasicDataAccess tests some functionality regarding capping/flattening.
   243  func TestPostCapBasicDataAccess(t *testing.T) {
   244  	t.Skip("Arbitrum: Test fails due to our modifications to Tree.Cap function.")
   245  	// setAccount is a helper to construct a random account entry and assign it to
   246  	// an account slot in a snapshot
   247  	setAccount := func(accKey string) map[common.Hash][]byte {
   248  		return map[common.Hash][]byte{
   249  			common.HexToHash(accKey): randomAccount(),
   250  		}
   251  	}
   252  	// Create a starting base layer and a snapshot tree out of it
   253  	base := &diskLayer{
   254  		diskdb: rawdb.NewMemoryDatabase(),
   255  		root:   common.HexToHash("0x01"),
   256  		cache:  fastcache.New(1024 * 500),
   257  	}
   258  	snaps := &Tree{
   259  		layers: map[common.Hash]snapshot{
   260  			base.root: base,
   261  		},
   262  	}
   263  	// The lowest difflayer
   264  	snaps.Update(common.HexToHash("0xa1"), common.HexToHash("0x01"), nil, setAccount("0xa1"), nil)
   265  	snaps.Update(common.HexToHash("0xa2"), common.HexToHash("0xa1"), nil, setAccount("0xa2"), nil)
   266  	snaps.Update(common.HexToHash("0xb2"), common.HexToHash("0xa1"), nil, setAccount("0xb2"), nil)
   267  
   268  	snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), nil, setAccount("0xa3"), nil)
   269  	snaps.Update(common.HexToHash("0xb3"), common.HexToHash("0xb2"), nil, setAccount("0xb3"), nil)
   270  
   271  	// checkExist verifies if an account exists in a snapshot
   272  	checkExist := func(layer *diffLayer, key string) error {
   273  		if data, _ := layer.Account(common.HexToHash(key)); data == nil {
   274  			return fmt.Errorf("expected %x to exist, got nil", common.HexToHash(key))
   275  		}
   276  		return nil
   277  	}
   278  	// shouldErr checks that an account access errors as expected
   279  	shouldErr := func(layer *diffLayer, key string) error {
   280  		if data, err := layer.Account(common.HexToHash(key)); err == nil {
   281  			return fmt.Errorf("expected error, got data %x", data)
   282  		}
   283  		return nil
   284  	}
   285  	// check basics
   286  	snap := snaps.Snapshot(common.HexToHash("0xb3")).(*diffLayer)
   287  
   288  	if err := checkExist(snap, "0xa1"); err != nil {
   289  		t.Error(err)
   290  	}
   291  	if err := checkExist(snap, "0xb2"); err != nil {
   292  		t.Error(err)
   293  	}
   294  	if err := checkExist(snap, "0xb3"); err != nil {
   295  		t.Error(err)
   296  	}
   297  	// Cap to a bad root should fail
   298  	if err := snaps.Cap(common.HexToHash("0x1337"), 0); err == nil {
   299  		t.Errorf("expected error, got none")
   300  	}
   301  	// Now, merge the a-chain
   302  	snaps.Cap(common.HexToHash("0xa3"), 0)
   303  
   304  	// At this point, a2 got merged into a1. Thus, a1 is now modified, and as a1 is
   305  	// the parent of b2, b2 should no longer be able to iterate into parent.
   306  
   307  	// These should still be accessible
   308  	if err := checkExist(snap, "0xb2"); err != nil {
   309  		t.Error(err)
   310  	}
   311  	if err := checkExist(snap, "0xb3"); err != nil {
   312  		t.Error(err)
   313  	}
   314  	// But these would need iteration into the modified parent
   315  	if err := shouldErr(snap, "0xa1"); err != nil {
   316  		t.Error(err)
   317  	}
   318  	if err := shouldErr(snap, "0xa2"); err != nil {
   319  		t.Error(err)
   320  	}
   321  	if err := shouldErr(snap, "0xa3"); err != nil {
   322  		t.Error(err)
   323  	}
   324  	// Now, merge it again, just for fun. It should now error, since a3
   325  	// is a disk layer
   326  	if err := snaps.Cap(common.HexToHash("0xa3"), 0); err == nil {
   327  		t.Error("expected error capping the disk layer, got none")
   328  	}
   329  }
   330  
   331  // TestSnaphots tests the functionality for retrieving the snapshot
   332  // with given head root and the desired depth.
   333  func TestSnaphots(t *testing.T) {
   334  	// setAccount is a helper to construct a random account entry and assign it to
   335  	// an account slot in a snapshot
   336  	setAccount := func(accKey string) map[common.Hash][]byte {
   337  		return map[common.Hash][]byte{
   338  			common.HexToHash(accKey): randomAccount(),
   339  		}
   340  	}
   341  	makeRoot := func(height uint64) common.Hash {
   342  		var buffer [8]byte
   343  		binary.BigEndian.PutUint64(buffer[:], height)
   344  		return common.BytesToHash(buffer[:])
   345  	}
   346  	// Create a starting base layer and a snapshot tree out of it
   347  	base := &diskLayer{
   348  		diskdb: rawdb.NewMemoryDatabase(),
   349  		root:   makeRoot(1),
   350  		cache:  fastcache.New(1024 * 500),
   351  	}
   352  	snaps := &Tree{
   353  		layers: map[common.Hash]snapshot{
   354  			base.root: base,
   355  		},
   356  	}
   357  	// Construct the snapshots with 129 layers, flattening whatever's above that
   358  	var (
   359  		last = common.HexToHash("0x01")
   360  		head common.Hash
   361  	)
   362  	for i := 0; i < 129; i++ {
   363  		head = makeRoot(uint64(i + 2))
   364  		snaps.Update(head, last, nil, setAccount(fmt.Sprintf("%d", i+2)), nil)
   365  		last = head
   366  		snaps.Cap(head, 128) // 130 layers (128 diffs + 1 accumulator + 1 disk)
   367  	}
   368  	var cases = []struct {
   369  		headRoot     common.Hash
   370  		limit        int
   371  		nodisk       bool
   372  		expected     int
   373  		expectBottom common.Hash
   374  	}{
   375  		{head, 0, false, 0, common.Hash{}},
   376  		{head, 64, false, 64, makeRoot(129 + 2 - 64)},
   377  		{head, 128, false, 128, makeRoot(3)}, // Normal diff layers, no accumulator
   378  		{head, 129, true, 129, makeRoot(2)},  // All diff layers, including accumulator
   379  		{head, 130, false, 130, makeRoot(1)}, // All diff layers + disk layer
   380  	}
   381  	for i, c := range cases {
   382  		layers := snaps.Snapshots(c.headRoot, c.limit, c.nodisk)
   383  		if len(layers) != c.expected {
   384  			t.Errorf("non-overflow test %d: returned snapshot layers are mismatched, want %v, got %v", i, c.expected, len(layers))
   385  		}
   386  		if len(layers) == 0 {
   387  			continue
   388  		}
   389  		bottommost := layers[len(layers)-1]
   390  		if bottommost.Root() != c.expectBottom {
   391  			t.Errorf("non-overflow test %d: snapshot mismatch, want %v, get %v", i, c.expectBottom, bottommost.Root())
   392  		}
   393  	}
   394  	// Above we've tested the normal capping, which leaves the accumulator live.
   395  	// Test that if the bottommost accumulator diff layer overflows the allowed
   396  	// memory limit, the snapshot tree gets capped to one less layer.
   397  	// Commit the diff layer onto the disk and ensure it's persisted
   398  	defer func(memcap uint64) { aggregatorMemoryLimit = memcap }(aggregatorMemoryLimit)
   399  	aggregatorMemoryLimit = 0
   400  
   401  	snaps.Cap(head, 128) // 129 (128 diffs + 1 overflown accumulator + 1 disk)
   402  
   403  	cases = []struct {
   404  		headRoot     common.Hash
   405  		limit        int
   406  		nodisk       bool
   407  		expected     int
   408  		expectBottom common.Hash
   409  	}{
   410  		{head, 0, false, 0, common.Hash{}},
   411  		{head, 64, false, 64, makeRoot(129 + 2 - 64)},
   412  		{head, 128, false, 128, makeRoot(3)}, // All diff layers, accumulator was flattened
   413  		{head, 129, true, 128, makeRoot(3)},  // All diff layers, accumulator was flattened
   414  		{head, 130, false, 129, makeRoot(2)}, // All diff layers + disk layer
   415  	}
   416  	for i, c := range cases {
   417  		layers := snaps.Snapshots(c.headRoot, c.limit, c.nodisk)
   418  		if len(layers) != c.expected {
   419  			t.Errorf("overflow test %d: returned snapshot layers are mismatched, want %v, got %v", i, c.expected, len(layers))
   420  		}
   421  		if len(layers) == 0 {
   422  			continue
   423  		}
   424  		bottommost := layers[len(layers)-1]
   425  		if bottommost.Root() != c.expectBottom {
   426  			t.Errorf("overflow test %d: snapshot mismatch, want %v, get %v", i, c.expectBottom, bottommost.Root())
   427  		}
   428  	}
   429  }
   430  
   431  // TestReadStateDuringFlattening tests the scenario that, during the
   432  // bottom diff layers are merging which tags these as stale, the read
   433  // happens via a pre-created top snapshot layer which tries to access
   434  // the state in these stale layers. Ensure this read can retrieve the
   435  // right state back(block until the flattening is finished) instead of
   436  // an unexpected error(snapshot layer is stale).
   437  func TestReadStateDuringFlattening(t *testing.T) {
   438  	// setAccount is a helper to construct a random account entry and assign it to
   439  	// an account slot in a snapshot
   440  	setAccount := func(accKey string) map[common.Hash][]byte {
   441  		return map[common.Hash][]byte{
   442  			common.HexToHash(accKey): randomAccount(),
   443  		}
   444  	}
   445  	// Create a starting base layer and a snapshot tree out of it
   446  	base := &diskLayer{
   447  		diskdb: rawdb.NewMemoryDatabase(),
   448  		root:   common.HexToHash("0x01"),
   449  		cache:  fastcache.New(1024 * 500),
   450  	}
   451  	snaps := &Tree{
   452  		layers: map[common.Hash]snapshot{
   453  			base.root: base,
   454  		},
   455  	}
   456  	// 4 layers in total, 3 diff layers and 1 disk layers
   457  	snaps.Update(common.HexToHash("0xa1"), common.HexToHash("0x01"), nil, setAccount("0xa1"), nil)
   458  	snaps.Update(common.HexToHash("0xa2"), common.HexToHash("0xa1"), nil, setAccount("0xa2"), nil)
   459  	snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), nil, setAccount("0xa3"), nil)
   460  
   461  	// Obtain the topmost snapshot handler for state accessing
   462  	snap := snaps.Snapshot(common.HexToHash("0xa3"))
   463  
   464  	// Register the testing hook to access the state after flattening
   465  	var result = make(chan *Account)
   466  	snaps.onFlatten = func() {
   467  		// Spin up a thread to read the account from the pre-created
   468  		// snapshot handler. It's expected to be blocked.
   469  		go func() {
   470  			account, _ := snap.Account(common.HexToHash("0xa1"))
   471  			result <- account
   472  		}()
   473  		select {
   474  		case res := <-result:
   475  			t.Fatalf("Unexpected return %v", res)
   476  		case <-time.NewTimer(time.Millisecond * 300).C:
   477  		}
   478  	}
   479  	// Cap the snap tree, which will mark the bottom-most layer as stale.
   480  	snaps.Cap(common.HexToHash("0xa3"), 1)
   481  	select {
   482  	case account := <-result:
   483  		if account == nil {
   484  			t.Fatal("Failed to retrieve account")
   485  		}
   486  	case <-time.NewTimer(time.Millisecond * 300).C:
   487  		t.Fatal("Unexpected blocker")
   488  	}
   489  }