github.com/fff-chain/go-fff@v0.0.0-20220726032732-1c84420b8a99/core/state/snapshot/difflayer.go (about)

     1  // Copyright 2019 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package snapshot
    18  
    19  import (
    20  	"encoding/binary"
    21  	"fmt"
    22  	"math"
    23  	"math/rand"
    24  	"sort"
    25  	"sync"
    26  	"sync/atomic"
    27  	"time"
    28  
    29  	"github.com/fff-chain/go-fff/common"
    30  	"github.com/fff-chain/go-fff/rlp"
    31  	bloomfilter "github.com/holiman/bloomfilter/v2"
    32  )
    33  
    34  var (
    35  	// aggregatorMemoryLimit is the maximum size of the bottom-most diff layer
    36  	// that aggregates the writes from above until it's flushed into the disk
    37  	// layer.
    38  	//
    39  	// Note, bumping this up might drastically increase the size of the bloom
    40  	// filters that's stored in every diff layer. Don't do that without fully
    41  	// understanding all the implications.
    42  	aggregatorMemoryLimit = uint64(4 * 1024 * 1024)
    43  
    44  	// aggregatorItemLimit is an approximate number of items that will end up
    45  	// in the agregator layer before it's flushed out to disk. A plain account
    46  	// weighs around 14B (+hash), a storage slot 32B (+hash), a deleted slot
    47  	// 0B (+hash). Slots are mostly set/unset in lockstep, so that average at
    48  	// 16B (+hash). All in all, the average entry seems to be 15+32=47B. Use a
    49  	// smaller number to be on the safe side.
    50  	aggregatorItemLimit = aggregatorMemoryLimit / 42
    51  
    52  	// bloomTargetError is the target false positive rate when the aggregator
    53  	// layer is at its fullest. The actual value will probably move around up
    54  	// and down from this number, it's mostly a ballpark figure.
    55  	//
    56  	// Note, dropping this down might drastically increase the size of the bloom
    57  	// filters that's stored in every diff layer. Don't do that without fully
    58  	// understanding all the implications.
    59  	bloomTargetError = 0.02
    60  
    61  	// bloomSize is the ideal bloom filter size given the maximum number of items
    62  	// it's expected to hold and the target false positive error rate.
    63  	bloomSize = math.Ceil(float64(aggregatorItemLimit) * math.Log(bloomTargetError) / math.Log(1/math.Pow(2, math.Log(2))))
    64  
    65  	// bloomFuncs is the ideal number of bits a single entry should set in the
    66  	// bloom filter to keep its size to a minimum (given it's size and maximum
    67  	// entry count).
    68  	bloomFuncs = math.Round((bloomSize / float64(aggregatorItemLimit)) * math.Log(2))
    69  
    70  	// the bloom offsets are runtime constants which determines which part of the
    71  	// the account/storage hash the hasher functions looks at, to determine the
    72  	// bloom key for an account/slot. This is randomized at init(), so that the
    73  	// global population of nodes do not all display the exact same behaviour with
    74  	// regards to bloom content
    75  	bloomDestructHasherOffset = 0
    76  	bloomAccountHasherOffset  = 0
    77  	bloomStorageHasherOffset  = 0
    78  )
    79  
    80  func init() {
    81  	// Init the bloom offsets in the range [0:24] (requires 8 bytes)
    82  	bloomDestructHasherOffset = rand.Intn(25)
    83  	bloomAccountHasherOffset = rand.Intn(25)
    84  	bloomStorageHasherOffset = rand.Intn(25)
    85  
    86  	// The destruct and account blooms must be different, as the storage slots
    87  	// will check for destruction too for every bloom miss. It should not collide
    88  	// with modified accounts.
    89  	for bloomAccountHasherOffset == bloomDestructHasherOffset {
    90  		bloomAccountHasherOffset = rand.Intn(25)
    91  	}
    92  }
    93  
    94  // diffLayer represents a collection of modifications made to a state snapshot
    95  // after running a block on top. It contains one sorted list for the account trie
    96  // and one-one list for each storage tries.
    97  //
    98  // The goal of a diff layer is to act as a journal, tracking recent modifications
    99  // made to the state, that have not yet graduated into a semi-immutable state.
   100  type diffLayer struct {
   101  	origin *diskLayer // Base disk layer to directly use on bloom misses
   102  	parent snapshot   // Parent snapshot modified by this one, never nil
   103  	memory uint64     // Approximate guess as to how much memory we use
   104  
   105  	root  common.Hash // Root hash to which this snapshot diff belongs to
   106  	stale uint32      // Signals that the layer became stale (state progressed)
   107  
   108  	// destructSet is a very special helper marker. If an account is marked as
   109  	// deleted, then it's recorded in this set. However it's allowed that an account
   110  	// is included here but still available in other sets(e.g. storageData). The
   111  	// reason is the diff layer includes all the changes in a *block*. It can
   112  	// happen that in the tx_1, account A is self-destructed while in the tx_2
   113  	// it's recreated. But we still need this marker to indicate the "old" A is
   114  	// deleted, all data in other set belongs to the "new" A.
   115  	destructSet map[common.Hash]struct{}               // Keyed markers for deleted (and potentially) recreated accounts
   116  	accountList []common.Hash                          // List of account for iteration. If it exists, it's sorted, otherwise it's nil
   117  	accountData map[common.Hash][]byte                 // Keyed accounts for direct retrieval (nil means deleted)
   118  	storageList map[common.Hash][]common.Hash          // List of storage slots for iterated retrievals, one per account. Any existing lists are sorted if non-nil
   119  	storageData map[common.Hash]map[common.Hash][]byte // Keyed storage slots for direct retrieval. one per account (nil means deleted)
   120  
   121  	verifiedCh chan struct{} // the difflayer is verified when verifiedCh is nil or closed
   122  	valid      bool          // mark the difflayer is valid or not.
   123  
   124  	diffed *bloomfilter.Filter // Bloom filter tracking all the diffed items up to the disk layer
   125  
   126  	lock sync.RWMutex
   127  }
   128  
   129  // destructBloomHasher is a wrapper around a common.Hash to satisfy the interface
   130  // API requirements of the bloom library used. It's used to convert a destruct
   131  // event into a 64 bit mini hash.
   132  type destructBloomHasher common.Hash
   133  
   134  func (h destructBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") }
   135  func (h destructBloomHasher) Sum(b []byte) []byte               { panic("not implemented") }
   136  func (h destructBloomHasher) Reset()                            { panic("not implemented") }
   137  func (h destructBloomHasher) BlockSize() int                    { panic("not implemented") }
   138  func (h destructBloomHasher) Size() int                         { return 8 }
   139  func (h destructBloomHasher) Sum64() uint64 {
   140  	return binary.BigEndian.Uint64(h[bloomDestructHasherOffset : bloomDestructHasherOffset+8])
   141  }
   142  
   143  // accountBloomHasher is a wrapper around a common.Hash to satisfy the interface
   144  // API requirements of the bloom library used. It's used to convert an account
   145  // hash into a 64 bit mini hash.
   146  type accountBloomHasher common.Hash
   147  
   148  func (h accountBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") }
   149  func (h accountBloomHasher) Sum(b []byte) []byte               { panic("not implemented") }
   150  func (h accountBloomHasher) Reset()                            { panic("not implemented") }
   151  func (h accountBloomHasher) BlockSize() int                    { panic("not implemented") }
   152  func (h accountBloomHasher) Size() int                         { return 8 }
   153  func (h accountBloomHasher) Sum64() uint64 {
   154  	return binary.BigEndian.Uint64(h[bloomAccountHasherOffset : bloomAccountHasherOffset+8])
   155  }
   156  
   157  // storageBloomHasher is a wrapper around a [2]common.Hash to satisfy the interface
   158  // API requirements of the bloom library used. It's used to convert an account
   159  // hash into a 64 bit mini hash.
   160  type storageBloomHasher [2]common.Hash
   161  
   162  func (h storageBloomHasher) Write(p []byte) (n int, err error) { panic("not implemented") }
   163  func (h storageBloomHasher) Sum(b []byte) []byte               { panic("not implemented") }
   164  func (h storageBloomHasher) Reset()                            { panic("not implemented") }
   165  func (h storageBloomHasher) BlockSize() int                    { panic("not implemented") }
   166  func (h storageBloomHasher) Size() int                         { return 8 }
   167  func (h storageBloomHasher) Sum64() uint64 {
   168  	return binary.BigEndian.Uint64(h[0][bloomStorageHasherOffset:bloomStorageHasherOffset+8]) ^
   169  		binary.BigEndian.Uint64(h[1][bloomStorageHasherOffset:bloomStorageHasherOffset+8])
   170  }
   171  
   172  // newDiffLayer creates a new diff on top of an existing snapshot, whether that's a low
   173  // level persistent database or a hierarchical diff already.
   174  func newDiffLayer(parent snapshot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte, verified chan struct{}) *diffLayer {
   175  	// Create the new layer with some pre-allocated data segments
   176  	dl := &diffLayer{
   177  		parent:      parent,
   178  		root:        root,
   179  		destructSet: destructs,
   180  		accountData: accounts,
   181  		storageData: storage,
   182  		storageList: make(map[common.Hash][]common.Hash),
   183  		verifiedCh:  verified,
   184  	}
   185  	switch parent := parent.(type) {
   186  	case *diskLayer:
   187  		dl.rebloom(parent)
   188  	case *diffLayer:
   189  		dl.rebloom(parent.origin)
   190  	default:
   191  		panic("unknown parent type")
   192  	}
   193  	// Sanity check that accounts or storage slots are never nil
   194  	for accountHash, blob := range accounts {
   195  		if blob == nil {
   196  			panic(fmt.Sprintf("account %#x nil", accountHash))
   197  		}
   198  		// Determine memory size and track the dirty writes
   199  		dl.memory += uint64(common.HashLength + len(blob))
   200  		snapshotDirtyAccountWriteMeter.Mark(int64(len(blob)))
   201  	}
   202  	for accountHash, slots := range storage {
   203  		if slots == nil {
   204  			panic(fmt.Sprintf("storage %#x nil", accountHash))
   205  		}
   206  		// Determine memory size and track the dirty writes
   207  		for _, data := range slots {
   208  			dl.memory += uint64(common.HashLength + len(data))
   209  			snapshotDirtyStorageWriteMeter.Mark(int64(len(data)))
   210  		}
   211  	}
   212  	dl.memory += uint64(len(destructs) * common.HashLength)
   213  	return dl
   214  }
   215  
   216  // rebloom discards the layer's current bloom and rebuilds it from scratch based
   217  // on the parent's and the local diffs.
   218  func (dl *diffLayer) rebloom(origin *diskLayer) {
   219  	dl.lock.Lock()
   220  	defer dl.lock.Unlock()
   221  
   222  	defer func(start time.Time) {
   223  		snapshotBloomIndexTimer.Update(time.Since(start))
   224  	}(time.Now())
   225  
   226  	// Inject the new origin that triggered the rebloom
   227  	dl.origin = origin
   228  
   229  	// Retrieve the parent bloom or create a fresh empty one
   230  	if parent, ok := dl.parent.(*diffLayer); ok {
   231  		parent.lock.RLock()
   232  		dl.diffed, _ = parent.diffed.Copy()
   233  		parent.lock.RUnlock()
   234  	} else {
   235  		dl.diffed, _ = bloomfilter.New(uint64(bloomSize), uint64(bloomFuncs))
   236  	}
   237  	// Iterate over all the accounts and storage slots and index them
   238  	for hash := range dl.destructSet {
   239  		dl.diffed.Add(destructBloomHasher(hash))
   240  	}
   241  	for hash := range dl.accountData {
   242  		dl.diffed.Add(accountBloomHasher(hash))
   243  	}
   244  	for accountHash, slots := range dl.storageData {
   245  		for storageHash := range slots {
   246  			dl.diffed.Add(storageBloomHasher{accountHash, storageHash})
   247  		}
   248  	}
   249  	// Calculate the current false positive rate and update the error rate meter.
   250  	// This is a bit cheating because subsequent layers will overwrite it, but it
   251  	// should be fine, we're only interested in ballpark figures.
   252  	k := float64(dl.diffed.K())
   253  	n := float64(dl.diffed.N())
   254  	m := float64(dl.diffed.M())
   255  	snapshotBloomErrorGauge.Update(math.Pow(1.0-math.Exp((-k)*(n+0.5)/(m-1)), k))
   256  }
   257  
   258  // Root returns the root hash for which this snapshot was made.
   259  func (dl *diffLayer) Root() common.Hash {
   260  	return dl.root
   261  }
   262  
   263  // WaitAndGetVerifyRes will wait until the diff layer been verified and return the verification result
   264  func (dl *diffLayer) WaitAndGetVerifyRes() bool {
   265  	if dl.verifiedCh == nil {
   266  		return true
   267  	}
   268  	<-dl.verifiedCh
   269  	return dl.valid
   270  }
   271  
   272  func (dl *diffLayer) MarkValid() {
   273  	dl.valid = true
   274  }
   275  
   276  // Represent whether the difflayer is been verified, does not means it is a valid or invalid difflayer
   277  func (dl *diffLayer) Verified() bool {
   278  	if dl.verifiedCh == nil {
   279  		return true
   280  	}
   281  	select {
   282  	case <-dl.verifiedCh:
   283  		return true
   284  	default:
   285  		return false
   286  	}
   287  }
   288  
   289  // Parent returns the subsequent layer of a diff layer.
   290  func (dl *diffLayer) Parent() snapshot {
   291  	return dl.parent
   292  }
   293  
   294  // Stale return whether this layer has become stale (was flattened across) or if
   295  // it's still live.
   296  func (dl *diffLayer) Stale() bool {
   297  	return atomic.LoadUint32(&dl.stale) != 0
   298  }
   299  
   300  // Account directly retrieves the account associated with a particular hash in
   301  // the snapshot slim data format.
   302  func (dl *diffLayer) Account(hash common.Hash) (*Account, error) {
   303  	data, err := dl.AccountRLP(hash)
   304  	if err != nil {
   305  		return nil, err
   306  	}
   307  	if len(data) == 0 { // can be both nil and []byte{}
   308  		return nil, nil
   309  	}
   310  	account := new(Account)
   311  	if err := rlp.DecodeBytes(data, account); err != nil {
   312  		panic(err)
   313  	}
   314  	return account, nil
   315  }
   316  
   317  // AccountRLP directly retrieves the account RLP associated with a particular
   318  // hash in the snapshot slim data format.
   319  //
   320  // Note the returned account is not a copy, please don't modify it.
   321  func (dl *diffLayer) AccountRLP(hash common.Hash) ([]byte, error) {
   322  	// Check the bloom filter first whether there's even a point in reaching into
   323  	// all the maps in all the layers below
   324  	dl.lock.RLock()
   325  	hit := dl.diffed.Contains(accountBloomHasher(hash))
   326  	if !hit {
   327  		hit = dl.diffed.Contains(destructBloomHasher(hash))
   328  	}
   329  	var origin *diskLayer
   330  	if !hit {
   331  		origin = dl.origin // extract origin while holding the lock
   332  	}
   333  	dl.lock.RUnlock()
   334  
   335  	// If the bloom filter misses, don't even bother with traversing the memory
   336  	// diff layers, reach straight into the bottom persistent disk layer
   337  	if origin != nil {
   338  		snapshotBloomAccountMissMeter.Mark(1)
   339  		return origin.AccountRLP(hash)
   340  	}
   341  	// The bloom filter hit, start poking in the internal maps
   342  	return dl.accountRLP(hash, 0)
   343  }
   344  
   345  // accountRLP is an internal version of AccountRLP that skips the bloom filter
   346  // checks and uses the internal maps to try and retrieve the data. It's meant
   347  // to be used if a higher layer's bloom filter hit already.
   348  func (dl *diffLayer) accountRLP(hash common.Hash, depth int) ([]byte, error) {
   349  	dl.lock.RLock()
   350  	defer dl.lock.RUnlock()
   351  
   352  	// If the layer was flattened into, consider it invalid (any live reference to
   353  	// the original should be marked as unusable).
   354  	if dl.Stale() {
   355  		return nil, ErrSnapshotStale
   356  	}
   357  	// If the account is known locally, return it
   358  	if data, ok := dl.accountData[hash]; ok {
   359  		snapshotDirtyAccountHitMeter.Mark(1)
   360  		snapshotDirtyAccountHitDepthHist.Update(int64(depth))
   361  		snapshotDirtyAccountReadMeter.Mark(int64(len(data)))
   362  		snapshotBloomAccountTrueHitMeter.Mark(1)
   363  		return data, nil
   364  	}
   365  	// If the account is known locally, but deleted, return it
   366  	if _, ok := dl.destructSet[hash]; ok {
   367  		snapshotDirtyAccountHitMeter.Mark(1)
   368  		snapshotDirtyAccountHitDepthHist.Update(int64(depth))
   369  		snapshotDirtyAccountInexMeter.Mark(1)
   370  		snapshotBloomAccountTrueHitMeter.Mark(1)
   371  		return nil, nil
   372  	}
   373  	// Account unknown to this diff, resolve from parent
   374  	if diff, ok := dl.parent.(*diffLayer); ok {
   375  		return diff.accountRLP(hash, depth+1)
   376  	}
   377  	// Failed to resolve through diff layers, mark a bloom error and use the disk
   378  	snapshotBloomAccountFalseHitMeter.Mark(1)
   379  	return dl.parent.AccountRLP(hash)
   380  }
   381  
   382  // Storage directly retrieves the storage data associated with a particular hash,
   383  // within a particular account. If the slot is unknown to this diff, it's parent
   384  // is consulted.
   385  //
   386  // Note the returned slot is not a copy, please don't modify it.
   387  func (dl *diffLayer) Storage(accountHash, storageHash common.Hash) ([]byte, error) {
   388  	// Check the bloom filter first whether there's even a point in reaching into
   389  	// all the maps in all the layers below
   390  	dl.lock.RLock()
   391  	hit := dl.diffed.Contains(storageBloomHasher{accountHash, storageHash})
   392  	if !hit {
   393  		hit = dl.diffed.Contains(destructBloomHasher(accountHash))
   394  	}
   395  	var origin *diskLayer
   396  	if !hit {
   397  		origin = dl.origin // extract origin while holding the lock
   398  	}
   399  	dl.lock.RUnlock()
   400  
   401  	// If the bloom filter misses, don't even bother with traversing the memory
   402  	// diff layers, reach straight into the bottom persistent disk layer
   403  	if origin != nil {
   404  		snapshotBloomStorageMissMeter.Mark(1)
   405  		return origin.Storage(accountHash, storageHash)
   406  	}
   407  	// The bloom filter hit, start poking in the internal maps
   408  	return dl.storage(accountHash, storageHash, 0)
   409  }
   410  
   411  // storage is an internal version of Storage that skips the bloom filter checks
   412  // and uses the internal maps to try and retrieve the data. It's meant  to be
   413  // used if a higher layer's bloom filter hit already.
   414  func (dl *diffLayer) storage(accountHash, storageHash common.Hash, depth int) ([]byte, error) {
   415  	dl.lock.RLock()
   416  	defer dl.lock.RUnlock()
   417  
   418  	// If the layer was flattened into, consider it invalid (any live reference to
   419  	// the original should be marked as unusable).
   420  	if dl.Stale() {
   421  		return nil, ErrSnapshotStale
   422  	}
   423  	// If the account is known locally, try to resolve the slot locally
   424  	if storage, ok := dl.storageData[accountHash]; ok {
   425  		if data, ok := storage[storageHash]; ok {
   426  			snapshotDirtyStorageHitMeter.Mark(1)
   427  			//snapshotDirtyStorageHitDepthHist.Update(int64(depth))
   428  			if n := len(data); n > 0 {
   429  				snapshotDirtyStorageReadMeter.Mark(int64(n))
   430  			} else {
   431  				snapshotDirtyStorageInexMeter.Mark(1)
   432  			}
   433  			snapshotBloomStorageTrueHitMeter.Mark(1)
   434  			return data, nil
   435  		}
   436  	}
   437  	// If the account is known locally, but deleted, return an empty slot
   438  	if _, ok := dl.destructSet[accountHash]; ok {
   439  		snapshotDirtyStorageHitMeter.Mark(1)
   440  		//snapshotDirtyStorageHitDepthHist.Update(int64(depth))
   441  		snapshotDirtyStorageInexMeter.Mark(1)
   442  		snapshotBloomStorageTrueHitMeter.Mark(1)
   443  		return nil, nil
   444  	}
   445  	// Storage slot unknown to this diff, resolve from parent
   446  	if diff, ok := dl.parent.(*diffLayer); ok {
   447  		return diff.storage(accountHash, storageHash, depth+1)
   448  	}
   449  	// Failed to resolve through diff layers, mark a bloom error and use the disk
   450  	snapshotBloomStorageFalseHitMeter.Mark(1)
   451  	return dl.parent.Storage(accountHash, storageHash)
   452  }
   453  
   454  // Update creates a new layer on top of the existing snapshot diff tree with
   455  // the specified data items.
   456  func (dl *diffLayer) Update(blockRoot common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte, verified chan struct{}) *diffLayer {
   457  	return newDiffLayer(dl, blockRoot, destructs, accounts, storage, verified)
   458  }
   459  
   460  // flatten pushes all data from this point downwards, flattening everything into
   461  // a single diff at the bottom. Since usually the lowermost diff is the largest,
   462  // the flattening builds up from there in reverse.
   463  func (dl *diffLayer) flatten() snapshot {
   464  	// If the parent is not diff, we're the first in line, return unmodified
   465  	parent, ok := dl.parent.(*diffLayer)
   466  	if !ok {
   467  		return dl
   468  	}
   469  	// Parent is a diff, flatten it first (note, apart from weird corned cases,
   470  	// flatten will realistically only ever merge 1 layer, so there's no need to
   471  	// be smarter about grouping flattens together).
   472  	parent = parent.flatten().(*diffLayer)
   473  
   474  	parent.lock.Lock()
   475  	defer parent.lock.Unlock()
   476  
   477  	// Before actually writing all our data to the parent, first ensure that the
   478  	// parent hasn't been 'corrupted' by someone else already flattening into it
   479  	if atomic.SwapUint32(&parent.stale, 1) != 0 {
   480  		panic("parent diff layer is stale") // we've flattened into the same parent from two children, boo
   481  	}
   482  	// Overwrite all the updated accounts blindly, merge the sorted list
   483  	for hash := range dl.destructSet {
   484  		parent.destructSet[hash] = struct{}{}
   485  		delete(parent.accountData, hash)
   486  		delete(parent.storageData, hash)
   487  	}
   488  	for hash, data := range dl.accountData {
   489  		parent.accountData[hash] = data
   490  	}
   491  	// Overwrite all the updated storage slots (individually)
   492  	for accountHash, storage := range dl.storageData {
   493  		// If storage didn't exist (or was deleted) in the parent, overwrite blindly
   494  		if _, ok := parent.storageData[accountHash]; !ok {
   495  			parent.storageData[accountHash] = storage
   496  			continue
   497  		}
   498  		// Storage exists in both parent and child, merge the slots
   499  		comboData := parent.storageData[accountHash]
   500  		for storageHash, data := range storage {
   501  			comboData[storageHash] = data
   502  		}
   503  		parent.storageData[accountHash] = comboData
   504  	}
   505  	// Return the combo parent
   506  	return &diffLayer{
   507  		parent:      parent.parent,
   508  		origin:      parent.origin,
   509  		root:        dl.root,
   510  		destructSet: parent.destructSet,
   511  		accountData: parent.accountData,
   512  		storageData: parent.storageData,
   513  		storageList: make(map[common.Hash][]common.Hash),
   514  		diffed:      dl.diffed,
   515  		memory:      parent.memory + dl.memory,
   516  	}
   517  }
   518  
   519  // AccountList returns a sorted list of all accounts in this diffLayer, including
   520  // the deleted ones.
   521  //
   522  // Note, the returned slice is not a copy, so do not modify it.
   523  func (dl *diffLayer) AccountList() []common.Hash {
   524  	// If an old list already exists, return it
   525  	dl.lock.RLock()
   526  	list := dl.accountList
   527  	dl.lock.RUnlock()
   528  
   529  	if list != nil {
   530  		return list
   531  	}
   532  	// No old sorted account list exists, generate a new one
   533  	dl.lock.Lock()
   534  	defer dl.lock.Unlock()
   535  
   536  	dl.accountList = make([]common.Hash, 0, len(dl.destructSet)+len(dl.accountData))
   537  	for hash := range dl.accountData {
   538  		dl.accountList = append(dl.accountList, hash)
   539  	}
   540  	for hash := range dl.destructSet {
   541  		if _, ok := dl.accountData[hash]; !ok {
   542  			dl.accountList = append(dl.accountList, hash)
   543  		}
   544  	}
   545  	sort.Sort(hashes(dl.accountList))
   546  	dl.memory += uint64(len(dl.accountList) * common.HashLength)
   547  	return dl.accountList
   548  }
   549  
   550  // StorageList returns a sorted list of all storage slot hashes in this diffLayer
   551  // for the given account. If the whole storage is destructed in this layer, then
   552  // an additional flag *destructed = true* will be returned, otherwise the flag is
   553  // false. Besides, the returned list will include the hash of deleted storage slot.
   554  // Note a special case is an account is deleted in a prior tx but is recreated in
   555  // the following tx with some storage slots set. In this case the returned list is
   556  // not empty but the flag is true.
   557  //
   558  // Note, the returned slice is not a copy, so do not modify it.
   559  func (dl *diffLayer) StorageList(accountHash common.Hash) ([]common.Hash, bool) {
   560  	dl.lock.RLock()
   561  	_, destructed := dl.destructSet[accountHash]
   562  	if _, ok := dl.storageData[accountHash]; !ok {
   563  		// Account not tracked by this layer
   564  		dl.lock.RUnlock()
   565  		return nil, destructed
   566  	}
   567  	// If an old list already exists, return it
   568  	if list, exist := dl.storageList[accountHash]; exist {
   569  		dl.lock.RUnlock()
   570  		return list, destructed // the cached list can't be nil
   571  	}
   572  	dl.lock.RUnlock()
   573  
   574  	// No old sorted account list exists, generate a new one
   575  	dl.lock.Lock()
   576  	defer dl.lock.Unlock()
   577  
   578  	storageMap := dl.storageData[accountHash]
   579  	storageList := make([]common.Hash, 0, len(storageMap))
   580  	for k := range storageMap {
   581  		storageList = append(storageList, k)
   582  	}
   583  	sort.Sort(hashes(storageList))
   584  	dl.storageList[accountHash] = storageList
   585  	dl.memory += uint64(len(dl.storageList)*common.HashLength + common.HashLength)
   586  	return storageList, destructed
   587  }