github.com/MetalBlockchain/subnet-evm@v0.4.9/core/state/snapshot/conversion.go (about)

     1  // (c) 2019-2020, Ava Labs, Inc.
     2  //
     3  // This file is a derived work, based on the go-ethereum library whose original
     4  // notices appear below.
     5  //
     6  // It is distributed under a license compatible with the licensing terms of the
     7  // original code from which it is derived.
     8  //
     9  // Much love to the original authors for their work.
    10  // **********
    11  // Copyright 2020 The go-ethereum Authors
    12  // This file is part of the go-ethereum library.
    13  //
    14  // The go-ethereum library is free software: you can redistribute it and/or modify
    15  // it under the terms of the GNU Lesser General Public License as published by
    16  // the Free Software Foundation, either version 3 of the License, or
    17  // (at your option) any later version.
    18  //
    19  // The go-ethereum library is distributed in the hope that it will be useful,
    20  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    21  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    22  // GNU Lesser General Public License for more details.
    23  //
    24  // You should have received a copy of the GNU Lesser General Public License
    25  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    26  
    27  package snapshot
    28  
    29  import (
    30  	"bytes"
    31  	"encoding/binary"
    32  	"errors"
    33  	"fmt"
    34  	"math"
    35  	"runtime"
    36  	"sync"
    37  	"time"
    38  
    39  	"github.com/MetalBlockchain/subnet-evm/core/rawdb"
    40  	"github.com/MetalBlockchain/subnet-evm/ethdb"
    41  	"github.com/MetalBlockchain/subnet-evm/trie"
    42  	"github.com/ethereum/go-ethereum/common"
    43  	"github.com/ethereum/go-ethereum/log"
    44  	"github.com/ethereum/go-ethereum/rlp"
    45  )
    46  
    47  // trieKV represents a trie key-value pair
    48  type trieKV struct {
    49  	key   common.Hash
    50  	value []byte
    51  }
    52  
    53  type (
    54  	// trieGeneratorFn is the interface of trie generation which can
    55  	// be implemented by different trie algorithm.
    56  	trieGeneratorFn func(db ethdb.KeyValueWriter, owner common.Hash, in chan (trieKV), out chan (common.Hash))
    57  
    58  	// leafCallbackFn is the callback invoked at the leaves of the trie,
    59  	// returns the subtrie root with the specified subtrie identifier.
    60  	leafCallbackFn func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error)
    61  )
    62  
    63  // GenerateAccountTrieRoot takes an account iterator and reproduces the root hash.
    64  func GenerateAccountTrieRoot(it AccountIterator) (common.Hash, error) {
    65  	return generateTrieRoot(nil, it, common.Hash{}, stackTrieGenerate, nil, newGenerateStats(), true)
    66  }
    67  
    68  // GenerateStorageTrieRoot takes a storage iterator and reproduces the root hash.
    69  func GenerateStorageTrieRoot(account common.Hash, it StorageIterator) (common.Hash, error) {
    70  	return generateTrieRoot(nil, it, account, stackTrieGenerate, nil, newGenerateStats(), true)
    71  }
    72  
    73  // GenerateTrie takes the whole snapshot tree as the input, traverses all the
    74  // accounts as well as the corresponding storages and regenerate the whole state
    75  // (account trie + all storage tries).
    76  func GenerateTrie(snaptree *Tree, root common.Hash, src ethdb.Database, dst ethdb.KeyValueWriter) error {
    77  	// Traverse all state by snapshot, re-generate the whole state trie
    78  	acctIt, err := snaptree.AccountIterator(root, common.Hash{}, false)
    79  	if err != nil {
    80  		return err // The required snapshot might not exist.
    81  	}
    82  	defer acctIt.Release()
    83  
    84  	got, err := generateTrieRoot(dst, acctIt, common.Hash{}, stackTrieGenerate, func(dst ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) {
    85  		// Migrate the code first, commit the contract code into the tmp db.
    86  		if codeHash != emptyCode {
    87  			code := rawdb.ReadCode(src, codeHash)
    88  			if len(code) == 0 {
    89  				return common.Hash{}, errors.New("failed to read contract code")
    90  			}
    91  			rawdb.WriteCode(dst, codeHash, code)
    92  		}
    93  		// Then migrate all storage trie nodes into the tmp db.
    94  		storageIt, err := snaptree.StorageIterator(root, accountHash, common.Hash{}, false)
    95  		if err != nil {
    96  			return common.Hash{}, err
    97  		}
    98  		defer storageIt.Release()
    99  
   100  		hash, err := generateTrieRoot(dst, storageIt, accountHash, stackTrieGenerate, nil, stat, false)
   101  		if err != nil {
   102  			return common.Hash{}, err
   103  		}
   104  		return hash, nil
   105  	}, newGenerateStats(), true)
   106  
   107  	if err != nil {
   108  		return err
   109  	}
   110  	if got != root {
   111  		return fmt.Errorf("state root hash mismatch: got %x, want %x", got, root)
   112  	}
   113  	return nil
   114  }
   115  
   116  // generateStats is a collection of statistics gathered by the trie generator
   117  // for logging purposes.
   118  type generateStats struct {
   119  	head  common.Hash
   120  	start time.Time
   121  
   122  	accounts uint64 // Number of accounts done (including those being crawled)
   123  	slots    uint64 // Number of storage slots done (including those being crawled)
   124  
   125  	slotsStart map[common.Hash]time.Time   // Start time for account slot crawling
   126  	slotsHead  map[common.Hash]common.Hash // Slot head for accounts being crawled
   127  
   128  	lock sync.RWMutex
   129  }
   130  
   131  // newGenerateStats creates a new generator stats.
   132  func newGenerateStats() *generateStats {
   133  	return &generateStats{
   134  		slotsStart: make(map[common.Hash]time.Time),
   135  		slotsHead:  make(map[common.Hash]common.Hash),
   136  		start:      time.Now(),
   137  	}
   138  }
   139  
   140  // progressAccounts updates the generator stats for the account range.
   141  func (stat *generateStats) progressAccounts(account common.Hash, done uint64) {
   142  	stat.lock.Lock()
   143  	defer stat.lock.Unlock()
   144  
   145  	stat.accounts += done
   146  	stat.head = account
   147  }
   148  
   149  // finishAccounts updates the gemerator stats for the finished account range.
   150  func (stat *generateStats) finishAccounts(done uint64) {
   151  	stat.lock.Lock()
   152  	defer stat.lock.Unlock()
   153  
   154  	stat.accounts += done
   155  }
   156  
   157  // progressContract updates the generator stats for a specific in-progress contract.
   158  func (stat *generateStats) progressContract(account common.Hash, slot common.Hash, done uint64) {
   159  	stat.lock.Lock()
   160  	defer stat.lock.Unlock()
   161  
   162  	stat.slots += done
   163  	stat.slotsHead[account] = slot
   164  	if _, ok := stat.slotsStart[account]; !ok {
   165  		stat.slotsStart[account] = time.Now()
   166  	}
   167  }
   168  
   169  // finishContract updates the generator stats for a specific just-finished contract.
   170  func (stat *generateStats) finishContract(account common.Hash, done uint64) {
   171  	stat.lock.Lock()
   172  	defer stat.lock.Unlock()
   173  
   174  	stat.slots += done
   175  	delete(stat.slotsHead, account)
   176  	delete(stat.slotsStart, account)
   177  }
   178  
   179  // report prints the cumulative progress statistic smartly.
   180  func (stat *generateStats) report() {
   181  	stat.lock.RLock()
   182  	defer stat.lock.RUnlock()
   183  
   184  	ctx := []interface{}{
   185  		"accounts", stat.accounts,
   186  		"slots", stat.slots,
   187  		"elapsed", common.PrettyDuration(time.Since(stat.start)),
   188  	}
   189  	if stat.accounts > 0 {
   190  		// If there's progress on the account trie, estimate the time to finish crawling it
   191  		if done := binary.BigEndian.Uint64(stat.head[:8]) / stat.accounts; done > 0 {
   192  			var (
   193  				left  = (math.MaxUint64 - binary.BigEndian.Uint64(stat.head[:8])) / stat.accounts
   194  				speed = done/uint64(time.Since(stat.start)/time.Millisecond+1) + 1 // +1s to avoid division by zero
   195  				eta   = time.Duration(left/speed) * time.Millisecond
   196  			)
   197  			// If there are large contract crawls in progress, estimate their finish time
   198  			for acc, head := range stat.slotsHead {
   199  				start := stat.slotsStart[acc]
   200  				if done := binary.BigEndian.Uint64(head[:8]); done > 0 {
   201  					var (
   202  						left  = math.MaxUint64 - binary.BigEndian.Uint64(head[:8])
   203  						speed = done/uint64(time.Since(start)/time.Millisecond+1) + 1 // +1s to avoid division by zero
   204  					)
   205  					// Override the ETA if larger than the largest until now
   206  					if slotETA := time.Duration(left/speed) * time.Millisecond; eta < slotETA {
   207  						eta = slotETA
   208  					}
   209  				}
   210  			}
   211  			ctx = append(ctx, []interface{}{
   212  				"eta", common.PrettyDuration(eta),
   213  			}...)
   214  		}
   215  	}
   216  	log.Info("Iterating state snapshot", ctx...)
   217  }
   218  
   219  // reportDone prints the last log when the whole generation is finished.
   220  func (stat *generateStats) reportDone() {
   221  	stat.lock.RLock()
   222  	defer stat.lock.RUnlock()
   223  
   224  	var ctx []interface{}
   225  	ctx = append(ctx, []interface{}{"accounts", stat.accounts}...)
   226  	if stat.slots != 0 {
   227  		ctx = append(ctx, []interface{}{"slots", stat.slots}...)
   228  	}
   229  	ctx = append(ctx, []interface{}{"elapsed", common.PrettyDuration(time.Since(stat.start))}...)
   230  	log.Info("Iterated snapshot", ctx...)
   231  }
   232  
   233  // runReport periodically prints the progress information.
   234  func runReport(stats *generateStats, stop chan bool) {
   235  	timer := time.NewTimer(0)
   236  	defer timer.Stop()
   237  
   238  	for {
   239  		select {
   240  		case <-timer.C:
   241  			stats.report()
   242  			timer.Reset(time.Second * 8)
   243  		case success := <-stop:
   244  			if success {
   245  				stats.reportDone()
   246  			}
   247  			return
   248  		}
   249  	}
   250  }
   251  
   252  // generateTrieRoot generates the trie hash based on the snapshot iterator.
   253  // It can be used for generating account trie, storage trie or even the
   254  // whole state which connects the accounts and the corresponding storages.
   255  func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, generatorFn trieGeneratorFn, leafCallback leafCallbackFn, stats *generateStats, report bool) (common.Hash, error) {
   256  	var (
   257  		in      = make(chan trieKV)         // chan to pass leaves
   258  		out     = make(chan common.Hash, 1) // chan to collect result
   259  		stoplog = make(chan bool, 1)        // 1-size buffer, works when logging is not enabled
   260  		wg      sync.WaitGroup
   261  	)
   262  	// Spin up a go-routine for trie hash re-generation
   263  	wg.Add(1)
   264  	go func() {
   265  		defer wg.Done()
   266  		generatorFn(db, account, in, out)
   267  	}()
   268  	// Spin up a go-routine for progress logging
   269  	if report && stats != nil {
   270  		wg.Add(1)
   271  		go func() {
   272  			defer wg.Done()
   273  			runReport(stats, stoplog)
   274  		}()
   275  	}
   276  	// Create a semaphore to assign tasks and collect results through. We'll pre-
   277  	// fill it with nils, thus using the same channel for both limiting concurrent
   278  	// processing and gathering results.
   279  	threads := runtime.NumCPU()
   280  	results := make(chan error, threads)
   281  	for i := 0; i < threads; i++ {
   282  		results <- nil // fill the semaphore
   283  	}
   284  	// stop is a helper function to shutdown the background threads
   285  	// and return the re-generated trie hash.
   286  	stop := func(fail error) (common.Hash, error) {
   287  		close(in)
   288  		result := <-out
   289  		for i := 0; i < threads; i++ {
   290  			if err := <-results; err != nil && fail == nil {
   291  				fail = err
   292  			}
   293  		}
   294  		stoplog <- fail == nil
   295  
   296  		wg.Wait()
   297  		return result, fail
   298  	}
   299  	var (
   300  		logged    = time.Now()
   301  		processed = uint64(0)
   302  		leaf      trieKV
   303  	)
   304  	// Start to feed leaves
   305  	for it.Next() {
   306  		if account == (common.Hash{}) {
   307  			var (
   308  				err      error
   309  				fullData []byte
   310  			)
   311  			if leafCallback == nil {
   312  				fullData, err = FullAccountRLP(it.(AccountIterator).Account())
   313  				if err != nil {
   314  					return stop(err)
   315  				}
   316  			} else {
   317  				// Wait until the semaphore allows us to continue, aborting if
   318  				// a sub-task failed
   319  				if err := <-results; err != nil {
   320  					results <- nil // stop will drain the results, add a noop back for this error we just consumed
   321  					return stop(err)
   322  				}
   323  				// Fetch the next account and process it concurrently
   324  				account, err := FullAccount(it.(AccountIterator).Account())
   325  				if err != nil {
   326  					return stop(err)
   327  				}
   328  				go func(hash common.Hash) {
   329  					subroot, err := leafCallback(db, hash, common.BytesToHash(account.CodeHash), stats)
   330  					if err != nil {
   331  						results <- err
   332  						return
   333  					}
   334  					if !bytes.Equal(account.Root, subroot.Bytes()) {
   335  						results <- fmt.Errorf("invalid subroot(path %x), want %x, have %x", hash, account.Root, subroot)
   336  						return
   337  					}
   338  					results <- nil
   339  				}(it.Hash())
   340  				fullData, err = rlp.EncodeToBytes(account)
   341  				if err != nil {
   342  					return stop(err)
   343  				}
   344  			}
   345  			leaf = trieKV{it.Hash(), fullData}
   346  		} else {
   347  			leaf = trieKV{it.Hash(), common.CopyBytes(it.(StorageIterator).Slot())}
   348  		}
   349  		in <- leaf
   350  
   351  		// Accumulate the generation statistic if it's required.
   352  		processed++
   353  		if time.Since(logged) > 3*time.Second && stats != nil {
   354  			if account == (common.Hash{}) {
   355  				stats.progressAccounts(it.Hash(), processed)
   356  			} else {
   357  				stats.progressContract(account, it.Hash(), processed)
   358  			}
   359  			logged, processed = time.Now(), 0
   360  		}
   361  	}
   362  	// Commit the last part statistic.
   363  	if processed > 0 && stats != nil {
   364  		if account == (common.Hash{}) {
   365  			stats.finishAccounts(processed)
   366  		} else {
   367  			stats.finishContract(account, processed)
   368  		}
   369  	}
   370  	return stop(nil)
   371  }
   372  
   373  func stackTrieGenerate(db ethdb.KeyValueWriter, owner common.Hash, in chan trieKV, out chan common.Hash) {
   374  	t := trie.NewStackTrieWithOwner(db, owner)
   375  	for leaf := range in {
   376  		t.TryUpdate(leaf.key[:], leaf.value)
   377  	}
   378  	var root common.Hash
   379  	if db == nil {
   380  		root = t.Hash()
   381  	} else {
   382  		root, _ = t.Commit()
   383  	}
   384  	out <- root
   385  }