gitlab.com/yannislg/go-pulse@v0.0.0-20210722055913-a3e24e95638d/core/state/snapshot/generate.go (about)

     1  // Copyright 2019 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package snapshot
    18  
    19  import (
    20  	"bytes"
    21  	"encoding/binary"
    22  	"math/big"
    23  	"time"
    24  
    25  	"github.com/VictoriaMetrics/fastcache"
    26  	"github.com/ethereum/go-ethereum/common"
    27  	"github.com/ethereum/go-ethereum/common/math"
    28  	"github.com/ethereum/go-ethereum/core/rawdb"
    29  	"github.com/ethereum/go-ethereum/crypto"
    30  	"github.com/ethereum/go-ethereum/ethdb"
    31  	"github.com/ethereum/go-ethereum/log"
    32  	"github.com/ethereum/go-ethereum/rlp"
    33  	"github.com/ethereum/go-ethereum/trie"
    34  )
    35  
    36  var (
    37  	// emptyRoot is the known root hash of an empty trie.
    38  	emptyRoot = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
    39  
    40  	// emptyCode is the known hash of the empty EVM bytecode.
    41  	emptyCode = crypto.Keccak256Hash(nil)
    42  )
    43  
    44  // generatorStats is a collection of statistics gathered by the snapshot generator
    45  // for  logging purposes.
    46  type generatorStats struct {
    47  	wiping   chan struct{}      // Notification channel if wiping is in progress
    48  	origin   uint64             // Origin prefix where generation started
    49  	start    time.Time          // Timestamp when generation started
    50  	accounts uint64             // Number of accounts indexed
    51  	slots    uint64             // Number of storage slots indexed
    52  	storage  common.StorageSize // Account and storage slot size
    53  }
    54  
    55  // Log creates an contextual log with the given message and the context pulled
    56  // from the internally maintained statistics.
    57  func (gs *generatorStats) Log(msg string, marker []byte) {
    58  	var ctx []interface{}
    59  
    60  	// Figure out whether we're after or within an account
    61  	switch len(marker) {
    62  	case common.HashLength:
    63  		ctx = append(ctx, []interface{}{"at", common.BytesToHash(marker)}...)
    64  	case 2 * common.HashLength:
    65  		ctx = append(ctx, []interface{}{
    66  			"in", common.BytesToHash(marker[:common.HashLength]),
    67  			"at", common.BytesToHash(marker[common.HashLength:]),
    68  		}...)
    69  	}
    70  	// Add the usual measurements
    71  	ctx = append(ctx, []interface{}{
    72  		"accounts", gs.accounts,
    73  		"slots", gs.slots,
    74  		"storage", gs.storage,
    75  		"elapsed", common.PrettyDuration(time.Since(gs.start)),
    76  	}...)
    77  	// Calculate the estimated indexing time based on current stats
    78  	if len(marker) > 0 {
    79  		if done := binary.BigEndian.Uint64(marker[:8]) - gs.origin; done > 0 {
    80  			left := math.MaxUint64 - binary.BigEndian.Uint64(marker[:8])
    81  
    82  			speed := done/uint64(time.Since(gs.start)/time.Millisecond+1) + 1 // +1s to avoid division by zero
    83  			ctx = append(ctx, []interface{}{
    84  				"eta", common.PrettyDuration(time.Duration(left/speed) * time.Millisecond),
    85  			}...)
    86  		}
    87  	}
    88  	log.Info(msg, ctx...)
    89  }
    90  
    91  // generateSnapshot regenerates a brand new snapshot based on an existing state
    92  // database and head block asynchronously. The snapshot is returned immediately
    93  // and generation is continued in the background until done.
    94  func generateSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, wiper chan struct{}) *diskLayer {
    95  	// Wipe any previously existing snapshot from the database if no wiper is
    96  	// currently in progress.
    97  	if wiper == nil {
    98  		wiper = wipeSnapshot(diskdb, true)
    99  	}
   100  	// Create a new disk layer with an initialized state marker at zero
   101  	rawdb.WriteSnapshotRoot(diskdb, root)
   102  
   103  	base := &diskLayer{
   104  		diskdb:     diskdb,
   105  		triedb:     triedb,
   106  		root:       root,
   107  		cache:      fastcache.New(cache * 1024 * 1024),
   108  		genMarker:  []byte{}, // Initialized but empty!
   109  		genPending: make(chan struct{}),
   110  		genAbort:   make(chan chan *generatorStats),
   111  	}
   112  	go base.generate(&generatorStats{wiping: wiper, start: time.Now()})
   113  	return base
   114  }
   115  
   116  // generate is a background thread that iterates over the state and storage tries,
   117  // constructing the state snapshot. All the arguments are purely for statistics
   118  // gethering and logging, since the method surfs the blocks as they arrive, often
   119  // being restarted.
   120  func (dl *diskLayer) generate(stats *generatorStats) {
   121  	// If a database wipe is in operation, wait until it's done
   122  	if stats.wiping != nil {
   123  		stats.Log("Wiper running, state snapshotting paused", dl.genMarker)
   124  		select {
   125  		// If wiper is done, resume normal mode of operation
   126  		case <-stats.wiping:
   127  			stats.wiping = nil
   128  			stats.start = time.Now()
   129  
   130  		// If generator was aboted during wipe, return
   131  		case abort := <-dl.genAbort:
   132  			abort <- stats
   133  			return
   134  		}
   135  	}
   136  	// Create an account and state iterator pointing to the current generator marker
   137  	accTrie, err := trie.NewSecure(dl.root, dl.triedb)
   138  	if err != nil {
   139  		// The account trie is missing (GC), surf the chain until one becomes available
   140  		stats.Log("Trie missing, state snapshotting paused", dl.genMarker)
   141  
   142  		abort := <-dl.genAbort
   143  		abort <- stats
   144  		return
   145  	}
   146  	stats.Log("Resuming state snapshot generation", dl.genMarker)
   147  
   148  	var accMarker []byte
   149  	if len(dl.genMarker) > 0 { // []byte{} is the start, use nil for that
   150  		accMarker = dl.genMarker[:common.HashLength]
   151  	}
   152  	accIt := trie.NewIterator(accTrie.NodeIterator(accMarker))
   153  	batch := dl.diskdb.NewBatch()
   154  
   155  	// Iterate from the previous marker and continue generating the state snapshot
   156  	logged := time.Now()
   157  	for accIt.Next() {
   158  		// Retrieve the current account and flatten it into the internal format
   159  		accountHash := common.BytesToHash(accIt.Key)
   160  
   161  		var acc struct {
   162  			Nonce    uint64
   163  			Balance  *big.Int
   164  			Root     common.Hash
   165  			CodeHash []byte
   166  		}
   167  		if err := rlp.DecodeBytes(accIt.Value, &acc); err != nil {
   168  			log.Crit("Invalid account encountered during snapshot creation", "err", err)
   169  		}
   170  		data := AccountRLP(acc.Nonce, acc.Balance, acc.Root, acc.CodeHash)
   171  
   172  		// If the account is not yet in-progress, write it out
   173  		if accMarker == nil || !bytes.Equal(accountHash[:], accMarker) {
   174  			rawdb.WriteAccountSnapshot(batch, accountHash, data)
   175  			stats.storage += common.StorageSize(1 + common.HashLength + len(data))
   176  			stats.accounts++
   177  		}
   178  		// If we've exceeded our batch allowance or termination was requested, flush to disk
   179  		var abort chan *generatorStats
   180  		select {
   181  		case abort = <-dl.genAbort:
   182  		default:
   183  		}
   184  		if batch.ValueSize() > ethdb.IdealBatchSize || abort != nil {
   185  			// Only write and set the marker if we actually did something useful
   186  			if batch.ValueSize() > 0 {
   187  				batch.Write()
   188  				batch.Reset()
   189  
   190  				dl.lock.Lock()
   191  				dl.genMarker = accountHash[:]
   192  				dl.lock.Unlock()
   193  			}
   194  			if abort != nil {
   195  				stats.Log("Aborting state snapshot generation", accountHash[:])
   196  				abort <- stats
   197  				return
   198  			}
   199  		}
   200  		// If the account is in-progress, continue where we left off (otherwise iterate all)
   201  		if acc.Root != emptyRoot {
   202  			storeTrie, err := trie.NewSecure(acc.Root, dl.triedb)
   203  			if err != nil {
   204  				log.Crit("Storage trie inaccessible for snapshot generation", "err", err)
   205  			}
   206  			var storeMarker []byte
   207  			if accMarker != nil && bytes.Equal(accountHash[:], accMarker) && len(dl.genMarker) > common.HashLength {
   208  				storeMarker = dl.genMarker[common.HashLength:]
   209  			}
   210  			storeIt := trie.NewIterator(storeTrie.NodeIterator(storeMarker))
   211  			for storeIt.Next() {
   212  				rawdb.WriteStorageSnapshot(batch, accountHash, common.BytesToHash(storeIt.Key), storeIt.Value)
   213  				stats.storage += common.StorageSize(1 + 2*common.HashLength + len(storeIt.Value))
   214  				stats.slots++
   215  
   216  				// If we've exceeded our batch allowance or termination was requested, flush to disk
   217  				var abort chan *generatorStats
   218  				select {
   219  				case abort = <-dl.genAbort:
   220  				default:
   221  				}
   222  				if batch.ValueSize() > ethdb.IdealBatchSize || abort != nil {
   223  					// Only write and set the marker if we actually did something useful
   224  					if batch.ValueSize() > 0 {
   225  						batch.Write()
   226  						batch.Reset()
   227  
   228  						dl.lock.Lock()
   229  						dl.genMarker = append(accountHash[:], storeIt.Key...)
   230  						dl.lock.Unlock()
   231  					}
   232  					if abort != nil {
   233  						stats.Log("Aborting state snapshot generation", append(accountHash[:], storeIt.Key...))
   234  						abort <- stats
   235  						return
   236  					}
   237  				}
   238  			}
   239  		}
   240  		if time.Since(logged) > 8*time.Second {
   241  			stats.Log("Generating state snapshot", accIt.Key)
   242  			logged = time.Now()
   243  		}
   244  		// Some account processed, unmark the marker
   245  		accMarker = nil
   246  	}
   247  	// Snapshot fully generated, set the marker to nil
   248  	if batch.ValueSize() > 0 {
   249  		batch.Write()
   250  	}
   251  	log.Info("Generated state snapshot", "accounts", stats.accounts, "slots", stats.slots,
   252  		"storage", stats.storage, "elapsed", common.PrettyDuration(time.Since(stats.start)))
   253  
   254  	dl.lock.Lock()
   255  	dl.genMarker = nil
   256  	close(dl.genPending)
   257  	dl.lock.Unlock()
   258  
   259  	// Someone will be looking for us, wait it out
   260  	abort := <-dl.genAbort
   261  	abort <- nil
   262  }