gitlab.com/yannislg/go-pulse@v0.0.0-20210722055913-a3e24e95638d/core/state/snapshot/generate.go (about) 1 // Copyright 2019 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package snapshot 18 19 import ( 20 "bytes" 21 "encoding/binary" 22 "math/big" 23 "time" 24 25 "github.com/VictoriaMetrics/fastcache" 26 "github.com/ethereum/go-ethereum/common" 27 "github.com/ethereum/go-ethereum/common/math" 28 "github.com/ethereum/go-ethereum/core/rawdb" 29 "github.com/ethereum/go-ethereum/crypto" 30 "github.com/ethereum/go-ethereum/ethdb" 31 "github.com/ethereum/go-ethereum/log" 32 "github.com/ethereum/go-ethereum/rlp" 33 "github.com/ethereum/go-ethereum/trie" 34 ) 35 36 var ( 37 // emptyRoot is the known root hash of an empty trie. 38 emptyRoot = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") 39 40 // emptyCode is the known hash of the empty EVM bytecode. 41 emptyCode = crypto.Keccak256Hash(nil) 42 ) 43 44 // generatorStats is a collection of statistics gathered by the snapshot generator 45 // for logging purposes. 46 type generatorStats struct { 47 wiping chan struct{} // Notification channel if wiping is in progress 48 origin uint64 // Origin prefix where generation started 49 start time.Time // Timestamp when generation started 50 accounts uint64 // Number of accounts indexed 51 slots uint64 // Number of storage slots indexed 52 storage common.StorageSize // Account and storage slot size 53 } 54 55 // Log creates an contextual log with the given message and the context pulled 56 // from the internally maintained statistics. 57 func (gs *generatorStats) Log(msg string, marker []byte) { 58 var ctx []interface{} 59 60 // Figure out whether we're after or within an account 61 switch len(marker) { 62 case common.HashLength: 63 ctx = append(ctx, []interface{}{"at", common.BytesToHash(marker)}...) 64 case 2 * common.HashLength: 65 ctx = append(ctx, []interface{}{ 66 "in", common.BytesToHash(marker[:common.HashLength]), 67 "at", common.BytesToHash(marker[common.HashLength:]), 68 }...) 69 } 70 // Add the usual measurements 71 ctx = append(ctx, []interface{}{ 72 "accounts", gs.accounts, 73 "slots", gs.slots, 74 "storage", gs.storage, 75 "elapsed", common.PrettyDuration(time.Since(gs.start)), 76 }...) 77 // Calculate the estimated indexing time based on current stats 78 if len(marker) > 0 { 79 if done := binary.BigEndian.Uint64(marker[:8]) - gs.origin; done > 0 { 80 left := math.MaxUint64 - binary.BigEndian.Uint64(marker[:8]) 81 82 speed := done/uint64(time.Since(gs.start)/time.Millisecond+1) + 1 // +1s to avoid division by zero 83 ctx = append(ctx, []interface{}{ 84 "eta", common.PrettyDuration(time.Duration(left/speed) * time.Millisecond), 85 }...) 86 } 87 } 88 log.Info(msg, ctx...) 89 } 90 91 // generateSnapshot regenerates a brand new snapshot based on an existing state 92 // database and head block asynchronously. The snapshot is returned immediately 93 // and generation is continued in the background until done. 94 func generateSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, wiper chan struct{}) *diskLayer { 95 // Wipe any previously existing snapshot from the database if no wiper is 96 // currently in progress. 97 if wiper == nil { 98 wiper = wipeSnapshot(diskdb, true) 99 } 100 // Create a new disk layer with an initialized state marker at zero 101 rawdb.WriteSnapshotRoot(diskdb, root) 102 103 base := &diskLayer{ 104 diskdb: diskdb, 105 triedb: triedb, 106 root: root, 107 cache: fastcache.New(cache * 1024 * 1024), 108 genMarker: []byte{}, // Initialized but empty! 109 genPending: make(chan struct{}), 110 genAbort: make(chan chan *generatorStats), 111 } 112 go base.generate(&generatorStats{wiping: wiper, start: time.Now()}) 113 return base 114 } 115 116 // generate is a background thread that iterates over the state and storage tries, 117 // constructing the state snapshot. All the arguments are purely for statistics 118 // gethering and logging, since the method surfs the blocks as they arrive, often 119 // being restarted. 120 func (dl *diskLayer) generate(stats *generatorStats) { 121 // If a database wipe is in operation, wait until it's done 122 if stats.wiping != nil { 123 stats.Log("Wiper running, state snapshotting paused", dl.genMarker) 124 select { 125 // If wiper is done, resume normal mode of operation 126 case <-stats.wiping: 127 stats.wiping = nil 128 stats.start = time.Now() 129 130 // If generator was aboted during wipe, return 131 case abort := <-dl.genAbort: 132 abort <- stats 133 return 134 } 135 } 136 // Create an account and state iterator pointing to the current generator marker 137 accTrie, err := trie.NewSecure(dl.root, dl.triedb) 138 if err != nil { 139 // The account trie is missing (GC), surf the chain until one becomes available 140 stats.Log("Trie missing, state snapshotting paused", dl.genMarker) 141 142 abort := <-dl.genAbort 143 abort <- stats 144 return 145 } 146 stats.Log("Resuming state snapshot generation", dl.genMarker) 147 148 var accMarker []byte 149 if len(dl.genMarker) > 0 { // []byte{} is the start, use nil for that 150 accMarker = dl.genMarker[:common.HashLength] 151 } 152 accIt := trie.NewIterator(accTrie.NodeIterator(accMarker)) 153 batch := dl.diskdb.NewBatch() 154 155 // Iterate from the previous marker and continue generating the state snapshot 156 logged := time.Now() 157 for accIt.Next() { 158 // Retrieve the current account and flatten it into the internal format 159 accountHash := common.BytesToHash(accIt.Key) 160 161 var acc struct { 162 Nonce uint64 163 Balance *big.Int 164 Root common.Hash 165 CodeHash []byte 166 } 167 if err := rlp.DecodeBytes(accIt.Value, &acc); err != nil { 168 log.Crit("Invalid account encountered during snapshot creation", "err", err) 169 } 170 data := AccountRLP(acc.Nonce, acc.Balance, acc.Root, acc.CodeHash) 171 172 // If the account is not yet in-progress, write it out 173 if accMarker == nil || !bytes.Equal(accountHash[:], accMarker) { 174 rawdb.WriteAccountSnapshot(batch, accountHash, data) 175 stats.storage += common.StorageSize(1 + common.HashLength + len(data)) 176 stats.accounts++ 177 } 178 // If we've exceeded our batch allowance or termination was requested, flush to disk 179 var abort chan *generatorStats 180 select { 181 case abort = <-dl.genAbort: 182 default: 183 } 184 if batch.ValueSize() > ethdb.IdealBatchSize || abort != nil { 185 // Only write and set the marker if we actually did something useful 186 if batch.ValueSize() > 0 { 187 batch.Write() 188 batch.Reset() 189 190 dl.lock.Lock() 191 dl.genMarker = accountHash[:] 192 dl.lock.Unlock() 193 } 194 if abort != nil { 195 stats.Log("Aborting state snapshot generation", accountHash[:]) 196 abort <- stats 197 return 198 } 199 } 200 // If the account is in-progress, continue where we left off (otherwise iterate all) 201 if acc.Root != emptyRoot { 202 storeTrie, err := trie.NewSecure(acc.Root, dl.triedb) 203 if err != nil { 204 log.Crit("Storage trie inaccessible for snapshot generation", "err", err) 205 } 206 var storeMarker []byte 207 if accMarker != nil && bytes.Equal(accountHash[:], accMarker) && len(dl.genMarker) > common.HashLength { 208 storeMarker = dl.genMarker[common.HashLength:] 209 } 210 storeIt := trie.NewIterator(storeTrie.NodeIterator(storeMarker)) 211 for storeIt.Next() { 212 rawdb.WriteStorageSnapshot(batch, accountHash, common.BytesToHash(storeIt.Key), storeIt.Value) 213 stats.storage += common.StorageSize(1 + 2*common.HashLength + len(storeIt.Value)) 214 stats.slots++ 215 216 // If we've exceeded our batch allowance or termination was requested, flush to disk 217 var abort chan *generatorStats 218 select { 219 case abort = <-dl.genAbort: 220 default: 221 } 222 if batch.ValueSize() > ethdb.IdealBatchSize || abort != nil { 223 // Only write and set the marker if we actually did something useful 224 if batch.ValueSize() > 0 { 225 batch.Write() 226 batch.Reset() 227 228 dl.lock.Lock() 229 dl.genMarker = append(accountHash[:], storeIt.Key...) 230 dl.lock.Unlock() 231 } 232 if abort != nil { 233 stats.Log("Aborting state snapshot generation", append(accountHash[:], storeIt.Key...)) 234 abort <- stats 235 return 236 } 237 } 238 } 239 } 240 if time.Since(logged) > 8*time.Second { 241 stats.Log("Generating state snapshot", accIt.Key) 242 logged = time.Now() 243 } 244 // Some account processed, unmark the marker 245 accMarker = nil 246 } 247 // Snapshot fully generated, set the marker to nil 248 if batch.ValueSize() > 0 { 249 batch.Write() 250 } 251 log.Info("Generated state snapshot", "accounts", stats.accounts, "slots", stats.slots, 252 "storage", stats.storage, "elapsed", common.PrettyDuration(time.Since(stats.start))) 253 254 dl.lock.Lock() 255 dl.genMarker = nil 256 close(dl.genPending) 257 dl.lock.Unlock() 258 259 // Someone will be looking for us, wait it out 260 abort := <-dl.genAbort 261 abort <- nil 262 }