github.com/MetalBlockchain/subnet-evm@v0.4.9/core/state/snapshot/conversion.go (about) 1 // (c) 2019-2020, Ava Labs, Inc. 2 // 3 // This file is a derived work, based on the go-ethereum library whose original 4 // notices appear below. 5 // 6 // It is distributed under a license compatible with the licensing terms of the 7 // original code from which it is derived. 8 // 9 // Much love to the original authors for their work. 10 // ********** 11 // Copyright 2020 The go-ethereum Authors 12 // This file is part of the go-ethereum library. 13 // 14 // The go-ethereum library is free software: you can redistribute it and/or modify 15 // it under the terms of the GNU Lesser General Public License as published by 16 // the Free Software Foundation, either version 3 of the License, or 17 // (at your option) any later version. 18 // 19 // The go-ethereum library is distributed in the hope that it will be useful, 20 // but WITHOUT ANY WARRANTY; without even the implied warranty of 21 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 // GNU Lesser General Public License for more details. 23 // 24 // You should have received a copy of the GNU Lesser General Public License 25 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 26 27 package snapshot 28 29 import ( 30 "bytes" 31 "encoding/binary" 32 "errors" 33 "fmt" 34 "math" 35 "runtime" 36 "sync" 37 "time" 38 39 "github.com/MetalBlockchain/subnet-evm/core/rawdb" 40 "github.com/MetalBlockchain/subnet-evm/ethdb" 41 "github.com/MetalBlockchain/subnet-evm/trie" 42 "github.com/ethereum/go-ethereum/common" 43 "github.com/ethereum/go-ethereum/log" 44 "github.com/ethereum/go-ethereum/rlp" 45 ) 46 47 // trieKV represents a trie key-value pair 48 type trieKV struct { 49 key common.Hash 50 value []byte 51 } 52 53 type ( 54 // trieGeneratorFn is the interface of trie generation which can 55 // be implemented by different trie algorithm. 56 trieGeneratorFn func(db ethdb.KeyValueWriter, owner common.Hash, in chan (trieKV), out chan (common.Hash)) 57 58 // leafCallbackFn is the callback invoked at the leaves of the trie, 59 // returns the subtrie root with the specified subtrie identifier. 60 leafCallbackFn func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) 61 ) 62 63 // GenerateAccountTrieRoot takes an account iterator and reproduces the root hash. 64 func GenerateAccountTrieRoot(it AccountIterator) (common.Hash, error) { 65 return generateTrieRoot(nil, it, common.Hash{}, stackTrieGenerate, nil, newGenerateStats(), true) 66 } 67 68 // GenerateStorageTrieRoot takes a storage iterator and reproduces the root hash. 69 func GenerateStorageTrieRoot(account common.Hash, it StorageIterator) (common.Hash, error) { 70 return generateTrieRoot(nil, it, account, stackTrieGenerate, nil, newGenerateStats(), true) 71 } 72 73 // GenerateTrie takes the whole snapshot tree as the input, traverses all the 74 // accounts as well as the corresponding storages and regenerate the whole state 75 // (account trie + all storage tries). 76 func GenerateTrie(snaptree *Tree, root common.Hash, src ethdb.Database, dst ethdb.KeyValueWriter) error { 77 // Traverse all state by snapshot, re-generate the whole state trie 78 acctIt, err := snaptree.AccountIterator(root, common.Hash{}, false) 79 if err != nil { 80 return err // The required snapshot might not exist. 81 } 82 defer acctIt.Release() 83 84 got, err := generateTrieRoot(dst, acctIt, common.Hash{}, stackTrieGenerate, func(dst ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) { 85 // Migrate the code first, commit the contract code into the tmp db. 86 if codeHash != emptyCode { 87 code := rawdb.ReadCode(src, codeHash) 88 if len(code) == 0 { 89 return common.Hash{}, errors.New("failed to read contract code") 90 } 91 rawdb.WriteCode(dst, codeHash, code) 92 } 93 // Then migrate all storage trie nodes into the tmp db. 94 storageIt, err := snaptree.StorageIterator(root, accountHash, common.Hash{}, false) 95 if err != nil { 96 return common.Hash{}, err 97 } 98 defer storageIt.Release() 99 100 hash, err := generateTrieRoot(dst, storageIt, accountHash, stackTrieGenerate, nil, stat, false) 101 if err != nil { 102 return common.Hash{}, err 103 } 104 return hash, nil 105 }, newGenerateStats(), true) 106 107 if err != nil { 108 return err 109 } 110 if got != root { 111 return fmt.Errorf("state root hash mismatch: got %x, want %x", got, root) 112 } 113 return nil 114 } 115 116 // generateStats is a collection of statistics gathered by the trie generator 117 // for logging purposes. 118 type generateStats struct { 119 head common.Hash 120 start time.Time 121 122 accounts uint64 // Number of accounts done (including those being crawled) 123 slots uint64 // Number of storage slots done (including those being crawled) 124 125 slotsStart map[common.Hash]time.Time // Start time for account slot crawling 126 slotsHead map[common.Hash]common.Hash // Slot head for accounts being crawled 127 128 lock sync.RWMutex 129 } 130 131 // newGenerateStats creates a new generator stats. 132 func newGenerateStats() *generateStats { 133 return &generateStats{ 134 slotsStart: make(map[common.Hash]time.Time), 135 slotsHead: make(map[common.Hash]common.Hash), 136 start: time.Now(), 137 } 138 } 139 140 // progressAccounts updates the generator stats for the account range. 141 func (stat *generateStats) progressAccounts(account common.Hash, done uint64) { 142 stat.lock.Lock() 143 defer stat.lock.Unlock() 144 145 stat.accounts += done 146 stat.head = account 147 } 148 149 // finishAccounts updates the gemerator stats for the finished account range. 150 func (stat *generateStats) finishAccounts(done uint64) { 151 stat.lock.Lock() 152 defer stat.lock.Unlock() 153 154 stat.accounts += done 155 } 156 157 // progressContract updates the generator stats for a specific in-progress contract. 158 func (stat *generateStats) progressContract(account common.Hash, slot common.Hash, done uint64) { 159 stat.lock.Lock() 160 defer stat.lock.Unlock() 161 162 stat.slots += done 163 stat.slotsHead[account] = slot 164 if _, ok := stat.slotsStart[account]; !ok { 165 stat.slotsStart[account] = time.Now() 166 } 167 } 168 169 // finishContract updates the generator stats for a specific just-finished contract. 170 func (stat *generateStats) finishContract(account common.Hash, done uint64) { 171 stat.lock.Lock() 172 defer stat.lock.Unlock() 173 174 stat.slots += done 175 delete(stat.slotsHead, account) 176 delete(stat.slotsStart, account) 177 } 178 179 // report prints the cumulative progress statistic smartly. 180 func (stat *generateStats) report() { 181 stat.lock.RLock() 182 defer stat.lock.RUnlock() 183 184 ctx := []interface{}{ 185 "accounts", stat.accounts, 186 "slots", stat.slots, 187 "elapsed", common.PrettyDuration(time.Since(stat.start)), 188 } 189 if stat.accounts > 0 { 190 // If there's progress on the account trie, estimate the time to finish crawling it 191 if done := binary.BigEndian.Uint64(stat.head[:8]) / stat.accounts; done > 0 { 192 var ( 193 left = (math.MaxUint64 - binary.BigEndian.Uint64(stat.head[:8])) / stat.accounts 194 speed = done/uint64(time.Since(stat.start)/time.Millisecond+1) + 1 // +1s to avoid division by zero 195 eta = time.Duration(left/speed) * time.Millisecond 196 ) 197 // If there are large contract crawls in progress, estimate their finish time 198 for acc, head := range stat.slotsHead { 199 start := stat.slotsStart[acc] 200 if done := binary.BigEndian.Uint64(head[:8]); done > 0 { 201 var ( 202 left = math.MaxUint64 - binary.BigEndian.Uint64(head[:8]) 203 speed = done/uint64(time.Since(start)/time.Millisecond+1) + 1 // +1s to avoid division by zero 204 ) 205 // Override the ETA if larger than the largest until now 206 if slotETA := time.Duration(left/speed) * time.Millisecond; eta < slotETA { 207 eta = slotETA 208 } 209 } 210 } 211 ctx = append(ctx, []interface{}{ 212 "eta", common.PrettyDuration(eta), 213 }...) 214 } 215 } 216 log.Info("Iterating state snapshot", ctx...) 217 } 218 219 // reportDone prints the last log when the whole generation is finished. 220 func (stat *generateStats) reportDone() { 221 stat.lock.RLock() 222 defer stat.lock.RUnlock() 223 224 var ctx []interface{} 225 ctx = append(ctx, []interface{}{"accounts", stat.accounts}...) 226 if stat.slots != 0 { 227 ctx = append(ctx, []interface{}{"slots", stat.slots}...) 228 } 229 ctx = append(ctx, []interface{}{"elapsed", common.PrettyDuration(time.Since(stat.start))}...) 230 log.Info("Iterated snapshot", ctx...) 231 } 232 233 // runReport periodically prints the progress information. 234 func runReport(stats *generateStats, stop chan bool) { 235 timer := time.NewTimer(0) 236 defer timer.Stop() 237 238 for { 239 select { 240 case <-timer.C: 241 stats.report() 242 timer.Reset(time.Second * 8) 243 case success := <-stop: 244 if success { 245 stats.reportDone() 246 } 247 return 248 } 249 } 250 } 251 252 // generateTrieRoot generates the trie hash based on the snapshot iterator. 253 // It can be used for generating account trie, storage trie or even the 254 // whole state which connects the accounts and the corresponding storages. 255 func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, generatorFn trieGeneratorFn, leafCallback leafCallbackFn, stats *generateStats, report bool) (common.Hash, error) { 256 var ( 257 in = make(chan trieKV) // chan to pass leaves 258 out = make(chan common.Hash, 1) // chan to collect result 259 stoplog = make(chan bool, 1) // 1-size buffer, works when logging is not enabled 260 wg sync.WaitGroup 261 ) 262 // Spin up a go-routine for trie hash re-generation 263 wg.Add(1) 264 go func() { 265 defer wg.Done() 266 generatorFn(db, account, in, out) 267 }() 268 // Spin up a go-routine for progress logging 269 if report && stats != nil { 270 wg.Add(1) 271 go func() { 272 defer wg.Done() 273 runReport(stats, stoplog) 274 }() 275 } 276 // Create a semaphore to assign tasks and collect results through. We'll pre- 277 // fill it with nils, thus using the same channel for both limiting concurrent 278 // processing and gathering results. 279 threads := runtime.NumCPU() 280 results := make(chan error, threads) 281 for i := 0; i < threads; i++ { 282 results <- nil // fill the semaphore 283 } 284 // stop is a helper function to shutdown the background threads 285 // and return the re-generated trie hash. 286 stop := func(fail error) (common.Hash, error) { 287 close(in) 288 result := <-out 289 for i := 0; i < threads; i++ { 290 if err := <-results; err != nil && fail == nil { 291 fail = err 292 } 293 } 294 stoplog <- fail == nil 295 296 wg.Wait() 297 return result, fail 298 } 299 var ( 300 logged = time.Now() 301 processed = uint64(0) 302 leaf trieKV 303 ) 304 // Start to feed leaves 305 for it.Next() { 306 if account == (common.Hash{}) { 307 var ( 308 err error 309 fullData []byte 310 ) 311 if leafCallback == nil { 312 fullData, err = FullAccountRLP(it.(AccountIterator).Account()) 313 if err != nil { 314 return stop(err) 315 } 316 } else { 317 // Wait until the semaphore allows us to continue, aborting if 318 // a sub-task failed 319 if err := <-results; err != nil { 320 results <- nil // stop will drain the results, add a noop back for this error we just consumed 321 return stop(err) 322 } 323 // Fetch the next account and process it concurrently 324 account, err := FullAccount(it.(AccountIterator).Account()) 325 if err != nil { 326 return stop(err) 327 } 328 go func(hash common.Hash) { 329 subroot, err := leafCallback(db, hash, common.BytesToHash(account.CodeHash), stats) 330 if err != nil { 331 results <- err 332 return 333 } 334 if !bytes.Equal(account.Root, subroot.Bytes()) { 335 results <- fmt.Errorf("invalid subroot(path %x), want %x, have %x", hash, account.Root, subroot) 336 return 337 } 338 results <- nil 339 }(it.Hash()) 340 fullData, err = rlp.EncodeToBytes(account) 341 if err != nil { 342 return stop(err) 343 } 344 } 345 leaf = trieKV{it.Hash(), fullData} 346 } else { 347 leaf = trieKV{it.Hash(), common.CopyBytes(it.(StorageIterator).Slot())} 348 } 349 in <- leaf 350 351 // Accumulate the generation statistic if it's required. 352 processed++ 353 if time.Since(logged) > 3*time.Second && stats != nil { 354 if account == (common.Hash{}) { 355 stats.progressAccounts(it.Hash(), processed) 356 } else { 357 stats.progressContract(account, it.Hash(), processed) 358 } 359 logged, processed = time.Now(), 0 360 } 361 } 362 // Commit the last part statistic. 363 if processed > 0 && stats != nil { 364 if account == (common.Hash{}) { 365 stats.finishAccounts(processed) 366 } else { 367 stats.finishContract(account, processed) 368 } 369 } 370 return stop(nil) 371 } 372 373 func stackTrieGenerate(db ethdb.KeyValueWriter, owner common.Hash, in chan trieKV, out chan common.Hash) { 374 t := trie.NewStackTrieWithOwner(db, owner) 375 for leaf := range in { 376 t.TryUpdate(leaf.key[:], leaf.value) 377 } 378 var root common.Hash 379 if db == nil { 380 root = t.Hash() 381 } else { 382 root, _ = t.Commit() 383 } 384 out <- root 385 }