github.com/klaytn/klaytn@v1.10.2/snapshot/conversion.go (about) 1 // Modifications Copyright 2021 The klaytn Authors 2 // Copyright 2020 The go-ethereum Authors 3 // This file is part of the go-ethereum library. 4 // 5 // The go-ethereum library is free software: you can redistribute it and/or modify 6 // it under the terms of the GNU Lesser General Public License as published by 7 // the Free Software Foundation, either version 3 of the License, or 8 // (at your option) any later version. 9 // 10 // The go-ethereum library is distributed in the hope that it will be useful, 11 // but WITHOUT ANY WARRANTY; without even the implied warranty of 12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 // GNU Lesser General Public License for more details. 14 // 15 // You should have received a copy of the GNU Lesser General Public License 16 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 17 // 18 // This file is derived from core/state/snapshot/conversion.go (2021/10/21). 19 // Modified and improved for the klaytn development. 20 21 package snapshot 22 23 import ( 24 "encoding/binary" 25 "fmt" 26 "math" 27 "runtime" 28 "sync" 29 "time" 30 31 "github.com/klaytn/klaytn/blockchain/types/account" 32 "github.com/klaytn/klaytn/common" 33 "github.com/klaytn/klaytn/rlp" 34 "github.com/klaytn/klaytn/storage/database" 35 "github.com/klaytn/klaytn/storage/statedb" 36 ) 37 38 // trieKV represents a trie key-value pair 39 type trieKV struct { 40 key common.Hash 41 value []byte 42 } 43 44 type ( 45 // trieGeneratorFn is the interface of trie generation which can 46 // be implemented by different trie algorithm. 47 trieGeneratorFn func(in chan (trieKV), out chan (common.Hash)) 48 49 // leafCallbackFn is the callback invoked at the leaves of the trie, 50 // returns the subtrie root with the specified subtrie identifier. 51 leafCallbackFn func(accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) 52 ) 53 54 // TODO-Klaytn-Snapshot port GenerateAccountTrieRoot/GenerateStorageTrieRoot/GenerateTrie 55 56 // generateStats is a collection of statistics gathered by the trie generator 57 // for logging purposes. 58 type generateStats struct { 59 head common.Hash 60 start time.Time 61 62 accounts uint64 // Number of accounts done (including those being crawled) 63 slots uint64 // Number of storage slots done (including those being crawled) 64 65 slotsStart map[common.Hash]time.Time // Start time for account slot crawling 66 slotsHead map[common.Hash]common.Hash // Slot head for accounts being crawled 67 68 lock sync.RWMutex 69 } 70 71 // newGenerateStats creates a new generator stats. 72 func newGenerateStats() *generateStats { 73 return &generateStats{ 74 slotsStart: make(map[common.Hash]time.Time), 75 slotsHead: make(map[common.Hash]common.Hash), 76 start: time.Now(), 77 } 78 } 79 80 // progressAccounts updates the generator stats for the account range. 81 func (stat *generateStats) progressAccounts(account common.Hash, done uint64) { 82 stat.lock.Lock() 83 defer stat.lock.Unlock() 84 85 stat.accounts += done 86 stat.head = account 87 } 88 89 // finishAccounts updates the gemerator stats for the finished account range. 90 func (stat *generateStats) finishAccounts(done uint64) { 91 stat.lock.Lock() 92 defer stat.lock.Unlock() 93 94 stat.accounts += done 95 } 96 97 // progressContract updates the generator stats for a specific in-progress contract. 98 func (stat *generateStats) progressContract(account common.Hash, slot common.Hash, done uint64) { 99 stat.lock.Lock() 100 defer stat.lock.Unlock() 101 102 stat.slots += done 103 stat.slotsHead[account] = slot 104 if _, ok := stat.slotsStart[account]; !ok { 105 stat.slotsStart[account] = time.Now() 106 } 107 } 108 109 // finishContract updates the generator stats for a specific just-finished contract. 110 func (stat *generateStats) finishContract(account common.Hash, done uint64) { 111 stat.lock.Lock() 112 defer stat.lock.Unlock() 113 114 stat.slots += done 115 delete(stat.slotsHead, account) 116 delete(stat.slotsStart, account) 117 } 118 119 // report prints the cumulative progress statistic smartly. 120 func (stat *generateStats) report() { 121 stat.lock.RLock() 122 defer stat.lock.RUnlock() 123 124 ctx := []interface{}{ 125 "accounts", stat.accounts, 126 "slots", stat.slots, 127 "elapsed", common.PrettyDuration(time.Since(stat.start)), 128 } 129 if stat.accounts > 0 { 130 // If there's progress on the account trie, estimate the time to finish crawling it 131 if done := binary.BigEndian.Uint64(stat.head[:8]) / stat.accounts; done > 0 { 132 var ( 133 left = (math.MaxUint64 - binary.BigEndian.Uint64(stat.head[:8])) / stat.accounts 134 speed = done/uint64(time.Since(stat.start)/time.Millisecond+1) + 1 // +1s to avoid division by zero 135 eta = time.Duration(left/speed) * time.Millisecond 136 ) 137 // If there are large contract crawls in progress, estimate their finish time 138 for acc, head := range stat.slotsHead { 139 start := stat.slotsStart[acc] 140 if done := binary.BigEndian.Uint64(head[:8]); done > 0 { 141 var ( 142 left = math.MaxUint64 - binary.BigEndian.Uint64(head[:8]) 143 speed = done/uint64(time.Since(start)/time.Millisecond+1) + 1 // +1s to avoid division by zero 144 ) 145 // Override the ETA if larger than the largest until now 146 if slotETA := time.Duration(left/speed) * time.Millisecond; eta < slotETA { 147 eta = slotETA 148 } 149 } 150 } 151 ctx = append(ctx, []interface{}{ 152 "eta", common.PrettyDuration(eta), 153 }...) 154 } 155 } 156 logger.Info("Iterating state snapshot", ctx...) 157 } 158 159 // reportDone prints the last log when the whole generation is finished. 160 func (stat *generateStats) reportDone() { 161 stat.lock.RLock() 162 defer stat.lock.RUnlock() 163 164 var ctx []interface{} 165 ctx = append(ctx, []interface{}{"accounts", stat.accounts}...) 166 if stat.slots != 0 { 167 ctx = append(ctx, []interface{}{"slots", stat.slots}...) 168 } 169 ctx = append(ctx, []interface{}{"elapsed", common.PrettyDuration(time.Since(stat.start))}...) 170 logger.Info("Iterated snapshot", ctx...) 171 } 172 173 // runReport periodically prints the progress information. 174 func runReport(stats *generateStats, stop chan bool) { 175 timer := time.NewTimer(0) 176 defer timer.Stop() 177 178 for { 179 select { 180 case <-timer.C: 181 stats.report() 182 timer.Reset(time.Second * 8) 183 case success := <-stop: 184 if success { 185 stats.reportDone() 186 } 187 return 188 } 189 } 190 } 191 192 // generateTrieRoot generates the trie hash based on the snapshot iterator. 193 // It can be used for generating account trie, storage trie or even the 194 // whole state which connects the accounts and the corresponding storages. 195 func generateTrieRoot(it Iterator, accountHash common.Hash, generatorFn trieGeneratorFn, leafCallback leafCallbackFn, stats *generateStats, report bool) (common.Hash, error) { 196 var ( 197 in = make(chan trieKV) // chan to pass leaves 198 out = make(chan common.Hash, 1) // chan to collect result 199 stoplog = make(chan bool, 1) // 1-size buffer, works when logging is not enabled 200 wg sync.WaitGroup 201 ) 202 // Spin up a go-routine for trie hash re-generation 203 wg.Add(1) 204 go func() { 205 defer wg.Done() 206 generatorFn(in, out) 207 }() 208 // Spin up a go-routine for progress logging 209 if report && stats != nil { 210 wg.Add(1) 211 go func() { 212 defer wg.Done() 213 runReport(stats, stoplog) 214 }() 215 } 216 // Create a semaphore to assign tasks and collect results through. We'll pre- 217 // fill it with nils, thus using the same channel for both limiting concurrent 218 // processing and gathering results. 219 threads := runtime.NumCPU() 220 results := make(chan error, threads) 221 for i := 0; i < threads; i++ { 222 results <- nil // fill the semaphore 223 } 224 // stop is a helper function to shutdown the background threads 225 // and return the re-generated trie hash. 226 stop := func(fail error) (common.Hash, error) { 227 close(in) 228 result := <-out 229 for i := 0; i < threads; i++ { 230 if err := <-results; err != nil && fail == nil { 231 fail = err 232 } 233 } 234 stoplog <- fail == nil 235 236 wg.Wait() 237 return result, fail 238 } 239 var ( 240 logged = time.Now() 241 processed = uint64(0) 242 leaf trieKV 243 ) 244 // Start to feed leaves 245 for it.Next() { 246 if accountHash == (common.Hash{}) { 247 var ( 248 err error 249 fullData []byte 250 ) 251 if leafCallback == nil { 252 fullData = it.(AccountIterator).Account() 253 } else { 254 // Wait until the semaphore allows us to continue, aborting if 255 // a sub-task failed 256 if err := <-results; err != nil { 257 results <- nil // stop will drain the results, add a noop back for this error we just consumed 258 return stop(err) 259 } 260 // Fetch the next account and process it concurrently 261 serializer := account.NewAccountSerializer() 262 if err := rlp.DecodeBytes(it.(AccountIterator).Account(), serializer); err != nil { 263 logger.Error("Failed to decode an account from iterator", "err", err) 264 return stop(err) 265 } 266 acc := serializer.GetAccount() 267 go func(hash common.Hash) { 268 contract, ok := acc.(*account.SmartContractAccount) 269 if !ok { 270 results <- nil 271 return 272 } 273 subroot, err := leafCallback(hash, common.BytesToHash(contract.GetCodeHash()), stats) 274 if err != nil { 275 results <- err 276 return 277 } 278 rootHash := contract.GetStorageRoot() 279 if rootHash != subroot { 280 results <- fmt.Errorf("invalid subroot(path %x), want %x, have %x", hash, rootHash, subroot) 281 return 282 } 283 results <- nil 284 }(it.Hash()) 285 fullData, err = rlp.EncodeToBytes(serializer) 286 if err != nil { 287 return stop(err) 288 } 289 } 290 leaf = trieKV{it.Hash(), fullData} 291 } else { 292 leaf = trieKV{it.Hash(), common.CopyBytes(it.(StorageIterator).Slot())} 293 } 294 in <- leaf 295 296 // Accumulate the generation statistic if it's required. 297 processed++ 298 if time.Since(logged) > 3*time.Second && stats != nil { 299 if accountHash == (common.Hash{}) { 300 stats.progressAccounts(it.Hash(), processed) 301 } else { 302 stats.progressContract(accountHash, it.Hash(), processed) 303 } 304 logged, processed = time.Now(), 0 305 } 306 } 307 // Commit the last part statistic. 308 if processed > 0 && stats != nil { 309 if accountHash == (common.Hash{}) { 310 stats.finishAccounts(processed) 311 } else { 312 stats.finishContract(accountHash, processed) 313 } 314 } 315 return stop(nil) 316 } 317 318 func trieGenerate(in chan trieKV, out chan common.Hash) { 319 db := statedb.NewDatabase(database.NewMemoryDBManager()) 320 t, _ := statedb.NewTrie(common.Hash{}, db) 321 for leaf := range in { 322 t.TryUpdate(leaf.key[:], leaf.value) 323 } 324 var root common.Hash 325 if db == nil { 326 root = t.Hash() 327 } else { 328 root, _ = t.Commit(nil) 329 } 330 out <- root 331 }