github.com/shrimpyuk/bor@v0.2.15-0.20220224151350-fb4ec6020bae/cmd/geth/snapshot.go (about) 1 // Copyright 2020 The go-ethereum Authors 2 // This file is part of go-ethereum. 3 // 4 // go-ethereum is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // go-ethereum is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU General Public License for more details. 13 // 14 // You should have received a copy of the GNU General Public License 15 // along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. 16 17 package main 18 19 import ( 20 "bytes" 21 "encoding/json" 22 "errors" 23 "os" 24 "time" 25 26 "github.com/ethereum/go-ethereum/cmd/utils" 27 "github.com/ethereum/go-ethereum/common" 28 "github.com/ethereum/go-ethereum/core/rawdb" 29 "github.com/ethereum/go-ethereum/core/state" 30 "github.com/ethereum/go-ethereum/core/state/pruner" 31 "github.com/ethereum/go-ethereum/core/state/snapshot" 32 "github.com/ethereum/go-ethereum/core/types" 33 "github.com/ethereum/go-ethereum/crypto" 34 "github.com/ethereum/go-ethereum/log" 35 "github.com/ethereum/go-ethereum/rlp" 36 "github.com/ethereum/go-ethereum/trie" 37 cli "gopkg.in/urfave/cli.v1" 38 ) 39 40 var ( 41 // emptyRoot is the known root hash of an empty trie. 42 emptyRoot = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") 43 44 // emptyCode is the known hash of the empty EVM bytecode. 45 emptyCode = crypto.Keccak256(nil) 46 ) 47 48 var ( 49 snapshotCommand = cli.Command{ 50 Name: "snapshot", 51 Usage: "A set of commands based on the snapshot", 52 Category: "MISCELLANEOUS COMMANDS", 53 Description: "", 54 Subcommands: []cli.Command{ 55 { 56 Name: "prune-state", 57 Usage: "Prune stale ethereum state data based on the snapshot", 58 ArgsUsage: "<root>", 59 Action: utils.MigrateFlags(pruneState), 60 Category: "MISCELLANEOUS COMMANDS", 61 Flags: []cli.Flag{ 62 utils.DataDirFlag, 63 utils.AncientFlag, 64 utils.RopstenFlag, 65 utils.RinkebyFlag, 66 utils.GoerliFlag, 67 utils.MumbaiFlag, 68 utils.BorMainnetFlag, 69 utils.CacheTrieJournalFlag, 70 utils.BloomFilterSizeFlag, 71 }, 72 Description: ` 73 geth snapshot prune-state <state-root> 74 will prune historical state data with the help of the state snapshot. 75 All trie nodes and contract codes that do not belong to the specified 76 version state will be deleted from the database. After pruning, only 77 two version states are available: genesis and the specific one. 78 79 The default pruning target is the HEAD-127 state. 80 81 WARNING: It's necessary to delete the trie clean cache after the pruning. 82 If you specify another directory for the trie clean cache via "--cache.trie.journal" 83 during the use of Geth, please also specify it here for correct deletion. Otherwise 84 the trie clean cache with default directory will be deleted. 85 `, 86 }, 87 { 88 Name: "verify-state", 89 Usage: "Recalculate state hash based on the snapshot for verification", 90 ArgsUsage: "<root>", 91 Action: utils.MigrateFlags(verifyState), 92 Category: "MISCELLANEOUS COMMANDS", 93 Flags: []cli.Flag{ 94 utils.DataDirFlag, 95 utils.AncientFlag, 96 utils.RopstenFlag, 97 utils.RinkebyFlag, 98 utils.GoerliFlag, 99 utils.MumbaiFlag, 100 utils.BorMainnetFlag, 101 }, 102 Description: ` 103 geth snapshot verify-state <state-root> 104 will traverse the whole accounts and storages set based on the specified 105 snapshot and recalculate the root hash of state for verification. 106 In other words, this command does the snapshot to trie conversion. 107 `, 108 }, 109 { 110 Name: "traverse-state", 111 Usage: "Traverse the state with given root hash for verification", 112 ArgsUsage: "<root>", 113 Action: utils.MigrateFlags(traverseState), 114 Category: "MISCELLANEOUS COMMANDS", 115 Flags: []cli.Flag{ 116 utils.DataDirFlag, 117 utils.AncientFlag, 118 utils.RopstenFlag, 119 utils.RinkebyFlag, 120 utils.GoerliFlag, 121 utils.MumbaiFlag, 122 utils.BorMainnetFlag, 123 }, 124 Description: ` 125 geth snapshot traverse-state <state-root> 126 will traverse the whole state from the given state root and will abort if any 127 referenced trie node or contract code is missing. This command can be used for 128 state integrity verification. The default checking target is the HEAD state. 129 130 It's also usable without snapshot enabled. 131 `, 132 }, 133 { 134 Name: "traverse-rawstate", 135 Usage: "Traverse the state with given root hash for verification", 136 ArgsUsage: "<root>", 137 Action: utils.MigrateFlags(traverseRawState), 138 Category: "MISCELLANEOUS COMMANDS", 139 Flags: []cli.Flag{ 140 utils.DataDirFlag, 141 utils.AncientFlag, 142 utils.RopstenFlag, 143 utils.RinkebyFlag, 144 utils.GoerliFlag, 145 utils.MumbaiFlag, 146 utils.BorMainnetFlag, 147 }, 148 Description: ` 149 geth snapshot traverse-rawstate <state-root> 150 will traverse the whole state from the given root and will abort if any referenced 151 trie node or contract code is missing. This command can be used for state integrity 152 verification. The default checking target is the HEAD state. It's basically identical 153 to traverse-state, but the check granularity is smaller. 154 155 It's also usable without snapshot enabled. 156 `, 157 }, 158 { 159 Name: "dump", 160 Usage: "Dump a specific block from storage (same as 'geth dump' but using snapshots)", 161 ArgsUsage: "[? <blockHash> | <blockNum>]", 162 Action: utils.MigrateFlags(dumpState), 163 Category: "MISCELLANEOUS COMMANDS", 164 Flags: []cli.Flag{ 165 utils.DataDirFlag, 166 utils.AncientFlag, 167 utils.RopstenFlag, 168 utils.RinkebyFlag, 169 utils.GoerliFlag, 170 utils.MumbaiFlag, 171 utils.BorMainnetFlag, 172 utils.ExcludeCodeFlag, 173 utils.ExcludeStorageFlag, 174 utils.StartKeyFlag, 175 utils.DumpLimitFlag, 176 }, 177 Description: ` 178 This command is semantically equivalent to 'geth dump', but uses the snapshots 179 as the backend data source, making this command a lot faster. 180 181 The argument is interpreted as block number or hash. If none is provided, the latest 182 block is used. 183 `, 184 }, 185 }, 186 } 187 ) 188 189 func pruneState(ctx *cli.Context) error { 190 stack, config := makeConfigNode(ctx) 191 defer stack.Close() 192 193 chaindb := utils.MakeChainDatabase(ctx, stack, false) 194 pruner, err := pruner.NewPruner(chaindb, stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal), ctx.GlobalUint64(utils.BloomFilterSizeFlag.Name)) 195 if err != nil { 196 log.Error("Failed to open snapshot tree", "err", err) 197 return err 198 } 199 if ctx.NArg() > 1 { 200 log.Error("Too many arguments given") 201 return errors.New("too many arguments") 202 } 203 var targetRoot common.Hash 204 if ctx.NArg() == 1 { 205 targetRoot, err = parseRoot(ctx.Args()[0]) 206 if err != nil { 207 log.Error("Failed to resolve state root", "err", err) 208 return err 209 } 210 } 211 if err = pruner.Prune(targetRoot); err != nil { 212 log.Error("Failed to prune state", "err", err) 213 return err 214 } 215 return nil 216 } 217 218 func verifyState(ctx *cli.Context) error { 219 stack, _ := makeConfigNode(ctx) 220 defer stack.Close() 221 222 chaindb := utils.MakeChainDatabase(ctx, stack, true) 223 headBlock := rawdb.ReadHeadBlock(chaindb) 224 if headBlock == nil { 225 log.Error("Failed to load head block") 226 return errors.New("no head block") 227 } 228 snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, headBlock.Root(), false, false, false) 229 if err != nil { 230 log.Error("Failed to open snapshot tree", "err", err) 231 return err 232 } 233 if ctx.NArg() > 1 { 234 log.Error("Too many arguments given") 235 return errors.New("too many arguments") 236 } 237 var root = headBlock.Root() 238 if ctx.NArg() == 1 { 239 root, err = parseRoot(ctx.Args()[0]) 240 if err != nil { 241 log.Error("Failed to resolve state root", "err", err) 242 return err 243 } 244 } 245 if err := snaptree.Verify(root); err != nil { 246 log.Error("Failed to verify state", "root", root, "err", err) 247 return err 248 } 249 log.Info("Verified the state", "root", root) 250 return nil 251 } 252 253 // traverseState is a helper function used for pruning verification. 254 // Basically it just iterates the trie, ensure all nodes and associated 255 // contract codes are present. 256 func traverseState(ctx *cli.Context) error { 257 stack, _ := makeConfigNode(ctx) 258 defer stack.Close() 259 260 chaindb := utils.MakeChainDatabase(ctx, stack, true) 261 headBlock := rawdb.ReadHeadBlock(chaindb) 262 if headBlock == nil { 263 log.Error("Failed to load head block") 264 return errors.New("no head block") 265 } 266 if ctx.NArg() > 1 { 267 log.Error("Too many arguments given") 268 return errors.New("too many arguments") 269 } 270 var ( 271 root common.Hash 272 err error 273 ) 274 if ctx.NArg() == 1 { 275 root, err = parseRoot(ctx.Args()[0]) 276 if err != nil { 277 log.Error("Failed to resolve state root", "err", err) 278 return err 279 } 280 log.Info("Start traversing the state", "root", root) 281 } else { 282 root = headBlock.Root() 283 log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) 284 } 285 triedb := trie.NewDatabase(chaindb) 286 t, err := trie.NewSecure(root, triedb) 287 if err != nil { 288 log.Error("Failed to open trie", "root", root, "err", err) 289 return err 290 } 291 var ( 292 accounts int 293 slots int 294 codes int 295 lastReport time.Time 296 start = time.Now() 297 ) 298 accIter := trie.NewIterator(t.NodeIterator(nil)) 299 for accIter.Next() { 300 accounts += 1 301 var acc types.StateAccount 302 if err := rlp.DecodeBytes(accIter.Value, &acc); err != nil { 303 log.Error("Invalid account encountered during traversal", "err", err) 304 return err 305 } 306 if acc.Root != emptyRoot { 307 storageTrie, err := trie.NewSecure(acc.Root, triedb) 308 if err != nil { 309 log.Error("Failed to open storage trie", "root", acc.Root, "err", err) 310 return err 311 } 312 storageIter := trie.NewIterator(storageTrie.NodeIterator(nil)) 313 for storageIter.Next() { 314 slots += 1 315 } 316 if storageIter.Err != nil { 317 log.Error("Failed to traverse storage trie", "root", acc.Root, "err", storageIter.Err) 318 return storageIter.Err 319 } 320 } 321 if !bytes.Equal(acc.CodeHash, emptyCode) { 322 code := rawdb.ReadCode(chaindb, common.BytesToHash(acc.CodeHash)) 323 if len(code) == 0 { 324 log.Error("Code is missing", "hash", common.BytesToHash(acc.CodeHash)) 325 return errors.New("missing code") 326 } 327 codes += 1 328 } 329 if time.Since(lastReport) > time.Second*8 { 330 log.Info("Traversing state", "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start))) 331 lastReport = time.Now() 332 } 333 } 334 if accIter.Err != nil { 335 log.Error("Failed to traverse state trie", "root", root, "err", accIter.Err) 336 return accIter.Err 337 } 338 log.Info("State is complete", "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start))) 339 return nil 340 } 341 342 // traverseRawState is a helper function used for pruning verification. 343 // Basically it just iterates the trie, ensure all nodes and associated 344 // contract codes are present. It's basically identical to traverseState 345 // but it will check each trie node. 346 func traverseRawState(ctx *cli.Context) error { 347 stack, _ := makeConfigNode(ctx) 348 defer stack.Close() 349 350 chaindb := utils.MakeChainDatabase(ctx, stack, true) 351 headBlock := rawdb.ReadHeadBlock(chaindb) 352 if headBlock == nil { 353 log.Error("Failed to load head block") 354 return errors.New("no head block") 355 } 356 if ctx.NArg() > 1 { 357 log.Error("Too many arguments given") 358 return errors.New("too many arguments") 359 } 360 var ( 361 root common.Hash 362 err error 363 ) 364 if ctx.NArg() == 1 { 365 root, err = parseRoot(ctx.Args()[0]) 366 if err != nil { 367 log.Error("Failed to resolve state root", "err", err) 368 return err 369 } 370 log.Info("Start traversing the state", "root", root) 371 } else { 372 root = headBlock.Root() 373 log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) 374 } 375 triedb := trie.NewDatabase(chaindb) 376 t, err := trie.NewSecure(root, triedb) 377 if err != nil { 378 log.Error("Failed to open trie", "root", root, "err", err) 379 return err 380 } 381 var ( 382 nodes int 383 accounts int 384 slots int 385 codes int 386 lastReport time.Time 387 start = time.Now() 388 ) 389 accIter := t.NodeIterator(nil) 390 for accIter.Next(true) { 391 nodes += 1 392 node := accIter.Hash() 393 394 if node != (common.Hash{}) { 395 // Check the present for non-empty hash node(embedded node doesn't 396 // have their own hash). 397 blob := rawdb.ReadTrieNode(chaindb, node) 398 if len(blob) == 0 { 399 log.Error("Missing trie node(account)", "hash", node) 400 return errors.New("missing account") 401 } 402 } 403 // If it's a leaf node, yes we are touching an account, 404 // dig into the storage trie further. 405 if accIter.Leaf() { 406 accounts += 1 407 var acc types.StateAccount 408 if err := rlp.DecodeBytes(accIter.LeafBlob(), &acc); err != nil { 409 log.Error("Invalid account encountered during traversal", "err", err) 410 return errors.New("invalid account") 411 } 412 if acc.Root != emptyRoot { 413 storageTrie, err := trie.NewSecure(acc.Root, triedb) 414 if err != nil { 415 log.Error("Failed to open storage trie", "root", acc.Root, "err", err) 416 return errors.New("missing storage trie") 417 } 418 storageIter := storageTrie.NodeIterator(nil) 419 for storageIter.Next(true) { 420 nodes += 1 421 node := storageIter.Hash() 422 423 // Check the present for non-empty hash node(embedded node doesn't 424 // have their own hash). 425 if node != (common.Hash{}) { 426 blob := rawdb.ReadTrieNode(chaindb, node) 427 if len(blob) == 0 { 428 log.Error("Missing trie node(storage)", "hash", node) 429 return errors.New("missing storage") 430 } 431 } 432 // Bump the counter if it's leaf node. 433 if storageIter.Leaf() { 434 slots += 1 435 } 436 } 437 if storageIter.Error() != nil { 438 log.Error("Failed to traverse storage trie", "root", acc.Root, "err", storageIter.Error()) 439 return storageIter.Error() 440 } 441 } 442 if !bytes.Equal(acc.CodeHash, emptyCode) { 443 code := rawdb.ReadCode(chaindb, common.BytesToHash(acc.CodeHash)) 444 if len(code) == 0 { 445 log.Error("Code is missing", "account", common.BytesToHash(accIter.LeafKey())) 446 return errors.New("missing code") 447 } 448 codes += 1 449 } 450 if time.Since(lastReport) > time.Second*8 { 451 log.Info("Traversing state", "nodes", nodes, "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start))) 452 lastReport = time.Now() 453 } 454 } 455 } 456 if accIter.Error() != nil { 457 log.Error("Failed to traverse state trie", "root", root, "err", accIter.Error()) 458 return accIter.Error() 459 } 460 log.Info("State is complete", "nodes", nodes, "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start))) 461 return nil 462 } 463 464 func parseRoot(input string) (common.Hash, error) { 465 var h common.Hash 466 if err := h.UnmarshalText([]byte(input)); err != nil { 467 return h, err 468 } 469 return h, nil 470 } 471 472 func dumpState(ctx *cli.Context) error { 473 stack, _ := makeConfigNode(ctx) 474 defer stack.Close() 475 476 conf, db, root, err := parseDumpConfig(ctx, stack) 477 if err != nil { 478 return err 479 } 480 snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, root, false, false, false) 481 if err != nil { 482 return err 483 } 484 accIt, err := snaptree.AccountIterator(root, common.BytesToHash(conf.Start)) 485 if err != nil { 486 return err 487 } 488 defer accIt.Release() 489 490 log.Info("Snapshot dumping started", "root", root) 491 var ( 492 start = time.Now() 493 logged = time.Now() 494 accounts uint64 495 ) 496 enc := json.NewEncoder(os.Stdout) 497 enc.Encode(struct { 498 Root common.Hash `json:"root"` 499 }{root}) 500 for accIt.Next() { 501 account, err := snapshot.FullAccount(accIt.Account()) 502 if err != nil { 503 return err 504 } 505 da := &state.DumpAccount{ 506 Balance: account.Balance.String(), 507 Nonce: account.Nonce, 508 Root: account.Root, 509 CodeHash: account.CodeHash, 510 SecureKey: accIt.Hash().Bytes(), 511 } 512 if !conf.SkipCode && !bytes.Equal(account.CodeHash, emptyCode) { 513 da.Code = rawdb.ReadCode(db, common.BytesToHash(account.CodeHash)) 514 } 515 if !conf.SkipStorage { 516 da.Storage = make(map[common.Hash]string) 517 518 stIt, err := snaptree.StorageIterator(root, accIt.Hash(), common.Hash{}) 519 if err != nil { 520 return err 521 } 522 for stIt.Next() { 523 da.Storage[stIt.Hash()] = common.Bytes2Hex(stIt.Slot()) 524 } 525 } 526 enc.Encode(da) 527 accounts++ 528 if time.Since(logged) > 8*time.Second { 529 log.Info("Snapshot dumping in progress", "at", accIt.Hash(), "accounts", accounts, 530 "elapsed", common.PrettyDuration(time.Since(start))) 531 logged = time.Now() 532 } 533 if conf.Max > 0 && accounts >= conf.Max { 534 break 535 } 536 } 537 log.Info("Snapshot dumping complete", "accounts", accounts, 538 "elapsed", common.PrettyDuration(time.Since(start))) 539 return nil 540 }