github.com/carter-ya/go-ethereum@v0.0.0-20230628080049-d2309be3983b/cmd/geth/snapshot.go (about) 1 // Copyright 2021 The go-ethereum Authors 2 // This file is part of go-ethereum. 3 // 4 // go-ethereum is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // go-ethereum is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU General Public License for more details. 13 // 14 // You should have received a copy of the GNU General Public License 15 // along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. 16 17 package main 18 19 import ( 20 "bytes" 21 "encoding/json" 22 "errors" 23 "os" 24 "time" 25 26 "github.com/ethereum/go-ethereum/cmd/utils" 27 "github.com/ethereum/go-ethereum/common" 28 "github.com/ethereum/go-ethereum/core/rawdb" 29 "github.com/ethereum/go-ethereum/core/state" 30 "github.com/ethereum/go-ethereum/core/state/pruner" 31 "github.com/ethereum/go-ethereum/core/state/snapshot" 32 "github.com/ethereum/go-ethereum/core/types" 33 "github.com/ethereum/go-ethereum/crypto" 34 "github.com/ethereum/go-ethereum/internal/flags" 35 "github.com/ethereum/go-ethereum/log" 36 "github.com/ethereum/go-ethereum/rlp" 37 "github.com/ethereum/go-ethereum/trie" 38 cli "github.com/urfave/cli/v2" 39 ) 40 41 var ( 42 // emptyRoot is the known root hash of an empty trie. 43 emptyRoot = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") 44 45 // emptyCode is the known hash of the empty EVM bytecode. 46 emptyCode = crypto.Keccak256(nil) 47 ) 48 49 var ( 50 snapshotCommand = &cli.Command{ 51 Name: "snapshot", 52 Usage: "A set of commands based on the snapshot", 53 Description: "", 54 Subcommands: []*cli.Command{ 55 { 56 Name: "prune-state", 57 Usage: "Prune stale ethereum state data based on the snapshot", 58 ArgsUsage: "<root>", 59 Action: pruneState, 60 Flags: flags.Merge([]cli.Flag{ 61 utils.CacheTrieJournalFlag, 62 utils.BloomFilterSizeFlag, 63 }, utils.NetworkFlags, utils.DatabasePathFlags), 64 Description: ` 65 geth snapshot prune-state <state-root> 66 will prune historical state data with the help of the state snapshot. 67 All trie nodes and contract codes that do not belong to the specified 68 version state will be deleted from the database. After pruning, only 69 two version states are available: genesis and the specific one. 70 71 The default pruning target is the HEAD-127 state. 72 73 WARNING: It's necessary to delete the trie clean cache after the pruning. 74 If you specify another directory for the trie clean cache via "--cache.trie.journal" 75 during the use of Geth, please also specify it here for correct deletion. Otherwise 76 the trie clean cache with default directory will be deleted. 77 `, 78 }, 79 { 80 Name: "verify-state", 81 Usage: "Recalculate state hash based on the snapshot for verification", 82 ArgsUsage: "<root>", 83 Action: verifyState, 84 Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), 85 Description: ` 86 geth snapshot verify-state <state-root> 87 will traverse the whole accounts and storages set based on the specified 88 snapshot and recalculate the root hash of state for verification. 89 In other words, this command does the snapshot to trie conversion. 90 `, 91 }, 92 { 93 Name: "check-dangling-storage", 94 Usage: "Check that there is no 'dangling' snap storage", 95 ArgsUsage: "<root>", 96 Action: checkDanglingStorage, 97 Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), 98 Description: ` 99 geth snapshot check-dangling-storage <state-root> traverses the snap storage 100 data, and verifies that all snapshot storage data has a corresponding account. 101 `, 102 }, 103 { 104 Name: "inspect-account", 105 Usage: "Check all snapshot layers for the a specific account", 106 ArgsUsage: "<address | hash>", 107 Action: checkAccount, 108 Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), 109 Description: ` 110 geth snapshot inspect-account <address | hash> checks all snapshot layers and prints out 111 information about the specified address. 112 `, 113 }, 114 { 115 Name: "traverse-state", 116 Usage: "Traverse the state with given root hash and perform quick verification", 117 ArgsUsage: "<root>", 118 Action: traverseState, 119 Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), 120 Description: ` 121 geth snapshot traverse-state <state-root> 122 will traverse the whole state from the given state root and will abort if any 123 referenced trie node or contract code is missing. This command can be used for 124 state integrity verification. The default checking target is the HEAD state. 125 126 It's also usable without snapshot enabled. 127 `, 128 }, 129 { 130 Name: "traverse-rawstate", 131 Usage: "Traverse the state with given root hash and perform detailed verification", 132 ArgsUsage: "<root>", 133 Action: traverseRawState, 134 Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), 135 Description: ` 136 geth snapshot traverse-rawstate <state-root> 137 will traverse the whole state from the given root and will abort if any referenced 138 trie node or contract code is missing. This command can be used for state integrity 139 verification. The default checking target is the HEAD state. It's basically identical 140 to traverse-state, but the check granularity is smaller. 141 142 It's also usable without snapshot enabled. 143 `, 144 }, 145 { 146 Name: "dump", 147 Usage: "Dump a specific block from storage (same as 'geth dump' but using snapshots)", 148 ArgsUsage: "[? <blockHash> | <blockNum>]", 149 Action: dumpState, 150 Flags: flags.Merge([]cli.Flag{ 151 utils.ExcludeCodeFlag, 152 utils.ExcludeStorageFlag, 153 utils.StartKeyFlag, 154 utils.DumpLimitFlag, 155 }, utils.NetworkFlags, utils.DatabasePathFlags), 156 Description: ` 157 This command is semantically equivalent to 'geth dump', but uses the snapshots 158 as the backend data source, making this command a lot faster. 159 160 The argument is interpreted as block number or hash. If none is provided, the latest 161 block is used. 162 `, 163 }, 164 }, 165 } 166 ) 167 168 func pruneState(ctx *cli.Context) error { 169 stack, config := makeConfigNode(ctx) 170 defer stack.Close() 171 172 chaindb := utils.MakeChainDatabase(ctx, stack, false) 173 defer chaindb.Close() 174 175 prunerconfig := pruner.Config{ 176 Datadir: stack.ResolvePath(""), 177 Cachedir: stack.ResolvePath(config.Eth.TrieCleanCacheJournal), 178 BloomSize: ctx.Uint64(utils.BloomFilterSizeFlag.Name), 179 } 180 pruner, err := pruner.NewPruner(chaindb, prunerconfig) 181 if err != nil { 182 log.Error("Failed to open snapshot tree", "err", err) 183 return err 184 } 185 if ctx.NArg() > 1 { 186 log.Error("Too many arguments given") 187 return errors.New("too many arguments") 188 } 189 var targetRoot common.Hash 190 if ctx.NArg() == 1 { 191 targetRoot, err = parseRoot(ctx.Args().First()) 192 if err != nil { 193 log.Error("Failed to resolve state root", "err", err) 194 return err 195 } 196 } 197 if err = pruner.Prune(targetRoot); err != nil { 198 log.Error("Failed to prune state", "err", err) 199 return err 200 } 201 return nil 202 } 203 204 func verifyState(ctx *cli.Context) error { 205 stack, _ := makeConfigNode(ctx) 206 defer stack.Close() 207 208 chaindb := utils.MakeChainDatabase(ctx, stack, true) 209 defer chaindb.Close() 210 211 headBlock := rawdb.ReadHeadBlock(chaindb) 212 if headBlock == nil { 213 log.Error("Failed to load head block") 214 return errors.New("no head block") 215 } 216 snapconfig := snapshot.Config{ 217 CacheSize: 256, 218 Recovery: false, 219 NoBuild: true, 220 AsyncBuild: false, 221 } 222 snaptree, err := snapshot.New(snapconfig, chaindb, trie.NewDatabase(chaindb), headBlock.Root()) 223 if err != nil { 224 log.Error("Failed to open snapshot tree", "err", err) 225 return err 226 } 227 if ctx.NArg() > 1 { 228 log.Error("Too many arguments given") 229 return errors.New("too many arguments") 230 } 231 var root = headBlock.Root() 232 if ctx.NArg() == 1 { 233 root, err = parseRoot(ctx.Args().First()) 234 if err != nil { 235 log.Error("Failed to resolve state root", "err", err) 236 return err 237 } 238 } 239 if err := snaptree.Verify(root); err != nil { 240 log.Error("Failed to verify state", "root", root, "err", err) 241 return err 242 } 243 log.Info("Verified the state", "root", root) 244 return snapshot.CheckDanglingStorage(chaindb) 245 } 246 247 // checkDanglingStorage iterates the snap storage data, and verifies that all 248 // storage also has corresponding account data. 249 func checkDanglingStorage(ctx *cli.Context) error { 250 stack, _ := makeConfigNode(ctx) 251 defer stack.Close() 252 253 return snapshot.CheckDanglingStorage(utils.MakeChainDatabase(ctx, stack, true)) 254 } 255 256 // traverseState is a helper function used for pruning verification. 257 // Basically it just iterates the trie, ensure all nodes and associated 258 // contract codes are present. 259 func traverseState(ctx *cli.Context) error { 260 stack, _ := makeConfigNode(ctx) 261 defer stack.Close() 262 263 chaindb := utils.MakeChainDatabase(ctx, stack, true) 264 headBlock := rawdb.ReadHeadBlock(chaindb) 265 if headBlock == nil { 266 log.Error("Failed to load head block") 267 return errors.New("no head block") 268 } 269 if ctx.NArg() > 1 { 270 log.Error("Too many arguments given") 271 return errors.New("too many arguments") 272 } 273 var ( 274 root common.Hash 275 err error 276 ) 277 if ctx.NArg() == 1 { 278 root, err = parseRoot(ctx.Args().First()) 279 if err != nil { 280 log.Error("Failed to resolve state root", "err", err) 281 return err 282 } 283 log.Info("Start traversing the state", "root", root) 284 } else { 285 root = headBlock.Root() 286 log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) 287 } 288 triedb := trie.NewDatabase(chaindb) 289 t, err := trie.NewStateTrie(trie.StateTrieID(root), triedb) 290 if err != nil { 291 log.Error("Failed to open trie", "root", root, "err", err) 292 return err 293 } 294 var ( 295 accounts int 296 slots int 297 codes int 298 lastReport time.Time 299 start = time.Now() 300 ) 301 accIter := trie.NewIterator(t.NodeIterator(nil)) 302 for accIter.Next() { 303 accounts += 1 304 var acc types.StateAccount 305 if err := rlp.DecodeBytes(accIter.Value, &acc); err != nil { 306 log.Error("Invalid account encountered during traversal", "err", err) 307 return err 308 } 309 if acc.Root != emptyRoot { 310 id := trie.StorageTrieID(root, common.BytesToHash(accIter.Key), acc.Root) 311 storageTrie, err := trie.NewStateTrie(id, triedb) 312 if err != nil { 313 log.Error("Failed to open storage trie", "root", acc.Root, "err", err) 314 return err 315 } 316 storageIter := trie.NewIterator(storageTrie.NodeIterator(nil)) 317 for storageIter.Next() { 318 slots += 1 319 } 320 if storageIter.Err != nil { 321 log.Error("Failed to traverse storage trie", "root", acc.Root, "err", storageIter.Err) 322 return storageIter.Err 323 } 324 } 325 if !bytes.Equal(acc.CodeHash, emptyCode) { 326 if !rawdb.HasCode(chaindb, common.BytesToHash(acc.CodeHash)) { 327 log.Error("Code is missing", "hash", common.BytesToHash(acc.CodeHash)) 328 return errors.New("missing code") 329 } 330 codes += 1 331 } 332 if time.Since(lastReport) > time.Second*8 { 333 log.Info("Traversing state", "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start))) 334 lastReport = time.Now() 335 } 336 } 337 if accIter.Err != nil { 338 log.Error("Failed to traverse state trie", "root", root, "err", accIter.Err) 339 return accIter.Err 340 } 341 log.Info("State is complete", "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start))) 342 return nil 343 } 344 345 // traverseRawState is a helper function used for pruning verification. 346 // Basically it just iterates the trie, ensure all nodes and associated 347 // contract codes are present. It's basically identical to traverseState 348 // but it will check each trie node. 349 func traverseRawState(ctx *cli.Context) error { 350 stack, _ := makeConfigNode(ctx) 351 defer stack.Close() 352 353 chaindb := utils.MakeChainDatabase(ctx, stack, true) 354 headBlock := rawdb.ReadHeadBlock(chaindb) 355 if headBlock == nil { 356 log.Error("Failed to load head block") 357 return errors.New("no head block") 358 } 359 if ctx.NArg() > 1 { 360 log.Error("Too many arguments given") 361 return errors.New("too many arguments") 362 } 363 var ( 364 root common.Hash 365 err error 366 ) 367 if ctx.NArg() == 1 { 368 root, err = parseRoot(ctx.Args().First()) 369 if err != nil { 370 log.Error("Failed to resolve state root", "err", err) 371 return err 372 } 373 log.Info("Start traversing the state", "root", root) 374 } else { 375 root = headBlock.Root() 376 log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) 377 } 378 triedb := trie.NewDatabase(chaindb) 379 t, err := trie.NewStateTrie(trie.StateTrieID(root), triedb) 380 if err != nil { 381 log.Error("Failed to open trie", "root", root, "err", err) 382 return err 383 } 384 var ( 385 nodes int 386 accounts int 387 slots int 388 codes int 389 lastReport time.Time 390 start = time.Now() 391 hasher = crypto.NewKeccakState() 392 got = make([]byte, 32) 393 ) 394 accIter := t.NodeIterator(nil) 395 for accIter.Next(true) { 396 nodes += 1 397 node := accIter.Hash() 398 399 // Check the present for non-empty hash node(embedded node doesn't 400 // have their own hash). 401 if node != (common.Hash{}) { 402 blob := rawdb.ReadTrieNode(chaindb, node) 403 if len(blob) == 0 { 404 log.Error("Missing trie node(account)", "hash", node) 405 return errors.New("missing account") 406 } 407 hasher.Reset() 408 hasher.Write(blob) 409 hasher.Read(got) 410 if !bytes.Equal(got, node.Bytes()) { 411 log.Error("Invalid trie node(account)", "hash", node.Hex(), "value", blob) 412 return errors.New("invalid account node") 413 } 414 } 415 // If it's a leaf node, yes we are touching an account, 416 // dig into the storage trie further. 417 if accIter.Leaf() { 418 accounts += 1 419 var acc types.StateAccount 420 if err := rlp.DecodeBytes(accIter.LeafBlob(), &acc); err != nil { 421 log.Error("Invalid account encountered during traversal", "err", err) 422 return errors.New("invalid account") 423 } 424 if acc.Root != emptyRoot { 425 id := trie.StorageTrieID(root, common.BytesToHash(accIter.LeafKey()), acc.Root) 426 storageTrie, err := trie.NewStateTrie(id, triedb) 427 if err != nil { 428 log.Error("Failed to open storage trie", "root", acc.Root, "err", err) 429 return errors.New("missing storage trie") 430 } 431 storageIter := storageTrie.NodeIterator(nil) 432 for storageIter.Next(true) { 433 nodes += 1 434 node := storageIter.Hash() 435 436 // Check the present for non-empty hash node(embedded node doesn't 437 // have their own hash). 438 if node != (common.Hash{}) { 439 blob := rawdb.ReadTrieNode(chaindb, node) 440 if len(blob) == 0 { 441 log.Error("Missing trie node(storage)", "hash", node) 442 return errors.New("missing storage") 443 } 444 hasher.Reset() 445 hasher.Write(blob) 446 hasher.Read(got) 447 if !bytes.Equal(got, node.Bytes()) { 448 log.Error("Invalid trie node(storage)", "hash", node.Hex(), "value", blob) 449 return errors.New("invalid storage node") 450 } 451 } 452 // Bump the counter if it's leaf node. 453 if storageIter.Leaf() { 454 slots += 1 455 } 456 } 457 if storageIter.Error() != nil { 458 log.Error("Failed to traverse storage trie", "root", acc.Root, "err", storageIter.Error()) 459 return storageIter.Error() 460 } 461 } 462 if !bytes.Equal(acc.CodeHash, emptyCode) { 463 if !rawdb.HasCode(chaindb, common.BytesToHash(acc.CodeHash)) { 464 log.Error("Code is missing", "account", common.BytesToHash(accIter.LeafKey())) 465 return errors.New("missing code") 466 } 467 codes += 1 468 } 469 if time.Since(lastReport) > time.Second*8 { 470 log.Info("Traversing state", "nodes", nodes, "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start))) 471 lastReport = time.Now() 472 } 473 } 474 } 475 if accIter.Error() != nil { 476 log.Error("Failed to traverse state trie", "root", root, "err", accIter.Error()) 477 return accIter.Error() 478 } 479 log.Info("State is complete", "nodes", nodes, "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start))) 480 return nil 481 } 482 483 func parseRoot(input string) (common.Hash, error) { 484 var h common.Hash 485 if err := h.UnmarshalText([]byte(input)); err != nil { 486 return h, err 487 } 488 return h, nil 489 } 490 491 func dumpState(ctx *cli.Context) error { 492 stack, _ := makeConfigNode(ctx) 493 defer stack.Close() 494 495 conf, db, root, err := parseDumpConfig(ctx, stack) 496 if err != nil { 497 return err 498 } 499 snapConfig := snapshot.Config{ 500 CacheSize: 256, 501 Recovery: false, 502 NoBuild: true, 503 AsyncBuild: false, 504 } 505 snaptree, err := snapshot.New(snapConfig, db, trie.NewDatabase(db), root) 506 if err != nil { 507 return err 508 } 509 accIt, err := snaptree.AccountIterator(root, common.BytesToHash(conf.Start)) 510 if err != nil { 511 return err 512 } 513 defer accIt.Release() 514 515 log.Info("Snapshot dumping started", "root", root) 516 var ( 517 start = time.Now() 518 logged = time.Now() 519 accounts uint64 520 ) 521 enc := json.NewEncoder(os.Stdout) 522 enc.Encode(struct { 523 Root common.Hash `json:"root"` 524 }{root}) 525 for accIt.Next() { 526 account, err := snapshot.FullAccount(accIt.Account()) 527 if err != nil { 528 return err 529 } 530 da := &state.DumpAccount{ 531 Balance: account.Balance.String(), 532 Nonce: account.Nonce, 533 Root: account.Root, 534 CodeHash: account.CodeHash, 535 SecureKey: accIt.Hash().Bytes(), 536 } 537 if !conf.SkipCode && !bytes.Equal(account.CodeHash, emptyCode) { 538 da.Code = rawdb.ReadCode(db, common.BytesToHash(account.CodeHash)) 539 } 540 if !conf.SkipStorage { 541 da.Storage = make(map[common.Hash]string) 542 543 stIt, err := snaptree.StorageIterator(root, accIt.Hash(), common.Hash{}) 544 if err != nil { 545 return err 546 } 547 for stIt.Next() { 548 da.Storage[stIt.Hash()] = common.Bytes2Hex(stIt.Slot()) 549 } 550 } 551 enc.Encode(da) 552 accounts++ 553 if time.Since(logged) > 8*time.Second { 554 log.Info("Snapshot dumping in progress", "at", accIt.Hash(), "accounts", accounts, 555 "elapsed", common.PrettyDuration(time.Since(start))) 556 logged = time.Now() 557 } 558 if conf.Max > 0 && accounts >= conf.Max { 559 break 560 } 561 } 562 log.Info("Snapshot dumping complete", "accounts", accounts, 563 "elapsed", common.PrettyDuration(time.Since(start))) 564 return nil 565 } 566 567 // checkAccount iterates the snap data layers, and looks up the given account 568 // across all layers. 569 func checkAccount(ctx *cli.Context) error { 570 if ctx.NArg() != 1 { 571 return errors.New("need <address|hash> arg") 572 } 573 var ( 574 hash common.Hash 575 addr common.Address 576 ) 577 switch arg := ctx.Args().First(); len(arg) { 578 case 40, 42: 579 addr = common.HexToAddress(arg) 580 hash = crypto.Keccak256Hash(addr.Bytes()) 581 case 64, 66: 582 hash = common.HexToHash(arg) 583 default: 584 return errors.New("malformed address or hash") 585 } 586 stack, _ := makeConfigNode(ctx) 587 defer stack.Close() 588 chaindb := utils.MakeChainDatabase(ctx, stack, true) 589 defer chaindb.Close() 590 start := time.Now() 591 log.Info("Checking difflayer journal", "address", addr, "hash", hash) 592 if err := snapshot.CheckJournalAccount(chaindb, hash); err != nil { 593 return err 594 } 595 log.Info("Checked the snapshot journalled storage", "time", common.PrettyDuration(time.Since(start))) 596 return nil 597 }