github.com/aiyaya188/klaytn@v0.0.0-20220629133911-2c66fd5546f4/cmd/geth/snapshot.go (about) 1 // Copyright 2021 The go-ethereum Authors 2 // This file is part of go-ethereum. 3 // 4 // go-ethereum is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // go-ethereum is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU General Public License for more details. 13 // 14 // You should have received a copy of the GNU General Public License 15 // along with go-ethereum. If not, see <http://www.gnu.org/licenses/>. 16 17 package main 18 19 import ( 20 "bytes" 21 "encoding/json" 22 "errors" 23 "os" 24 "time" 25 26 "github.com/aiyaya188/klaytn/cmd/utils" 27 "github.com/aiyaya188/klaytn/common" 28 "github.com/aiyaya188/klaytn/core/rawdb" 29 "github.com/aiyaya188/klaytn/core/state" 30 "github.com/aiyaya188/klaytn/core/state/pruner" 31 "github.com/aiyaya188/klaytn/core/state/snapshot" 32 "github.com/aiyaya188/klaytn/core/types" 33 "github.com/aiyaya188/klaytn/crypto" 34 "github.com/aiyaya188/klaytn/log" 35 "github.com/aiyaya188/klaytn/rlp" 36 "github.com/aiyaya188/klaytn/trie" 37 cli "github.com/urfave/cli/v2" 38 ) 39 40 var ( 41 // emptyRoot is the known root hash of an empty trie. 42 emptyRoot = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") 43 44 // emptyCode is the known hash of the empty EVM bytecode. 45 emptyCode = crypto.Keccak256(nil) 46 ) 47 48 var ( 49 snapshotCommand = &cli.Command{ 50 Name: "snapshot", 51 Usage: "A set of commands based on the snapshot", 52 Description: "", 53 Subcommands: []*cli.Command{ 54 { 55 Name: "prune-state", 56 Usage: "Prune stale ethereum state data based on the snapshot", 57 ArgsUsage: "<root>", 58 Action: pruneState, 59 Flags: utils.GroupFlags([]cli.Flag{ 60 utils.CacheTrieJournalFlag, 61 utils.BloomFilterSizeFlag, 62 }, utils.NetworkFlags, utils.DatabasePathFlags), 63 Description: ` 64 geth snapshot prune-state <state-root> 65 will prune historical state data with the help of the state snapshot. 66 All trie nodes and contract codes that do not belong to the specified 67 version state will be deleted from the database. After pruning, only 68 two version states are available: genesis and the specific one. 69 70 The default pruning target is the HEAD-127 state. 71 72 WARNING: It's necessary to delete the trie clean cache after the pruning. 73 If you specify another directory for the trie clean cache via "--cache.trie.journal" 74 during the use of Geth, please also specify it here for correct deletion. Otherwise 75 the trie clean cache with default directory will be deleted. 76 `, 77 }, 78 { 79 Name: "verify-state", 80 Usage: "Recalculate state hash based on the snapshot for verification", 81 ArgsUsage: "<root>", 82 Action: verifyState, 83 Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), 84 Description: ` 85 geth snapshot verify-state <state-root> 86 will traverse the whole accounts and storages set based on the specified 87 snapshot and recalculate the root hash of state for verification. 88 In other words, this command does the snapshot to trie conversion. 89 `, 90 }, 91 { 92 Name: "check-dangling-storage", 93 Usage: "Check that there is no 'dangling' snap storage", 94 ArgsUsage: "<root>", 95 Action: checkDanglingStorage, 96 Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), 97 Description: ` 98 geth snapshot check-dangling-storage <state-root> traverses the snap storage 99 data, and verifies that all snapshot storage data has a corresponding account. 100 `, 101 }, 102 { 103 Name: "inspect-account", 104 Usage: "Check all snapshot layers for the a specific account", 105 ArgsUsage: "<address | hash>", 106 Action: checkAccount, 107 Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), 108 Description: ` 109 geth snapshot inspect-account <address | hash> checks all snapshot layers and prints out 110 information about the specified address. 111 `, 112 }, 113 { 114 Name: "traverse-state", 115 Usage: "Traverse the state with given root hash and perform quick verification", 116 ArgsUsage: "<root>", 117 Action: traverseState, 118 Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), 119 Description: ` 120 geth snapshot traverse-state <state-root> 121 will traverse the whole state from the given state root and will abort if any 122 referenced trie node or contract code is missing. This command can be used for 123 state integrity verification. The default checking target is the HEAD state. 124 125 It's also usable without snapshot enabled. 126 `, 127 }, 128 { 129 Name: "traverse-rawstate", 130 Usage: "Traverse the state with given root hash and perform detailed verification", 131 ArgsUsage: "<root>", 132 Action: traverseRawState, 133 Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), 134 Description: ` 135 geth snapshot traverse-rawstate <state-root> 136 will traverse the whole state from the given root and will abort if any referenced 137 trie node or contract code is missing. This command can be used for state integrity 138 verification. The default checking target is the HEAD state. It's basically identical 139 to traverse-state, but the check granularity is smaller. 140 141 It's also usable without snapshot enabled. 142 `, 143 }, 144 { 145 Name: "dump", 146 Usage: "Dump a specific block from storage (same as 'geth dump' but using snapshots)", 147 ArgsUsage: "[? <blockHash> | <blockNum>]", 148 Action: dumpState, 149 Flags: utils.GroupFlags([]cli.Flag{ 150 utils.ExcludeCodeFlag, 151 utils.ExcludeStorageFlag, 152 utils.StartKeyFlag, 153 utils.DumpLimitFlag, 154 }, utils.NetworkFlags, utils.DatabasePathFlags), 155 Description: ` 156 This command is semantically equivalent to 'geth dump', but uses the snapshots 157 as the backend data source, making this command a lot faster. 158 159 The argument is interpreted as block number or hash. If none is provided, the latest 160 block is used. 161 `, 162 }, 163 }, 164 } 165 ) 166 167 func pruneState(ctx *cli.Context) error { 168 stack, config := makeConfigNode(ctx) 169 defer stack.Close() 170 171 chaindb := utils.MakeChainDatabase(ctx, stack, false) 172 pruner, err := pruner.NewPruner(chaindb, stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal), ctx.Uint64(utils.BloomFilterSizeFlag.Name)) 173 if err != nil { 174 log.Error("Failed to open snapshot tree", "err", err) 175 return err 176 } 177 if ctx.NArg() > 1 { 178 log.Error("Too many arguments given") 179 return errors.New("too many arguments") 180 } 181 var targetRoot common.Hash 182 if ctx.NArg() == 1 { 183 targetRoot, err = parseRoot(ctx.Args().First()) 184 if err != nil { 185 log.Error("Failed to resolve state root", "err", err) 186 return err 187 } 188 } 189 if err = pruner.Prune(targetRoot); err != nil { 190 log.Error("Failed to prune state", "err", err) 191 return err 192 } 193 return nil 194 } 195 196 func verifyState(ctx *cli.Context) error { 197 stack, _ := makeConfigNode(ctx) 198 defer stack.Close() 199 200 chaindb := utils.MakeChainDatabase(ctx, stack, true) 201 headBlock := rawdb.ReadHeadBlock(chaindb) 202 if headBlock == nil { 203 log.Error("Failed to load head block") 204 return errors.New("no head block") 205 } 206 snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, headBlock.Root(), false, false, false) 207 if err != nil { 208 log.Error("Failed to open snapshot tree", "err", err) 209 return err 210 } 211 if ctx.NArg() > 1 { 212 log.Error("Too many arguments given") 213 return errors.New("too many arguments") 214 } 215 var root = headBlock.Root() 216 if ctx.NArg() == 1 { 217 root, err = parseRoot(ctx.Args().First()) 218 if err != nil { 219 log.Error("Failed to resolve state root", "err", err) 220 return err 221 } 222 } 223 if err := snaptree.Verify(root); err != nil { 224 log.Error("Failed to verify state", "root", root, "err", err) 225 return err 226 } 227 log.Info("Verified the state", "root", root) 228 return snapshot.CheckDanglingStorage(chaindb) 229 } 230 231 // checkDanglingStorage iterates the snap storage data, and verifies that all 232 // storage also has corresponding account data. 233 func checkDanglingStorage(ctx *cli.Context) error { 234 stack, _ := makeConfigNode(ctx) 235 defer stack.Close() 236 237 return snapshot.CheckDanglingStorage(utils.MakeChainDatabase(ctx, stack, true)) 238 } 239 240 // traverseState is a helper function used for pruning verification. 241 // Basically it just iterates the trie, ensure all nodes and associated 242 // contract codes are present. 243 func traverseState(ctx *cli.Context) error { 244 stack, _ := makeConfigNode(ctx) 245 defer stack.Close() 246 247 chaindb := utils.MakeChainDatabase(ctx, stack, true) 248 headBlock := rawdb.ReadHeadBlock(chaindb) 249 if headBlock == nil { 250 log.Error("Failed to load head block") 251 return errors.New("no head block") 252 } 253 if ctx.NArg() > 1 { 254 log.Error("Too many arguments given") 255 return errors.New("too many arguments") 256 } 257 var ( 258 root common.Hash 259 err error 260 ) 261 if ctx.NArg() == 1 { 262 root, err = parseRoot(ctx.Args().First()) 263 if err != nil { 264 log.Error("Failed to resolve state root", "err", err) 265 return err 266 } 267 log.Info("Start traversing the state", "root", root) 268 } else { 269 root = headBlock.Root() 270 log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) 271 } 272 triedb := trie.NewDatabase(chaindb) 273 t, err := trie.NewSecure(common.Hash{}, root, triedb) 274 if err != nil { 275 log.Error("Failed to open trie", "root", root, "err", err) 276 return err 277 } 278 var ( 279 accounts int 280 slots int 281 codes int 282 lastReport time.Time 283 start = time.Now() 284 ) 285 accIter := trie.NewIterator(t.NodeIterator(nil)) 286 for accIter.Next() { 287 accounts += 1 288 var acc types.StateAccount 289 if err := rlp.DecodeBytes(accIter.Value, &acc); err != nil { 290 log.Error("Invalid account encountered during traversal", "err", err) 291 return err 292 } 293 if acc.Root != emptyRoot { 294 storageTrie, err := trie.NewSecure(common.BytesToHash(accIter.Key), acc.Root, triedb) 295 if err != nil { 296 log.Error("Failed to open storage trie", "root", acc.Root, "err", err) 297 return err 298 } 299 storageIter := trie.NewIterator(storageTrie.NodeIterator(nil)) 300 for storageIter.Next() { 301 slots += 1 302 } 303 if storageIter.Err != nil { 304 log.Error("Failed to traverse storage trie", "root", acc.Root, "err", storageIter.Err) 305 return storageIter.Err 306 } 307 } 308 if !bytes.Equal(acc.CodeHash, emptyCode) { 309 if !rawdb.HasCode(chaindb, common.BytesToHash(acc.CodeHash)) { 310 log.Error("Code is missing", "hash", common.BytesToHash(acc.CodeHash)) 311 return errors.New("missing code") 312 } 313 codes += 1 314 } 315 if time.Since(lastReport) > time.Second*8 { 316 log.Info("Traversing state", "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start))) 317 lastReport = time.Now() 318 } 319 } 320 if accIter.Err != nil { 321 log.Error("Failed to traverse state trie", "root", root, "err", accIter.Err) 322 return accIter.Err 323 } 324 log.Info("State is complete", "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start))) 325 return nil 326 } 327 328 // traverseRawState is a helper function used for pruning verification. 329 // Basically it just iterates the trie, ensure all nodes and associated 330 // contract codes are present. It's basically identical to traverseState 331 // but it will check each trie node. 332 func traverseRawState(ctx *cli.Context) error { 333 stack, _ := makeConfigNode(ctx) 334 defer stack.Close() 335 336 chaindb := utils.MakeChainDatabase(ctx, stack, true) 337 headBlock := rawdb.ReadHeadBlock(chaindb) 338 if headBlock == nil { 339 log.Error("Failed to load head block") 340 return errors.New("no head block") 341 } 342 if ctx.NArg() > 1 { 343 log.Error("Too many arguments given") 344 return errors.New("too many arguments") 345 } 346 var ( 347 root common.Hash 348 err error 349 ) 350 if ctx.NArg() == 1 { 351 root, err = parseRoot(ctx.Args().First()) 352 if err != nil { 353 log.Error("Failed to resolve state root", "err", err) 354 return err 355 } 356 log.Info("Start traversing the state", "root", root) 357 } else { 358 root = headBlock.Root() 359 log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) 360 } 361 triedb := trie.NewDatabase(chaindb) 362 t, err := trie.NewSecure(common.Hash{}, root, triedb) 363 if err != nil { 364 log.Error("Failed to open trie", "root", root, "err", err) 365 return err 366 } 367 var ( 368 nodes int 369 accounts int 370 slots int 371 codes int 372 lastReport time.Time 373 start = time.Now() 374 hasher = crypto.NewKeccakState() 375 got = make([]byte, 32) 376 ) 377 accIter := t.NodeIterator(nil) 378 for accIter.Next(true) { 379 nodes += 1 380 node := accIter.Hash() 381 382 // Check the present for non-empty hash node(embedded node doesn't 383 // have their own hash). 384 if node != (common.Hash{}) { 385 blob := rawdb.ReadTrieNode(chaindb, node) 386 if len(blob) == 0 { 387 log.Error("Missing trie node(account)", "hash", node) 388 return errors.New("missing account") 389 } 390 hasher.Reset() 391 hasher.Write(blob) 392 hasher.Read(got) 393 if !bytes.Equal(got, node.Bytes()) { 394 log.Error("Invalid trie node(account)", "hash", node.Hex(), "value", blob) 395 return errors.New("invalid account node") 396 } 397 } 398 // If it's a leaf node, yes we are touching an account, 399 // dig into the storage trie further. 400 if accIter.Leaf() { 401 accounts += 1 402 var acc types.StateAccount 403 if err := rlp.DecodeBytes(accIter.LeafBlob(), &acc); err != nil { 404 log.Error("Invalid account encountered during traversal", "err", err) 405 return errors.New("invalid account") 406 } 407 if acc.Root != emptyRoot { 408 storageTrie, err := trie.NewSecure(common.BytesToHash(accIter.LeafKey()), acc.Root, triedb) 409 if err != nil { 410 log.Error("Failed to open storage trie", "root", acc.Root, "err", err) 411 return errors.New("missing storage trie") 412 } 413 storageIter := storageTrie.NodeIterator(nil) 414 for storageIter.Next(true) { 415 nodes += 1 416 node := storageIter.Hash() 417 418 // Check the present for non-empty hash node(embedded node doesn't 419 // have their own hash). 420 if node != (common.Hash{}) { 421 blob := rawdb.ReadTrieNode(chaindb, node) 422 if len(blob) == 0 { 423 log.Error("Missing trie node(storage)", "hash", node) 424 return errors.New("missing storage") 425 } 426 hasher.Reset() 427 hasher.Write(blob) 428 hasher.Read(got) 429 if !bytes.Equal(got, node.Bytes()) { 430 log.Error("Invalid trie node(storage)", "hash", node.Hex(), "value", blob) 431 return errors.New("invalid storage node") 432 } 433 } 434 // Bump the counter if it's leaf node. 435 if storageIter.Leaf() { 436 slots += 1 437 } 438 } 439 if storageIter.Error() != nil { 440 log.Error("Failed to traverse storage trie", "root", acc.Root, "err", storageIter.Error()) 441 return storageIter.Error() 442 } 443 } 444 if !bytes.Equal(acc.CodeHash, emptyCode) { 445 if !rawdb.HasCode(chaindb, common.BytesToHash(acc.CodeHash)) { 446 log.Error("Code is missing", "account", common.BytesToHash(accIter.LeafKey())) 447 return errors.New("missing code") 448 } 449 codes += 1 450 } 451 if time.Since(lastReport) > time.Second*8 { 452 log.Info("Traversing state", "nodes", nodes, "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start))) 453 lastReport = time.Now() 454 } 455 } 456 } 457 if accIter.Error() != nil { 458 log.Error("Failed to traverse state trie", "root", root, "err", accIter.Error()) 459 return accIter.Error() 460 } 461 log.Info("State is complete", "nodes", nodes, "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start))) 462 return nil 463 } 464 465 func parseRoot(input string) (common.Hash, error) { 466 var h common.Hash 467 if err := h.UnmarshalText([]byte(input)); err != nil { 468 return h, err 469 } 470 return h, nil 471 } 472 473 func dumpState(ctx *cli.Context) error { 474 stack, _ := makeConfigNode(ctx) 475 defer stack.Close() 476 477 conf, db, root, err := parseDumpConfig(ctx, stack) 478 if err != nil { 479 return err 480 } 481 snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, root, false, false, false) 482 if err != nil { 483 return err 484 } 485 accIt, err := snaptree.AccountIterator(root, common.BytesToHash(conf.Start)) 486 if err != nil { 487 return err 488 } 489 defer accIt.Release() 490 491 log.Info("Snapshot dumping started", "root", root) 492 var ( 493 start = time.Now() 494 logged = time.Now() 495 accounts uint64 496 ) 497 enc := json.NewEncoder(os.Stdout) 498 enc.Encode(struct { 499 Root common.Hash `json:"root"` 500 }{root}) 501 for accIt.Next() { 502 account, err := snapshot.FullAccount(accIt.Account()) 503 if err != nil { 504 return err 505 } 506 da := &state.DumpAccount{ 507 Balance: account.Balance.String(), 508 Nonce: account.Nonce, 509 Root: account.Root, 510 CodeHash: account.CodeHash, 511 SecureKey: accIt.Hash().Bytes(), 512 } 513 if !conf.SkipCode && !bytes.Equal(account.CodeHash, emptyCode) { 514 da.Code = rawdb.ReadCode(db, common.BytesToHash(account.CodeHash)) 515 } 516 if !conf.SkipStorage { 517 da.Storage = make(map[common.Hash]string) 518 519 stIt, err := snaptree.StorageIterator(root, accIt.Hash(), common.Hash{}) 520 if err != nil { 521 return err 522 } 523 for stIt.Next() { 524 da.Storage[stIt.Hash()] = common.Bytes2Hex(stIt.Slot()) 525 } 526 } 527 enc.Encode(da) 528 accounts++ 529 if time.Since(logged) > 8*time.Second { 530 log.Info("Snapshot dumping in progress", "at", accIt.Hash(), "accounts", accounts, 531 "elapsed", common.PrettyDuration(time.Since(start))) 532 logged = time.Now() 533 } 534 if conf.Max > 0 && accounts >= conf.Max { 535 break 536 } 537 } 538 log.Info("Snapshot dumping complete", "accounts", accounts, 539 "elapsed", common.PrettyDuration(time.Since(start))) 540 return nil 541 } 542 543 // checkAccount iterates the snap data layers, and looks up the given account 544 // across all layers. 545 func checkAccount(ctx *cli.Context) error { 546 if ctx.NArg() != 1 { 547 return errors.New("need <address|hash> arg") 548 } 549 var ( 550 hash common.Hash 551 addr common.Address 552 ) 553 switch arg := ctx.Args().First(); len(arg) { 554 case 40, 42: 555 addr = common.HexToAddress(arg) 556 hash = crypto.Keccak256Hash(addr.Bytes()) 557 case 64, 66: 558 hash = common.HexToHash(arg) 559 default: 560 return errors.New("malformed address or hash") 561 } 562 stack, _ := makeConfigNode(ctx) 563 defer stack.Close() 564 chaindb := utils.MakeChainDatabase(ctx, stack, true) 565 defer chaindb.Close() 566 start := time.Now() 567 log.Info("Checking difflayer journal", "address", addr, "hash", hash) 568 if err := snapshot.CheckJournalAccount(chaindb, hash); err != nil { 569 return err 570 } 571 log.Info("Checked the snapshot journalled storage", "time", common.PrettyDuration(time.Since(start))) 572 return nil 573 }