github.com/neatio-net/neatio@v1.7.3-0.20231114194659-f4d7a2226baa/chain/core/datareduction/prune_processor.go (about) 1 package datareduction 2 3 import ( 4 "bytes" 5 "fmt" 6 "sort" 7 "sync/atomic" 8 "time" 9 10 "github.com/neatio-net/neatio/chain/core" 11 "github.com/neatio-net/neatio/chain/core/rawdb" 12 "github.com/neatio-net/neatio/chain/core/state" 13 "github.com/neatio-net/neatio/chain/log" 14 "github.com/neatio-net/neatio/chain/trie" 15 "github.com/neatio-net/neatio/neatdb" 16 "github.com/neatio-net/neatio/utilities/common" 17 "github.com/neatio-net/neatio/utilities/rlp" 18 ) 19 20 var ( 21 // max scan trie height 22 max_count_trie uint64 = 1000 23 // max retain trie height 24 max_remain_trie uint64 = 1000 25 // emptyRoot is the known root hash of an empty trie. 26 emptyRoot = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") 27 28 pruning int32 // indicate pruning is running or not 29 ) 30 31 type NodeCount map[common.Hash]uint64 32 33 type PruneProcessor struct { 34 db neatdb.Database // Low level persistent database to store prune counting statistics 35 prunedb PruneDatabase 36 37 bc *core.BlockChain 38 chainDb neatdb.Database // database instance to delete the state/block data 39 40 pruneBodyData bool 41 42 nodeCount NodeCount 43 } 44 45 type PruneStatus struct { 46 Running bool `json:"is_running"` 47 LatestBlockNumber uint64 `json:"latest_block_number"` 48 LatestScanNumber uint64 `json:"latest_scan_number"` 49 LatestPruneNumber uint64 `json:"latest_prune_number"` 50 } 51 52 type processLeafTrie func(addr common.Address, account state.Account) 53 54 func StartPruning() bool { 55 return atomic.CompareAndSwapInt32(&pruning, 0, 1) 56 } 57 58 func StopPruning() bool { 59 return atomic.CompareAndSwapInt32(&pruning, 1, 0) 60 } 61 62 func NewPruneProcessor(chaindb, prunedb neatdb.Database, bc *core.BlockChain, pruneBodyData bool) *PruneProcessor { 63 return &PruneProcessor{ 64 db: prunedb, 65 prunedb: NewDatabase(prunedb), 66 bc: bc, 67 chainDb: chaindb, 68 pruneBodyData: pruneBodyData, 69 nodeCount: make(NodeCount), 70 } 71 } 72 73 func (p *PruneProcessor) Process(blockNumber, scanNumber, pruneNumber uint64) (uint64, uint64) { 74 75 var needScan bool 76 var scanStart, scanEnd uint64 77 for { 78 // Step 1. determine the scan height 79 needScan, scanStart, scanEnd = calculateScan(scanNumber, blockNumber) 80 81 log.Infof("Data Reduction - scan ? %v , %d - %d", needScan, scanStart, scanEnd) 82 83 if needScan { 84 85 // Step 2. Read Latest Node Count 86 pruneBodyStart := uint64(0) 87 88 if pruneNumber > 0 { 89 90 pruneBodyStart = pruneNumber + 1 91 92 // Add previous state root for prune 93 for i := pruneNumber + 1; i <= scanNumber; i++ { 94 header := p.bc.GetHeaderByNumber(i) 95 p.countBlockChainTrie(header.Root, false) 96 log.Infof("countBlockChainTrie for block %d", i) 97 } 98 } else { 99 pruneBodyStart = scanStart 100 } 101 102 for i := scanStart; i <= scanEnd+1; i++ { 103 104 //TODO Cache the header 105 header := p.bc.GetHeaderByNumber(i) 106 p.countBlockChainTrie(header.Root, false) 107 108 //log.Printf("Block: %v, Root %x", i, header.Root) 109 if i%max_count_trie == max_count_trie-1 || i == scanEnd { 110 111 p.bc.MuLock() 112 113 header := p.bc.CurrentBlock().Header() 114 log.Infof("lastest block number is %v\n", header.Number.Uint64()) 115 p.countBlockChainTrie(header.Root, true) 116 p.processScanData(i) 117 118 p.bc.MuUnLock() 119 120 if p.pruneBodyData { 121 for j := pruneBodyStart; j <= i; j++ { 122 rawdb.DeleteBody(p.chainDb, rawdb.ReadCanonicalHash(p.chainDb, i), i) 123 } 124 log.Infof("deleted block from %v to %v", pruneBodyStart, i-1) 125 pruneBodyStart = i 126 } 127 } 128 //log.Infof("countBlockChainTrie for block %d", i) 129 } 130 131 blockNumber = p.bc.CurrentBlock().NumberU64() 132 scanNumber = scanEnd + 1 133 pruneNumber = scanEnd 134 135 } else { 136 time.Sleep(5 * time.Second) //sleep 5 seconds to wait more blocks to prune 137 blockNumber = p.bc.CurrentBlock().NumberU64() 138 } 139 } 140 141 return scanEnd + 1, scanEnd 142 } 143 144 func calculateScan(scan, latestBlockHeight uint64) (scanOrNot bool, from, to uint64) { 145 146 from = scan 147 to = 0 148 149 unscanHeight := latestBlockHeight - scan 150 if unscanHeight > max_remain_trie { 151 to = latestBlockHeight - max_remain_trie 152 } 153 154 if to != 0 { 155 scanOrNot = true 156 } 157 158 return 159 } 160 161 func (p *PruneProcessor) readLatestNodeCount(scanNumber, pruneNumber uint64) NodeCount { 162 nodeCount := make(NodeCount) 163 164 lastHash := rawdb.ReadDataPruneTrieRootHash(p.db, scanNumber, pruneNumber) 165 if (lastHash != common.Hash{}) { 166 lastPruneTrie, openErr := p.prunedb.OpenPruneTrie(lastHash) 167 if openErr != nil { 168 log.Error("Data Reduction - Unable read the last Prune Trie.", "err", openErr) 169 } else { 170 it := trie.NewIterator(lastPruneTrie.NodeIterator(nil)) 171 for it.Next() { 172 nodeHash := common.BytesToHash(lastPruneTrie.GetKey(it.Key)) 173 var nodeHashCount uint64 174 rlp.DecodeBytes(it.Value, &nodeHashCount) 175 nodeCount[nodeHash] = nodeHashCount 176 } 177 } 178 } 179 return nodeCount 180 } 181 182 func (p *PruneProcessor) countBlockChainTrie(root common.Hash, markNoPrune bool) (skip bool) { 183 t, openErr := p.bc.StateCache().OpenTrie(root) 184 if openErr != nil { 185 if _, ok := openErr.(*trie.MissingNodeError); ok { 186 // Missing Node Error means the root node of the trie has been removed earlier, so skip the trie and return 187 skip = true 188 } else { 189 log.Error("Data Reduction - Error when open the Main Trie", "err", openErr, "stateroot", root) 190 } 191 return 192 } 193 194 countTrie(t, p.nodeCount, markNoPrune, func(addr common.Address, account state.Account) { 195 if account.Root != emptyRoot { 196 if storageTrie, stErr := p.bc.StateCache().OpenStorageTrie(common.Hash{}, account.Root); stErr == nil { 197 countTrie(storageTrie, p.nodeCount, markNoPrune, nil) 198 } else { 199 log.Error("Data Reduction - Error when open the Storage Trie", "err", stErr, "storageroot", account.Root, "account", addr) 200 } 201 } 202 203 if account.TX1Root != emptyRoot { 204 if tx1Trie, tx1Err := p.bc.StateCache().OpenTX1Trie(common.Hash{}, account.TX1Root); tx1Err == nil { 205 countTrie(tx1Trie, p.nodeCount, markNoPrune, nil) 206 } else { 207 log.Error("Data Reduction - Error when open the TX1 Trie", "err", tx1Err, "tx1root", account.TX1Root, "account", addr) 208 } 209 } 210 211 if account.TX3Root != emptyRoot { 212 if tx3Trie, tx3Err := p.bc.StateCache().OpenTX3Trie(common.Hash{}, account.TX3Root); tx3Err == nil { 213 countTrie(tx3Trie, p.nodeCount, markNoPrune, nil) 214 } else { 215 log.Error("Data Reduction - Error when open the TX3 Trie", "err", tx3Err, "tx3root", account.TX3Root, "account", addr) 216 } 217 } 218 219 if account.ProxiedRoot != emptyRoot { 220 if proxiedTrie, proxiedErr := p.bc.StateCache().OpenProxiedTrie(common.Hash{}, account.ProxiedRoot); proxiedErr == nil { 221 countTrie(proxiedTrie, p.nodeCount, markNoPrune, nil) 222 } else { 223 log.Error("Data Reduction - Error when open the Proxied Trie", "err", proxiedErr, "proxiedroot", account.ProxiedRoot, "account", addr) 224 } 225 } 226 227 if account.RewardRoot != emptyRoot { 228 if rewardTrie, rewardErr := p.bc.StateCache().OpenRewardTrie(common.Hash{}, account.RewardRoot); rewardErr == nil { 229 countTrie(rewardTrie, p.nodeCount, markNoPrune, nil) 230 } else { 231 log.Error("Data Reduction - Error when open the Reward Trie", "err", rewardErr, "rewardroot", account.RewardRoot, "account", addr) 232 } 233 } 234 }) 235 return 236 } 237 238 func countTrie(t state.Trie, nodeCount NodeCount, markNoPrune bool, processLeaf processLeafTrie) { 239 240 side := true 241 if !markNoPrune { 242 for it := t.NodeIterator(nil); it.Next(side); { 243 if !it.Leaf() { 244 nodeHash := it.Hash() 245 if _, exist := nodeCount[nodeHash]; exist { 246 side = false 247 } else { 248 nodeCount[nodeHash] = 0 //this node occurs, may need prune 249 side = true 250 } 251 } else { 252 // Process the Account -> Inner Trie 253 if processLeaf != nil { 254 addr := t.GetKey(it.LeafKey()) 255 if len(addr) == 20 { 256 var data state.Account 257 rlp.DecodeBytes(it.LeafBlob(), &data) 258 259 processLeaf(common.BytesToAddress(addr), data) 260 } 261 } 262 } 263 } 264 } else { 265 for it := t.NodeIterator(nil); it.Next(side); { 266 if !it.Leaf() { 267 nodeHash := it.Hash() 268 nodeCount[nodeHash] = 1 //this node occurs in the latest block, mark no prune 269 } else { 270 // Process the Account -> Inner Trie 271 if processLeaf != nil { 272 addr := t.GetKey(it.LeafKey()) 273 if len(addr) == 20 { 274 var data state.Account 275 rlp.DecodeBytes(it.LeafBlob(), &data) 276 277 processLeaf(common.BytesToAddress(addr), data) 278 } 279 } 280 } 281 } 282 } 283 } 284 285 func (p *PruneProcessor) processScanData(latestScanNumber uint64) uint64 { 286 287 log.Infof("Data Reduction - After Scan, lastest scan number: %d", latestScanNumber) 288 289 // Prune State Data 290 p.pruneData() 291 292 newPruneNumber := latestScanNumber 293 294 // Commit the new scaned/pruned node count to trie 295 p.writeLastNumber(latestScanNumber, newPruneNumber) 296 297 log.Infof("Data Reduction - Scan/Prune Completed for trie %d %d", latestScanNumber, newPruneNumber) 298 return newPruneNumber 299 } 300 301 func (p *PruneProcessor) pruneData() { 302 303 count := 0 304 305 batch := p.chainDb.NewBatch() 306 for node, latest := range p.nodeCount { 307 if latest == 0 { 308 if batchDeleteError := batch.Delete(node.Bytes()); batchDeleteError != nil { 309 log.Error("Data Reduction - Error when delete the hash from chaindb", "err", batchDeleteError, "hash", node) 310 } 311 delete(p.nodeCount, node) 312 count++ 313 } else { 314 p.nodeCount[node] = 0 315 } 316 } 317 318 log.Infof("Data Reduction - %d hashes will be deleted from chaindb", count) 319 if writeErr := batch.Write(); writeErr != nil { 320 log.Error("Data Reduction - Error when write the deletion batch", "err", writeErr) 321 } else { 322 log.Infof("Data Reduction - write the deletion batch success, delete %v hashes", count) 323 } 324 } 325 326 func (p *PruneProcessor) writeLastNumber(lastScanNumber, lastPruneNumber uint64) { 327 rawdb.WriteHeadScanNumber(p.db, lastScanNumber) 328 rawdb.WriteHeadPruneNumber(p.db, lastPruneNumber) 329 } 330 331 func (nc NodeCount) String() string { 332 list := make([]common.Hash, 0, len(nc)) 333 for key := range nc { 334 list = append(list, key) 335 } 336 sort.Slice(list, func(i, j int) bool { 337 return bytes.Compare(list[i].Bytes(), list[j].Bytes()) == 1 338 }) 339 340 result := "" 341 for _, key := range list { 342 result += fmt.Sprintf("%v: %d \n", key.Hex(), nc[key]) 343 } 344 return result 345 } 346 347 func GetLatestStatus(prunedb neatdb.Database) *PruneStatus { 348 var scanNo, pruneNo uint64 349 if ps := rawdb.ReadHeadScanNumber(prunedb); ps != nil { 350 scanNo = *ps 351 } 352 if pp := rawdb.ReadHeadPruneNumber(prunedb); pp != nil { 353 pruneNo = *pp 354 } 355 356 return &PruneStatus{ 357 Running: atomic.LoadInt32(&pruning) == 1, 358 LatestScanNumber: scanNo, 359 LatestPruneNumber: pruneNo, 360 } 361 } 362 363 /* 364 func (p *PruneProcessor) pruneBlockChainTrie(root common.Hash, nodeCount NodeCount) { 365 t, openErr := p.bc.StateCache().OpenTrie(root) 366 if openErr != nil { 367 log.Error("Data Reduction - Error when open the Main Trie", "err", openErr, "stateroot", root) 368 return 369 } 370 371 pruneTrie(t, nodeCount, &p.pendingDeleteHashList, func(addr common.Address, account state.Account) { 372 if account.Root != emptyRoot { 373 if storageTrie, stErr := p.bc.StateCache().OpenStorageTrie(common.Hash{}, account.Root); stErr == nil { 374 pruneTrie(storageTrie, nodeCount, &p.pendingDeleteHashList, nil) 375 } else { 376 log.Error("Data Reduction - Error when open the Storage Trie", "err", stErr, "storageroot", account.Root, "account", addr) 377 } 378 } 379 380 if account.TX1Root != emptyRoot { 381 if tx1Trie, tx1Err := p.bc.StateCache().OpenTX1Trie(common.Hash{}, account.TX1Root); tx1Err == nil { 382 pruneTrie(tx1Trie, nodeCount, &p.pendingDeleteHashList, nil) 383 } else { 384 log.Error("Data Reduction - Error when open the TX1 Trie", "err", tx1Err, "tx1root", account.TX1Root, "account", addr) 385 } 386 } 387 388 if account.TX3Root != emptyRoot { 389 if tx3Trie, tx3Err := p.bc.StateCache().OpenTX3Trie(common.Hash{}, account.TX3Root); tx3Err == nil { 390 pruneTrie(tx3Trie, nodeCount, &p.pendingDeleteHashList, nil) 391 } else { 392 log.Error("Data Reduction - Error when open the TX3 Trie", "err", tx3Err, "tx3root", account.TX3Root, "account", addr) 393 } 394 } 395 396 if account.ProxiedRoot != emptyRoot { 397 if proxiedTrie, proxiedErr := p.bc.StateCache().OpenProxiedTrie(common.Hash{}, account.ProxiedRoot); proxiedErr == nil { 398 pruneTrie(proxiedTrie, nodeCount, &p.pendingDeleteHashList, nil) 399 } else { 400 log.Error("Data Reduction - Error when open the Proxied Trie", "err", proxiedErr, "proxiedroot", account.ProxiedRoot, "account", addr) 401 } 402 } 403 404 if account.RewardRoot != emptyRoot { 405 if rewardTrie, rewardErr := p.bc.StateCache().OpenRewardTrie(common.Hash{}, account.RewardRoot); rewardErr == nil { 406 pruneTrie(rewardTrie, nodeCount, &p.pendingDeleteHashList, nil) 407 } else { 408 log.Error("Data Reduction - Error when open the Reward Trie", "err", rewardErr, "rewardroot", account.RewardRoot, "account", addr) 409 } 410 } 411 }) 412 413 } 414 415 func pruneTrie(t state.Trie, nodeCount NodeCount, pendingDeleteHashList *[]common.Hash, processLeaf processLeafTrie) { 416 side := true 417 for it := t.NodeIterator(nil); it.Next(side); { 418 if !it.Leaf() { 419 nodeHash := it.Hash() 420 if nodeCount[nodeHash] > 0 { 421 nodeCount[nodeHash]-- 422 } 423 424 if nodeCount[nodeHash] == 0 { 425 side = true 426 *pendingDeleteHashList = append(*pendingDeleteHashList, nodeHash) 427 delete(nodeCount, nodeHash) 428 } else { 429 side = false 430 } 431 } else { 432 // Process the Account -> Inner Trie 433 if processLeaf != nil { 434 addr := t.GetKey(it.LeafKey()) 435 if len(addr) == 20 { 436 var data state.Account 437 rlp.DecodeBytes(it.LeafBlob(), &data) 438 439 processLeaf(common.BytesToAddress(addr), data) 440 } 441 } 442 } 443 } 444 } 445 446 func (p *PruneProcessor) commitDataPruneTrie(nodeCount NodeCount, lastScanNumber, lastPruneNumber uint64) { 447 // Store the Node Count into data prune trie 448 // Commit the Prune Trie 449 pruneTrie, _ := p.prunedb.OpenPruneTrie(common.Hash{}) 450 451 for key, count := range nodeCount { 452 value, _ := rlp.EncodeToBytes(count) 453 pruneTrie.TryUpdate(key[:], value) 454 } 455 pruneTrieRoot, commit_err := pruneTrie.Commit(nil) 456 log.Info("Data Reduction - Commit Prune Trie", "hash", pruneTrieRoot.Hex(), "err", commit_err) 457 // Commit to Prune DB 458 db_commit_err := p.prunedb.TrieDB().Commit(pruneTrieRoot, true) 459 log.Info("Data Reduction - Write to Prune DB", "err", db_commit_err) 460 461 // Write the Root Hash of Prune Trie 462 rawdb.WriteDataPruneTrieRootHash(p.db, pruneTrieRoot, lastScanNumber, lastPruneNumber) 463 // Write the last number 464 p.writeLastNumber(lastScanNumber, lastPruneNumber) 465 } 466 */