github.com/haliliceylan/bsc@v1.1.10-0.20220501224556-eb78d644ebcb/core/rawdb/chain_iterator.go (about) 1 // Copyright 2019 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package rawdb 18 19 import ( 20 "runtime" 21 "sync/atomic" 22 "time" 23 24 "github.com/ethereum/go-ethereum/common" 25 "github.com/ethereum/go-ethereum/common/gopool" 26 "github.com/ethereum/go-ethereum/common/prque" 27 "github.com/ethereum/go-ethereum/core/types" 28 "github.com/ethereum/go-ethereum/ethdb" 29 "github.com/ethereum/go-ethereum/log" 30 "github.com/ethereum/go-ethereum/rlp" 31 ) 32 33 // InitDatabaseFromFreezer reinitializes an empty database from a previous batch 34 // of frozen ancient blocks. The method iterates over all the frozen blocks and 35 // injects into the database the block hash->number mappings. 36 func InitDatabaseFromFreezer(db ethdb.Database) { 37 // If we can't access the freezer or it's empty, abort 38 frozen, err := db.ItemAmountInAncient() 39 if err != nil || frozen == 0 { 40 return 41 } 42 var ( 43 batch = db.NewBatch() 44 start = time.Now() 45 logged = start.Add(-7 * time.Second) // Unindex during import is fast, don't double log 46 hash common.Hash 47 offset = db.AncientOffSet() 48 ) 49 for i := uint64(0) + offset; i < frozen+offset; i++ { 50 // Since the freezer has all data in sequential order on a file, 51 // it would be 'neat' to read more data in one go, and let the 52 // freezerdb return N items (e.g up to 1000 items per go) 53 // That would require an API change in Ancients though 54 if h, err := db.Ancient(freezerHashTable, i); err != nil { 55 log.Crit("Failed to init database from freezer", "err", err) 56 } else { 57 hash = common.BytesToHash(h) 58 } 59 WriteHeaderNumber(batch, hash, i) 60 // If enough data was accumulated in memory or we're at the last block, dump to disk 61 if batch.ValueSize() > ethdb.IdealBatchSize { 62 if err := batch.Write(); err != nil { 63 log.Crit("Failed to write data to db", "err", err) 64 } 65 batch.Reset() 66 } 67 // If we've spent too much time already, notify the user of what we're doing 68 if time.Since(logged) > 8*time.Second { 69 log.Info("Initializing database from freezer", "total", frozen, "number", i, "hash", hash, "elapsed", common.PrettyDuration(time.Since(start))) 70 logged = time.Now() 71 } 72 } 73 if err := batch.Write(); err != nil { 74 log.Crit("Failed to write data to db", "err", err) 75 } 76 batch.Reset() 77 78 WriteHeadHeaderHash(db, hash) 79 WriteHeadFastBlockHash(db, hash) 80 log.Info("Initialized database from freezer", "blocks", frozen, "elapsed", common.PrettyDuration(time.Since(start))) 81 } 82 83 type blockTxHashes struct { 84 number uint64 85 hashes []common.Hash 86 } 87 88 // iterateTransactions iterates over all transactions in the (canon) block 89 // number(s) given, and yields the hashes on a channel. If there is a signal 90 // received from interrupt channel, the iteration will be aborted and result 91 // channel will be closed. 92 func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool, interrupt chan struct{}) chan *blockTxHashes { 93 // One thread sequentially reads data from db 94 type numberRlp struct { 95 number uint64 96 rlp rlp.RawValue 97 } 98 if offset := db.AncientOffSet(); offset > from { 99 from = offset 100 } 101 if to <= from { 102 return nil 103 } 104 threads := to - from 105 if cpus := runtime.NumCPU(); threads > uint64(cpus) { 106 threads = uint64(cpus) 107 } 108 var ( 109 rlpCh = make(chan *numberRlp, threads*2) // we send raw rlp over this channel 110 hashesCh = make(chan *blockTxHashes, threads*2) // send hashes over hashesCh 111 ) 112 // lookup runs in one instance 113 lookup := func() { 114 n, end := from, to 115 if reverse { 116 n, end = to-1, from-1 117 } 118 defer close(rlpCh) 119 for n != end { 120 data := ReadCanonicalBodyRLP(db, n) 121 // Feed the block to the aggregator, or abort on interrupt 122 select { 123 case rlpCh <- &numberRlp{n, data}: 124 case <-interrupt: 125 return 126 } 127 if reverse { 128 n-- 129 } else { 130 n++ 131 } 132 } 133 } 134 // process runs in parallel 135 nThreadsAlive := int32(threads) 136 process := func() { 137 defer func() { 138 // Last processor closes the result channel 139 if atomic.AddInt32(&nThreadsAlive, -1) == 0 { 140 close(hashesCh) 141 } 142 }() 143 for data := range rlpCh { 144 var body types.Body 145 if err := rlp.DecodeBytes(data.rlp, &body); err != nil { 146 log.Warn("Failed to decode block body", "block", data.number, "error", err) 147 return 148 } 149 var hashes []common.Hash 150 for _, tx := range body.Transactions { 151 hashes = append(hashes, tx.Hash()) 152 } 153 result := &blockTxHashes{ 154 hashes: hashes, 155 number: data.number, 156 } 157 // Feed the block to the aggregator, or abort on interrupt 158 select { 159 case hashesCh <- result: 160 case <-interrupt: 161 return 162 } 163 } 164 } 165 go lookup() // start the sequential db accessor 166 for i := 0; i < int(threads); i++ { 167 gopool.Submit(func() { 168 process() 169 }) 170 } 171 return hashesCh 172 } 173 174 // indexTransactions creates txlookup indices of the specified block range. 175 // 176 // This function iterates canonical chain in reverse order, it has one main advantage: 177 // We can write tx index tail flag periodically even without the whole indexing 178 // procedure is finished. So that we can resume indexing procedure next time quickly. 179 // 180 // There is a passed channel, the whole procedure will be interrupted if any 181 // signal received. 182 func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { 183 // short circuit for invalid range 184 if from >= to { 185 return 186 } 187 var ( 188 hashesCh = iterateTransactions(db, from, to, true, interrupt) 189 batch = db.NewBatch() 190 start = time.Now() 191 logged = start.Add(-7 * time.Second) 192 // Since we iterate in reverse, we expect the first number to come 193 // in to be [to-1]. Therefore, setting lastNum to means that the 194 // prqueue gap-evaluation will work correctly 195 lastNum = to 196 queue = prque.New(nil) 197 // for stats reporting 198 blocks, txs = 0, 0 199 ) 200 for chanDelivery := range hashesCh { 201 // Push the delivery into the queue and process contiguous ranges. 202 // Since we iterate in reverse, so lower numbers have lower prio, and 203 // we can use the number directly as prio marker 204 queue.Push(chanDelivery, int64(chanDelivery.number)) 205 for !queue.Empty() { 206 // If the next available item is gapped, return 207 if _, priority := queue.Peek(); priority != int64(lastNum-1) { 208 break 209 } 210 // For testing 211 if hook != nil && !hook(lastNum-1) { 212 break 213 } 214 // Next block available, pop it off and index it 215 delivery := queue.PopItem().(*blockTxHashes) 216 lastNum = delivery.number 217 WriteTxLookupEntries(batch, delivery.number, delivery.hashes) 218 blocks++ 219 txs += len(delivery.hashes) 220 // If enough data was accumulated in memory or we're at the last block, dump to disk 221 if batch.ValueSize() > ethdb.IdealBatchSize { 222 WriteTxIndexTail(batch, lastNum) // Also write the tail here 223 if err := batch.Write(); err != nil { 224 log.Crit("Failed writing batch to db", "error", err) 225 return 226 } 227 batch.Reset() 228 } 229 // If we've spent too much time already, notify the user of what we're doing 230 if time.Since(logged) > 8*time.Second { 231 log.Info("Indexing transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "total", to-from, "elapsed", common.PrettyDuration(time.Since(start))) 232 logged = time.Now() 233 } 234 } 235 } 236 // Flush the new indexing tail and the last committed data. It can also happen 237 // that the last batch is empty because nothing to index, but the tail has to 238 // be flushed anyway. 239 WriteTxIndexTail(batch, lastNum) 240 if err := batch.Write(); err != nil { 241 log.Crit("Failed writing batch to db", "error", err) 242 return 243 } 244 select { 245 case <-interrupt: 246 log.Debug("Transaction indexing interrupted", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start))) 247 default: 248 log.Info("Indexed transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start))) 249 } 250 } 251 252 // IndexTransactions creates txlookup indices of the specified block range. 253 // 254 // This function iterates canonical chain in reverse order, it has one main advantage: 255 // We can write tx index tail flag periodically even without the whole indexing 256 // procedure is finished. So that we can resume indexing procedure next time quickly. 257 // 258 // There is a passed channel, the whole procedure will be interrupted if any 259 // signal received. 260 func IndexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}) { 261 indexTransactions(db, from, to, interrupt, nil) 262 } 263 264 // indexTransactionsForTesting is the internal debug version with an additional hook. 265 func indexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { 266 indexTransactions(db, from, to, interrupt, hook) 267 } 268 269 // unindexTransactions removes txlookup indices of the specified block range. 270 // 271 // There is a passed channel, the whole procedure will be interrupted if any 272 // signal received. 273 func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { 274 // short circuit for invalid range 275 if from >= to { 276 return 277 } 278 var ( 279 hashesCh = iterateTransactions(db, from, to, false, interrupt) 280 batch = db.NewBatch() 281 start = time.Now() 282 logged = start.Add(-7 * time.Second) 283 // we expect the first number to come in to be [from]. Therefore, setting 284 // nextNum to from means that the prqueue gap-evaluation will work correctly 285 nextNum = from 286 queue = prque.New(nil) 287 // for stats reporting 288 blocks, txs = 0, 0 289 ) 290 // Otherwise spin up the concurrent iterator and unindexer 291 for delivery := range hashesCh { 292 // Push the delivery into the queue and process contiguous ranges. 293 queue.Push(delivery, -int64(delivery.number)) 294 for !queue.Empty() { 295 // If the next available item is gapped, return 296 if _, priority := queue.Peek(); -priority != int64(nextNum) { 297 break 298 } 299 // For testing 300 if hook != nil && !hook(nextNum) { 301 break 302 } 303 delivery := queue.PopItem().(*blockTxHashes) 304 nextNum = delivery.number + 1 305 DeleteTxLookupEntries(batch, delivery.hashes) 306 txs += len(delivery.hashes) 307 blocks++ 308 309 // If enough data was accumulated in memory or we're at the last block, dump to disk 310 // A batch counts the size of deletion as '1', so we need to flush more 311 // often than that. 312 if blocks%1000 == 0 { 313 WriteTxIndexTail(batch, nextNum) 314 if err := batch.Write(); err != nil { 315 log.Crit("Failed writing batch to db", "error", err) 316 return 317 } 318 batch.Reset() 319 } 320 // If we've spent too much time already, notify the user of what we're doing 321 if time.Since(logged) > 8*time.Second { 322 log.Info("Unindexing transactions", "blocks", blocks, "txs", txs, "total", to-from, "elapsed", common.PrettyDuration(time.Since(start))) 323 logged = time.Now() 324 } 325 } 326 } 327 // Flush the new indexing tail and the last committed data. It can also happen 328 // that the last batch is empty because nothing to unindex, but the tail has to 329 // be flushed anyway. 330 WriteTxIndexTail(batch, nextNum) 331 if err := batch.Write(); err != nil { 332 log.Crit("Failed writing batch to db", "error", err) 333 return 334 } 335 select { 336 case <-interrupt: 337 log.Debug("Transaction unindexing interrupted", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) 338 default: 339 log.Info("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) 340 } 341 } 342 343 // UnindexTransactions removes txlookup indices of the specified block range. 344 // 345 // There is a passed channel, the whole procedure will be interrupted if any 346 // signal received. 347 func UnindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}) { 348 unindexTransactions(db, from, to, interrupt, nil) 349 } 350 351 // unindexTransactionsForTesting is the internal debug version with an additional hook. 352 func unindexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { 353 unindexTransactions(db, from, to, interrupt, hook) 354 }