github.com/fff-chain/go-fff@v0.0.0-20220726032732-1c84420b8a99/core/rawdb/chain_iterator.go (about) 1 // Copyright 2019 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package rawdb 18 19 import ( 20 "runtime" 21 "sync/atomic" 22 "time" 23 24 "github.com/fff-chain/go-fff/common" 25 "github.com/fff-chain/go-fff/common/gopool" 26 "github.com/fff-chain/go-fff/common/prque" 27 "github.com/fff-chain/go-fff/core/types" 28 "github.com/fff-chain/go-fff/ethdb" 29 "github.com/fff-chain/go-fff/log" 30 "github.com/fff-chain/go-fff/rlp" 31 ) 32 33 // InitDatabaseFromFreezer reinitializes an empty database from a previous batch 34 // of frozen ancient blocks. The method iterates over all the frozen blocks and 35 // injects into the database the block hash->number mappings. 36 func InitDatabaseFromFreezer(db ethdb.Database) { 37 // If we can't access the freezer or it's empty, abort 38 frozen, err := db.ItemAmountInAncient() 39 if err != nil || frozen == 0 { 40 return 41 } 42 var ( 43 batch = db.NewBatch() 44 start = time.Now() 45 logged = start.Add(-7 * time.Second) // Unindex during import is fast, don't double log 46 hash common.Hash 47 offset = db.AncientOffSet() 48 ) 49 for i := uint64(0) + offset; i < frozen+offset; i++ { 50 // Since the freezer has all data in sequential order on a file, 51 // it would be 'neat' to read more data in one go, and let the 52 // freezerdb return N items (e.g up to 1000 items per go) 53 // That would require an API change in Ancients though 54 if h, err := db.Ancient(freezerHashTable, i); err != nil { 55 log.Crit("Failed to init database from freezer", "err", err) 56 } else { 57 hash = common.BytesToHash(h) 58 } 59 WriteHeaderNumber(batch, hash, i) 60 // If enough data was accumulated in memory or we're at the last block, dump to disk 61 if batch.ValueSize() > ethdb.IdealBatchSize { 62 if err := batch.Write(); err != nil { 63 log.Crit("Failed to write data to db", "err", err) 64 } 65 batch.Reset() 66 } 67 // If we've spent too much time already, notify the user of what we're doing 68 if time.Since(logged) > 8*time.Second { 69 log.Info("Initializing database from freezer", "total", frozen, "number", i, "hash", hash, "elapsed", common.PrettyDuration(time.Since(start))) 70 logged = time.Now() 71 } 72 } 73 if err := batch.Write(); err != nil { 74 log.Crit("Failed to write data to db", "err", err) 75 } 76 batch.Reset() 77 78 WriteHeadHeaderHash(db, hash) 79 WriteHeadFastBlockHash(db, hash) 80 log.Info("Initialized database from freezer", "blocks", frozen, "elapsed", common.PrettyDuration(time.Since(start))) 81 } 82 83 type blockTxHashes struct { 84 number uint64 85 hashes []common.Hash 86 } 87 88 // iterateTransactions iterates over all transactions in the (canon) block 89 // number(s) given, and yields the hashes on a channel. If there is a signal 90 // received from interrupt channel, the iteration will be aborted and result 91 // channel will be closed. 92 func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool, interrupt chan struct{}) chan *blockTxHashes { 93 // One thread sequentially reads data from db 94 type numberRlp struct { 95 number uint64 96 rlp rlp.RawValue 97 } 98 if to == from { 99 return nil 100 } 101 threads := to - from 102 if cpus := runtime.NumCPU(); threads > uint64(cpus) { 103 threads = uint64(cpus) 104 } 105 var ( 106 rlpCh = make(chan *numberRlp, threads*2) // we send raw rlp over this channel 107 hashesCh = make(chan *blockTxHashes, threads*2) // send hashes over hashesCh 108 ) 109 // lookup runs in one instance 110 lookup := func() { 111 n, end := from, to 112 if reverse { 113 n, end = to-1, from-1 114 } 115 defer close(rlpCh) 116 for n != end { 117 data := ReadCanonicalBodyRLP(db, n) 118 // Feed the block to the aggregator, or abort on interrupt 119 select { 120 case rlpCh <- &numberRlp{n, data}: 121 case <-interrupt: 122 return 123 } 124 if reverse { 125 n-- 126 } else { 127 n++ 128 } 129 } 130 } 131 // process runs in parallel 132 nThreadsAlive := int32(threads) 133 process := func() { 134 defer func() { 135 // Last processor closes the result channel 136 if atomic.AddInt32(&nThreadsAlive, -1) == 0 { 137 close(hashesCh) 138 } 139 }() 140 for data := range rlpCh { 141 var body types.Body 142 if err := rlp.DecodeBytes(data.rlp, &body); err != nil { 143 log.Warn("Failed to decode block body", "block", data.number, "error", err) 144 return 145 } 146 var hashes []common.Hash 147 for _, tx := range body.Transactions { 148 hashes = append(hashes, tx.Hash()) 149 } 150 result := &blockTxHashes{ 151 hashes: hashes, 152 number: data.number, 153 } 154 // Feed the block to the aggregator, or abort on interrupt 155 select { 156 case hashesCh <- result: 157 case <-interrupt: 158 return 159 } 160 } 161 } 162 go lookup() // start the sequential db accessor 163 for i := 0; i < int(threads); i++ { 164 gopool.Submit(func() { 165 process() 166 }) 167 } 168 return hashesCh 169 } 170 171 // indexTransactions creates txlookup indices of the specified block range. 172 // 173 // This function iterates canonical chain in reverse order, it has one main advantage: 174 // We can write tx index tail flag periodically even without the whole indexing 175 // procedure is finished. So that we can resume indexing procedure next time quickly. 176 // 177 // There is a passed channel, the whole procedure will be interrupted if any 178 // signal received. 179 func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { 180 // short circuit for invalid range 181 if from >= to { 182 return 183 } 184 var ( 185 hashesCh = iterateTransactions(db, from, to, true, interrupt) 186 batch = db.NewBatch() 187 start = time.Now() 188 logged = start.Add(-7 * time.Second) 189 // Since we iterate in reverse, we expect the first number to come 190 // in to be [to-1]. Therefore, setting lastNum to means that the 191 // prqueue gap-evaluation will work correctly 192 lastNum = to 193 queue = prque.New(nil) 194 // for stats reporting 195 blocks, txs = 0, 0 196 ) 197 for chanDelivery := range hashesCh { 198 // Push the delivery into the queue and process contiguous ranges. 199 // Since we iterate in reverse, so lower numbers have lower prio, and 200 // we can use the number directly as prio marker 201 queue.Push(chanDelivery, int64(chanDelivery.number)) 202 for !queue.Empty() { 203 // If the next available item is gapped, return 204 if _, priority := queue.Peek(); priority != int64(lastNum-1) { 205 break 206 } 207 // For testing 208 if hook != nil && !hook(lastNum-1) { 209 break 210 } 211 // Next block available, pop it off and index it 212 delivery := queue.PopItem().(*blockTxHashes) 213 lastNum = delivery.number 214 WriteTxLookupEntries(batch, delivery.number, delivery.hashes) 215 blocks++ 216 txs += len(delivery.hashes) 217 // If enough data was accumulated in memory or we're at the last block, dump to disk 218 if batch.ValueSize() > ethdb.IdealBatchSize { 219 WriteTxIndexTail(batch, lastNum) // Also write the tail here 220 if err := batch.Write(); err != nil { 221 log.Crit("Failed writing batch to db", "error", err) 222 return 223 } 224 batch.Reset() 225 } 226 // If we've spent too much time already, notify the user of what we're doing 227 if time.Since(logged) > 8*time.Second { 228 log.Info("Indexing transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "total", to-from, "elapsed", common.PrettyDuration(time.Since(start))) 229 logged = time.Now() 230 } 231 } 232 } 233 // Flush the new indexing tail and the last committed data. It can also happen 234 // that the last batch is empty because nothing to index, but the tail has to 235 // be flushed anyway. 236 WriteTxIndexTail(batch, lastNum) 237 if err := batch.Write(); err != nil { 238 log.Crit("Failed writing batch to db", "error", err) 239 return 240 } 241 select { 242 case <-interrupt: 243 log.Debug("Transaction indexing interrupted", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start))) 244 default: 245 log.Info("Indexed transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start))) 246 } 247 } 248 249 // IndexTransactions creates txlookup indices of the specified block range. 250 // 251 // This function iterates canonical chain in reverse order, it has one main advantage: 252 // We can write tx index tail flag periodically even without the whole indexing 253 // procedure is finished. So that we can resume indexing procedure next time quickly. 254 // 255 // There is a passed channel, the whole procedure will be interrupted if any 256 // signal received. 257 func IndexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}) { 258 indexTransactions(db, from, to, interrupt, nil) 259 } 260 261 // indexTransactionsForTesting is the internal debug version with an additional hook. 262 func indexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { 263 indexTransactions(db, from, to, interrupt, hook) 264 } 265 266 // unindexTransactions removes txlookup indices of the specified block range. 267 // 268 // There is a passed channel, the whole procedure will be interrupted if any 269 // signal received. 270 func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { 271 // short circuit for invalid range 272 if from >= to { 273 return 274 } 275 var ( 276 hashesCh = iterateTransactions(db, from, to, false, interrupt) 277 batch = db.NewBatch() 278 start = time.Now() 279 logged = start.Add(-7 * time.Second) 280 // we expect the first number to come in to be [from]. Therefore, setting 281 // nextNum to from means that the prqueue gap-evaluation will work correctly 282 nextNum = from 283 queue = prque.New(nil) 284 // for stats reporting 285 blocks, txs = 0, 0 286 ) 287 // Otherwise spin up the concurrent iterator and unindexer 288 for delivery := range hashesCh { 289 // Push the delivery into the queue and process contiguous ranges. 290 queue.Push(delivery, -int64(delivery.number)) 291 for !queue.Empty() { 292 // If the next available item is gapped, return 293 if _, priority := queue.Peek(); -priority != int64(nextNum) { 294 break 295 } 296 // For testing 297 if hook != nil && !hook(nextNum) { 298 break 299 } 300 delivery := queue.PopItem().(*blockTxHashes) 301 nextNum = delivery.number + 1 302 DeleteTxLookupEntries(batch, delivery.hashes) 303 txs += len(delivery.hashes) 304 blocks++ 305 306 // If enough data was accumulated in memory or we're at the last block, dump to disk 307 // A batch counts the size of deletion as '1', so we need to flush more 308 // often than that. 309 if blocks%1000 == 0 { 310 WriteTxIndexTail(batch, nextNum) 311 if err := batch.Write(); err != nil { 312 log.Crit("Failed writing batch to db", "error", err) 313 return 314 } 315 batch.Reset() 316 } 317 // If we've spent too much time already, notify the user of what we're doing 318 if time.Since(logged) > 8*time.Second { 319 log.Info("Unindexing transactions", "blocks", blocks, "txs", txs, "total", to-from, "elapsed", common.PrettyDuration(time.Since(start))) 320 logged = time.Now() 321 } 322 } 323 } 324 // Flush the new indexing tail and the last committed data. It can also happen 325 // that the last batch is empty because nothing to unindex, but the tail has to 326 // be flushed anyway. 327 WriteTxIndexTail(batch, nextNum) 328 if err := batch.Write(); err != nil { 329 log.Crit("Failed writing batch to db", "error", err) 330 return 331 } 332 select { 333 case <-interrupt: 334 log.Debug("Transaction unindexing interrupted", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) 335 default: 336 log.Info("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) 337 } 338 } 339 340 // UnindexTransactions removes txlookup indices of the specified block range. 341 // 342 // There is a passed channel, the whole procedure will be interrupted if any 343 // signal received. 344 func UnindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}) { 345 unindexTransactions(db, from, to, interrupt, nil) 346 } 347 348 // unindexTransactionsForTesting is the internal debug version with an additional hook. 349 func unindexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { 350 unindexTransactions(db, from, to, interrupt, hook) 351 }