github.com/MetalBlockchain/subnet-evm@v0.6.3/core/rawdb/chain_iterator.go (about) 1 // (c) 2019-2022, Ava Labs, Inc. 2 // 3 // This file is a derived work, based on the go-ethereum library whose original 4 // notices appear below. 5 // 6 // It is distributed under a license compatible with the licensing terms of the 7 // original code from which it is derived. 8 // 9 // Much love to the original authors for their work. 10 // ********** 11 // Copyright 2020 The go-ethereum Authors 12 // This file is part of the go-ethereum library. 13 // 14 // The go-ethereum library is free software: you can redistribute it and/or modify 15 // it under the terms of the GNU Lesser General Public License as published by 16 // the Free Software Foundation, either version 3 of the License, or 17 // (at your option) any later version. 18 // 19 // The go-ethereum library is distributed in the hope that it will be useful, 20 // but WITHOUT ANY WARRANTY; without even the implied warranty of 21 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 // GNU Lesser General Public License for more details. 23 // 24 // You should have received a copy of the GNU Lesser General Public License 25 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 26 27 package rawdb 28 29 import ( 30 "runtime" 31 "sync/atomic" 32 "time" 33 34 "github.com/MetalBlockchain/subnet-evm/core/types" 35 "github.com/ethereum/go-ethereum/common" 36 "github.com/ethereum/go-ethereum/common/prque" 37 "github.com/ethereum/go-ethereum/ethdb" 38 "github.com/ethereum/go-ethereum/log" 39 "github.com/ethereum/go-ethereum/rlp" 40 ) 41 42 type blockTxHashes struct { 43 number uint64 44 hashes []common.Hash 45 } 46 47 // iterateTransactions iterates over all transactions in the (canon) block 48 // number(s) given, and yields the hashes on a channel. If there is a signal 49 // received from interrupt channel, the iteration will be aborted and result 50 // channel will be closed. 51 // Iterates blocks in the range [from, to) 52 func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool, interrupt chan struct{}) chan *blockTxHashes { 53 // One thread sequentially reads data from db 54 type numberRlp struct { 55 number uint64 56 rlp rlp.RawValue 57 } 58 if to == from { 59 return nil 60 } 61 threads := to - from 62 if cpus := runtime.NumCPU(); threads > uint64(cpus) { 63 threads = uint64(cpus) 64 } 65 var ( 66 rlpCh = make(chan *numberRlp, threads*2) // we send raw rlp over this channel 67 hashesCh = make(chan *blockTxHashes, threads*2) // send hashes over hashesCh 68 ) 69 // lookup runs in one instance 70 lookup := func() { 71 n, end := from, to 72 if reverse { 73 n, end = to-1, from-1 74 } 75 defer close(rlpCh) 76 for n != end { 77 data := ReadCanonicalBodyRLP(db, n) 78 // Feed the block to the aggregator, or abort on interrupt 79 select { 80 case rlpCh <- &numberRlp{n, data}: 81 case <-interrupt: 82 return 83 } 84 if reverse { 85 n-- 86 } else { 87 n++ 88 } 89 } 90 } 91 // process runs in parallel 92 var nThreadsAlive atomic.Int32 93 nThreadsAlive.Store(int32(threads)) 94 process := func() { 95 defer func() { 96 // Last processor closes the result channel 97 if nThreadsAlive.Add(-1) == 0 { 98 close(hashesCh) 99 } 100 }() 101 for data := range rlpCh { 102 var body types.Body 103 if err := rlp.DecodeBytes(data.rlp, &body); err != nil { 104 log.Warn("Failed to decode block body", "block", data.number, "error", err) 105 return 106 } 107 var hashes []common.Hash 108 for _, tx := range body.Transactions { 109 hashes = append(hashes, tx.Hash()) 110 } 111 result := &blockTxHashes{ 112 hashes: hashes, 113 number: data.number, 114 } 115 // Feed the block to the aggregator, or abort on interrupt 116 select { 117 case hashesCh <- result: 118 case <-interrupt: 119 return 120 } 121 } 122 } 123 go lookup() // start the sequential db accessor 124 for i := 0; i < int(threads); i++ { 125 go process() 126 } 127 return hashesCh 128 } 129 130 // indexTransactions creates txlookup indices of the specified block range. 131 // 132 // This function iterates canonical chain in reverse order, it has one main advantage: 133 // We can write tx index tail flag periodically even without the whole indexing 134 // procedure is finished. So that we can resume indexing procedure next time quickly. 135 // 136 // There is a passed channel, the whole procedure will be interrupted if any 137 // signal received. 138 func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { 139 // short circuit for invalid range 140 if from >= to { 141 return 142 } 143 var ( 144 hashesCh = iterateTransactions(db, from, to, true, interrupt) 145 batch = db.NewBatch() 146 start = time.Now() 147 logged = start.Add(-7 * time.Second) 148 // Since we iterate in reverse, we expect the first number to come 149 // in to be [to-1]. Therefore, setting lastNum to means that the 150 // prqueue gap-evaluation will work correctly 151 lastNum = to 152 queue = prque.New[int64, *blockTxHashes](nil) 153 // for stats reporting 154 blocks, txs = 0, 0 155 ) 156 for chanDelivery := range hashesCh { 157 // Push the delivery into the queue and process contiguous ranges. 158 // Since we iterate in reverse, so lower numbers have lower prio, and 159 // we can use the number directly as prio marker 160 queue.Push(chanDelivery, int64(chanDelivery.number)) 161 for !queue.Empty() { 162 // If the next available item is gapped, return 163 if _, priority := queue.Peek(); priority != int64(lastNum-1) { 164 break 165 } 166 // For testing 167 if hook != nil && !hook(lastNum-1) { 168 break 169 } 170 // Next block available, pop it off and index it 171 delivery := queue.PopItem() 172 lastNum = delivery.number 173 WriteTxLookupEntries(batch, delivery.number, delivery.hashes) 174 blocks++ 175 txs += len(delivery.hashes) 176 // If enough data was accumulated in memory or we're at the last block, dump to disk 177 if batch.ValueSize() > ethdb.IdealBatchSize { 178 WriteTxIndexTail(batch, lastNum) // Also write the tail here 179 if err := batch.Write(); err != nil { 180 log.Crit("Failed writing batch to db", "error", err) 181 return 182 } 183 batch.Reset() 184 } 185 // If we've spent too much time already, notify the user of what we're doing 186 if time.Since(logged) > 8*time.Second { 187 log.Info("Indexing transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "total", to-from, "elapsed", common.PrettyDuration(time.Since(start))) 188 logged = time.Now() 189 } 190 } 191 } 192 // Flush the new indexing tail and the last committed data. It can also happen 193 // that the last batch is empty because nothing to index, but the tail has to 194 // be flushed anyway. 195 WriteTxIndexTail(batch, lastNum) 196 if err := batch.Write(); err != nil { 197 log.Crit("Failed writing batch to db", "error", err) 198 return 199 } 200 select { 201 case <-interrupt: 202 log.Debug("Transaction indexing interrupted", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start))) 203 default: 204 log.Debug("Indexed transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start))) 205 } 206 } 207 208 // // IndexTransactions creates txlookup indices of the specified block range. The from 209 // // is included while to is excluded. 210 // // 211 // // This function iterates canonical chain in reverse order, it has one main advantage: 212 // // We can write tx index tail flag periodically even without the whole indexing 213 // // procedure is finished. So that we can resume indexing procedure next time quickly. 214 // // 215 // // There is a passed channel, the whole procedure will be interrupted if any 216 // // signal received. 217 // func IndexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}) { 218 // indexTransactions(db, from, to, interrupt, nil) 219 // } 220 221 // indexTransactionsForTesting is the internal debug version with an additional hook. 222 func indexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { 223 indexTransactions(db, from, to, interrupt, hook) 224 } 225 226 // unindexTransactions removes txlookup indices of the specified block range. 227 // 228 // There is a passed channel, the whole procedure will be interrupted if any 229 // signal received. 230 func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { 231 // short circuit for invalid range 232 if from >= to { 233 return 234 } 235 var ( 236 hashesCh = iterateTransactions(db, from, to, false, interrupt) 237 batch = db.NewBatch() 238 start = time.Now() 239 logged = start.Add(-7 * time.Second) 240 // we expect the first number to come in to be [from]. Therefore, setting 241 // nextNum to from means that the prqueue gap-evaluation will work correctly 242 nextNum = from 243 queue = prque.New[int64, *blockTxHashes](nil) 244 // for stats reporting 245 blocks, txs = 0, 0 246 ) 247 // Otherwise spin up the concurrent iterator and unindexer 248 for delivery := range hashesCh { 249 // Push the delivery into the queue and process contiguous ranges. 250 queue.Push(delivery, -int64(delivery.number)) 251 for !queue.Empty() { 252 // If the next available item is gapped, return 253 if _, priority := queue.Peek(); -priority != int64(nextNum) { 254 break 255 } 256 // For testing 257 if hook != nil && !hook(nextNum) { 258 break 259 } 260 delivery := queue.PopItem() 261 nextNum = delivery.number + 1 262 DeleteTxLookupEntries(batch, delivery.hashes) 263 txs += len(delivery.hashes) 264 blocks++ 265 266 // If enough data was accumulated in memory or we're at the last block, dump to disk 267 // A batch counts the size of deletion as '1', so we need to flush more 268 // often than that. 269 if blocks%1000 == 0 { 270 WriteTxIndexTail(batch, nextNum) 271 if err := batch.Write(); err != nil { 272 log.Crit("Failed writing batch to db", "error", err) 273 return 274 } 275 batch.Reset() 276 } 277 // If we've spent too much time already, notify the user of what we're doing 278 if time.Since(logged) > 8*time.Second { 279 log.Info("Unindexing transactions", "blocks", blocks, "txs", txs, "total", to-from, "elapsed", common.PrettyDuration(time.Since(start))) 280 logged = time.Now() 281 } 282 } 283 } 284 // Flush the new indexing tail and the last committed data. It can also happen 285 // that the last batch is empty because nothing to unindex, but the tail has to 286 // be flushed anyway. 287 WriteTxIndexTail(batch, nextNum) 288 if err := batch.Write(); err != nil { 289 log.Crit("Failed writing batch to db", "error", err) 290 return 291 } 292 select { 293 case <-interrupt: 294 log.Debug("Transaction unindexing interrupted", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) 295 default: 296 log.Debug("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) 297 } 298 } 299 300 // UnindexTransactions removes txlookup indices of the specified block range. 301 // The from is included while to is excluded. 302 // 303 // There is a passed channel, the whole procedure will be interrupted if any 304 // signal received. 305 func UnindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}) { 306 unindexTransactions(db, from, to, interrupt, nil) 307 } 308 309 // unindexTransactionsForTesting is the internal debug version with an additional hook. 310 func unindexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { 311 unindexTransactions(db, from, to, interrupt, hook) 312 }