github.com/haliliceylan/bsc@v1.1.10-0.20220501224556-eb78d644ebcb/core/rawdb/chain_iterator.go (about)

     1  // Copyright 2019 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package rawdb
    18  
    19  import (
    20  	"runtime"
    21  	"sync/atomic"
    22  	"time"
    23  
    24  	"github.com/ethereum/go-ethereum/common"
    25  	"github.com/ethereum/go-ethereum/common/gopool"
    26  	"github.com/ethereum/go-ethereum/common/prque"
    27  	"github.com/ethereum/go-ethereum/core/types"
    28  	"github.com/ethereum/go-ethereum/ethdb"
    29  	"github.com/ethereum/go-ethereum/log"
    30  	"github.com/ethereum/go-ethereum/rlp"
    31  )
    32  
    33  // InitDatabaseFromFreezer reinitializes an empty database from a previous batch
    34  // of frozen ancient blocks. The method iterates over all the frozen blocks and
    35  // injects into the database the block hash->number mappings.
    36  func InitDatabaseFromFreezer(db ethdb.Database) {
    37  	// If we can't access the freezer or it's empty, abort
    38  	frozen, err := db.ItemAmountInAncient()
    39  	if err != nil || frozen == 0 {
    40  		return
    41  	}
    42  	var (
    43  		batch  = db.NewBatch()
    44  		start  = time.Now()
    45  		logged = start.Add(-7 * time.Second) // Unindex during import is fast, don't double log
    46  		hash   common.Hash
    47  		offset = db.AncientOffSet()
    48  	)
    49  	for i := uint64(0) + offset; i < frozen+offset; i++ {
    50  		// Since the freezer has all data in sequential order on a file,
    51  		// it would be 'neat' to read more data in one go, and let the
    52  		// freezerdb return N items (e.g up to 1000 items per go)
    53  		// That would require an API change in Ancients though
    54  		if h, err := db.Ancient(freezerHashTable, i); err != nil {
    55  			log.Crit("Failed to init database from freezer", "err", err)
    56  		} else {
    57  			hash = common.BytesToHash(h)
    58  		}
    59  		WriteHeaderNumber(batch, hash, i)
    60  		// If enough data was accumulated in memory or we're at the last block, dump to disk
    61  		if batch.ValueSize() > ethdb.IdealBatchSize {
    62  			if err := batch.Write(); err != nil {
    63  				log.Crit("Failed to write data to db", "err", err)
    64  			}
    65  			batch.Reset()
    66  		}
    67  		// If we've spent too much time already, notify the user of what we're doing
    68  		if time.Since(logged) > 8*time.Second {
    69  			log.Info("Initializing database from freezer", "total", frozen, "number", i, "hash", hash, "elapsed", common.PrettyDuration(time.Since(start)))
    70  			logged = time.Now()
    71  		}
    72  	}
    73  	if err := batch.Write(); err != nil {
    74  		log.Crit("Failed to write data to db", "err", err)
    75  	}
    76  	batch.Reset()
    77  
    78  	WriteHeadHeaderHash(db, hash)
    79  	WriteHeadFastBlockHash(db, hash)
    80  	log.Info("Initialized database from freezer", "blocks", frozen, "elapsed", common.PrettyDuration(time.Since(start)))
    81  }
    82  
    83  type blockTxHashes struct {
    84  	number uint64
    85  	hashes []common.Hash
    86  }
    87  
    88  // iterateTransactions iterates over all transactions in the (canon) block
    89  // number(s) given, and yields the hashes on a channel. If there is a signal
    90  // received from interrupt channel, the iteration will be aborted and result
    91  // channel will be closed.
    92  func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool, interrupt chan struct{}) chan *blockTxHashes {
    93  	// One thread sequentially reads data from db
    94  	type numberRlp struct {
    95  		number uint64
    96  		rlp    rlp.RawValue
    97  	}
    98  	if offset := db.AncientOffSet(); offset > from {
    99  		from = offset
   100  	}
   101  	if to <= from {
   102  		return nil
   103  	}
   104  	threads := to - from
   105  	if cpus := runtime.NumCPU(); threads > uint64(cpus) {
   106  		threads = uint64(cpus)
   107  	}
   108  	var (
   109  		rlpCh    = make(chan *numberRlp, threads*2)     // we send raw rlp over this channel
   110  		hashesCh = make(chan *blockTxHashes, threads*2) // send hashes over hashesCh
   111  	)
   112  	// lookup runs in one instance
   113  	lookup := func() {
   114  		n, end := from, to
   115  		if reverse {
   116  			n, end = to-1, from-1
   117  		}
   118  		defer close(rlpCh)
   119  		for n != end {
   120  			data := ReadCanonicalBodyRLP(db, n)
   121  			// Feed the block to the aggregator, or abort on interrupt
   122  			select {
   123  			case rlpCh <- &numberRlp{n, data}:
   124  			case <-interrupt:
   125  				return
   126  			}
   127  			if reverse {
   128  				n--
   129  			} else {
   130  				n++
   131  			}
   132  		}
   133  	}
   134  	// process runs in parallel
   135  	nThreadsAlive := int32(threads)
   136  	process := func() {
   137  		defer func() {
   138  			// Last processor closes the result channel
   139  			if atomic.AddInt32(&nThreadsAlive, -1) == 0 {
   140  				close(hashesCh)
   141  			}
   142  		}()
   143  		for data := range rlpCh {
   144  			var body types.Body
   145  			if err := rlp.DecodeBytes(data.rlp, &body); err != nil {
   146  				log.Warn("Failed to decode block body", "block", data.number, "error", err)
   147  				return
   148  			}
   149  			var hashes []common.Hash
   150  			for _, tx := range body.Transactions {
   151  				hashes = append(hashes, tx.Hash())
   152  			}
   153  			result := &blockTxHashes{
   154  				hashes: hashes,
   155  				number: data.number,
   156  			}
   157  			// Feed the block to the aggregator, or abort on interrupt
   158  			select {
   159  			case hashesCh <- result:
   160  			case <-interrupt:
   161  				return
   162  			}
   163  		}
   164  	}
   165  	go lookup() // start the sequential db accessor
   166  	for i := 0; i < int(threads); i++ {
   167  		gopool.Submit(func() {
   168  			process()
   169  		})
   170  	}
   171  	return hashesCh
   172  }
   173  
   174  // indexTransactions creates txlookup indices of the specified block range.
   175  //
   176  // This function iterates canonical chain in reverse order, it has one main advantage:
   177  // We can write tx index tail flag periodically even without the whole indexing
   178  // procedure is finished. So that we can resume indexing procedure next time quickly.
   179  //
   180  // There is a passed channel, the whole procedure will be interrupted if any
   181  // signal received.
   182  func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) {
   183  	// short circuit for invalid range
   184  	if from >= to {
   185  		return
   186  	}
   187  	var (
   188  		hashesCh = iterateTransactions(db, from, to, true, interrupt)
   189  		batch    = db.NewBatch()
   190  		start    = time.Now()
   191  		logged   = start.Add(-7 * time.Second)
   192  		// Since we iterate in reverse, we expect the first number to come
   193  		// in to be [to-1]. Therefore, setting lastNum to means that the
   194  		// prqueue gap-evaluation will work correctly
   195  		lastNum = to
   196  		queue   = prque.New(nil)
   197  		// for stats reporting
   198  		blocks, txs = 0, 0
   199  	)
   200  	for chanDelivery := range hashesCh {
   201  		// Push the delivery into the queue and process contiguous ranges.
   202  		// Since we iterate in reverse, so lower numbers have lower prio, and
   203  		// we can use the number directly as prio marker
   204  		queue.Push(chanDelivery, int64(chanDelivery.number))
   205  		for !queue.Empty() {
   206  			// If the next available item is gapped, return
   207  			if _, priority := queue.Peek(); priority != int64(lastNum-1) {
   208  				break
   209  			}
   210  			// For testing
   211  			if hook != nil && !hook(lastNum-1) {
   212  				break
   213  			}
   214  			// Next block available, pop it off and index it
   215  			delivery := queue.PopItem().(*blockTxHashes)
   216  			lastNum = delivery.number
   217  			WriteTxLookupEntries(batch, delivery.number, delivery.hashes)
   218  			blocks++
   219  			txs += len(delivery.hashes)
   220  			// If enough data was accumulated in memory or we're at the last block, dump to disk
   221  			if batch.ValueSize() > ethdb.IdealBatchSize {
   222  				WriteTxIndexTail(batch, lastNum) // Also write the tail here
   223  				if err := batch.Write(); err != nil {
   224  					log.Crit("Failed writing batch to db", "error", err)
   225  					return
   226  				}
   227  				batch.Reset()
   228  			}
   229  			// If we've spent too much time already, notify the user of what we're doing
   230  			if time.Since(logged) > 8*time.Second {
   231  				log.Info("Indexing transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "total", to-from, "elapsed", common.PrettyDuration(time.Since(start)))
   232  				logged = time.Now()
   233  			}
   234  		}
   235  	}
   236  	// Flush the new indexing tail and the last committed data. It can also happen
   237  	// that the last batch is empty because nothing to index, but the tail has to
   238  	// be flushed anyway.
   239  	WriteTxIndexTail(batch, lastNum)
   240  	if err := batch.Write(); err != nil {
   241  		log.Crit("Failed writing batch to db", "error", err)
   242  		return
   243  	}
   244  	select {
   245  	case <-interrupt:
   246  		log.Debug("Transaction indexing interrupted", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start)))
   247  	default:
   248  		log.Info("Indexed transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start)))
   249  	}
   250  }
   251  
   252  // IndexTransactions creates txlookup indices of the specified block range.
   253  //
   254  // This function iterates canonical chain in reverse order, it has one main advantage:
   255  // We can write tx index tail flag periodically even without the whole indexing
   256  // procedure is finished. So that we can resume indexing procedure next time quickly.
   257  //
   258  // There is a passed channel, the whole procedure will be interrupted if any
   259  // signal received.
   260  func IndexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}) {
   261  	indexTransactions(db, from, to, interrupt, nil)
   262  }
   263  
   264  // indexTransactionsForTesting is the internal debug version with an additional hook.
   265  func indexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) {
   266  	indexTransactions(db, from, to, interrupt, hook)
   267  }
   268  
   269  // unindexTransactions removes txlookup indices of the specified block range.
   270  //
   271  // There is a passed channel, the whole procedure will be interrupted if any
   272  // signal received.
   273  func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) {
   274  	// short circuit for invalid range
   275  	if from >= to {
   276  		return
   277  	}
   278  	var (
   279  		hashesCh = iterateTransactions(db, from, to, false, interrupt)
   280  		batch    = db.NewBatch()
   281  		start    = time.Now()
   282  		logged   = start.Add(-7 * time.Second)
   283  		// we expect the first number to come in to be [from]. Therefore, setting
   284  		// nextNum to from means that the prqueue gap-evaluation will work correctly
   285  		nextNum = from
   286  		queue   = prque.New(nil)
   287  		// for stats reporting
   288  		blocks, txs = 0, 0
   289  	)
   290  	// Otherwise spin up the concurrent iterator and unindexer
   291  	for delivery := range hashesCh {
   292  		// Push the delivery into the queue and process contiguous ranges.
   293  		queue.Push(delivery, -int64(delivery.number))
   294  		for !queue.Empty() {
   295  			// If the next available item is gapped, return
   296  			if _, priority := queue.Peek(); -priority != int64(nextNum) {
   297  				break
   298  			}
   299  			// For testing
   300  			if hook != nil && !hook(nextNum) {
   301  				break
   302  			}
   303  			delivery := queue.PopItem().(*blockTxHashes)
   304  			nextNum = delivery.number + 1
   305  			DeleteTxLookupEntries(batch, delivery.hashes)
   306  			txs += len(delivery.hashes)
   307  			blocks++
   308  
   309  			// If enough data was accumulated in memory or we're at the last block, dump to disk
   310  			// A batch counts the size of deletion as '1', so we need to flush more
   311  			// often than that.
   312  			if blocks%1000 == 0 {
   313  				WriteTxIndexTail(batch, nextNum)
   314  				if err := batch.Write(); err != nil {
   315  					log.Crit("Failed writing batch to db", "error", err)
   316  					return
   317  				}
   318  				batch.Reset()
   319  			}
   320  			// If we've spent too much time already, notify the user of what we're doing
   321  			if time.Since(logged) > 8*time.Second {
   322  				log.Info("Unindexing transactions", "blocks", blocks, "txs", txs, "total", to-from, "elapsed", common.PrettyDuration(time.Since(start)))
   323  				logged = time.Now()
   324  			}
   325  		}
   326  	}
   327  	// Flush the new indexing tail and the last committed data. It can also happen
   328  	// that the last batch is empty because nothing to unindex, but the tail has to
   329  	// be flushed anyway.
   330  	WriteTxIndexTail(batch, nextNum)
   331  	if err := batch.Write(); err != nil {
   332  		log.Crit("Failed writing batch to db", "error", err)
   333  		return
   334  	}
   335  	select {
   336  	case <-interrupt:
   337  		log.Debug("Transaction unindexing interrupted", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start)))
   338  	default:
   339  		log.Info("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start)))
   340  	}
   341  }
   342  
   343  // UnindexTransactions removes txlookup indices of the specified block range.
   344  //
   345  // There is a passed channel, the whole procedure will be interrupted if any
   346  // signal received.
   347  func UnindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}) {
   348  	unindexTransactions(db, from, to, interrupt, nil)
   349  }
   350  
   351  // unindexTransactionsForTesting is the internal debug version with an additional hook.
   352  func unindexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) {
   353  	unindexTransactions(db, from, to, interrupt, hook)
   354  }