github.com/fff-chain/go-fff@v0.0.0-20220726032732-1c84420b8a99/core/rawdb/chain_iterator.go (about)

     1  // Copyright 2019 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package rawdb
    18  
    19  import (
    20  	"runtime"
    21  	"sync/atomic"
    22  	"time"
    23  
    24  	"github.com/fff-chain/go-fff/common"
    25  	"github.com/fff-chain/go-fff/common/gopool"
    26  	"github.com/fff-chain/go-fff/common/prque"
    27  	"github.com/fff-chain/go-fff/core/types"
    28  	"github.com/fff-chain/go-fff/ethdb"
    29  	"github.com/fff-chain/go-fff/log"
    30  	"github.com/fff-chain/go-fff/rlp"
    31  )
    32  
    33  // InitDatabaseFromFreezer reinitializes an empty database from a previous batch
    34  // of frozen ancient blocks. The method iterates over all the frozen blocks and
    35  // injects into the database the block hash->number mappings.
    36  func InitDatabaseFromFreezer(db ethdb.Database) {
    37  	// If we can't access the freezer or it's empty, abort
    38  	frozen, err := db.ItemAmountInAncient()
    39  	if err != nil || frozen == 0 {
    40  		return
    41  	}
    42  	var (
    43  		batch  = db.NewBatch()
    44  		start  = time.Now()
    45  		logged = start.Add(-7 * time.Second) // Unindex during import is fast, don't double log
    46  		hash   common.Hash
    47  		offset = db.AncientOffSet()
    48  	)
    49  	for i := uint64(0) + offset; i < frozen+offset; i++ {
    50  		// Since the freezer has all data in sequential order on a file,
    51  		// it would be 'neat' to read more data in one go, and let the
    52  		// freezerdb return N items (e.g up to 1000 items per go)
    53  		// That would require an API change in Ancients though
    54  		if h, err := db.Ancient(freezerHashTable, i); err != nil {
    55  			log.Crit("Failed to init database from freezer", "err", err)
    56  		} else {
    57  			hash = common.BytesToHash(h)
    58  		}
    59  		WriteHeaderNumber(batch, hash, i)
    60  		// If enough data was accumulated in memory or we're at the last block, dump to disk
    61  		if batch.ValueSize() > ethdb.IdealBatchSize {
    62  			if err := batch.Write(); err != nil {
    63  				log.Crit("Failed to write data to db", "err", err)
    64  			}
    65  			batch.Reset()
    66  		}
    67  		// If we've spent too much time already, notify the user of what we're doing
    68  		if time.Since(logged) > 8*time.Second {
    69  			log.Info("Initializing database from freezer", "total", frozen, "number", i, "hash", hash, "elapsed", common.PrettyDuration(time.Since(start)))
    70  			logged = time.Now()
    71  		}
    72  	}
    73  	if err := batch.Write(); err != nil {
    74  		log.Crit("Failed to write data to db", "err", err)
    75  	}
    76  	batch.Reset()
    77  
    78  	WriteHeadHeaderHash(db, hash)
    79  	WriteHeadFastBlockHash(db, hash)
    80  	log.Info("Initialized database from freezer", "blocks", frozen, "elapsed", common.PrettyDuration(time.Since(start)))
    81  }
    82  
    83  type blockTxHashes struct {
    84  	number uint64
    85  	hashes []common.Hash
    86  }
    87  
    88  // iterateTransactions iterates over all transactions in the (canon) block
    89  // number(s) given, and yields the hashes on a channel. If there is a signal
    90  // received from interrupt channel, the iteration will be aborted and result
    91  // channel will be closed.
    92  func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool, interrupt chan struct{}) chan *blockTxHashes {
    93  	// One thread sequentially reads data from db
    94  	type numberRlp struct {
    95  		number uint64
    96  		rlp    rlp.RawValue
    97  	}
    98  	if to == from {
    99  		return nil
   100  	}
   101  	threads := to - from
   102  	if cpus := runtime.NumCPU(); threads > uint64(cpus) {
   103  		threads = uint64(cpus)
   104  	}
   105  	var (
   106  		rlpCh    = make(chan *numberRlp, threads*2)     // we send raw rlp over this channel
   107  		hashesCh = make(chan *blockTxHashes, threads*2) // send hashes over hashesCh
   108  	)
   109  	// lookup runs in one instance
   110  	lookup := func() {
   111  		n, end := from, to
   112  		if reverse {
   113  			n, end = to-1, from-1
   114  		}
   115  		defer close(rlpCh)
   116  		for n != end {
   117  			data := ReadCanonicalBodyRLP(db, n)
   118  			// Feed the block to the aggregator, or abort on interrupt
   119  			select {
   120  			case rlpCh <- &numberRlp{n, data}:
   121  			case <-interrupt:
   122  				return
   123  			}
   124  			if reverse {
   125  				n--
   126  			} else {
   127  				n++
   128  			}
   129  		}
   130  	}
   131  	// process runs in parallel
   132  	nThreadsAlive := int32(threads)
   133  	process := func() {
   134  		defer func() {
   135  			// Last processor closes the result channel
   136  			if atomic.AddInt32(&nThreadsAlive, -1) == 0 {
   137  				close(hashesCh)
   138  			}
   139  		}()
   140  		for data := range rlpCh {
   141  			var body types.Body
   142  			if err := rlp.DecodeBytes(data.rlp, &body); err != nil {
   143  				log.Warn("Failed to decode block body", "block", data.number, "error", err)
   144  				return
   145  			}
   146  			var hashes []common.Hash
   147  			for _, tx := range body.Transactions {
   148  				hashes = append(hashes, tx.Hash())
   149  			}
   150  			result := &blockTxHashes{
   151  				hashes: hashes,
   152  				number: data.number,
   153  			}
   154  			// Feed the block to the aggregator, or abort on interrupt
   155  			select {
   156  			case hashesCh <- result:
   157  			case <-interrupt:
   158  				return
   159  			}
   160  		}
   161  	}
   162  	go lookup() // start the sequential db accessor
   163  	for i := 0; i < int(threads); i++ {
   164  		gopool.Submit(func() {
   165  			process()
   166  		})
   167  	}
   168  	return hashesCh
   169  }
   170  
   171  // indexTransactions creates txlookup indices of the specified block range.
   172  //
   173  // This function iterates canonical chain in reverse order, it has one main advantage:
   174  // We can write tx index tail flag periodically even without the whole indexing
   175  // procedure is finished. So that we can resume indexing procedure next time quickly.
   176  //
   177  // There is a passed channel, the whole procedure will be interrupted if any
   178  // signal received.
   179  func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) {
   180  	// short circuit for invalid range
   181  	if from >= to {
   182  		return
   183  	}
   184  	var (
   185  		hashesCh = iterateTransactions(db, from, to, true, interrupt)
   186  		batch    = db.NewBatch()
   187  		start    = time.Now()
   188  		logged   = start.Add(-7 * time.Second)
   189  		// Since we iterate in reverse, we expect the first number to come
   190  		// in to be [to-1]. Therefore, setting lastNum to means that the
   191  		// prqueue gap-evaluation will work correctly
   192  		lastNum = to
   193  		queue   = prque.New(nil)
   194  		// for stats reporting
   195  		blocks, txs = 0, 0
   196  	)
   197  	for chanDelivery := range hashesCh {
   198  		// Push the delivery into the queue and process contiguous ranges.
   199  		// Since we iterate in reverse, so lower numbers have lower prio, and
   200  		// we can use the number directly as prio marker
   201  		queue.Push(chanDelivery, int64(chanDelivery.number))
   202  		for !queue.Empty() {
   203  			// If the next available item is gapped, return
   204  			if _, priority := queue.Peek(); priority != int64(lastNum-1) {
   205  				break
   206  			}
   207  			// For testing
   208  			if hook != nil && !hook(lastNum-1) {
   209  				break
   210  			}
   211  			// Next block available, pop it off and index it
   212  			delivery := queue.PopItem().(*blockTxHashes)
   213  			lastNum = delivery.number
   214  			WriteTxLookupEntries(batch, delivery.number, delivery.hashes)
   215  			blocks++
   216  			txs += len(delivery.hashes)
   217  			// If enough data was accumulated in memory or we're at the last block, dump to disk
   218  			if batch.ValueSize() > ethdb.IdealBatchSize {
   219  				WriteTxIndexTail(batch, lastNum) // Also write the tail here
   220  				if err := batch.Write(); err != nil {
   221  					log.Crit("Failed writing batch to db", "error", err)
   222  					return
   223  				}
   224  				batch.Reset()
   225  			}
   226  			// If we've spent too much time already, notify the user of what we're doing
   227  			if time.Since(logged) > 8*time.Second {
   228  				log.Info("Indexing transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "total", to-from, "elapsed", common.PrettyDuration(time.Since(start)))
   229  				logged = time.Now()
   230  			}
   231  		}
   232  	}
   233  	// Flush the new indexing tail and the last committed data. It can also happen
   234  	// that the last batch is empty because nothing to index, but the tail has to
   235  	// be flushed anyway.
   236  	WriteTxIndexTail(batch, lastNum)
   237  	if err := batch.Write(); err != nil {
   238  		log.Crit("Failed writing batch to db", "error", err)
   239  		return
   240  	}
   241  	select {
   242  	case <-interrupt:
   243  		log.Debug("Transaction indexing interrupted", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start)))
   244  	default:
   245  		log.Info("Indexed transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start)))
   246  	}
   247  }
   248  
   249  // IndexTransactions creates txlookup indices of the specified block range.
   250  //
   251  // This function iterates canonical chain in reverse order, it has one main advantage:
   252  // We can write tx index tail flag periodically even without the whole indexing
   253  // procedure is finished. So that we can resume indexing procedure next time quickly.
   254  //
   255  // There is a passed channel, the whole procedure will be interrupted if any
   256  // signal received.
   257  func IndexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}) {
   258  	indexTransactions(db, from, to, interrupt, nil)
   259  }
   260  
   261  // indexTransactionsForTesting is the internal debug version with an additional hook.
   262  func indexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) {
   263  	indexTransactions(db, from, to, interrupt, hook)
   264  }
   265  
   266  // unindexTransactions removes txlookup indices of the specified block range.
   267  //
   268  // There is a passed channel, the whole procedure will be interrupted if any
   269  // signal received.
   270  func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) {
   271  	// short circuit for invalid range
   272  	if from >= to {
   273  		return
   274  	}
   275  	var (
   276  		hashesCh = iterateTransactions(db, from, to, false, interrupt)
   277  		batch    = db.NewBatch()
   278  		start    = time.Now()
   279  		logged   = start.Add(-7 * time.Second)
   280  		// we expect the first number to come in to be [from]. Therefore, setting
   281  		// nextNum to from means that the prqueue gap-evaluation will work correctly
   282  		nextNum = from
   283  		queue   = prque.New(nil)
   284  		// for stats reporting
   285  		blocks, txs = 0, 0
   286  	)
   287  	// Otherwise spin up the concurrent iterator and unindexer
   288  	for delivery := range hashesCh {
   289  		// Push the delivery into the queue and process contiguous ranges.
   290  		queue.Push(delivery, -int64(delivery.number))
   291  		for !queue.Empty() {
   292  			// If the next available item is gapped, return
   293  			if _, priority := queue.Peek(); -priority != int64(nextNum) {
   294  				break
   295  			}
   296  			// For testing
   297  			if hook != nil && !hook(nextNum) {
   298  				break
   299  			}
   300  			delivery := queue.PopItem().(*blockTxHashes)
   301  			nextNum = delivery.number + 1
   302  			DeleteTxLookupEntries(batch, delivery.hashes)
   303  			txs += len(delivery.hashes)
   304  			blocks++
   305  
   306  			// If enough data was accumulated in memory or we're at the last block, dump to disk
   307  			// A batch counts the size of deletion as '1', so we need to flush more
   308  			// often than that.
   309  			if blocks%1000 == 0 {
   310  				WriteTxIndexTail(batch, nextNum)
   311  				if err := batch.Write(); err != nil {
   312  					log.Crit("Failed writing batch to db", "error", err)
   313  					return
   314  				}
   315  				batch.Reset()
   316  			}
   317  			// If we've spent too much time already, notify the user of what we're doing
   318  			if time.Since(logged) > 8*time.Second {
   319  				log.Info("Unindexing transactions", "blocks", blocks, "txs", txs, "total", to-from, "elapsed", common.PrettyDuration(time.Since(start)))
   320  				logged = time.Now()
   321  			}
   322  		}
   323  	}
   324  	// Flush the new indexing tail and the last committed data. It can also happen
   325  	// that the last batch is empty because nothing to unindex, but the tail has to
   326  	// be flushed anyway.
   327  	WriteTxIndexTail(batch, nextNum)
   328  	if err := batch.Write(); err != nil {
   329  		log.Crit("Failed writing batch to db", "error", err)
   330  		return
   331  	}
   332  	select {
   333  	case <-interrupt:
   334  		log.Debug("Transaction unindexing interrupted", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start)))
   335  	default:
   336  		log.Info("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start)))
   337  	}
   338  }
   339  
   340  // UnindexTransactions removes txlookup indices of the specified block range.
   341  //
   342  // There is a passed channel, the whole procedure will be interrupted if any
   343  // signal received.
   344  func UnindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}) {
   345  	unindexTransactions(db, from, to, interrupt, nil)
   346  }
   347  
   348  // unindexTransactionsForTesting is the internal debug version with an additional hook.
   349  func unindexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) {
   350  	unindexTransactions(db, from, to, interrupt, hook)
   351  }