gitee.com/liu-zhao234568/cntest@v1.0.0/core/chain_indexer.go (about)

     1  // Copyright 2017 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package core
    18  
    19  import (
    20  	"context"
    21  	"encoding/binary"
    22  	"fmt"
    23  	"sync"
    24  	"sync/atomic"
    25  	"time"
    26  
    27  	"gitee.com/liu-zhao234568/cntest/common"
    28  	"gitee.com/liu-zhao234568/cntest/core/rawdb"
    29  	"gitee.com/liu-zhao234568/cntest/core/types"
    30  	"gitee.com/liu-zhao234568/cntest/ethdb"
    31  	"gitee.com/liu-zhao234568/cntest/event"
    32  	"gitee.com/liu-zhao234568/cntest/log"
    33  )
    34  
    35  // ChainIndexerBackend defines the methods needed to process chain segments in
    36  // the background and write the segment results into the database. These can be
    37  // used to create filter blooms or CHTs.
    38  type ChainIndexerBackend interface {
    39  	// Reset initiates the processing of a new chain segment, potentially terminating
    40  	// any partially completed operations (in case of a reorg).
    41  	Reset(ctx context.Context, section uint64, prevHead common.Hash) error
    42  
    43  	// Process crunches through the next header in the chain segment. The caller
    44  	// will ensure a sequential order of headers.
    45  	Process(ctx context.Context, header *types.Header) error
    46  
    47  	// Commit finalizes the section metadata and stores it into the database.
    48  	Commit() error
    49  
    50  	// Prune deletes the chain index older than the given threshold.
    51  	Prune(threshold uint64) error
    52  }
    53  
    54  // ChainIndexerChain interface is used for connecting the indexer to a blockchain
    55  type ChainIndexerChain interface {
    56  	// CurrentHeader retrieves the latest locally known header.
    57  	CurrentHeader() *types.Header
    58  
    59  	// SubscribeChainHeadEvent subscribes to new head header notifications.
    60  	SubscribeChainHeadEvent(ch chan<- ChainHeadEvent) event.Subscription
    61  }
    62  
    63  // ChainIndexer does a post-processing job for equally sized sections of the
    64  // canonical chain (like BlooomBits and CHT structures). A ChainIndexer is
    65  // connected to the blockchain through the event system by starting a
    66  // ChainHeadEventLoop in a goroutine.
    67  //
    68  // Further child ChainIndexers can be added which use the output of the parent
    69  // section indexer. These child indexers receive new head notifications only
    70  // after an entire section has been finished or in case of rollbacks that might
    71  // affect already finished sections.
    72  type ChainIndexer struct {
    73  	chainDb  ethdb.Database      // Chain database to index the data from
    74  	indexDb  ethdb.Database      // Prefixed table-view of the db to write index metadata into
    75  	backend  ChainIndexerBackend // Background processor generating the index data content
    76  	children []*ChainIndexer     // Child indexers to cascade chain updates to
    77  
    78  	active    uint32          // Flag whether the event loop was started
    79  	update    chan struct{}   // Notification channel that headers should be processed
    80  	quit      chan chan error // Quit channel to tear down running goroutines
    81  	ctx       context.Context
    82  	ctxCancel func()
    83  
    84  	sectionSize uint64 // Number of blocks in a single chain segment to process
    85  	confirmsReq uint64 // Number of confirmations before processing a completed segment
    86  
    87  	storedSections uint64 // Number of sections successfully indexed into the database
    88  	knownSections  uint64 // Number of sections known to be complete (block wise)
    89  	cascadedHead   uint64 // Block number of the last completed section cascaded to subindexers
    90  
    91  	checkpointSections uint64      // Number of sections covered by the checkpoint
    92  	checkpointHead     common.Hash // Section head belonging to the checkpoint
    93  
    94  	throttling time.Duration // Disk throttling to prevent a heavy upgrade from hogging resources
    95  
    96  	log  log.Logger
    97  	lock sync.Mutex
    98  }
    99  
   100  // NewChainIndexer creates a new chain indexer to do background processing on
   101  // chain segments of a given size after certain number of confirmations passed.
   102  // The throttling parameter might be used to prevent database thrashing.
   103  func NewChainIndexer(chainDb ethdb.Database, indexDb ethdb.Database, backend ChainIndexerBackend, section, confirm uint64, throttling time.Duration, kind string) *ChainIndexer {
   104  	c := &ChainIndexer{
   105  		chainDb:     chainDb,
   106  		indexDb:     indexDb,
   107  		backend:     backend,
   108  		update:      make(chan struct{}, 1),
   109  		quit:        make(chan chan error),
   110  		sectionSize: section,
   111  		confirmsReq: confirm,
   112  		throttling:  throttling,
   113  		log:         log.New("type", kind),
   114  	}
   115  	// Initialize database dependent fields and start the updater
   116  	c.loadValidSections()
   117  	c.ctx, c.ctxCancel = context.WithCancel(context.Background())
   118  
   119  	go c.updateLoop()
   120  
   121  	return c
   122  }
   123  
   124  // AddCheckpoint adds a checkpoint. Sections are never processed and the chain
   125  // is not expected to be available before this point. The indexer assumes that
   126  // the backend has sufficient information available to process subsequent sections.
   127  //
   128  // Note: knownSections == 0 and storedSections == checkpointSections until
   129  // syncing reaches the checkpoint
   130  func (c *ChainIndexer) AddCheckpoint(section uint64, shead common.Hash) {
   131  	c.lock.Lock()
   132  	defer c.lock.Unlock()
   133  
   134  	// Short circuit if the given checkpoint is below than local's.
   135  	if c.checkpointSections >= section+1 || section < c.storedSections {
   136  		return
   137  	}
   138  	c.checkpointSections = section + 1
   139  	c.checkpointHead = shead
   140  
   141  	c.setSectionHead(section, shead)
   142  	c.setValidSections(section + 1)
   143  }
   144  
   145  // Start creates a goroutine to feed chain head events into the indexer for
   146  // cascading background processing. Children do not need to be started, they
   147  // are notified about new events by their parents.
   148  func (c *ChainIndexer) Start(chain ChainIndexerChain) {
   149  	events := make(chan ChainHeadEvent, 10)
   150  	sub := chain.SubscribeChainHeadEvent(events)
   151  
   152  	go c.eventLoop(chain.CurrentHeader(), events, sub)
   153  }
   154  
   155  // Close tears down all goroutines belonging to the indexer and returns any error
   156  // that might have occurred internally.
   157  func (c *ChainIndexer) Close() error {
   158  	var errs []error
   159  
   160  	c.ctxCancel()
   161  
   162  	// Tear down the primary update loop
   163  	errc := make(chan error)
   164  	c.quit <- errc
   165  	if err := <-errc; err != nil {
   166  		errs = append(errs, err)
   167  	}
   168  	// If needed, tear down the secondary event loop
   169  	if atomic.LoadUint32(&c.active) != 0 {
   170  		c.quit <- errc
   171  		if err := <-errc; err != nil {
   172  			errs = append(errs, err)
   173  		}
   174  	}
   175  	// Close all children
   176  	for _, child := range c.children {
   177  		if err := child.Close(); err != nil {
   178  			errs = append(errs, err)
   179  		}
   180  	}
   181  	// Return any failures
   182  	switch {
   183  	case len(errs) == 0:
   184  		return nil
   185  
   186  	case len(errs) == 1:
   187  		return errs[0]
   188  
   189  	default:
   190  		return fmt.Errorf("%v", errs)
   191  	}
   192  }
   193  
   194  // eventLoop is a secondary - optional - event loop of the indexer which is only
   195  // started for the outermost indexer to push chain head events into a processing
   196  // queue.
   197  func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainHeadEvent, sub event.Subscription) {
   198  	// Mark the chain indexer as active, requiring an additional teardown
   199  	atomic.StoreUint32(&c.active, 1)
   200  
   201  	defer sub.Unsubscribe()
   202  
   203  	// Fire the initial new head event to start any outstanding processing
   204  	c.newHead(currentHeader.Number.Uint64(), false)
   205  
   206  	var (
   207  		prevHeader = currentHeader
   208  		prevHash   = currentHeader.Hash()
   209  	)
   210  	for {
   211  		select {
   212  		case errc := <-c.quit:
   213  			// Chain indexer terminating, report no failure and abort
   214  			errc <- nil
   215  			return
   216  
   217  		case ev, ok := <-events:
   218  			// Received a new event, ensure it's not nil (closing) and update
   219  			if !ok {
   220  				errc := <-c.quit
   221  				errc <- nil
   222  				return
   223  			}
   224  			header := ev.Block.Header()
   225  			if header.ParentHash != prevHash {
   226  				// Reorg to the common ancestor if needed (might not exist in light sync mode, skip reorg then)
   227  				// TODO(karalabe, zsfelfoldi): This seems a bit brittle, can we detect this case explicitly?
   228  
   229  				if rawdb.ReadCanonicalHash(c.chainDb, prevHeader.Number.Uint64()) != prevHash {
   230  					if h := rawdb.FindCommonAncestor(c.chainDb, prevHeader, header); h != nil {
   231  						c.newHead(h.Number.Uint64(), true)
   232  					}
   233  				}
   234  			}
   235  			c.newHead(header.Number.Uint64(), false)
   236  
   237  			prevHeader, prevHash = header, header.Hash()
   238  		}
   239  	}
   240  }
   241  
   242  // newHead notifies the indexer about new chain heads and/or reorgs.
   243  func (c *ChainIndexer) newHead(head uint64, reorg bool) {
   244  	c.lock.Lock()
   245  	defer c.lock.Unlock()
   246  
   247  	// If a reorg happened, invalidate all sections until that point
   248  	if reorg {
   249  		// Revert the known section number to the reorg point
   250  		known := (head + 1) / c.sectionSize
   251  		stored := known
   252  		if known < c.checkpointSections {
   253  			known = 0
   254  		}
   255  		if stored < c.checkpointSections {
   256  			stored = c.checkpointSections
   257  		}
   258  		if known < c.knownSections {
   259  			c.knownSections = known
   260  		}
   261  		// Revert the stored sections from the database to the reorg point
   262  		if stored < c.storedSections {
   263  			c.setValidSections(stored)
   264  		}
   265  		// Update the new head number to the finalized section end and notify children
   266  		head = known * c.sectionSize
   267  
   268  		if head < c.cascadedHead {
   269  			c.cascadedHead = head
   270  			for _, child := range c.children {
   271  				child.newHead(c.cascadedHead, true)
   272  			}
   273  		}
   274  		return
   275  	}
   276  	// No reorg, calculate the number of newly known sections and update if high enough
   277  	var sections uint64
   278  	if head >= c.confirmsReq {
   279  		sections = (head + 1 - c.confirmsReq) / c.sectionSize
   280  		if sections < c.checkpointSections {
   281  			sections = 0
   282  		}
   283  		if sections > c.knownSections {
   284  			if c.knownSections < c.checkpointSections {
   285  				// syncing reached the checkpoint, verify section head
   286  				syncedHead := rawdb.ReadCanonicalHash(c.chainDb, c.checkpointSections*c.sectionSize-1)
   287  				if syncedHead != c.checkpointHead {
   288  					c.log.Error("Synced chain does not match checkpoint", "number", c.checkpointSections*c.sectionSize-1, "expected", c.checkpointHead, "synced", syncedHead)
   289  					return
   290  				}
   291  			}
   292  			c.knownSections = sections
   293  
   294  			select {
   295  			case c.update <- struct{}{}:
   296  			default:
   297  			}
   298  		}
   299  	}
   300  }
   301  
   302  // updateLoop is the main event loop of the indexer which pushes chain segments
   303  // down into the processing backend.
   304  func (c *ChainIndexer) updateLoop() {
   305  	var (
   306  		updating bool
   307  		updated  time.Time
   308  	)
   309  
   310  	for {
   311  		select {
   312  		case errc := <-c.quit:
   313  			// Chain indexer terminating, report no failure and abort
   314  			errc <- nil
   315  			return
   316  
   317  		case <-c.update:
   318  			// Section headers completed (or rolled back), update the index
   319  			c.lock.Lock()
   320  			if c.knownSections > c.storedSections {
   321  				// Periodically print an upgrade log message to the user
   322  				if time.Since(updated) > 8*time.Second {
   323  					if c.knownSections > c.storedSections+1 {
   324  						updating = true
   325  						c.log.Info("Upgrading chain index", "percentage", c.storedSections*100/c.knownSections)
   326  					}
   327  					updated = time.Now()
   328  				}
   329  				// Cache the current section count and head to allow unlocking the mutex
   330  				c.verifyLastHead()
   331  				section := c.storedSections
   332  				var oldHead common.Hash
   333  				if section > 0 {
   334  					oldHead = c.SectionHead(section - 1)
   335  				}
   336  				// Process the newly defined section in the background
   337  				c.lock.Unlock()
   338  				newHead, err := c.processSection(section, oldHead)
   339  				if err != nil {
   340  					select {
   341  					case <-c.ctx.Done():
   342  						<-c.quit <- nil
   343  						return
   344  					default:
   345  					}
   346  					c.log.Error("Section processing failed", "error", err)
   347  				}
   348  				c.lock.Lock()
   349  
   350  				// If processing succeeded and no reorgs occurred, mark the section completed
   351  				if err == nil && (section == 0 || oldHead == c.SectionHead(section-1)) {
   352  					c.setSectionHead(section, newHead)
   353  					c.setValidSections(section + 1)
   354  					if c.storedSections == c.knownSections && updating {
   355  						updating = false
   356  						c.log.Info("Finished upgrading chain index")
   357  					}
   358  					c.cascadedHead = c.storedSections*c.sectionSize - 1
   359  					for _, child := range c.children {
   360  						c.log.Trace("Cascading chain index update", "head", c.cascadedHead)
   361  						child.newHead(c.cascadedHead, false)
   362  					}
   363  				} else {
   364  					// If processing failed, don't retry until further notification
   365  					c.log.Debug("Chain index processing failed", "section", section, "err", err)
   366  					c.verifyLastHead()
   367  					c.knownSections = c.storedSections
   368  				}
   369  			}
   370  			// If there are still further sections to process, reschedule
   371  			if c.knownSections > c.storedSections {
   372  				time.AfterFunc(c.throttling, func() {
   373  					select {
   374  					case c.update <- struct{}{}:
   375  					default:
   376  					}
   377  				})
   378  			}
   379  			c.lock.Unlock()
   380  		}
   381  	}
   382  }
   383  
   384  // processSection processes an entire section by calling backend functions while
   385  // ensuring the continuity of the passed headers. Since the chain mutex is not
   386  // held while processing, the continuity can be broken by a long reorg, in which
   387  // case the function returns with an error.
   388  func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (common.Hash, error) {
   389  	c.log.Trace("Processing new chain section", "section", section)
   390  
   391  	// Reset and partial processing
   392  	if err := c.backend.Reset(c.ctx, section, lastHead); err != nil {
   393  		c.setValidSections(0)
   394  		return common.Hash{}, err
   395  	}
   396  
   397  	for number := section * c.sectionSize; number < (section+1)*c.sectionSize; number++ {
   398  		hash := rawdb.ReadCanonicalHash(c.chainDb, number)
   399  		if hash == (common.Hash{}) {
   400  			return common.Hash{}, fmt.Errorf("canonical block #%d unknown", number)
   401  		}
   402  		header := rawdb.ReadHeader(c.chainDb, hash, number)
   403  		if header == nil {
   404  			return common.Hash{}, fmt.Errorf("block #%d [%x..] not found", number, hash[:4])
   405  		} else if header.ParentHash != lastHead {
   406  			return common.Hash{}, fmt.Errorf("chain reorged during section processing")
   407  		}
   408  		if err := c.backend.Process(c.ctx, header); err != nil {
   409  			return common.Hash{}, err
   410  		}
   411  		lastHead = header.Hash()
   412  	}
   413  	if err := c.backend.Commit(); err != nil {
   414  		return common.Hash{}, err
   415  	}
   416  	return lastHead, nil
   417  }
   418  
   419  // verifyLastHead compares last stored section head with the corresponding block hash in the
   420  // actual canonical chain and rolls back reorged sections if necessary to ensure that stored
   421  // sections are all valid
   422  func (c *ChainIndexer) verifyLastHead() {
   423  	for c.storedSections > 0 && c.storedSections > c.checkpointSections {
   424  		if c.SectionHead(c.storedSections-1) == rawdb.ReadCanonicalHash(c.chainDb, c.storedSections*c.sectionSize-1) {
   425  			return
   426  		}
   427  		c.setValidSections(c.storedSections - 1)
   428  	}
   429  }
   430  
   431  // Sections returns the number of processed sections maintained by the indexer
   432  // and also the information about the last header indexed for potential canonical
   433  // verifications.
   434  func (c *ChainIndexer) Sections() (uint64, uint64, common.Hash) {
   435  	c.lock.Lock()
   436  	defer c.lock.Unlock()
   437  
   438  	c.verifyLastHead()
   439  	return c.storedSections, c.storedSections*c.sectionSize - 1, c.SectionHead(c.storedSections - 1)
   440  }
   441  
   442  // AddChildIndexer adds a child ChainIndexer that can use the output of this one
   443  func (c *ChainIndexer) AddChildIndexer(indexer *ChainIndexer) {
   444  	if indexer == c {
   445  		panic("can't add indexer as a child of itself")
   446  	}
   447  	c.lock.Lock()
   448  	defer c.lock.Unlock()
   449  
   450  	c.children = append(c.children, indexer)
   451  
   452  	// Cascade any pending updates to new children too
   453  	sections := c.storedSections
   454  	if c.knownSections < sections {
   455  		// if a section is "stored" but not "known" then it is a checkpoint without
   456  		// available chain data so we should not cascade it yet
   457  		sections = c.knownSections
   458  	}
   459  	if sections > 0 {
   460  		indexer.newHead(sections*c.sectionSize-1, false)
   461  	}
   462  }
   463  
   464  // Prune deletes all chain data older than given threshold.
   465  func (c *ChainIndexer) Prune(threshold uint64) error {
   466  	return c.backend.Prune(threshold)
   467  }
   468  
   469  // loadValidSections reads the number of valid sections from the index database
   470  // and caches is into the local state.
   471  func (c *ChainIndexer) loadValidSections() {
   472  	data, _ := c.indexDb.Get([]byte("count"))
   473  	if len(data) == 8 {
   474  		c.storedSections = binary.BigEndian.Uint64(data)
   475  	}
   476  }
   477  
   478  // setValidSections writes the number of valid sections to the index database
   479  func (c *ChainIndexer) setValidSections(sections uint64) {
   480  	// Set the current number of valid sections in the database
   481  	var data [8]byte
   482  	binary.BigEndian.PutUint64(data[:], sections)
   483  	c.indexDb.Put([]byte("count"), data[:])
   484  
   485  	// Remove any reorged sections, caching the valids in the mean time
   486  	for c.storedSections > sections {
   487  		c.storedSections--
   488  		c.removeSectionHead(c.storedSections)
   489  	}
   490  	c.storedSections = sections // needed if new > old
   491  }
   492  
   493  // SectionHead retrieves the last block hash of a processed section from the
   494  // index database.
   495  func (c *ChainIndexer) SectionHead(section uint64) common.Hash {
   496  	var data [8]byte
   497  	binary.BigEndian.PutUint64(data[:], section)
   498  
   499  	hash, _ := c.indexDb.Get(append([]byte("shead"), data[:]...))
   500  	if len(hash) == len(common.Hash{}) {
   501  		return common.BytesToHash(hash)
   502  	}
   503  	return common.Hash{}
   504  }
   505  
   506  // setSectionHead writes the last block hash of a processed section to the index
   507  // database.
   508  func (c *ChainIndexer) setSectionHead(section uint64, hash common.Hash) {
   509  	var data [8]byte
   510  	binary.BigEndian.PutUint64(data[:], section)
   511  
   512  	c.indexDb.Put(append([]byte("shead"), data[:]...), hash.Bytes())
   513  }
   514  
   515  // removeSectionHead removes the reference to a processed section from the index
   516  // database.
   517  func (c *ChainIndexer) removeSectionHead(section uint64) {
   518  	var data [8]byte
   519  	binary.BigEndian.PutUint64(data[:], section)
   520  
   521  	c.indexDb.Delete(append([]byte("shead"), data[:]...))
   522  }