github.com/onflow/flow-go@v0.33.17/engine/verification/assigner/blockconsumer/consumer.go (about)

     1  package blockconsumer
     2  
     3  import (
     4  	"fmt"
     5  
     6  	"github.com/rs/zerolog"
     7  
     8  	"github.com/onflow/flow-go/consensus/hotstuff/model"
     9  	"github.com/onflow/flow-go/engine"
    10  	"github.com/onflow/flow-go/engine/verification/assigner"
    11  	"github.com/onflow/flow-go/module"
    12  	"github.com/onflow/flow-go/module/jobqueue"
    13  	"github.com/onflow/flow-go/state/protocol"
    14  	"github.com/onflow/flow-go/storage"
    15  )
    16  
    17  // DefaultBlockWorkers is the number of blocks processed in parallel.
    18  const DefaultBlockWorkers = uint64(2)
    19  
    20  // BlockConsumer listens to the OnFinalizedBlock event
    21  // and notifies the consumer to check in the job queue
    22  // (i.e., its block reader) for new block jobs.
    23  type BlockConsumer struct {
    24  	consumer module.JobConsumer
    25  	unit     *engine.Unit
    26  	metrics  module.VerificationMetrics
    27  }
    28  
    29  // defaultProcessedIndex returns the last sealed block height from the protocol state.
    30  //
    31  // The BlockConsumer utilizes this return height to fetch and consume block jobs from
    32  // jobs queue the first time it initializes.
    33  func defaultProcessedIndex(state protocol.State) (uint64, error) {
    34  	final, err := state.Sealed().Head()
    35  	if err != nil {
    36  		return 0, fmt.Errorf("could not get finalized height: %w", err)
    37  	}
    38  	return final.Height, nil
    39  }
    40  
    41  // NewBlockConsumer creates a new consumer and returns the default processed
    42  // index for initializing the processed index in storage.
    43  func NewBlockConsumer(log zerolog.Logger,
    44  	metrics module.VerificationMetrics,
    45  	processedHeight storage.ConsumerProgress,
    46  	blocks storage.Blocks,
    47  	state protocol.State,
    48  	blockProcessor assigner.FinalizedBlockProcessor,
    49  	maxProcessing uint64) (*BlockConsumer, uint64, error) {
    50  
    51  	lg := log.With().Str("module", "block_consumer").Logger()
    52  
    53  	// wires blockProcessor as the worker. The block consumer will
    54  	// invoke instances of worker concurrently to process block jobs.
    55  	worker := newWorker(blockProcessor)
    56  	blockProcessor.WithBlockConsumerNotifier(worker)
    57  
    58  	// the block reader is where the consumer reads new finalized blocks from (i.e., jobs).
    59  	jobs := jobqueue.NewFinalizedBlockReader(state, blocks)
    60  
    61  	defaultIndex, err := defaultProcessedIndex(state)
    62  	if err != nil {
    63  		return nil, 0, fmt.Errorf("could not read default processed index: %w", err)
    64  	}
    65  
    66  	consumer, err := jobqueue.NewConsumer(lg, jobs, processedHeight, worker, maxProcessing, 0, defaultIndex)
    67  	if err != nil {
    68  		return nil, 0, fmt.Errorf("could not create block consumer: %w", err)
    69  	}
    70  
    71  	blockConsumer := &BlockConsumer{
    72  		consumer: consumer,
    73  		unit:     engine.NewUnit(),
    74  		metrics:  metrics,
    75  	}
    76  	worker.withBlockConsumer(blockConsumer)
    77  
    78  	return blockConsumer, defaultIndex, nil
    79  }
    80  
    81  // NotifyJobIsDone is invoked by the worker to let the consumer know that it is done
    82  // processing a (block) job.
    83  func (c *BlockConsumer) NotifyJobIsDone(jobID module.JobID) {
    84  	processedIndex := c.consumer.NotifyJobIsDone(jobID)
    85  	c.metrics.OnBlockConsumerJobDone(processedIndex)
    86  }
    87  
    88  // Size returns number of in-memory block jobs that block consumer is processing.
    89  func (c *BlockConsumer) Size() uint {
    90  	return c.consumer.Size()
    91  }
    92  
    93  // OnFinalizedBlock implements FinalizationConsumer, and is invoked by the follower engine whenever
    94  // a new block is finalized.
    95  // In this implementation for block consumer, invoking OnFinalizedBlock is enough to only notify the consumer
    96  // to check its internal queue and move its processing index ahead to the next height if there are workers available.
    97  // The consumer retrieves the new blocks from its block reader module, hence it does not need to use the parameter
    98  // of OnFinalizedBlock here.
    99  func (c *BlockConsumer) OnFinalizedBlock(*model.Block) {
   100  	c.unit.Launch(c.consumer.Check)
   101  }
   102  
   103  func (c *BlockConsumer) Ready() <-chan struct{} {
   104  	err := c.consumer.Start()
   105  	if err != nil {
   106  		panic(fmt.Errorf("could not start block consumer for finder engine: %w", err))
   107  	}
   108  
   109  	ready := make(chan struct{})
   110  	close(ready)
   111  	return ready
   112  }
   113  
   114  func (c *BlockConsumer) Done() <-chan struct{} {
   115  	ready := make(chan struct{})
   116  	go func() {
   117  		completeChan := c.unit.Done()
   118  		c.consumer.Stop()
   119  		<-completeChan
   120  		close(ready)
   121  	}()
   122  	return ready
   123  }