github.com/onflow/flow-go@v0.35.7-crescendo-preview.23-atree-inlining/engine/verification/fetcher/chunkconsumer/consumer.go (about)

     1  package chunkconsumer
     2  
     3  import (
     4  	"fmt"
     5  
     6  	"github.com/rs/zerolog"
     7  
     8  	"github.com/onflow/flow-go/engine/verification/fetcher"
     9  	"github.com/onflow/flow-go/module"
    10  	"github.com/onflow/flow-go/module/jobqueue"
    11  	"github.com/onflow/flow-go/storage"
    12  )
    13  
    14  const (
    15  	DefaultJobIndex     = uint64(0)
    16  	DefaultChunkWorkers = uint64(5)
    17  )
    18  
    19  // ChunkConsumer consumes the jobs from the job queue, and pass it to the
    20  // Worker for processing.
    21  // It wraps the generic job consumer in order to be used as a ReadyDoneAware
    22  // on startup
    23  type ChunkConsumer struct {
    24  	consumer       module.JobConsumer
    25  	chunkProcessor fetcher.AssignedChunkProcessor
    26  	metrics        module.VerificationMetrics
    27  }
    28  
    29  func NewChunkConsumer(
    30  	log zerolog.Logger,
    31  	metrics module.VerificationMetrics,
    32  	processedIndex storage.ConsumerProgress, // to persist the processed index
    33  	chunksQueue storage.ChunksQueue, // to read jobs (chunks) from
    34  	chunkProcessor fetcher.AssignedChunkProcessor, // to process jobs (chunks)
    35  	maxProcessing uint64, // max number of jobs to be processed in parallel
    36  ) (*ChunkConsumer, error) {
    37  	worker := NewWorker(chunkProcessor)
    38  	chunkProcessor.WithChunkConsumerNotifier(worker)
    39  
    40  	jobs := &ChunkJobs{locators: chunksQueue}
    41  
    42  	lg := log.With().Str("module", "chunk_consumer").Logger()
    43  	consumer, err := jobqueue.NewConsumer(lg, jobs, processedIndex, worker, maxProcessing, 0, DefaultJobIndex)
    44  	if err != nil {
    45  		return nil, err
    46  	}
    47  
    48  	chunkConsumer := &ChunkConsumer{
    49  		consumer:       consumer,
    50  		chunkProcessor: chunkProcessor,
    51  		metrics:        metrics,
    52  	}
    53  
    54  	worker.consumer = chunkConsumer
    55  
    56  	return chunkConsumer, nil
    57  }
    58  
    59  func (c *ChunkConsumer) NotifyJobIsDone(jobID module.JobID) {
    60  	processedIndex := c.consumer.NotifyJobIsDone(jobID)
    61  	c.metrics.OnChunkConsumerJobDone(processedIndex)
    62  }
    63  
    64  // Size returns number of in-memory chunk jobs that chunk consumer is processing.
    65  func (c *ChunkConsumer) Size() uint {
    66  	return c.consumer.Size()
    67  }
    68  
    69  func (c ChunkConsumer) Check() {
    70  	c.consumer.Check()
    71  }
    72  
    73  func (c *ChunkConsumer) Ready() <-chan struct{} {
    74  	err := c.consumer.Start()
    75  	if err != nil {
    76  		panic(fmt.Errorf("could not start the chunk consumer for match engine: %w", err))
    77  	}
    78  
    79  	return c.chunkProcessor.Ready()
    80  }
    81  
    82  func (c *ChunkConsumer) Done() <-chan struct{} {
    83  	c.consumer.Stop()
    84  
    85  	return c.chunkProcessor.Done()
    86  }