github.com/koko1123/flow-go-1@v0.29.6/engine/verification/fetcher/chunkconsumer/consumer.go (about)

     1  package chunkconsumer
     2  
     3  import (
     4  	"fmt"
     5  
     6  	"github.com/rs/zerolog"
     7  
     8  	"github.com/koko1123/flow-go-1/engine/verification/fetcher"
     9  	"github.com/koko1123/flow-go-1/module"
    10  	"github.com/koko1123/flow-go-1/module/jobqueue"
    11  	"github.com/koko1123/flow-go-1/storage"
    12  )
    13  
    14  const (
    15  	DefaultJobIndex     = uint64(0)
    16  	DefaultChunkWorkers = uint64(5)
    17  )
    18  
    19  // ChunkConsumer consumes the jobs from the job queue, and pass it to the
    20  // Worker for processing.
    21  // It wraps the generic job consumer in order to be used as a ReadyDoneAware
    22  // on startup
    23  type ChunkConsumer struct {
    24  	consumer       module.JobConsumer
    25  	chunkProcessor fetcher.AssignedChunkProcessor
    26  	metrics        module.VerificationMetrics
    27  }
    28  
    29  func NewChunkConsumer(
    30  	log zerolog.Logger,
    31  	metrics module.VerificationMetrics,
    32  	processedIndex storage.ConsumerProgress, // to persist the processed index
    33  	chunksQueue storage.ChunksQueue, // to read jobs (chunks) from
    34  	chunkProcessor fetcher.AssignedChunkProcessor, // to process jobs (chunks)
    35  	maxProcessing uint64, // max number of jobs to be processed in parallel
    36  ) *ChunkConsumer {
    37  	worker := NewWorker(chunkProcessor)
    38  	chunkProcessor.WithChunkConsumerNotifier(worker)
    39  
    40  	jobs := &ChunkJobs{locators: chunksQueue}
    41  
    42  	lg := log.With().Str("module", "chunk_consumer").Logger()
    43  	consumer := jobqueue.NewConsumer(lg, jobs, processedIndex, worker, maxProcessing, 0)
    44  
    45  	chunkConsumer := &ChunkConsumer{
    46  		consumer:       consumer,
    47  		chunkProcessor: chunkProcessor,
    48  		metrics:        metrics,
    49  	}
    50  
    51  	worker.consumer = chunkConsumer
    52  
    53  	return chunkConsumer
    54  }
    55  
    56  func (c *ChunkConsumer) NotifyJobIsDone(jobID module.JobID) {
    57  	processedIndex := c.consumer.NotifyJobIsDone(jobID)
    58  	c.metrics.OnChunkConsumerJobDone(processedIndex)
    59  }
    60  
    61  // Size returns number of in-memory chunk jobs that chunk consumer is processing.
    62  func (c *ChunkConsumer) Size() uint {
    63  	return c.consumer.Size()
    64  }
    65  
    66  func (c ChunkConsumer) Check() {
    67  	c.consumer.Check()
    68  }
    69  
    70  func (c *ChunkConsumer) Ready() <-chan struct{} {
    71  	err := c.consumer.Start(DefaultJobIndex)
    72  	if err != nil {
    73  		panic(fmt.Errorf("could not start the chunk consumer for match engine: %w", err))
    74  	}
    75  
    76  	return c.chunkProcessor.Ready()
    77  }
    78  
    79  func (c *ChunkConsumer) Done() <-chan struct{} {
    80  	c.consumer.Stop()
    81  
    82  	return c.chunkProcessor.Done()
    83  }