github.com/onflow/flow-go@v0.35.7-crescendo-preview.23-atree-inlining/module/jobqueue.go (about)

     1  package module
     2  
     3  import (
     4  	"github.com/onflow/flow-go/model/flow"
     5  )
     6  
     7  const (
     8  	ConsumeProgressVerificationBlockHeight = "ConsumeProgressVerificationBlockHeight"
     9  	ConsumeProgressVerificationChunkIndex  = "ConsumeProgressVerificationChunkIndex"
    10  
    11  	ConsumeProgressExecutionDataRequesterBlockHeight  = "ConsumeProgressExecutionDataRequesterBlockHeight"
    12  	ConsumeProgressExecutionDataRequesterNotification = "ConsumeProgressExecutionDataRequesterNotification"
    13  
    14  	ConsumeProgressExecutionDataIndexerBlockHeight = "ConsumeProgressExecutionDataIndexerBlockHeight"
    15  
    16  	ConsumeProgressIngestionEngineBlockHeight = "ConsumeProgressIngestionEngineBlockHeight"
    17  	ConsumeProgressLastFullBlockHeight        = "ConsumeProgressLastFullBlockHeight"
    18  )
    19  
    20  // JobID is a unique ID of the job.
    21  type JobID string
    22  
    23  type NewJobListener interface {
    24  	// Check let the producer notify the consumer that a new job has been added, so that the consumer
    25  	// can check if there is worker available to process that job.
    26  	Check()
    27  }
    28  
    29  // JobConsumer consumes jobs from a job queue, and it remembers which job it has processed, and is able
    30  // to resume processing from the next.
    31  type JobConsumer interface {
    32  	NewJobListener
    33  
    34  	// Start starts processing jobs from a job queue.
    35  	Start() error
    36  
    37  	// Stop gracefully stops the consumer from reading new jobs from the job queue. It does not stop
    38  	// the existing worker finishing their jobs
    39  	// It blocks until the existing worker finish processing the job
    40  	Stop()
    41  
    42  	// LastProcessedIndex returns the last processed job index
    43  	LastProcessedIndex() uint64
    44  
    45  	// NotifyJobIsDone let the consumer know a job has been finished, so that consumer will take
    46  	// the next job from the job queue if there are workers available. It returns the last processed job index.
    47  	NotifyJobIsDone(JobID) uint64
    48  
    49  	// Size returns the number of processing jobs in consumer.
    50  	Size() uint
    51  }
    52  
    53  type Job interface {
    54  	// each job has a unique ID for deduplication
    55  	ID() JobID
    56  }
    57  
    58  // Jobs is the reader for an ordered job queue. Job can be fetched by the index,
    59  // which start from 0
    60  type Jobs interface {
    61  	// AtIndex returns the job at the given index.
    62  	// Error returns:
    63  	//   * storage.ErrNotFound if a job at the provided index is not available
    64  	AtIndex(index uint64) (Job, error)
    65  
    66  	// Head returns the index of the last job
    67  	Head() (uint64, error)
    68  }
    69  
    70  type JobQueue interface {
    71  	// Add a job to the job queue
    72  	Add(job Job) error
    73  }
    74  
    75  // ProcessingNotifier is for the worker's underneath engine to report an entity
    76  // has been processed without knowing the job queue.
    77  // It is a callback so that the worker can convert the entity id into a job
    78  // id, and notify the consumer about a finished job.
    79  //
    80  // At the current version, entities used in this interface are chunks and blocks ids.
    81  type ProcessingNotifier interface {
    82  	Notify(entityID flow.Identifier)
    83  }