github.com/sequix/cortex@v1.1.6/pkg/ruler/worker.go (about)

     1  package ruler
     2  
     3  import (
     4  	"time"
     5  
     6  	"github.com/sequix/cortex/pkg/util"
     7  	"github.com/go-kit/kit/log/level"
     8  	"github.com/prometheus/client_golang/prometheus"
     9  )
    10  
    11  var (
    12  	blockedWorkers = prometheus.NewGauge(prometheus.GaugeOpts{
    13  		Namespace: "cortex",
    14  		Name:      "blocked_workers",
    15  		Help:      "How many workers are waiting on an item to be ready.",
    16  	})
    17  	workerIdleTime = prometheus.NewCounter(prometheus.CounterOpts{
    18  		Namespace: "cortex",
    19  		Name:      "worker_idle_seconds_total",
    20  		Help:      "How long workers have spent waiting for work.",
    21  	})
    22  	evalLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
    23  		Namespace: "cortex",
    24  		Name:      "group_evaluation_latency_seconds",
    25  		Help:      "How far behind the target time each rule group executed.",
    26  		Buckets:   []float64{.1, .25, .5, 1, 2.5, 5, 10, 25},
    27  	})
    28  )
    29  
    30  // Worker does a thing until it's told to stop.
    31  type Worker interface {
    32  	Run()
    33  	Stop()
    34  }
    35  
    36  type worker struct {
    37  	scheduler *scheduler
    38  	ruler     *Ruler
    39  }
    40  
    41  func newWorker(ruler *Ruler) worker {
    42  	return worker{
    43  		scheduler: ruler.scheduler,
    44  		ruler:     ruler,
    45  	}
    46  }
    47  
    48  func (w *worker) Run() {
    49  	for {
    50  		waitStart := time.Now()
    51  		blockedWorkers.Inc()
    52  		level.Debug(util.Logger).Log("msg", "waiting for next work item")
    53  		item := w.scheduler.nextWorkItem()
    54  		blockedWorkers.Dec()
    55  		waitElapsed := time.Now().Sub(waitStart)
    56  		if item == nil {
    57  			level.Debug(util.Logger).Log("msg", "queue closed and empty; terminating worker")
    58  			return
    59  		}
    60  		evalLatency.Observe(time.Since(item.scheduled).Seconds())
    61  		workerIdleTime.Add(waitElapsed.Seconds())
    62  		level.Debug(util.Logger).Log("msg", "processing item", "item", item)
    63  		w.ruler.Evaluate(item.userID, item)
    64  		w.scheduler.workItemDone(*item)
    65  		level.Debug(util.Logger).Log("msg", "item handed back to queue", "item", item)
    66  	}
    67  }