github.com/sequix/cortex@v1.1.6/pkg/ruler/worker.go (about) 1 package ruler 2 3 import ( 4 "time" 5 6 "github.com/sequix/cortex/pkg/util" 7 "github.com/go-kit/kit/log/level" 8 "github.com/prometheus/client_golang/prometheus" 9 ) 10 11 var ( 12 blockedWorkers = prometheus.NewGauge(prometheus.GaugeOpts{ 13 Namespace: "cortex", 14 Name: "blocked_workers", 15 Help: "How many workers are waiting on an item to be ready.", 16 }) 17 workerIdleTime = prometheus.NewCounter(prometheus.CounterOpts{ 18 Namespace: "cortex", 19 Name: "worker_idle_seconds_total", 20 Help: "How long workers have spent waiting for work.", 21 }) 22 evalLatency = prometheus.NewHistogram(prometheus.HistogramOpts{ 23 Namespace: "cortex", 24 Name: "group_evaluation_latency_seconds", 25 Help: "How far behind the target time each rule group executed.", 26 Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 10, 25}, 27 }) 28 ) 29 30 // Worker does a thing until it's told to stop. 31 type Worker interface { 32 Run() 33 Stop() 34 } 35 36 type worker struct { 37 scheduler *scheduler 38 ruler *Ruler 39 } 40 41 func newWorker(ruler *Ruler) worker { 42 return worker{ 43 scheduler: ruler.scheduler, 44 ruler: ruler, 45 } 46 } 47 48 func (w *worker) Run() { 49 for { 50 waitStart := time.Now() 51 blockedWorkers.Inc() 52 level.Debug(util.Logger).Log("msg", "waiting for next work item") 53 item := w.scheduler.nextWorkItem() 54 blockedWorkers.Dec() 55 waitElapsed := time.Now().Sub(waitStart) 56 if item == nil { 57 level.Debug(util.Logger).Log("msg", "queue closed and empty; terminating worker") 58 return 59 } 60 evalLatency.Observe(time.Since(item.scheduled).Seconds()) 61 workerIdleTime.Add(waitElapsed.Seconds()) 62 level.Debug(util.Logger).Log("msg", "processing item", "item", item) 63 w.ruler.Evaluate(item.userID, item) 64 w.scheduler.workItemDone(*item) 65 level.Debug(util.Logger).Log("msg", "item handed back to queue", "item", item) 66 } 67 }