github.com/grafana/pyroscope@v1.18.0/pkg/metastore/compaction/scheduler/metrics.go (about)

     1  package scheduler
     2  
     3  import (
     4  	"strconv"
     5  
     6  	"github.com/prometheus/client_golang/prometheus"
     7  )
     8  
     9  type statsCollector struct {
    10  	s *Scheduler
    11  
    12  	addedTotal      *prometheus.Desc
    13  	completedTotal  *prometheus.Desc
    14  	assignedTotal   *prometheus.Desc
    15  	reassignedTotal *prometheus.Desc
    16  	evictedTotal    *prometheus.Desc
    17  
    18  	// Gauge showing the job queue status breakdown.
    19  	jobs *prometheus.Desc
    20  }
    21  
    22  const schedulerQueueMetricsPrefix = "compaction_scheduler_queue_"
    23  
    24  func newStatsCollector(s *Scheduler) *statsCollector {
    25  	variableLabels := []string{"level"}
    26  	statusGaugeLabels := append(variableLabels, "status")
    27  	return &statsCollector{
    28  		s: s,
    29  
    30  		jobs: prometheus.NewDesc(
    31  			schedulerQueueMetricsPrefix+"jobs",
    32  			"The total number of jobs in the queue.",
    33  			statusGaugeLabels, nil,
    34  		),
    35  
    36  		addedTotal: prometheus.NewDesc(
    37  			schedulerQueueMetricsPrefix+"added_jobs_total",
    38  			"The total number of jobs added to the queue.",
    39  			variableLabels, nil,
    40  		),
    41  		completedTotal: prometheus.NewDesc(
    42  			schedulerQueueMetricsPrefix+"completed_jobs_total",
    43  			"The total number of jobs completed.",
    44  			variableLabels, nil,
    45  		),
    46  		assignedTotal: prometheus.NewDesc(
    47  			schedulerQueueMetricsPrefix+"assigned_jobs_total",
    48  			"The total number of jobs assigned.",
    49  			variableLabels, nil,
    50  		),
    51  		reassignedTotal: prometheus.NewDesc(
    52  			schedulerQueueMetricsPrefix+"reassigned_jobs_total",
    53  			"The total number of jobs reassigned.",
    54  			variableLabels, nil,
    55  		),
    56  		evictedTotal: prometheus.NewDesc(
    57  			schedulerQueueMetricsPrefix+"evicted_jobs_total",
    58  			"The total number of jobs evicted.",
    59  			variableLabels, nil,
    60  		),
    61  	}
    62  }
    63  
    64  func (c *statsCollector) Describe(ch chan<- *prometheus.Desc) {
    65  	ch <- c.jobs
    66  	ch <- c.addedTotal
    67  	ch <- c.completedTotal
    68  	ch <- c.assignedTotal
    69  	ch <- c.reassignedTotal
    70  	ch <- c.evictedTotal
    71  }
    72  
    73  func (c *statsCollector) Collect(ch chan<- prometheus.Metric) {
    74  	for _, m := range c.collectMetrics() {
    75  		ch <- m
    76  	}
    77  }
    78  
    79  func (c *statsCollector) collectStats(fn func(level int, stats queueStats)) {
    80  	for i, q := range c.s.queue.levels {
    81  		// Note that some levels may be empty.
    82  		if q == nil || q.jobs == nil {
    83  			continue
    84  		}
    85  		var stats queueStats
    86  		for _, e := range *q.jobs {
    87  			switch {
    88  			case e.Status == 0:
    89  				stats.unassigned++
    90  			case c.s.config.MaxFailures > 0 && uint64(e.Failures) >= c.s.config.MaxFailures:
    91  				stats.failed++
    92  			case e.Failures > 0:
    93  				stats.reassigned++
    94  			default:
    95  				stats.assigned++
    96  			}
    97  		}
    98  
    99  		// Update stored gauges. Those are not used at the moment,
   100  		// but can help planning schedule updates in the future.
   101  		q.stats.assigned = stats.assigned
   102  		q.stats.unassigned = stats.unassigned
   103  		q.stats.reassigned = stats.reassigned
   104  		q.stats.failed = stats.failed
   105  
   106  		// Counters are updated on access.
   107  		stats.addedTotal = q.stats.addedTotal
   108  		stats.completedTotal = q.stats.completedTotal
   109  		stats.assignedTotal = q.stats.assignedTotal
   110  		stats.reassignedTotal = q.stats.reassignedTotal
   111  		stats.evictedTotal = q.stats.evictedTotal
   112  
   113  		fn(i, stats)
   114  	}
   115  }
   116  
   117  func (c *statsCollector) collectMetrics() []prometheus.Metric {
   118  	c.s.mu.Lock()
   119  	defer c.s.mu.Unlock()
   120  
   121  	var metrics = make([]prometheus.Metric, 0, 8*len(c.s.queue.levels))
   122  	c.collectStats(func(l int, stats queueStats) {
   123  		level := strconv.Itoa(l)
   124  		metrics = append(metrics,
   125  			prometheus.MustNewConstMetric(c.jobs, prometheus.GaugeValue, float64(stats.assigned), level, "assigned"),
   126  			prometheus.MustNewConstMetric(c.jobs, prometheus.GaugeValue, float64(stats.unassigned), level, "unassigned"),
   127  			prometheus.MustNewConstMetric(c.jobs, prometheus.GaugeValue, float64(stats.reassigned), level, "reassigned"),
   128  			prometheus.MustNewConstMetric(c.jobs, prometheus.GaugeValue, float64(stats.failed), level, "failed"),
   129  			prometheus.MustNewConstMetric(c.addedTotal, prometheus.CounterValue, float64(stats.addedTotal), level),
   130  			prometheus.MustNewConstMetric(c.completedTotal, prometheus.CounterValue, float64(stats.completedTotal), level),
   131  			prometheus.MustNewConstMetric(c.assignedTotal, prometheus.CounterValue, float64(stats.assignedTotal), level),
   132  			prometheus.MustNewConstMetric(c.reassignedTotal, prometheus.CounterValue, float64(stats.reassignedTotal), level),
   133  			prometheus.MustNewConstMetric(c.evictedTotal, prometheus.CounterValue, float64(stats.evictedTotal), level),
   134  		)
   135  	})
   136  	return metrics
   137  }