github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/pkg/compactor/syncer_metrics.go (about)

     1  package compactor
     2  
     3  import (
     4  	"github.com/go-kit/log/level"
     5  	"github.com/prometheus/client_golang/prometheus"
     6  	"github.com/prometheus/client_golang/prometheus/promauto"
     7  
     8  	"github.com/cortexproject/cortex/pkg/util"
     9  	util_log "github.com/cortexproject/cortex/pkg/util/log"
    10  )
    11  
    12  // Copied from Thanos, pkg/compact/compact.go.
    13  // Here we aggregate metrics from all finished syncers.
    14  type syncerMetrics struct {
    15  	metaSync                  prometheus.Counter
    16  	metaSyncFailures          prometheus.Counter
    17  	metaSyncDuration          *util.HistogramDataCollector // was prometheus.Histogram before
    18  	metaSyncConsistencyDelay  prometheus.Gauge
    19  	garbageCollections        prometheus.Counter
    20  	garbageCollectionFailures prometheus.Counter
    21  	garbageCollectionDuration *util.HistogramDataCollector // was prometheus.Histogram before
    22  	compactions               prometheus.Counter
    23  	compactionRunsStarted     prometheus.Counter
    24  	compactionRunsCompleted   prometheus.Counter
    25  	compactionFailures        prometheus.Counter
    26  	verticalCompactions       prometheus.Counter
    27  }
    28  
    29  // Copied (and modified with Cortex prefix) from Thanos, pkg/compact/compact.go
    30  // We also ignore "group" label, since we only use a single group.
    31  func newSyncerMetrics(reg prometheus.Registerer) *syncerMetrics {
    32  	var m syncerMetrics
    33  
    34  	m.metaSync = promauto.With(reg).NewCounter(prometheus.CounterOpts{
    35  		Name: "cortex_compactor_meta_syncs_total",
    36  		Help: "Total blocks metadata synchronization attempts.",
    37  	})
    38  	m.metaSyncFailures = promauto.With(reg).NewCounter(prometheus.CounterOpts{
    39  		Name: "cortex_compactor_meta_sync_failures_total",
    40  		Help: "Total blocks metadata synchronization failures.",
    41  	})
    42  	m.metaSyncDuration = util.NewHistogramDataCollector(prometheus.NewDesc(
    43  		"cortex_compactor_meta_sync_duration_seconds",
    44  		"Duration of the blocks metadata synchronization in seconds.",
    45  		nil, nil))
    46  	m.metaSyncConsistencyDelay = promauto.With(reg).NewGauge(prometheus.GaugeOpts{
    47  		Name: "cortex_compactor_meta_sync_consistency_delay_seconds",
    48  		Help: "Configured consistency delay in seconds.",
    49  	})
    50  
    51  	m.garbageCollections = promauto.With(reg).NewCounter(prometheus.CounterOpts{
    52  		Name: "cortex_compactor_garbage_collection_total",
    53  		Help: "Total number of garbage collection operations.",
    54  	})
    55  	m.garbageCollectionFailures = promauto.With(reg).NewCounter(prometheus.CounterOpts{
    56  		Name: "cortex_compactor_garbage_collection_failures_total",
    57  		Help: "Total number of failed garbage collection operations.",
    58  	})
    59  	m.garbageCollectionDuration = util.NewHistogramDataCollector(prometheus.NewDesc(
    60  		"cortex_compactor_garbage_collection_duration_seconds",
    61  		"Time it took to perform garbage collection iteration.",
    62  		nil, nil))
    63  
    64  	m.compactions = promauto.With(reg).NewCounter(prometheus.CounterOpts{
    65  		Name: "cortex_compactor_group_compactions_total",
    66  		Help: "Total number of group compaction attempts that resulted in a new block.",
    67  	})
    68  	m.compactionRunsStarted = promauto.With(reg).NewCounter(prometheus.CounterOpts{
    69  		Name: "cortex_compactor_group_compaction_runs_started_total",
    70  		Help: "Total number of group compaction attempts.",
    71  	})
    72  	m.compactionRunsCompleted = promauto.With(reg).NewCounter(prometheus.CounterOpts{
    73  		Name: "cortex_compactor_group_compaction_runs_completed_total",
    74  		Help: "Total number of group completed compaction runs. This also includes compactor group runs that resulted with no compaction.",
    75  	})
    76  	m.compactionFailures = promauto.With(reg).NewCounter(prometheus.CounterOpts{
    77  		Name: "cortex_compactor_group_compactions_failures_total",
    78  		Help: "Total number of failed group compactions.",
    79  	})
    80  	m.verticalCompactions = promauto.With(reg).NewCounter(prometheus.CounterOpts{
    81  		Name: "cortex_compactor_group_vertical_compactions_total",
    82  		Help: "Total number of group compaction attempts that resulted in a new block based on overlapping blocks.",
    83  	})
    84  
    85  	if reg != nil {
    86  		reg.MustRegister(m.metaSyncDuration, m.garbageCollectionDuration)
    87  	}
    88  
    89  	return &m
    90  }
    91  
    92  func (m *syncerMetrics) gatherThanosSyncerMetrics(reg *prometheus.Registry) {
    93  	if m == nil {
    94  		return
    95  	}
    96  
    97  	mf, err := reg.Gather()
    98  	if err != nil {
    99  		level.Warn(util_log.Logger).Log("msg", "failed to gather metrics from syncer registry after compaction", "err", err)
   100  		return
   101  	}
   102  
   103  	mfm, err := util.NewMetricFamilyMap(mf)
   104  	if err != nil {
   105  		level.Warn(util_log.Logger).Log("msg", "failed to gather metrics from syncer registry after compaction", "err", err)
   106  		return
   107  	}
   108  
   109  	m.metaSync.Add(mfm.SumCounters("blocks_meta_syncs_total"))
   110  	m.metaSyncFailures.Add(mfm.SumCounters("blocks_meta_sync_failures_total"))
   111  	m.metaSyncDuration.Add(mfm.SumHistograms("blocks_meta_sync_duration_seconds"))
   112  	m.metaSyncConsistencyDelay.Set(mfm.MaxGauges("consistency_delay_seconds"))
   113  
   114  	m.garbageCollections.Add(mfm.SumCounters("thanos_compact_garbage_collection_total"))
   115  	m.garbageCollectionFailures.Add(mfm.SumCounters("thanos_compact_garbage_collection_failures_total"))
   116  	m.garbageCollectionDuration.Add(mfm.SumHistograms("thanos_compact_garbage_collection_duration_seconds"))
   117  
   118  	// These metrics have "group" label, but we sum them all together.
   119  	m.compactions.Add(mfm.SumCounters("thanos_compact_group_compactions_total"))
   120  	m.compactionRunsStarted.Add(mfm.SumCounters("thanos_compact_group_compaction_runs_started_total"))
   121  	m.compactionRunsCompleted.Add(mfm.SumCounters("thanos_compact_group_compaction_runs_completed_total"))
   122  	m.compactionFailures.Add(mfm.SumCounters("thanos_compact_group_compactions_failures_total"))
   123  	m.verticalCompactions.Add(mfm.SumCounters("thanos_compact_group_vertical_compactions_total"))
   124  }