github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/pkg/alertmanager/alertmanager_metrics_test.go (about) 1 package alertmanager 2 3 import ( 4 "bytes" 5 "testing" 6 7 "github.com/prometheus/alertmanager/types" 8 "github.com/prometheus/client_golang/prometheus" 9 "github.com/prometheus/client_golang/prometheus/promauto" 10 "github.com/prometheus/client_golang/prometheus/testutil" 11 "github.com/stretchr/testify/require" 12 ) 13 14 var integrations = []string{ 15 "email", 16 "pagerduty", 17 "wechat", 18 "pushover", 19 "slack", 20 "opsgenie", 21 "webhook", 22 "victorops", 23 } 24 25 func TestAlertmanagerMetricsStore(t *testing.T) { 26 mainReg := prometheus.NewPedanticRegistry() 27 28 alertmanangerMetrics := newAlertmanagerMetrics() 29 mainReg.MustRegister(alertmanangerMetrics) 30 alertmanangerMetrics.addUserRegistry("user1", populateAlertmanager(1)) 31 alertmanangerMetrics.addUserRegistry("user2", populateAlertmanager(10)) 32 alertmanangerMetrics.addUserRegistry("user3", populateAlertmanager(100)) 33 34 //noinspection ALL 35 err := testutil.GatherAndCompare(mainReg, bytes.NewBufferString(` 36 # HELP cortex_alertmanager_alerts How many alerts by state. 37 # TYPE cortex_alertmanager_alerts gauge 38 cortex_alertmanager_alerts{state="active",user="user1"} 1 39 cortex_alertmanager_alerts{state="active",user="user2"} 10 40 cortex_alertmanager_alerts{state="active",user="user3"} 100 41 cortex_alertmanager_alerts{state="suppressed",user="user1"} 2 42 cortex_alertmanager_alerts{state="suppressed",user="user2"} 20 43 cortex_alertmanager_alerts{state="suppressed",user="user3"} 200 44 # HELP cortex_alertmanager_alerts_invalid_total The total number of received alerts that were invalid. 45 # TYPE cortex_alertmanager_alerts_invalid_total counter 46 cortex_alertmanager_alerts_invalid_total{user="user1"} 2 47 cortex_alertmanager_alerts_invalid_total{user="user2"} 20 48 cortex_alertmanager_alerts_invalid_total{user="user3"} 200 49 # HELP cortex_alertmanager_alerts_received_total The total number of received alerts. 50 # TYPE cortex_alertmanager_alerts_received_total counter 51 cortex_alertmanager_alerts_received_total{user="user1"} 10 52 cortex_alertmanager_alerts_received_total{user="user2"} 100 53 cortex_alertmanager_alerts_received_total{user="user3"} 1000 54 # HELP cortex_alertmanager_config_hash Hash of the currently loaded alertmanager configuration. 55 # TYPE cortex_alertmanager_config_hash gauge 56 cortex_alertmanager_config_hash{user="user1"} 0 57 cortex_alertmanager_config_hash{user="user2"} 0 58 cortex_alertmanager_config_hash{user="user3"} 0 59 # HELP cortex_alertmanager_nflog_gc_duration_seconds Duration of the last notification log garbage collection cycle. 60 # TYPE cortex_alertmanager_nflog_gc_duration_seconds summary 61 cortex_alertmanager_nflog_gc_duration_seconds_sum 111 62 cortex_alertmanager_nflog_gc_duration_seconds_count 3 63 # HELP cortex_alertmanager_nflog_gossip_messages_propagated_total Number of received gossip messages that have been further gossiped. 64 # TYPE cortex_alertmanager_nflog_gossip_messages_propagated_total counter 65 cortex_alertmanager_nflog_gossip_messages_propagated_total 111 66 # HELP cortex_alertmanager_nflog_queries_total Number of notification log queries were received. 67 # TYPE cortex_alertmanager_nflog_queries_total counter 68 cortex_alertmanager_nflog_queries_total 111 69 # HELP cortex_alertmanager_nflog_query_duration_seconds Duration of notification log query evaluation. 70 # TYPE cortex_alertmanager_nflog_query_duration_seconds histogram 71 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="0.005"} 0 72 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="0.01"} 0 73 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="0.025"} 0 74 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="0.05"} 0 75 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="0.1"} 0 76 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="0.25"} 0 77 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="0.5"} 0 78 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="1"} 1 79 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="2.5"} 1 80 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="5"} 1 81 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="10"} 2 82 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="+Inf"} 3 83 cortex_alertmanager_nflog_query_duration_seconds_sum 111 84 cortex_alertmanager_nflog_query_duration_seconds_count 3 85 # HELP cortex_alertmanager_nflog_query_errors_total Number notification log received queries that failed. 86 # TYPE cortex_alertmanager_nflog_query_errors_total counter 87 cortex_alertmanager_nflog_query_errors_total 111 88 # HELP cortex_alertmanager_nflog_snapshot_duration_seconds Duration of the last notification log snapshot. 89 # TYPE cortex_alertmanager_nflog_snapshot_duration_seconds summary 90 cortex_alertmanager_nflog_snapshot_duration_seconds_sum 111 91 cortex_alertmanager_nflog_snapshot_duration_seconds_count 3 92 # HELP cortex_alertmanager_nflog_snapshot_size_bytes Size of the last notification log snapshot in bytes. 93 # TYPE cortex_alertmanager_nflog_snapshot_size_bytes gauge 94 cortex_alertmanager_nflog_snapshot_size_bytes 111 95 # HELP cortex_alertmanager_notification_latency_seconds The latency of notifications in seconds. 96 # TYPE cortex_alertmanager_notification_latency_seconds histogram 97 cortex_alertmanager_notification_latency_seconds_bucket{le="1"} 14 98 cortex_alertmanager_notification_latency_seconds_bucket{le="5"} 19 99 cortex_alertmanager_notification_latency_seconds_bucket{le="10"} 21 100 cortex_alertmanager_notification_latency_seconds_bucket{le="15"} 23 101 cortex_alertmanager_notification_latency_seconds_bucket{le="20"} 24 102 cortex_alertmanager_notification_latency_seconds_bucket{le="+Inf"} 24 103 cortex_alertmanager_notification_latency_seconds_sum 77.7 104 cortex_alertmanager_notification_latency_seconds_count 24 105 # HELP cortex_alertmanager_notifications_failed_total The total number of failed notifications. 106 # TYPE cortex_alertmanager_notifications_failed_total counter 107 cortex_alertmanager_notifications_failed_total{integration="email",user="user1"} 0 108 cortex_alertmanager_notifications_failed_total{integration="email",user="user2"} 0 109 cortex_alertmanager_notifications_failed_total{integration="email",user="user3"} 0 110 cortex_alertmanager_notifications_failed_total{integration="opsgenie",user="user1"} 5 111 cortex_alertmanager_notifications_failed_total{integration="opsgenie",user="user2"} 50 112 cortex_alertmanager_notifications_failed_total{integration="opsgenie",user="user3"} 500 113 cortex_alertmanager_notifications_failed_total{integration="pagerduty",user="user1"} 1 114 cortex_alertmanager_notifications_failed_total{integration="pagerduty",user="user2"} 10 115 cortex_alertmanager_notifications_failed_total{integration="pagerduty",user="user3"} 100 116 cortex_alertmanager_notifications_failed_total{integration="pushover",user="user1"} 3 117 cortex_alertmanager_notifications_failed_total{integration="pushover",user="user2"} 30 118 cortex_alertmanager_notifications_failed_total{integration="pushover",user="user3"} 300 119 cortex_alertmanager_notifications_failed_total{integration="slack",user="user1"} 4 120 cortex_alertmanager_notifications_failed_total{integration="slack",user="user2"} 40 121 cortex_alertmanager_notifications_failed_total{integration="slack",user="user3"} 400 122 cortex_alertmanager_notifications_failed_total{integration="victorops",user="user1"} 7 123 cortex_alertmanager_notifications_failed_total{integration="victorops",user="user2"} 70 124 cortex_alertmanager_notifications_failed_total{integration="victorops",user="user3"} 700 125 cortex_alertmanager_notifications_failed_total{integration="webhook",user="user1"} 6 126 cortex_alertmanager_notifications_failed_total{integration="webhook",user="user2"} 60 127 cortex_alertmanager_notifications_failed_total{integration="webhook",user="user3"} 600 128 cortex_alertmanager_notifications_failed_total{integration="wechat",user="user1"} 2 129 cortex_alertmanager_notifications_failed_total{integration="wechat",user="user2"} 20 130 cortex_alertmanager_notifications_failed_total{integration="wechat",user="user3"} 200 131 # HELP cortex_alertmanager_notification_requests_total The total number of attempted notification requests. 132 # TYPE cortex_alertmanager_notification_requests_total counter 133 cortex_alertmanager_notification_requests_total{integration="email",user="user1"} 0 134 cortex_alertmanager_notification_requests_total{integration="email",user="user2"} 0 135 cortex_alertmanager_notification_requests_total{integration="email",user="user3"} 0 136 cortex_alertmanager_notification_requests_total{integration="opsgenie",user="user1"} 5 137 cortex_alertmanager_notification_requests_total{integration="opsgenie",user="user2"} 50 138 cortex_alertmanager_notification_requests_total{integration="opsgenie",user="user3"} 500 139 cortex_alertmanager_notification_requests_total{integration="pagerduty",user="user1"} 1 140 cortex_alertmanager_notification_requests_total{integration="pagerduty",user="user2"} 10 141 cortex_alertmanager_notification_requests_total{integration="pagerduty",user="user3"} 100 142 cortex_alertmanager_notification_requests_total{integration="pushover",user="user1"} 3 143 cortex_alertmanager_notification_requests_total{integration="pushover",user="user2"} 30 144 cortex_alertmanager_notification_requests_total{integration="pushover",user="user3"} 300 145 cortex_alertmanager_notification_requests_total{integration="slack",user="user1"} 4 146 cortex_alertmanager_notification_requests_total{integration="slack",user="user2"} 40 147 cortex_alertmanager_notification_requests_total{integration="slack",user="user3"} 400 148 cortex_alertmanager_notification_requests_total{integration="victorops",user="user1"} 7 149 cortex_alertmanager_notification_requests_total{integration="victorops",user="user2"} 70 150 cortex_alertmanager_notification_requests_total{integration="victorops",user="user3"} 700 151 cortex_alertmanager_notification_requests_total{integration="webhook",user="user1"} 6 152 cortex_alertmanager_notification_requests_total{integration="webhook",user="user2"} 60 153 cortex_alertmanager_notification_requests_total{integration="webhook",user="user3"} 600 154 cortex_alertmanager_notification_requests_total{integration="wechat",user="user1"} 2 155 cortex_alertmanager_notification_requests_total{integration="wechat",user="user2"} 20 156 cortex_alertmanager_notification_requests_total{integration="wechat",user="user3"} 200 157 # HELP cortex_alertmanager_notification_requests_failed_total The total number of failed notification requests. 158 # TYPE cortex_alertmanager_notification_requests_failed_total counter 159 cortex_alertmanager_notification_requests_failed_total{integration="email",user="user1"} 0 160 cortex_alertmanager_notification_requests_failed_total{integration="email",user="user2"} 0 161 cortex_alertmanager_notification_requests_failed_total{integration="email",user="user3"} 0 162 cortex_alertmanager_notification_requests_failed_total{integration="opsgenie",user="user1"} 5 163 cortex_alertmanager_notification_requests_failed_total{integration="opsgenie",user="user2"} 50 164 cortex_alertmanager_notification_requests_failed_total{integration="opsgenie",user="user3"} 500 165 cortex_alertmanager_notification_requests_failed_total{integration="pagerduty",user="user1"} 1 166 cortex_alertmanager_notification_requests_failed_total{integration="pagerduty",user="user2"} 10 167 cortex_alertmanager_notification_requests_failed_total{integration="pagerduty",user="user3"} 100 168 cortex_alertmanager_notification_requests_failed_total{integration="pushover",user="user1"} 3 169 cortex_alertmanager_notification_requests_failed_total{integration="pushover",user="user2"} 30 170 cortex_alertmanager_notification_requests_failed_total{integration="pushover",user="user3"} 300 171 cortex_alertmanager_notification_requests_failed_total{integration="slack",user="user1"} 4 172 cortex_alertmanager_notification_requests_failed_total{integration="slack",user="user2"} 40 173 cortex_alertmanager_notification_requests_failed_total{integration="slack",user="user3"} 400 174 cortex_alertmanager_notification_requests_failed_total{integration="victorops",user="user1"} 7 175 cortex_alertmanager_notification_requests_failed_total{integration="victorops",user="user2"} 70 176 cortex_alertmanager_notification_requests_failed_total{integration="victorops",user="user3"} 700 177 cortex_alertmanager_notification_requests_failed_total{integration="webhook",user="user1"} 6 178 cortex_alertmanager_notification_requests_failed_total{integration="webhook",user="user2"} 60 179 cortex_alertmanager_notification_requests_failed_total{integration="webhook",user="user3"} 600 180 cortex_alertmanager_notification_requests_failed_total{integration="wechat",user="user1"} 2 181 cortex_alertmanager_notification_requests_failed_total{integration="wechat",user="user2"} 20 182 cortex_alertmanager_notification_requests_failed_total{integration="wechat",user="user3"} 200 183 # HELP cortex_alertmanager_notifications_total The total number of attempted notifications. 184 # TYPE cortex_alertmanager_notifications_total counter 185 cortex_alertmanager_notifications_total{integration="email",user="user1"} 0 186 cortex_alertmanager_notifications_total{integration="email",user="user2"} 0 187 cortex_alertmanager_notifications_total{integration="email",user="user3"} 0 188 cortex_alertmanager_notifications_total{integration="opsgenie",user="user1"} 5 189 cortex_alertmanager_notifications_total{integration="opsgenie",user="user2"} 50 190 cortex_alertmanager_notifications_total{integration="opsgenie",user="user3"} 500 191 cortex_alertmanager_notifications_total{integration="pagerduty",user="user1"} 1 192 cortex_alertmanager_notifications_total{integration="pagerduty",user="user2"} 10 193 cortex_alertmanager_notifications_total{integration="pagerduty",user="user3"} 100 194 cortex_alertmanager_notifications_total{integration="pushover",user="user1"} 3 195 cortex_alertmanager_notifications_total{integration="pushover",user="user2"} 30 196 cortex_alertmanager_notifications_total{integration="pushover",user="user3"} 300 197 cortex_alertmanager_notifications_total{integration="slack",user="user1"} 4 198 cortex_alertmanager_notifications_total{integration="slack",user="user2"} 40 199 cortex_alertmanager_notifications_total{integration="slack",user="user3"} 400 200 cortex_alertmanager_notifications_total{integration="victorops",user="user1"} 7 201 cortex_alertmanager_notifications_total{integration="victorops",user="user2"} 70 202 cortex_alertmanager_notifications_total{integration="victorops",user="user3"} 700 203 cortex_alertmanager_notifications_total{integration="webhook",user="user1"} 6 204 cortex_alertmanager_notifications_total{integration="webhook",user="user2"} 60 205 cortex_alertmanager_notifications_total{integration="webhook",user="user3"} 600 206 cortex_alertmanager_notifications_total{integration="wechat",user="user1"} 2 207 cortex_alertmanager_notifications_total{integration="wechat",user="user2"} 20 208 cortex_alertmanager_notifications_total{integration="wechat",user="user3"} 200 209 210 # HELP cortex_alertmanager_silences How many silences by state. 211 # TYPE cortex_alertmanager_silences gauge 212 cortex_alertmanager_silences{state="active",user="user1"} 1 213 cortex_alertmanager_silences{state="active",user="user2"} 10 214 cortex_alertmanager_silences{state="active",user="user3"} 100 215 cortex_alertmanager_silences{state="expired",user="user1"} 2 216 cortex_alertmanager_silences{state="expired",user="user2"} 20 217 cortex_alertmanager_silences{state="expired",user="user3"} 200 218 cortex_alertmanager_silences{state="pending",user="user1"} 3 219 cortex_alertmanager_silences{state="pending",user="user2"} 30 220 cortex_alertmanager_silences{state="pending",user="user3"} 300 221 # HELP cortex_alertmanager_silences_gc_duration_seconds Duration of the last silence garbage collection cycle. 222 # TYPE cortex_alertmanager_silences_gc_duration_seconds summary 223 cortex_alertmanager_silences_gc_duration_seconds_sum 111 224 cortex_alertmanager_silences_gc_duration_seconds_count 3 225 # HELP cortex_alertmanager_silences_gossip_messages_propagated_total Number of received gossip messages that have been further gossiped. 226 # TYPE cortex_alertmanager_silences_gossip_messages_propagated_total counter 227 cortex_alertmanager_silences_gossip_messages_propagated_total 111 228 # HELP cortex_alertmanager_silences_queries_total How many silence queries were received. 229 # TYPE cortex_alertmanager_silences_queries_total counter 230 cortex_alertmanager_silences_queries_total 111 231 # HELP cortex_alertmanager_silences_query_duration_seconds Duration of silence query evaluation. 232 # TYPE cortex_alertmanager_silences_query_duration_seconds histogram 233 cortex_alertmanager_silences_query_duration_seconds_bucket{le="0.005"} 0 234 cortex_alertmanager_silences_query_duration_seconds_bucket{le="0.01"} 0 235 cortex_alertmanager_silences_query_duration_seconds_bucket{le="0.025"} 0 236 cortex_alertmanager_silences_query_duration_seconds_bucket{le="0.05"} 0 237 cortex_alertmanager_silences_query_duration_seconds_bucket{le="0.1"} 0 238 cortex_alertmanager_silences_query_duration_seconds_bucket{le="0.25"} 0 239 cortex_alertmanager_silences_query_duration_seconds_bucket{le="0.5"} 0 240 cortex_alertmanager_silences_query_duration_seconds_bucket{le="1"} 1 241 cortex_alertmanager_silences_query_duration_seconds_bucket{le="2.5"} 1 242 cortex_alertmanager_silences_query_duration_seconds_bucket{le="5"} 1 243 cortex_alertmanager_silences_query_duration_seconds_bucket{le="10"} 2 244 cortex_alertmanager_silences_query_duration_seconds_bucket{le="+Inf"} 3 245 cortex_alertmanager_silences_query_duration_seconds_sum 111 246 cortex_alertmanager_silences_query_duration_seconds_count 3 247 # HELP cortex_alertmanager_silences_query_errors_total How many silence received queries did not succeed. 248 # TYPE cortex_alertmanager_silences_query_errors_total counter 249 cortex_alertmanager_silences_query_errors_total 111 250 # HELP cortex_alertmanager_silences_snapshot_duration_seconds Duration of the last silence snapshot. 251 # TYPE cortex_alertmanager_silences_snapshot_duration_seconds summary 252 cortex_alertmanager_silences_snapshot_duration_seconds_sum 111 253 cortex_alertmanager_silences_snapshot_duration_seconds_count 3 254 # HELP cortex_alertmanager_silences_snapshot_size_bytes Size of the last silence snapshot in bytes. 255 # TYPE cortex_alertmanager_silences_snapshot_size_bytes gauge 256 cortex_alertmanager_silences_snapshot_size_bytes 111 257 # HELP cortex_alertmanager_state_fetch_replica_state_failed_total Number of times we have failed to read and merge the full state from another replica. 258 # TYPE cortex_alertmanager_state_fetch_replica_state_failed_total counter 259 cortex_alertmanager_state_fetch_replica_state_failed_total 0 260 # HELP cortex_alertmanager_state_fetch_replica_state_total Number of times we have tried to read and merge the full state from another replica. 261 # TYPE cortex_alertmanager_state_fetch_replica_state_total counter 262 cortex_alertmanager_state_fetch_replica_state_total 0 263 # HELP cortex_alertmanager_state_initial_sync_duration_seconds Time spent syncing initial state from peers or storage. 264 # TYPE cortex_alertmanager_state_initial_sync_duration_seconds histogram 265 cortex_alertmanager_state_initial_sync_duration_seconds_bucket{le="+Inf"} 0 266 cortex_alertmanager_state_initial_sync_duration_seconds_sum 0 267 cortex_alertmanager_state_initial_sync_duration_seconds_count 0 268 # HELP cortex_alertmanager_state_initial_sync_total Number of times we have tried to sync initial state from peers or storage. 269 # TYPE cortex_alertmanager_state_initial_sync_total counter 270 cortex_alertmanager_state_initial_sync_total 0 271 # HELP cortex_alertmanager_state_persist_failed_total Number of times we have failed to persist the running state to storage. 272 # TYPE cortex_alertmanager_state_persist_failed_total counter 273 cortex_alertmanager_state_persist_failed_total 0 274 # HELP cortex_alertmanager_state_persist_total Number of times we have tried to persist the running state to storage. 275 # TYPE cortex_alertmanager_state_persist_total counter 276 cortex_alertmanager_state_persist_total 0 277 278 # HELP cortex_alertmanager_alerts_limiter_current_alerts Number of alerts tracked by alerts limiter. 279 # TYPE cortex_alertmanager_alerts_limiter_current_alerts gauge 280 cortex_alertmanager_alerts_limiter_current_alerts{user="user1"} 10 281 cortex_alertmanager_alerts_limiter_current_alerts{user="user2"} 100 282 cortex_alertmanager_alerts_limiter_current_alerts{user="user3"} 1000 283 # HELP cortex_alertmanager_alerts_limiter_current_alerts_size_bytes Total size of alerts tracked by alerts limiter. 284 # TYPE cortex_alertmanager_alerts_limiter_current_alerts_size_bytes gauge 285 cortex_alertmanager_alerts_limiter_current_alerts_size_bytes{user="user1"} 100 286 cortex_alertmanager_alerts_limiter_current_alerts_size_bytes{user="user2"} 1000 287 cortex_alertmanager_alerts_limiter_current_alerts_size_bytes{user="user3"} 10000 288 # HELP cortex_alertmanager_alerts_insert_limited_total Total number of failures to store alert due to hitting alertmanager limits. 289 # TYPE cortex_alertmanager_alerts_insert_limited_total counter 290 cortex_alertmanager_alerts_insert_limited_total{user="user1"} 7 291 cortex_alertmanager_alerts_insert_limited_total{user="user2"} 70 292 cortex_alertmanager_alerts_insert_limited_total{user="user3"} 700 293 `)) 294 require.NoError(t, err) 295 } 296 297 func TestAlertmanagerMetricsRemoval(t *testing.T) { 298 mainReg := prometheus.NewPedanticRegistry() 299 300 alertmanagerMetrics := newAlertmanagerMetrics() 301 mainReg.MustRegister(alertmanagerMetrics) 302 alertmanagerMetrics.addUserRegistry("user1", populateAlertmanager(1)) 303 alertmanagerMetrics.addUserRegistry("user2", populateAlertmanager(10)) 304 alertmanagerMetrics.addUserRegistry("user3", populateAlertmanager(100)) 305 306 // In this test, we assert that metrics are "soft deleted" per the registry removal. 307 // In practice, this means several things: 308 // a) counters of removed registries are not reset. 309 // b) gauges of removed registries are removed. 310 // c) histograms/summaries (as these are just counters) are not reset. 311 // Instead of just asserting a few of these metrics we go for the whole payload to ensure our tests are sensitive to change and avoid regressions. 312 err := testutil.GatherAndCompare(mainReg, bytes.NewBufferString(` 313 # HELP cortex_alertmanager_alerts How many alerts by state. 314 # TYPE cortex_alertmanager_alerts gauge 315 cortex_alertmanager_alerts{state="active",user="user1"} 1 316 cortex_alertmanager_alerts{state="active",user="user2"} 10 317 cortex_alertmanager_alerts{state="active",user="user3"} 100 318 cortex_alertmanager_alerts{state="suppressed",user="user1"} 2 319 cortex_alertmanager_alerts{state="suppressed",user="user2"} 20 320 cortex_alertmanager_alerts{state="suppressed",user="user3"} 200 321 322 # HELP cortex_alertmanager_alerts_invalid_total The total number of received alerts that were invalid. 323 # TYPE cortex_alertmanager_alerts_invalid_total counter 324 cortex_alertmanager_alerts_invalid_total{user="user1"} 2 325 cortex_alertmanager_alerts_invalid_total{user="user2"} 20 326 cortex_alertmanager_alerts_invalid_total{user="user3"} 200 327 328 # HELP cortex_alertmanager_alerts_received_total The total number of received alerts. 329 # TYPE cortex_alertmanager_alerts_received_total counter 330 cortex_alertmanager_alerts_received_total{user="user1"} 10 331 cortex_alertmanager_alerts_received_total{user="user2"} 100 332 cortex_alertmanager_alerts_received_total{user="user3"} 1000 333 334 # HELP cortex_alertmanager_config_hash Hash of the currently loaded alertmanager configuration. 335 # TYPE cortex_alertmanager_config_hash gauge 336 cortex_alertmanager_config_hash{user="user1"} 0 337 cortex_alertmanager_config_hash{user="user2"} 0 338 cortex_alertmanager_config_hash{user="user3"} 0 339 340 # HELP cortex_alertmanager_nflog_gc_duration_seconds Duration of the last notification log garbage collection cycle. 341 # TYPE cortex_alertmanager_nflog_gc_duration_seconds summary 342 cortex_alertmanager_nflog_gc_duration_seconds_sum 111 343 cortex_alertmanager_nflog_gc_duration_seconds_count 3 344 345 # HELP cortex_alertmanager_nflog_gossip_messages_propagated_total Number of received gossip messages that have been further gossiped. 346 # TYPE cortex_alertmanager_nflog_gossip_messages_propagated_total counter 347 cortex_alertmanager_nflog_gossip_messages_propagated_total 111 348 349 # HELP cortex_alertmanager_nflog_queries_total Number of notification log queries were received. 350 # TYPE cortex_alertmanager_nflog_queries_total counter 351 cortex_alertmanager_nflog_queries_total 111 352 353 # HELP cortex_alertmanager_nflog_query_duration_seconds Duration of notification log query evaluation. 354 # TYPE cortex_alertmanager_nflog_query_duration_seconds histogram 355 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="0.005"} 0 356 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="0.01"} 0 357 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="0.025"} 0 358 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="0.05"} 0 359 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="0.1"} 0 360 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="0.25"} 0 361 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="0.5"} 0 362 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="1"} 1 363 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="2.5"} 1 364 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="5"} 1 365 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="10"} 2 366 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="+Inf"} 3 367 cortex_alertmanager_nflog_query_duration_seconds_sum 111 368 cortex_alertmanager_nflog_query_duration_seconds_count 3 369 370 # HELP cortex_alertmanager_nflog_query_errors_total Number notification log received queries that failed. 371 # TYPE cortex_alertmanager_nflog_query_errors_total counter 372 cortex_alertmanager_nflog_query_errors_total 111 373 374 # HELP cortex_alertmanager_nflog_snapshot_duration_seconds Duration of the last notification log snapshot. 375 # TYPE cortex_alertmanager_nflog_snapshot_duration_seconds summary 376 cortex_alertmanager_nflog_snapshot_duration_seconds_sum 111 377 cortex_alertmanager_nflog_snapshot_duration_seconds_count 3 378 379 # HELP cortex_alertmanager_nflog_snapshot_size_bytes Size of the last notification log snapshot in bytes. 380 # TYPE cortex_alertmanager_nflog_snapshot_size_bytes gauge 381 cortex_alertmanager_nflog_snapshot_size_bytes 111 382 383 # HELP cortex_alertmanager_notification_latency_seconds The latency of notifications in seconds. 384 # TYPE cortex_alertmanager_notification_latency_seconds histogram 385 cortex_alertmanager_notification_latency_seconds_bucket{le="1"} 14 386 cortex_alertmanager_notification_latency_seconds_bucket{le="5"} 19 387 cortex_alertmanager_notification_latency_seconds_bucket{le="10"} 21 388 cortex_alertmanager_notification_latency_seconds_bucket{le="15"} 23 389 cortex_alertmanager_notification_latency_seconds_bucket{le="20"} 24 390 cortex_alertmanager_notification_latency_seconds_bucket{le="+Inf"} 24 391 cortex_alertmanager_notification_latency_seconds_sum 77.7 392 cortex_alertmanager_notification_latency_seconds_count 24 393 394 # HELP cortex_alertmanager_notification_requests_failed_total The total number of failed notification requests. 395 # TYPE cortex_alertmanager_notification_requests_failed_total counter 396 cortex_alertmanager_notification_requests_failed_total{integration="email",user="user1"} 0 397 cortex_alertmanager_notification_requests_failed_total{integration="email",user="user2"} 0 398 cortex_alertmanager_notification_requests_failed_total{integration="email",user="user3"} 0 399 cortex_alertmanager_notification_requests_failed_total{integration="opsgenie",user="user1"} 5 400 cortex_alertmanager_notification_requests_failed_total{integration="opsgenie",user="user2"} 50 401 cortex_alertmanager_notification_requests_failed_total{integration="opsgenie",user="user3"} 500 402 cortex_alertmanager_notification_requests_failed_total{integration="pagerduty",user="user1"} 1 403 cortex_alertmanager_notification_requests_failed_total{integration="pagerduty",user="user2"} 10 404 cortex_alertmanager_notification_requests_failed_total{integration="pagerduty",user="user3"} 100 405 cortex_alertmanager_notification_requests_failed_total{integration="pushover",user="user1"} 3 406 cortex_alertmanager_notification_requests_failed_total{integration="pushover",user="user2"} 30 407 cortex_alertmanager_notification_requests_failed_total{integration="pushover",user="user3"} 300 408 cortex_alertmanager_notification_requests_failed_total{integration="slack",user="user1"} 4 409 cortex_alertmanager_notification_requests_failed_total{integration="slack",user="user2"} 40 410 cortex_alertmanager_notification_requests_failed_total{integration="slack",user="user3"} 400 411 cortex_alertmanager_notification_requests_failed_total{integration="victorops",user="user1"} 7 412 cortex_alertmanager_notification_requests_failed_total{integration="victorops",user="user2"} 70 413 cortex_alertmanager_notification_requests_failed_total{integration="victorops",user="user3"} 700 414 cortex_alertmanager_notification_requests_failed_total{integration="webhook",user="user1"} 6 415 cortex_alertmanager_notification_requests_failed_total{integration="webhook",user="user2"} 60 416 cortex_alertmanager_notification_requests_failed_total{integration="webhook",user="user3"} 600 417 cortex_alertmanager_notification_requests_failed_total{integration="wechat",user="user1"} 2 418 cortex_alertmanager_notification_requests_failed_total{integration="wechat",user="user2"} 20 419 cortex_alertmanager_notification_requests_failed_total{integration="wechat",user="user3"} 200 420 421 # HELP cortex_alertmanager_notification_requests_total The total number of attempted notification requests. 422 # TYPE cortex_alertmanager_notification_requests_total counter 423 cortex_alertmanager_notification_requests_total{integration="email",user="user1"} 0 424 cortex_alertmanager_notification_requests_total{integration="email",user="user2"} 0 425 cortex_alertmanager_notification_requests_total{integration="email",user="user3"} 0 426 cortex_alertmanager_notification_requests_total{integration="opsgenie",user="user1"} 5 427 cortex_alertmanager_notification_requests_total{integration="opsgenie",user="user2"} 50 428 cortex_alertmanager_notification_requests_total{integration="opsgenie",user="user3"} 500 429 cortex_alertmanager_notification_requests_total{integration="pagerduty",user="user1"} 1 430 cortex_alertmanager_notification_requests_total{integration="pagerduty",user="user2"} 10 431 cortex_alertmanager_notification_requests_total{integration="pagerduty",user="user3"} 100 432 cortex_alertmanager_notification_requests_total{integration="pushover",user="user1"} 3 433 cortex_alertmanager_notification_requests_total{integration="pushover",user="user2"} 30 434 cortex_alertmanager_notification_requests_total{integration="pushover",user="user3"} 300 435 cortex_alertmanager_notification_requests_total{integration="slack",user="user1"} 4 436 cortex_alertmanager_notification_requests_total{integration="slack",user="user2"} 40 437 cortex_alertmanager_notification_requests_total{integration="slack",user="user3"} 400 438 cortex_alertmanager_notification_requests_total{integration="victorops",user="user1"} 7 439 cortex_alertmanager_notification_requests_total{integration="victorops",user="user2"} 70 440 cortex_alertmanager_notification_requests_total{integration="victorops",user="user3"} 700 441 cortex_alertmanager_notification_requests_total{integration="webhook",user="user1"} 6 442 cortex_alertmanager_notification_requests_total{integration="webhook",user="user2"} 60 443 cortex_alertmanager_notification_requests_total{integration="webhook",user="user3"} 600 444 cortex_alertmanager_notification_requests_total{integration="wechat",user="user1"} 2 445 cortex_alertmanager_notification_requests_total{integration="wechat",user="user2"} 20 446 cortex_alertmanager_notification_requests_total{integration="wechat",user="user3"} 200 447 448 # HELP cortex_alertmanager_notifications_failed_total The total number of failed notifications. 449 # TYPE cortex_alertmanager_notifications_failed_total counter 450 cortex_alertmanager_notifications_failed_total{integration="email",user="user1"} 0 451 cortex_alertmanager_notifications_failed_total{integration="email",user="user2"} 0 452 cortex_alertmanager_notifications_failed_total{integration="email",user="user3"} 0 453 cortex_alertmanager_notifications_failed_total{integration="opsgenie",user="user1"} 5 454 cortex_alertmanager_notifications_failed_total{integration="opsgenie",user="user2"} 50 455 cortex_alertmanager_notifications_failed_total{integration="opsgenie",user="user3"} 500 456 cortex_alertmanager_notifications_failed_total{integration="pagerduty",user="user1"} 1 457 cortex_alertmanager_notifications_failed_total{integration="pagerduty",user="user2"} 10 458 cortex_alertmanager_notifications_failed_total{integration="pagerduty",user="user3"} 100 459 cortex_alertmanager_notifications_failed_total{integration="pushover",user="user1"} 3 460 cortex_alertmanager_notifications_failed_total{integration="pushover",user="user2"} 30 461 cortex_alertmanager_notifications_failed_total{integration="pushover",user="user3"} 300 462 cortex_alertmanager_notifications_failed_total{integration="slack",user="user1"} 4 463 cortex_alertmanager_notifications_failed_total{integration="slack",user="user2"} 40 464 cortex_alertmanager_notifications_failed_total{integration="slack",user="user3"} 400 465 cortex_alertmanager_notifications_failed_total{integration="victorops",user="user1"} 7 466 cortex_alertmanager_notifications_failed_total{integration="victorops",user="user2"} 70 467 cortex_alertmanager_notifications_failed_total{integration="victorops",user="user3"} 700 468 cortex_alertmanager_notifications_failed_total{integration="webhook",user="user1"} 6 469 cortex_alertmanager_notifications_failed_total{integration="webhook",user="user2"} 60 470 cortex_alertmanager_notifications_failed_total{integration="webhook",user="user3"} 600 471 cortex_alertmanager_notifications_failed_total{integration="wechat",user="user1"} 2 472 cortex_alertmanager_notifications_failed_total{integration="wechat",user="user2"} 20 473 cortex_alertmanager_notifications_failed_total{integration="wechat",user="user3"} 200 474 475 # HELP cortex_alertmanager_notifications_total The total number of attempted notifications. 476 # TYPE cortex_alertmanager_notifications_total counter 477 cortex_alertmanager_notifications_total{integration="email",user="user1"} 0 478 cortex_alertmanager_notifications_total{integration="email",user="user2"} 0 479 cortex_alertmanager_notifications_total{integration="email",user="user3"} 0 480 cortex_alertmanager_notifications_total{integration="opsgenie",user="user1"} 5 481 cortex_alertmanager_notifications_total{integration="opsgenie",user="user2"} 50 482 cortex_alertmanager_notifications_total{integration="opsgenie",user="user3"} 500 483 cortex_alertmanager_notifications_total{integration="pagerduty",user="user1"} 1 484 cortex_alertmanager_notifications_total{integration="pagerduty",user="user2"} 10 485 cortex_alertmanager_notifications_total{integration="pagerduty",user="user3"} 100 486 cortex_alertmanager_notifications_total{integration="pushover",user="user1"} 3 487 cortex_alertmanager_notifications_total{integration="pushover",user="user2"} 30 488 cortex_alertmanager_notifications_total{integration="pushover",user="user3"} 300 489 cortex_alertmanager_notifications_total{integration="slack",user="user1"} 4 490 cortex_alertmanager_notifications_total{integration="slack",user="user2"} 40 491 cortex_alertmanager_notifications_total{integration="slack",user="user3"} 400 492 cortex_alertmanager_notifications_total{integration="victorops",user="user1"} 7 493 cortex_alertmanager_notifications_total{integration="victorops",user="user2"} 70 494 cortex_alertmanager_notifications_total{integration="victorops",user="user3"} 700 495 cortex_alertmanager_notifications_total{integration="webhook",user="user1"} 6 496 cortex_alertmanager_notifications_total{integration="webhook",user="user2"} 60 497 cortex_alertmanager_notifications_total{integration="webhook",user="user3"} 600 498 cortex_alertmanager_notifications_total{integration="wechat",user="user1"} 2 499 cortex_alertmanager_notifications_total{integration="wechat",user="user2"} 20 500 cortex_alertmanager_notifications_total{integration="wechat",user="user3"} 200 501 502 # HELP cortex_alertmanager_silences How many silences by state. 503 # TYPE cortex_alertmanager_silences gauge 504 cortex_alertmanager_silences{state="active",user="user1"} 1 505 cortex_alertmanager_silences{state="active",user="user2"} 10 506 cortex_alertmanager_silences{state="active",user="user3"} 100 507 cortex_alertmanager_silences{state="expired",user="user1"} 2 508 cortex_alertmanager_silences{state="expired",user="user2"} 20 509 cortex_alertmanager_silences{state="expired",user="user3"} 200 510 cortex_alertmanager_silences{state="pending",user="user1"} 3 511 cortex_alertmanager_silences{state="pending",user="user2"} 30 512 cortex_alertmanager_silences{state="pending",user="user3"} 300 513 514 # HELP cortex_alertmanager_silences_gc_duration_seconds Duration of the last silence garbage collection cycle. 515 # TYPE cortex_alertmanager_silences_gc_duration_seconds summary 516 cortex_alertmanager_silences_gc_duration_seconds_sum 111 517 cortex_alertmanager_silences_gc_duration_seconds_count 3 518 519 # HELP cortex_alertmanager_silences_gossip_messages_propagated_total Number of received gossip messages that have been further gossiped. 520 # TYPE cortex_alertmanager_silences_gossip_messages_propagated_total counter 521 cortex_alertmanager_silences_gossip_messages_propagated_total 111 522 523 # HELP cortex_alertmanager_silences_queries_total How many silence queries were received. 524 # TYPE cortex_alertmanager_silences_queries_total counter 525 cortex_alertmanager_silences_queries_total 111 526 527 # HELP cortex_alertmanager_silences_query_duration_seconds Duration of silence query evaluation. 528 # TYPE cortex_alertmanager_silences_query_duration_seconds histogram 529 cortex_alertmanager_silences_query_duration_seconds_bucket{le="0.005"} 0 530 cortex_alertmanager_silences_query_duration_seconds_bucket{le="0.01"} 0 531 cortex_alertmanager_silences_query_duration_seconds_bucket{le="0.025"} 0 532 cortex_alertmanager_silences_query_duration_seconds_bucket{le="0.05"} 0 533 cortex_alertmanager_silences_query_duration_seconds_bucket{le="0.1"} 0 534 cortex_alertmanager_silences_query_duration_seconds_bucket{le="0.25"} 0 535 cortex_alertmanager_silences_query_duration_seconds_bucket{le="0.5"} 0 536 cortex_alertmanager_silences_query_duration_seconds_bucket{le="1"} 1 537 cortex_alertmanager_silences_query_duration_seconds_bucket{le="2.5"} 1 538 cortex_alertmanager_silences_query_duration_seconds_bucket{le="5"} 1 539 cortex_alertmanager_silences_query_duration_seconds_bucket{le="10"} 2 540 cortex_alertmanager_silences_query_duration_seconds_bucket{le="+Inf"} 3 541 cortex_alertmanager_silences_query_duration_seconds_sum 111 542 cortex_alertmanager_silences_query_duration_seconds_count 3 543 544 # HELP cortex_alertmanager_silences_query_errors_total How many silence received queries did not succeed. 545 # TYPE cortex_alertmanager_silences_query_errors_total counter 546 cortex_alertmanager_silences_query_errors_total 111 547 548 # HELP cortex_alertmanager_silences_snapshot_duration_seconds Duration of the last silence snapshot. 549 # TYPE cortex_alertmanager_silences_snapshot_duration_seconds summary 550 cortex_alertmanager_silences_snapshot_duration_seconds_sum 111 551 cortex_alertmanager_silences_snapshot_duration_seconds_count 3 552 553 # HELP cortex_alertmanager_silences_snapshot_size_bytes Size of the last silence snapshot in bytes. 554 # TYPE cortex_alertmanager_silences_snapshot_size_bytes gauge 555 cortex_alertmanager_silences_snapshot_size_bytes 111 556 # HELP cortex_alertmanager_state_fetch_replica_state_failed_total Number of times we have failed to read and merge the full state from another replica. 557 # TYPE cortex_alertmanager_state_fetch_replica_state_failed_total counter 558 cortex_alertmanager_state_fetch_replica_state_failed_total 0 559 # HELP cortex_alertmanager_state_fetch_replica_state_total Number of times we have tried to read and merge the full state from another replica. 560 # TYPE cortex_alertmanager_state_fetch_replica_state_total counter 561 cortex_alertmanager_state_fetch_replica_state_total 0 562 # HELP cortex_alertmanager_state_initial_sync_duration_seconds Time spent syncing initial state from peers or storage. 563 # TYPE cortex_alertmanager_state_initial_sync_duration_seconds histogram 564 cortex_alertmanager_state_initial_sync_duration_seconds_bucket{le="+Inf"} 0 565 cortex_alertmanager_state_initial_sync_duration_seconds_sum 0 566 cortex_alertmanager_state_initial_sync_duration_seconds_count 0 567 # HELP cortex_alertmanager_state_initial_sync_total Number of times we have tried to sync initial state from peers or storage. 568 # TYPE cortex_alertmanager_state_initial_sync_total counter 569 cortex_alertmanager_state_initial_sync_total 0 570 # HELP cortex_alertmanager_state_persist_failed_total Number of times we have failed to persist the running state to storage. 571 # TYPE cortex_alertmanager_state_persist_failed_total counter 572 cortex_alertmanager_state_persist_failed_total 0 573 # HELP cortex_alertmanager_state_persist_total Number of times we have tried to persist the running state to storage. 574 # TYPE cortex_alertmanager_state_persist_total counter 575 cortex_alertmanager_state_persist_total 0 576 577 # HELP cortex_alertmanager_alerts_limiter_current_alerts Number of alerts tracked by alerts limiter. 578 # TYPE cortex_alertmanager_alerts_limiter_current_alerts gauge 579 cortex_alertmanager_alerts_limiter_current_alerts{user="user1"} 10 580 cortex_alertmanager_alerts_limiter_current_alerts{user="user2"} 100 581 cortex_alertmanager_alerts_limiter_current_alerts{user="user3"} 1000 582 # HELP cortex_alertmanager_alerts_limiter_current_alerts_size_bytes Total size of alerts tracked by alerts limiter. 583 # TYPE cortex_alertmanager_alerts_limiter_current_alerts_size_bytes gauge 584 cortex_alertmanager_alerts_limiter_current_alerts_size_bytes{user="user1"} 100 585 cortex_alertmanager_alerts_limiter_current_alerts_size_bytes{user="user2"} 1000 586 cortex_alertmanager_alerts_limiter_current_alerts_size_bytes{user="user3"} 10000 587 # HELP cortex_alertmanager_alerts_insert_limited_total Total number of failures to store alert due to hitting alertmanager limits. 588 # TYPE cortex_alertmanager_alerts_insert_limited_total counter 589 cortex_alertmanager_alerts_insert_limited_total{user="user1"} 7 590 cortex_alertmanager_alerts_insert_limited_total{user="user2"} 70 591 cortex_alertmanager_alerts_insert_limited_total{user="user3"} 700 592 593 `)) 594 require.NoError(t, err) 595 596 alertmanagerMetrics.removeUserRegistry("user3") 597 598 // After removal counters shouldn't reset. 599 err = testutil.GatherAndCompare(mainReg, bytes.NewBufferString(` 600 # HELP cortex_alertmanager_alerts How many alerts by state. 601 # TYPE cortex_alertmanager_alerts gauge 602 cortex_alertmanager_alerts{state="active",user="user1"} 1 603 cortex_alertmanager_alerts{state="active",user="user2"} 10 604 cortex_alertmanager_alerts{state="suppressed",user="user1"} 2 605 cortex_alertmanager_alerts{state="suppressed",user="user2"} 20 606 607 # HELP cortex_alertmanager_alerts_invalid_total The total number of received alerts that were invalid. 608 # TYPE cortex_alertmanager_alerts_invalid_total counter 609 cortex_alertmanager_alerts_invalid_total{user="user1"} 2 610 cortex_alertmanager_alerts_invalid_total{user="user2"} 20 611 612 # HELP cortex_alertmanager_alerts_received_total The total number of received alerts. 613 # TYPE cortex_alertmanager_alerts_received_total counter 614 cortex_alertmanager_alerts_received_total{user="user1"} 10 615 cortex_alertmanager_alerts_received_total{user="user2"} 100 616 617 # HELP cortex_alertmanager_config_hash Hash of the currently loaded alertmanager configuration. 618 # TYPE cortex_alertmanager_config_hash gauge 619 cortex_alertmanager_config_hash{user="user1"} 0 620 cortex_alertmanager_config_hash{user="user2"} 0 621 622 # HELP cortex_alertmanager_nflog_gc_duration_seconds Duration of the last notification log garbage collection cycle. 623 # TYPE cortex_alertmanager_nflog_gc_duration_seconds summary 624 cortex_alertmanager_nflog_gc_duration_seconds_sum 111 625 cortex_alertmanager_nflog_gc_duration_seconds_count 3 626 627 # HELP cortex_alertmanager_nflog_gossip_messages_propagated_total Number of received gossip messages that have been further gossiped. 628 # TYPE cortex_alertmanager_nflog_gossip_messages_propagated_total counter 629 cortex_alertmanager_nflog_gossip_messages_propagated_total 111 630 631 # HELP cortex_alertmanager_nflog_queries_total Number of notification log queries were received. 632 # TYPE cortex_alertmanager_nflog_queries_total counter 633 cortex_alertmanager_nflog_queries_total 111 634 635 # HELP cortex_alertmanager_nflog_query_duration_seconds Duration of notification log query evaluation. 636 # TYPE cortex_alertmanager_nflog_query_duration_seconds histogram 637 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="0.005"} 0 638 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="0.01"} 0 639 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="0.025"} 0 640 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="0.05"} 0 641 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="0.1"} 0 642 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="0.25"} 0 643 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="0.5"} 0 644 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="1"} 1 645 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="2.5"} 1 646 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="5"} 1 647 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="10"} 2 648 cortex_alertmanager_nflog_query_duration_seconds_bucket{le="+Inf"} 3 649 cortex_alertmanager_nflog_query_duration_seconds_sum 111 650 cortex_alertmanager_nflog_query_duration_seconds_count 3 651 652 # HELP cortex_alertmanager_nflog_query_errors_total Number notification log received queries that failed. 653 # TYPE cortex_alertmanager_nflog_query_errors_total counter 654 cortex_alertmanager_nflog_query_errors_total 111 655 656 # HELP cortex_alertmanager_nflog_snapshot_duration_seconds Duration of the last notification log snapshot. 657 # TYPE cortex_alertmanager_nflog_snapshot_duration_seconds summary 658 cortex_alertmanager_nflog_snapshot_duration_seconds_sum 111 659 cortex_alertmanager_nflog_snapshot_duration_seconds_count 3 660 661 # HELP cortex_alertmanager_nflog_snapshot_size_bytes Size of the last notification log snapshot in bytes. 662 # TYPE cortex_alertmanager_nflog_snapshot_size_bytes gauge 663 cortex_alertmanager_nflog_snapshot_size_bytes 11 664 665 # HELP cortex_alertmanager_notification_latency_seconds The latency of notifications in seconds. 666 # TYPE cortex_alertmanager_notification_latency_seconds histogram 667 cortex_alertmanager_notification_latency_seconds_bucket{le="1"} 14 668 cortex_alertmanager_notification_latency_seconds_bucket{le="5"} 19 669 cortex_alertmanager_notification_latency_seconds_bucket{le="10"} 21 670 cortex_alertmanager_notification_latency_seconds_bucket{le="15"} 23 671 cortex_alertmanager_notification_latency_seconds_bucket{le="20"} 24 672 cortex_alertmanager_notification_latency_seconds_bucket{le="+Inf"} 24 673 cortex_alertmanager_notification_latency_seconds_sum 77.7 674 cortex_alertmanager_notification_latency_seconds_count 24 675 676 # HELP cortex_alertmanager_notification_requests_failed_total The total number of failed notification requests. 677 # TYPE cortex_alertmanager_notification_requests_failed_total counter 678 cortex_alertmanager_notification_requests_failed_total{integration="email",user="user1"} 0 679 cortex_alertmanager_notification_requests_failed_total{integration="email",user="user2"} 0 680 cortex_alertmanager_notification_requests_failed_total{integration="opsgenie",user="user1"} 5 681 cortex_alertmanager_notification_requests_failed_total{integration="opsgenie",user="user2"} 50 682 cortex_alertmanager_notification_requests_failed_total{integration="pagerduty",user="user1"} 1 683 cortex_alertmanager_notification_requests_failed_total{integration="pagerduty",user="user2"} 10 684 cortex_alertmanager_notification_requests_failed_total{integration="pushover",user="user1"} 3 685 cortex_alertmanager_notification_requests_failed_total{integration="pushover",user="user2"} 30 686 cortex_alertmanager_notification_requests_failed_total{integration="slack",user="user1"} 4 687 cortex_alertmanager_notification_requests_failed_total{integration="slack",user="user2"} 40 688 cortex_alertmanager_notification_requests_failed_total{integration="victorops",user="user1"} 7 689 cortex_alertmanager_notification_requests_failed_total{integration="victorops",user="user2"} 70 690 cortex_alertmanager_notification_requests_failed_total{integration="webhook",user="user1"} 6 691 cortex_alertmanager_notification_requests_failed_total{integration="webhook",user="user2"} 60 692 cortex_alertmanager_notification_requests_failed_total{integration="wechat",user="user1"} 2 693 cortex_alertmanager_notification_requests_failed_total{integration="wechat",user="user2"} 20 694 695 # HELP cortex_alertmanager_notification_requests_total The total number of attempted notification requests. 696 # TYPE cortex_alertmanager_notification_requests_total counter 697 cortex_alertmanager_notification_requests_total{integration="email",user="user1"} 0 698 cortex_alertmanager_notification_requests_total{integration="email",user="user2"} 0 699 cortex_alertmanager_notification_requests_total{integration="opsgenie",user="user1"} 5 700 cortex_alertmanager_notification_requests_total{integration="opsgenie",user="user2"} 50 701 cortex_alertmanager_notification_requests_total{integration="pagerduty",user="user1"} 1 702 cortex_alertmanager_notification_requests_total{integration="pagerduty",user="user2"} 10 703 cortex_alertmanager_notification_requests_total{integration="pushover",user="user1"} 3 704 cortex_alertmanager_notification_requests_total{integration="pushover",user="user2"} 30 705 cortex_alertmanager_notification_requests_total{integration="slack",user="user1"} 4 706 cortex_alertmanager_notification_requests_total{integration="slack",user="user2"} 40 707 cortex_alertmanager_notification_requests_total{integration="victorops",user="user1"} 7 708 cortex_alertmanager_notification_requests_total{integration="victorops",user="user2"} 70 709 cortex_alertmanager_notification_requests_total{integration="webhook",user="user1"} 6 710 cortex_alertmanager_notification_requests_total{integration="webhook",user="user2"} 60 711 cortex_alertmanager_notification_requests_total{integration="wechat",user="user1"} 2 712 cortex_alertmanager_notification_requests_total{integration="wechat",user="user2"} 20 713 714 # HELP cortex_alertmanager_notifications_failed_total The total number of failed notifications. 715 # TYPE cortex_alertmanager_notifications_failed_total counter 716 cortex_alertmanager_notifications_failed_total{integration="email",user="user1"} 0 717 cortex_alertmanager_notifications_failed_total{integration="email",user="user2"} 0 718 cortex_alertmanager_notifications_failed_total{integration="opsgenie",user="user1"} 5 719 cortex_alertmanager_notifications_failed_total{integration="opsgenie",user="user2"} 50 720 cortex_alertmanager_notifications_failed_total{integration="pagerduty",user="user1"} 1 721 cortex_alertmanager_notifications_failed_total{integration="pagerduty",user="user2"} 10 722 cortex_alertmanager_notifications_failed_total{integration="pushover",user="user1"} 3 723 cortex_alertmanager_notifications_failed_total{integration="pushover",user="user2"} 30 724 cortex_alertmanager_notifications_failed_total{integration="slack",user="user1"} 4 725 cortex_alertmanager_notifications_failed_total{integration="slack",user="user2"} 40 726 cortex_alertmanager_notifications_failed_total{integration="victorops",user="user1"} 7 727 cortex_alertmanager_notifications_failed_total{integration="victorops",user="user2"} 70 728 cortex_alertmanager_notifications_failed_total{integration="webhook",user="user1"} 6 729 cortex_alertmanager_notifications_failed_total{integration="webhook",user="user2"} 60 730 cortex_alertmanager_notifications_failed_total{integration="wechat",user="user1"} 2 731 cortex_alertmanager_notifications_failed_total{integration="wechat",user="user2"} 20 732 733 # HELP cortex_alertmanager_notifications_total The total number of attempted notifications. 734 # TYPE cortex_alertmanager_notifications_total counter 735 cortex_alertmanager_notifications_total{integration="email",user="user1"} 0 736 cortex_alertmanager_notifications_total{integration="email",user="user2"} 0 737 cortex_alertmanager_notifications_total{integration="opsgenie",user="user1"} 5 738 cortex_alertmanager_notifications_total{integration="opsgenie",user="user2"} 50 739 cortex_alertmanager_notifications_total{integration="pagerduty",user="user1"} 1 740 cortex_alertmanager_notifications_total{integration="pagerduty",user="user2"} 10 741 cortex_alertmanager_notifications_total{integration="pushover",user="user1"} 3 742 cortex_alertmanager_notifications_total{integration="pushover",user="user2"} 30 743 cortex_alertmanager_notifications_total{integration="slack",user="user1"} 4 744 cortex_alertmanager_notifications_total{integration="slack",user="user2"} 40 745 cortex_alertmanager_notifications_total{integration="victorops",user="user1"} 7 746 cortex_alertmanager_notifications_total{integration="victorops",user="user2"} 70 747 cortex_alertmanager_notifications_total{integration="webhook",user="user1"} 6 748 cortex_alertmanager_notifications_total{integration="webhook",user="user2"} 60 749 cortex_alertmanager_notifications_total{integration="wechat",user="user1"} 2 750 cortex_alertmanager_notifications_total{integration="wechat",user="user2"} 20 751 752 # HELP cortex_alertmanager_silences How many silences by state. 753 # TYPE cortex_alertmanager_silences gauge 754 cortex_alertmanager_silences{state="active",user="user1"} 1 755 cortex_alertmanager_silences{state="active",user="user2"} 10 756 cortex_alertmanager_silences{state="expired",user="user1"} 2 757 cortex_alertmanager_silences{state="expired",user="user2"} 20 758 cortex_alertmanager_silences{state="pending",user="user1"} 3 759 cortex_alertmanager_silences{state="pending",user="user2"} 30 760 761 # HELP cortex_alertmanager_silences_gc_duration_seconds Duration of the last silence garbage collection cycle. 762 # TYPE cortex_alertmanager_silences_gc_duration_seconds summary 763 cortex_alertmanager_silences_gc_duration_seconds_sum 111 764 cortex_alertmanager_silences_gc_duration_seconds_count 3 765 766 # HELP cortex_alertmanager_silences_gossip_messages_propagated_total Number of received gossip messages that have been further gossiped. 767 # TYPE cortex_alertmanager_silences_gossip_messages_propagated_total counter 768 cortex_alertmanager_silences_gossip_messages_propagated_total 111 769 770 # HELP cortex_alertmanager_silences_queries_total How many silence queries were received. 771 # TYPE cortex_alertmanager_silences_queries_total counter 772 cortex_alertmanager_silences_queries_total 111 773 774 # HELP cortex_alertmanager_silences_query_duration_seconds Duration of silence query evaluation. 775 # TYPE cortex_alertmanager_silences_query_duration_seconds histogram 776 cortex_alertmanager_silences_query_duration_seconds_bucket{le="0.005"} 0 777 cortex_alertmanager_silences_query_duration_seconds_bucket{le="0.01"} 0 778 cortex_alertmanager_silences_query_duration_seconds_bucket{le="0.025"} 0 779 cortex_alertmanager_silences_query_duration_seconds_bucket{le="0.05"} 0 780 cortex_alertmanager_silences_query_duration_seconds_bucket{le="0.1"} 0 781 cortex_alertmanager_silences_query_duration_seconds_bucket{le="0.25"} 0 782 cortex_alertmanager_silences_query_duration_seconds_bucket{le="0.5"} 0 783 cortex_alertmanager_silences_query_duration_seconds_bucket{le="1"} 1 784 cortex_alertmanager_silences_query_duration_seconds_bucket{le="2.5"} 1 785 cortex_alertmanager_silences_query_duration_seconds_bucket{le="5"} 1 786 cortex_alertmanager_silences_query_duration_seconds_bucket{le="10"} 2 787 cortex_alertmanager_silences_query_duration_seconds_bucket{le="+Inf"} 3 788 cortex_alertmanager_silences_query_duration_seconds_sum 111 789 cortex_alertmanager_silences_query_duration_seconds_count 3 790 791 # HELP cortex_alertmanager_silences_query_errors_total How many silence received queries did not succeed. 792 # TYPE cortex_alertmanager_silences_query_errors_total counter 793 cortex_alertmanager_silences_query_errors_total 111 794 795 # HELP cortex_alertmanager_silences_snapshot_duration_seconds Duration of the last silence snapshot. 796 # TYPE cortex_alertmanager_silences_snapshot_duration_seconds summary 797 cortex_alertmanager_silences_snapshot_duration_seconds_sum 111 798 cortex_alertmanager_silences_snapshot_duration_seconds_count 3 799 800 # HELP cortex_alertmanager_silences_snapshot_size_bytes Size of the last silence snapshot in bytes. 801 # TYPE cortex_alertmanager_silences_snapshot_size_bytes gauge 802 cortex_alertmanager_silences_snapshot_size_bytes 11 803 804 # HELP cortex_alertmanager_state_fetch_replica_state_failed_total Number of times we have failed to read and merge the full state from another replica. 805 # TYPE cortex_alertmanager_state_fetch_replica_state_failed_total counter 806 cortex_alertmanager_state_fetch_replica_state_failed_total 0 807 # HELP cortex_alertmanager_state_fetch_replica_state_total Number of times we have tried to read and merge the full state from another replica. 808 # TYPE cortex_alertmanager_state_fetch_replica_state_total counter 809 cortex_alertmanager_state_fetch_replica_state_total 0 810 # HELP cortex_alertmanager_state_initial_sync_duration_seconds Time spent syncing initial state from peers or storage. 811 # TYPE cortex_alertmanager_state_initial_sync_duration_seconds histogram 812 cortex_alertmanager_state_initial_sync_duration_seconds_bucket{le="+Inf"} 0 813 cortex_alertmanager_state_initial_sync_duration_seconds_sum 0 814 cortex_alertmanager_state_initial_sync_duration_seconds_count 0 815 # HELP cortex_alertmanager_state_initial_sync_total Number of times we have tried to sync initial state from peers or storage. 816 # TYPE cortex_alertmanager_state_initial_sync_total counter 817 cortex_alertmanager_state_initial_sync_total 0 818 # HELP cortex_alertmanager_state_persist_failed_total Number of times we have failed to persist the running state to storage. 819 # TYPE cortex_alertmanager_state_persist_failed_total counter 820 cortex_alertmanager_state_persist_failed_total 0 821 # HELP cortex_alertmanager_state_persist_total Number of times we have tried to persist the running state to storage. 822 # TYPE cortex_alertmanager_state_persist_total counter 823 cortex_alertmanager_state_persist_total 0 824 825 # HELP cortex_alertmanager_alerts_limiter_current_alerts Number of alerts tracked by alerts limiter. 826 # TYPE cortex_alertmanager_alerts_limiter_current_alerts gauge 827 cortex_alertmanager_alerts_limiter_current_alerts{user="user1"} 10 828 cortex_alertmanager_alerts_limiter_current_alerts{user="user2"} 100 829 # HELP cortex_alertmanager_alerts_limiter_current_alerts_size_bytes Total size of alerts tracked by alerts limiter. 830 # TYPE cortex_alertmanager_alerts_limiter_current_alerts_size_bytes gauge 831 cortex_alertmanager_alerts_limiter_current_alerts_size_bytes{user="user1"} 100 832 cortex_alertmanager_alerts_limiter_current_alerts_size_bytes{user="user2"} 1000 833 # HELP cortex_alertmanager_alerts_insert_limited_total Total number of failures to store alert due to hitting alertmanager limits. 834 # TYPE cortex_alertmanager_alerts_insert_limited_total counter 835 cortex_alertmanager_alerts_insert_limited_total{user="user1"} 7 836 cortex_alertmanager_alerts_insert_limited_total{user="user2"} 70 837 `)) 838 require.NoError(t, err) 839 } 840 841 func populateAlertmanager(base float64) *prometheus.Registry { 842 reg := prometheus.NewRegistry() 843 s := newSilenceMetrics(reg) 844 s.gcDuration.Observe(base) 845 s.snapshotDuration.Observe(base) 846 s.snapshotSize.Add(base) 847 s.queriesTotal.Add(base) 848 s.queryErrorsTotal.Add(base) 849 s.queryDuration.Observe(base) 850 s.propagatedMessagesTotal.Add(base) 851 s.silencesActive.Set(base) 852 s.silencesExpired.Set(base * 2) 853 s.silencesPending.Set(base * 3) 854 855 n := newNflogMetrics(reg) 856 n.gcDuration.Observe(base) 857 n.snapshotDuration.Observe(base) 858 n.snapshotSize.Add(base) 859 n.queriesTotal.Add(base) 860 n.queryErrorsTotal.Add(base) 861 n.queryDuration.Observe(base) 862 n.propagatedMessagesTotal.Add(base) 863 864 nm := newNotifyMetrics(reg) 865 for i, integration := range integrations { 866 nm.numNotifications.WithLabelValues(integration).Add(base * float64(i)) 867 nm.numFailedNotifications.WithLabelValues(integration).Add(base * float64(i)) 868 nm.numNotificationRequestsTotal.WithLabelValues(integration).Add(base * float64(i)) 869 nm.numNotificationRequestsFailedTotal.WithLabelValues(integration).Add(base * float64(i)) 870 nm.notificationLatencySeconds.WithLabelValues(integration).Observe(base * float64(i) * 0.025) 871 } 872 873 m := newMarkerMetrics(reg) 874 m.alerts.WithLabelValues(string(types.AlertStateActive)).Add(base) 875 m.alerts.WithLabelValues(string(types.AlertStateSuppressed)).Add(base * 2) 876 877 v1APIMetrics := newAPIMetrics("v1", reg) 878 v1APIMetrics.firing.Add(base * 2) 879 v1APIMetrics.invalid.Add(base) 880 v1APIMetrics.resolved.Add(base * 3) 881 882 v2APIMetrics := newAPIMetrics("v2", reg) 883 v2APIMetrics.firing.Add(base * 2) 884 v2APIMetrics.invalid.Add(base) 885 v2APIMetrics.resolved.Add(base * 3) 886 887 lm := newLimiterMetrics(reg) 888 lm.count.Set(10 * base) 889 lm.size.Set(100 * base) 890 lm.insertFailures.Add(7 * base) 891 892 return reg 893 } 894 895 // Copied from github.com/alertmanager/nflog/nflog.go 896 type nflogMetrics struct { 897 gcDuration prometheus.Summary 898 snapshotDuration prometheus.Summary 899 snapshotSize prometheus.Gauge 900 queriesTotal prometheus.Counter 901 queryErrorsTotal prometheus.Counter 902 queryDuration prometheus.Histogram 903 propagatedMessagesTotal prometheus.Counter 904 } 905 906 func newNflogMetrics(r prometheus.Registerer) *nflogMetrics { 907 m := &nflogMetrics{} 908 909 m.gcDuration = promauto.With(r).NewSummary(prometheus.SummaryOpts{ 910 Name: "alertmanager_nflog_gc_duration_seconds", 911 Help: "Duration of the last notification log garbage collection cycle.", 912 Objectives: map[float64]float64{}, 913 }) 914 m.snapshotDuration = promauto.With(r).NewSummary(prometheus.SummaryOpts{ 915 Name: "alertmanager_nflog_snapshot_duration_seconds", 916 Help: "Duration of the last notification log snapshot.", 917 Objectives: map[float64]float64{}, 918 }) 919 m.snapshotSize = promauto.With(r).NewGauge(prometheus.GaugeOpts{ 920 Name: "alertmanager_nflog_snapshot_size_bytes", 921 Help: "Size of the last notification log snapshot in bytes.", 922 }) 923 m.queriesTotal = promauto.With(r).NewCounter(prometheus.CounterOpts{ 924 Name: "alertmanager_nflog_queries_total", 925 Help: "Number of notification log queries were received.", 926 }) 927 m.queryErrorsTotal = promauto.With(r).NewCounter(prometheus.CounterOpts{ 928 Name: "alertmanager_nflog_query_errors_total", 929 Help: "Number notification log received queries that failed.", 930 }) 931 m.queryDuration = promauto.With(r).NewHistogram(prometheus.HistogramOpts{ 932 Name: "alertmanager_nflog_query_duration_seconds", 933 Help: "Duration of notification log query evaluation.", 934 }) 935 m.propagatedMessagesTotal = promauto.With(r).NewCounter(prometheus.CounterOpts{ 936 Name: "alertmanager_nflog_gossip_messages_propagated_total", 937 Help: "Number of received gossip messages that have been further gossiped.", 938 }) 939 940 return m 941 } 942 943 // Copied from github.com/alertmanager/silence/silence.go 944 type silenceMetrics struct { 945 gcDuration prometheus.Summary 946 snapshotDuration prometheus.Summary 947 snapshotSize prometheus.Gauge 948 queriesTotal prometheus.Counter 949 queryErrorsTotal prometheus.Counter 950 queryDuration prometheus.Histogram 951 silencesActive prometheus.Gauge 952 silencesPending prometheus.Gauge 953 silencesExpired prometheus.Gauge 954 propagatedMessagesTotal prometheus.Counter 955 } 956 957 func newSilenceMetrics(r prometheus.Registerer) *silenceMetrics { 958 m := &silenceMetrics{} 959 960 m.gcDuration = promauto.With(r).NewSummary(prometheus.SummaryOpts{ 961 Name: "alertmanager_silences_gc_duration_seconds", 962 Help: "Duration of the last silence garbage collection cycle.", 963 Objectives: map[float64]float64{}, 964 }) 965 m.snapshotDuration = promauto.With(r).NewSummary(prometheus.SummaryOpts{ 966 Name: "alertmanager_silences_snapshot_duration_seconds", 967 Help: "Duration of the last silence snapshot.", 968 Objectives: map[float64]float64{}, 969 }) 970 m.snapshotSize = promauto.With(r).NewGauge(prometheus.GaugeOpts{ 971 Name: "alertmanager_silences_snapshot_size_bytes", 972 Help: "Size of the last silence snapshot in bytes.", 973 }) 974 m.queriesTotal = promauto.With(r).NewCounter(prometheus.CounterOpts{ 975 Name: "alertmanager_silences_queries_total", 976 Help: "How many silence queries were received.", 977 }) 978 m.queryErrorsTotal = promauto.With(r).NewCounter(prometheus.CounterOpts{ 979 Name: "alertmanager_silences_query_errors_total", 980 Help: "How many silence received queries did not succeed.", 981 }) 982 m.queryDuration = promauto.With(r).NewHistogram(prometheus.HistogramOpts{ 983 Name: "alertmanager_silences_query_duration_seconds", 984 Help: "Duration of silence query evaluation.", 985 }) 986 m.propagatedMessagesTotal = promauto.With(r).NewCounter(prometheus.CounterOpts{ 987 Name: "alertmanager_silences_gossip_messages_propagated_total", 988 Help: "Number of received gossip messages that have been further gossiped.", 989 }) 990 m.silencesActive = promauto.With(r).NewGauge(prometheus.GaugeOpts{ 991 Name: "alertmanager_silences", 992 Help: "How many silences by state.", 993 ConstLabels: prometheus.Labels{"state": string(types.SilenceStateActive)}, 994 }) 995 m.silencesPending = promauto.With(r).NewGauge(prometheus.GaugeOpts{ 996 Name: "alertmanager_silences", 997 Help: "How many silences by state.", 998 ConstLabels: prometheus.Labels{"state": string(types.SilenceStatePending)}, 999 }) 1000 m.silencesExpired = promauto.With(r).NewGauge(prometheus.GaugeOpts{ 1001 Name: "alertmanager_silences", 1002 Help: "How many silences by state.", 1003 ConstLabels: prometheus.Labels{"state": string(types.SilenceStateExpired)}, 1004 }) 1005 1006 return m 1007 } 1008 1009 // Copied from github.com/alertmanager/notify/notify.go 1010 type notifyMetrics struct { 1011 numNotifications *prometheus.CounterVec 1012 numFailedNotifications *prometheus.CounterVec 1013 numNotificationRequestsTotal *prometheus.CounterVec 1014 numNotificationRequestsFailedTotal *prometheus.CounterVec 1015 notificationLatencySeconds *prometheus.HistogramVec 1016 } 1017 1018 func newNotifyMetrics(r prometheus.Registerer) *notifyMetrics { 1019 m := ¬ifyMetrics{ 1020 numNotifications: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ 1021 Namespace: "alertmanager", 1022 Name: "notifications_total", 1023 Help: "The total number of attempted notifications.", 1024 }, []string{"integration"}), 1025 numFailedNotifications: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ 1026 Namespace: "alertmanager", 1027 Name: "notifications_failed_total", 1028 Help: "The total number of failed notifications.", 1029 }, []string{"integration"}), 1030 numNotificationRequestsTotal: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ 1031 Namespace: "alertmanager", 1032 Name: "notification_requests_total", 1033 Help: "The total number of attempted notification requests.", 1034 }, []string{"integration"}), 1035 numNotificationRequestsFailedTotal: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ 1036 Namespace: "alertmanager", 1037 Name: "notification_requests_failed_total", 1038 Help: "The total number of failed notification requests.", 1039 }, []string{"integration"}), 1040 notificationLatencySeconds: promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{ 1041 Namespace: "alertmanager", 1042 Name: "notification_latency_seconds", 1043 Help: "The latency of notifications in seconds.", 1044 Buckets: []float64{1, 5, 10, 15, 20}, 1045 }, []string{"integration"}), 1046 } 1047 for _, integration := range integrations { 1048 m.numNotifications.WithLabelValues(integration) 1049 m.numFailedNotifications.WithLabelValues(integration) 1050 m.numNotificationRequestsTotal.WithLabelValues(integration) 1051 m.numNotificationRequestsFailedTotal.WithLabelValues(integration) 1052 m.notificationLatencySeconds.WithLabelValues(integration) 1053 } 1054 return m 1055 } 1056 1057 type markerMetrics struct { 1058 alerts *prometheus.GaugeVec 1059 } 1060 1061 func newMarkerMetrics(r prometheus.Registerer) *markerMetrics { 1062 return &markerMetrics{ 1063 alerts: promauto.With(r).NewGaugeVec(prometheus.GaugeOpts{ 1064 Name: "alertmanager_alerts", 1065 Help: "How many alerts by state.", 1066 }, []string{"state"}), 1067 } 1068 } 1069 1070 // Copied from github.com/alertmanager/api/metrics/metrics.go 1071 type apiMetrics struct { 1072 firing prometheus.Counter 1073 resolved prometheus.Counter 1074 invalid prometheus.Counter 1075 } 1076 1077 func newAPIMetrics(version string, r prometheus.Registerer) *apiMetrics { 1078 numReceivedAlerts := promauto.With(r).NewCounterVec(prometheus.CounterOpts{ 1079 Name: "alertmanager_alerts_received_total", 1080 Help: "The total number of received alerts.", 1081 ConstLabels: prometheus.Labels{"version": version}, 1082 }, []string{"status"}) 1083 numInvalidAlerts := promauto.With(r).NewCounter(prometheus.CounterOpts{ 1084 Name: "alertmanager_alerts_invalid_total", 1085 Help: "The total number of received alerts that were invalid.", 1086 ConstLabels: prometheus.Labels{"version": version}, 1087 }) 1088 1089 return &apiMetrics{ 1090 firing: numReceivedAlerts.WithLabelValues("firing"), 1091 resolved: numReceivedAlerts.WithLabelValues("resolved"), 1092 invalid: numInvalidAlerts, 1093 } 1094 } 1095 1096 type limiterMetrics struct { 1097 count prometheus.Gauge 1098 size prometheus.Gauge 1099 insertFailures prometheus.Counter 1100 } 1101 1102 func newLimiterMetrics(r prometheus.Registerer) *limiterMetrics { 1103 count := promauto.With(r).NewGauge(prometheus.GaugeOpts{ 1104 Name: "alertmanager_alerts_limiter_current_alerts", 1105 Help: "Number of alerts tracked by alerts limiter.", 1106 }) 1107 1108 size := promauto.With(r).NewGauge(prometheus.GaugeOpts{ 1109 Name: "alertmanager_alerts_limiter_current_alerts_size_bytes", 1110 Help: "Total size of alerts tracked by alerts limiter.", 1111 }) 1112 1113 insertAlertFailures := promauto.With(r).NewCounter(prometheus.CounterOpts{ 1114 Name: "alertmanager_alerts_insert_limited_total", 1115 Help: "Number of failures to insert new alerts to in-memory alert store.", 1116 }) 1117 1118 return &limiterMetrics{ 1119 count: count, 1120 size: size, 1121 insertFailures: insertAlertFailures, 1122 } 1123 }