github.com/thanos-io/thanos@v0.32.5/mixin/alerts/sidecar.libsonnet (about)

     1  {
     2    local thanos = self,
     3    sidecar+:: {
     4      selector: error 'must provide selector for Thanos Sidecar alerts',
     5      thanosPrometheusCommonDimensions: error 'must provide commonDimensions between Thanos and Prometheus metrics for Sidecar alerts',
     6      dimensions: std.join(', ', std.objectFields(thanos.targetGroups) + ['job', 'instance']),
     7    },
     8    prometheusAlerts+:: {
     9      groups+: if thanos.sidecar == null then [] else [
    10        local location = if std.length(std.objectFields(thanos.targetGroups)) > 0 then ' in %s' % std.join('/', ['{{$labels.%s}}' % level for level in std.objectFields(thanos.targetGroups)]) else '';
    11        {
    12          name: 'thanos-sidecar',
    13          rules: [
    14            {
    15              alert: 'ThanosSidecarBucketOperationsFailed',
    16              annotations: {
    17                description: 'Thanos Sidecar {{$labels.instance}}%s bucket operations are failing' % location,
    18                summary: 'Thanos Sidecar bucket operations are failing',
    19              },
    20              expr: |||
    21                sum by (%(dimensions)s) (rate(thanos_objstore_bucket_operation_failures_total{%(selector)s}[5m])) > 0
    22              ||| % thanos.sidecar,
    23              'for': '5m',
    24              labels: {
    25                severity: 'critical',
    26              },
    27            },
    28            {
    29              alert: 'ThanosSidecarNoConnectionToStartedPrometheus',
    30              annotations: {
    31                description: 'Thanos Sidecar {{$labels.instance}}%s is unhealthy.' % location,
    32                summary: 'Thanos Sidecar cannot access Prometheus, even though Prometheus seems healthy and has reloaded WAL.',
    33              },
    34              expr: |||
    35                thanos_sidecar_prometheus_up{%(selector)s} == 0
    36                AND on (%(thanosPrometheusCommonDimensions)s)
    37                prometheus_tsdb_data_replay_duration_seconds != 0
    38              ||| % thanos.sidecar,
    39              'for': '5m',
    40              labels: {
    41                severity: 'critical',
    42              },
    43            },
    44          ],
    45        },
    46      ],
    47    },
    48  }