k8s.io/test-infra@v0.0.0-20240520184403-27c6b4c223d8/config/prow/cluster/monitoring/mixins/prometheus/sinker_alerts.libsonnet (about)

     1  {
     2    prometheusAlerts+:: {
     3      local componentName = $._config.components.sinker,
     4      groups+: [
     5        {
     6          name: 'sinker-missing',
     7          rules: [
     8            {
     9              alert: 'SinkerNotRemovingPods',
    10              expr: |||
    11                absent(sum(rate(sinker_pods_removed[1h]))) == 1
    12              |||,
    13              'for': '5m',
    14              labels: {
    15                severity: 'high',
    16                slo: componentName,
    17              },
    18              annotations: {
    19                message: 'Sinker has not removed any Pods in the last hour, likely indicating an outage in the service.',
    20              },
    21            },
    22            {
    23              alert: 'SinkerNotRemovingProwJobs',
    24              expr: |||
    25                absent(sum(rate(sinker_prow_jobs_cleaned[1h]))) == 1
    26              |||,
    27              'for': '5m',
    28              labels: {
    29                severity: 'high',
    30                slo: componentName,
    31              },
    32              annotations: {
    33                message: 'Sinker has not removed any Prow jobs in the last hour, likely indicating an outage in the service.',
    34              },
    35            }
    36          ],
    37        },
    38      ],
    39    },
    40  }