github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/production/promtail-mixin/alerts.libsonnet (about)

     1  {
     2    prometheusAlerts+:: {
     3      groups+: [
     4        {
     5          name: 'promtail_alerts',
     6          rules: [
     7            {
     8              alert: 'PromtailRequestsErrors',
     9              expr: |||
    10                100 * sum(rate(promtail_request_duration_seconds_count{status_code=~"5..|failed"}[1m])) by (namespace, job, route, instance)
    11                  /
    12                sum(rate(promtail_request_duration_seconds_count[1m])) by (namespace, job, route, instance)
    13                  > 10
    14              |||,
    15              'for': '15m',
    16              labels: {
    17                severity: 'critical',
    18              },
    19              annotations: {
    20                message: |||
    21                  {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}% errors.
    22                |||,
    23              },
    24            },
    25            {
    26              alert: 'PromtailRequestLatency',
    27              expr: |||
    28                job_status_code_namespace:promtail_request_duration_seconds:99quantile > 1
    29              |||,
    30              'for': '15m',
    31              labels: {
    32                severity: 'critical',
    33              },
    34              annotations: {
    35                message: |||
    36                  {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}s 99th percentile latency.
    37                |||,
    38              },
    39            },
    40            {
    41              alert: 'PromtailFileMissing',
    42              expr: |||
    43                promtail_file_bytes_total unless promtail_read_bytes_total
    44              |||,
    45              'for': '15m',
    46              labels: {
    47                severity: 'critical',
    48              },
    49              annotations: {
    50                message: |||
    51                  {{ $labels.instance }} {{ $labels.job }} {{ $labels.path }} matches the glob but is not being tailed.
    52                |||,
    53              },
    54            },
    55          ],
    56        },
    57      ],
    58    },
    59  }