github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/production/loki-mixin/dashboards/dashboard-recording-rules.json (about)

     1  {
     2    "annotations": {
     3      "list": [
     4        {
     5          "builtIn": 1,
     6          "datasource": "-- Grafana --",
     7          "enable": true,
     8          "hide": true,
     9          "iconColor": "rgba(0, 211, 255, 1)",
    10          "name": "Annotations & Alerts",
    11          "target": {
    12            "limit": 100,
    13            "matchAny": false,
    14            "tags": [],
    15            "type": "dashboard"
    16          },
    17          "type": "dashboard"
    18        },
    19        {
    20          "datasource": "${datasource}",
    21          "enable": false,
    22          "expr": "sum by (tenant) (changes(loki_ruler_wal_prometheus_tsdb_wal_truncations_total{tenant=~\"${tenant}\"}[$__rate_interval]))",
    23          "iconColor": "red",
    24          "name": "WAL Truncations",
    25          "target": {
    26            "queryType": "Azure Monitor",
    27            "refId": "Anno"
    28          },
    29          "titleFormat": "{{tenant}}"
    30        }
    31      ]
    32    },
    33    "editable": true,
    34    "fiscalYearStartMonth": 0,
    35    "gnetId": null,
    36    "graphTooltip": 0,
    37    "iteration": 1635347545534,
    38    "links": [],
    39    "liveNow": false,
    40    "panels": [
    41      {
    42        "datasource": "${datasource}",
    43        "fieldConfig": {
    44          "defaults": {
    45            "color": {
    46              "mode": "thresholds"
    47            },
    48            "mappings": [],
    49            "noValue": "0",
    50            "thresholds": {
    51              "mode": "absolute",
    52              "steps": [
    53                {
    54                  "color": "green",
    55                  "value": null
    56                },
    57                {
    58                  "color": "red",
    59                  "value": 1
    60                }
    61              ]
    62            }
    63          },
    64          "overrides": []
    65        },
    66        "gridPos": {
    67          "h": 10,
    68          "w": 2,
    69          "x": 0,
    70          "y": 0
    71        },
    72        "id": 2,
    73        "options": {
    74          "colorMode": "value",
    75          "graphMode": "area",
    76          "justifyMode": "auto",
    77          "orientation": "auto",
    78          "reduceOptions": {
    79            "calcs": [
    80              "lastNotNull"
    81            ],
    82            "fields": "",
    83            "values": false
    84          },
    85          "textMode": "auto"
    86        },
    87        "pluginVersion": "8.3.0-38205pre",
    88        "targets": [
    89          {
    90            "datasource": "${datasource}",
    91            "exemplar": false,
    92            "expr": "sum(loki_ruler_wal_appender_ready) by (pod, tenant) == 0",
    93            "instant": true,
    94            "interval": "",
    95            "legendFormat": "",
    96            "refId": "A"
    97          }
    98        ],
    99        "title": "Appenders Not Ready",
   100        "type": "stat"
   101      },
   102      {
   103        "datasource": "${datasource}",
   104        "description": "",
   105        "fieldConfig": {
   106          "defaults": {
   107            "color": {
   108              "mode": "palette-classic"
   109            },
   110            "custom": {
   111              "axisLabel": "",
   112              "axisPlacement": "auto",
   113              "barAlignment": 0,
   114              "drawStyle": "line",
   115              "fillOpacity": 0,
   116              "gradientMode": "none",
   117              "hideFrom": {
   118                "legend": false,
   119                "tooltip": false,
   120                "viz": false
   121              },
   122              "lineInterpolation": "linear",
   123              "lineWidth": 1,
   124              "pointSize": 5,
   125              "scaleDistribution": {
   126                "type": "linear"
   127              },
   128              "showPoints": "auto",
   129              "spanNulls": false,
   130              "stacking": {
   131                "group": "A",
   132                "mode": "none"
   133              },
   134              "thresholdsStyle": {
   135                "mode": "off"
   136              }
   137            },
   138            "mappings": [],
   139            "thresholds": {
   140              "mode": "absolute",
   141              "steps": [
   142                {
   143                  "color": "green",
   144                  "value": null
   145                },
   146                {
   147                  "color": "red",
   148                  "value": 80
   149                }
   150              ]
   151            }
   152          },
   153          "overrides": []
   154        },
   155        "gridPos": {
   156          "h": 10,
   157          "w": 11,
   158          "x": 2,
   159          "y": 0
   160        },
   161        "id": 4,
   162        "options": {
   163          "legend": {
   164            "calcs": [],
   165            "displayMode": "list",
   166            "placement": "bottom"
   167          },
   168          "tooltip": {
   169            "mode": "single"
   170          }
   171        },
   172        "targets": [
   173          {
   174            "datasource": "${datasource}",
   175            "exemplar": true,
   176            "expr": "sum(rate(loki_ruler_wal_samples_appended_total{tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0",
   177            "interval": "",
   178            "legendFormat": "{{tenant}}",
   179            "refId": "A"
   180          }
   181        ],
   182        "title": "Samples Appended to WAL per Second",
   183        "type": "timeseries"
   184      },
   185      {
   186        "datasource": "${datasource}",
   187        "description": "Series are unique combinations of labels",
   188        "fieldConfig": {
   189          "defaults": {
   190            "color": {
   191              "mode": "palette-classic"
   192            },
   193            "custom": {
   194              "axisLabel": "",
   195              "axisPlacement": "auto",
   196              "barAlignment": 0,
   197              "drawStyle": "line",
   198              "fillOpacity": 0,
   199              "gradientMode": "none",
   200              "hideFrom": {
   201                "legend": false,
   202                "tooltip": false,
   203                "viz": false
   204              },
   205              "lineInterpolation": "linear",
   206              "lineWidth": 1,
   207              "pointSize": 5,
   208              "scaleDistribution": {
   209                "type": "linear"
   210              },
   211              "showPoints": "auto",
   212              "spanNulls": false,
   213              "stacking": {
   214                "group": "A",
   215                "mode": "none"
   216              },
   217              "thresholdsStyle": {
   218                "mode": "off"
   219              }
   220            },
   221            "mappings": [],
   222            "thresholds": {
   223              "mode": "absolute",
   224              "steps": [
   225                {
   226                  "color": "green",
   227                  "value": null
   228                },
   229                {
   230                  "color": "red",
   231                  "value": 80
   232                }
   233              ]
   234            }
   235          },
   236          "overrides": []
   237        },
   238        "gridPos": {
   239          "h": 10,
   240          "w": 11,
   241          "x": 13,
   242          "y": 0
   243        },
   244        "id": 5,
   245        "options": {
   246          "legend": {
   247            "calcs": [],
   248            "displayMode": "list",
   249            "placement": "bottom"
   250          },
   251          "tooltip": {
   252            "mode": "single"
   253          }
   254        },
   255        "targets": [
   256          {
   257            "datasource": "${datasource}",
   258            "exemplar": true,
   259            "expr": "sum(rate(loki_ruler_wal_storage_created_series_total{tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0",
   260            "interval": "",
   261            "legendFormat": "{{tenant}}",
   262            "refId": "A"
   263          }
   264        ],
   265        "title": "Series Created per Second",
   266        "type": "timeseries"
   267      },
   268      {
   269        "datasource": "${datasource}",
   270        "description": "Difference between highest timestamp appended to WAL and highest timestamp successfully written to remote storage",
   271        "fieldConfig": {
   272          "defaults": {
   273            "color": {
   274              "mode": "palette-classic"
   275            },
   276            "custom": {
   277              "axisLabel": "",
   278              "axisPlacement": "auto",
   279              "barAlignment": 0,
   280              "drawStyle": "line",
   281              "fillOpacity": 0,
   282              "gradientMode": "none",
   283              "hideFrom": {
   284                "legend": false,
   285                "tooltip": false,
   286                "viz": false
   287              },
   288              "lineInterpolation": "linear",
   289              "lineWidth": 1,
   290              "pointSize": 5,
   291              "scaleDistribution": {
   292                "type": "linear"
   293              },
   294              "showPoints": "auto",
   295              "spanNulls": false,
   296              "stacking": {
   297                "group": "A",
   298                "mode": "none"
   299              },
   300              "thresholdsStyle": {
   301                "mode": "off"
   302              }
   303            },
   304            "mappings": [],
   305            "thresholds": {
   306              "mode": "absolute",
   307              "steps": [
   308                {
   309                  "color": "green",
   310                  "value": null
   311                },
   312                {
   313                  "color": "red",
   314                  "value": 80
   315                }
   316              ]
   317            }
   318          },
   319          "overrides": []
   320        },
   321        "gridPos": {
   322          "h": 10,
   323          "w": 12,
   324          "x": 0,
   325          "y": 10
   326        },
   327        "id": 6,
   328        "options": {
   329          "legend": {
   330            "calcs": [],
   331            "displayMode": "list",
   332            "placement": "bottom"
   333          },
   334          "tooltip": {
   335            "mode": "single"
   336          }
   337        },
   338        "targets": [
   339          {
   340            "datasource": "${datasource}",
   341            "exemplar": true,
   342            "expr": "loki_ruler_wal_prometheus_remote_storage_highest_timestamp_in_seconds{tenant=~\"${tenant}\"}\n- on (tenant)\n  (\n    loki_ruler_wal_prometheus_remote_storage_queue_highest_sent_timestamp_seconds{tenant=~\"${tenant}\"}\n    or vector(0)\n  )",
   343            "interval": "",
   344            "legendFormat": "{{tenant}}",
   345            "refId": "A"
   346          }
   347        ],
   348        "title": "Write Behind",
   349        "type": "timeseries"
   350      },
   351      {
   352        "datasource": "${datasource}",
   353        "description": "",
   354        "fieldConfig": {
   355          "defaults": {
   356            "color": {
   357              "mode": "palette-classic"
   358            },
   359            "custom": {
   360              "axisLabel": "",
   361              "axisPlacement": "auto",
   362              "barAlignment": 0,
   363              "drawStyle": "line",
   364              "fillOpacity": 0,
   365              "gradientMode": "none",
   366              "hideFrom": {
   367                "legend": false,
   368                "tooltip": false,
   369                "viz": false
   370              },
   371              "lineInterpolation": "linear",
   372              "lineWidth": 1,
   373              "pointSize": 5,
   374              "scaleDistribution": {
   375                "type": "linear"
   376              },
   377              "showPoints": "auto",
   378              "spanNulls": false,
   379              "stacking": {
   380                "group": "A",
   381                "mode": "none"
   382              },
   383              "thresholdsStyle": {
   384                "mode": "off"
   385              }
   386            },
   387            "mappings": [],
   388            "thresholds": {
   389              "mode": "absolute",
   390              "steps": [
   391                {
   392                  "color": "green",
   393                  "value": null
   394                },
   395                {
   396                  "color": "red",
   397                  "value": 80
   398                }
   399              ]
   400            }
   401          },
   402          "overrides": []
   403        },
   404        "gridPos": {
   405          "h": 10,
   406          "w": 12,
   407          "x": 12,
   408          "y": 10
   409        },
   410        "id": 7,
   411        "options": {
   412          "legend": {
   413            "calcs": [],
   414            "displayMode": "list",
   415            "placement": "bottom"
   416          },
   417          "tooltip": {
   418            "mode": "single"
   419          }
   420        },
   421        "targets": [
   422          {
   423            "datasource": "${datasource}",
   424            "exemplar": true,
   425            "expr": "sum(rate(loki_ruler_wal_prometheus_remote_storage_samples_total{tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0",
   426            "interval": "",
   427            "legendFormat": "{{tenant}}",
   428            "refId": "A"
   429          }
   430        ],
   431        "title": "Samples Sent per Second",
   432        "type": "timeseries"
   433      },
   434      {
   435        "datasource": "${datasource}",
   436        "description": "\n",
   437        "fieldConfig": {
   438          "defaults": {
   439            "color": {
   440              "mode": "palette-classic"
   441            },
   442            "custom": {
   443              "axisLabel": "",
   444              "axisPlacement": "auto",
   445              "barAlignment": 0,
   446              "drawStyle": "line",
   447              "fillOpacity": 0,
   448              "gradientMode": "none",
   449              "hideFrom": {
   450                "legend": false,
   451                "tooltip": false,
   452                "viz": false
   453              },
   454              "lineInterpolation": "linear",
   455              "lineWidth": 1,
   456              "pointSize": 5,
   457              "scaleDistribution": {
   458                "type": "linear"
   459              },
   460              "showPoints": "auto",
   461              "spanNulls": false,
   462              "stacking": {
   463                "group": "A",
   464                "mode": "none"
   465              },
   466              "thresholdsStyle": {
   467                "mode": "off"
   468              }
   469            },
   470            "mappings": [],
   471            "thresholds": {
   472              "mode": "absolute",
   473              "steps": [
   474                {
   475                  "color": "green",
   476                  "value": null
   477                },
   478                {
   479                  "color": "red",
   480                  "value": 80
   481                }
   482              ]
   483            },
   484            "unit": "bytes"
   485          },
   486          "overrides": []
   487        },
   488        "gridPos": {
   489          "h": 10,
   490          "w": 12,
   491          "x": 0,
   492          "y": 20
   493        },
   494        "id": 8,
   495        "options": {
   496          "legend": {
   497            "calcs": [],
   498            "displayMode": "list",
   499            "placement": "bottom"
   500          },
   501          "tooltip": {
   502            "mode": "single"
   503          }
   504        },
   505        "targets": [
   506          {
   507            "datasource": "${datasource}",
   508            "exemplar": true,
   509            "expr": "sum by (tenant) (loki_ruler_wal_disk_size{tenant=~\"${tenant}\"})",
   510            "interval": "",
   511            "legendFormat": "{{tenant}}",
   512            "refId": "A"
   513          }
   514        ],
   515        "title": "WAL Disk Size",
   516        "type": "timeseries"
   517      },
   518      {
   519        "datasource": "${datasource}",
   520        "description": "Some number of pending samples is expected, but if remote-write is failing this value will remain high",
   521        "fieldConfig": {
   522          "defaults": {
   523            "color": {
   524              "mode": "palette-classic"
   525            },
   526            "custom": {
   527              "axisLabel": "",
   528              "axisPlacement": "auto",
   529              "barAlignment": 0,
   530              "drawStyle": "line",
   531              "fillOpacity": 0,
   532              "gradientMode": "none",
   533              "hideFrom": {
   534                "legend": false,
   535                "tooltip": false,
   536                "viz": false
   537              },
   538              "lineInterpolation": "linear",
   539              "lineWidth": 1,
   540              "pointSize": 5,
   541              "scaleDistribution": {
   542                "type": "linear"
   543              },
   544              "showPoints": "auto",
   545              "spanNulls": false,
   546              "stacking": {
   547                "group": "A",
   548                "mode": "none"
   549              },
   550              "thresholdsStyle": {
   551                "mode": "off"
   552              }
   553            },
   554            "mappings": [],
   555            "thresholds": {
   556              "mode": "absolute",
   557              "steps": [
   558                {
   559                  "color": "green",
   560                  "value": null
   561                },
   562                {
   563                  "color": "red",
   564                  "value": 80
   565                }
   566              ]
   567            }
   568          },
   569          "overrides": []
   570        },
   571        "gridPos": {
   572          "h": 10,
   573          "w": 12,
   574          "x": 12,
   575          "y": 20
   576        },
   577        "id": 9,
   578        "options": {
   579          "legend": {
   580            "calcs": [],
   581            "displayMode": "list",
   582            "placement": "bottom"
   583          },
   584          "tooltip": {
   585            "mode": "single"
   586          }
   587        },
   588        "targets": [
   589          {
   590            "datasource": "${datasource}",
   591            "exemplar": true,
   592            "expr": "max(loki_ruler_wal_prometheus_remote_storage_samples_pending{tenant=~\"${tenant}\"}) by (tenant,pod) > 0",
   593            "interval": "",
   594            "legendFormat": "{{tenant}}",
   595            "refId": "A"
   596          }
   597        ],
   598        "title": "Pending Samples",
   599        "type": "timeseries"
   600      }
   601    ],
   602    "schemaVersion": 31,
   603    "style": "dark",
   604    "tags": [],
   605    "templating": {
   606      "list": [
   607        {
   608          "description": null,
   609          "error": null,
   610          "hide": 0,
   611          "includeAll": false,
   612          "label": "Datasource",
   613          "multi": false,
   614          "name": "datasource",
   615          "options": [],
   616          "query": "prometheus",
   617          "queryValue": "",
   618          "refresh": 1,
   619          "regex": "",
   620          "skipUrlSync": false,
   621          "type": "datasource"
   622        },
   623        {
   624          "allValue": null,
   625          "datasource": "${datasource}",
   626          "definition": "label_values(loki_ruler_wal_samples_appended_total, tenant)",
   627          "description": null,
   628          "error": null,
   629          "hide": 0,
   630          "includeAll": true,
   631          "label": "Tenant",
   632          "multi": true,
   633          "name": "tenant",
   634          "options": [],
   635          "query": {
   636            "query": "label_values(loki_ruler_wal_samples_appended_total, tenant)",
   637            "refId": "StandardVariableQuery"
   638          },
   639          "refresh": 2,
   640          "regex": "",
   641          "skipUrlSync": false,
   642          "sort": 0,
   643          "type": "query"
   644        }
   645      ]
   646    },
   647    "time": {
   648      "from": "now-6h",
   649      "to": "now"
   650    },
   651    "timepicker": {},
   652    "timezone": "",
   653    "title": "Recording Rules",
   654    "uid": "2xKA_ZK7k",
   655    "version": 9,
   656    "weekStart": ""
   657  }