github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/production/loki-mixin-compiled/dashboards/loki-mixin-recording-rules.json (about)

     1  {
     2        "annotations": {
     3           "list": [
     4              {
     5                 "builtIn": 1,
     6                 "datasource": "-- Grafana --",
     7                 "enable": true,
     8                 "hide": true,
     9                 "iconColor": "rgba(0, 211, 255, 1)",
    10                 "name": "Annotations & Alerts",
    11                 "target": {
    12                    "limit": 100,
    13                    "matchAny": false,
    14                    "tags": [ ],
    15                    "type": "dashboard"
    16                 },
    17                 "type": "dashboard"
    18              },
    19              {
    20                 "datasource": "${datasource}",
    21                 "enable": false,
    22                 "expr": "sum by (tenant) (changes(loki_ruler_wal_prometheus_tsdb_wal_truncations_total{tenant=~\"${tenant}\"}[$__rate_interval]))",
    23                 "iconColor": "red",
    24                 "name": "WAL Truncations",
    25                 "target": {
    26                    "queryType": "Azure Monitor",
    27                    "refId": "Anno"
    28                 },
    29                 "titleFormat": "{{tenant}}"
    30              }
    31           ]
    32        },
    33        "editable": true,
    34        "fiscalYearStartMonth": 0,
    35        "gnetId": null,
    36        "graphTooltip": 0,
    37        "iteration": 1635347545534,
    38        "links": [ ],
    39        "liveNow": false,
    40        "panels": [
    41           {
    42              "datasource": "${datasource}",
    43              "fieldConfig": {
    44                 "defaults": {
    45                    "color": {
    46                       "mode": "thresholds"
    47                    },
    48                    "mappings": [ ],
    49                    "noValue": "0",
    50                    "thresholds": {
    51                       "mode": "absolute",
    52                       "steps": [
    53                          {
    54                             "color": "green",
    55                             "value": null
    56                          },
    57                          {
    58                             "color": "red",
    59                             "value": 1
    60                          }
    61                       ]
    62                    }
    63                 },
    64                 "overrides": [ ]
    65              },
    66              "gridPos": {
    67                 "h": 10,
    68                 "w": 2,
    69                 "x": 0,
    70                 "y": 0
    71              },
    72              "id": 2,
    73              "options": {
    74                 "colorMode": "value",
    75                 "graphMode": "area",
    76                 "justifyMode": "auto",
    77                 "orientation": "auto",
    78                 "reduceOptions": {
    79                    "calcs": [
    80                       "lastNotNull"
    81                    ],
    82                    "fields": "",
    83                    "values": false
    84                 },
    85                 "textMode": "auto"
    86              },
    87              "pluginVersion": "8.3.0-38205pre",
    88              "targets": [
    89                 {
    90                    "datasource": "${datasource}",
    91                    "exemplar": false,
    92                    "expr": "sum(loki_ruler_wal_appender_ready) by (pod, tenant) == 0",
    93                    "instant": true,
    94                    "interval": "",
    95                    "legendFormat": "",
    96                    "refId": "A"
    97                 }
    98              ],
    99              "title": "Appenders Not Ready",
   100              "type": "stat"
   101           },
   102           {
   103              "datasource": "${datasource}",
   104              "description": "",
   105              "fieldConfig": {
   106                 "defaults": {
   107                    "color": {
   108                       "mode": "palette-classic"
   109                    },
   110                    "custom": {
   111                       "axisLabel": "",
   112                       "axisPlacement": "auto",
   113                       "barAlignment": 0,
   114                       "drawStyle": "line",
   115                       "fillOpacity": 0,
   116                       "gradientMode": "none",
   117                       "hideFrom": {
   118                          "legend": false,
   119                          "tooltip": false,
   120                          "viz": false
   121                       },
   122                       "lineInterpolation": "linear",
   123                       "lineWidth": 1,
   124                       "pointSize": 5,
   125                       "scaleDistribution": {
   126                          "type": "linear"
   127                       },
   128                       "showPoints": "auto",
   129                       "spanNulls": false,
   130                       "stacking": {
   131                          "group": "A",
   132                          "mode": "none"
   133                       },
   134                       "thresholdsStyle": {
   135                          "mode": "off"
   136                       }
   137                    },
   138                    "mappings": [ ],
   139                    "thresholds": {
   140                       "mode": "absolute",
   141                       "steps": [
   142                          {
   143                             "color": "green",
   144                             "value": null
   145                          },
   146                          {
   147                             "color": "red",
   148                             "value": 80
   149                          }
   150                       ]
   151                    }
   152                 },
   153                 "overrides": [ ]
   154              },
   155              "gridPos": {
   156                 "h": 10,
   157                 "w": 11,
   158                 "x": 2,
   159                 "y": 0
   160              },
   161              "id": 4,
   162              "options": {
   163                 "legend": {
   164                    "calcs": [ ],
   165                    "displayMode": "list",
   166                    "placement": "bottom"
   167                 },
   168                 "tooltip": {
   169                    "mode": "single"
   170                 }
   171              },
   172              "targets": [
   173                 {
   174                    "datasource": "${datasource}",
   175                    "exemplar": true,
   176                    "expr": "sum(rate(loki_ruler_wal_samples_appended_total{tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0",
   177                    "interval": "",
   178                    "legendFormat": "{{tenant}}",
   179                    "refId": "A"
   180                 }
   181              ],
   182              "title": "Samples Appended to WAL per Second",
   183              "type": "timeseries"
   184           },
   185           {
   186              "datasource": "${datasource}",
   187              "description": "Series are unique combinations of labels",
   188              "fieldConfig": {
   189                 "defaults": {
   190                    "color": {
   191                       "mode": "palette-classic"
   192                    },
   193                    "custom": {
   194                       "axisLabel": "",
   195                       "axisPlacement": "auto",
   196                       "barAlignment": 0,
   197                       "drawStyle": "line",
   198                       "fillOpacity": 0,
   199                       "gradientMode": "none",
   200                       "hideFrom": {
   201                          "legend": false,
   202                          "tooltip": false,
   203                          "viz": false
   204                       },
   205                       "lineInterpolation": "linear",
   206                       "lineWidth": 1,
   207                       "pointSize": 5,
   208                       "scaleDistribution": {
   209                          "type": "linear"
   210                       },
   211                       "showPoints": "auto",
   212                       "spanNulls": false,
   213                       "stacking": {
   214                          "group": "A",
   215                          "mode": "none"
   216                       },
   217                       "thresholdsStyle": {
   218                          "mode": "off"
   219                       }
   220                    },
   221                    "mappings": [ ],
   222                    "thresholds": {
   223                       "mode": "absolute",
   224                       "steps": [
   225                          {
   226                             "color": "green",
   227                             "value": null
   228                          },
   229                          {
   230                             "color": "red",
   231                             "value": 80
   232                          }
   233                       ]
   234                    }
   235                 },
   236                 "overrides": [ ]
   237              },
   238              "gridPos": {
   239                 "h": 10,
   240                 "w": 11,
   241                 "x": 13,
   242                 "y": 0
   243              },
   244              "id": 5,
   245              "options": {
   246                 "legend": {
   247                    "calcs": [ ],
   248                    "displayMode": "list",
   249                    "placement": "bottom"
   250                 },
   251                 "tooltip": {
   252                    "mode": "single"
   253                 }
   254              },
   255              "targets": [
   256                 {
   257                    "datasource": "${datasource}",
   258                    "exemplar": true,
   259                    "expr": "sum(rate(loki_ruler_wal_storage_created_series_total{tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0",
   260                    "interval": "",
   261                    "legendFormat": "{{tenant}}",
   262                    "refId": "A"
   263                 }
   264              ],
   265              "title": "Series Created per Second",
   266              "type": "timeseries"
   267           },
   268           {
   269              "datasource": "${datasource}",
   270              "description": "Difference between highest timestamp appended to WAL and highest timestamp successfully written to remote storage",
   271              "fieldConfig": {
   272                 "defaults": {
   273                    "color": {
   274                       "mode": "palette-classic"
   275                    },
   276                    "custom": {
   277                       "axisLabel": "",
   278                       "axisPlacement": "auto",
   279                       "barAlignment": 0,
   280                       "drawStyle": "line",
   281                       "fillOpacity": 0,
   282                       "gradientMode": "none",
   283                       "hideFrom": {
   284                          "legend": false,
   285                          "tooltip": false,
   286                          "viz": false
   287                       },
   288                       "lineInterpolation": "linear",
   289                       "lineWidth": 1,
   290                       "pointSize": 5,
   291                       "scaleDistribution": {
   292                          "type": "linear"
   293                       },
   294                       "showPoints": "auto",
   295                       "spanNulls": false,
   296                       "stacking": {
   297                          "group": "A",
   298                          "mode": "none"
   299                       },
   300                       "thresholdsStyle": {
   301                          "mode": "off"
   302                       }
   303                    },
   304                    "mappings": [ ],
   305                    "thresholds": {
   306                       "mode": "absolute",
   307                       "steps": [
   308                          {
   309                             "color": "green",
   310                             "value": null
   311                          },
   312                          {
   313                             "color": "red",
   314                             "value": 80
   315                          }
   316                       ]
   317                    }
   318                 },
   319                 "overrides": [ ]
   320              },
   321              "gridPos": {
   322                 "h": 10,
   323                 "w": 12,
   324                 "x": 0,
   325                 "y": 10
   326              },
   327              "id": 6,
   328              "options": {
   329                 "legend": {
   330                    "calcs": [ ],
   331                    "displayMode": "list",
   332                    "placement": "bottom"
   333                 },
   334                 "tooltip": {
   335                    "mode": "single"
   336                 }
   337              },
   338              "targets": [
   339                 {
   340                    "datasource": "${datasource}",
   341                    "exemplar": true,
   342                    "expr": "loki_ruler_wal_prometheus_remote_storage_highest_timestamp_in_seconds{tenant=~\"${tenant}\"}\n- on (tenant)\n  (\n    loki_ruler_wal_prometheus_remote_storage_queue_highest_sent_timestamp_seconds{tenant=~\"${tenant}\"}\n    or vector(0)\n  )",
   343                    "interval": "",
   344                    "legendFormat": "{{tenant}}",
   345                    "refId": "A"
   346                 }
   347              ],
   348              "title": "Write Behind",
   349              "type": "timeseries"
   350           },
   351           {
   352              "datasource": "${datasource}",
   353              "description": "",
   354              "fieldConfig": {
   355                 "defaults": {
   356                    "color": {
   357                       "mode": "palette-classic"
   358                    },
   359                    "custom": {
   360                       "axisLabel": "",
   361                       "axisPlacement": "auto",
   362                       "barAlignment": 0,
   363                       "drawStyle": "line",
   364                       "fillOpacity": 0,
   365                       "gradientMode": "none",
   366                       "hideFrom": {
   367                          "legend": false,
   368                          "tooltip": false,
   369                          "viz": false
   370                       },
   371                       "lineInterpolation": "linear",
   372                       "lineWidth": 1,
   373                       "pointSize": 5,
   374                       "scaleDistribution": {
   375                          "type": "linear"
   376                       },
   377                       "showPoints": "auto",
   378                       "spanNulls": false,
   379                       "stacking": {
   380                          "group": "A",
   381                          "mode": "none"
   382                       },
   383                       "thresholdsStyle": {
   384                          "mode": "off"
   385                       }
   386                    },
   387                    "mappings": [ ],
   388                    "thresholds": {
   389                       "mode": "absolute",
   390                       "steps": [
   391                          {
   392                             "color": "green",
   393                             "value": null
   394                          },
   395                          {
   396                             "color": "red",
   397                             "value": 80
   398                          }
   399                       ]
   400                    }
   401                 },
   402                 "overrides": [ ]
   403              },
   404              "gridPos": {
   405                 "h": 10,
   406                 "w": 12,
   407                 "x": 12,
   408                 "y": 10
   409              },
   410              "id": 7,
   411              "options": {
   412                 "legend": {
   413                    "calcs": [ ],
   414                    "displayMode": "list",
   415                    "placement": "bottom"
   416                 },
   417                 "tooltip": {
   418                    "mode": "single"
   419                 }
   420              },
   421              "targets": [
   422                 {
   423                    "datasource": "${datasource}",
   424                    "exemplar": true,
   425                    "expr": "sum(rate(loki_ruler_wal_prometheus_remote_storage_samples_total{tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0",
   426                    "interval": "",
   427                    "legendFormat": "{{tenant}}",
   428                    "refId": "A"
   429                 }
   430              ],
   431              "title": "Samples Sent per Second",
   432              "type": "timeseries"
   433           },
   434           {
   435              "datasource": "${datasource}",
   436              "description": "\n",
   437              "fieldConfig": {
   438                 "defaults": {
   439                    "color": {
   440                       "mode": "palette-classic"
   441                    },
   442                    "custom": {
   443                       "axisLabel": "",
   444                       "axisPlacement": "auto",
   445                       "barAlignment": 0,
   446                       "drawStyle": "line",
   447                       "fillOpacity": 0,
   448                       "gradientMode": "none",
   449                       "hideFrom": {
   450                          "legend": false,
   451                          "tooltip": false,
   452                          "viz": false
   453                       },
   454                       "lineInterpolation": "linear",
   455                       "lineWidth": 1,
   456                       "pointSize": 5,
   457                       "scaleDistribution": {
   458                          "type": "linear"
   459                       },
   460                       "showPoints": "auto",
   461                       "spanNulls": false,
   462                       "stacking": {
   463                          "group": "A",
   464                          "mode": "none"
   465                       },
   466                       "thresholdsStyle": {
   467                          "mode": "off"
   468                       }
   469                    },
   470                    "mappings": [ ],
   471                    "thresholds": {
   472                       "mode": "absolute",
   473                       "steps": [
   474                          {
   475                             "color": "green",
   476                             "value": null
   477                          },
   478                          {
   479                             "color": "red",
   480                             "value": 80
   481                          }
   482                       ]
   483                    },
   484                    "unit": "bytes"
   485                 },
   486                 "overrides": [ ]
   487              },
   488              "gridPos": {
   489                 "h": 10,
   490                 "w": 12,
   491                 "x": 0,
   492                 "y": 20
   493              },
   494              "id": 8,
   495              "options": {
   496                 "legend": {
   497                    "calcs": [ ],
   498                    "displayMode": "list",
   499                    "placement": "bottom"
   500                 },
   501                 "tooltip": {
   502                    "mode": "single"
   503                 }
   504              },
   505              "targets": [
   506                 {
   507                    "datasource": "${datasource}",
   508                    "exemplar": true,
   509                    "expr": "sum by (tenant) (loki_ruler_wal_disk_size{tenant=~\"${tenant}\"})",
   510                    "interval": "",
   511                    "legendFormat": "{{tenant}}",
   512                    "refId": "A"
   513                 }
   514              ],
   515              "title": "WAL Disk Size",
   516              "type": "timeseries"
   517           },
   518           {
   519              "datasource": "${datasource}",
   520              "description": "Some number of pending samples is expected, but if remote-write is failing this value will remain high",
   521              "fieldConfig": {
   522                 "defaults": {
   523                    "color": {
   524                       "mode": "palette-classic"
   525                    },
   526                    "custom": {
   527                       "axisLabel": "",
   528                       "axisPlacement": "auto",
   529                       "barAlignment": 0,
   530                       "drawStyle": "line",
   531                       "fillOpacity": 0,
   532                       "gradientMode": "none",
   533                       "hideFrom": {
   534                          "legend": false,
   535                          "tooltip": false,
   536                          "viz": false
   537                       },
   538                       "lineInterpolation": "linear",
   539                       "lineWidth": 1,
   540                       "pointSize": 5,
   541                       "scaleDistribution": {
   542                          "type": "linear"
   543                       },
   544                       "showPoints": "auto",
   545                       "spanNulls": false,
   546                       "stacking": {
   547                          "group": "A",
   548                          "mode": "none"
   549                       },
   550                       "thresholdsStyle": {
   551                          "mode": "off"
   552                       }
   553                    },
   554                    "mappings": [ ],
   555                    "thresholds": {
   556                       "mode": "absolute",
   557                       "steps": [
   558                          {
   559                             "color": "green",
   560                             "value": null
   561                          },
   562                          {
   563                             "color": "red",
   564                             "value": 80
   565                          }
   566                       ]
   567                    }
   568                 },
   569                 "overrides": [ ]
   570              },
   571              "gridPos": {
   572                 "h": 10,
   573                 "w": 12,
   574                 "x": 12,
   575                 "y": 20
   576              },
   577              "id": 9,
   578              "options": {
   579                 "legend": {
   580                    "calcs": [ ],
   581                    "displayMode": "list",
   582                    "placement": "bottom"
   583                 },
   584                 "tooltip": {
   585                    "mode": "single"
   586                 }
   587              },
   588              "targets": [
   589                 {
   590                    "datasource": "${datasource}",
   591                    "exemplar": true,
   592                    "expr": "max(loki_ruler_wal_prometheus_remote_storage_samples_pending{tenant=~\"${tenant}\"}) by (tenant,pod) > 0",
   593                    "interval": "",
   594                    "legendFormat": "{{tenant}}",
   595                    "refId": "A"
   596                 }
   597              ],
   598              "title": "Pending Samples",
   599              "type": "timeseries"
   600           }
   601        ],
   602        "schemaVersion": 31,
   603        "style": "dark",
   604        "tags": [ ],
   605        "templating": {
   606           "list": [
   607              {
   608                 "description": null,
   609                 "error": null,
   610                 "hide": 0,
   611                 "includeAll": false,
   612                 "label": "Datasource",
   613                 "multi": false,
   614                 "name": "datasource",
   615                 "options": [ ],
   616                 "query": "prometheus",
   617                 "queryValue": "",
   618                 "refresh": 1,
   619                 "regex": "",
   620                 "skipUrlSync": false,
   621                 "type": "datasource"
   622              },
   623              {
   624                 "allValue": null,
   625                 "datasource": "${datasource}",
   626                 "definition": "label_values(loki_ruler_wal_samples_appended_total, tenant)",
   627                 "description": null,
   628                 "error": null,
   629                 "hide": 0,
   630                 "includeAll": true,
   631                 "label": "Tenant",
   632                 "multi": true,
   633                 "name": "tenant",
   634                 "options": [ ],
   635                 "query": {
   636                    "query": "label_values(loki_ruler_wal_samples_appended_total, tenant)",
   637                    "refId": "StandardVariableQuery"
   638                 },
   639                 "refresh": 2,
   640                 "regex": "",
   641                 "skipUrlSync": false,
   642                 "sort": 0,
   643                 "type": "query"
   644              }
   645           ]
   646        },
   647        "time": {
   648           "from": "now-6h",
   649           "to": "now"
   650        },
   651        "timepicker": { },
   652        "timezone": "",
   653        "title": "Recording Rules",
   654        "uid": "2xKA_ZK7k",
   655        "version": 9,
   656        "weekStart": ""
   657     }