github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/production/loki-mixin-compiled/dashboards/loki-retention.json (about)

     1  {
     2        "annotations": {
     3           "list": [ ]
     4        },
     5        "editable": true,
     6        "gnetId": null,
     7        "graphTooltip": 0,
     8        "hideControls": false,
     9        "links": [
    10           {
    11              "asDropdown": true,
    12              "icon": "external link",
    13              "includeVars": true,
    14              "keepTime": true,
    15              "tags": [
    16                 "loki"
    17              ],
    18              "targetBlank": false,
    19              "title": "Loki Dashboards",
    20              "type": "dashboards"
    21           }
    22        ],
    23        "refresh": "10s",
    24        "rows": [
    25           {
    26              "collapse": false,
    27              "height": "250px",
    28              "panels": [
    29                 {
    30                    "aliasColors": { },
    31                    "bars": false,
    32                    "dashLength": 10,
    33                    "dashes": false,
    34                    "datasource": "$datasource",
    35                    "fill": 1,
    36                    "id": 1,
    37                    "legend": {
    38                       "avg": false,
    39                       "current": false,
    40                       "max": false,
    41                       "min": false,
    42                       "show": true,
    43                       "total": false,
    44                       "values": false
    45                    },
    46                    "lines": true,
    47                    "linewidth": 1,
    48                    "links": [ ],
    49                    "nullPointMode": "null as zero",
    50                    "percentage": false,
    51                    "pointradius": 5,
    52                    "points": false,
    53                    "renderer": "flot",
    54                    "seriesOverrides": [
    55                       {
    56                          "alias": "limit",
    57                          "color": "#E02F44",
    58                          "fill": 0
    59                       }
    60                    ],
    61                    "spaceLength": 10,
    62                    "span": 4,
    63                    "stack": false,
    64                    "steppedLine": false,
    65                    "targets": [
    66                       {
    67                          "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"}[$__rate_interval]))",
    68                          "format": "time_series",
    69                          "intervalFactor": 2,
    70                          "legendFormat": "{{pod}}",
    71                          "legendLink": null,
    72                          "step": 10
    73                       },
    74                       {
    75                          "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"})",
    76                          "format": "time_series",
    77                          "intervalFactor": 2,
    78                          "legendFormat": "limit",
    79                          "legendLink": null,
    80                          "step": 10
    81                       }
    82                    ],
    83                    "thresholds": [ ],
    84                    "timeFrom": null,
    85                    "timeShift": null,
    86                    "title": "CPU",
    87                    "tooltip": {
    88                       "sort": 2
    89                    },
    90                    "type": "graph",
    91                    "xaxis": {
    92                       "buckets": null,
    93                       "mode": "time",
    94                       "name": null,
    95                       "show": true,
    96                       "values": [ ]
    97                    },
    98                    "yaxes": [
    99                       {
   100                          "format": "short",
   101                          "label": null,
   102                          "logBase": 1,
   103                          "max": null,
   104                          "min": 0,
   105                          "show": true
   106                       },
   107                       {
   108                          "format": "short",
   109                          "label": null,
   110                          "logBase": 1,
   111                          "max": null,
   112                          "min": null,
   113                          "show": false
   114                       }
   115                    ]
   116                 },
   117                 {
   118                    "aliasColors": { },
   119                    "bars": false,
   120                    "dashLength": 10,
   121                    "dashes": false,
   122                    "datasource": "$datasource",
   123                    "fill": 1,
   124                    "id": 2,
   125                    "legend": {
   126                       "avg": false,
   127                       "current": false,
   128                       "max": false,
   129                       "min": false,
   130                       "show": true,
   131                       "total": false,
   132                       "values": false
   133                    },
   134                    "lines": true,
   135                    "linewidth": 1,
   136                    "links": [ ],
   137                    "nullPointMode": "null as zero",
   138                    "percentage": false,
   139                    "pointradius": 5,
   140                    "points": false,
   141                    "renderer": "flot",
   142                    "seriesOverrides": [
   143                       {
   144                          "alias": "limit",
   145                          "color": "#E02F44",
   146                          "fill": 0
   147                       }
   148                    ],
   149                    "spaceLength": 10,
   150                    "span": 4,
   151                    "stack": false,
   152                    "steppedLine": false,
   153                    "targets": [
   154                       {
   155                          "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"})",
   156                          "format": "time_series",
   157                          "intervalFactor": 2,
   158                          "legendFormat": "{{pod}}",
   159                          "legendLink": null,
   160                          "step": 10
   161                       },
   162                       {
   163                          "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} > 0)",
   164                          "format": "time_series",
   165                          "intervalFactor": 2,
   166                          "legendFormat": "limit",
   167                          "legendLink": null,
   168                          "step": 10
   169                       }
   170                    ],
   171                    "thresholds": [ ],
   172                    "timeFrom": null,
   173                    "timeShift": null,
   174                    "title": "Memory (workingset)",
   175                    "tooltip": {
   176                       "sort": 2
   177                    },
   178                    "type": "graph",
   179                    "xaxis": {
   180                       "buckets": null,
   181                       "mode": "time",
   182                       "name": null,
   183                       "show": true,
   184                       "values": [ ]
   185                    },
   186                    "yaxes": [
   187                       {
   188                          "format": "bytes",
   189                          "label": null,
   190                          "logBase": 1,
   191                          "max": null,
   192                          "min": 0,
   193                          "show": true
   194                       },
   195                       {
   196                          "format": "short",
   197                          "label": null,
   198                          "logBase": 1,
   199                          "max": null,
   200                          "min": null,
   201                          "show": false
   202                       }
   203                    ]
   204                 },
   205                 {
   206                    "aliasColors": { },
   207                    "bars": false,
   208                    "dashLength": 10,
   209                    "dashes": false,
   210                    "datasource": "$datasource",
   211                    "fill": 1,
   212                    "id": 3,
   213                    "legend": {
   214                       "avg": false,
   215                       "current": false,
   216                       "max": false,
   217                       "min": false,
   218                       "show": true,
   219                       "total": false,
   220                       "values": false
   221                    },
   222                    "lines": true,
   223                    "linewidth": 1,
   224                    "links": [ ],
   225                    "nullPointMode": "null as zero",
   226                    "percentage": false,
   227                    "pointradius": 5,
   228                    "points": false,
   229                    "renderer": "flot",
   230                    "seriesOverrides": [ ],
   231                    "spaceLength": 10,
   232                    "span": 4,
   233                    "stack": false,
   234                    "steppedLine": false,
   235                    "targets": [
   236                       {
   237                          "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/compactor\"})",
   238                          "format": "time_series",
   239                          "intervalFactor": 2,
   240                          "legendFormat": "{{pod}}",
   241                          "legendLink": null,
   242                          "step": 10
   243                       }
   244                    ],
   245                    "thresholds": [ ],
   246                    "timeFrom": null,
   247                    "timeShift": null,
   248                    "title": "Memory (go heap inuse)",
   249                    "tooltip": {
   250                       "sort": 2
   251                    },
   252                    "type": "graph",
   253                    "xaxis": {
   254                       "buckets": null,
   255                       "mode": "time",
   256                       "name": null,
   257                       "show": true,
   258                       "values": [ ]
   259                    },
   260                    "yaxes": [
   261                       {
   262                          "format": "bytes",
   263                          "label": null,
   264                          "logBase": 1,
   265                          "max": null,
   266                          "min": 0,
   267                          "show": true
   268                       },
   269                       {
   270                          "format": "short",
   271                          "label": null,
   272                          "logBase": 1,
   273                          "max": null,
   274                          "min": null,
   275                          "show": false
   276                       }
   277                    ]
   278                 }
   279              ],
   280              "repeat": null,
   281              "repeatIteration": null,
   282              "repeatRowId": null,
   283              "showTitle": true,
   284              "title": "Resource Usage",
   285              "titleSize": "h6"
   286           },
   287           {
   288              "collapse": false,
   289              "height": "250px",
   290              "panels": [
   291                 {
   292                    "aliasColors": { },
   293                    "bars": false,
   294                    "dashLength": 10,
   295                    "dashes": false,
   296                    "datasource": "$datasource",
   297                    "fieldConfig": {
   298                       "defaults": {
   299                          "color": {
   300                             "fixedColor": "blue",
   301                             "mode": "fixed"
   302                          },
   303                          "custom": { },
   304                          "thresholds": {
   305                             "mode": "absolute",
   306                             "steps": [
   307                                {
   308                                   "color": "green",
   309                                   "value": null
   310                                }
   311                             ]
   312                          },
   313                          "unit": "dateTimeFromNow"
   314                       }
   315                    },
   316                    "fill": 1,
   317                    "id": 4,
   318                    "legend": {
   319                       "avg": false,
   320                       "current": false,
   321                       "max": false,
   322                       "min": false,
   323                       "show": true,
   324                       "total": false,
   325                       "values": false
   326                    },
   327                    "lines": true,
   328                    "linewidth": 1,
   329                    "links": [ ],
   330                    "nullPointMode": "null as zero",
   331                    "options": {
   332                       "colorMode": "value",
   333                       "graphMode": "area",
   334                       "justifyMode": "auto",
   335                       "orientation": "auto",
   336                       "reduceOptions": {
   337                          "calcs": [
   338                             "lastNotNull"
   339                          ],
   340                          "fields": "",
   341                          "values": false
   342                       },
   343                       "text": { },
   344                       "textMode": "auto"
   345                    },
   346                    "percentage": false,
   347                    "pointradius": 5,
   348                    "points": false,
   349                    "renderer": "flot",
   350                    "seriesOverrides": [ ],
   351                    "spaceLength": 10,
   352                    "span": 4,
   353                    "stack": false,
   354                    "steppedLine": false,
   355                    "targets": [
   356                       {
   357                          "expr": "loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"} * 1e3",
   358                          "format": "time_series",
   359                          "instant": true,
   360                          "refId": "A"
   361                       }
   362                    ],
   363                    "thresholds": [ ],
   364                    "timeFrom": null,
   365                    "timeShift": null,
   366                    "title": "Last Compact and Mark Operation Success",
   367                    "tooltip": {
   368                       "shared": true,
   369                       "sort": 2,
   370                       "value_type": "individual"
   371                    },
   372                    "type": "stat",
   373                    "xaxis": {
   374                       "buckets": null,
   375                       "mode": "time",
   376                       "name": null,
   377                       "show": true,
   378                       "values": [ ]
   379                    },
   380                    "yaxes": [
   381                       {
   382                          "format": "short",
   383                          "label": null,
   384                          "logBase": 1,
   385                          "max": null,
   386                          "min": 0,
   387                          "show": true
   388                       },
   389                       {
   390                          "format": "short",
   391                          "label": null,
   392                          "logBase": 1,
   393                          "max": null,
   394                          "min": null,
   395                          "show": false
   396                       }
   397                    ]
   398                 },
   399                 {
   400                    "aliasColors": { },
   401                    "bars": false,
   402                    "dashLength": 10,
   403                    "dashes": false,
   404                    "datasource": "$datasource",
   405                    "fill": 1,
   406                    "id": 5,
   407                    "legend": {
   408                       "avg": false,
   409                       "current": false,
   410                       "max": false,
   411                       "min": false,
   412                       "show": true,
   413                       "total": false,
   414                       "values": false
   415                    },
   416                    "lines": true,
   417                    "linewidth": 1,
   418                    "links": [ ],
   419                    "nullPointMode": "null as zero",
   420                    "percentage": false,
   421                    "pointradius": 5,
   422                    "points": false,
   423                    "renderer": "flot",
   424                    "seriesOverrides": [ ],
   425                    "spaceLength": 10,
   426                    "span": 4,
   427                    "stack": false,
   428                    "steppedLine": false,
   429                    "targets": [
   430                       {
   431                          "expr": "loki_boltdb_shipper_compact_tables_operation_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}",
   432                          "format": "time_series",
   433                          "intervalFactor": 2,
   434                          "legendFormat": "duration",
   435                          "legendLink": null,
   436                          "step": 10
   437                       }
   438                    ],
   439                    "thresholds": [ ],
   440                    "timeFrom": null,
   441                    "timeShift": null,
   442                    "title": "Compact and Mark Operations Duration",
   443                    "tooltip": {
   444                       "shared": true,
   445                       "sort": 2,
   446                       "value_type": "individual"
   447                    },
   448                    "type": "graph",
   449                    "xaxis": {
   450                       "buckets": null,
   451                       "mode": "time",
   452                       "name": null,
   453                       "show": true,
   454                       "values": [ ]
   455                    },
   456                    "yaxes": [
   457                       {
   458                          "format": "s",
   459                          "label": null,
   460                          "logBase": 1,
   461                          "max": null,
   462                          "min": 0,
   463                          "show": true
   464                       },
   465                       {
   466                          "format": "short",
   467                          "label": null,
   468                          "logBase": 1,
   469                          "max": null,
   470                          "min": null,
   471                          "show": false
   472                       }
   473                    ]
   474                 },
   475                 {
   476                    "aliasColors": { },
   477                    "bars": false,
   478                    "dashLength": 10,
   479                    "dashes": false,
   480                    "datasource": "$datasource",
   481                    "fill": 1,
   482                    "id": 6,
   483                    "legend": {
   484                       "avg": false,
   485                       "current": false,
   486                       "max": false,
   487                       "min": false,
   488                       "show": true,
   489                       "total": false,
   490                       "values": false
   491                    },
   492                    "lines": true,
   493                    "linewidth": 1,
   494                    "links": [ ],
   495                    "nullPointMode": "null as zero",
   496                    "percentage": false,
   497                    "pointradius": 5,
   498                    "points": false,
   499                    "renderer": "flot",
   500                    "seriesOverrides": [ ],
   501                    "spaceLength": 10,
   502                    "span": 4,
   503                    "stack": false,
   504                    "steppedLine": false,
   505                    "targets": [
   506                       {
   507                          "expr": "sum by (status)(rate(loki_boltdb_shipper_compact_tables_operation_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
   508                          "format": "time_series",
   509                          "intervalFactor": 2,
   510                          "legendFormat": "{{success}}",
   511                          "legendLink": null,
   512                          "step": 10
   513                       }
   514                    ],
   515                    "thresholds": [ ],
   516                    "timeFrom": null,
   517                    "timeShift": null,
   518                    "title": "Compact and Mark Operations Per Status",
   519                    "tooltip": {
   520                       "shared": true,
   521                       "sort": 2,
   522                       "value_type": "individual"
   523                    },
   524                    "type": "graph",
   525                    "xaxis": {
   526                       "buckets": null,
   527                       "mode": "time",
   528                       "name": null,
   529                       "show": true,
   530                       "values": [ ]
   531                    },
   532                    "yaxes": [
   533                       {
   534                          "format": "short",
   535                          "label": null,
   536                          "logBase": 1,
   537                          "max": null,
   538                          "min": 0,
   539                          "show": true
   540                       },
   541                       {
   542                          "format": "short",
   543                          "label": null,
   544                          "logBase": 1,
   545                          "max": null,
   546                          "min": null,
   547                          "show": false
   548                       }
   549                    ]
   550                 }
   551              ],
   552              "repeat": null,
   553              "repeatIteration": null,
   554              "repeatRowId": null,
   555              "showTitle": true,
   556              "title": "Compact and Mark",
   557              "titleSize": "h6"
   558           },
   559           {
   560              "collapse": false,
   561              "height": "250px",
   562              "panels": [
   563                 {
   564                    "aliasColors": { },
   565                    "bars": false,
   566                    "dashLength": 10,
   567                    "dashes": false,
   568                    "datasource": "$datasource",
   569                    "fill": 10,
   570                    "id": 7,
   571                    "legend": {
   572                       "avg": false,
   573                       "current": false,
   574                       "max": false,
   575                       "min": false,
   576                       "show": true,
   577                       "total": false,
   578                       "values": false
   579                    },
   580                    "lines": true,
   581                    "linewidth": 0,
   582                    "links": [ ],
   583                    "nullPointMode": "null as zero",
   584                    "percentage": false,
   585                    "pointradius": 5,
   586                    "points": false,
   587                    "renderer": "flot",
   588                    "seriesOverrides": [ ],
   589                    "spaceLength": 10,
   590                    "span": 4,
   591                    "stack": true,
   592                    "steppedLine": false,
   593                    "targets": [
   594                       {
   595                          "expr": "count by(action)(loki_boltdb_shipper_retention_marker_table_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"})",
   596                          "format": "time_series",
   597                          "intervalFactor": 2,
   598                          "legendFormat": "{{action}}",
   599                          "legendLink": null,
   600                          "step": 10
   601                       }
   602                    ],
   603                    "thresholds": [ ],
   604                    "timeFrom": null,
   605                    "timeShift": null,
   606                    "title": "Processed Tables Per Action",
   607                    "tooltip": {
   608                       "shared": true,
   609                       "sort": 2,
   610                       "value_type": "individual"
   611                    },
   612                    "type": "graph",
   613                    "xaxis": {
   614                       "buckets": null,
   615                       "mode": "time",
   616                       "name": null,
   617                       "show": true,
   618                       "values": [ ]
   619                    },
   620                    "yaxes": [
   621                       {
   622                          "format": "short",
   623                          "label": null,
   624                          "logBase": 1,
   625                          "max": null,
   626                          "min": 0,
   627                          "show": true
   628                       },
   629                       {
   630                          "format": "short",
   631                          "label": null,
   632                          "logBase": 1,
   633                          "max": null,
   634                          "min": null,
   635                          "show": false
   636                       }
   637                    ]
   638                 },
   639                 {
   640                    "aliasColors": { },
   641                    "bars": false,
   642                    "dashLength": 10,
   643                    "dashes": false,
   644                    "datasource": "$datasource",
   645                    "fill": 10,
   646                    "id": 8,
   647                    "legend": {
   648                       "avg": false,
   649                       "current": false,
   650                       "max": false,
   651                       "min": false,
   652                       "show": true,
   653                       "total": false,
   654                       "values": false
   655                    },
   656                    "lines": true,
   657                    "linewidth": 0,
   658                    "links": [ ],
   659                    "nullPointMode": "null as zero",
   660                    "percentage": false,
   661                    "pointradius": 5,
   662                    "points": false,
   663                    "renderer": "flot",
   664                    "seriesOverrides": [ ],
   665                    "spaceLength": 10,
   666                    "span": 4,
   667                    "stack": true,
   668                    "steppedLine": false,
   669                    "targets": [
   670                       {
   671                          "expr": "count by(table,action)(loki_boltdb_shipper_retention_marker_table_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\" , action=~\"modified|deleted\"})",
   672                          "format": "time_series",
   673                          "intervalFactor": 2,
   674                          "legendFormat": "{{table}}-{{action}}",
   675                          "legendLink": null,
   676                          "step": 10
   677                       }
   678                    ],
   679                    "thresholds": [ ],
   680                    "timeFrom": null,
   681                    "timeShift": null,
   682                    "title": "Modified Tables",
   683                    "tooltip": {
   684                       "shared": true,
   685                       "sort": 2,
   686                       "value_type": "individual"
   687                    },
   688                    "type": "graph",
   689                    "xaxis": {
   690                       "buckets": null,
   691                       "mode": "time",
   692                       "name": null,
   693                       "show": true,
   694                       "values": [ ]
   695                    },
   696                    "yaxes": [
   697                       {
   698                          "format": "short",
   699                          "label": null,
   700                          "logBase": 1,
   701                          "max": null,
   702                          "min": 0,
   703                          "show": true
   704                       },
   705                       {
   706                          "format": "short",
   707                          "label": null,
   708                          "logBase": 1,
   709                          "max": null,
   710                          "min": null,
   711                          "show": false
   712                       }
   713                    ]
   714                 },
   715                 {
   716                    "aliasColors": { },
   717                    "bars": false,
   718                    "dashLength": 10,
   719                    "dashes": false,
   720                    "datasource": "$datasource",
   721                    "fill": 10,
   722                    "id": 9,
   723                    "legend": {
   724                       "avg": false,
   725                       "current": false,
   726                       "max": false,
   727                       "min": false,
   728                       "show": true,
   729                       "total": false,
   730                       "values": false
   731                    },
   732                    "lines": true,
   733                    "linewidth": 0,
   734                    "links": [ ],
   735                    "nullPointMode": "null as zero",
   736                    "percentage": false,
   737                    "pointradius": 5,
   738                    "points": false,
   739                    "renderer": "flot",
   740                    "seriesOverrides": [ ],
   741                    "spaceLength": 10,
   742                    "span": 4,
   743                    "stack": true,
   744                    "steppedLine": false,
   745                    "targets": [
   746                       {
   747                          "expr": "sum by (table)(rate(loki_boltdb_shipper_retention_marker_count_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) >0",
   748                          "format": "time_series",
   749                          "intervalFactor": 2,
   750                          "legendFormat": "{{table}}",
   751                          "legendLink": null,
   752                          "step": 10
   753                       }
   754                    ],
   755                    "thresholds": [ ],
   756                    "timeFrom": null,
   757                    "timeShift": null,
   758                    "title": "Marks Creation Rate Per Table",
   759                    "tooltip": {
   760                       "shared": true,
   761                       "sort": 2,
   762                       "value_type": "individual"
   763                    },
   764                    "type": "graph",
   765                    "xaxis": {
   766                       "buckets": null,
   767                       "mode": "time",
   768                       "name": null,
   769                       "show": true,
   770                       "values": [ ]
   771                    },
   772                    "yaxes": [
   773                       {
   774                          "format": "short",
   775                          "label": null,
   776                          "logBase": 1,
   777                          "max": null,
   778                          "min": 0,
   779                          "show": true
   780                       },
   781                       {
   782                          "format": "short",
   783                          "label": null,
   784                          "logBase": 1,
   785                          "max": null,
   786                          "min": null,
   787                          "show": false
   788                       }
   789                    ]
   790                 }
   791              ],
   792              "repeat": null,
   793              "repeatIteration": null,
   794              "repeatRowId": null,
   795              "showTitle": true,
   796              "title": "Per Table Marker",
   797              "titleSize": "h6"
   798           },
   799           {
   800              "collapse": false,
   801              "height": "250px",
   802              "panels": [
   803                 {
   804                    "aliasColors": { },
   805                    "bars": false,
   806                    "dashLength": 10,
   807                    "dashes": false,
   808                    "datasource": "$datasource",
   809                    "fill": 1,
   810                    "format": "short",
   811                    "id": 10,
   812                    "legend": {
   813                       "avg": false,
   814                       "current": false,
   815                       "max": false,
   816                       "min": false,
   817                       "show": true,
   818                       "total": false,
   819                       "values": false
   820                    },
   821                    "lines": true,
   822                    "linewidth": 1,
   823                    "links": [ ],
   824                    "nullPointMode": "null as zero",
   825                    "percentage": false,
   826                    "pointradius": 5,
   827                    "points": false,
   828                    "renderer": "flot",
   829                    "seriesOverrides": [ ],
   830                    "spaceLength": 10,
   831                    "span": 6,
   832                    "stack": false,
   833                    "steppedLine": false,
   834                    "targets": [
   835                       {
   836                          "expr": "sum (increase(loki_boltdb_shipper_retention_marker_count_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[24h]))",
   837                          "format": "time_series",
   838                          "instant": true,
   839                          "intervalFactor": 2,
   840                          "refId": "A"
   841                       }
   842                    ],
   843                    "thresholds": "70,80",
   844                    "timeFrom": null,
   845                    "timeShift": null,
   846                    "title": "Marked Chunks (24h)",
   847                    "tooltip": {
   848                       "shared": true,
   849                       "sort": 2,
   850                       "value_type": "individual"
   851                    },
   852                    "type": "singlestat",
   853                    "xaxis": {
   854                       "buckets": null,
   855                       "mode": "time",
   856                       "name": null,
   857                       "show": true,
   858                       "values": [ ]
   859                    },
   860                    "yaxes": [
   861                       {
   862                          "format": "short",
   863                          "label": null,
   864                          "logBase": 1,
   865                          "max": null,
   866                          "min": 0,
   867                          "show": true
   868                       },
   869                       {
   870                          "format": "short",
   871                          "label": null,
   872                          "logBase": 1,
   873                          "max": null,
   874                          "min": null,
   875                          "show": false
   876                       }
   877                    ]
   878                 },
   879                 {
   880                    "aliasColors": { },
   881                    "bars": false,
   882                    "dashLength": 10,
   883                    "dashes": false,
   884                    "datasource": "$datasource",
   885                    "fill": 1,
   886                    "id": 11,
   887                    "legend": {
   888                       "avg": false,
   889                       "current": false,
   890                       "max": false,
   891                       "min": false,
   892                       "show": true,
   893                       "total": false,
   894                       "values": false
   895                    },
   896                    "lines": true,
   897                    "linewidth": 1,
   898                    "links": [ ],
   899                    "nullPointMode": "null as zero",
   900                    "percentage": false,
   901                    "pointradius": 5,
   902                    "points": false,
   903                    "renderer": "flot",
   904                    "seriesOverrides": [ ],
   905                    "spaceLength": 10,
   906                    "span": 6,
   907                    "stack": false,
   908                    "steppedLine": false,
   909                    "targets": [
   910                       {
   911                          "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3",
   912                          "format": "time_series",
   913                          "intervalFactor": 2,
   914                          "legendFormat": "99th Percentile",
   915                          "refId": "A",
   916                          "step": 10
   917                       },
   918                       {
   919                          "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3",
   920                          "format": "time_series",
   921                          "intervalFactor": 2,
   922                          "legendFormat": "50th Percentile",
   923                          "refId": "B",
   924                          "step": 10
   925                       },
   926                       {
   927                          "expr": "sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
   928                          "format": "time_series",
   929                          "intervalFactor": 2,
   930                          "legendFormat": "Average",
   931                          "refId": "C",
   932                          "step": 10
   933                       }
   934                    ],
   935                    "thresholds": [ ],
   936                    "timeFrom": null,
   937                    "timeShift": null,
   938                    "title": "Mark Table Latency",
   939                    "tooltip": {
   940                       "shared": true,
   941                       "sort": 2,
   942                       "value_type": "individual"
   943                    },
   944                    "type": "graph",
   945                    "xaxis": {
   946                       "buckets": null,
   947                       "mode": "time",
   948                       "name": null,
   949                       "show": true,
   950                       "values": [ ]
   951                    },
   952                    "yaxes": [
   953                       {
   954                          "format": "ms",
   955                          "label": null,
   956                          "logBase": 1,
   957                          "max": null,
   958                          "min": 0,
   959                          "show": true
   960                       },
   961                       {
   962                          "format": "short",
   963                          "label": null,
   964                          "logBase": 1,
   965                          "max": null,
   966                          "min": null,
   967                          "show": false
   968                       }
   969                    ]
   970                 }
   971              ],
   972              "repeat": null,
   973              "repeatIteration": null,
   974              "repeatRowId": null,
   975              "showTitle": true,
   976              "title": "",
   977              "titleSize": "h6"
   978           },
   979           {
   980              "collapse": false,
   981              "height": "250px",
   982              "panels": [
   983                 {
   984                    "aliasColors": { },
   985                    "bars": false,
   986                    "dashLength": 10,
   987                    "dashes": false,
   988                    "datasource": "$datasource",
   989                    "fill": 1,
   990                    "format": "short",
   991                    "id": 12,
   992                    "legend": {
   993                       "avg": false,
   994                       "current": false,
   995                       "max": false,
   996                       "min": false,
   997                       "show": true,
   998                       "total": false,
   999                       "values": false
  1000                    },
  1001                    "lines": true,
  1002                    "linewidth": 1,
  1003                    "links": [ ],
  1004                    "nullPointMode": "null as zero",
  1005                    "percentage": false,
  1006                    "pointradius": 5,
  1007                    "points": false,
  1008                    "renderer": "flot",
  1009                    "seriesOverrides": [ ],
  1010                    "spaceLength": 10,
  1011                    "span": 6,
  1012                    "stack": false,
  1013                    "steppedLine": false,
  1014                    "targets": [
  1015                       {
  1016                          "expr": "sum (increase(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[24h]))",
  1017                          "format": "time_series",
  1018                          "instant": true,
  1019                          "intervalFactor": 2,
  1020                          "refId": "A"
  1021                       }
  1022                    ],
  1023                    "thresholds": "70,80",
  1024                    "timeFrom": null,
  1025                    "timeShift": null,
  1026                    "title": "Delete Chunks (24h)",
  1027                    "tooltip": {
  1028                       "shared": true,
  1029                       "sort": 2,
  1030                       "value_type": "individual"
  1031                    },
  1032                    "type": "singlestat",
  1033                    "xaxis": {
  1034                       "buckets": null,
  1035                       "mode": "time",
  1036                       "name": null,
  1037                       "show": true,
  1038                       "values": [ ]
  1039                    },
  1040                    "yaxes": [
  1041                       {
  1042                          "format": "short",
  1043                          "label": null,
  1044                          "logBase": 1,
  1045                          "max": null,
  1046                          "min": 0,
  1047                          "show": true
  1048                       },
  1049                       {
  1050                          "format": "short",
  1051                          "label": null,
  1052                          "logBase": 1,
  1053                          "max": null,
  1054                          "min": null,
  1055                          "show": false
  1056                       }
  1057                    ]
  1058                 },
  1059                 {
  1060                    "aliasColors": { },
  1061                    "bars": false,
  1062                    "dashLength": 10,
  1063                    "dashes": false,
  1064                    "datasource": "$datasource",
  1065                    "fill": 1,
  1066                    "id": 13,
  1067                    "legend": {
  1068                       "avg": false,
  1069                       "current": false,
  1070                       "max": false,
  1071                       "min": false,
  1072                       "show": true,
  1073                       "total": false,
  1074                       "values": false
  1075                    },
  1076                    "lines": true,
  1077                    "linewidth": 1,
  1078                    "links": [ ],
  1079                    "nullPointMode": "null as zero",
  1080                    "percentage": false,
  1081                    "pointradius": 5,
  1082                    "points": false,
  1083                    "renderer": "flot",
  1084                    "seriesOverrides": [ ],
  1085                    "spaceLength": 10,
  1086                    "span": 6,
  1087                    "stack": false,
  1088                    "steppedLine": false,
  1089                    "targets": [
  1090                       {
  1091                          "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3",
  1092                          "format": "time_series",
  1093                          "intervalFactor": 2,
  1094                          "legendFormat": "99th Percentile",
  1095                          "refId": "A",
  1096                          "step": 10
  1097                       },
  1098                       {
  1099                          "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3",
  1100                          "format": "time_series",
  1101                          "intervalFactor": 2,
  1102                          "legendFormat": "50th Percentile",
  1103                          "refId": "B",
  1104                          "step": 10
  1105                       },
  1106                       {
  1107                          "expr": "sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
  1108                          "format": "time_series",
  1109                          "intervalFactor": 2,
  1110                          "legendFormat": "Average",
  1111                          "refId": "C",
  1112                          "step": 10
  1113                       }
  1114                    ],
  1115                    "thresholds": [ ],
  1116                    "timeFrom": null,
  1117                    "timeShift": null,
  1118                    "title": "Delete Latency",
  1119                    "tooltip": {
  1120                       "shared": true,
  1121                       "sort": 2,
  1122                       "value_type": "individual"
  1123                    },
  1124                    "type": "graph",
  1125                    "xaxis": {
  1126                       "buckets": null,
  1127                       "mode": "time",
  1128                       "name": null,
  1129                       "show": true,
  1130                       "values": [ ]
  1131                    },
  1132                    "yaxes": [
  1133                       {
  1134                          "format": "ms",
  1135                          "label": null,
  1136                          "logBase": 1,
  1137                          "max": null,
  1138                          "min": 0,
  1139                          "show": true
  1140                       },
  1141                       {
  1142                          "format": "short",
  1143                          "label": null,
  1144                          "logBase": 1,
  1145                          "max": null,
  1146                          "min": null,
  1147                          "show": false
  1148                       }
  1149                    ]
  1150                 }
  1151              ],
  1152              "repeat": null,
  1153              "repeatIteration": null,
  1154              "repeatRowId": null,
  1155              "showTitle": true,
  1156              "title": "Sweeper",
  1157              "titleSize": "h6"
  1158           },
  1159           {
  1160              "collapse": false,
  1161              "height": "250px",
  1162              "panels": [
  1163                 {
  1164                    "aliasColors": { },
  1165                    "bars": false,
  1166                    "dashLength": 10,
  1167                    "dashes": false,
  1168                    "datasource": "$datasource",
  1169                    "fill": 1,
  1170                    "id": 14,
  1171                    "legend": {
  1172                       "avg": false,
  1173                       "current": false,
  1174                       "max": false,
  1175                       "min": false,
  1176                       "show": true,
  1177                       "total": false,
  1178                       "values": false
  1179                    },
  1180                    "lines": true,
  1181                    "linewidth": 1,
  1182                    "links": [ ],
  1183                    "nullPointMode": "null as zero",
  1184                    "percentage": false,
  1185                    "pointradius": 5,
  1186                    "points": false,
  1187                    "renderer": "flot",
  1188                    "seriesOverrides": [ ],
  1189                    "spaceLength": 10,
  1190                    "span": 4,
  1191                    "stack": false,
  1192                    "steppedLine": false,
  1193                    "targets": [
  1194                       {
  1195                          "expr": "time() - (loki_boltdb_shipper_retention_sweeper_marker_file_processing_current_time{cluster=~\"$cluster\", namespace=~\"$namespace\"} > 0)",
  1196                          "format": "time_series",
  1197                          "intervalFactor": 2,
  1198                          "legendFormat": "lag",
  1199                          "legendLink": null,
  1200                          "step": 10
  1201                       }
  1202                    ],
  1203                    "thresholds": [ ],
  1204                    "timeFrom": null,
  1205                    "timeShift": null,
  1206                    "title": "Sweeper Lag",
  1207                    "tooltip": {
  1208                       "shared": true,
  1209                       "sort": 2,
  1210                       "value_type": "individual"
  1211                    },
  1212                    "type": "graph",
  1213                    "xaxis": {
  1214                       "buckets": null,
  1215                       "mode": "time",
  1216                       "name": null,
  1217                       "show": true,
  1218                       "values": [ ]
  1219                    },
  1220                    "yaxes": [
  1221                       {
  1222                          "format": "s",
  1223                          "label": null,
  1224                          "logBase": 1,
  1225                          "max": null,
  1226                          "min": null,
  1227                          "show": true
  1228                       },
  1229                       {
  1230                          "format": "short",
  1231                          "label": null,
  1232                          "logBase": 1,
  1233                          "max": null,
  1234                          "min": null,
  1235                          "show": false
  1236                       }
  1237                    ]
  1238                 },
  1239                 {
  1240                    "aliasColors": { },
  1241                    "bars": false,
  1242                    "dashLength": 10,
  1243                    "dashes": false,
  1244                    "datasource": "$datasource",
  1245                    "fill": 1,
  1246                    "id": 15,
  1247                    "legend": {
  1248                       "avg": false,
  1249                       "current": false,
  1250                       "max": false,
  1251                       "min": false,
  1252                       "show": true,
  1253                       "total": false,
  1254                       "values": false
  1255                    },
  1256                    "lines": true,
  1257                    "linewidth": 1,
  1258                    "links": [ ],
  1259                    "nullPointMode": "null as zero",
  1260                    "percentage": false,
  1261                    "pointradius": 5,
  1262                    "points": false,
  1263                    "renderer": "flot",
  1264                    "seriesOverrides": [ ],
  1265                    "spaceLength": 10,
  1266                    "span": 4,
  1267                    "stack": false,
  1268                    "steppedLine": false,
  1269                    "targets": [
  1270                       {
  1271                          "expr": "sum(loki_boltdb_shipper_retention_sweeper_marker_files_current{cluster=~\"$cluster\", namespace=~\"$namespace\"})",
  1272                          "format": "time_series",
  1273                          "intervalFactor": 2,
  1274                          "legendFormat": "count",
  1275                          "legendLink": null,
  1276                          "step": 10
  1277                       }
  1278                    ],
  1279                    "thresholds": [ ],
  1280                    "timeFrom": null,
  1281                    "timeShift": null,
  1282                    "title": "Marks Files to Process",
  1283                    "tooltip": {
  1284                       "shared": true,
  1285                       "sort": 2,
  1286                       "value_type": "individual"
  1287                    },
  1288                    "type": "graph",
  1289                    "xaxis": {
  1290                       "buckets": null,
  1291                       "mode": "time",
  1292                       "name": null,
  1293                       "show": true,
  1294                       "values": [ ]
  1295                    },
  1296                    "yaxes": [
  1297                       {
  1298                          "format": "short",
  1299                          "label": null,
  1300                          "logBase": 1,
  1301                          "max": null,
  1302                          "min": 0,
  1303                          "show": true
  1304                       },
  1305                       {
  1306                          "format": "short",
  1307                          "label": null,
  1308                          "logBase": 1,
  1309                          "max": null,
  1310                          "min": null,
  1311                          "show": false
  1312                       }
  1313                    ]
  1314                 },
  1315                 {
  1316                    "aliasColors": { },
  1317                    "bars": false,
  1318                    "dashLength": 10,
  1319                    "dashes": false,
  1320                    "datasource": "$datasource",
  1321                    "fill": 1,
  1322                    "id": 16,
  1323                    "legend": {
  1324                       "avg": false,
  1325                       "current": false,
  1326                       "max": false,
  1327                       "min": false,
  1328                       "show": true,
  1329                       "total": false,
  1330                       "values": false
  1331                    },
  1332                    "lines": true,
  1333                    "linewidth": 1,
  1334                    "links": [ ],
  1335                    "nullPointMode": "null as zero",
  1336                    "percentage": false,
  1337                    "pointradius": 5,
  1338                    "points": false,
  1339                    "renderer": "flot",
  1340                    "seriesOverrides": [ ],
  1341                    "spaceLength": 10,
  1342                    "span": 4,
  1343                    "stack": false,
  1344                    "steppedLine": false,
  1345                    "targets": [
  1346                       {
  1347                          "expr": "sum by (status)(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
  1348                          "format": "time_series",
  1349                          "intervalFactor": 2,
  1350                          "legendFormat": "{{status}}",
  1351                          "legendLink": null,
  1352                          "step": 10
  1353                       }
  1354                    ],
  1355                    "thresholds": [ ],
  1356                    "timeFrom": null,
  1357                    "timeShift": null,
  1358                    "title": "Delete Rate Per Status",
  1359                    "tooltip": {
  1360                       "shared": true,
  1361                       "sort": 2,
  1362                       "value_type": "individual"
  1363                    },
  1364                    "type": "graph",
  1365                    "xaxis": {
  1366                       "buckets": null,
  1367                       "mode": "time",
  1368                       "name": null,
  1369                       "show": true,
  1370                       "values": [ ]
  1371                    },
  1372                    "yaxes": [
  1373                       {
  1374                          "format": "short",
  1375                          "label": null,
  1376                          "logBase": 1,
  1377                          "max": null,
  1378                          "min": 0,
  1379                          "show": true
  1380                       },
  1381                       {
  1382                          "format": "short",
  1383                          "label": null,
  1384                          "logBase": 1,
  1385                          "max": null,
  1386                          "min": null,
  1387                          "show": false
  1388                       }
  1389                    ]
  1390                 }
  1391              ],
  1392              "repeat": null,
  1393              "repeatIteration": null,
  1394              "repeatRowId": null,
  1395              "showTitle": true,
  1396              "title": "",
  1397              "titleSize": "h6"
  1398           },
  1399           {
  1400              "collapse": false,
  1401              "height": "250px",
  1402              "panels": [
  1403                 {
  1404                    "datasource": "$logs",
  1405                    "id": 17,
  1406                    "span": 12,
  1407                    "targets": [
  1408                       {
  1409                          "expr": "{cluster=~\"$cluster\", job=~\"($namespace)/compactor\"}",
  1410                          "refId": "A"
  1411                       }
  1412                    ],
  1413                    "title": "Compactor Logs",
  1414                    "type": "logs"
  1415                 }
  1416              ],
  1417              "repeat": null,
  1418              "repeatIteration": null,
  1419              "repeatRowId": null,
  1420              "showTitle": true,
  1421              "title": "Logs",
  1422              "titleSize": "h6"
  1423           }
  1424        ],
  1425        "schemaVersion": 14,
  1426        "style": "dark",
  1427        "tags": [
  1428           "loki"
  1429        ],
  1430        "templating": {
  1431           "list": [
  1432              {
  1433                 "current": {
  1434                    "text": "default",
  1435                    "value": "default"
  1436                 },
  1437                 "hide": 0,
  1438                 "label": "Data Source",
  1439                 "name": "datasource",
  1440                 "options": [ ],
  1441                 "query": "prometheus",
  1442                 "refresh": 1,
  1443                 "regex": "",
  1444                 "type": "datasource"
  1445              },
  1446              {
  1447                 "allValue": null,
  1448                 "current": {
  1449                    "text": "prod",
  1450                    "value": "prod"
  1451                 },
  1452                 "datasource": "$datasource",
  1453                 "hide": 0,
  1454                 "includeAll": false,
  1455                 "label": "cluster",
  1456                 "multi": false,
  1457                 "name": "cluster",
  1458                 "options": [ ],
  1459                 "query": "label_values(loki_build_info, cluster)",
  1460                 "refresh": 1,
  1461                 "regex": "",
  1462                 "sort": 2,
  1463                 "tagValuesQuery": "",
  1464                 "tags": [ ],
  1465                 "tagsQuery": "",
  1466                 "type": "query",
  1467                 "useTags": false
  1468              },
  1469              {
  1470                 "allValue": null,
  1471                 "current": {
  1472                    "text": "prod",
  1473                    "value": "prod"
  1474                 },
  1475                 "datasource": "$datasource",
  1476                 "hide": 0,
  1477                 "includeAll": false,
  1478                 "label": "namespace",
  1479                 "multi": false,
  1480                 "name": "namespace",
  1481                 "options": [ ],
  1482                 "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)",
  1483                 "refresh": 1,
  1484                 "regex": "",
  1485                 "sort": 2,
  1486                 "tagValuesQuery": "",
  1487                 "tags": [ ],
  1488                 "tagsQuery": "",
  1489                 "type": "query",
  1490                 "useTags": false
  1491              },
  1492              {
  1493                 "hide": 0,
  1494                 "label": null,
  1495                 "name": "logs",
  1496                 "options": [ ],
  1497                 "query": "loki",
  1498                 "refresh": 1,
  1499                 "regex": "",
  1500                 "type": "datasource"
  1501              }
  1502           ]
  1503        },
  1504        "time": {
  1505           "from": "now-1h",
  1506           "to": "now"
  1507        },
  1508        "timepicker": {
  1509           "refresh_intervals": [
  1510              "5s",
  1511              "10s",
  1512              "30s",
  1513              "1m",
  1514              "5m",
  1515              "15m",
  1516              "30m",
  1517              "1h",
  1518              "2h",
  1519              "1d"
  1520           ],
  1521           "time_options": [
  1522              "5m",
  1523              "15m",
  1524              "1h",
  1525              "6h",
  1526              "12h",
  1527              "24h",
  1528              "2d",
  1529              "7d",
  1530              "30d"
  1531           ]
  1532        },
  1533        "timezone": "utc",
  1534        "title": "Loki / Retention",
  1535        "uid": "retention",
  1536        "version": 0
  1537     }