sigs.k8s.io/kubebuilder/v3@v3.14.0/pkg/plugins/optional/grafana/v1alpha/scaffolds/internal/templates/runtime.go (about)

     1  /*
     2  Copyright 2022 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package templates
    18  
    19  import (
    20  	"path/filepath"
    21  
    22  	"sigs.k8s.io/kubebuilder/v3/pkg/machinery"
    23  )
    24  
    25  var _ machinery.Template = &RuntimeManifest{}
    26  
    27  // Kustomization scaffolds a file that defines the kustomization scheme for the prometheus folder
    28  type RuntimeManifest struct {
    29  	machinery.TemplateMixin
    30  }
    31  
    32  // SetTemplateDefaults implements file.Template
    33  func (f *RuntimeManifest) SetTemplateDefaults() error {
    34  	if f.Path == "" {
    35  		f.Path = filepath.Join("grafana", "controller-runtime-metrics.json")
    36  	}
    37  
    38  	// Grafana syntax use {{ }} quite often, which is collided with default delimiter for go template parsing.
    39  	// Provide an alternative delimiter here to avoid overlaps.
    40  	f.SetDelim("[[", "]]")
    41  	f.TemplateBody = controllerRuntimeTemplate
    42  	f.IfExistsAction = machinery.OverwriteFile
    43  
    44  	return nil
    45  }
    46  
    47  // nolint: lll
    48  const controllerRuntimeTemplate = `{
    49    "__inputs": [
    50      {
    51        "name": "DS_PROMETHEUS",
    52        "label": "Prometheus",
    53        "description": "",
    54        "type": "datasource",
    55        "pluginId": "prometheus",
    56        "pluginName": "Prometheus"
    57      }
    58    ],
    59    "__requires": [
    60      {
    61        "type": "datasource",
    62        "id": "prometheus",
    63        "name": "Prometheus",
    64        "version": "1.0.0"
    65      }
    66    ],
    67    "annotations": {
    68      "list": [
    69        {
    70          "builtIn": 1,
    71          "datasource": {
    72            "type": "datasource",
    73            "uid": "grafana"
    74          },
    75          "enable": true,
    76          "hide": true,
    77          "iconColor": "rgba(0, 211, 255, 1)",
    78          "name": "Annotations & Alerts",
    79          "target": {
    80            "limit": 100,
    81            "matchAny": false,
    82            "tags": [],
    83            "type": "dashboard"
    84          },
    85          "type": "dashboard"
    86        }
    87      ]
    88    },
    89    "editable": true,
    90    "fiscalYearStartMonth": 0,
    91    "graphTooltip": 0,
    92    "links": [],
    93    "liveNow": false,
    94    "panels": [
    95      {
    96        "collapsed": false,
    97        "gridPos": {
    98          "h": 1,
    99          "w": 24,
   100          "x": 0,
   101          "y": 0
   102        },
   103        "id": 9,
   104        "panels": [],
   105        "title": "Reconciliation Metrics",
   106        "type": "row"
   107      },
   108      {
   109        "datasource": "${DS_PROMETHEUS}",
   110        "fieldConfig": {
   111          "defaults": {
   112            "mappings": [],
   113            "thresholds": {
   114              "mode": "percentage",
   115              "steps": [
   116                {
   117                  "color": "green",
   118                  "value": null
   119                },
   120                {
   121                  "color": "orange",
   122                  "value": 70
   123                },
   124                {
   125                  "color": "red",
   126                  "value": 85
   127                }
   128              ]
   129            }
   130          },
   131          "overrides": []
   132        },
   133        "gridPos": {
   134          "h": 8,
   135          "w": 3,
   136          "x": 0,
   137          "y": 1
   138        },
   139        "id": 24,
   140        "options": {
   141          "orientation": "auto",
   142          "reduceOptions": {
   143            "calcs": ["lastNotNull"],
   144            "fields": "",
   145            "values": false
   146          },
   147          "showThresholdLabels": false,
   148          "showThresholdMarkers": true
   149        },
   150        "pluginVersion": "9.5.3",
   151        "targets": [
   152          {
   153            "datasource": "${DS_PROMETHEUS}",
   154            "exemplar": true,
   155            "expr": "controller_runtime_active_workers{job=\"$job\", namespace=\"$namespace\"}",
   156            "interval": "",
   157            "legendFormat": "{{controller}} {{instance}}",
   158            "refId": "A"
   159          }
   160        ],
   161        "title": "Number of workers in use",
   162        "type": "gauge"
   163      },
   164      {
   165        "datasource": "${DS_PROMETHEUS}",
   166        "description": "Total number of reconciliations per controller",
   167        "fieldConfig": {
   168          "defaults": {
   169            "color": {
   170              "mode": "continuous-GrYlRd"
   171            },
   172            "custom": {
   173              "axisCenteredZero": false,
   174              "axisColorMode": "text",
   175              "axisLabel": "",
   176              "axisPlacement": "auto",
   177              "barAlignment": 0,
   178              "drawStyle": "line",
   179              "fillOpacity": 20,
   180              "gradientMode": "scheme",
   181              "hideFrom": {
   182                "legend": false,
   183                "tooltip": false,
   184                "viz": false
   185              },
   186              "lineInterpolation": "smooth",
   187              "lineWidth": 3,
   188              "pointSize": 5,
   189              "scaleDistribution": {
   190                "type": "linear"
   191              },
   192              "showPoints": "auto",
   193              "spanNulls": false,
   194              "stacking": {
   195                "group": "A",
   196                "mode": "none"
   197              },
   198              "thresholdsStyle": {
   199                "mode": "off"
   200              }
   201            },
   202            "mappings": [],
   203            "thresholds": {
   204              "mode": "absolute",
   205              "steps": [
   206                {
   207                  "color": "green",
   208                  "value": null
   209                },
   210                {
   211                  "color": "red",
   212                  "value": 80
   213                }
   214              ]
   215            },
   216            "unit": "cpm"
   217          },
   218          "overrides": []
   219        },
   220        "gridPos": {
   221          "h": 8,
   222          "w": 11,
   223          "x": 3,
   224          "y": 1
   225        },
   226        "id": 7,
   227        "options": {
   228          "legend": {
   229            "calcs": [],
   230            "displayMode": "table",
   231            "placement": "bottom",
   232            "showLegend": true
   233          },
   234          "tooltip": {
   235            "mode": "single",
   236            "sort": "none"
   237          }
   238        },
   239        "targets": [
   240          {
   241            "datasource": "${DS_PROMETHEUS}",
   242            "editorMode": "code",
   243            "exemplar": true,
   244            "expr": "sum(rate(controller_runtime_reconcile_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, pod)",
   245            "interval": "",
   246            "legendFormat": "{{instance}} {{pod}}",
   247            "range": true,
   248            "refId": "A"
   249          }
   250        ],
   251        "title": "Total Reconciliation Count Per Controller",
   252        "type": "timeseries"
   253      },
   254      {
   255        "datasource": "${DS_PROMETHEUS}",
   256        "description": "Total number of reconciliation errors per controller",
   257        "fieldConfig": {
   258          "defaults": {
   259            "color": {
   260              "mode": "continuous-GrYlRd"
   261            },
   262            "custom": {
   263              "axisCenteredZero": false,
   264              "axisColorMode": "text",
   265              "axisLabel": "",
   266              "axisPlacement": "auto",
   267              "barAlignment": 0,
   268              "drawStyle": "line",
   269              "fillOpacity": 20,
   270              "gradientMode": "scheme",
   271              "hideFrom": {
   272                "legend": false,
   273                "tooltip": false,
   274                "viz": false
   275              },
   276              "lineInterpolation": "smooth",
   277              "lineWidth": 3,
   278              "pointSize": 5,
   279              "scaleDistribution": {
   280                "type": "linear"
   281              },
   282              "showPoints": "auto",
   283              "spanNulls": false,
   284              "stacking": {
   285                "group": "A",
   286                "mode": "none"
   287              },
   288              "thresholdsStyle": {
   289                "mode": "off"
   290              }
   291            },
   292            "mappings": [],
   293            "thresholds": {
   294              "mode": "absolute",
   295              "steps": [
   296                {
   297                  "color": "green",
   298                  "value": null
   299                },
   300                {
   301                  "color": "red",
   302                  "value": 80
   303                }
   304              ]
   305            },
   306            "unit": "cpm"
   307          },
   308          "overrides": []
   309        },
   310        "gridPos": {
   311          "h": 8,
   312          "w": 10,
   313          "x": 14,
   314          "y": 1
   315        },
   316        "id": 6,
   317        "options": {
   318          "legend": {
   319            "calcs": [],
   320            "displayMode": "table",
   321            "placement": "bottom",
   322            "showLegend": true
   323          },
   324          "tooltip": {
   325            "mode": "single",
   326            "sort": "none"
   327          }
   328        },
   329        "targets": [
   330          {
   331            "datasource": "${DS_PROMETHEUS}",
   332            "editorMode": "code",
   333            "exemplar": true,
   334            "expr": "sum(rate(controller_runtime_reconcile_errors_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, pod)",
   335            "interval": "",
   336            "legendFormat": "{{instance}} {{pod}}",
   337            "range": true,
   338            "refId": "A"
   339          }
   340        ],
   341        "title": "Reconciliation Error Count Per Controller",
   342        "type": "timeseries"
   343      },
   344      {
   345        "collapsed": false,
   346        "gridPos": {
   347          "h": 1,
   348          "w": 24,
   349          "x": 0,
   350          "y": 9
   351        },
   352        "id": 11,
   353        "panels": [],
   354        "title": "Work Queue Metrics",
   355        "type": "row"
   356      },
   357      {
   358        "datasource": "${DS_PROMETHEUS}",
   359        "fieldConfig": {
   360          "defaults": {
   361            "mappings": [],
   362            "thresholds": {
   363              "mode": "percentage",
   364              "steps": [
   365                {
   366                  "color": "green",
   367                  "value": null
   368                },
   369                {
   370                  "color": "orange",
   371                  "value": 70
   372                },
   373                {
   374                  "color": "red",
   375                  "value": 85
   376                }
   377              ]
   378            }
   379          },
   380          "overrides": []
   381        },
   382        "gridPos": {
   383          "h": 8,
   384          "w": 3,
   385          "x": 0,
   386          "y": 10
   387        },
   388        "id": 22,
   389        "options": {
   390          "orientation": "auto",
   391          "reduceOptions": {
   392            "calcs": ["lastNotNull"],
   393            "fields": "",
   394            "values": false
   395          },
   396          "showThresholdLabels": false,
   397          "showThresholdMarkers": true
   398        },
   399        "pluginVersion": "9.5.3",
   400        "targets": [
   401          {
   402            "datasource": "${DS_PROMETHEUS}",
   403            "exemplar": true,
   404            "expr": "workqueue_depth{job=\"$job\", namespace=\"$namespace\"}",
   405            "interval": "",
   406            "legendFormat": "",
   407            "refId": "A"
   408          }
   409        ],
   410        "title": "WorkQueue Depth",
   411        "type": "gauge"
   412      },
   413      {
   414        "datasource": "${DS_PROMETHEUS}",
   415        "description": "How long in seconds an item stays in workqueue before being requested",
   416        "fieldConfig": {
   417          "defaults": {
   418            "color": {
   419              "mode": "palette-classic"
   420            },
   421            "custom": {
   422              "axisCenteredZero": false,
   423              "axisColorMode": "text",
   424              "axisLabel": "",
   425              "axisPlacement": "auto",
   426              "barAlignment": 0,
   427              "drawStyle": "line",
   428              "fillOpacity": 10,
   429              "gradientMode": "none",
   430              "hideFrom": {
   431                "legend": false,
   432                "tooltip": false,
   433                "viz": false
   434              },
   435              "lineInterpolation": "linear",
   436              "lineWidth": 1,
   437              "pointSize": 5,
   438              "scaleDistribution": {
   439                "type": "linear"
   440              },
   441              "showPoints": "auto",
   442              "spanNulls": false,
   443              "stacking": {
   444                "group": "A",
   445                "mode": "normal"
   446              },
   447              "thresholdsStyle": {
   448                "mode": "off"
   449              }
   450            },
   451            "mappings": [],
   452            "thresholds": {
   453              "mode": "absolute",
   454              "steps": [
   455                {
   456                  "color": "green",
   457                  "value": null
   458                },
   459                {
   460                  "color": "red",
   461                  "value": 80
   462                }
   463              ]
   464            },
   465            "unit": "s"
   466          },
   467          "overrides": []
   468        },
   469        "gridPos": {
   470          "h": 8,
   471          "w": 11,
   472          "x": 3,
   473          "y": 10
   474        },
   475        "id": 13,
   476        "options": {
   477          "legend": {
   478            "calcs": [
   479              "max",
   480              "mean"
   481            ],
   482            "displayMode": "table",
   483            "placement": "bottom",
   484            "showLegend": true
   485          },
   486          "tooltip": {
   487            "mode": "single",
   488            "sort": "none"
   489          }
   490        },
   491        "targets": [
   492          {
   493            "datasource": "${DS_PROMETHEUS}",
   494            "exemplar": true,
   495            "expr": "histogram_quantile(0.50, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
   496            "interval": "",
   497            "legendFormat": "P50 {{name}} {{instance}} ",
   498            "refId": "A"
   499          },
   500          {
   501            "datasource": "${DS_PROMETHEUS}",
   502            "exemplar": true,
   503            "expr": "histogram_quantile(0.90, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
   504            "hide": false,
   505            "interval": "",
   506            "legendFormat": "P90 {{name}} {{instance}} ",
   507            "refId": "B"
   508          },
   509          {
   510            "datasource": "${DS_PROMETHEUS}",
   511            "exemplar": true,
   512            "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
   513            "hide": false,
   514            "interval": "",
   515            "legendFormat": "P99 {{name}} {{instance}} ",
   516            "refId": "C"
   517          }
   518        ],
   519        "title": "Seconds For Items Stay In Queue (before being requested) (P50, P90, P99)",
   520        "type": "timeseries"
   521      },
   522      {
   523        "datasource": "${DS_PROMETHEUS}",
   524        "fieldConfig": {
   525          "defaults": {
   526            "color": {
   527              "mode": "continuous-GrYlRd"
   528            },
   529            "custom": {
   530              "axisCenteredZero": false,
   531              "axisColorMode": "text",
   532              "axisLabel": "",
   533              "axisPlacement": "auto",
   534              "barAlignment": 0,
   535              "drawStyle": "line",
   536              "fillOpacity": 20,
   537              "gradientMode": "scheme",
   538              "hideFrom": {
   539                "legend": false,
   540                "tooltip": false,
   541                "viz": false
   542              },
   543              "lineInterpolation": "smooth",
   544              "lineWidth": 3,
   545              "pointSize": 5,
   546              "scaleDistribution": {
   547                "type": "linear"
   548              },
   549              "showPoints": "auto",
   550              "spanNulls": false,
   551              "stacking": {
   552                "group": "A",
   553                "mode": "none"
   554              },
   555              "thresholdsStyle": {
   556                "mode": "off"
   557              }
   558            },
   559            "mappings": [],
   560            "thresholds": {
   561              "mode": "absolute",
   562              "steps": [
   563                {
   564                  "color": "green",
   565                  "value": null
   566                },
   567                {
   568                  "color": "red",
   569                  "value": 80
   570                }
   571              ]
   572            },
   573            "unit": "ops"
   574          },
   575          "overrides": []
   576        },
   577        "gridPos": {
   578          "h": 8,
   579          "w": 10,
   580          "x": 14,
   581          "y": 10
   582        },
   583        "id": 15,
   584        "options": {
   585          "legend": {
   586            "calcs": [],
   587            "displayMode": "table",
   588            "placement": "bottom",
   589            "showLegend": true
   590          },
   591          "tooltip": {
   592            "mode": "single",
   593            "sort": "none"
   594          }
   595        },
   596        "pluginVersion": "8.4.3",
   597        "targets": [
   598          {
   599            "datasource": "${DS_PROMETHEUS}",
   600            "exemplar": true,
   601            "expr": "sum(rate(workqueue_adds_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name)",
   602            "interval": "",
   603            "legendFormat": "{{name}} {{instance}}",
   604            "refId": "A"
   605          }
   606        ],
   607        "title": "Work Queue Add Rate",
   608        "type": "timeseries"
   609      },
   610      {
   611        "datasource": "${DS_PROMETHEUS}",
   612        "description": "How many seconds of work has done that is in progress and hasn't been observed by work_duration.\nLarge values indicate stuck threads.\nOne can deduce the number of stuck threads by observing the rate at which this increases.",
   613        "fieldConfig": {
   614          "defaults": {
   615            "mappings": [],
   616            "thresholds": {
   617              "mode": "percentage",
   618              "steps": [
   619                {
   620                  "color": "green",
   621                  "value": null
   622                },
   623                {
   624                  "color": "orange",
   625                  "value": 70
   626                },
   627                {
   628                  "color": "red",
   629                  "value": 85
   630                }
   631              ]
   632            },
   633            "unit": "s"
   634          },
   635          "overrides": []
   636        },
   637        "gridPos": {
   638          "h": 9,
   639          "w": 3,
   640          "x": 0,
   641          "y": 18
   642        },
   643        "id": 23,
   644        "options": {
   645          "orientation": "auto",
   646          "reduceOptions": {
   647            "calcs": ["lastNotNull"],
   648            "fields": "",
   649            "values": false
   650          },
   651          "showThresholdLabels": false,
   652          "showThresholdMarkers": true
   653        },
   654        "pluginVersion": "9.5.3",
   655        "targets": [
   656          {
   657            "datasource": "${DS_PROMETHEUS}",
   658            "exemplar": true,
   659            "expr": "rate(workqueue_unfinished_work_seconds{job=\"$job\", namespace=\"$namespace\"}[5m])",
   660            "interval": "",
   661            "legendFormat": "",
   662            "refId": "A"
   663          }
   664        ],
   665        "title": "Unfinished Seconds",
   666        "type": "gauge"
   667      },
   668      {
   669        "datasource": "${DS_PROMETHEUS}",
   670        "description": "How long in seconds processing an item from workqueue takes.",
   671        "fieldConfig": {
   672          "defaults": {
   673            "color": {
   674              "mode": "palette-classic"
   675            },
   676            "custom": {
   677              "axisCenteredZero": false,
   678              "axisColorMode": "text",
   679              "axisLabel": "",
   680              "axisPlacement": "auto",
   681              "barAlignment": 0,
   682              "drawStyle": "line",
   683              "fillOpacity": 10,
   684              "gradientMode": "none",
   685              "hideFrom": {
   686                "legend": false,
   687                "tooltip": false,
   688                "viz": false
   689              },
   690              "lineInterpolation": "linear",
   691              "lineWidth": 1,
   692              "pointSize": 5,
   693              "scaleDistribution": {
   694                "type": "linear"
   695              },
   696              "showPoints": "auto",
   697              "spanNulls": false,
   698              "stacking": {
   699                "group": "A",
   700                "mode": "none"
   701              },
   702              "thresholdsStyle": {
   703                "mode": "off"
   704              }
   705            },
   706            "mappings": [],
   707            "thresholds": {
   708              "mode": "absolute",
   709              "steps": [
   710                {
   711                  "color": "green",
   712                  "value": null
   713                },
   714                {
   715                  "color": "red",
   716                  "value": 80
   717                }
   718              ]
   719            },
   720            "unit": "s"
   721          },
   722          "overrides": []
   723        },
   724        "gridPos": {
   725          "h": 9,
   726          "w": 11,
   727          "x": 3,
   728          "y": 18
   729        },
   730        "id": 19,
   731        "options": {
   732          "legend": {
   733            "calcs": [
   734              "max",
   735              "mean"
   736            ],
   737            "displayMode": "table",
   738            "placement": "bottom",
   739            "showLegend": true
   740          },
   741          "tooltip": {
   742            "mode": "single",
   743            "sort": "none"
   744          }
   745        },
   746        "targets": [
   747          {
   748            "datasource": "${DS_PROMETHEUS}",
   749            "exemplar": true,
   750            "expr": "histogram_quantile(0.50, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
   751            "interval": "",
   752            "legendFormat": "P50 {{name}} {{instance}} ",
   753            "refId": "A"
   754          },
   755          {
   756            "datasource": "${DS_PROMETHEUS}",
   757            "exemplar": true,
   758            "expr": "histogram_quantile(0.90, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
   759            "hide": false,
   760            "interval": "",
   761            "legendFormat": "P90 {{name}} {{instance}} ",
   762            "refId": "B"
   763          },
   764          {
   765            "datasource": "${DS_PROMETHEUS}",
   766            "exemplar": true,
   767            "expr": "histogram_quantile(0.99, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
   768            "hide": false,
   769            "interval": "",
   770            "legendFormat": "P99 {{name}} {{instance}} ",
   771            "refId": "C"
   772          }
   773        ],
   774        "title": "Seconds Processing Items From WorkQueue (P50, P90, P99)",
   775        "type": "timeseries"
   776      },
   777      {
   778        "datasource": "${DS_PROMETHEUS}",
   779        "description": "Total number of retries handled by workqueue",
   780        "fieldConfig": {
   781          "defaults": {
   782            "color": {
   783              "mode": "continuous-GrYlRd"
   784            },
   785            "custom": {
   786              "axisCenteredZero": false,
   787              "axisColorMode": "text",
   788              "axisLabel": "",
   789              "axisPlacement": "auto",
   790              "barAlignment": 0,
   791              "drawStyle": "line",
   792              "fillOpacity": 20,
   793              "gradientMode": "scheme",
   794              "hideFrom": {
   795                "legend": false,
   796                "tooltip": false,
   797                "viz": false
   798              },
   799              "lineInterpolation": "smooth",
   800              "lineWidth": 3,
   801              "pointSize": 5,
   802              "scaleDistribution": {
   803                "type": "linear"
   804              },
   805              "showPoints": "auto",
   806              "spanNulls": false,
   807              "stacking": {
   808                "group": "A",
   809                "mode": "none"
   810              },
   811              "thresholdsStyle": {
   812                "mode": "off"
   813              }
   814            },
   815            "mappings": [],
   816            "thresholds": {
   817              "mode": "absolute",
   818              "steps": [
   819                {
   820                  "color": "green",
   821                  "value": null
   822                },
   823                {
   824                  "color": "red",
   825                  "value": 80
   826                }
   827              ]
   828            },
   829            "unit": "ops"
   830          },
   831          "overrides": []
   832        },
   833        "gridPos": {
   834          "h": 9,
   835          "w": 10,
   836          "x": 14,
   837          "y": 18
   838        },
   839        "id": 17,
   840        "options": {
   841          "legend": {
   842            "calcs": [],
   843            "displayMode": "table",
   844            "placement": "bottom",
   845            "showLegend": true
   846          },
   847          "tooltip": {
   848            "mode": "single",
   849            "sort": "none"
   850          }
   851        },
   852        "targets": [
   853          {
   854            "datasource": "${DS_PROMETHEUS}",
   855            "exemplar": true,
   856            "expr": "sum(rate(workqueue_retries_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name)",
   857            "interval": "",
   858            "legendFormat": "{{name}} {{instance}} ",
   859            "refId": "A"
   860          }
   861        ],
   862        "title": "Work Queue Retries Rate",
   863        "type": "timeseries"
   864      }
   865    ],
   866    "refresh": "",
   867    "style": "dark",
   868    "tags": [],
   869    "templating": {
   870      "list": [
   871        {
   872          "datasource": "${DS_PROMETHEUS}",
   873          "definition": "label_values(controller_runtime_reconcile_total{namespace=~\"$namespace\"}, job)",
   874          "hide": 0,
   875          "includeAll": false,
   876          "multi": false,
   877          "name": "job",
   878          "options": [],
   879          "query": {
   880            "query": "label_values(controller_runtime_reconcile_total{namespace=~\"$namespace\"}, job)",
   881            "refId": "StandardVariableQuery"
   882          },
   883          "refresh": 2,
   884          "regex": "",
   885          "skipUrlSync": false,
   886          "sort": 0,
   887          "type": "query"
   888        },
   889        {
   890          "datasource": "${DS_PROMETHEUS}",
   891          "definition": "label_values(controller_runtime_reconcile_total, namespace)",
   892          "hide": 0,
   893          "includeAll": false,
   894          "multi": false,
   895          "name": "namespace",
   896          "options": [],
   897          "query": {
   898            "query": "label_values(controller_runtime_reconcile_total, namespace)",
   899            "refId": "StandardVariableQuery"
   900          },
   901          "refresh": 1,
   902          "regex": "",
   903          "skipUrlSync": false,
   904          "sort": 0,
   905          "type": "query"
   906        },
   907        {
   908          "current": {
   909            "selected": true,
   910            "text": [
   911              "All"
   912            ],
   913            "value": [
   914              "$__all"
   915            ]
   916          },
   917          "datasource": "${DS_PROMETHEUS}",
   918          "definition": "label_values(controller_runtime_reconcile_total{namespace=~\"$namespace\", job=~\"$job\"}, pod)",
   919          "hide": 2,
   920          "includeAll": true,
   921          "label": "pod",
   922          "multi": true,
   923          "name": "pod",
   924          "options": [],
   925          "query": {
   926            "query": "label_values(controller_runtime_reconcile_total{namespace=~\"$namespace\", job=~\"$job\"}, pod)",
   927            "refId": "StandardVariableQuery"
   928          },
   929          "refresh": 2,
   930          "regex": "",
   931          "skipUrlSync": false,
   932          "sort": 0,
   933          "type": "query"
   934        }
   935      ]
   936    },
   937    "time": {
   938      "from": "now-15m",
   939      "to": "now"
   940    },
   941    "timepicker": {},
   942    "timezone": "",
   943    "title": "Controller-Runtime-Metrics",
   944    "weekStart": ""
   945  }
   946  `