github.com/ironcore-dev/gardener-extension-provider-ironcore@v0.3.2-0.20240314231816-8336447fb9a0/charts/internal/machine-controller-manager/seed/mcm-monitoring-dashboard.json (about)

     1  {
     2    "description": "Information about the operations of the Machine Controller Manager",
     3    "editable": false,
     4    "gnetId": null,
     5    "graphTooltip": 0,
     6    "id": 16,
     7    "iteration": 1564731005347,
     8    "links": [
     9      {
    10        "icon": "external link",
    11        "tags": [],
    12        "targetBlank": true,
    13        "title": "Machine Controller Manager",
    14        "tooltip": "",
    15        "type": "link",
    16        "url": "https://github.com/gardener/machine-controller-manager"
    17      }
    18    ],
    19    "panels": [
    20      {
    21        "aliasColors": {},
    22        "bars": false,
    23        "dashLength": 10,
    24        "dashes": false,
    25        "decimals": null,
    26        "description": "State of the managed machines.\n\n| Code | Machine State |\n|---|---|\n| 0 | Running |\n| 1 | Terminating |\n| 2 | Unknown |\n| 3 | Failed |\n| -1 | Available |\n| -2 | Pending |",
    27        "fill": 0,
    28        "gridPos": {
    29          "h": 7,
    30          "w": 24,
    31          "x": 0,
    32          "y": 0
    33        },
    34        "id": 5,
    35        "legend": {
    36          "alignAsTable": true,
    37          "avg": false,
    38          "current": false,
    39          "max": false,
    40          "min": false,
    41          "rightSide": true,
    42          "show": true,
    43          "total": false,
    44          "values": false
    45        },
    46        "lines": true,
    47        "linewidth": 1,
    48        "links": [],
    49        "nullPointMode": "null",
    50        "percentage": false,
    51        "pointradius": 2,
    52        "points": false,
    53        "renderer": "flot",
    54        "seriesOverrides": [],
    55        "spaceLength": 10,
    56        "stack": false,
    57        "steppedLine": false,
    58        "targets": [
    59          {
    60            "expr": "mcm_machine_current_status_phase",
    61            "format": "time_series",
    62            "intervalFactor": 1,
    63            "legendFormat": "{{name}}",
    64            "refId": "A"
    65          }
    66        ],
    67        "thresholds": [],
    68        "timeFrom": null,
    69        "timeRegions": [],
    70        "timeShift": null,
    71        "title": "Managed Machines States",
    72        "tooltip": {
    73          "shared": true,
    74          "sort": 0,
    75          "value_type": "individual"
    76        },
    77        "type": "graph",
    78        "xaxis": {
    79          "buckets": null,
    80          "mode": "time",
    81          "name": null,
    82          "show": true,
    83          "values": []
    84        },
    85        "yaxes": [
    86          {
    87            "decimals": null,
    88            "format": "short",
    89            "label": null,
    90            "logBase": 1,
    91            "max": "3.2",
    92            "min": "-2.2",
    93            "show": true
    94          },
    95          {
    96            "format": "short",
    97            "label": null,
    98            "logBase": 1,
    99            "max": null,
   100            "min": null,
   101            "show": false
   102          }
   103        ],
   104        "yaxis": {
   105          "align": false,
   106          "alignLevel": null
   107        }
   108      },
   109      {
   110        "aliasColors": {},
   111        "bars": false,
   112        "dashLength": 10,
   113        "dashes": false,
   114        "description": "Shows the CPU usage of the Machine Controller Manager and shows the requests and limits.",
   115        "fill": 0,
   116        "gridPos": {
   117          "h": 6,
   118          "w": 12,
   119          "x": 0,
   120          "y": 7
   121        },
   122        "id": 13,
   123        "legend": {
   124          "avg": false,
   125          "current": false,
   126          "max": false,
   127          "min": false,
   128          "show": true,
   129          "total": false,
   130          "values": false
   131        },
   132        "lines": true,
   133        "linewidth": 1,
   134        "links": [],
   135        "nullPointMode": "null",
   136        "percentage": false,
   137        "pointradius": 2,
   138        "points": false,
   139        "renderer": "flot",
   140        "seriesOverrides": [],
   141        "spaceLength": 10,
   142        "stack": false,
   143        "steppedLine": false,
   144        "targets": [
   145          {
   146            "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"machine-controller-manager-(.+)\"}[5m])) by (pod)",
   147            "format": "time_series",
   148            "intervalFactor": 1,
   149            "legendFormat": "Current ({{pod}})",
   150            "refId": "A"
   151          },
   152          {
   153            "expr": "sum(kube_pod_container_resource_limits{resource=\"cpu\", unit=\"core\", pod=~\"machine-controller-manager-(.+)\"}) by (pod)",
   154            "format": "time_series",
   155            "intervalFactor": 1,
   156            "legendFormat": "Limits ({{pod}})",
   157            "refId": "C"
   158          },
   159          {
   160            "expr": "sum(kube_pod_container_resource_requests{resource=\"cpu\", unit=\"core\", pod=~\"machine-controller-manager-(.+)\"}) by (pod)",
   161            "format": "time_series",
   162            "intervalFactor": 1,
   163            "legendFormat": "Requests ({{pod}})",
   164            "refId": "B"
   165          }
   166        ],
   167        "thresholds": [],
   168        "timeFrom": null,
   169        "timeRegions": [],
   170        "timeShift": null,
   171        "title": "MCM CPU usage",
   172        "tooltip": {
   173          "shared": true,
   174          "sort": 0,
   175          "value_type": "individual"
   176        },
   177        "type": "graph",
   178        "xaxis": {
   179          "buckets": null,
   180          "mode": "time",
   181          "name": null,
   182          "show": true,
   183          "values": []
   184        },
   185        "yaxes": [
   186          {
   187            "decimals": null,
   188            "format": "short",
   189            "label": null,
   190            "logBase": 1,
   191            "max": null,
   192            "min": "0",
   193            "show": true
   194          },
   195          {
   196            "format": "short",
   197            "label": null,
   198            "logBase": 1,
   199            "max": null,
   200            "min": null,
   201            "show": true
   202          }
   203        ],
   204        "yaxis": {
   205          "align": false,
   206          "alignLevel": null
   207        }
   208      },
   209      {
   210        "aliasColors": {},
   211        "bars": false,
   212        "dashLength": 10,
   213        "dashes": false,
   214        "description": "Shows the memory usage of the Machine Controller Manager.",
   215        "fill": 0,
   216        "gridPos": {
   217          "h": 6,
   218          "w": 12,
   219          "x": 12,
   220          "y": 7
   221        },
   222        "id": 11,
   223        "legend": {
   224          "avg": false,
   225          "current": false,
   226          "max": false,
   227          "min": false,
   228          "show": true,
   229          "total": false,
   230          "values": false
   231        },
   232        "lines": true,
   233        "linewidth": 1,
   234        "links": [],
   235        "nullPointMode": "null",
   236        "percentage": false,
   237        "pointradius": 2,
   238        "points": false,
   239        "renderer": "flot",
   240        "seriesOverrides": [],
   241        "spaceLength": 10,
   242        "stack": false,
   243        "steppedLine": false,
   244        "targets": [
   245          {
   246            "expr": "sum(container_memory_working_set_bytes{pod=~\"machine-controller-manager-(.+)\"}) by (pod)",
   247            "format": "time_series",
   248            "intervalFactor": 1,
   249            "legendFormat": "Current ({{pod}})",
   250            "refId": "A"
   251          },
   252          {
   253            "expr": "sum(kube_pod_container_resource_limits{resource=\"memory\", unit=\"byte\", pod=~\"machine-controller-manager-(.+)\"}) by (pod)",
   254            "format": "time_series",
   255            "intervalFactor": 1,
   256            "legendFormat": "Limits ({{pod}})",
   257            "refId": "B"
   258          },
   259          {
   260            "expr": "sum(kube_pod_container_resource_requests{resource=\"memory\", unit=\"byte\", pod=~\"machine-controller-manager-(.+)\"}) by (pod)",
   261            "format": "time_series",
   262            "intervalFactor": 1,
   263            "legendFormat": "Requests ({{pod}})",
   264            "refId": "C"
   265          }
   266        ],
   267        "thresholds": [],
   268        "timeFrom": null,
   269        "timeRegions": [],
   270        "timeShift": null,
   271        "title": "MCM Memory Usage",
   272        "tooltip": {
   273          "shared": true,
   274          "sort": 0,
   275          "value_type": "individual"
   276        },
   277        "type": "graph",
   278        "xaxis": {
   279          "buckets": null,
   280          "mode": "time",
   281          "name": null,
   282          "show": true,
   283          "values": []
   284        },
   285        "yaxes": [
   286          {
   287            "format": "bytes",
   288            "label": null,
   289            "logBase": 2,
   290            "max": null,
   291            "min": null,
   292            "show": true
   293          },
   294          {
   295            "format": "none",
   296            "label": null,
   297            "logBase": 1,
   298            "max": null,
   299            "min": null,
   300            "show": false
   301          }
   302        ],
   303        "yaxis": {
   304          "align": false,
   305          "alignLevel": null
   306        }
   307      },
   308      {
   309        "aliasColors": {},
   310        "bars": false,
   311        "dashLength": 10,
   312        "dashes": false,
   313        "description": "Indicates if the Machine Controller Manager is frozen due to unreachable API server.\n\n0 = ok; 1= frozen",
   314        "fill": 0,
   315        "gridPos": {
   316          "h": 6,
   317          "w": 12,
   318          "x": 0,
   319          "y": 13
   320        },
   321        "id": 2,
   322        "legend": {
   323          "avg": false,
   324          "current": false,
   325          "max": false,
   326          "min": false,
   327          "show": true,
   328          "total": false,
   329          "values": false
   330        },
   331        "lines": true,
   332        "linewidth": 2,
   333        "links": [],
   334        "nullPointMode": "null",
   335        "percentage": false,
   336        "pointradius": 2,
   337        "points": false,
   338        "renderer": "flot",
   339        "seriesOverrides": [],
   340        "spaceLength": 10,
   341        "stack": false,
   342        "steppedLine": false,
   343        "targets": [
   344          {
   345            "expr": "mcm_machine_controller_frozen",
   346            "format": "time_series",
   347            "intervalFactor": 1,
   348            "legendFormat": "{{pod}}",
   349            "refId": "A"
   350          }
   351        ],
   352        "thresholds": [
   353          {
   354            "colorMode": "critical",
   355            "fill": true,
   356            "line": true,
   357            "op": "gt",
   358            "value": 0.5,
   359            "yaxis": "left"
   360          },
   361          {
   362            "colorMode": "ok",
   363            "fill": true,
   364            "line": true,
   365            "op": "lt",
   366            "value": 0.5,
   367            "yaxis": "left"
   368          }
   369        ],
   370        "timeFrom": null,
   371        "timeRegions": [],
   372        "timeShift": null,
   373        "title": "MCM Frozen Status (API Server reachable)",
   374        "tooltip": {
   375          "shared": true,
   376          "sort": 0,
   377          "value_type": "individual"
   378        },
   379        "type": "graph",
   380        "xaxis": {
   381          "buckets": null,
   382          "mode": "time",
   383          "name": null,
   384          "show": true,
   385          "values": []
   386        },
   387        "yaxes": [
   388          {
   389            "decimals": 0,
   390            "format": "short",
   391            "label": "",
   392            "logBase": 1,
   393            "max": "1.2",
   394            "min": "-0.2",
   395            "show": true
   396          },
   397          {
   398            "format": "short",
   399            "label": null,
   400            "logBase": 1,
   401            "max": null,
   402            "min": null,
   403            "show": true
   404          }
   405        ],
   406        "yaxis": {
   407          "align": false,
   408          "alignLevel": null
   409        }
   410      },
   411      {
   412        "aliasColors": {},
   413        "bars": false,
   414        "dashLength": 10,
   415        "dashes": false,
   416        "decimals": null,
   417        "description": "Average per Second rate over 1m of IaaS provider api calls split by services. \n\nShows also the rate of failed iaas calls if at least one failed.",
   418        "fill": 0,
   419        "gridPos": {
   420          "h": 6,
   421          "w": 12,
   422          "x": 12,
   423          "y": 13
   424        },
   425        "id": 4,
   426        "legend": {
   427          "avg": false,
   428          "current": false,
   429          "max": false,
   430          "min": false,
   431          "show": true,
   432          "total": false,
   433          "values": false
   434        },
   435        "lines": true,
   436        "linewidth": 1,
   437        "links": [],
   438        "nullPointMode": "null",
   439        "percentage": false,
   440        "pointradius": 2,
   441        "points": false,
   442        "renderer": "flot",
   443        "seriesOverrides": [],
   444        "spaceLength": 10,
   445        "stack": false,
   446        "steppedLine": false,
   447        "targets": [
   448          {
   449            "expr": "rate(mcm_cloud_api_requests_total[1m])",
   450            "format": "time_series",
   451            "intervalFactor": 1,
   452            "legendFormat": "{{provider}} / {{service}} ({{pod}})",
   453            "refId": "A"
   454          },
   455          {
   456            "expr": "rate(mcm_cloud_api_requests_failed_total[1m])",
   457            "format": "time_series",
   458            "intervalFactor": 1,
   459            "legendFormat": "Error: {{provider}} / {{service}} ({{pod}})",
   460            "refId": "B"
   461          }
   462        ],
   463        "thresholds": [],
   464        "timeFrom": null,
   465        "timeRegions": [],
   466        "timeShift": null,
   467        "title": "IaaS API Calls",
   468        "tooltip": {
   469          "shared": true,
   470          "sort": 0,
   471          "value_type": "individual"
   472        },
   473        "type": "graph",
   474        "xaxis": {
   475          "buckets": null,
   476          "mode": "time",
   477          "name": null,
   478          "show": true,
   479          "values": []
   480        },
   481        "yaxes": [
   482          {
   483            "decimals": null,
   484            "format": "short",
   485            "label": null,
   486            "logBase": 1,
   487            "max": null,
   488            "min": "0",
   489            "show": true
   490          },
   491          {
   492            "format": "short",
   493            "label": null,
   494            "logBase": 1,
   495            "max": null,
   496            "min": null,
   497            "show": true
   498          }
   499        ],
   500        "yaxis": {
   501          "align": false,
   502          "alignLevel": null
   503        }
   504      },
   505      {
   506        "aliasColors": {},
   507        "bars": false,
   508        "dashLength": 10,
   509        "dashes": false,
   510        "decimals": 0,
   511        "description": "The count of kubernetes resources managed by the Machine Controller Manager.",
   512        "fill": 0,
   513        "gridPos": {
   514          "h": 6,
   515          "w": 12,
   516          "x": 0,
   517          "y": 19
   518        },
   519        "id": 3,
   520        "legend": {
   521          "avg": false,
   522          "current": false,
   523          "max": false,
   524          "min": false,
   525          "show": true,
   526          "total": false,
   527          "values": false
   528        },
   529        "lines": true,
   530        "linewidth": 1,
   531        "links": [],
   532        "nullPointMode": "null",
   533        "percentage": false,
   534        "pointradius": 2,
   535        "points": false,
   536        "renderer": "flot",
   537        "seriesOverrides": [],
   538        "spaceLength": 10,
   539        "stack": false,
   540        "steppedLine": false,
   541        "targets": [
   542          {
   543            "expr": "mcm_machine_items_total",
   544            "format": "time_series",
   545            "intervalFactor": 1,
   546            "legendFormat": "machine(s)",
   547            "refId": "A"
   548          },
   549          {
   550            "expr": "mcm_machine_set_items_total",
   551            "format": "time_series",
   552            "intervalFactor": 1,
   553            "legendFormat": "machine set(s)",
   554            "refId": "B"
   555          },
   556          {
   557            "expr": "mcm_machine_deployment_items_total",
   558            "format": "time_series",
   559            "intervalFactor": 1,
   560            "legendFormat": "machine deployment(s)",
   561            "refId": "C"
   562          }
   563        ],
   564        "thresholds": [],
   565        "timeFrom": null,
   566        "timeRegions": [],
   567        "timeShift": null,
   568        "title": "Count of Managed Resouces",
   569        "tooltip": {
   570          "shared": true,
   571          "sort": 0,
   572          "value_type": "individual"
   573        },
   574        "type": "graph",
   575        "xaxis": {
   576          "buckets": null,
   577          "mode": "time",
   578          "name": null,
   579          "show": true,
   580          "values": []
   581        },
   582        "yaxes": [
   583          {
   584            "decimals": 0,
   585            "format": "short",
   586            "label": null,
   587            "logBase": 1,
   588            "max": null,
   589            "min": "0",
   590            "show": true
   591          },
   592          {
   593            "format": "short",
   594            "label": null,
   595            "logBase": 1,
   596            "max": null,
   597            "min": null,
   598            "show": true
   599          }
   600        ],
   601        "yaxis": {
   602          "align": false,
   603          "alignLevel": null
   604        }
   605      },
   606      {
   607        "collapsed": false,
   608        "gridPos": {
   609          "h": 1,
   610          "w": 24,
   611          "x": 0,
   612          "y": 25
   613        },
   614        "id": 9,
   615        "panels": [],
   616        "title": "Control Loops",
   617        "type": "row"
   618      },
   619      {
   620        "aliasColors": {},
   621        "bars": false,
   622        "dashLength": 10,
   623        "dashes": false,
   624        "description": "Average processing time of items in the workqueue.",
   625        "fill": 1,
   626        "gridPos": {
   627          "h": 7,
   628          "w": 12,
   629          "x": 0,
   630          "y": 26
   631        },
   632        "id": 19,
   633        "legend": {
   634          "avg": false,
   635          "current": false,
   636          "max": false,
   637          "min": false,
   638          "show": true,
   639          "total": false,
   640          "values": false
   641        },
   642        "lines": true,
   643        "linewidth": 1,
   644        "links": [],
   645        "nullPointMode": "null",
   646        "percentage": false,
   647        "pointradius": 2,
   648        "points": false,
   649        "renderer": "flot",
   650        "seriesOverrides": [],
   651        "spaceLength": 10,
   652        "stack": false,
   653        "steppedLine": false,
   654        "targets": [
   655          {
   656            "expr": "${controlloop}_work_duration{quantile=\"0.5\"}",
   657            "format": "time_series",
   658            "intervalFactor": 1,
   659            "legendFormat": "p50 ({{pod}})",
   660            "refId": "A"
   661          },
   662          {
   663            "expr": "${controlloop}_work_duration{quantile=\"0.9\"}",
   664            "format": "time_series",
   665            "intervalFactor": 1,
   666            "legendFormat": "p90 ({{pod}})",
   667            "refId": "B"
   668          },
   669          {
   670            "expr": "${controlloop}_work_duration{quantile=\"0.99\"}",
   671            "format": "time_series",
   672            "intervalFactor": 1,
   673            "legendFormat": "p99 ({{pod}})",
   674            "refId": "C"
   675          }
   676        ],
   677        "thresholds": [],
   678        "timeFrom": null,
   679        "timeRegions": [],
   680        "timeShift": null,
   681        "title": "Workqueue item processing time: ${controlloop}",
   682        "tooltip": {
   683          "shared": true,
   684          "sort": 0,
   685          "value_type": "individual"
   686        },
   687        "type": "graph",
   688        "xaxis": {
   689          "buckets": null,
   690          "mode": "time",
   691          "name": null,
   692          "show": true,
   693          "values": []
   694        },
   695        "yaxes": [
   696          {
   697            "format": "ms",
   698            "label": null,
   699            "logBase": 1,
   700            "max": null,
   701            "min": null,
   702            "show": true
   703          },
   704          {
   705            "format": "short",
   706            "label": null,
   707            "logBase": 1,
   708            "max": null,
   709            "min": null,
   710            "show": false
   711          }
   712        ],
   713        "yaxis": {
   714          "align": false,
   715          "alignLevel": null
   716        }
   717      },
   718      {
   719        "aliasColors": {},
   720        "bars": false,
   721        "dashLength": 10,
   722        "dashes": false,
   723        "description": "How long items stay in the workqueue before they get processed.",
   724        "fill": 1,
   725        "gridPos": {
   726          "h": 7,
   727          "w": 12,
   728          "x": 12,
   729          "y": 26
   730        },
   731        "id": 18,
   732        "legend": {
   733          "avg": false,
   734          "current": false,
   735          "max": false,
   736          "min": false,
   737          "show": true,
   738          "total": false,
   739          "values": false
   740        },
   741        "lines": true,
   742        "linewidth": 1,
   743        "links": [],
   744        "nullPointMode": "null",
   745        "percentage": false,
   746        "pointradius": 2,
   747        "points": false,
   748        "renderer": "flot",
   749        "seriesOverrides": [],
   750        "spaceLength": 10,
   751        "stack": false,
   752        "steppedLine": false,
   753        "targets": [
   754          {
   755            "expr": "${controlloop}_queue_latency{quantile=\"0.5\"}",
   756            "format": "time_series",
   757            "intervalFactor": 1,
   758            "legendFormat": "p50 ({{pod}})",
   759            "refId": "A"
   760          },
   761          {
   762            "expr": "${controlloop}_queue_latency{quantile=\"0.9\"}",
   763            "format": "time_series",
   764            "intervalFactor": 1,
   765            "legendFormat": "p90 ({{pod}})",
   766            "refId": "B"
   767          },
   768          {
   769            "expr": "${controlloop}_queue_latency{quantile=\"0.99\"}",
   770            "format": "time_series",
   771            "intervalFactor": 1,
   772            "legendFormat": "p99 ({{pod}})",
   773            "refId": "C"
   774          }
   775        ],
   776        "thresholds": [],
   777        "timeFrom": null,
   778        "timeRegions": [],
   779        "timeShift": null,
   780        "title": "Workqueue item latency: ${controlloop}",
   781        "tooltip": {
   782          "shared": true,
   783          "sort": 0,
   784          "value_type": "individual"
   785        },
   786        "type": "graph",
   787        "xaxis": {
   788          "buckets": null,
   789          "mode": "time",
   790          "name": null,
   791          "show": true,
   792          "values": []
   793        },
   794        "yaxes": [
   795          {
   796            "format": "ms",
   797            "label": null,
   798            "logBase": 1,
   799            "max": null,
   800            "min": null,
   801            "show": true
   802          },
   803          {
   804            "format": "short",
   805            "label": null,
   806            "logBase": 1,
   807            "max": null,
   808            "min": null,
   809            "show": false
   810          }
   811        ],
   812        "yaxis": {
   813          "align": false,
   814          "alignLevel": null
   815        }
   816      },
   817      {
   818        "aliasColors": {},
   819        "bars": false,
   820        "dashLength": 10,
   821        "dashes": false,
   822        "description": "Current amount of items in the workqueue.",
   823        "fill": 1,
   824        "gridPos": {
   825          "h": 6,
   826          "w": 8,
   827          "x": 0,
   828          "y": 33
   829        },
   830        "id": 16,
   831        "legend": {
   832          "avg": false,
   833          "current": false,
   834          "max": false,
   835          "min": false,
   836          "show": false,
   837          "total": false,
   838          "values": false
   839        },
   840        "lines": true,
   841        "linewidth": 1,
   842        "links": [],
   843        "nullPointMode": "null",
   844        "percentage": false,
   845        "pointradius": 2,
   846        "points": false,
   847        "renderer": "flot",
   848        "seriesOverrides": [],
   849        "spaceLength": 10,
   850        "stack": false,
   851        "steppedLine": false,
   852        "targets": [
   853          {
   854            "expr": "${controlloop}_depth",
   855            "format": "time_series",
   856            "intervalFactor": 1,
   857            "legendFormat": "${controlloop} ({{pod}})",
   858            "refId": "A"
   859          }
   860        ],
   861        "thresholds": [],
   862        "timeFrom": null,
   863        "timeRegions": [],
   864        "timeShift": null,
   865        "title": "Items in Workqueue: ${controlloop}",
   866        "tooltip": {
   867          "shared": true,
   868          "sort": 0,
   869          "value_type": "individual"
   870        },
   871        "type": "graph",
   872        "xaxis": {
   873          "buckets": null,
   874          "mode": "time",
   875          "name": null,
   876          "show": true,
   877          "values": []
   878        },
   879        "yaxes": [
   880          {
   881            "format": "short",
   882            "label": null,
   883            "logBase": 1,
   884            "max": null,
   885            "min": null,
   886            "show": true
   887          },
   888          {
   889            "format": "short",
   890            "label": null,
   891            "logBase": 1,
   892            "max": null,
   893            "min": null,
   894            "show": false
   895          }
   896        ],
   897        "yaxis": {
   898          "align": false,
   899          "alignLevel": null
   900        }
   901      },
   902      {
   903        "aliasColors": {},
   904        "bars": false,
   905        "dashLength": 10,
   906        "dashes": false,
   907        "description": "Average per second rate over 5m of workqueue item adds.",
   908        "fill": 1,
   909        "gridPos": {
   910          "h": 6,
   911          "w": 8,
   912          "x": 8,
   913          "y": 33
   914        },
   915        "id": 7,
   916        "legend": {
   917          "avg": false,
   918          "current": false,
   919          "max": false,
   920          "min": false,
   921          "show": false,
   922          "total": false,
   923          "values": false
   924        },
   925        "lines": true,
   926        "linewidth": 1,
   927        "links": [],
   928        "nullPointMode": "null",
   929        "percentage": false,
   930        "pointradius": 2,
   931        "points": false,
   932        "renderer": "flot",
   933        "seriesOverrides": [],
   934        "spaceLength": 10,
   935        "stack": false,
   936        "steppedLine": false,
   937        "targets": [
   938          {
   939            "expr": "rate(${controlloop}_adds[5m])",
   940            "format": "time_series",
   941            "intervalFactor": 1,
   942            "legendFormat": "${controlloop} ({{pod}})",
   943            "refId": "A"
   944          }
   945        ],
   946        "thresholds": [],
   947        "timeFrom": null,
   948        "timeRegions": [],
   949        "timeShift": null,
   950        "title": "Adds to Workqueue: ${controlloop}",
   951        "tooltip": {
   952          "shared": true,
   953          "sort": 0,
   954          "value_type": "individual"
   955        },
   956        "type": "graph",
   957        "xaxis": {
   958          "buckets": null,
   959          "mode": "time",
   960          "name": null,
   961          "show": true,
   962          "values": []
   963        },
   964        "yaxes": [
   965          {
   966            "format": "short",
   967            "label": null,
   968            "logBase": 1,
   969            "max": null,
   970            "min": null,
   971            "show": true
   972          },
   973          {
   974            "format": "short",
   975            "label": null,
   976            "logBase": 1,
   977            "max": null,
   978            "min": null,
   979            "show": false
   980          }
   981        ],
   982        "yaxis": {
   983          "align": false,
   984          "alignLevel": null
   985        }
   986      },
   987      {
   988        "aliasColors": {},
   989        "bars": false,
   990        "dashLength": 10,
   991        "dashes": false,
   992        "description": "Average per second rate over 5m of workqueue item retries.",
   993        "fill": 1,
   994        "gridPos": {
   995          "h": 6,
   996          "w": 8,
   997          "x": 16,
   998          "y": 33
   999        },
  1000        "id": 17,
  1001        "legend": {
  1002          "avg": false,
  1003          "current": false,
  1004          "max": false,
  1005          "min": false,
  1006          "show": false,
  1007          "total": false,
  1008          "values": false
  1009        },
  1010        "lines": true,
  1011        "linewidth": 1,
  1012        "links": [],
  1013        "nullPointMode": "null",
  1014        "percentage": false,
  1015        "pointradius": 2,
  1016        "points": false,
  1017        "renderer": "flot",
  1018        "seriesOverrides": [],
  1019        "spaceLength": 10,
  1020        "stack": false,
  1021        "steppedLine": false,
  1022        "targets": [
  1023          {
  1024            "expr": "rate(${controlloop}_retries[5m])",
  1025            "format": "time_series",
  1026            "intervalFactor": 1,
  1027            "legendFormat": "${controlloop} ({{pod}})",
  1028            "refId": "A"
  1029          }
  1030        ],
  1031        "thresholds": [],
  1032        "timeFrom": null,
  1033        "timeRegions": [],
  1034        "timeShift": null,
  1035        "title": "Workqueue item retries: ${controlloop}",
  1036        "tooltip": {
  1037          "shared": true,
  1038          "sort": 0,
  1039          "value_type": "individual"
  1040        },
  1041        "type": "graph",
  1042        "xaxis": {
  1043          "buckets": null,
  1044          "mode": "time",
  1045          "name": null,
  1046          "show": true,
  1047          "values": []
  1048        },
  1049        "yaxes": [
  1050          {
  1051            "format": "short",
  1052            "label": null,
  1053            "logBase": 1,
  1054            "max": null,
  1055            "min": null,
  1056            "show": true
  1057          },
  1058          {
  1059            "format": "short",
  1060            "label": null,
  1061            "logBase": 1,
  1062            "max": null,
  1063            "min": null,
  1064            "show": false
  1065          }
  1066        ],
  1067        "yaxis": {
  1068          "align": false,
  1069          "alignLevel": null
  1070        }
  1071      }
  1072    ],
  1073    "schemaVersion": 18,
  1074    "style": "dark",
  1075    "tags": [
  1076      "controlplane",
  1077      "seed"
  1078    ],
  1079    "templating": {
  1080      "list": [
  1081        {
  1082          "allValue": null,
  1083          "current": {
  1084            "tags": [],
  1085            "text": "machine",
  1086            "value": "machine"
  1087          },
  1088          "hide": 0,
  1089          "includeAll": false,
  1090          "label": "Control Loop",
  1091          "multi": false,
  1092          "name": "controlloop",
  1093          "options": [
  1094            {
  1095              "selected": true,
  1096              "text": "machine",
  1097              "value": "machine"
  1098            },
  1099            {
  1100              "selected": false,
  1101              "text": "machineset",
  1102              "value": "machineset"
  1103            },
  1104            {
  1105              "selected": false,
  1106              "text": "machinedeployment",
  1107              "value": "machinedeployment"
  1108            },
  1109            {
  1110              "selected": false,
  1111              "text": "node",
  1112              "value": "node"
  1113            },
  1114            {
  1115              "selected": false,
  1116              "text": "secret",
  1117              "value": "secret"
  1118            },
  1119            {
  1120              "selected": false,
  1121              "text": "machinesafetyapiserver",
  1122              "value": "machinesafetyapiserver"
  1123            },
  1124            {
  1125              "selected": false,
  1126              "text": "machinesafetyorphanvms",
  1127              "value": "machinesafetyorphanvms"
  1128            },
  1129            {
  1130              "selected": false,
  1131              "text": "machinesafetyovershooting",
  1132              "value": "machinesafetyovershooting"
  1133            }
  1134          ],
  1135          "query": "machine, machineset, machinedeployment, node, secret, machinesafetyapiserver, machinesafetyorphanvms, machinesafetyovershooting",
  1136          "skipUrlSync": false,
  1137          "type": "custom"
  1138        }
  1139      ]
  1140    },
  1141    "time": {
  1142      "from": "now-3h",
  1143      "to": "now"
  1144    },
  1145    "timepicker": {
  1146      "refresh_intervals": [
  1147        "5s",
  1148        "10s",
  1149        "30s",
  1150        "1m",
  1151        "5m",
  1152        "15m",
  1153        "30m",
  1154        "1h",
  1155        "2h",
  1156        "1d"
  1157      ],
  1158      "time_options": [
  1159        "5m",
  1160        "15m",
  1161        "1h",
  1162        "6h",
  1163        "12h",
  1164        "24h",
  1165        "2d",
  1166        "7d",
  1167        "14d"
  1168      ]
  1169    },
  1170    "timezone": "utc",
  1171    "title": "Machine Controller Manager",
  1172    "uid": "machine-controller-manager",
  1173    "version": 1
  1174  }