github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/metrics/grafana/tiflow.json (about)

     1  {
     2    "__inputs": [
     3      {
     4        "name": "DS_PROMETHEUS_TIFLOW",
     5        "label": "Prometheus_tiflow",
     6        "description": "",
     7        "type": "datasource",
     8        "pluginId": "prometheus",
     9        "pluginName": "Prometheus"
    10      }
    11    ],
    12    "__elements": {
    13      "WBa0-4KVz": {
    14        "name": "Uptime",
    15        "uid": "WBa0-4KVz",
    16        "kind": 1,
    17        "model": {
    18          "aliasColors": {},
    19          "bars": false,
    20          "dashLength": 10,
    21          "dashes": false,
    22          "datasource": {
    23            "type": "prometheus",
    24            "uid": "${DS_PROMETHEUS_TIFLOW}"
    25          },
    26          "description": "Uptime of Tiflow Engine",
    27          "fieldConfig": {
    28            "defaults": {
    29              "links": []
    30            },
    31            "overrides": []
    32          },
    33          "fill": 1,
    34          "fillGradient": 0,
    35          "hiddenSeries": false,
    36          "legend": {
    37            "alignAsTable": true,
    38            "avg": false,
    39            "current": true,
    40            "max": false,
    41            "min": false,
    42            "rightSide": true,
    43            "show": true,
    44            "total": false,
    45            "values": true
    46          },
    47          "lines": true,
    48          "linewidth": 1,
    49          "links": [],
    50          "nullPointMode": "null",
    51          "options": {
    52            "alertThreshold": true
    53          },
    54          "paceLength": 10,
    55          "percentage": false,
    56          "pluginVersion": "9.2.1",
    57          "pointradius": 2,
    58          "points": false,
    59          "renderer": "flot",
    60          "seriesOverrides": [],
    61          "spaceLength": 10,
    62          "stack": false,
    63          "steppedLine": false,
    64          "targets": [
    65            {
    66              "datasource": {
    67                "type": "prometheus",
    68                "uid": "${DS_PROMETHEUS_TIFLOW}"
    69              },
    70              "editorMode": "code",
    71              "expr": "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"})",
    72              "format": "time_series",
    73              "intervalFactor": 1,
    74              "legendFormat": "Tiflow-{{instance}}",
    75              "range": true,
    76              "refId": "A"
    77            }
    78          ],
    79          "thresholds": [],
    80          "timeRegions": [],
    81          "title": "Uptime",
    82          "tooltip": {
    83            "shared": true,
    84            "sort": 0,
    85            "value_type": "individual"
    86          },
    87          "type": "graph",
    88          "xaxis": {
    89            "mode": "time",
    90            "show": true,
    91            "values": []
    92          },
    93          "yaxes": [
    94            {
    95              "$$hashKey": "object:49",
    96              "format": "dtdurations",
    97              "logBase": 1,
    98              "show": true
    99            },
   100            {
   101              "$$hashKey": "object:50",
   102              "format": "short",
   103              "logBase": 1,
   104              "show": true
   105            }
   106          ],
   107          "yaxis": {
   108            "align": false
   109          }
   110        }
   111      },
   112      "XeEAa4FVk": {
   113        "name": "Goroutine count",
   114        "uid": "XeEAa4FVk",
   115        "kind": 1,
   116        "model": {
   117          "aliasColors": {},
   118          "bars": false,
   119          "dashLength": 10,
   120          "dashes": false,
   121          "datasource": {
   122            "type": "prometheus",
   123            "uid": "${DS_PROMETHEUS_TIFLOW}"
   124          },
   125          "description": "Goroutine count of Tiflow Engine",
   126          "fieldConfig": {
   127            "defaults": {
   128              "links": []
   129            },
   130            "overrides": []
   131          },
   132          "fill": 1,
   133          "fillGradient": 0,
   134          "hiddenSeries": false,
   135          "legend": {
   136            "alignAsTable": true,
   137            "avg": false,
   138            "current": true,
   139            "max": true,
   140            "min": false,
   141            "rightSide": false,
   142            "show": true,
   143            "total": false,
   144            "values": true
   145          },
   146          "lines": true,
   147          "linewidth": 1,
   148          "links": [],
   149          "nullPointMode": "null",
   150          "options": {
   151            "alertThreshold": true
   152          },
   153          "paceLength": 10,
   154          "percentage": false,
   155          "pluginVersion": "9.2.1",
   156          "pointradius": 2,
   157          "points": false,
   158          "renderer": "flot",
   159          "seriesOverrides": [],
   160          "spaceLength": 10,
   161          "stack": false,
   162          "steppedLine": false,
   163          "targets": [
   164            {
   165              "datasource": {
   166                "type": "prometheus",
   167                "uid": "${DS_PROMETHEUS_TIFLOW}"
   168              },
   169              "editorMode": "code",
   170              "expr": " go_goroutines{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}",
   171              "format": "time_series",
   172              "hide": false,
   173              "intervalFactor": 1,
   174              "legendFormat": "{{instance}}",
   175              "range": true,
   176              "refId": "A"
   177            },
   178            {
   179              "datasource": {
   180                "type": "prometheus",
   181                "uid": "${DS_PROMETHEUS_TIFLOW}"
   182              },
   183              "editorMode": "code",
   184              "expr": "go_threads{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}",
   185              "format": "time_series",
   186              "hide": true,
   187              "intervalFactor": 1,
   188              "legendFormat": "threads-{{instance}}",
   189              "range": true,
   190              "refId": "B"
   191            }
   192          ],
   193          "thresholds": [],
   194          "timeRegions": [],
   195          "title": "Goroutine count",
   196          "tooltip": {
   197            "shared": true,
   198            "sort": 0,
   199            "value_type": "individual"
   200          },
   201          "type": "graph",
   202          "xaxis": {
   203            "mode": "time",
   204            "show": true,
   205            "values": []
   206          },
   207          "yaxes": [
   208            {
   209              "format": "short",
   210              "logBase": 1,
   211              "show": true
   212            },
   213            {
   214              "format": "short",
   215              "logBase": 1,
   216              "show": true
   217            }
   218          ],
   219          "yaxis": {
   220            "align": false
   221          }
   222        }
   223      }
   224    },
   225    "__requires": [
   226      {
   227        "type": "grafana",
   228        "id": "grafana",
   229        "name": "Grafana",
   230        "version": "9.2.1"
   231      },
   232      {
   233        "type": "panel",
   234        "id": "graph",
   235        "name": "Graph (old)",
   236        "version": ""
   237      },
   238      {
   239        "type": "datasource",
   240        "id": "prometheus",
   241        "name": "Prometheus",
   242        "version": "1.0.0"
   243      },
   244      {
   245        "type": "panel",
   246        "id": "timeseries",
   247        "name": "Time series",
   248        "version": ""
   249      }
   250    ],
   251    "annotations": {
   252      "list": [
   253        {
   254          "builtIn": 1,
   255          "datasource": {
   256            "type": "grafana",
   257            "uid": "-- Grafana --"
   258          },
   259          "enable": true,
   260          "hide": true,
   261          "iconColor": "rgba(0, 211, 255, 1)",
   262          "name": "Annotations & Alerts",
   263          "target": {
   264            "limit": 100,
   265            "matchAny": false,
   266            "tags": [],
   267            "type": "dashboard"
   268          },
   269          "type": "dashboard"
   270        }
   271      ]
   272    },
   273    "description": "dashboard for tiflow cluster",
   274    "editable": true,
   275    "fiscalYearStartMonth": 0,
   276    "graphTooltip": 0,
   277    "id": null,
   278    "links": [],
   279    "liveNow": false,
   280    "panels": [
   281      {
   282        "collapsed": false,
   283        "gridPos": {
   284          "h": 1,
   285          "w": 24,
   286          "x": 0,
   287          "y": 0
   288        },
   289        "id": 18,
   290        "panels": [],
   291        "title": "Server",
   292        "type": "row"
   293      },
   294      {
   295        "gridPos": {
   296          "h": 8,
   297          "w": 12,
   298          "x": 0,
   299          "y": 1
   300        },
   301        "id": 20,
   302        "libraryPanel": {
   303          "uid": "WBa0-4KVz",
   304          "name": "Uptime"
   305        }
   306      },
   307      {
   308        "gridPos": {
   309          "h": 8,
   310          "w": 12,
   311          "x": 12,
   312          "y": 1
   313        },
   314        "id": 22,
   315        "libraryPanel": {
   316          "uid": "XeEAa4FVk",
   317          "name": "Goroutine count"
   318        }
   319      },
   320      {
   321        "aliasColors": {},
   322        "bars": false,
   323        "dashLength": 10,
   324        "dashes": false,
   325        "datasource": {
   326          "type": "prometheus",
   327          "uid": "${DS_PROMETHEUS_TIFLOW}"
   328        },
   329        "description": "CPU usage of Tiflow Engine",
   330        "fieldConfig": {
   331          "defaults": {
   332            "links": []
   333          },
   334          "overrides": []
   335        },
   336        "fill": 1,
   337        "fillGradient": 0,
   338        "gridPos": {
   339          "h": 8,
   340          "w": 12,
   341          "x": 0,
   342          "y": 9
   343        },
   344        "hiddenSeries": false,
   345        "id": 24,
   346        "legend": {
   347          "alignAsTable": true,
   348          "avg": false,
   349          "current": true,
   350          "hideEmpty": true,
   351          "max": true,
   352          "min": false,
   353          "rightSide": false,
   354          "show": true,
   355          "total": false,
   356          "values": true
   357        },
   358        "lines": true,
   359        "linewidth": 1,
   360        "links": [],
   361        "nullPointMode": "null",
   362        "options": {
   363          "alertThreshold": true
   364        },
   365        "paceLength": 10,
   366        "percentage": false,
   367        "pluginVersion": "9.2.1",
   368        "pointradius": 2,
   369        "points": false,
   370        "renderer": "flot",
   371        "seriesOverrides": [
   372          {
   373            "alias": "/.*MaxProcs/",
   374            "fill": 0,
   375            "linewidth": 2,
   376            "yaxis": 2
   377          }
   378        ],
   379        "spaceLength": 10,
   380        "stack": false,
   381        "steppedLine": false,
   382        "targets": [
   383          {
   384            "datasource": {
   385              "type": "prometheus",
   386              "uid": "${DS_PROMETHEUS_TIFLOW}"
   387            },
   388            "editorMode": "code",
   389            "expr": "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}[2m])",
   390            "format": "time_series",
   391            "intervalFactor": 1,
   392            "legendFormat": "{{instance}}",
   393            "range": true,
   394            "refId": "A"
   395          }
   396        ],
   397        "thresholds": [],
   398        "timeRegions": [],
   399        "title": "CPU usage",
   400        "tooltip": {
   401          "shared": true,
   402          "sort": 0,
   403          "value_type": "individual"
   404        },
   405        "type": "graph",
   406        "xaxis": {
   407          "mode": "time",
   408          "show": true,
   409          "values": []
   410        },
   411        "yaxes": [
   412          {
   413            "format": "percentunit",
   414            "logBase": 1,
   415            "show": true
   416          },
   417          {
   418            "format": "short",
   419            "logBase": 1,
   420            "min": "0",
   421            "show": true
   422          }
   423        ],
   424        "yaxis": {
   425          "align": false
   426        }
   427      },
   428      {
   429        "aliasColors": {},
   430        "bars": false,
   431        "dashLength": 10,
   432        "dashes": false,
   433        "datasource": {
   434          "type": "prometheus",
   435          "uid": "${DS_PROMETHEUS_TIFLOW}"
   436        },
   437        "description": "Memory usage of Tiflow Engine",
   438        "fieldConfig": {
   439          "defaults": {
   440            "links": []
   441          },
   442          "overrides": []
   443        },
   444        "fill": 1,
   445        "fillGradient": 0,
   446        "gridPos": {
   447          "h": 8,
   448          "w": 12,
   449          "x": 12,
   450          "y": 9
   451        },
   452        "hiddenSeries": false,
   453        "id": 26,
   454        "legend": {
   455          "alignAsTable": true,
   456          "avg": false,
   457          "current": true,
   458          "hideEmpty": true,
   459          "max": true,
   460          "min": false,
   461          "rightSide": false,
   462          "show": true,
   463          "total": false,
   464          "values": true
   465        },
   466        "lines": true,
   467        "linewidth": 1,
   468        "links": [],
   469        "nullPointMode": "null",
   470        "options": {
   471          "alertThreshold": true
   472        },
   473        "paceLength": 10,
   474        "percentage": false,
   475        "pluginVersion": "9.2.1",
   476        "pointradius": 2,
   477        "points": false,
   478        "renderer": "flot",
   479        "seriesOverrides": [],
   480        "spaceLength": 10,
   481        "stack": false,
   482        "steppedLine": false,
   483        "targets": [
   484          {
   485            "datasource": {
   486              "type": "prometheus",
   487              "uid": "${DS_PROMETHEUS_TIFLOW}"
   488            },
   489            "editorMode": "code",
   490            "expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}",
   491            "format": "time_series",
   492            "hide": false,
   493            "intervalFactor": 1,
   494            "legendFormat": "process-{{instance}}",
   495            "range": true,
   496            "refId": "A"
   497          },
   498          {
   499            "datasource": {
   500              "type": "prometheus",
   501              "uid": "${DS_PROMETHEUS_TIFLOW}"
   502            },
   503            "editorMode": "code",
   504            "expr": "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}",
   505            "format": "time_series",
   506            "hide": false,
   507            "intervalFactor": 1,
   508            "legendFormat": "heap-{{instance}}",
   509            "range": true,
   510            "refId": "B"
   511          }
   512        ],
   513        "thresholds": [],
   514        "timeRegions": [],
   515        "title": "Memory usage",
   516        "tooltip": {
   517          "shared": true,
   518          "sort": 0,
   519          "value_type": "individual"
   520        },
   521        "type": "graph",
   522        "xaxis": {
   523          "mode": "time",
   524          "show": true,
   525          "values": []
   526        },
   527        "yaxes": [
   528          {
   529            "format": "bytes",
   530            "logBase": 1,
   531            "show": true
   532          },
   533          {
   534            "format": "short",
   535            "logBase": 1,
   536            "show": true
   537          }
   538        ],
   539        "yaxis": {
   540          "align": false
   541        }
   542      },
   543      {
   544        "collapsed": false,
   545        "gridPos": {
   546          "h": 1,
   547          "w": 24,
   548          "x": 0,
   549          "y": 17
   550        },
   551        "id": 12,
   552        "panels": [],
   553        "title": "General",
   554        "type": "row"
   555      },
   556      {
   557        "datasource": {
   558          "type": "prometheus",
   559          "uid": "${DS_PROMETHEUS_TIFLOW}"
   560        },
   561        "description": "num of different job state",
   562        "fieldConfig": {
   563          "defaults": {
   564            "color": {
   565              "mode": "palette-classic"
   566            },
   567            "custom": {
   568              "axisCenteredZero": false,
   569              "axisColorMode": "text",
   570              "axisLabel": "",
   571              "axisPlacement": "auto",
   572              "barAlignment": 0,
   573              "drawStyle": "line",
   574              "fillOpacity": 0,
   575              "gradientMode": "none",
   576              "hideFrom": {
   577                "legend": false,
   578                "tooltip": false,
   579                "viz": false
   580              },
   581              "lineInterpolation": "linear",
   582              "lineWidth": 1,
   583              "pointSize": 5,
   584              "scaleDistribution": {
   585                "type": "linear"
   586              },
   587              "showPoints": "auto",
   588              "spanNulls": false,
   589              "stacking": {
   590                "group": "A",
   591                "mode": "none"
   592              },
   593              "thresholdsStyle": {
   594                "mode": "off"
   595              }
   596            },
   597            "mappings": [],
   598            "thresholds": {
   599              "mode": "absolute",
   600              "steps": [
   601                {
   602                  "color": "green",
   603                  "value": null
   604                },
   605                {
   606                  "color": "red",
   607                  "value": 80
   608                }
   609              ]
   610            }
   611          },
   612          "overrides": []
   613        },
   614        "gridPos": {
   615          "h": 8,
   616          "w": 12,
   617          "x": 0,
   618          "y": 18
   619        },
   620        "id": 14,
   621        "options": {
   622          "legend": {
   623            "calcs": [],
   624            "displayMode": "list",
   625            "placement": "bottom",
   626            "showLegend": true
   627          },
   628          "tooltip": {
   629            "mode": "single",
   630            "sort": "none"
   631          }
   632        },
   633        "targets": [
   634          {
   635            "datasource": {
   636              "type": "prometheus",
   637              "uid": "${DS_PROMETHEUS_TIFLOW}"
   638            },
   639            "editorMode": "code",
   640            "expr": "max(dataflow_tiflow_server_master_job_num{k8s_cluster=\"$k8s_cluster\"}) by (status)",
   641            "legendFormat": "{{status}}",
   642            "range": true,
   643            "refId": "A"
   644          }
   645        ],
   646        "title": "Num of Job State",
   647        "type": "timeseries"
   648      },
   649      {
   650        "datasource": {
   651          "type": "prometheus",
   652          "uid": "${DS_PROMETHEUS_TIFLOW}"
   653        },
   654        "description": "num of different executor state",
   655        "fieldConfig": {
   656          "defaults": {
   657            "color": {
   658              "mode": "palette-classic"
   659            },
   660            "custom": {
   661              "axisCenteredZero": false,
   662              "axisColorMode": "text",
   663              "axisLabel": "",
   664              "axisPlacement": "auto",
   665              "barAlignment": 0,
   666              "drawStyle": "line",
   667              "fillOpacity": 0,
   668              "gradientMode": "none",
   669              "hideFrom": {
   670                "legend": false,
   671                "tooltip": false,
   672                "viz": false
   673              },
   674              "lineInterpolation": "linear",
   675              "lineWidth": 1,
   676              "pointSize": 5,
   677              "scaleDistribution": {
   678                "type": "linear"
   679              },
   680              "showPoints": "auto",
   681              "spanNulls": false,
   682              "stacking": {
   683                "group": "A",
   684                "mode": "none"
   685              },
   686              "thresholdsStyle": {
   687                "mode": "off"
   688              }
   689            },
   690            "mappings": [],
   691            "thresholds": {
   692              "mode": "absolute",
   693              "steps": [
   694                {
   695                  "color": "green",
   696                  "value": null
   697                },
   698                {
   699                  "color": "red",
   700                  "value": 80
   701                }
   702              ]
   703            }
   704          },
   705          "overrides": []
   706        },
   707        "gridPos": {
   708          "h": 8,
   709          "w": 12,
   710          "x": 12,
   711          "y": 18
   712        },
   713        "id": 16,
   714        "options": {
   715          "legend": {
   716            "calcs": [],
   717            "displayMode": "list",
   718            "placement": "bottom",
   719            "showLegend": true
   720          },
   721          "tooltip": {
   722            "mode": "single",
   723            "sort": "none"
   724          }
   725        },
   726        "targets": [
   727          {
   728            "datasource": {
   729              "type": "prometheus",
   730              "uid": "${DS_PROMETHEUS_TIFLOW}"
   731            },
   732            "editorMode": "code",
   733            "expr": "max(dataflow_tiflow_server_master_executor_num{k8s_cluster=\"$k8s_cluster\"}) by (status)",
   734            "interval": "1",
   735            "legendFormat": "{{status}}",
   736            "range": true,
   737            "refId": "A"
   738          }
   739        ],
   740        "title": "Num of Executor State",
   741        "type": "timeseries"
   742      },
   743      {
   744        "collapsed": false,
   745        "gridPos": {
   746          "h": 1,
   747          "w": 24,
   748          "x": 0,
   749          "y": 26
   750        },
   751        "id": 4,
   752        "panels": [],
   753        "title": "Go Runtime",
   754        "type": "row"
   755      },
   756      {
   757        "datasource": {
   758          "type": "prometheus",
   759          "uid": "${DS_PROMETHEUS_TIFLOW}"
   760        },
   761        "description": "Tiflow process rss memory usage. Tiflow heap memory size in use.",
   762        "fieldConfig": {
   763          "defaults": {
   764            "color": {
   765              "mode": "palette-classic"
   766            },
   767            "custom": {
   768              "axisCenteredZero": false,
   769              "axisColorMode": "text",
   770              "axisLabel": "",
   771              "axisPlacement": "auto",
   772              "axisSoftMin": -1,
   773              "axisWidth": 60,
   774              "barAlignment": 0,
   775              "drawStyle": "line",
   776              "fillOpacity": 0,
   777              "gradientMode": "none",
   778              "hideFrom": {
   779                "legend": false,
   780                "tooltip": false,
   781                "viz": false
   782              },
   783              "lineInterpolation": "linear",
   784              "lineStyle": {
   785                "fill": "solid"
   786              },
   787              "lineWidth": 1,
   788              "pointSize": 5,
   789              "scaleDistribution": {
   790                "type": "linear"
   791              },
   792              "showPoints": "auto",
   793              "spanNulls": false,
   794              "stacking": {
   795                "group": "A",
   796                "mode": "none"
   797              },
   798              "thresholdsStyle": {
   799                "mode": "off"
   800              }
   801            },
   802            "mappings": [],
   803            "thresholds": {
   804              "mode": "absolute",
   805              "steps": [
   806                {
   807                  "color": "green",
   808                  "value": null
   809                },
   810                {
   811                  "color": "red",
   812                  "value": 80
   813                }
   814              ]
   815            },
   816            "unit": "decbytes"
   817          },
   818          "overrides": []
   819        },
   820        "gridPos": {
   821          "h": 8,
   822          "w": 12,
   823          "x": 0,
   824          "y": 27
   825        },
   826        "id": 2,
   827        "options": {
   828          "legend": {
   829            "calcs": [],
   830            "displayMode": "list",
   831            "placement": "bottom",
   832            "showLegend": true
   833          },
   834          "tooltip": {
   835            "mode": "single",
   836            "sort": "none"
   837          }
   838        },
   839        "targets": [
   840          {
   841            "datasource": {
   842              "type": "prometheus",
   843              "uid": "${DS_PROMETHEUS_TIFLOW}"
   844            },
   845            "editorMode": "code",
   846            "expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}",
   847            "hide": false,
   848            "legendFormat": "alloc-from-os-{{instance}}",
   849            "range": true,
   850            "refId": "A"
   851          },
   852          {
   853            "datasource": {
   854              "type": "prometheus",
   855              "uid": "${DS_PROMETHEUS_TIFLOW}"
   856            },
   857            "editorMode": "code",
   858            "expr": "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"} - go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"} + go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}",
   859            "hide": false,
   860            "legendFormat": "reserved-by-go-{{instance}}",
   861            "range": true,
   862            "refId": "B"
   863          },
   864          {
   865            "datasource": {
   866              "type": "prometheus",
   867              "uid": "${DS_PROMETHEUS_TIFLOW}"
   868            },
   869            "editorMode": "code",
   870            "expr": "go_memstats_stack_sys_bytes{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"} + go_memstats_mspan_sys_bytes{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"} + go_memstats_mcache_sys_bytes{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"} + go_memstats_buck_hash_sys_bytes{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"} + go_memstats_gc_sys_bytes{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"} + go_memstats_other_sys_bytes{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}",
   871            "hide": false,
   872            "legendFormat": "used-by-go-{{instance}}",
   873            "range": true,
   874            "refId": "C"
   875          },
   876          {
   877            "datasource": {
   878              "type": "prometheus",
   879              "uid": "${DS_PROMETHEUS_TIFLOW}"
   880            },
   881            "editorMode": "code",
   882            "expr": "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}",
   883            "hide": false,
   884            "legendFormat": "gc-threshold-{{instance}}",
   885            "range": true,
   886            "refId": "D"
   887          },
   888          {
   889            "datasource": {
   890              "type": "prometheus",
   891              "uid": "${DS_PROMETHEUS_TIFLOW}"
   892            },
   893            "editorMode": "code",
   894            "expr": "(clamp_max(idelta(go_memstats_last_gc_time_seconds{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}[1m]), 1) * go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}) > 0",
   895            "hide": false,
   896            "legendFormat": "gc-{{instance}}",
   897            "range": true,
   898            "refId": "E"
   899          }
   900        ],
   901        "title": "Memory Usage",
   902        "type": "timeseries"
   903      },
   904      {
   905        "datasource": {
   906          "type": "prometheus",
   907          "uid": "${DS_PROMETHEUS_TIFLOW}"
   908        },
   909        "description": "Count of live objects.",
   910        "fieldConfig": {
   911          "defaults": {
   912            "color": {
   913              "mode": "palette-classic"
   914            },
   915            "custom": {
   916              "axisCenteredZero": false,
   917              "axisColorMode": "text",
   918              "axisLabel": "",
   919              "axisPlacement": "auto",
   920              "barAlignment": 0,
   921              "drawStyle": "line",
   922              "fillOpacity": 0,
   923              "gradientMode": "none",
   924              "hideFrom": {
   925                "legend": false,
   926                "tooltip": false,
   927                "viz": false
   928              },
   929              "lineInterpolation": "linear",
   930              "lineWidth": 1,
   931              "pointSize": 5,
   932              "scaleDistribution": {
   933                "type": "linear"
   934              },
   935              "showPoints": "auto",
   936              "spanNulls": false,
   937              "stacking": {
   938                "group": "A",
   939                "mode": "none"
   940              },
   941              "thresholdsStyle": {
   942                "mode": "off"
   943              }
   944            },
   945            "mappings": [],
   946            "thresholds": {
   947              "mode": "absolute",
   948              "steps": [
   949                {
   950                  "color": "green",
   951                  "value": null
   952                },
   953                {
   954                  "color": "red",
   955                  "value": 80
   956                }
   957              ]
   958            },
   959            "unit": "locale"
   960          },
   961          "overrides": []
   962        },
   963        "gridPos": {
   964          "h": 8,
   965          "w": 12,
   966          "x": 12,
   967          "y": 27
   968        },
   969        "id": 6,
   970        "options": {
   971          "legend": {
   972            "calcs": [],
   973            "displayMode": "list",
   974            "placement": "bottom",
   975            "showLegend": true
   976          },
   977          "tooltip": {
   978            "mode": "single",
   979            "sort": "none"
   980          }
   981        },
   982        "targets": [
   983          {
   984            "datasource": {
   985              "type": "prometheus",
   986              "uid": "${DS_PROMETHEUS_TIFLOW}"
   987            },
   988            "editorMode": "code",
   989            "expr": "go_memstats_heap_objects{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}",
   990            "legendFormat": "objects-{{instance}}",
   991            "range": true,
   992            "refId": "A"
   993          }
   994        ],
   995        "title": "Estimated Live Objects",
   996        "type": "timeseries"
   997      },
   998      {
   999        "datasource": {
  1000          "type": "prometheus",
  1001          "uid": "${DS_PROMETHEUS_TIFLOW}"
  1002        },
  1003        "description": "Tiflow Engine process Go garbage collection STW pause duration",
  1004        "fieldConfig": {
  1005          "defaults": {
  1006            "color": {
  1007              "mode": "palette-classic"
  1008            },
  1009            "custom": {
  1010              "axisCenteredZero": false,
  1011              "axisColorMode": "text",
  1012              "axisLabel": "",
  1013              "axisPlacement": "auto",
  1014              "barAlignment": 0,
  1015              "drawStyle": "line",
  1016              "fillOpacity": 0,
  1017              "gradientMode": "none",
  1018              "hideFrom": {
  1019                "legend": false,
  1020                "tooltip": false,
  1021                "viz": false
  1022              },
  1023              "lineInterpolation": "linear",
  1024              "lineWidth": 1,
  1025              "pointSize": 5,
  1026              "scaleDistribution": {
  1027                "type": "linear"
  1028              },
  1029              "showPoints": "auto",
  1030              "spanNulls": false,
  1031              "stacking": {
  1032                "group": "A",
  1033                "mode": "none"
  1034              },
  1035              "thresholdsStyle": {
  1036                "mode": "off"
  1037              }
  1038            },
  1039            "decimals": 2,
  1040            "mappings": [],
  1041            "thresholds": {
  1042              "mode": "absolute",
  1043              "steps": [
  1044                {
  1045                  "color": "green",
  1046                  "value": null
  1047                },
  1048                {
  1049                  "color": "red",
  1050                  "value": 80
  1051                }
  1052              ]
  1053            },
  1054            "unit": "s"
  1055          },
  1056          "overrides": []
  1057        },
  1058        "gridPos": {
  1059          "h": 8,
  1060          "w": 12,
  1061          "x": 0,
  1062          "y": 35
  1063        },
  1064        "id": 8,
  1065        "options": {
  1066          "legend": {
  1067            "calcs": [],
  1068            "displayMode": "list",
  1069            "placement": "bottom",
  1070            "showLegend": true
  1071          },
  1072          "tooltip": {
  1073            "mode": "single",
  1074            "sort": "none"
  1075          }
  1076        },
  1077        "targets": [
  1078          {
  1079            "datasource": {
  1080              "type": "prometheus",
  1081              "uid": "${DS_PROMETHEUS_TIFLOW}"
  1082            },
  1083            "editorMode": "code",
  1084            "expr": "go_gc_duration_seconds{quantile=\"0\", k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}",
  1085            "legendFormat": "min-{{instance}}",
  1086            "range": true,
  1087            "refId": "A"
  1088          },
  1089          {
  1090            "datasource": {
  1091              "type": "prometheus",
  1092              "uid": "${DS_PROMETHEUS_TIFLOW}"
  1093            },
  1094            "editorMode": "code",
  1095            "expr": "go_gc_duration_seconds{quantile!~\"0|1\", k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}",
  1096            "hide": false,
  1097            "legendFormat": "{{quantile}}-{{instance}}",
  1098            "range": true,
  1099            "refId": "B"
  1100          },
  1101          {
  1102            "datasource": {
  1103              "type": "prometheus",
  1104              "uid": "${DS_PROMETHEUS_TIFLOW}"
  1105            },
  1106            "editorMode": "code",
  1107            "expr": "go_gc_duration_seconds{quantile=\"1\", k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}",
  1108            "hide": false,
  1109            "legendFormat": "max-{{instance}}",
  1110            "range": true,
  1111            "refId": "C"
  1112          }
  1113        ],
  1114        "title": "GC STW Duration (last 256 GC cycles)",
  1115        "type": "timeseries"
  1116      },
  1117      {
  1118        "datasource": {
  1119          "type": "prometheus",
  1120          "uid": "${DS_PROMETHEUS_TIFLOW}"
  1121        },
  1122        "description": "The throughput of Go's memory allocator.",
  1123        "fieldConfig": {
  1124          "defaults": {
  1125            "color": {
  1126              "mode": "palette-classic"
  1127            },
  1128            "custom": {
  1129              "axisCenteredZero": false,
  1130              "axisColorMode": "text",
  1131              "axisLabel": "",
  1132              "axisPlacement": "auto",
  1133              "barAlignment": 0,
  1134              "drawStyle": "line",
  1135              "fillOpacity": 0,
  1136              "gradientMode": "none",
  1137              "hideFrom": {
  1138                "legend": false,
  1139                "tooltip": false,
  1140                "viz": false
  1141              },
  1142              "lineInterpolation": "linear",
  1143              "lineWidth": 1,
  1144              "pointSize": 5,
  1145              "scaleDistribution": {
  1146                "type": "linear"
  1147              },
  1148              "showPoints": "auto",
  1149              "spanNulls": false,
  1150              "stacking": {
  1151                "group": "A",
  1152                "mode": "none"
  1153              },
  1154              "thresholdsStyle": {
  1155                "mode": "off"
  1156              }
  1157            },
  1158            "mappings": [],
  1159            "thresholds": {
  1160              "mode": "absolute",
  1161              "steps": [
  1162                {
  1163                  "color": "green",
  1164                  "value": null
  1165                },
  1166                {
  1167                  "color": "red",
  1168                  "value": 80
  1169                }
  1170              ]
  1171            },
  1172            "unit": "Bps"
  1173          },
  1174          "overrides": []
  1175        },
  1176        "gridPos": {
  1177          "h": 8,
  1178          "w": 12,
  1179          "x": 12,
  1180          "y": 35
  1181        },
  1182        "id": 10,
  1183        "options": {
  1184          "legend": {
  1185            "calcs": [],
  1186            "displayMode": "list",
  1187            "placement": "bottom",
  1188            "showLegend": true
  1189          },
  1190          "tooltip": {
  1191            "mode": "single",
  1192            "sort": "none"
  1193          }
  1194        },
  1195        "targets": [
  1196          {
  1197            "datasource": {
  1198              "type": "prometheus",
  1199              "uid": "${DS_PROMETHEUS_TIFLOW}"
  1200            },
  1201            "editorMode": "code",
  1202            "expr": "irate(go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}[30s])",
  1203            "hide": false,
  1204            "legendFormat": "alloc-{{instance}}",
  1205            "range": true,
  1206            "refId": "A"
  1207          },
  1208          {
  1209            "datasource": {
  1210              "type": "prometheus",
  1211              "uid": "${DS_PROMETHEUS_TIFLOW}"
  1212            },
  1213            "editorMode": "code",
  1214            "expr": "irate((go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"})[30s:])",
  1215            "hide": false,
  1216            "legendFormat": "sweep-{{instance}}",
  1217            "range": true,
  1218            "refId": "B"
  1219          },
  1220          {
  1221            "datasource": {
  1222              "type": "prometheus",
  1223              "uid": "${DS_PROMETHEUS_TIFLOW}"
  1224            },
  1225            "editorMode": "code",
  1226            "expr": "irate(go_memstats_mallocs_total{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}[30s])",
  1227            "hide": false,
  1228            "legendFormat": "alloc-ops-{{instance}}",
  1229            "range": true,
  1230            "refId": "C"
  1231          },
  1232          {
  1233            "datasource": {
  1234              "type": "prometheus",
  1235              "uid": "${DS_PROMETHEUS_TIFLOW}"
  1236            },
  1237            "editorMode": "code",
  1238            "expr": "irate(go_memstats_frees_total{k8s_cluster=\"$k8s_cluster\", instance=~\"$instance\"}[30s])",
  1239            "hide": false,
  1240            "legendFormat": "sweep-ops-{{instance}}",
  1241            "range": true,
  1242            "refId": "D"
  1243          }
  1244        ],
  1245        "title": "Allocator Throughput",
  1246        "type": "timeseries"
  1247      }
  1248    ],
  1249    "refresh": false,
  1250    "schemaVersion": 37,
  1251    "style": "dark",
  1252    "tags": [],
  1253    "templating": {
  1254      "list": [
  1255        {
  1256          "current": {},
  1257          "datasource": {
  1258            "type": "prometheus",
  1259            "uid": "${DS_PROMETHEUS_TIFLOW}"
  1260          },
  1261          "definition": "label_values(go_goroutines, k8s_cluster)",
  1262          "hide": 0,
  1263          "includeAll": false,
  1264          "label": "K8s_cluster",
  1265          "multi": false,
  1266          "name": "k8s_cluster",
  1267          "options": [],
  1268          "query": {
  1269            "query": "label_values(go_goroutines, k8s_cluster)",
  1270            "refId": "StandardVariableQuery"
  1271          },
  1272          "refresh": 1,
  1273          "regex": "",
  1274          "skipUrlSync": false,
  1275          "sort": 1,
  1276          "type": "query"
  1277        },
  1278        {
  1279          "allValue": ".*",
  1280          "current": {},
  1281          "datasource": {
  1282            "type": "prometheus",
  1283            "uid": "${DS_PROMETHEUS_TIFLOW}"
  1284          },
  1285          "definition": "label_values(go_goroutines, instance)",
  1286          "hide": 0,
  1287          "includeAll": true,
  1288          "label": "instance",
  1289          "multi": true,
  1290          "name": "instance",
  1291          "options": [],
  1292          "query": {
  1293            "query": "label_values(go_goroutines, instance)",
  1294            "refId": "StandardVariableQuery"
  1295          },
  1296          "refresh": 1,
  1297          "regex": "",
  1298          "skipUrlSync": false,
  1299          "sort": 1,
  1300          "type": "query"
  1301        }
  1302      ]
  1303    },
  1304    "time": {
  1305      "from": "now-6h",
  1306      "to": "now"
  1307    },
  1308    "timepicker": {},
  1309    "timezone": "",
  1310    "title": "Cluster-Tiflow",
  1311    "uid": "tFhyuFG4z",
  1312    "version": 16,
  1313    "weekStart": ""
  1314  }