github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/metrics/grafana/ticdc.json (about)

     1  {
     2    "__inputs": [
     3      {
     4        "name": "DS_TEST-CLUSTER",
     5        "label": "${DS_TEST-CLUSTER}",
     6        "description": "",
     7        "type": "datasource",
     8        "pluginId": "prometheus",
     9        "pluginName": "Prometheus"
    10      }
    11    ],
    12    "__requires": [
    13      {
    14        "type": "grafana",
    15        "id": "grafana",
    16        "name": "Grafana",
    17        "version": "7.5.11"
    18      },
    19      {
    20        "type": "panel",
    21        "id": "graph",
    22        "name": "Graph",
    23        "version": ""
    24      },
    25      {
    26        "type": "panel",
    27        "id": "heatmap",
    28        "name": "Heatmap",
    29        "version": ""
    30      },
    31      {
    32        "type": "datasource",
    33        "id": "prometheus",
    34        "name": "Prometheus",
    35        "version": "1.0.0"
    36      },
    37      {
    38        "type": "panel",
    39        "id": "table",
    40        "name": "Table",
    41        "version": ""
    42      },
    43      {
    44        "type": "panel",
    45        "id": "timeseries",
    46        "name": "Time series",
    47        "version": ""
    48      }
    49    ],
    50    "annotations": {
    51      "list": [
    52        {
    53          "builtIn": 1,
    54          "datasource": "-- Grafana --",
    55          "enable": false,
    56          "expr": "",
    57          "hide": true,
    58          "iconColor": "#F2495C",
    59          "limit": 100,
    60          "name": "",
    61          "showIn": 0,
    62          "tagKeys": "",
    63          "textFormat": "",
    64          "titleFormat": "",
    65          "type": "dashboard",
    66          "useValueForTime": false
    67        },
    68        {
    69          "datasource": "${DS_TEST-CLUSTER}",
    70          "enable": true,
    71          "expr": "max(ticdc_owner_checkpoint_ts_lag) by (changefeed, instance) > BOOL $spike_threshold",
    72          "hide": true,
    73          "iconColor": "#F2495C",
    74          "limit": 100,
    75          "name": "Latency spike",
    76          "showIn": 0,
    77          "tagKeys": "changefeed",
    78          "tags": [],
    79          "titleFormat": "Latency spike",
    80          "type": "tags",
    81          "useValueForTime": false
    82        },
    83        {
    84          "datasource": "${DS_TEST-CLUSTER}",
    85          "enable": false,
    86          "expr": "delta(up{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=~\"tikv|ticdc|tidb|pd\"}[30s]) < BOOL 0",
    87          "hide": false,
    88          "iconColor": "#FF9830",
    89          "limit": 100,
    90          "name": "Server down",
    91          "showIn": 0,
    92          "step": "15s",
    93          "tagKeys": "instance,job",
    94          "tags": [],
    95          "textFormat": "",
    96          "titleFormat": "Down",
    97          "type": "tags"
    98        },
    99        {
   100          "datasource": "${DS_TEST-CLUSTER}",
   101          "enable": false,
   102          "expr": "sum(ALERTS{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", alertstate=\"firing\", alertname=~\"ticdc.*\"}) by (alertname) > BOOL 0",
   103          "hide": false,
   104          "iconColor": "#B877D9",
   105          "limit": 100,
   106          "name": "All TiCDC alerts",
   107          "showIn": 0,
   108          "tagKeys": "alertname",
   109          "tags": [],
   110          "titleFormat": "Alert Name",
   111          "type": "tags"
   112        },
   113        {
   114          "datasource": "${DS_TEST-CLUSTER}",
   115          "enable": false,
   116          "expr": "delta(tikv_cdc_region_resolve_status{status=\"resolved\"}[30s]) < BOOL -800",
   117          "hide": false,
   118          "iconColor": "rgba(255, 96, 96, 1)",
   119          "limit": 100,
   120          "name": "Resolved region drop",
   121          "showIn": 0,
   122          "step": "15s",
   123          "tagKeys": "instance",
   124          "tags": [],
   125          "titleFormat": "Resolved region drop",
   126          "type": "tags"
   127        }
   128      ]
   129    },
   130    "editable": true,
   131    "gnetId": null,
   132    "graphTooltip": 1,
   133    "id": null,
   134    "iteration": 1705029240982,
   135    "links": [],
   136    "panels": [
   137      {
   138        "collapsed": true,
   139        "datasource": null,
   140        "gridPos": {
   141          "h": 1,
   142          "w": 24,
   143          "x": 0,
   144          "y": 0
   145        },
   146        "id": 397,
   147        "panels": [
   148          {
   149            "aliasColors": {},
   150            "bars": false,
   151            "dashLength": 10,
   152            "dashes": false,
   153            "datasource": "${DS_TEST-CLUSTER}",
   154            "description": "The lag between changefeed checkpoint ts and the latest ts of upstream TiDB.",
   155            "fieldConfig": {
   156              "defaults": {
   157                "links": []
   158              },
   159              "overrides": []
   160            },
   161            "fill": 1,
   162            "fillGradient": 0,
   163            "gridPos": {
   164              "h": 4,
   165              "w": 12,
   166              "x": 0,
   167              "y": 1
   168            },
   169            "hiddenSeries": false,
   170            "id": 398,
   171            "legend": {
   172              "alignAsTable": true,
   173              "avg": false,
   174              "current": true,
   175              "max": false,
   176              "min": false,
   177              "rightSide": true,
   178              "show": false,
   179              "total": false,
   180              "values": true
   181            },
   182            "lines": true,
   183            "linewidth": 1,
   184            "links": [],
   185            "nullPointMode": "null",
   186            "options": {
   187              "alertThreshold": true
   188            },
   189            "paceLength": 10,
   190            "percentage": false,
   191            "pluginVersion": "7.5.11",
   192            "pointradius": 2,
   193            "points": false,
   194            "renderer": "flot",
   195            "seriesOverrides": [],
   196            "spaceLength": 10,
   197            "stack": false,
   198            "steppedLine": false,
   199            "targets": [
   200              {
   201                "exemplar": true,
   202                "expr": "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",  changefeed=~\"$changefeed\"}) by (namespace, changefeed)",
   203                "format": "time_series",
   204                "interval": "",
   205                "intervalFactor": 1,
   206                "legendFormat": "{{namespace}}-{{changefeed}}",
   207                "refId": "A"
   208              }
   209            ],
   210            "thresholds": [],
   211            "timeFrom": null,
   212            "timeRegions": [],
   213            "timeShift": null,
   214            "title": "Changefeed checkpoint lag",
   215            "tooltip": {
   216              "shared": true,
   217              "sort": 0,
   218              "value_type": "individual"
   219            },
   220            "type": "graph",
   221            "xaxis": {
   222              "buckets": null,
   223              "mode": "time",
   224              "name": null,
   225              "show": true,
   226              "values": []
   227            },
   228            "yaxes": [
   229              {
   230                "format": "s",
   231                "label": null,
   232                "logBase": 1,
   233                "max": null,
   234                "min": "0",
   235                "show": true
   236              },
   237              {
   238                "format": "short",
   239                "label": null,
   240                "logBase": 1,
   241                "max": null,
   242                "min": null,
   243                "show": true
   244              }
   245            ],
   246            "yaxis": {
   247              "align": false,
   248              "alignLevel": null
   249            }
   250          },
   251          {
   252            "aliasColors": {},
   253            "bars": false,
   254            "dashLength": 10,
   255            "dashes": false,
   256            "datasource": "${DS_TEST-CLUSTER}",
   257            "description": "The lag between changefeed resolved ts and the latest ts of upstream TiDB.",
   258            "fieldConfig": {
   259              "defaults": {
   260                "links": []
   261              },
   262              "overrides": []
   263            },
   264            "fill": 1,
   265            "fillGradient": 0,
   266            "gridPos": {
   267              "h": 4,
   268              "w": 12,
   269              "x": 12,
   270              "y": 1
   271            },
   272            "hiddenSeries": false,
   273            "id": 468,
   274            "legend": {
   275              "alignAsTable": true,
   276              "avg": false,
   277              "current": true,
   278              "max": false,
   279              "min": false,
   280              "rightSide": true,
   281              "show": false,
   282              "total": false,
   283              "values": true
   284            },
   285            "lines": true,
   286            "linewidth": 1,
   287            "links": [],
   288            "nullPointMode": "null",
   289            "options": {
   290              "alertThreshold": true
   291            },
   292            "paceLength": 10,
   293            "percentage": false,
   294            "pluginVersion": "7.5.11",
   295            "pointradius": 2,
   296            "points": false,
   297            "renderer": "flot",
   298            "seriesOverrides": [],
   299            "spaceLength": 10,
   300            "stack": false,
   301            "steppedLine": false,
   302            "targets": [
   303              {
   304                "exemplar": true,
   305                "expr": "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namesapce=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)",
   306                "format": "time_series",
   307                "interval": "",
   308                "intervalFactor": 1,
   309                "legendFormat": "{{namespace}}-{{changefeed}}-resolvedts",
   310                "refId": "A"
   311              }
   312            ],
   313            "thresholds": [],
   314            "timeFrom": null,
   315            "timeRegions": [],
   316            "timeShift": null,
   317            "title": "Changefeed resolved ts lag",
   318            "tooltip": {
   319              "shared": true,
   320              "sort": 0,
   321              "value_type": "individual"
   322            },
   323            "type": "graph",
   324            "xaxis": {
   325              "buckets": null,
   326              "mode": "time",
   327              "name": null,
   328              "show": true,
   329              "values": []
   330            },
   331            "yaxes": [
   332              {
   333                "format": "s",
   334                "label": null,
   335                "logBase": 1,
   336                "max": null,
   337                "min": "0",
   338                "show": true
   339              },
   340              {
   341                "format": "short",
   342                "label": null,
   343                "logBase": 1,
   344                "max": null,
   345                "min": null,
   346                "show": true
   347              }
   348            ],
   349            "yaxis": {
   350              "align": false,
   351              "alignLevel": null
   352            }
   353          },
   354          {
   355            "aliasColors": {},
   356            "bars": true,
   357            "dashLength": 10,
   358            "dashes": false,
   359            "datasource": "${DS_TEST-CLUSTER}",
   360            "description": "The number of errors that interrupt Eventfeed RPC",
   361            "fieldConfig": {
   362              "defaults": {
   363                "links": []
   364              },
   365              "overrides": []
   366            },
   367            "fill": 1,
   368            "fillGradient": 0,
   369            "gridPos": {
   370              "h": 4,
   371              "w": 12,
   372              "x": 0,
   373              "y": 5
   374            },
   375            "hiddenSeries": false,
   376            "id": 469,
   377            "legend": {
   378              "alignAsTable": false,
   379              "avg": false,
   380              "current": false,
   381              "hideEmpty": true,
   382              "hideZero": true,
   383              "max": false,
   384              "min": false,
   385              "rightSide": false,
   386              "show": true,
   387              "total": false,
   388              "values": false
   389            },
   390            "lines": true,
   391            "linewidth": 1,
   392            "links": [],
   393            "nullPointMode": "null",
   394            "options": {
   395              "alertThreshold": true
   396            },
   397            "paceLength": 10,
   398            "percentage": false,
   399            "pluginVersion": "7.5.11",
   400            "pointradius": 2,
   401            "points": false,
   402            "renderer": "flot",
   403            "seriesOverrides": [],
   404            "spaceLength": 10,
   405            "stack": false,
   406            "steppedLine": false,
   407            "targets": [
   408              {
   409                "expr": "sum(increase(ticdc_kvclient_event_feed_error_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (type)",
   410                "format": "time_series",
   411                "hide": false,
   412                "interval": "1m",
   413                "intervalFactor": 1,
   414                "legendFormat": "{{type}}",
   415                "refId": "A"
   416              }
   417            ],
   418            "thresholds": [],
   419            "timeFrom": null,
   420            "timeRegions": [],
   421            "timeShift": null,
   422            "title": "Eventfeed error/m",
   423            "tooltip": {
   424              "shared": true,
   425              "sort": 0,
   426              "value_type": "individual"
   427            },
   428            "type": "graph",
   429            "xaxis": {
   430              "buckets": null,
   431              "mode": "time",
   432              "name": null,
   433              "show": true,
   434              "values": []
   435            },
   436            "yaxes": [
   437              {
   438                "format": "short",
   439                "label": null,
   440                "logBase": 1,
   441                "max": null,
   442                "min": null,
   443                "show": true
   444              },
   445              {
   446                "format": "short",
   447                "label": null,
   448                "logBase": 1,
   449                "max": null,
   450                "min": null,
   451                "show": true
   452              }
   453            ],
   454            "yaxis": {
   455              "align": false,
   456              "alignLevel": null
   457            }
   458          },
   459          {
   460            "aliasColors": {},
   461            "bars": true,
   462            "dashLength": 10,
   463            "dashes": false,
   464            "datasource": "${DS_TEST-CLUSTER}",
   465            "description": "The number of PD scheduling operator.",
   466            "fieldConfig": {
   467              "defaults": {
   468                "links": []
   469              },
   470              "overrides": []
   471            },
   472            "fill": 1,
   473            "fillGradient": 0,
   474            "gridPos": {
   475              "h": 4,
   476              "w": 12,
   477              "x": 12,
   478              "y": 5
   479            },
   480            "hiddenSeries": false,
   481            "id": 466,
   482            "legend": {
   483              "alignAsTable": false,
   484              "avg": false,
   485              "current": false,
   486              "hideEmpty": true,
   487              "hideZero": true,
   488              "max": false,
   489              "min": false,
   490              "rightSide": false,
   491              "show": true,
   492              "total": false,
   493              "values": false
   494            },
   495            "lines": true,
   496            "linewidth": 1,
   497            "links": [],
   498            "nullPointMode": "null",
   499            "options": {
   500              "alertThreshold": true
   501            },
   502            "paceLength": 10,
   503            "percentage": false,
   504            "pluginVersion": "7.5.11",
   505            "pointradius": 2,
   506            "points": false,
   507            "renderer": "flot",
   508            "seriesOverrides": [],
   509            "spaceLength": 10,
   510            "stack": false,
   511            "steppedLine": false,
   512            "targets": [
   513              {
   514                "expr": "sum(increase(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", event=\"create\"}[1m])) by (type)",
   515                "format": "time_series",
   516                "hide": false,
   517                "interval": "1m",
   518                "intervalFactor": 1,
   519                "legendFormat": "{{type}}",
   520                "refId": "A"
   521              }
   522            ],
   523            "thresholds": [],
   524            "timeFrom": null,
   525            "timeRegions": [],
   526            "timeShift": null,
   527            "title": "PD operator/m",
   528            "tooltip": {
   529              "shared": true,
   530              "sort": 0,
   531              "value_type": "individual"
   532            },
   533            "type": "graph",
   534            "xaxis": {
   535              "buckets": null,
   536              "mode": "time",
   537              "name": null,
   538              "show": true,
   539              "values": []
   540            },
   541            "yaxes": [
   542              {
   543                "format": "short",
   544                "label": null,
   545                "logBase": 1,
   546                "max": null,
   547                "min": null,
   548                "show": true
   549              },
   550              {
   551                "format": "short",
   552                "label": null,
   553                "logBase": 1,
   554                "max": null,
   555                "min": null,
   556                "show": true
   557              }
   558            ],
   559            "yaxis": {
   560              "align": false,
   561              "alignLevel": null
   562            }
   563          },
   564          {
   565            "aliasColors": {},
   566            "bars": false,
   567            "dashLength": 10,
   568            "dashes": false,
   569            "datasource": "${DS_TEST-CLUSTER}",
   570            "description": "99.9% of TiDB query durations.",
   571            "fieldConfig": {
   572              "defaults": {},
   573              "overrides": []
   574            },
   575            "fill": 1,
   576            "fillGradient": 0,
   577            "gridPos": {
   578              "h": 4,
   579              "w": 12,
   580              "x": 0,
   581              "y": 9
   582            },
   583            "hiddenSeries": false,
   584            "id": 479,
   585            "legend": {
   586              "alignAsTable": false,
   587              "avg": false,
   588              "current": false,
   589              "hideEmpty": false,
   590              "hideZero": false,
   591              "max": false,
   592              "min": false,
   593              "rightSide": false,
   594              "show": true,
   595              "total": false,
   596              "values": false
   597            },
   598            "lines": true,
   599            "linewidth": 1,
   600            "links": [],
   601            "nullPointMode": "null as zero",
   602            "options": {
   603              "alertThreshold": true
   604            },
   605            "percentage": false,
   606            "pluginVersion": "7.5.11",
   607            "pointradius": 5,
   608            "points": false,
   609            "renderer": "flot",
   610            "seriesOverrides": [],
   611            "spaceLength": 10,
   612            "stack": false,
   613            "steppedLine": false,
   614            "targets": [
   615              {
   616                "expr": "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", sql_type!~\"internal|Use|Show\"}[1m])) by (le, sql_type))",
   617                "format": "time_series",
   618                "intervalFactor": 2,
   619                "legendFormat": "{{sql_type}}",
   620                "refId": "B"
   621              }
   622            ],
   623            "thresholds": [],
   624            "timeFrom": null,
   625            "timeRegions": [],
   626            "timeShift": null,
   627            "title": "TiDB query duration",
   628            "tooltip": {
   629              "shared": true,
   630              "sort": 0,
   631              "value_type": "individual"
   632            },
   633            "type": "graph",
   634            "xaxis": {
   635              "buckets": null,
   636              "mode": "time",
   637              "name": null,
   638              "show": true,
   639              "values": []
   640            },
   641            "yaxes": [
   642              {
   643                "format": "s",
   644                "label": null,
   645                "logBase": 1,
   646                "max": null,
   647                "min": "0",
   648                "show": true
   649              },
   650              {
   651                "format": "short",
   652                "label": null,
   653                "logBase": 1,
   654                "max": null,
   655                "min": null,
   656                "show": true
   657              }
   658            ],
   659            "yaxis": {
   660              "align": false,
   661              "alignLevel": null
   662            }
   663          },
   664          {
   665            "aliasColors": {},
   666            "bars": false,
   667            "dashLength": 10,
   668            "dashes": false,
   669            "datasource": "${DS_TEST-CLUSTER}",
   670            "decimals": 1,
   671            "description": "The min resolved ts lag of each TiKV",
   672            "editable": true,
   673            "error": false,
   674            "fieldConfig": {
   675              "defaults": {},
   676              "overrides": []
   677            },
   678            "fill": 0,
   679            "fillGradient": 0,
   680            "grid": {},
   681            "gridPos": {
   682              "h": 4,
   683              "w": 12,
   684              "x": 12,
   685              "y": 9
   686            },
   687            "hiddenSeries": false,
   688            "id": 473,
   689            "legend": {
   690              "alignAsTable": true,
   691              "avg": false,
   692              "current": true,
   693              "max": false,
   694              "min": false,
   695              "rightSide": false,
   696              "show": false,
   697              "sideWidth": null,
   698              "sort": null,
   699              "sortDesc": null,
   700              "total": false,
   701              "values": true
   702            },
   703            "lines": true,
   704            "linewidth": 1,
   705            "links": [],
   706            "nullPointMode": "null",
   707            "options": {
   708              "alertThreshold": true
   709            },
   710            "paceLength": 10,
   711            "percentage": false,
   712            "pluginVersion": "7.5.11",
   713            "pointradius": 1,
   714            "points": false,
   715            "renderer": "flot",
   716            "seriesOverrides": [
   717              {
   718                "alias": "/.*method/",
   719                "lines": true,
   720                "linewidth": 3,
   721                "points": false,
   722                "yaxis": 2
   723              },
   724              {
   725                "alias": "/.*-lag/",
   726                "bars": true,
   727                "fill": 1
   728              }
   729            ],
   730            "spaceLength": 10,
   731            "stack": false,
   732            "steppedLine": false,
   733            "targets": [
   734              {
   735                "expr": "scalar(max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}))/1000 - avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000) by (instance) > 0",
   736                "format": "time_series",
   737                "hide": false,
   738                "instant": false,
   739                "intervalFactor": 2,
   740                "legendFormat": "{{instance}}-min-resolved-lag",
   741                "refId": "A",
   742                "step": 10
   743              },
   744              {
   745                "expr": "tikv_cdc_resolved_ts_advance_method{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}",
   746                "format": "time_series",
   747                "hide": false,
   748                "intervalFactor": 2,
   749                "legendFormat": "{{instance}}-resolved_ts_advance_method",
   750                "refId": "B",
   751                "step": 10
   752              }
   753            ],
   754            "thresholds": [],
   755            "timeFrom": null,
   756            "timeRegions": [],
   757            "timeShift": null,
   758            "title": "TiKV min resolved ts lag",
   759            "tooltip": {
   760              "msResolution": false,
   761              "shared": true,
   762              "sort": 0,
   763              "value_type": "individual"
   764            },
   765            "type": "graph",
   766            "xaxis": {
   767              "buckets": null,
   768              "mode": "time",
   769              "name": null,
   770              "show": true,
   771              "values": []
   772            },
   773            "yaxes": [
   774              {
   775                "format": "s",
   776                "label": null,
   777                "logBase": 1,
   778                "max": null,
   779                "min": "0",
   780                "show": true
   781              },
   782              {
   783                "format": "none",
   784                "label": null,
   785                "logBase": 1,
   786                "max": null,
   787                "min": null,
   788                "show": true
   789              }
   790            ],
   791            "yaxis": {
   792              "align": false,
   793              "alignLevel": null
   794            }
   795          },
   796          {
   797            "aliasColors": {},
   798            "bars": false,
   799            "dashLength": 10,
   800            "dashes": false,
   801            "datasource": "${DS_TEST-CLUSTER}",
   802            "description": "The number of changed rows that are written to downstream per second",
   803            "fieldConfig": {
   804              "defaults": {
   805                "links": []
   806              },
   807              "overrides": []
   808            },
   809            "fill": 1,
   810            "fillGradient": 0,
   811            "gridPos": {
   812              "h": 4,
   813              "w": 12,
   814              "x": 0,
   815              "y": 13
   816            },
   817            "hiddenSeries": false,
   818            "id": 486,
   819            "legend": {
   820              "alignAsTable": false,
   821              "avg": false,
   822              "current": false,
   823              "max": false,
   824              "min": false,
   825              "rightSide": false,
   826              "show": true,
   827              "total": false,
   828              "values": false
   829            },
   830            "lines": true,
   831            "linewidth": 1,
   832            "links": [],
   833            "nullPointMode": "null",
   834            "options": {
   835              "alertThreshold": true
   836            },
   837            "paceLength": 10,
   838            "percentage": false,
   839            "pluginVersion": "7.5.11",
   840            "pointradius": 2,
   841            "points": false,
   842            "renderer": "flot",
   843            "seriesOverrides": [],
   844            "spaceLength": 10,
   845            "stack": false,
   846            "steppedLine": false,
   847            "targets": [
   848              {
   849                "exemplar": true,
   850                "expr": "sum (rate(ticdc_sink_txn_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed)",
   851                "format": "time_series",
   852                "interval": "",
   853                "intervalFactor": 1,
   854                "legendFormat": "{{namespace}}-{{changefeed}}",
   855                "refId": "A"
   856              },
   857              {
   858                "exemplar": true,
   859                "expr": "sum (rate(ticdc_sink_ddl_exec_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed)",
   860                "format": "time_series",
   861                "interval": "",
   862                "intervalFactor": 1,
   863                "legendFormat": "{{namespace}}-{{changefeed}}-ddl",
   864                "refId": "B"
   865              }
   866            ],
   867            "thresholds": [],
   868            "timeFrom": null,
   869            "timeRegions": [],
   870            "timeShift": null,
   871            "title": "Sink write rows/s",
   872            "tooltip": {
   873              "shared": true,
   874              "sort": 0,
   875              "value_type": "individual"
   876            },
   877            "type": "graph",
   878            "xaxis": {
   879              "buckets": null,
   880              "mode": "time",
   881              "name": null,
   882              "show": true,
   883              "values": []
   884            },
   885            "yaxes": [
   886              {
   887                "format": "none",
   888                "label": null,
   889                "logBase": 1,
   890                "max": null,
   891                "min": null,
   892                "show": true
   893              },
   894              {
   895                "format": "short",
   896                "label": null,
   897                "logBase": 1,
   898                "max": null,
   899                "min": null,
   900                "show": true
   901              }
   902            ],
   903            "yaxis": {
   904              "align": false,
   905              "alignLevel": null
   906            }
   907          },
   908          {
   909            "aliasColors": {},
   910            "bars": false,
   911            "dashLength": 10,
   912            "dashes": false,
   913            "datasource": "${DS_TEST-CLUSTER}",
   914            "description": "99.9% of TiCDC sink write durations.",
   915            "fieldConfig": {
   916              "defaults": {},
   917              "overrides": []
   918            },
   919            "fill": 1,
   920            "fillGradient": 0,
   921            "gridPos": {
   922              "h": 4,
   923              "w": 12,
   924              "x": 12,
   925              "y": 13
   926            },
   927            "hiddenSeries": false,
   928            "id": 485,
   929            "legend": {
   930              "alignAsTable": false,
   931              "avg": false,
   932              "current": false,
   933              "hideEmpty": false,
   934              "hideZero": false,
   935              "max": false,
   936              "min": false,
   937              "rightSide": false,
   938              "show": true,
   939              "total": false,
   940              "values": false
   941            },
   942            "lines": true,
   943            "linewidth": 1,
   944            "links": [],
   945            "nullPointMode": "null as zero",
   946            "options": {
   947              "alertThreshold": true
   948            },
   949            "percentage": false,
   950            "pluginVersion": "7.5.11",
   951            "pointradius": 5,
   952            "points": false,
   953            "renderer": "flot",
   954            "seriesOverrides": [],
   955            "spaceLength": 10,
   956            "stack": false,
   957            "steppedLine": false,
   958            "targets": [
   959              {
   960                "exemplar": true,
   961                "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_worker_flush_duration{tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[1m])) by (le, namespace,changefeed))",
   962                "format": "time_series",
   963                "interval": "",
   964                "intervalFactor": 2,
   965                "legendFormat": "{{namespace}}-{{changefeed}}",
   966                "refId": "B"
   967              },
   968              {
   969                "exemplar": true,
   970                "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[1m])) by (le, namespace, changefeed))",
   971                "format": "time_series",
   972                "interval": "",
   973                "intervalFactor": 2,
   974                "legendFormat": "{{namespace}}-{{changefeed}}-ddl",
   975                "refId": "A"
   976              }
   977            ],
   978            "thresholds": [],
   979            "timeFrom": null,
   980            "timeRegions": [],
   981            "timeShift": null,
   982            "title": "Sink write duration",
   983            "tooltip": {
   984              "shared": true,
   985              "sort": 0,
   986              "value_type": "individual"
   987            },
   988            "type": "graph",
   989            "xaxis": {
   990              "buckets": null,
   991              "mode": "time",
   992              "name": null,
   993              "show": true,
   994              "values": []
   995            },
   996            "yaxes": [
   997              {
   998                "format": "s",
   999                "label": null,
  1000                "logBase": 1,
  1001                "max": null,
  1002                "min": "0",
  1003                "show": true
  1004              },
  1005              {
  1006                "format": "short",
  1007                "label": null,
  1008                "logBase": 1,
  1009                "max": null,
  1010                "min": null,
  1011                "show": true
  1012              }
  1013            ],
  1014            "yaxis": {
  1015              "align": false,
  1016              "alignLevel": null
  1017            }
  1018          },
  1019          {
  1020            "aliasColors": {},
  1021            "bars": false,
  1022            "dashLength": 10,
  1023            "dashes": false,
  1024            "datasource": "${DS_TEST-CLUSTER}",
  1025            "decimals": 1,
  1026            "description": "The number of incremental scan tasks per minute",
  1027            "fieldConfig": {
  1028              "defaults": {},
  1029              "overrides": []
  1030            },
  1031            "fill": 1,
  1032            "fillGradient": 0,
  1033            "gridPos": {
  1034              "h": 4,
  1035              "w": 12,
  1036              "x": 0,
  1037              "y": 17
  1038            },
  1039            "hiddenSeries": false,
  1040            "id": 401,
  1041            "legend": {
  1042              "alignAsTable": true,
  1043              "avg": false,
  1044              "current": true,
  1045              "hideEmpty": false,
  1046              "hideZero": true,
  1047              "max": true,
  1048              "min": false,
  1049              "rightSide": false,
  1050              "show": false,
  1051              "sideWidth": null,
  1052              "sort": "current",
  1053              "sortDesc": true,
  1054              "total": false,
  1055              "values": true
  1056            },
  1057            "lines": true,
  1058            "linewidth": 1,
  1059            "links": [],
  1060            "nullPointMode": "null",
  1061            "options": {
  1062              "alertThreshold": true
  1063            },
  1064            "paceLength": 10,
  1065            "percentage": false,
  1066            "pluginVersion": "7.5.11",
  1067            "pointradius": 5,
  1068            "points": false,
  1069            "renderer": "flot",
  1070            "seriesOverrides": [],
  1071            "spaceLength": 10,
  1072            "stack": false,
  1073            "steppedLine": false,
  1074            "targets": [
  1075              {
  1076                "expr": "sum(increase(tikv_cdc_scan_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (type, instance)",
  1077                "format": "time_series",
  1078                "hide": false,
  1079                "intervalFactor": 1,
  1080                "legendFormat": "{{instance}}",
  1081                "refId": "A"
  1082              }
  1083            ],
  1084            "thresholds": [],
  1085            "timeFrom": null,
  1086            "timeRegions": [],
  1087            "timeShift": null,
  1088            "title": "TiKV scan tasks/m",
  1089            "tooltip": {
  1090              "shared": true,
  1091              "sort": 0,
  1092              "value_type": "individual"
  1093            },
  1094            "type": "graph",
  1095            "xaxis": {
  1096              "buckets": null,
  1097              "mode": "time",
  1098              "name": null,
  1099              "show": true,
  1100              "values": []
  1101            },
  1102            "yaxes": [
  1103              {
  1104                "format": "none",
  1105                "label": null,
  1106                "logBase": 1,
  1107                "max": null,
  1108                "min": null,
  1109                "show": true
  1110              },
  1111              {
  1112                "format": "none",
  1113                "label": null,
  1114                "logBase": 1,
  1115                "max": null,
  1116                "min": null,
  1117                "show": true
  1118              }
  1119            ],
  1120            "yaxis": {
  1121              "align": false,
  1122              "alignLevel": null
  1123            }
  1124          },
  1125          {
  1126            "aliasColors": {},
  1127            "bars": false,
  1128            "dashLength": 10,
  1129            "dashes": false,
  1130            "datasource": "${DS_TEST-CLUSTER}",
  1131            "decimals": 1,
  1132            "description": "The total time of incremental scan region takes per minute",
  1133            "editable": true,
  1134            "error": false,
  1135            "fieldConfig": {
  1136              "defaults": {},
  1137              "overrides": []
  1138            },
  1139            "fill": 0,
  1140            "fillGradient": 0,
  1141            "grid": {},
  1142            "gridPos": {
  1143              "h": 4,
  1144              "w": 12,
  1145              "x": 12,
  1146              "y": 17
  1147            },
  1148            "hiddenSeries": false,
  1149            "id": 467,
  1150            "legend": {
  1151              "alignAsTable": true,
  1152              "avg": false,
  1153              "current": true,
  1154              "max": true,
  1155              "min": false,
  1156              "rightSide": true,
  1157              "show": false,
  1158              "sideWidth": null,
  1159              "sort": null,
  1160              "sortDesc": null,
  1161              "total": false,
  1162              "values": true
  1163            },
  1164            "lines": true,
  1165            "linewidth": 1,
  1166            "links": [],
  1167            "nullPointMode": "null",
  1168            "options": {
  1169              "alertThreshold": true
  1170            },
  1171            "paceLength": 10,
  1172            "percentage": false,
  1173            "pluginVersion": "7.5.11",
  1174            "pointradius": 5,
  1175            "points": false,
  1176            "renderer": "flot",
  1177            "seriesOverrides": [],
  1178            "spaceLength": 10,
  1179            "stack": false,
  1180            "steppedLine": false,
  1181            "targets": [
  1182              {
  1183                "expr": "sum(increase(tikv_cdc_scan_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)",
  1184                "format": "time_series",
  1185                "hide": false,
  1186                "intervalFactor": 2,
  1187                "legendFormat": "{{instance}}",
  1188                "refId": "B",
  1189                "step": 10
  1190              }
  1191            ],
  1192            "thresholds": [],
  1193            "timeFrom": null,
  1194            "timeRegions": [],
  1195            "timeShift": null,
  1196            "title": "TiKV scan region time/m",
  1197            "tooltip": {
  1198              "msResolution": false,
  1199              "shared": true,
  1200              "sort": 0,
  1201              "value_type": "individual"
  1202            },
  1203            "type": "graph",
  1204            "xaxis": {
  1205              "buckets": null,
  1206              "mode": "time",
  1207              "name": null,
  1208              "show": true,
  1209              "values": []
  1210            },
  1211            "yaxes": [
  1212              {
  1213                "format": "s",
  1214                "label": null,
  1215                "logBase": 1,
  1216                "max": null,
  1217                "min": "0",
  1218                "show": true
  1219              },
  1220              {
  1221                "format": "short",
  1222                "label": null,
  1223                "logBase": 1,
  1224                "max": null,
  1225                "min": null,
  1226                "show": true
  1227              }
  1228            ],
  1229            "yaxis": {
  1230              "align": false,
  1231              "alignLevel": null
  1232            }
  1233          },
  1234          {
  1235            "aliasColors": {},
  1236            "bars": false,
  1237            "dashLength": 10,
  1238            "dashes": false,
  1239            "datasource": "${DS_TEST-CLUSTER}",
  1240            "decimals": 1,
  1241            "description": "The number of leaders on each TiKV instance",
  1242            "editable": true,
  1243            "error": false,
  1244            "fieldConfig": {
  1245              "defaults": {},
  1246              "overrides": []
  1247            },
  1248            "fill": 0,
  1249            "fillGradient": 0,
  1250            "grid": {},
  1251            "gridPos": {
  1252              "h": 4,
  1253              "w": 12,
  1254              "x": 0,
  1255              "y": 21
  1256            },
  1257            "hiddenSeries": false,
  1258            "id": 407,
  1259            "legend": {
  1260              "alignAsTable": true,
  1261              "avg": false,
  1262              "current": true,
  1263              "max": true,
  1264              "min": false,
  1265              "rightSide": true,
  1266              "show": false,
  1267              "sideWidth": null,
  1268              "sort": "current",
  1269              "sortDesc": true,
  1270              "total": false,
  1271              "values": true
  1272            },
  1273            "lines": true,
  1274            "linewidth": 1,
  1275            "links": [],
  1276            "nullPointMode": "null",
  1277            "options": {
  1278              "alertThreshold": true
  1279            },
  1280            "percentage": false,
  1281            "pluginVersion": "7.5.11",
  1282            "pointradius": 5,
  1283            "points": false,
  1284            "renderer": "flot",
  1285            "seriesOverrides": [],
  1286            "spaceLength": 10,
  1287            "stack": false,
  1288            "steppedLine": false,
  1289            "targets": [
  1290              {
  1291                "expr": "sum(delta(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"leader\"}[30s]))",
  1292                "format": "time_series",
  1293                "hide": false,
  1294                "intervalFactor": 2,
  1295                "legendFormat": "changed",
  1296                "refId": "A",
  1297                "step": 10
  1298              }
  1299            ],
  1300            "thresholds": [],
  1301            "timeFrom": null,
  1302            "timeRegions": [],
  1303            "timeShift": null,
  1304            "title": "TiKV leader change",
  1305            "tooltip": {
  1306              "msResolution": false,
  1307              "shared": true,
  1308              "sort": 0,
  1309              "value_type": "cumulative"
  1310            },
  1311            "type": "graph",
  1312            "xaxis": {
  1313              "buckets": null,
  1314              "mode": "time",
  1315              "name": null,
  1316              "show": true,
  1317              "values": []
  1318            },
  1319            "yaxes": [
  1320              {
  1321                "format": "short",
  1322                "label": null,
  1323                "logBase": 1,
  1324                "max": null,
  1325                "min": null,
  1326                "show": true
  1327              },
  1328              {
  1329                "format": "short",
  1330                "label": null,
  1331                "logBase": 1,
  1332                "max": null,
  1333                "min": null,
  1334                "show": true
  1335              }
  1336            ],
  1337            "yaxis": {
  1338              "align": false,
  1339              "alignLevel": null
  1340            }
  1341          },
  1342          {
  1343            "aliasColors": {},
  1344            "bars": false,
  1345            "dashLength": 10,
  1346            "dashes": false,
  1347            "datasource": "${DS_TEST-CLUSTER}",
  1348            "decimals": 1,
  1349            "description": "The number of the processed TiKV admin command",
  1350            "editable": true,
  1351            "error": false,
  1352            "fieldConfig": {
  1353              "defaults": {},
  1354              "overrides": []
  1355            },
  1356            "fill": 0,
  1357            "fillGradient": 0,
  1358            "grid": {},
  1359            "gridPos": {
  1360              "h": 4,
  1361              "w": 12,
  1362              "x": 12,
  1363              "y": 21
  1364            },
  1365            "hiddenSeries": false,
  1366            "id": 470,
  1367            "legend": {
  1368              "alignAsTable": false,
  1369              "avg": false,
  1370              "current": false,
  1371              "max": false,
  1372              "min": false,
  1373              "rightSide": false,
  1374              "show": true,
  1375              "sideWidth": null,
  1376              "sort": null,
  1377              "sortDesc": null,
  1378              "total": false,
  1379              "values": false
  1380            },
  1381            "lines": true,
  1382            "linewidth": 1,
  1383            "links": [],
  1384            "nullPointMode": "null",
  1385            "options": {
  1386              "alertThreshold": true
  1387            },
  1388            "paceLength": 10,
  1389            "percentage": false,
  1390            "pluginVersion": "7.5.11",
  1391            "pointradius": 5,
  1392            "points": false,
  1393            "renderer": "flot",
  1394            "seriesOverrides": [],
  1395            "spaceLength": 10,
  1396            "stack": false,
  1397            "steppedLine": false,
  1398            "targets": [
  1399              {
  1400                "expr": "sum(rate(tikv_raftstore_admin_cmd_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", status=\"success\", type!=\"compact\"}[1m])) by (type)",
  1401                "format": "time_series",
  1402                "hide": false,
  1403                "intervalFactor": 2,
  1404                "legendFormat": "{{type}}",
  1405                "refId": "B",
  1406                "step": 10
  1407              }
  1408            ],
  1409            "thresholds": [],
  1410            "timeFrom": null,
  1411            "timeRegions": [],
  1412            "timeShift": null,
  1413            "title": "TiKV admin apply/s",
  1414            "tooltip": {
  1415              "msResolution": false,
  1416              "shared": true,
  1417              "sort": 0,
  1418              "value_type": "individual"
  1419            },
  1420            "type": "graph",
  1421            "xaxis": {
  1422              "buckets": null,
  1423              "mode": "time",
  1424              "name": null,
  1425              "show": true,
  1426              "values": []
  1427            },
  1428            "yaxes": [
  1429              {
  1430                "format": "none",
  1431                "label": null,
  1432                "logBase": 1,
  1433                "max": null,
  1434                "min": "0",
  1435                "show": true
  1436              },
  1437              {
  1438                "format": "short",
  1439                "label": null,
  1440                "logBase": 1,
  1441                "max": null,
  1442                "min": null,
  1443                "show": true
  1444              }
  1445            ],
  1446            "yaxis": {
  1447              "align": false,
  1448              "alignLevel": null
  1449            }
  1450          },
  1451          {
  1452            "aliasColors": {},
  1453            "bars": false,
  1454            "dashLength": 10,
  1455            "dashes": false,
  1456            "datasource": "${DS_TEST-CLUSTER}",
  1457            "decimals": 1,
  1458            "description": "The rate of TiKV advancing resolved ts.",
  1459            "fieldConfig": {
  1460              "defaults": {},
  1461              "overrides": []
  1462            },
  1463            "fill": 1,
  1464            "fillGradient": 0,
  1465            "gridPos": {
  1466              "h": 4,
  1467              "w": 12,
  1468              "x": 0,
  1469              "y": 25
  1470            },
  1471            "hiddenSeries": false,
  1472            "id": 471,
  1473            "legend": {
  1474              "alignAsTable": true,
  1475              "avg": false,
  1476              "current": true,
  1477              "hideEmpty": false,
  1478              "hideZero": true,
  1479              "max": true,
  1480              "min": false,
  1481              "rightSide": false,
  1482              "show": false,
  1483              "sideWidth": null,
  1484              "sort": "current",
  1485              "sortDesc": true,
  1486              "total": false,
  1487              "values": true
  1488            },
  1489            "lines": true,
  1490            "linewidth": 1,
  1491            "links": [],
  1492            "nullPointMode": "null",
  1493            "options": {
  1494              "alertThreshold": true
  1495            },
  1496            "paceLength": 10,
  1497            "percentage": false,
  1498            "pluginVersion": "7.5.11",
  1499            "pointradius": 5,
  1500            "points": false,
  1501            "renderer": "flot",
  1502            "seriesOverrides": [],
  1503            "spaceLength": 10,
  1504            "stack": false,
  1505            "steppedLine": false,
  1506            "targets": [
  1507              {
  1508                "expr": "sum(rate(tikv_cdc_resolved_ts_gap_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance) / sum(tikv_cdc_region_resolve_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", status=~\"resolved\"}) by (instance) ",
  1509                "format": "time_series",
  1510                "hide": false,
  1511                "intervalFactor": 1,
  1512                "legendFormat": "{{instance}}",
  1513                "refId": "A"
  1514              }
  1515            ],
  1516            "thresholds": [],
  1517            "timeFrom": null,
  1518            "timeRegions": [],
  1519            "timeShift": null,
  1520            "title": "TiKV advance resovled ts/s",
  1521            "tooltip": {
  1522              "shared": true,
  1523              "sort": 0,
  1524              "value_type": "individual"
  1525            },
  1526            "type": "graph",
  1527            "xaxis": {
  1528              "buckets": null,
  1529              "mode": "time",
  1530              "name": null,
  1531              "show": true,
  1532              "values": []
  1533            },
  1534            "yaxes": [
  1535              {
  1536                "format": "none",
  1537                "label": null,
  1538                "logBase": 1,
  1539                "max": null,
  1540                "min": null,
  1541                "show": true
  1542              },
  1543              {
  1544                "format": "none",
  1545                "label": null,
  1546                "logBase": 1,
  1547                "max": null,
  1548                "min": null,
  1549                "show": true
  1550              }
  1551            ],
  1552            "yaxis": {
  1553              "align": false,
  1554              "alignLevel": null
  1555            }
  1556          },
  1557          {
  1558            "aliasColors": {},
  1559            "bars": false,
  1560            "dashLength": 10,
  1561            "dashes": false,
  1562            "datasource": "${DS_TEST-CLUSTER}",
  1563            "decimals": 1,
  1564            "description": "The number of unresolved region per TiKV",
  1565            "editable": true,
  1566            "error": false,
  1567            "fieldConfig": {
  1568              "defaults": {},
  1569              "overrides": []
  1570            },
  1571            "fill": 0,
  1572            "fillGradient": 0,
  1573            "grid": {},
  1574            "gridPos": {
  1575              "h": 4,
  1576              "w": 12,
  1577              "x": 12,
  1578              "y": 25
  1579            },
  1580            "hiddenSeries": false,
  1581            "id": 400,
  1582            "legend": {
  1583              "alignAsTable": true,
  1584              "avg": false,
  1585              "current": true,
  1586              "max": true,
  1587              "min": false,
  1588              "rightSide": true,
  1589              "show": false,
  1590              "sideWidth": null,
  1591              "sort": null,
  1592              "sortDesc": null,
  1593              "total": false,
  1594              "values": true
  1595            },
  1596            "lines": true,
  1597            "linewidth": 1,
  1598            "links": [],
  1599            "nullPointMode": "null",
  1600            "options": {
  1601              "alertThreshold": true
  1602            },
  1603            "paceLength": 10,
  1604            "percentage": false,
  1605            "pluginVersion": "7.5.11",
  1606            "pointradius": 5,
  1607            "points": false,
  1608            "renderer": "flot",
  1609            "seriesOverrides": [],
  1610            "spaceLength": 10,
  1611            "stack": false,
  1612            "steppedLine": false,
  1613            "targets": [
  1614              {
  1615                "expr": "sum(tikv_cdc_region_resolve_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", status=\"unresolved\"}) by (status)",
  1616                "format": "time_series",
  1617                "hide": false,
  1618                "intervalFactor": 2,
  1619                "legendFormat": "{{status}}",
  1620                "refId": "B",
  1621                "step": 10
  1622              }
  1623            ],
  1624            "thresholds": [],
  1625            "timeFrom": null,
  1626            "timeRegions": [],
  1627            "timeShift": null,
  1628            "title": "TiKV unresolved region count",
  1629            "tooltip": {
  1630              "msResolution": false,
  1631              "shared": true,
  1632              "sort": 0,
  1633              "value_type": "individual"
  1634            },
  1635            "type": "graph",
  1636            "xaxis": {
  1637              "buckets": null,
  1638              "mode": "time",
  1639              "name": null,
  1640              "show": true,
  1641              "values": []
  1642            },
  1643            "yaxes": [
  1644              {
  1645                "format": "none",
  1646                "label": null,
  1647                "logBase": 1,
  1648                "max": null,
  1649                "min": "0",
  1650                "show": true
  1651              },
  1652              {
  1653                "format": "short",
  1654                "label": null,
  1655                "logBase": 1,
  1656                "max": null,
  1657                "min": null,
  1658                "show": true
  1659              }
  1660            ],
  1661            "yaxis": {
  1662              "align": false,
  1663              "alignLevel": null
  1664            }
  1665          },
  1666          {
  1667            "aliasColors": {},
  1668            "bars": false,
  1669            "dashLength": 10,
  1670            "dashes": false,
  1671            "datasource": "${DS_TEST-CLUSTER}",
  1672            "decimals": 1,
  1673            "description": "99.99% of the number of regions that TiKV checks. ",
  1674            "fieldConfig": {
  1675              "defaults": {},
  1676              "overrides": []
  1677            },
  1678            "fill": 1,
  1679            "fillGradient": 0,
  1680            "gridPos": {
  1681              "h": 4,
  1682              "w": 12,
  1683              "x": 0,
  1684              "y": 29
  1685            },
  1686            "hiddenSeries": false,
  1687            "id": 433,
  1688            "legend": {
  1689              "alignAsTable": true,
  1690              "avg": false,
  1691              "current": true,
  1692              "hideEmpty": false,
  1693              "hideZero": true,
  1694              "max": true,
  1695              "min": false,
  1696              "rightSide": false,
  1697              "show": false,
  1698              "sideWidth": null,
  1699              "sort": "current",
  1700              "sortDesc": true,
  1701              "total": false,
  1702              "values": true
  1703            },
  1704            "lines": true,
  1705            "linewidth": 1,
  1706            "links": [],
  1707            "nullPointMode": "null",
  1708            "options": {
  1709              "alertThreshold": true
  1710            },
  1711            "paceLength": 10,
  1712            "percentage": false,
  1713            "pluginVersion": "7.5.11",
  1714            "pointradius": 5,
  1715            "points": false,
  1716            "renderer": "flot",
  1717            "seriesOverrides": [],
  1718            "spaceLength": 10,
  1719            "stack": false,
  1720            "steppedLine": false,
  1721            "targets": [
  1722              {
  1723                "expr": "histogram_quantile(0.9999, sum(rate(tikv_check_leader_request_item_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance))",
  1724                "format": "time_series",
  1725                "intervalFactor": 1,
  1726                "legendFormat": "{{instance}}",
  1727                "refId": "A"
  1728              }
  1729            ],
  1730            "thresholds": [],
  1731            "timeFrom": null,
  1732            "timeRegions": [],
  1733            "timeShift": null,
  1734            "title": "TiKV check leader region count percentile",
  1735            "tooltip": {
  1736              "shared": true,
  1737              "sort": 0,
  1738              "value_type": "individual"
  1739            },
  1740            "type": "graph",
  1741            "xaxis": {
  1742              "buckets": null,
  1743              "mode": "time",
  1744              "name": null,
  1745              "show": true,
  1746              "values": []
  1747            },
  1748            "yaxes": [
  1749              {
  1750                "format": "none",
  1751                "label": null,
  1752                "logBase": 1,
  1753                "max": null,
  1754                "min": null,
  1755                "show": true
  1756              },
  1757              {
  1758                "format": "short",
  1759                "label": null,
  1760                "logBase": 1,
  1761                "max": null,
  1762                "min": null,
  1763                "show": false
  1764              }
  1765            ],
  1766            "yaxis": {
  1767              "align": false,
  1768              "alignLevel": null
  1769            }
  1770          },
  1771          {
  1772            "aliasColors": {},
  1773            "bars": false,
  1774            "dashLength": 10,
  1775            "dashes": false,
  1776            "datasource": "${DS_TEST-CLUSTER}",
  1777            "description": "The lag between the commit ts of the events and the time when they are flushed to downstream. ",
  1778            "fieldConfig": {
  1779              "defaults": {},
  1780              "overrides": []
  1781            },
  1782            "fill": 1,
  1783            "fillGradient": 0,
  1784            "gridPos": {
  1785              "h": 4,
  1786              "w": 12,
  1787              "x": 0,
  1788              "y": 37
  1789            },
  1790            "hiddenSeries": false,
  1791            "id": 727,
  1792            "legend": {
  1793              "alignAsTable": true,
  1794              "avg": false,
  1795              "current": true,
  1796              "max": true,
  1797              "min": false,
  1798              "rightSide": true,
  1799              "show": true,
  1800              "total": false,
  1801              "values": true
  1802            },
  1803            "lines": true,
  1804            "linewidth": 1,
  1805            "nullPointMode": "null",
  1806            "options": {
  1807              "alertThreshold": true
  1808            },
  1809            "percentage": false,
  1810            "pluginVersion": "7.5.11",
  1811            "pointradius": 2,
  1812            "points": false,
  1813            "renderer": "flot",
  1814            "seriesOverrides": [],
  1815            "spaceLength": 10,
  1816            "stack": false,
  1817            "steppedLine": false,
  1818            "targets": [
  1819              {
  1820                "exemplar": true,
  1821                "expr": "histogram_quantile(1.00, sum(rate(ticdc_sink_flush_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))",
  1822                "interval": "",
  1823                "legendFormat": "{{instance}}-{{type}}-max",
  1824                "queryType": "randomWalk",
  1825                "refId": "A"
  1826              },
  1827              {
  1828                "exemplar": true,
  1829                "expr": "histogram_quantile(0.95, sum(rate(ticdc_sink_flush_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))",
  1830                "hide": false,
  1831                "interval": "",
  1832                "legendFormat": "{{instance}}-{{type}}-95",
  1833                "refId": "B"
  1834              },
  1835              {
  1836                "exemplar": true,
  1837                "expr": "histogram_quantile(0.8, sum(rate(ticdc_sink_flush_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))",
  1838                "hide": false,
  1839                "interval": "",
  1840                "legendFormat": "{{instance}}-{{type}}-80",
  1841                "refId": "C"
  1842              }
  1843            ],
  1844            "thresholds": [],
  1845            "timeFrom": null,
  1846            "timeRegions": [],
  1847            "timeShift": null,
  1848            "title": "Events Flush Lag",
  1849            "tooltip": {
  1850              "shared": true,
  1851              "sort": 0,
  1852              "value_type": "individual"
  1853            },
  1854            "type": "graph",
  1855            "xaxis": {
  1856              "buckets": null,
  1857              "mode": "time",
  1858              "name": null,
  1859              "show": true,
  1860              "values": []
  1861            },
  1862            "yaxes": [
  1863              {
  1864                "$$hashKey": "object:869",
  1865                "format": "s",
  1866                "label": null,
  1867                "logBase": 1,
  1868                "max": null,
  1869                "min": "0",
  1870                "show": true
  1871              },
  1872              {
  1873                "$$hashKey": "object:870",
  1874                "format": "short",
  1875                "label": null,
  1876                "logBase": 1,
  1877                "max": null,
  1878                "min": null,
  1879                "show": true
  1880              }
  1881            ],
  1882            "yaxis": {
  1883              "align": false,
  1884              "alignLevel": null
  1885            }
  1886          },
  1887          {
  1888            "aliasColors": {},
  1889            "bars": false,
  1890            "dashLength": 10,
  1891            "dashes": false,
  1892            "datasource": "${DS_TEST-CLUSTER}",
  1893            "decimals": 1,
  1894            "description": "The number of failed count of advancing resolved ts.",
  1895            "fieldConfig": {
  1896              "defaults": {},
  1897              "overrides": []
  1898            },
  1899            "fill": 1,
  1900            "fillGradient": 0,
  1901            "gridPos": {
  1902              "h": 4,
  1903              "w": 12,
  1904              "x": 12,
  1905              "y": 29
  1906            },
  1907            "hiddenSeries": false,
  1908            "id": 427,
  1909            "legend": {
  1910              "alignAsTable": true,
  1911              "avg": false,
  1912              "current": true,
  1913              "hideEmpty": false,
  1914              "hideZero": true,
  1915              "max": true,
  1916              "min": false,
  1917              "rightSide": false,
  1918              "show": false,
  1919              "sideWidth": null,
  1920              "sort": "current",
  1921              "sortDesc": true,
  1922              "total": false,
  1923              "values": true
  1924            },
  1925            "lines": true,
  1926            "linewidth": 1,
  1927            "links": [],
  1928            "nullPointMode": "null",
  1929            "options": {
  1930              "alertThreshold": true
  1931            },
  1932            "paceLength": 10,
  1933            "percentage": false,
  1934            "pluginVersion": "7.5.11",
  1935            "pointradius": 5,
  1936            "points": false,
  1937            "renderer": "flot",
  1938            "seriesOverrides": [],
  1939            "spaceLength": 10,
  1940            "stack": false,
  1941            "steppedLine": false,
  1942            "targets": [
  1943              {
  1944                "expr": "sum(increase(tikv_resolved_ts_fail_advance_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (reason, instance)",
  1945                "format": "time_series",
  1946                "hide": false,
  1947                "intervalFactor": 1,
  1948                "legendFormat": "{{reason}}-{{instance}}",
  1949                "refId": "A"
  1950              }
  1951            ],
  1952            "thresholds": [],
  1953            "timeFrom": null,
  1954            "timeRegions": [],
  1955            "timeShift": null,
  1956            "title": "TiKV advance resolved ts fail/m",
  1957            "tooltip": {
  1958              "shared": true,
  1959              "sort": 0,
  1960              "value_type": "individual"
  1961            },
  1962            "type": "graph",
  1963            "xaxis": {
  1964              "buckets": null,
  1965              "mode": "time",
  1966              "name": null,
  1967              "show": true,
  1968              "values": []
  1969            },
  1970            "yaxes": [
  1971              {
  1972                "format": "none",
  1973                "label": null,
  1974                "logBase": 1,
  1975                "max": null,
  1976                "min": null,
  1977                "show": true
  1978              },
  1979              {
  1980                "format": "none",
  1981                "label": null,
  1982                "logBase": 1,
  1983                "max": null,
  1984                "min": null,
  1985                "show": true
  1986              }
  1987            ],
  1988            "yaxis": {
  1989              "align": false,
  1990              "alignLevel": null
  1991            }
  1992          },
  1993          {
  1994            "aliasColors": {},
  1995            "bars": false,
  1996            "dashLength": 10,
  1997            "dashes": false,
  1998            "datasource": "${DS_TEST-CLUSTER}",
  1999            "decimals": 1,
  2000            "description": "The number of tables rebalance on each TiCDC instance",
  2001            "editable": true,
  2002            "error": false,
  2003            "fieldConfig": {
  2004              "defaults": {},
  2005              "overrides": []
  2006            },
  2007            "fill": 0,
  2008            "fillGradient": 0,
  2009            "grid": {},
  2010            "gridPos": {
  2011              "h": 4,
  2012              "w": 12,
  2013              "x": 0,
  2014              "y": 33
  2015            },
  2016            "hiddenSeries": false,
  2017            "id": 399,
  2018            "legend": {
  2019              "alignAsTable": true,
  2020              "avg": false,
  2021              "current": true,
  2022              "max": true,
  2023              "min": false,
  2024              "rightSide": true,
  2025              "show": false,
  2026              "sideWidth": null,
  2027              "sort": "current",
  2028              "sortDesc": true,
  2029              "total": false,
  2030              "values": true
  2031            },
  2032            "lines": true,
  2033            "linewidth": 1,
  2034            "links": [],
  2035            "nullPointMode": "null",
  2036            "options": {
  2037              "alertThreshold": true
  2038            },
  2039            "percentage": false,
  2040            "pluginVersion": "7.5.11",
  2041            "pointradius": 5,
  2042            "points": false,
  2043            "renderer": "flot",
  2044            "seriesOverrides": [],
  2045            "spaceLength": 10,
  2046            "stack": false,
  2047            "steppedLine": false,
  2048            "targets": [
  2049              {
  2050                "expr": "sum(delta(ticdc_processor_num_of_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[30s])) by (instance)",
  2051                "format": "time_series",
  2052                "hide": false,
  2053                "intervalFactor": 2,
  2054                "legendFormat": "{{instance}}",
  2055                "refId": "A",
  2056                "step": 10
  2057              }
  2058            ],
  2059            "thresholds": [],
  2060            "timeFrom": null,
  2061            "timeRegions": [],
  2062            "timeShift": null,
  2063            "title": "Table rebalance",
  2064            "tooltip": {
  2065              "msResolution": false,
  2066              "shared": true,
  2067              "sort": 0,
  2068              "value_type": "cumulative"
  2069            },
  2070            "type": "graph",
  2071            "xaxis": {
  2072              "buckets": null,
  2073              "mode": "time",
  2074              "name": null,
  2075              "show": true,
  2076              "values": []
  2077            },
  2078            "yaxes": [
  2079              {
  2080                "format": "short",
  2081                "label": null,
  2082                "logBase": 1,
  2083                "max": null,
  2084                "min": null,
  2085                "show": true
  2086              },
  2087              {
  2088                "format": "short",
  2089                "label": null,
  2090                "logBase": 1,
  2091                "max": null,
  2092                "min": null,
  2093                "show": true
  2094              }
  2095            ],
  2096            "yaxis": {
  2097              "align": false,
  2098              "alignLevel": null
  2099            }
  2100          },
  2101          {
  2102            "aliasColors": {},
  2103            "bars": false,
  2104            "dashLength": 10,
  2105            "dashes": false,
  2106            "datasource": "${DS_TEST-CLUSTER}",
  2107            "decimals": 1,
  2108            "description": "99.99% of the duration that TiKV check leader takes. ",
  2109            "fieldConfig": {
  2110              "defaults": {},
  2111              "overrides": []
  2112            },
  2113            "fill": 1,
  2114            "fillGradient": 0,
  2115            "gridPos": {
  2116              "h": 4,
  2117              "w": 12,
  2118              "x": 12,
  2119              "y": 33
  2120            },
  2121            "hiddenSeries": false,
  2122            "id": 472,
  2123            "legend": {
  2124              "alignAsTable": true,
  2125              "avg": false,
  2126              "current": true,
  2127              "hideEmpty": false,
  2128              "hideZero": true,
  2129              "max": true,
  2130              "min": false,
  2131              "rightSide": false,
  2132              "show": false,
  2133              "sideWidth": null,
  2134              "sort": "current",
  2135              "sortDesc": true,
  2136              "total": false,
  2137              "values": true
  2138            },
  2139            "lines": true,
  2140            "linewidth": 1,
  2141            "links": [],
  2142            "nullPointMode": "null",
  2143            "options": {
  2144              "alertThreshold": true
  2145            },
  2146            "paceLength": 10,
  2147            "percentage": false,
  2148            "pluginVersion": "7.5.11",
  2149            "pointradius": 5,
  2150            "points": false,
  2151            "renderer": "flot",
  2152            "seriesOverrides": [],
  2153            "spaceLength": 10,
  2154            "stack": false,
  2155            "steppedLine": false,
  2156            "targets": [
  2157              {
  2158                "expr": "histogram_quantile(0.9999, sum(rate(tikv_resolved_ts_tikv_client_init_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le))",
  2159                "format": "time_series",
  2160                "intervalFactor": 1,
  2161                "legendFormat": "new-client",
  2162                "refId": "A"
  2163              },
  2164              {
  2165                "expr": "histogram_quantile(0.9999, sum(rate(tikv_resolved_ts_check_leader_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, type))",
  2166                "format": "time_series",
  2167                "intervalFactor": 1,
  2168                "legendFormat": "{{type}}",
  2169                "refId": "B"
  2170              }
  2171            ],
  2172            "thresholds": [],
  2173            "timeFrom": null,
  2174            "timeRegions": [],
  2175            "timeShift": null,
  2176            "title": "TiKV check leader duration percentile",
  2177            "tooltip": {
  2178              "shared": true,
  2179              "sort": 0,
  2180              "value_type": "individual"
  2181            },
  2182            "type": "graph",
  2183            "xaxis": {
  2184              "buckets": null,
  2185              "mode": "time",
  2186              "name": null,
  2187              "show": true,
  2188              "values": []
  2189            },
  2190            "yaxes": [
  2191              {
  2192                "format": "s",
  2193                "label": null,
  2194                "logBase": 1,
  2195                "max": null,
  2196                "min": null,
  2197                "show": true
  2198              },
  2199              {
  2200                "format": "short",
  2201                "label": null,
  2202                "logBase": 1,
  2203                "max": null,
  2204                "min": null,
  2205                "show": false
  2206              }
  2207            ],
  2208            "yaxis": {
  2209              "align": false,
  2210              "alignLevel": null
  2211            }
  2212          },
  2213          {
  2214            "aliasColors": {},
  2215            "dashLength": 10,
  2216            "datasource": "${DS_TEST-CLUSTER}",
  2217            "decimals": 1,
  2218            "description": "The number of regions that take a long time(more than 60s) to scan",
  2219            "editable": true,
  2220            "fieldConfig": {
  2221              "defaults": {},
  2222              "overrides": []
  2223            },
  2224            "grid": {},
  2225            "gridPos": {
  2226              "h": 4,
  2227              "w": 12,
  2228              "x": 12,
  2229              "y": 37
  2230            },
  2231            "id": 10038,
  2232            "legend": {
  2233              "alignAsTable": true,
  2234              "avg": false,
  2235              "current": true,
  2236              "max": true,
  2237              "min": false,
  2238              "rightSide": true,
  2239              "show": false,
  2240              "sideWidth": null,
  2241              "sort": null,
  2242              "sortDesc": null,
  2243              "total": false,
  2244              "values": true
  2245            },
  2246            "lines": true,
  2247            "linewidth": 1,
  2248            "links": [],
  2249            "nullPointMode": "null",
  2250            "options": {
  2251              "alertThreshold": true
  2252            },
  2253            "paceLength": 10,
  2254            "pluginVersion": "7.5.17",
  2255            "pointradius": 5,
  2256            "renderer": "flot",
  2257            "seriesOverrides": [],
  2258            "spaceLength": 10,
  2259            "targets": [
  2260              {
  2261                "expr": "tikv_cdc_scan_long_duration_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}",
  2262                "legendFormat": "{{instance}}",
  2263                "interval": "",
  2264                "exemplar": true,
  2265                "format": "time_series",
  2266                "hide": false,
  2267                "intervalFactor": 2,
  2268                "refId": "B",
  2269                "step": 10
  2270              }
  2271            ],
  2272            "thresholds": [],
  2273            "timeRegions": [],
  2274            "title": "TiKV CDC Incremental Scan Long Duration Region Count",
  2275            "tooltip": {
  2276              "msResolution": false,
  2277              "shared": true,
  2278              "sort": 0,
  2279              "value_type": "individual"
  2280            },
  2281            "type": "graph",
  2282            "xaxis": {
  2283              "buckets": null,
  2284              "mode": "time",
  2285              "name": null,
  2286              "show": true,
  2287              "values": []
  2288            },
  2289            "yaxes": [
  2290              {
  2291                "format": "none",
  2292                "label": null,
  2293                "logBase": 1,
  2294                "max": null,
  2295                "min": "0",
  2296                "show": true,
  2297                "$$hashKey": "object:115"
  2298              },
  2299              {
  2300                "format": "short",
  2301                "label": null,
  2302                "logBase": 1,
  2303                "max": null,
  2304                "min": null,
  2305                "show": true,
  2306                "$$hashKey": "object:116"
  2307              }
  2308            ],
  2309            "yaxis": {
  2310              "align": false,
  2311              "alignLevel": null
  2312            },
  2313            "bars": false,
  2314            "dashes": false,
  2315            "error": false,
  2316            "fill": 0,
  2317            "fillGradient": 0,
  2318            "hiddenSeries": false,
  2319            "percentage": false,
  2320            "points": false,
  2321            "stack": false,
  2322            "steppedLine": false,
  2323            "timeFrom": null,
  2324            "timeShift": null
  2325          }
  2326        ],
  2327        "title": "Lag analyze",
  2328        "type": "row"
  2329      },
  2330      {
  2331        "collapsed": true,
  2332        "datasource": null,
  2333        "gridPos": {
  2334          "h": 1,
  2335          "w": 24,
  2336          "x": 0,
  2337          "y": 1
  2338        },
  2339        "id": 21,
  2340        "panels": [
  2341          {
  2342            "aliasColors": {},
  2343            "bars": false,
  2344            "dashLength": 10,
  2345            "dashes": false,
  2346            "datasource": "${DS_TEST-CLUSTER}",
  2347            "description": "Uptime of TiCDC and TiKV",
  2348            "fieldConfig": {
  2349              "defaults": {
  2350                "links": []
  2351              },
  2352              "overrides": []
  2353            },
  2354            "fill": 1,
  2355            "fillGradient": 0,
  2356            "gridPos": {
  2357              "h": 7,
  2358              "w": 8,
  2359              "x": 0,
  2360              "y": 2
  2361            },
  2362            "hiddenSeries": false,
  2363            "id": 19,
  2364            "legend": {
  2365              "alignAsTable": true,
  2366              "avg": false,
  2367              "current": true,
  2368              "max": false,
  2369              "min": false,
  2370              "rightSide": true,
  2371              "show": true,
  2372              "total": false,
  2373              "values": true
  2374            },
  2375            "lines": true,
  2376            "linewidth": 1,
  2377            "links": [],
  2378            "nullPointMode": "null",
  2379            "options": {
  2380              "alertThreshold": true
  2381            },
  2382            "paceLength": 10,
  2383            "percentage": false,
  2384            "pluginVersion": "7.5.11",
  2385            "pointradius": 2,
  2386            "points": false,
  2387            "renderer": "flot",
  2388            "seriesOverrides": [],
  2389            "spaceLength": 10,
  2390            "stack": false,
  2391            "steppedLine": false,
  2392            "targets": [
  2393              {
  2394                "expr": "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"})",
  2395                "format": "time_series",
  2396                "intervalFactor": 1,
  2397                "legendFormat": "TiCDC-{{instance}}",
  2398                "refId": "A"
  2399              },
  2400              {
  2401                "expr": "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tikv\"})",
  2402                "format": "time_series",
  2403                "intervalFactor": 1,
  2404                "legendFormat": "TiKV-{{instance}}",
  2405                "refId": "B"
  2406              },
  2407              {
  2408                "expr": "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"pd\"})",
  2409                "format": "time_series",
  2410                "intervalFactor": 1,
  2411                "legendFormat": "PD-{{instance}}",
  2412                "refId": "C"
  2413              }
  2414            ],
  2415            "thresholds": [],
  2416            "timeFrom": null,
  2417            "timeRegions": [],
  2418            "timeShift": null,
  2419            "title": "Uptime",
  2420            "tooltip": {
  2421              "shared": true,
  2422              "sort": 0,
  2423              "value_type": "individual"
  2424            },
  2425            "type": "graph",
  2426            "xaxis": {
  2427              "buckets": null,
  2428              "mode": "time",
  2429              "name": null,
  2430              "show": true,
  2431              "values": []
  2432            },
  2433            "yaxes": [
  2434              {
  2435                "format": "dtdurations",
  2436                "label": null,
  2437                "logBase": 1,
  2438                "max": null,
  2439                "min": null,
  2440                "show": true
  2441              },
  2442              {
  2443                "format": "short",
  2444                "label": null,
  2445                "logBase": 1,
  2446                "max": null,
  2447                "min": null,
  2448                "show": true
  2449              }
  2450            ],
  2451            "yaxis": {
  2452              "align": false,
  2453              "alignLevel": null
  2454            }
  2455          },
  2456          {
  2457            "aliasColors": {},
  2458            "bars": false,
  2459            "dashLength": 10,
  2460            "dashes": false,
  2461            "datasource": "${DS_TEST-CLUSTER}",
  2462            "description": "Goroutine count of TiCDC",
  2463            "fieldConfig": {
  2464              "defaults": {
  2465                "links": []
  2466              },
  2467              "overrides": []
  2468            },
  2469            "fill": 1,
  2470            "fillGradient": 0,
  2471            "gridPos": {
  2472              "h": 7,
  2473              "w": 6,
  2474              "x": 8,
  2475              "y": 2
  2476            },
  2477            "hiddenSeries": false,
  2478            "id": 26,
  2479            "legend": {
  2480              "alignAsTable": true,
  2481              "avg": false,
  2482              "current": true,
  2483              "max": true,
  2484              "min": false,
  2485              "rightSide": false,
  2486              "show": true,
  2487              "total": false,
  2488              "values": true
  2489            },
  2490            "lines": true,
  2491            "linewidth": 1,
  2492            "links": [],
  2493            "nullPointMode": "null",
  2494            "options": {
  2495              "alertThreshold": true
  2496            },
  2497            "paceLength": 10,
  2498            "percentage": false,
  2499            "pluginVersion": "7.5.11",
  2500            "pointradius": 2,
  2501            "points": false,
  2502            "renderer": "flot",
  2503            "seriesOverrides": [],
  2504            "spaceLength": 10,
  2505            "stack": false,
  2506            "steppedLine": false,
  2507            "targets": [
  2508              {
  2509                "expr": " go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}",
  2510                "format": "time_series",
  2511                "intervalFactor": 1,
  2512                "legendFormat": "{{instance}}",
  2513                "refId": "A"
  2514              },
  2515              {
  2516                "expr": "go_threads{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}",
  2517                "format": "time_series",
  2518                "hide": true,
  2519                "intervalFactor": 1,
  2520                "legendFormat": "threads-{{instance}}",
  2521                "refId": "B"
  2522              }
  2523            ],
  2524            "thresholds": [],
  2525            "timeFrom": null,
  2526            "timeRegions": [],
  2527            "timeShift": null,
  2528            "title": "Goroutine count",
  2529            "tooltip": {
  2530              "shared": true,
  2531              "sort": 0,
  2532              "value_type": "individual"
  2533            },
  2534            "type": "graph",
  2535            "xaxis": {
  2536              "buckets": null,
  2537              "mode": "time",
  2538              "name": null,
  2539              "show": true,
  2540              "values": []
  2541            },
  2542            "yaxes": [
  2543              {
  2544                "format": "short",
  2545                "label": null,
  2546                "logBase": 1,
  2547                "max": null,
  2548                "min": null,
  2549                "show": true
  2550              },
  2551              {
  2552                "format": "short",
  2553                "label": null,
  2554                "logBase": 1,
  2555                "max": null,
  2556                "min": null,
  2557                "show": true
  2558              }
  2559            ],
  2560            "yaxis": {
  2561              "align": false,
  2562              "alignLevel": null
  2563            }
  2564          },
  2565          {
  2566            "aliasColors": {},
  2567            "bars": false,
  2568            "dashLength": 10,
  2569            "dashes": false,
  2570            "datasource": "${DS_TEST-CLUSTER}",
  2571            "description": "The count of open FD count of TiCDC",
  2572            "fieldConfig": {
  2573              "defaults": {
  2574                "links": []
  2575              },
  2576              "overrides": []
  2577            },
  2578            "fill": 1,
  2579            "fillGradient": 0,
  2580            "gridPos": {
  2581              "h": 7,
  2582              "w": 6,
  2583              "x": 14,
  2584              "y": 2
  2585            },
  2586            "hiddenSeries": false,
  2587            "id": 27,
  2588            "legend": {
  2589              "alignAsTable": true,
  2590              "avg": false,
  2591              "current": true,
  2592              "max": true,
  2593              "min": false,
  2594              "rightSide": false,
  2595              "show": true,
  2596              "total": false,
  2597              "values": true
  2598            },
  2599            "lines": true,
  2600            "linewidth": 1,
  2601            "links": [],
  2602            "nullPointMode": "null",
  2603            "options": {
  2604              "alertThreshold": true
  2605            },
  2606            "paceLength": 10,
  2607            "percentage": false,
  2608            "pluginVersion": "7.5.11",
  2609            "pointradius": 2,
  2610            "points": false,
  2611            "renderer": "flot",
  2612            "seriesOverrides": [],
  2613            "spaceLength": 10,
  2614            "stack": false,
  2615            "steppedLine": false,
  2616            "targets": [
  2617              {
  2618                "expr": "process_open_fds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}",
  2619                "format": "time_series",
  2620                "intervalFactor": 1,
  2621                "legendFormat": "{{instance}}",
  2622                "refId": "A"
  2623              }
  2624            ],
  2625            "thresholds": [],
  2626            "timeFrom": null,
  2627            "timeRegions": [],
  2628            "timeShift": null,
  2629            "title": "Open FD count",
  2630            "tooltip": {
  2631              "shared": true,
  2632              "sort": 0,
  2633              "value_type": "individual"
  2634            },
  2635            "type": "graph",
  2636            "xaxis": {
  2637              "buckets": null,
  2638              "mode": "time",
  2639              "name": null,
  2640              "show": true,
  2641              "values": []
  2642            },
  2643            "yaxes": [
  2644              {
  2645                "format": "short",
  2646                "label": null,
  2647                "logBase": 1,
  2648                "max": null,
  2649                "min": null,
  2650                "show": true
  2651              },
  2652              {
  2653                "format": "short",
  2654                "label": null,
  2655                "logBase": 1,
  2656                "max": null,
  2657                "min": null,
  2658                "show": true
  2659              }
  2660            ],
  2661            "yaxis": {
  2662              "align": false,
  2663              "alignLevel": null
  2664            }
  2665          },
  2666          {
  2667            "datasource": "${DS_TEST-CLUSTER}",
  2668            "description": "TiCDC cluster ownership status",
  2669            "fieldConfig": {
  2670              "defaults": {
  2671                "color": {
  2672                  "mode": "thresholds"
  2673                },
  2674                "custom": {
  2675                  "align": "center",
  2676                  "filterable": false
  2677                },
  2678                "mappings": [],
  2679                "thresholds": {
  2680                  "mode": "absolute",
  2681                  "steps": [
  2682                    {
  2683                      "color": "green",
  2684                      "value": null
  2685                    },
  2686                    {
  2687                      "color": "red",
  2688                      "value": 80
  2689                    }
  2690                  ]
  2691                }
  2692              },
  2693              "overrides": [
  2694                {
  2695                  "matcher": {
  2696                    "id": "byName",
  2697                    "options": "Field"
  2698                  },
  2699                  "properties": [
  2700                    {
  2701                      "id": "displayName",
  2702                      "value": "Instance"
  2703                    },
  2704                    {
  2705                      "id": "unit",
  2706                      "value": "string"
  2707                    },
  2708                    {
  2709                      "id": "decimals",
  2710                      "value": 2
  2711                    },
  2712                    {
  2713                      "id": "custom.align",
  2714                      "value": null
  2715                    },
  2716                    {
  2717                      "id": "thresholds",
  2718                      "value": {
  2719                        "mode": "absolute",
  2720                        "steps": [
  2721                          {
  2722                            "color": "#8AB8FF",
  2723                            "value": null
  2724                          },
  2725                          {
  2726                            "color": "#73BF69",
  2727                            "value": 0
  2728                          },
  2729                          {
  2730                            "color": "rgba(245, 54, 54, 0.9)",
  2731                            "value": 2
  2732                          }
  2733                        ]
  2734                      }
  2735                    }
  2736                  ]
  2737                },
  2738                {
  2739                  "matcher": {
  2740                    "id": "byName",
  2741                    "options": "Last"
  2742                  },
  2743                  "properties": [
  2744                    {
  2745                      "id": "displayName",
  2746                      "value": "Role"
  2747                    },
  2748                    {
  2749                      "id": "decimals",
  2750                      "value": 0
  2751                    },
  2752                    {
  2753                      "id": "mappings",
  2754                      "value": [
  2755                        {
  2756                          "from": "0.1",
  2757                          "id": 1,
  2758                          "text": "Owner",
  2759                          "to": "2",
  2760                          "type": 2,
  2761                          "value": "1"
  2762                        },
  2763                        {
  2764                          "from": "0",
  2765                          "id": 2,
  2766                          "text": "Processor",
  2767                          "to": "0.1",
  2768                          "type": 2
  2769                        }
  2770                      ]
  2771                    }
  2772                  ]
  2773                }
  2774              ]
  2775            },
  2776            "gridPos": {
  2777              "h": 7,
  2778              "w": 4,
  2779              "x": 20,
  2780              "y": 2
  2781            },
  2782            "hideTimeOverride": true,
  2783            "id": 113,
  2784            "links": [],
  2785            "options": {
  2786              "showHeader": true
  2787            },
  2788            "pluginVersion": "7.5.11",
  2789            "targets": [
  2790              {
  2791                "expr": "rate(ticdc_owner_ownership_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[240s])",
  2792                "format": "time_series",
  2793                "intervalFactor": 2,
  2794                "legendFormat": "{{instance}}",
  2795                "refId": "A"
  2796              }
  2797            ],
  2798            "timeFrom": "1s",
  2799            "title": "Ownership",
  2800            "transformations": [
  2801              {
  2802                "id": "reduce",
  2803                "options": {
  2804                  "includeTimeField": false,
  2805                  "reducers": [
  2806                    "last"
  2807                  ]
  2808                }
  2809              }
  2810            ],
  2811            "type": "table"
  2812          },
  2813          {
  2814            "aliasColors": {},
  2815            "bars": false,
  2816            "dashLength": 10,
  2817            "dashes": false,
  2818            "datasource": "${DS_TEST-CLUSTER}",
  2819            "description": "CPU usage of TiCDC",
  2820            "fieldConfig": {
  2821              "defaults": {
  2822                "links": []
  2823              },
  2824              "overrides": []
  2825            },
  2826            "fill": 1,
  2827            "fillGradient": 0,
  2828            "gridPos": {
  2829              "h": 7,
  2830              "w": 6,
  2831              "x": 0,
  2832              "y": 9
  2833            },
  2834            "hiddenSeries": false,
  2835            "id": 24,
  2836            "legend": {
  2837              "alignAsTable": true,
  2838              "avg": false,
  2839              "current": true,
  2840              "hideEmpty": true,
  2841              "max": true,
  2842              "min": false,
  2843              "rightSide": false,
  2844              "show": true,
  2845              "total": false,
  2846              "values": true
  2847            },
  2848            "lines": true,
  2849            "linewidth": 1,
  2850            "links": [],
  2851            "nullPointMode": "null",
  2852            "options": {
  2853              "alertThreshold": true
  2854            },
  2855            "paceLength": 10,
  2856            "percentage": false,
  2857            "pluginVersion": "7.5.11",
  2858            "pointradius": 2,
  2859            "points": false,
  2860            "renderer": "flot",
  2861            "seriesOverrides": [
  2862              {
  2863                "alias": "/.*MaxProcs/",
  2864                "fill": 0,
  2865                "linewidth": 2,
  2866                "yaxis": 2
  2867              }
  2868            ],
  2869            "spaceLength": 10,
  2870            "stack": false,
  2871            "steppedLine": false,
  2872            "targets": [
  2873              {
  2874                "expr": "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}[1m])",
  2875                "format": "time_series",
  2876                "intervalFactor": 1,
  2877                "legendFormat": "{{instance}}",
  2878                "refId": "A"
  2879              },
  2880              {
  2881                "expr": "ticdc_server_go_max_procs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}",
  2882                "format": "time_series",
  2883                "intervalFactor": 1,
  2884                "legendFormat": "quota-{{instance}}",
  2885                "refId": "B"
  2886              }
  2887            ],
  2888            "thresholds": [],
  2889            "timeFrom": null,
  2890            "timeRegions": [],
  2891            "timeShift": null,
  2892            "title": "CPU usage",
  2893            "tooltip": {
  2894              "shared": true,
  2895              "sort": 0,
  2896              "value_type": "individual"
  2897            },
  2898            "type": "graph",
  2899            "xaxis": {
  2900              "buckets": null,
  2901              "mode": "time",
  2902              "name": null,
  2903              "show": true,
  2904              "values": []
  2905            },
  2906            "yaxes": [
  2907              {
  2908                "format": "percentunit",
  2909                "label": null,
  2910                "logBase": 1,
  2911                "max": null,
  2912                "min": null,
  2913                "show": true
  2914              },
  2915              {
  2916                "format": "percentunit",
  2917                "label": null,
  2918                "logBase": 1,
  2919                "max": null,
  2920                "min": null,
  2921                "show": true
  2922              }
  2923            ],
  2924            "yaxis": {
  2925              "align": false,
  2926              "alignLevel": null
  2927            }
  2928          },
  2929          {
  2930            "aliasColors": {},
  2931            "bars": false,
  2932            "dashLength": 10,
  2933            "dashes": false,
  2934            "datasource": "${DS_TEST-CLUSTER}",
  2935            "description": "Memory usage of TiCDC",
  2936            "fieldConfig": {
  2937              "defaults": {
  2938                "links": []
  2939              },
  2940              "overrides": []
  2941            },
  2942            "fill": 1,
  2943            "fillGradient": 0,
  2944            "gridPos": {
  2945              "h": 7,
  2946              "w": 6,
  2947              "x": 6,
  2948              "y": 9
  2949            },
  2950            "hiddenSeries": false,
  2951            "id": 23,
  2952            "legend": {
  2953              "alignAsTable": true,
  2954              "avg": false,
  2955              "current": true,
  2956              "hideEmpty": true,
  2957              "max": true,
  2958              "min": false,
  2959              "rightSide": false,
  2960              "show": true,
  2961              "total": false,
  2962              "values": true
  2963            },
  2964            "lines": true,
  2965            "linewidth": 1,
  2966            "links": [],
  2967            "nullPointMode": "null",
  2968            "options": {
  2969              "alertThreshold": true
  2970            },
  2971            "paceLength": 10,
  2972            "percentage": false,
  2973            "pluginVersion": "7.5.11",
  2974            "pointradius": 2,
  2975            "points": false,
  2976            "renderer": "flot",
  2977            "seriesOverrides": [],
  2978            "spaceLength": 10,
  2979            "stack": false,
  2980            "steppedLine": false,
  2981            "targets": [
  2982              {
  2983                "expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}",
  2984                "format": "time_series",
  2985                "intervalFactor": 1,
  2986                "legendFormat": "process-{{instance}}",
  2987                "refId": "A"
  2988              },
  2989              {
  2990                "expr": "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}",
  2991                "format": "time_series",
  2992                "intervalFactor": 1,
  2993                "legendFormat": "heap-{{instance}}",
  2994                "refId": "B"
  2995              }
  2996            ],
  2997            "thresholds": [],
  2998            "timeFrom": null,
  2999            "timeRegions": [],
  3000            "timeShift": null,
  3001            "title": "Memory usage",
  3002            "tooltip": {
  3003              "shared": true,
  3004              "sort": 0,
  3005              "value_type": "individual"
  3006            },
  3007            "type": "graph",
  3008            "xaxis": {
  3009              "buckets": null,
  3010              "mode": "time",
  3011              "name": null,
  3012              "show": true,
  3013              "values": []
  3014            },
  3015            "yaxes": [
  3016              {
  3017                "format": "bytes",
  3018                "label": null,
  3019                "logBase": 1,
  3020                "max": null,
  3021                "min": null,
  3022                "show": true
  3023              },
  3024              {
  3025                "format": "short",
  3026                "label": null,
  3027                "logBase": 1,
  3028                "max": null,
  3029                "min": null,
  3030                "show": true
  3031              }
  3032            ],
  3033            "yaxis": {
  3034              "align": false,
  3035              "alignLevel": null
  3036            }
  3037          },
  3038          {
  3039            "aliasColors": {},
  3040            "bars": false,
  3041            "dashLength": 10,
  3042            "dashes": false,
  3043            "datasource": "${DS_TEST-CLUSTER}",
  3044            "description": "The history of TiCDC cluster ownership, owner node has a value that is great than 0",
  3045            "fieldConfig": {
  3046              "defaults": {},
  3047              "overrides": []
  3048            },
  3049            "fill": 1,
  3050            "fillGradient": 0,
  3051            "gridPos": {
  3052              "h": 7,
  3053              "w": 6,
  3054              "x": 12,
  3055              "y": 9
  3056            },
  3057            "hiddenSeries": false,
  3058            "id": 110,
  3059            "legend": {
  3060              "alignAsTable": true,
  3061              "avg": false,
  3062              "current": true,
  3063              "max": false,
  3064              "min": false,
  3065              "rightSide": false,
  3066              "show": true,
  3067              "total": false,
  3068              "values": true
  3069            },
  3070            "lines": true,
  3071            "linewidth": 1,
  3072            "links": [],
  3073            "nullPointMode": "null",
  3074            "options": {
  3075              "alertThreshold": true
  3076            },
  3077            "percentage": false,
  3078            "pluginVersion": "7.5.11",
  3079            "pointradius": 2,
  3080            "points": false,
  3081            "renderer": "flot",
  3082            "seriesOverrides": [],
  3083            "spaceLength": 10,
  3084            "stack": false,
  3085            "steppedLine": true,
  3086            "targets": [
  3087              {
  3088                "expr": "sum(rate(ticdc_owner_ownership_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[240s])) by (instance) > BOOL 0.0625",
  3089                "format": "time_series",
  3090                "interval": "30s",
  3091                "intervalFactor": 1,
  3092                "legendFormat": "{{instance}}",
  3093                "refId": "A"
  3094              }
  3095            ],
  3096            "thresholds": [],
  3097            "timeFrom": null,
  3098            "timeRegions": [],
  3099            "timeShift": null,
  3100            "title": "Ownership history",
  3101            "tooltip": {
  3102              "shared": true,
  3103              "sort": 0,
  3104              "value_type": "individual"
  3105            },
  3106            "type": "graph",
  3107            "xaxis": {
  3108              "buckets": null,
  3109              "mode": "time",
  3110              "name": null,
  3111              "show": true,
  3112              "values": []
  3113            },
  3114            "yaxes": [
  3115              {
  3116                "format": "short",
  3117                "label": null,
  3118                "logBase": 1,
  3119                "max": null,
  3120                "min": null,
  3121                "show": true
  3122              },
  3123              {
  3124                "format": "short",
  3125                "label": null,
  3126                "logBase": 1,
  3127                "max": null,
  3128                "min": null,
  3129                "show": true
  3130              }
  3131            ],
  3132            "yaxis": {
  3133              "align": false,
  3134              "alignLevel": null
  3135            }
  3136          },
  3137          {
  3138            "aliasColors": {},
  3139            "bars": false,
  3140            "dashLength": 10,
  3141            "dashes": false,
  3142            "datasource": "${DS_TEST-CLUSTER}",
  3143            "description": "The history of PD cluster leadership, leader node has a value that is great than 0",
  3144            "fieldConfig": {
  3145              "defaults": {},
  3146              "overrides": []
  3147            },
  3148            "fill": 1,
  3149            "fillGradient": 0,
  3150            "gridPos": {
  3151              "h": 7,
  3152              "w": 6,
  3153              "x": 18,
  3154              "y": 9
  3155            },
  3156            "hiddenSeries": false,
  3157            "id": 293,
  3158            "legend": {
  3159              "alignAsTable": true,
  3160              "avg": false,
  3161              "current": true,
  3162              "max": false,
  3163              "min": false,
  3164              "rightSide": false,
  3165              "show": true,
  3166              "total": false,
  3167              "values": true
  3168            },
  3169            "lines": true,
  3170            "linewidth": 1,
  3171            "links": [],
  3172            "nullPointMode": "null",
  3173            "options": {
  3174              "alertThreshold": true
  3175            },
  3176            "percentage": false,
  3177            "pluginVersion": "7.5.11",
  3178            "pointradius": 2,
  3179            "points": false,
  3180            "renderer": "flot",
  3181            "seriesOverrides": [],
  3182            "spaceLength": 10,
  3183            "stack": false,
  3184            "steppedLine": true,
  3185            "targets": [
  3186              {
  3187                "expr": "pd_tso_role{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", dc=\"global\"} > BOOL 0.5",
  3188                "format": "time_series",
  3189                "hide": false,
  3190                "interval": "30s",
  3191                "intervalFactor": 1,
  3192                "legendFormat": "PD-{{instance}}",
  3193                "refId": "A"
  3194              }
  3195            ],
  3196            "thresholds": [],
  3197            "timeFrom": null,
  3198            "timeRegions": [],
  3199            "timeShift": null,
  3200            "title": "PD leader history",
  3201            "tooltip": {
  3202              "shared": true,
  3203              "sort": 0,
  3204              "value_type": "individual"
  3205            },
  3206            "type": "graph",
  3207            "xaxis": {
  3208              "buckets": null,
  3209              "mode": "time",
  3210              "name": null,
  3211              "show": true,
  3212              "values": []
  3213            },
  3214            "yaxes": [
  3215              {
  3216                "format": "short",
  3217                "label": null,
  3218                "logBase": 1,
  3219                "max": null,
  3220                "min": null,
  3221                "show": true
  3222              },
  3223              {
  3224                "format": "short",
  3225                "label": null,
  3226                "logBase": 1,
  3227                "max": null,
  3228                "min": null,
  3229                "show": true
  3230              }
  3231            ],
  3232            "yaxis": {
  3233              "align": false,
  3234              "alignLevel": null
  3235            }
  3236          }
  3237        ],
  3238        "title": "Server",
  3239        "type": "row"
  3240      },
  3241      {
  3242        "collapsed": true,
  3243        "datasource": null,
  3244        "gridPos": {
  3245          "h": 1,
  3246          "w": 24,
  3247          "x": 0,
  3248          "y": 2
  3249        },
  3250        "id": 11,
  3251        "panels": [
  3252          {
  3253            "datasource": "${DS_TEST-CLUSTER}",
  3254            "description": "The number of captured table of TiCDC nodes ",
  3255            "fieldConfig": {
  3256              "defaults": {
  3257                "color": {
  3258                  "mode": "thresholds"
  3259                },
  3260                "custom": {
  3261                  "align": "left",
  3262                  "filterable": false
  3263                },
  3264                "decimals": 2,
  3265                "mappings": [],
  3266                "thresholds": {
  3267                  "mode": "absolute",
  3268                  "steps": [
  3269                    {
  3270                      "color": "green",
  3271                      "value": null
  3272                    },
  3273                    {
  3274                      "color": "red",
  3275                      "value": 80
  3276                    }
  3277                  ]
  3278                },
  3279                "unit": "short"
  3280              },
  3281              "overrides": [
  3282                {
  3283                  "matcher": {
  3284                    "id": "byName",
  3285                    "options": "Field"
  3286                  },
  3287                  "properties": [
  3288                    {
  3289                      "id": "displayName",
  3290                      "value": "Instance"
  3291                    },
  3292                    {
  3293                      "id": "unit",
  3294                      "value": "string"
  3295                    },
  3296                    {
  3297                      "id": "custom.align",
  3298                      "value": null
  3299                    }
  3300                  ]
  3301                },
  3302                {
  3303                  "matcher": {
  3304                    "id": "byName",
  3305                    "options": "Last"
  3306                  },
  3307                  "properties": [
  3308                    {
  3309                      "id": "displayName",
  3310                      "value": "Count"
  3311                    }
  3312                  ]
  3313                }
  3314              ]
  3315            },
  3316            "gridPos": {
  3317              "h": 7,
  3318              "w": 6,
  3319              "x": 0,
  3320              "y": 3
  3321            },
  3322            "id": 4,
  3323            "links": [],
  3324            "options": {
  3325              "showHeader": true
  3326            },
  3327            "pluginVersion": "7.5.11",
  3328            "targets": [
  3329              {
  3330                "exemplar": true,
  3331                "expr": "sum(ticdc_processor_num_of_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}) by (instance)",
  3332                "format": "time_series",
  3333                "interval": "",
  3334                "intervalFactor": 1,
  3335                "legendFormat": "{{instance}}",
  3336                "refId": "A"
  3337              }
  3338            ],
  3339            "timeFrom": null,
  3340            "timeShift": null,
  3341            "title": "Changefeed table count",
  3342            "transformations": [
  3343              {
  3344                "id": "reduce",
  3345                "options": {
  3346                  "includeTimeField": false,
  3347                  "reducers": [
  3348                    "last"
  3349                  ]
  3350                }
  3351              }
  3352            ],
  3353            "type": "table"
  3354          },
  3355          {
  3356            "aliasColors": {},
  3357            "bars": false,
  3358            "dashLength": 10,
  3359            "dashes": false,
  3360            "datasource": "${DS_TEST-CLUSTER}",
  3361            "decimals": 1,
  3362            "description": "The number of tables in different capture.",
  3363            "editable": true,
  3364            "error": false,
  3365            "fieldConfig": {
  3366              "defaults": {
  3367                "unit": "none"
  3368              },
  3369              "overrides": []
  3370            },
  3371            "fill": 0,
  3372            "fillGradient": 0,
  3373            "grid": {},
  3374            "gridPos": {
  3375              "h": 7,
  3376              "w": 6,
  3377              "x": 6,
  3378              "y": 3
  3379            },
  3380            "hiddenSeries": false,
  3381            "id": 642,
  3382            "legend": {
  3383              "alignAsTable": true,
  3384              "avg": false,
  3385              "current": true,
  3386              "max": false,
  3387              "min": false,
  3388              "rightSide": false,
  3389              "show": true,
  3390              "sideWidth": null,
  3391              "sort": null,
  3392              "sortDesc": null,
  3393              "total": false,
  3394              "values": true
  3395            },
  3396            "lines": true,
  3397            "linewidth": 1,
  3398            "links": [],
  3399            "nullPointMode": "null",
  3400            "options": {
  3401              "alertThreshold": true
  3402            },
  3403            "paceLength": 10,
  3404            "percentage": false,
  3405            "pluginVersion": "7.5.11",
  3406            "pointradius": 1,
  3407            "points": true,
  3408            "renderer": "flot",
  3409            "seriesOverrides": [],
  3410            "spaceLength": 10,
  3411            "stack": false,
  3412            "steppedLine": false,
  3413            "targets": [
  3414              {
  3415                "exemplar": false,
  3416                "expr": "sum(ticdc_scheduler_capture_table{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, addr)",
  3417                "format": "time_series",
  3418                "hide": false,
  3419                "instant": false,
  3420                "interval": "",
  3421                "intervalFactor": 2,
  3422                "legendFormat": "{{namespace}}-{{changefeed}}-{{addr}}",
  3423                "refId": "A",
  3424                "step": 10
  3425              }
  3426            ],
  3427            "thresholds": [],
  3428            "timeFrom": null,
  3429            "timeRegions": [],
  3430            "timeShift": null,
  3431            "title": "Capture tables count",
  3432            "tooltip": {
  3433              "msResolution": false,
  3434              "shared": true,
  3435              "sort": 0,
  3436              "value_type": "individual"
  3437            },
  3438            "type": "graph",
  3439            "xaxis": {
  3440              "buckets": null,
  3441              "mode": "time",
  3442              "name": null,
  3443              "show": true,
  3444              "values": []
  3445            },
  3446            "yaxes": [
  3447              {
  3448                "format": "none",
  3449                "label": null,
  3450                "logBase": 1,
  3451                "max": null,
  3452                "min": "0",
  3453                "show": true
  3454              },
  3455              {
  3456                "format": "none",
  3457                "label": null,
  3458                "logBase": 1,
  3459                "max": null,
  3460                "min": null,
  3461                "show": false
  3462              }
  3463            ],
  3464            "yaxis": {
  3465              "align": false,
  3466              "alignLevel": null
  3467            }
  3468          },
  3469          {
  3470            "aliasColors": {},
  3471            "bars": false,
  3472            "dashLength": 10,
  3473            "dashes": false,
  3474            "datasource": "${DS_TEST-CLUSTER}",
  3475            "fieldConfig": {
  3476              "defaults": {
  3477                "unit": "dateTimeAsIso"
  3478              },
  3479              "overrides": []
  3480            },
  3481            "fill": 1,
  3482            "fillGradient": 0,
  3483            "gridPos": {
  3484              "h": 7,
  3485              "w": 12,
  3486              "x": 12,
  3487              "y": 3
  3488            },
  3489            "hiddenSeries": false,
  3490            "id": 10037,
  3491            "legend": {
  3492              "avg": false,
  3493              "current": true,
  3494              "max": false,
  3495              "min": false,
  3496              "show": true,
  3497              "total": false,
  3498              "values": true
  3499            },
  3500            "lines": true,
  3501            "linewidth": 1,
  3502            "nullPointMode": "null",
  3503            "options": {
  3504              "alertThreshold": true
  3505            },
  3506            "percentage": false,
  3507            "pluginVersion": "7.5.10",
  3508            "pointradius": 2,
  3509            "points": false,
  3510            "renderer": "flot",
  3511            "seriesOverrides": [],
  3512            "spaceLength": 10,
  3513            "stack": false,
  3514            "steppedLine": false,
  3515            "targets": [
  3516              {
  3517                "exemplar": true,
  3518                "expr": "max(ticdc_gc_min_service_gc_safepoint{})",
  3519                "interval": "",
  3520                "legendFormat": "gc time",
  3521                "queryType": "randomWalk",
  3522                "refId": "A"
  3523              },
  3524              {
  3525                "exemplar": true,
  3526                "expr": "max(ticdc_gc_cdc_gc_safepoint{})",
  3527                "hide": false,
  3528                "interval": "",
  3529                "legendFormat": "cdc service safepoint",
  3530                "refId": "B"
  3531              }
  3532            ],
  3533            "thresholds": [],
  3534            "timeFrom": null,
  3535            "timeRegions": [],
  3536            "timeShift": null,
  3537            "title": "GC Time",
  3538            "tooltip": {
  3539              "shared": true,
  3540              "sort": 0,
  3541              "value_type": "individual"
  3542            },
  3543            "type": "graph",
  3544            "xaxis": {
  3545              "buckets": null,
  3546              "mode": "time",
  3547              "name": null,
  3548              "show": true,
  3549              "values": []
  3550            },
  3551            "yaxes": [
  3552              {
  3553                "format": "dateTimeAsIso",
  3554                "label": null,
  3555                "logBase": 1,
  3556                "max": null,
  3557                "min": null,
  3558                "show": true
  3559              },
  3560              {
  3561                "format": "short",
  3562                "label": null,
  3563                "logBase": 1,
  3564                "max": null,
  3565                "min": null,
  3566                "show": true
  3567              }
  3568            ],
  3569            "yaxis": {
  3570              "align": false,
  3571              "alignLevel": null
  3572            }
  3573          },
  3574          {
  3575            "aliasColors": {},
  3576            "bars": true,
  3577            "cacheTimeout": null,
  3578            "dashLength": 10,
  3579            "dashes": false,
  3580            "datasource": "${DS_TEST-CLUSTER}",
  3581            "description": "The checkpoint ts of changefeeds.",
  3582            "fieldConfig": {
  3583              "defaults": {
  3584                "unit": "dateTimeAsIso"
  3585              },
  3586              "overrides": []
  3587            },
  3588            "fill": 0,
  3589            "fillGradient": 0,
  3590            "gridPos": {
  3591              "h": 7,
  3592              "w": 12,
  3593              "x": 0,
  3594              "y": 10
  3595            },
  3596            "hiddenSeries": false,
  3597            "id": 86,
  3598            "legend": {
  3599              "alignAsTable": true,
  3600              "avg": false,
  3601              "current": true,
  3602              "max": false,
  3603              "min": false,
  3604              "rightSide": false,
  3605              "show": true,
  3606              "sideWidth": null,
  3607              "total": false,
  3608              "values": true
  3609            },
  3610            "lines": true,
  3611            "linewidth": 2,
  3612            "links": [],
  3613            "nullPointMode": "null",
  3614            "options": {
  3615              "alertThreshold": true
  3616            },
  3617            "percentage": false,
  3618            "pluginVersion": "7.5.11",
  3619            "pointradius": 2,
  3620            "points": false,
  3621            "renderer": "flot",
  3622            "seriesOverrides": [
  3623              {
  3624                "alias": "/approximate current time.*/",
  3625                "bars": false
  3626              }
  3627            ],
  3628            "spaceLength": 10,
  3629            "stack": false,
  3630            "steppedLine": false,
  3631            "targets": [
  3632              {
  3633                "expr": "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})",
  3634                "format": "time_series",
  3635                "hide": false,
  3636                "interval": "",
  3637                "intervalFactor": 1,
  3638                "legendFormat": "approximate current time (s)",
  3639                "refId": "A"
  3640              },
  3641              {
  3642                "exemplar": true,
  3643                "expr": "max(ticdc_owner_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)",
  3644                "format": "time_series",
  3645                "hide": false,
  3646                "interval": "",
  3647                "intervalFactor": 1,
  3648                "legendFormat": "{{namespace}}-{{changefeed}}",
  3649                "refId": "B"
  3650              },
  3651              {
  3652                "exemplar": true,
  3653                "expr": "max(ticdc_owner_barrier_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)",
  3654                "hide": false,
  3655                "interval": "",
  3656                "legendFormat": "{{namespace}}-{{changefeed}}-barrierTs",
  3657                "refId": "D"
  3658              }
  3659            ],
  3660            "thresholds": [],
  3661            "timeFrom": null,
  3662            "timeRegions": [],
  3663            "timeShift": null,
  3664            "title": "Changefeed checkpoint",
  3665            "tooltip": {
  3666              "shared": true,
  3667              "sort": 0,
  3668              "value_type": "individual"
  3669            },
  3670            "type": "graph",
  3671            "xaxis": {
  3672              "buckets": null,
  3673              "max": null,
  3674              "mode": "time",
  3675              "name": null,
  3676              "show": true,
  3677              "values": []
  3678            },
  3679            "yaxes": [
  3680              {
  3681                "format": "dateTimeAsIso",
  3682                "label": null,
  3683                "logBase": 1,
  3684                "max": null,
  3685                "min": null,
  3686                "show": true
  3687              },
  3688              {
  3689                "format": "short",
  3690                "label": null,
  3691                "logBase": 1,
  3692                "max": null,
  3693                "min": null,
  3694                "show": false
  3695              }
  3696            ],
  3697            "yaxis": {
  3698              "align": false,
  3699              "alignLevel": null
  3700            }
  3701          },
  3702          {
  3703            "aliasColors": {},
  3704            "bars": true,
  3705            "cacheTimeout": null,
  3706            "dashLength": 10,
  3707            "dashes": false,
  3708            "datasource": "${DS_TEST-CLUSTER}",
  3709            "description": "The resolved ts of changefeeds.",
  3710            "fieldConfig": {
  3711              "defaults": {},
  3712              "overrides": []
  3713            },
  3714            "fill": 0,
  3715            "fillGradient": 0,
  3716            "gridPos": {
  3717              "h": 7,
  3718              "w": 12,
  3719              "x": 12,
  3720              "y": 10
  3721            },
  3722            "hiddenSeries": false,
  3723            "id": 512,
  3724            "legend": {
  3725              "alignAsTable": true,
  3726              "avg": false,
  3727              "current": true,
  3728              "max": false,
  3729              "min": false,
  3730              "rightSide": false,
  3731              "show": true,
  3732              "sideWidth": null,
  3733              "total": false,
  3734              "values": true
  3735            },
  3736            "lines": true,
  3737            "linewidth": 2,
  3738            "links": [],
  3739            "nullPointMode": "null",
  3740            "options": {
  3741              "alertThreshold": true
  3742            },
  3743            "percentage": false,
  3744            "pluginVersion": "7.5.11",
  3745            "pointradius": 2,
  3746            "points": false,
  3747            "renderer": "flot",
  3748            "seriesOverrides": [
  3749              {
  3750                "alias": "/approximate current time.*/",
  3751                "bars": false
  3752              }
  3753            ],
  3754            "spaceLength": 10,
  3755            "stack": false,
  3756            "steppedLine": false,
  3757            "targets": [
  3758              {
  3759                "expr": "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})",
  3760                "format": "time_series",
  3761                "hide": false,
  3762                "interval": "",
  3763                "intervalFactor": 1,
  3764                "legendFormat": "approximate current time (s)",
  3765                "refId": "A"
  3766              },
  3767              {
  3768                "exemplar": true,
  3769                "expr": "max(ticdc_owner_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)",
  3770                "format": "time_series",
  3771                "hide": false,
  3772                "interval": "",
  3773                "intervalFactor": 1,
  3774                "legendFormat": "{{namespace}}-{{changefeed}}-barrier",
  3775                "refId": "C"
  3776              },
  3777              {
  3778                "exemplar": true,
  3779                "expr": "max(ticdc_scheduler_slow_table_puller_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)",
  3780                "hide": false,
  3781                "interval": "",
  3782                "legendFormat": "{{namespace}}-{{changefeed}}-puller",
  3783                "refId": "B"
  3784              }
  3785            ],
  3786            "thresholds": [],
  3787            "timeFrom": null,
  3788            "timeRegions": [],
  3789            "timeShift": null,
  3790            "title": "Changefeed resolved ts",
  3791            "tooltip": {
  3792              "shared": true,
  3793              "sort": 0,
  3794              "value_type": "individual"
  3795            },
  3796            "type": "graph",
  3797            "xaxis": {
  3798              "buckets": null,
  3799              "max": null,
  3800              "mode": "time",
  3801              "name": null,
  3802              "show": true,
  3803              "values": []
  3804            },
  3805            "yaxes": [
  3806              {
  3807                "format": "dateTimeAsIso",
  3808                "label": null,
  3809                "logBase": 1,
  3810                "max": null,
  3811                "min": null,
  3812                "show": true
  3813              },
  3814              {
  3815                "format": "short",
  3816                "label": null,
  3817                "logBase": 1,
  3818                "max": null,
  3819                "min": null,
  3820                "show": false
  3821              }
  3822            ],
  3823            "yaxis": {
  3824              "align": false,
  3825              "alignLevel": null
  3826            }
  3827          },
  3828          {
  3829            "aliasColors": {},
  3830            "bars": false,
  3831            "dashLength": 10,
  3832            "dashes": false,
  3833            "datasource": "${DS_TEST-CLUSTER}",
  3834            "description": "The lag between changefeed checkpoint ts and the latest ts of upstream TiDB.",
  3835            "fieldConfig": {
  3836              "defaults": {
  3837                "links": []
  3838              },
  3839              "overrides": []
  3840            },
  3841            "fill": 1,
  3842            "fillGradient": 0,
  3843            "gridPos": {
  3844              "h": 7,
  3845              "w": 12,
  3846              "x": 0,
  3847              "y": 17
  3848            },
  3849            "hiddenSeries": false,
  3850            "id": 3,
  3851            "legend": {
  3852              "alignAsTable": true,
  3853              "avg": false,
  3854              "current": true,
  3855              "max": false,
  3856              "min": false,
  3857              "rightSide": false,
  3858              "show": true,
  3859              "total": false,
  3860              "values": true
  3861            },
  3862            "lines": true,
  3863            "linewidth": 1,
  3864            "links": [],
  3865            "nullPointMode": "null",
  3866            "options": {
  3867              "alertThreshold": true
  3868            },
  3869            "paceLength": 10,
  3870            "percentage": false,
  3871            "pluginVersion": "7.5.11",
  3872            "pointradius": 2,
  3873            "points": false,
  3874            "renderer": "flot",
  3875            "seriesOverrides": [],
  3876            "spaceLength": 10,
  3877            "stack": false,
  3878            "steppedLine": false,
  3879            "targets": [
  3880              {
  3881                "exemplar": true,
  3882                "expr": "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)",
  3883                "format": "time_series",
  3884                "interval": "",
  3885                "intervalFactor": 1,
  3886                "legendFormat": "{{namespace}}-{{changefeed}}",
  3887                "refId": "A"
  3888              }
  3889            ],
  3890            "thresholds": [],
  3891            "timeFrom": null,
  3892            "timeRegions": [],
  3893            "timeShift": null,
  3894            "title": "Changefeed checkpoint lag",
  3895            "tooltip": {
  3896              "shared": true,
  3897              "sort": 0,
  3898              "value_type": "individual"
  3899            },
  3900            "type": "graph",
  3901            "xaxis": {
  3902              "buckets": null,
  3903              "mode": "time",
  3904              "name": null,
  3905              "show": true,
  3906              "values": []
  3907            },
  3908            "yaxes": [
  3909              {
  3910                "format": "s",
  3911                "label": null,
  3912                "logBase": 1,
  3913                "max": null,
  3914                "min": "0",
  3915                "show": true
  3916              },
  3917              {
  3918                "format": "short",
  3919                "label": null,
  3920                "logBase": 1,
  3921                "max": null,
  3922                "min": null,
  3923                "show": true
  3924              }
  3925            ],
  3926            "yaxis": {
  3927              "align": false,
  3928              "alignLevel": null
  3929            }
  3930          },
  3931          {
  3932            "aliasColors": {},
  3933            "bars": false,
  3934            "dashLength": 10,
  3935            "dashes": false,
  3936            "datasource": "${DS_TEST-CLUSTER}",
  3937            "description": "The lag between changefeed resolved ts and the latest ts of upstream TiDB.",
  3938            "fieldConfig": {
  3939              "defaults": {
  3940                "links": []
  3941              },
  3942              "overrides": []
  3943            },
  3944            "fill": 1,
  3945            "fillGradient": 0,
  3946            "gridPos": {
  3947              "h": 7,
  3948              "w": 12,
  3949              "x": 12,
  3950              "y": 17
  3951            },
  3952            "hiddenSeries": false,
  3953            "id": 513,
  3954            "legend": {
  3955              "alignAsTable": true,
  3956              "avg": false,
  3957              "current": true,
  3958              "max": false,
  3959              "min": false,
  3960              "rightSide": false,
  3961              "show": true,
  3962              "total": false,
  3963              "values": true
  3964            },
  3965            "lines": true,
  3966            "linewidth": 1,
  3967            "links": [],
  3968            "nullPointMode": "null",
  3969            "options": {
  3970              "alertThreshold": true
  3971            },
  3972            "paceLength": 10,
  3973            "percentage": false,
  3974            "pluginVersion": "7.5.11",
  3975            "pointradius": 2,
  3976            "points": false,
  3977            "renderer": "flot",
  3978            "seriesOverrides": [],
  3979            "spaceLength": 10,
  3980            "stack": false,
  3981            "steppedLine": false,
  3982            "targets": [
  3983              {
  3984                "exemplar": true,
  3985                "expr": "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)",
  3986                "format": "time_series",
  3987                "interval": "",
  3988                "intervalFactor": 1,
  3989                "legendFormat": "{{namespace}}-{{changefeed}}-barrier",
  3990                "refId": "C"
  3991              },
  3992              {
  3993                "exemplar": true,
  3994                "expr": "max(ticdc_scheduler_slow_table_puller_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)",
  3995                "hide": false,
  3996                "interval": "",
  3997                "legendFormat": "{{namespace}}-{{changefeed}}-puller",
  3998                "refId": "A"
  3999              }
  4000            ],
  4001            "thresholds": [],
  4002            "timeFrom": null,
  4003            "timeRegions": [],
  4004            "timeShift": null,
  4005            "title": "Changefeed resolved ts lag",
  4006            "tooltip": {
  4007              "shared": true,
  4008              "sort": 0,
  4009              "value_type": "individual"
  4010            },
  4011            "type": "graph",
  4012            "xaxis": {
  4013              "buckets": null,
  4014              "mode": "time",
  4015              "name": null,
  4016              "show": true,
  4017              "values": []
  4018            },
  4019            "yaxes": [
  4020              {
  4021                "format": "s",
  4022                "label": null,
  4023                "logBase": 1,
  4024                "max": null,
  4025                "min": "0",
  4026                "show": true
  4027              },
  4028              {
  4029                "format": "short",
  4030                "label": null,
  4031                "logBase": 1,
  4032                "max": null,
  4033                "min": null,
  4034                "show": true
  4035              }
  4036            ],
  4037            "yaxis": {
  4038              "align": false,
  4039              "alignLevel": null
  4040            }
  4041          },
  4042          {
  4043            "aliasColors": {},
  4044            "bars": false,
  4045            "dashLength": 10,
  4046            "dashes": false,
  4047            "datasource": "${DS_TEST-CLUSTER}",
  4048            "description": "The difference between changefeed resolved ts and changefeed checkpointTs.",
  4049            "fieldConfig": {
  4050              "defaults": {
  4051                "links": []
  4052              },
  4053              "overrides": []
  4054            },
  4055            "fill": 1,
  4056            "fillGradient": 0,
  4057            "gridPos": {
  4058              "h": 7,
  4059              "w": 12,
  4060              "x": 0,
  4061              "y": 24
  4062            },
  4063            "hiddenSeries": false,
  4064            "id": 603,
  4065            "legend": {
  4066              "alignAsTable": true,
  4067              "avg": false,
  4068              "current": true,
  4069              "max": false,
  4070              "min": false,
  4071              "rightSide": false,
  4072              "show": true,
  4073              "total": false,
  4074              "values": true
  4075            },
  4076            "lines": true,
  4077            "linewidth": 1,
  4078            "links": [],
  4079            "nullPointMode": "null",
  4080            "options": {
  4081              "alertThreshold": true
  4082            },
  4083            "paceLength": 10,
  4084            "percentage": false,
  4085            "pluginVersion": "7.5.11",
  4086            "pointradius": 2,
  4087            "points": false,
  4088            "renderer": "flot",
  4089            "seriesOverrides": [],
  4090            "spaceLength": 10,
  4091            "stack": false,
  4092            "steppedLine": false,
  4093            "targets": [
  4094              {
  4095                "exemplar": true,
  4096                "expr": "(max(ticdc_owner_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed) - max(ticdc_owner_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)) / 1000 > 0",
  4097                "format": "time_series",
  4098                "interval": "",
  4099                "intervalFactor": 1,
  4100                "legendFormat": "{{namespace}}-{{changefeed}}",
  4101                "refId": "C"
  4102              }
  4103            ],
  4104            "thresholds": [],
  4105            "timeFrom": null,
  4106            "timeRegions": [],
  4107            "timeShift": null,
  4108            "title": "Difference of resolvedTs and checkpointTs",
  4109            "tooltip": {
  4110              "shared": true,
  4111              "sort": 0,
  4112              "value_type": "individual"
  4113            },
  4114            "type": "graph",
  4115            "xaxis": {
  4116              "buckets": null,
  4117              "mode": "time",
  4118              "name": null,
  4119              "show": true,
  4120              "values": []
  4121            },
  4122            "yaxes": [
  4123              {
  4124                "format": "s",
  4125                "label": null,
  4126                "logBase": 1,
  4127                "max": null,
  4128                "min": "0",
  4129                "show": true
  4130              },
  4131              {
  4132                "format": "short",
  4133                "label": null,
  4134                "logBase": 1,
  4135                "max": null,
  4136                "min": null,
  4137                "show": true
  4138              }
  4139            ],
  4140            "yaxis": {
  4141              "align": false,
  4142              "alignLevel": 1
  4143            }
  4144          },
  4145          {
  4146            "aliasColors": {},
  4147            "bars": false,
  4148            "dashLength": 10,
  4149            "dashes": false,
  4150            "datasource": "${DS_TEST-CLUSTER}",
  4151            "description": "each changefeed tick periodically, the time cost of each tick",
  4152            "fieldConfig": {
  4153              "defaults": {},
  4154              "overrides": []
  4155            },
  4156            "fill": 1,
  4157            "fillGradient": 0,
  4158            "gridPos": {
  4159              "h": 7,
  4160              "w": 12,
  4161              "x": 12,
  4162              "y": 24
  4163            },
  4164            "hiddenSeries": false,
  4165            "id": 597,
  4166            "legend": {
  4167              "alignAsTable": true,
  4168              "avg": false,
  4169              "current": true,
  4170              "max": true,
  4171              "min": false,
  4172              "rightSide": false,
  4173              "show": true,
  4174              "total": false,
  4175              "values": true
  4176            },
  4177            "lines": true,
  4178            "linewidth": 1,
  4179            "nullPointMode": "null",
  4180            "options": {
  4181              "alertThreshold": true
  4182            },
  4183            "percentage": false,
  4184            "pluginVersion": "7.5.11",
  4185            "pointradius": 2,
  4186            "points": false,
  4187            "renderer": "flot",
  4188            "seriesOverrides": [],
  4189            "spaceLength": 10,
  4190            "stack": false,
  4191            "steppedLine": false,
  4192            "targets": [
  4193              {
  4194                "exemplar": true,
  4195                "expr": "histogram_quantile(0.999, sum(rate(ticdc_owner_changefeed_tick_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le,instance,namespace,changefeed))",
  4196                "hide": false,
  4197                "interval": "",
  4198                "legendFormat": "{{namespace}}-{{changefeed}}-p999",
  4199                "refId": "A"
  4200              },
  4201              {
  4202                "exemplar": true,
  4203                "expr": "histogram_quantile(0.99, sum(rate(ticdc_processor_processor_tick_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le,instance,namespace,changefeed))",
  4204                "hide": false,
  4205                "interval": "",
  4206                "legendFormat": "{{namespace}}-{{changefeed}}-p99",
  4207                "refId": "B"
  4208              }
  4209            ],
  4210            "thresholds": [],
  4211            "timeFrom": null,
  4212            "timeRegions": [],
  4213            "timeShift": null,
  4214            "title": "changefeed tick duration percentile",
  4215            "tooltip": {
  4216              "shared": true,
  4217              "sort": 0,
  4218              "value_type": "individual"
  4219            },
  4220            "type": "graph",
  4221            "xaxis": {
  4222              "buckets": null,
  4223              "mode": "time",
  4224              "name": null,
  4225              "show": true,
  4226              "values": []
  4227            },
  4228            "yaxes": [
  4229              {
  4230                "format": "s",
  4231                "label": null,
  4232                "logBase": 1,
  4233                "max": null,
  4234                "min": null,
  4235                "show": true
  4236              },
  4237              {
  4238                "format": "none",
  4239                "label": null,
  4240                "logBase": 1,
  4241                "max": null,
  4242                "min": null,
  4243                "show": true
  4244              }
  4245            ],
  4246            "yaxis": {
  4247              "align": false,
  4248              "alignLevel": null
  4249            }
  4250          },
  4251          {
  4252            "datasource": "${DS_TEST-CLUSTER}",
  4253            "description": "The status of each changefeed.\n\n0: Normal\n\n1 and 6: Warning\n\n2: Failed\n\n3: Stopped\n\n4: Finished\n\n-1: Unknown",
  4254            "fieldConfig": {
  4255              "defaults": {
  4256                "color": {
  4257                  "mode": "palette-classic"
  4258                },
  4259                "custom": {
  4260                  "axisLabel": "",
  4261                  "axisPlacement": "auto",
  4262                  "barAlignment": 0,
  4263                  "drawStyle": "line",
  4264                  "fillOpacity": 10,
  4265                  "gradientMode": "none",
  4266                  "hideFrom": {
  4267                    "graph": false,
  4268                    "legend": false,
  4269                    "tooltip": false
  4270                  },
  4271                  "lineInterpolation": "stepAfter",
  4272                  "lineWidth": 1,
  4273                  "pointSize": 4,
  4274                  "scaleDistribution": {
  4275                    "type": "linear"
  4276                  },
  4277                  "showPoints": "always",
  4278                  "spanNulls": true
  4279                },
  4280                "mappings": [
  4281                  {
  4282                    "from": "",
  4283                    "id": 1,
  4284                    "text": "Normal",
  4285                    "to": "",
  4286                    "type": 1,
  4287                    "value": "0"
  4288                  },
  4289                  {
  4290                    "from": "",
  4291                    "id": 2,
  4292                    "text": "Warning",
  4293                    "to": "",
  4294                    "type": 1,
  4295                    "value": "1"
  4296                  },
  4297                  {
  4298                    "from": "",
  4299                    "id": 3,
  4300                    "text": "Failed",
  4301                    "to": "",
  4302                    "type": 1,
  4303                    "value": "2"
  4304                  },
  4305                  {
  4306                    "from": "",
  4307                    "id": 4,
  4308                    "text": "Stopped",
  4309                    "to": "",
  4310                    "type": 1,
  4311                    "value": "3"
  4312                  },
  4313                  {
  4314                    "from": "",
  4315                    "id": 5,
  4316                    "text": "Finished",
  4317                    "to": "",
  4318                    "type": 1,
  4319                    "value": "4"
  4320                  },
  4321                  {
  4322                    "from": "",
  4323                    "id": 6,
  4324                    "text": "Unknown",
  4325                    "to": "",
  4326                    "type": 1,
  4327                    "value": "-1"
  4328                  },
  4329                  {
  4330                    "from": "5",
  4331                    "id": 7,
  4332                    "text": "Other",
  4333                    "to": "10000",
  4334                    "type": 1,
  4335                    "value": "5"
  4336                  },
  4337                  {
  4338                    "from": "6",
  4339                    "id": 8,
  4340                    "text": "-",
  4341                    "to": "1000",
  4342                    "type": 2
  4343                  }
  4344                ],
  4345                "max": 8,
  4346                "min": -1,
  4347                "noValue": "-",
  4348                "thresholds": {
  4349                  "mode": "absolute",
  4350                  "steps": [
  4351                    {
  4352                      "color": "green",
  4353                      "value": null
  4354                    },
  4355                    {
  4356                      "color": "light-yellow",
  4357                      "value": -1
  4358                    },
  4359                    {
  4360                      "color": "green",
  4361                      "value": 0
  4362                    },
  4363                    {
  4364                      "color": "semi-dark-red",
  4365                      "value": 1
  4366                    },
  4367                    {
  4368                      "color": "dark-red",
  4369                      "value": 2
  4370                    },
  4371                    {
  4372                      "color": "dark-yellow",
  4373                      "value": 3
  4374                    },
  4375                    {
  4376                      "color": "#6ED0E0",
  4377                      "value": 4
  4378                    }
  4379                  ]
  4380                },
  4381                "unit": "short"
  4382              },
  4383              "overrides": []
  4384            },
  4385            "gridPos": {
  4386              "h": 7,
  4387              "w": 12,
  4388              "x": 0,
  4389              "y": 31
  4390            },
  4391            "id": 163,
  4392            "links": [],
  4393            "options": {
  4394              "graph": {},
  4395              "legend": {
  4396                "calcs": [
  4397                  "lastNotNull"
  4398                ],
  4399                "displayMode": "table",
  4400                "placement": "bottom"
  4401              },
  4402              "tooltipOptions": {
  4403                "mode": "single"
  4404              }
  4405            },
  4406            "pluginVersion": "7.5.11",
  4407            "targets": [
  4408              {
  4409                "exemplar": true,
  4410                "expr": "max(ticdc_owner_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)",
  4411                "format": "time_series",
  4412                "instant": false,
  4413                "interval": "",
  4414                "intervalFactor": 1,
  4415                "legendFormat": "{{namespace}}-{{changefeed}}",
  4416                "refId": "A"
  4417              }
  4418            ],
  4419            "timeFrom": null,
  4420            "timeShift": null,
  4421            "title": "The status of changefeeds",
  4422            "type": "timeseries"
  4423          },
  4424          {
  4425            "aliasColors": {},
  4426            "bars": true,
  4427            "cacheTimeout": null,
  4428            "dashLength": 10,
  4429            "dashes": false,
  4430            "datasource": "${DS_TEST-CLUSTER}",
  4431            "description": "",
  4432            "fieldConfig": {
  4433              "defaults": {},
  4434              "overrides": []
  4435            },
  4436            "fill": 0,
  4437            "fillGradient": 0,
  4438            "gridPos": {
  4439              "h": 7,
  4440              "w": 12,
  4441              "x": 12,
  4442              "y": 31
  4443            },
  4444            "hiddenSeries": false,
  4445            "id": 514,
  4446            "legend": {
  4447              "alignAsTable": true,
  4448              "avg": false,
  4449              "current": true,
  4450              "max": false,
  4451              "min": false,
  4452              "rightSide": false,
  4453              "show": true,
  4454              "sideWidth": null,
  4455              "total": false,
  4456              "values": true
  4457            },
  4458            "lines": true,
  4459            "linewidth": 2,
  4460            "links": [],
  4461            "nullPointMode": "null",
  4462            "options": {
  4463              "alertThreshold": true
  4464            },
  4465            "percentage": false,
  4466            "pluginVersion": "7.5.11",
  4467            "pointradius": 2,
  4468            "points": false,
  4469            "renderer": "flot",
  4470            "seriesOverrides": [
  4471              {
  4472                "alias": "/approximate current time.*/",
  4473                "bars": false
  4474              }
  4475            ],
  4476            "spaceLength": 10,
  4477            "stack": false,
  4478            "steppedLine": false,
  4479            "targets": [
  4480              {
  4481                "expr": "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})",
  4482                "format": "time_series",
  4483                "hide": false,
  4484                "interval": "",
  4485                "intervalFactor": 1,
  4486                "legendFormat": "approximate current time (s)",
  4487                "refId": "A"
  4488              },
  4489              {
  4490                "exemplar": true,
  4491                "expr": "max(ticdc_processor_schema_storage_gc_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)",
  4492                "format": "time_series",
  4493                "hide": false,
  4494                "interval": "",
  4495                "intervalFactor": 1,
  4496                "legendFormat": "{{namespace}}-{{changefeed}}",
  4497                "refId": "C"
  4498              }
  4499            ],
  4500            "thresholds": [],
  4501            "timeFrom": null,
  4502            "timeRegions": [],
  4503            "timeShift": null,
  4504            "title": "Schema Storage GC progress",
  4505            "tooltip": {
  4506              "shared": true,
  4507              "sort": 0,
  4508              "value_type": "individual"
  4509            },
  4510            "type": "graph",
  4511            "xaxis": {
  4512              "buckets": null,
  4513              "max": null,
  4514              "mode": "time",
  4515              "name": null,
  4516              "show": true,
  4517              "values": []
  4518            },
  4519            "yaxes": [
  4520              {
  4521                "format": "dateTimeAsIso",
  4522                "label": null,
  4523                "logBase": 1,
  4524                "max": null,
  4525                "min": null,
  4526                "show": true
  4527              },
  4528              {
  4529                "format": "short",
  4530                "label": null,
  4531                "logBase": 1,
  4532                "max": null,
  4533                "min": null,
  4534                "show": false
  4535              }
  4536            ],
  4537            "yaxis": {
  4538              "align": false,
  4539              "alignLevel": null
  4540            }
  4541          },
  4542          {
  4543            "aliasColors": {},
  4544            "bars": false,
  4545            "dashLength": 10,
  4546            "dashes": false,
  4547            "datasource": "${DS_TEST-CLUSTER}",
  4548            "description": "Request count of etcd operation per second",
  4549            "fieldConfig": {
  4550              "defaults": {
  4551                "links": []
  4552              },
  4553              "overrides": []
  4554            },
  4555            "fill": 1,
  4556            "fillGradient": 0,
  4557            "gridPos": {
  4558              "h": 7,
  4559              "w": 12,
  4560              "x": 0,
  4561              "y": 38
  4562            },
  4563            "hiddenSeries": false,
  4564            "id": 102,
  4565            "legend": {
  4566              "alignAsTable": true,
  4567              "avg": false,
  4568              "current": true,
  4569              "hideEmpty": true,
  4570              "hideZero": true,
  4571              "max": true,
  4572              "min": false,
  4573              "rightSide": true,
  4574              "show": true,
  4575              "total": false,
  4576              "values": true
  4577            },
  4578            "lines": true,
  4579            "linewidth": 1,
  4580            "links": [],
  4581            "nullPointMode": "null",
  4582            "options": {
  4583              "alertThreshold": true
  4584            },
  4585            "paceLength": 10,
  4586            "percentage": false,
  4587            "pluginVersion": "7.5.11",
  4588            "pointradius": 2,
  4589            "points": false,
  4590            "renderer": "flot",
  4591            "seriesOverrides": [],
  4592            "spaceLength": 10,
  4593            "stack": false,
  4594            "steppedLine": false,
  4595            "targets": [
  4596              {
  4597                "exemplar": true,
  4598                "expr": "sum(rate(ticdc_etcd_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance, type)",
  4599                "format": "time_series",
  4600                "interval": "",
  4601                "intervalFactor": 1,
  4602                "legendFormat": "{{instance}}-{{type}}",
  4603                "refId": "A"
  4604              }
  4605            ],
  4606            "thresholds": [],
  4607            "timeFrom": null,
  4608            "timeRegions": [],
  4609            "timeShift": null,
  4610            "title": "PD etcd requests/s",
  4611            "tooltip": {
  4612              "shared": true,
  4613              "sort": 0,
  4614              "value_type": "individual"
  4615            },
  4616            "type": "graph",
  4617            "xaxis": {
  4618              "buckets": null,
  4619              "mode": "time",
  4620              "name": null,
  4621              "show": true,
  4622              "values": []
  4623            },
  4624            "yaxes": [
  4625              {
  4626                "format": "short",
  4627                "label": null,
  4628                "logBase": 1,
  4629                "max": null,
  4630                "min": null,
  4631                "show": true
  4632              },
  4633              {
  4634                "format": "short",
  4635                "label": null,
  4636                "logBase": 1,
  4637                "max": null,
  4638                "min": null,
  4639                "show": true
  4640              }
  4641            ],
  4642            "yaxis": {
  4643              "align": false,
  4644              "alignLevel": null
  4645            }
  4646          },
  4647          {
  4648            "aliasColors": {},
  4649            "bars": false,
  4650            "dashLength": 10,
  4651            "dashes": false,
  4652            "datasource": "${DS_TEST-CLUSTER}",
  4653            "description": "The number of errors that interrupt changefeed per minute ",
  4654            "fieldConfig": {
  4655              "defaults": {},
  4656              "overrides": []
  4657            },
  4658            "fill": 1,
  4659            "fillGradient": 0,
  4660            "gridPos": {
  4661              "h": 7,
  4662              "w": 12,
  4663              "x": 12,
  4664              "y": 38
  4665            },
  4666            "hiddenSeries": false,
  4667            "id": 82,
  4668            "legend": {
  4669              "alignAsTable": true,
  4670              "avg": false,
  4671              "current": true,
  4672              "max": false,
  4673              "min": false,
  4674              "rightSide": false,
  4675              "show": true,
  4676              "total": false,
  4677              "values": true
  4678            },
  4679            "lines": true,
  4680            "linewidth": 1,
  4681            "links": [],
  4682            "nullPointMode": "null",
  4683            "options": {
  4684              "alertThreshold": true
  4685            },
  4686            "percentage": false,
  4687            "pluginVersion": "7.5.11",
  4688            "pointradius": 2,
  4689            "points": false,
  4690            "renderer": "flot",
  4691            "seriesOverrides": [],
  4692            "spaceLength": 10,
  4693            "stack": false,
  4694            "steppedLine": false,
  4695            "targets": [
  4696              {
  4697                "exemplar": true,
  4698                "expr": "sum(rate(ticdc_processor_exit_with_error_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (instance, namespace,changefeed)",
  4699                "hide": false,
  4700                "interval": "",
  4701                "legendFormat": "{{instance}}-{{namespace}}-{{changefeed}}",
  4702                "refId": "B"
  4703              }
  4704            ],
  4705            "thresholds": [],
  4706            "timeFrom": null,
  4707            "timeRegions": [],
  4708            "timeShift": null,
  4709            "title": "Exit error count/m",
  4710            "tooltip": {
  4711              "shared": true,
  4712              "sort": 0,
  4713              "value_type": "individual"
  4714            },
  4715            "type": "graph",
  4716            "xaxis": {
  4717              "buckets": null,
  4718              "mode": "time",
  4719              "name": null,
  4720              "show": true,
  4721              "values": []
  4722            },
  4723            "yaxes": [
  4724              {
  4725                "format": "short",
  4726                "label": null,
  4727                "logBase": 1,
  4728                "max": null,
  4729                "min": null,
  4730                "show": true
  4731              },
  4732              {
  4733                "format": "short",
  4734                "label": null,
  4735                "logBase": 1,
  4736                "max": null,
  4737                "min": null,
  4738                "show": true
  4739              }
  4740            ],
  4741            "yaxis": {
  4742              "align": false,
  4743              "alignLevel": null
  4744            }
  4745          }
  4746        ],
  4747        "title": "Changefeed",
  4748        "type": "row"
  4749      },
  4750      {
  4751        "collapsed": true,
  4752        "datasource": null,
  4753        "gridPos": {
  4754          "h": 1,
  4755          "w": 24,
  4756          "x": 0,
  4757          "y": 3
  4758        },
  4759        "id": 206,
  4760        "panels": [
  4761          {
  4762            "aliasColors": {},
  4763            "bars": false,
  4764            "dashLength": 10,
  4765            "dashes": false,
  4766            "datasource": "${DS_TEST-CLUSTER}",
  4767            "description": "The number of events that puller outputs to sorter \n per second",
  4768            "fieldConfig": {
  4769              "defaults": {
  4770                "links": []
  4771              },
  4772              "overrides": []
  4773            },
  4774            "fill": 1,
  4775            "fillGradient": 0,
  4776            "gridPos": {
  4777              "h": 4,
  4778              "w": 12,
  4779              "x": 0,
  4780              "y": 4
  4781            },
  4782            "hiddenSeries": false,
  4783            "id": 218,
  4784            "legend": {
  4785              "alignAsTable": true,
  4786              "avg": false,
  4787              "current": true,
  4788              "max": true,
  4789              "min": false,
  4790              "rightSide": false,
  4791              "show": true,
  4792              "total": false,
  4793              "values": true
  4794            },
  4795            "lines": true,
  4796            "linewidth": 1,
  4797            "links": [],
  4798            "nullPointMode": "null",
  4799            "options": {
  4800              "alertThreshold": true
  4801            },
  4802            "paceLength": 10,
  4803            "percentage": false,
  4804            "pluginVersion": "7.5.11",
  4805            "pointradius": 2,
  4806            "points": false,
  4807            "renderer": "flot",
  4808            "seriesOverrides": [],
  4809            "spaceLength": 10,
  4810            "stack": false,
  4811            "steppedLine": false,
  4812            "targets": [
  4813              {
  4814                "exemplar": true,
  4815                "expr": "sum (rate(ticdc_puller_txn_collect_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\", changefeed!~\".*ddl_puller\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, type)",
  4816                "format": "time_series",
  4817                "interval": "",
  4818                "intervalFactor": 1,
  4819                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{type}}",
  4820                "refId": "A"
  4821              }
  4822            ],
  4823            "thresholds": [],
  4824            "timeFrom": null,
  4825            "timeRegions": [],
  4826            "timeShift": null,
  4827            "title": "Puller output events/s",
  4828            "tooltip": {
  4829              "shared": true,
  4830              "sort": 0,
  4831              "value_type": "individual"
  4832            },
  4833            "type": "graph",
  4834            "xaxis": {
  4835              "buckets": null,
  4836              "mode": "time",
  4837              "name": null,
  4838              "show": true,
  4839              "values": []
  4840            },
  4841            "yaxes": [
  4842              {
  4843                "$$hashKey": "object:643",
  4844                "format": "short",
  4845                "label": "",
  4846                "logBase": 1,
  4847                "max": null,
  4848                "min": null,
  4849                "show": true
  4850              },
  4851              {
  4852                "$$hashKey": "object:644",
  4853                "format": "short",
  4854                "label": null,
  4855                "logBase": 1,
  4856                "max": null,
  4857                "min": null,
  4858                "show": false
  4859              }
  4860            ],
  4861            "yaxis": {
  4862              "align": false,
  4863              "alignLevel": null
  4864            }
  4865          },
  4866          {
  4867            "aliasColors": {},
  4868            "bars": false,
  4869            "dashLength": 10,
  4870            "dashes": false,
  4871            "datasource": "${DS_TEST-CLUSTER}",
  4872            "description": "The total number of events that puller outputs",
  4873            "fieldConfig": {
  4874              "defaults": {
  4875                "links": []
  4876              },
  4877              "overrides": []
  4878            },
  4879            "fill": 1,
  4880            "fillGradient": 0,
  4881            "gridPos": {
  4882              "h": 4,
  4883              "w": 12,
  4884              "x": 12,
  4885              "y": 4
  4886            },
  4887            "hiddenSeries": false,
  4888            "id": 229,
  4889            "legend": {
  4890              "alignAsTable": true,
  4891              "avg": false,
  4892              "current": true,
  4893              "max": true,
  4894              "min": false,
  4895              "rightSide": false,
  4896              "show": true,
  4897              "sort": "max",
  4898              "sortDesc": true,
  4899              "total": false,
  4900              "values": true
  4901            },
  4902            "lines": true,
  4903            "linewidth": 1,
  4904            "links": [],
  4905            "nullPointMode": "null",
  4906            "options": {
  4907              "alertThreshold": true
  4908            },
  4909            "paceLength": 10,
  4910            "percentage": false,
  4911            "pluginVersion": "7.5.11",
  4912            "pointradius": 2,
  4913            "points": false,
  4914            "renderer": "flot",
  4915            "seriesOverrides": [],
  4916            "spaceLength": 10,
  4917            "stack": false,
  4918            "steppedLine": false,
  4919            "targets": [
  4920              {
  4921                "exemplar": true,
  4922                "expr": "sum(ticdc_puller_txn_collect_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, type)",
  4923                "format": "time_series",
  4924                "interval": "",
  4925                "intervalFactor": 1,
  4926                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{type}}",
  4927                "refId": "A"
  4928              }
  4929            ],
  4930            "thresholds": [],
  4931            "timeFrom": null,
  4932            "timeRegions": [],
  4933            "timeShift": null,
  4934            "title": "Puller output events",
  4935            "tooltip": {
  4936              "shared": true,
  4937              "sort": 0,
  4938              "value_type": "individual"
  4939            },
  4940            "type": "graph",
  4941            "xaxis": {
  4942              "buckets": null,
  4943              "mode": "time",
  4944              "name": null,
  4945              "show": true,
  4946              "values": []
  4947            },
  4948            "yaxes": [
  4949              {
  4950                "$$hashKey": "object:915",
  4951                "format": "none",
  4952                "label": null,
  4953                "logBase": 1,
  4954                "max": null,
  4955                "min": null,
  4956                "show": true
  4957              },
  4958              {
  4959                "$$hashKey": "object:916",
  4960                "format": "short",
  4961                "label": null,
  4962                "logBase": 1,
  4963                "max": null,
  4964                "min": null,
  4965                "show": true
  4966              }
  4967            ],
  4968            "yaxis": {
  4969              "align": false,
  4970              "alignLevel": null
  4971            }
  4972          },
  4973          {
  4974            "aliasColors": {},
  4975            "bars": false,
  4976            "dashLength": 10,
  4977            "dashes": false,
  4978            "datasource": "${DS_TEST-CLUSTER}",
  4979            "description": "The number of events that sorter outputs to mounter(or table sink worker) per second",
  4980            "fieldConfig": {
  4981              "defaults": {
  4982                "links": []
  4983              },
  4984              "overrides": []
  4985            },
  4986            "fill": 1,
  4987            "fillGradient": 0,
  4988            "gridPos": {
  4989              "h": 4,
  4990              "w": 12,
  4991              "x": 0,
  4992              "y": 8
  4993            },
  4994            "hiddenSeries": false,
  4995            "id": 228,
  4996            "legend": {
  4997              "alignAsTable": true,
  4998              "avg": false,
  4999              "current": true,
  5000              "max": true,
  5001              "min": false,
  5002              "rightSide": false,
  5003              "show": true,
  5004              "total": false,
  5005              "values": true
  5006            },
  5007            "lines": true,
  5008            "linewidth": 1,
  5009            "links": [],
  5010            "nullPointMode": "null",
  5011            "options": {
  5012              "alertThreshold": true
  5013            },
  5014            "paceLength": 10,
  5015            "percentage": false,
  5016            "pluginVersion": "7.5.11",
  5017            "pointradius": 2,
  5018            "points": false,
  5019            "renderer": "flot",
  5020            "seriesOverrides": [],
  5021            "spaceLength": 10,
  5022            "stack": false,
  5023            "steppedLine": false,
  5024            "targets": [
  5025              {
  5026                "exemplar": true,
  5027                "expr": "sum(rate(ticdc_sorter_output_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, type)",
  5028                "format": "time_series",
  5029                "interval": "",
  5030                "intervalFactor": 1,
  5031                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{type}}",
  5032                "refId": "A"
  5033              }
  5034            ],
  5035            "thresholds": [],
  5036            "timeFrom": null,
  5037            "timeRegions": [],
  5038            "timeShift": null,
  5039            "title": "Sorter output events/s",
  5040            "tooltip": {
  5041              "shared": true,
  5042              "sort": 0,
  5043              "value_type": "individual"
  5044            },
  5045            "type": "graph",
  5046            "xaxis": {
  5047              "buckets": null,
  5048              "mode": "time",
  5049              "name": null,
  5050              "show": true,
  5051              "values": []
  5052            },
  5053            "yaxes": [
  5054              {
  5055                "$$hashKey": "object:1089",
  5056                "format": "short",
  5057                "label": null,
  5058                "logBase": 1,
  5059                "max": null,
  5060                "min": null,
  5061                "show": true
  5062              },
  5063              {
  5064                "$$hashKey": "object:1090",
  5065                "format": "short",
  5066                "label": null,
  5067                "logBase": 1,
  5068                "max": null,
  5069                "min": null,
  5070                "show": false
  5071              }
  5072            ],
  5073            "yaxis": {
  5074              "align": false,
  5075              "alignLevel": null
  5076            }
  5077          },
  5078          {
  5079            "aliasColors": {},
  5080            "bars": false,
  5081            "dashLength": 10,
  5082            "dashes": false,
  5083            "datasource": "${DS_TEST-CLUSTER}",
  5084            "description": "The total number of events that sorter outputs",
  5085            "fieldConfig": {
  5086              "defaults": {
  5087                "links": []
  5088              },
  5089              "overrides": []
  5090            },
  5091            "fill": 1,
  5092            "fillGradient": 0,
  5093            "gridPos": {
  5094              "h": 4,
  5095              "w": 12,
  5096              "x": 12,
  5097              "y": 8
  5098            },
  5099            "hiddenSeries": false,
  5100            "id": 220,
  5101            "legend": {
  5102              "alignAsTable": true,
  5103              "avg": false,
  5104              "current": true,
  5105              "max": true,
  5106              "min": false,
  5107              "rightSide": false,
  5108              "show": true,
  5109              "total": false,
  5110              "values": true
  5111            },
  5112            "lines": true,
  5113            "linewidth": 1,
  5114            "links": [],
  5115            "nullPointMode": "null",
  5116            "options": {
  5117              "alertThreshold": true
  5118            },
  5119            "paceLength": 10,
  5120            "percentage": false,
  5121            "pluginVersion": "7.5.11",
  5122            "pointradius": 2,
  5123            "points": false,
  5124            "renderer": "flot",
  5125            "seriesOverrides": [],
  5126            "spaceLength": 10,
  5127            "stack": false,
  5128            "steppedLine": false,
  5129            "targets": [
  5130              {
  5131                "exemplar": true,
  5132                "expr": "sum(ticdc_sorter_output_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace, instance, changefeed, type)",
  5133                "format": "time_series",
  5134                "interval": "",
  5135                "intervalFactor": 1,
  5136                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{type}}",
  5137                "refId": "A"
  5138              }
  5139            ],
  5140            "thresholds": [],
  5141            "timeFrom": null,
  5142            "timeRegions": [],
  5143            "timeShift": null,
  5144            "title": "Sorter output events",
  5145            "tooltip": {
  5146              "shared": true,
  5147              "sort": 0,
  5148              "value_type": "individual"
  5149            },
  5150            "type": "graph",
  5151            "xaxis": {
  5152              "buckets": null,
  5153              "mode": "time",
  5154              "name": null,
  5155              "show": true,
  5156              "values": []
  5157            },
  5158            "yaxes": [
  5159              {
  5160                "$$hashKey": "object:1403",
  5161                "format": "none",
  5162                "label": null,
  5163                "logBase": 1,
  5164                "max": null,
  5165                "min": null,
  5166                "show": true
  5167              },
  5168              {
  5169                "$$hashKey": "object:1404",
  5170                "format": "short",
  5171                "label": null,
  5172                "logBase": 1,
  5173                "max": null,
  5174                "min": null,
  5175                "show": true
  5176              }
  5177            ],
  5178            "yaxis": {
  5179              "align": false,
  5180              "alignLevel": null
  5181            }
  5182          },
  5183          {
  5184            "aliasColors": {},
  5185            "bars": false,
  5186            "dashLength": 10,
  5187            "dashes": false,
  5188            "datasource": "${DS_TEST-CLUSTER}",
  5189            "description": "The number of events that mounter outputs to sink per second",
  5190            "fieldConfig": {
  5191              "defaults": {
  5192                "links": []
  5193              },
  5194              "overrides": []
  5195            },
  5196            "fill": 1,
  5197            "fillGradient": 0,
  5198            "gridPos": {
  5199              "h": 4,
  5200              "w": 12,
  5201              "x": 0,
  5202              "y": 12
  5203            },
  5204            "hiddenSeries": false,
  5205            "id": 219,
  5206            "legend": {
  5207              "alignAsTable": true,
  5208              "avg": false,
  5209              "current": true,
  5210              "max": true,
  5211              "min": false,
  5212              "rightSide": false,
  5213              "show": true,
  5214              "total": false,
  5215              "values": true
  5216            },
  5217            "lines": true,
  5218            "linewidth": 1,
  5219            "links": [],
  5220            "nullPointMode": "null",
  5221            "options": {
  5222              "alertThreshold": true
  5223            },
  5224            "paceLength": 10,
  5225            "percentage": false,
  5226            "pluginVersion": "7.5.11",
  5227            "pointradius": 2,
  5228            "points": false,
  5229            "renderer": "flot",
  5230            "seriesOverrides": [],
  5231            "spaceLength": 10,
  5232            "stack": false,
  5233            "steppedLine": false,
  5234            "targets": [
  5235              {
  5236                "exemplar": true,
  5237                "expr": "sum(rate(ticdc_mounter_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (instance,namespace,changefeed)",
  5238                "format": "time_series",
  5239                "interval": "",
  5240                "intervalFactor": 1,
  5241                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}",
  5242                "refId": "A"
  5243              }
  5244            ],
  5245            "thresholds": [],
  5246            "timeFrom": null,
  5247            "timeRegions": [],
  5248            "timeShift": null,
  5249            "title": "Mounter output events/s",
  5250            "tooltip": {
  5251              "shared": true,
  5252              "sort": 0,
  5253              "value_type": "individual"
  5254            },
  5255            "type": "graph",
  5256            "xaxis": {
  5257              "buckets": null,
  5258              "mode": "time",
  5259              "name": null,
  5260              "show": true,
  5261              "values": []
  5262            },
  5263            "yaxes": [
  5264              {
  5265                "$$hashKey": "object:1121",
  5266                "format": "short",
  5267                "label": null,
  5268                "logBase": 1,
  5269                "max": null,
  5270                "min": null,
  5271                "show": true
  5272              },
  5273              {
  5274                "$$hashKey": "object:1122",
  5275                "format": "short",
  5276                "label": null,
  5277                "logBase": 1,
  5278                "max": null,
  5279                "min": null,
  5280                "show": false
  5281              }
  5282            ],
  5283            "yaxis": {
  5284              "align": false,
  5285              "alignLevel": null
  5286            }
  5287          },
  5288          {
  5289            "aliasColors": {},
  5290            "bars": false,
  5291            "dashLength": 10,
  5292            "dashes": false,
  5293            "datasource": "${DS_TEST-CLUSTER}",
  5294            "description": "The total number of events that mounter outputs",
  5295            "fieldConfig": {
  5296              "defaults": {
  5297                "links": []
  5298              },
  5299              "overrides": []
  5300            },
  5301            "fill": 1,
  5302            "fillGradient": 0,
  5303            "gridPos": {
  5304              "h": 4,
  5305              "w": 12,
  5306              "x": 12,
  5307              "y": 12
  5308            },
  5309            "hiddenSeries": false,
  5310            "id": 224,
  5311            "legend": {
  5312              "alignAsTable": true,
  5313              "avg": false,
  5314              "current": true,
  5315              "max": true,
  5316              "min": false,
  5317              "rightSide": false,
  5318              "show": true,
  5319              "total": false,
  5320              "values": true
  5321            },
  5322            "lines": true,
  5323            "linewidth": 1,
  5324            "links": [],
  5325            "nullPointMode": "null",
  5326            "options": {
  5327              "alertThreshold": true
  5328            },
  5329            "paceLength": 10,
  5330            "percentage": false,
  5331            "pluginVersion": "7.5.11",
  5332            "pointradius": 2,
  5333            "points": false,
  5334            "renderer": "flot",
  5335            "seriesOverrides": [],
  5336            "spaceLength": 10,
  5337            "stack": false,
  5338            "steppedLine": false,
  5339            "targets": [
  5340              {
  5341                "exemplar": true,
  5342                "expr": "sum(ticdc_mounter_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)",
  5343                "format": "time_series",
  5344                "interval": "",
  5345                "intervalFactor": 1,
  5346                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}",
  5347                "refId": "A"
  5348              }
  5349            ],
  5350            "thresholds": [],
  5351            "timeFrom": null,
  5352            "timeRegions": [],
  5353            "timeShift": null,
  5354            "title": "Mounter output events",
  5355            "tooltip": {
  5356              "shared": true,
  5357              "sort": 0,
  5358              "value_type": "individual"
  5359            },
  5360            "type": "graph",
  5361            "xaxis": {
  5362              "buckets": null,
  5363              "mode": "time",
  5364              "name": null,
  5365              "show": true,
  5366              "values": []
  5367            },
  5368            "yaxes": [
  5369              {
  5370                "$$hashKey": "object:1568",
  5371                "format": "none",
  5372                "label": null,
  5373                "logBase": 1,
  5374                "max": null,
  5375                "min": null,
  5376                "show": true
  5377              },
  5378              {
  5379                "$$hashKey": "object:1569",
  5380                "format": "short",
  5381                "label": null,
  5382                "logBase": 1,
  5383                "max": null,
  5384                "min": null,
  5385                "show": true
  5386              }
  5387            ],
  5388            "yaxis": {
  5389              "align": false,
  5390              "alignLevel": null
  5391            }
  5392          },
  5393          {
  5394            "aliasColors": {},
  5395            "bars": false,
  5396            "dashLength": 10,
  5397            "dashes": false,
  5398            "datasource": "${DS_TEST-CLUSTER}",
  5399            "description": "The number of events that table sorter outputs to sink per second",
  5400            "fieldConfig": {
  5401              "defaults": {
  5402                "links": []
  5403              },
  5404              "overrides": []
  5405            },
  5406            "fill": 1,
  5407            "fillGradient": 0,
  5408            "gridPos": {
  5409              "h": 4,
  5410              "w": 12,
  5411              "x": 0,
  5412              "y": 16
  5413            },
  5414            "hiddenSeries": false,
  5415            "id": 223,
  5416            "legend": {
  5417              "alignAsTable": true,
  5418              "avg": false,
  5419              "current": true,
  5420              "max": true,
  5421              "min": false,
  5422              "rightSide": false,
  5423              "show": true,
  5424              "total": false,
  5425              "values": true
  5426            },
  5427            "lines": true,
  5428            "linewidth": 1,
  5429            "links": [],
  5430            "nullPointMode": "null",
  5431            "options": {
  5432              "alertThreshold": true
  5433            },
  5434            "paceLength": 10,
  5435            "percentage": false,
  5436            "pluginVersion": "7.5.11",
  5437            "pointradius": 2,
  5438            "points": false,
  5439            "renderer": "flot",
  5440            "seriesOverrides": [],
  5441            "spaceLength": 10,
  5442            "stack": false,
  5443            "steppedLine": false,
  5444            "targets": [
  5445              {
  5446                "exemplar": true,
  5447                "expr": "sum(rate(ticdc_sink_table_sink_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)",
  5448                "format": "time_series",
  5449                "interval": "",
  5450                "intervalFactor": 1,
  5451                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}",
  5452                "refId": "A"
  5453              }
  5454            ],
  5455            "thresholds": [],
  5456            "timeFrom": null,
  5457            "timeRegions": [],
  5458            "timeShift": null,
  5459            "title": "Table sink output events/s",
  5460            "tooltip": {
  5461              "shared": true,
  5462              "sort": 0,
  5463              "value_type": "individual"
  5464            },
  5465            "type": "graph",
  5466            "xaxis": {
  5467              "buckets": null,
  5468              "mode": "time",
  5469              "name": null,
  5470              "show": true,
  5471              "values": []
  5472            },
  5473            "yaxes": [
  5474              {
  5475                "$$hashKey": "object:1273",
  5476                "format": "short",
  5477                "label": null,
  5478                "logBase": 1,
  5479                "max": null,
  5480                "min": null,
  5481                "show": true
  5482              },
  5483              {
  5484                "$$hashKey": "object:1274",
  5485                "format": "short",
  5486                "label": null,
  5487                "logBase": 1,
  5488                "max": null,
  5489                "min": null,
  5490                "show": false
  5491              }
  5492            ],
  5493            "yaxis": {
  5494              "align": false,
  5495              "alignLevel": null
  5496            }
  5497          },
  5498          {
  5499            "aliasColors": {},
  5500            "bars": false,
  5501            "dashLength": 10,
  5502            "dashes": false,
  5503            "datasource": "${DS_TEST-CLUSTER}",
  5504            "description": "The total number of events that table sinks emit",
  5505            "fieldConfig": {
  5506              "defaults": {
  5507                "links": []
  5508              },
  5509              "overrides": []
  5510            },
  5511            "fill": 1,
  5512            "fillGradient": 0,
  5513            "gridPos": {
  5514              "h": 4,
  5515              "w": 12,
  5516              "x": 12,
  5517              "y": 16
  5518            },
  5519            "hiddenSeries": false,
  5520            "id": 221,
  5521            "legend": {
  5522              "alignAsTable": true,
  5523              "avg": false,
  5524              "current": true,
  5525              "max": true,
  5526              "min": false,
  5527              "rightSide": false,
  5528              "show": true,
  5529              "total": false,
  5530              "values": true
  5531            },
  5532            "lines": true,
  5533            "linewidth": 1,
  5534            "links": [],
  5535            "nullPointMode": "null",
  5536            "options": {
  5537              "alertThreshold": true
  5538            },
  5539            "paceLength": 10,
  5540            "percentage": false,
  5541            "pluginVersion": "7.5.11",
  5542            "pointradius": 2,
  5543            "points": false,
  5544            "renderer": "flot",
  5545            "seriesOverrides": [],
  5546            "spaceLength": 10,
  5547            "stack": false,
  5548            "steppedLine": false,
  5549            "targets": [
  5550              {
  5551                "exemplar": true,
  5552                "expr": "sum(ticdc_sink_table_sink_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)",
  5553                "format": "time_series",
  5554                "interval": "",
  5555                "intervalFactor": 1,
  5556                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}",
  5557                "refId": "A"
  5558              }
  5559            ],
  5560            "thresholds": [],
  5561            "timeFrom": null,
  5562            "timeRegions": [],
  5563            "timeShift": null,
  5564            "title": "Table sink output events",
  5565            "tooltip": {
  5566              "shared": true,
  5567              "sort": 0,
  5568              "value_type": "individual"
  5569            },
  5570            "type": "graph",
  5571            "xaxis": {
  5572              "buckets": null,
  5573              "mode": "time",
  5574              "name": null,
  5575              "show": true,
  5576              "values": []
  5577            },
  5578            "yaxes": [
  5579              {
  5580                "format": "none",
  5581                "label": null,
  5582                "logBase": 1,
  5583                "max": null,
  5584                "min": null,
  5585                "show": true
  5586              },
  5587              {
  5588                "format": "short",
  5589                "label": null,
  5590                "logBase": 1,
  5591                "max": null,
  5592                "min": null,
  5593                "show": true
  5594              }
  5595            ],
  5596            "yaxis": {
  5597              "align": false,
  5598              "alignLevel": null
  5599            }
  5600          },
  5601          {
  5602            "aliasColors": {},
  5603            "bars": false,
  5604            "dashLength": 10,
  5605            "dashes": false,
  5606            "datasource": "${DS_TEST-CLUSTER}",
  5607            "description": "The number of rows that sink flushes to downstream per second.",
  5608            "fieldConfig": {
  5609              "defaults": {
  5610                "links": []
  5611              },
  5612              "overrides": []
  5613            },
  5614            "fill": 1,
  5615            "fillGradient": 0,
  5616            "gridPos": {
  5617              "h": 4,
  5618              "w": 12,
  5619              "x": 0,
  5620              "y": 20
  5621            },
  5622            "hiddenSeries": false,
  5623            "id": 664,
  5624            "legend": {
  5625              "alignAsTable": true,
  5626              "avg": false,
  5627              "current": true,
  5628              "max": true,
  5629              "min": false,
  5630              "rightSide": false,
  5631              "show": true,
  5632              "sort": "current",
  5633              "sortDesc": true,
  5634              "total": false,
  5635              "values": true
  5636            },
  5637            "lines": true,
  5638            "linewidth": 1,
  5639            "links": [],
  5640            "nullPointMode": "null",
  5641            "options": {
  5642              "alertThreshold": true
  5643            },
  5644            "percentage": false,
  5645            "pluginVersion": "7.5.11",
  5646            "pointradius": 2,
  5647            "points": false,
  5648            "renderer": "flot",
  5649            "seriesOverrides": [],
  5650            "spaceLength": 10,
  5651            "stack": false,
  5652            "steppedLine": false,
  5653            "targets": [
  5654              {
  5655                "exemplar": true,
  5656                "expr": "sum(rate(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)",
  5657                "format": "time_series",
  5658                "interval": "",
  5659                "intervalFactor": 1,
  5660                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}",
  5661                "refId": "A"
  5662              }
  5663            ],
  5664            "thresholds": [],
  5665            "timeFrom": null,
  5666            "timeRegions": [],
  5667            "timeShift": null,
  5668            "title": "Sink flush rows/s",
  5669            "tooltip": {
  5670              "shared": true,
  5671              "sort": 2,
  5672              "value_type": "individual"
  5673            },
  5674            "type": "graph",
  5675            "xaxis": {
  5676              "buckets": null,
  5677              "mode": "time",
  5678              "name": null,
  5679              "show": true,
  5680              "values": []
  5681            },
  5682            "yaxes": [
  5683              {
  5684                "$$hashKey": "object:1318",
  5685                "format": "short",
  5686                "label": null,
  5687                "logBase": 1,
  5688                "max": null,
  5689                "min": null,
  5690                "show": true
  5691              },
  5692              {
  5693                "$$hashKey": "object:1319",
  5694                "format": "none",
  5695                "label": null,
  5696                "logBase": 1,
  5697                "max": null,
  5698                "min": null,
  5699                "show": false
  5700              }
  5701            ],
  5702            "yaxis": {
  5703              "align": false,
  5704              "alignLevel": null
  5705            }
  5706          },
  5707          {
  5708            "aliasColors": {},
  5709            "bars": false,
  5710            "dashLength": 10,
  5711            "dashes": false,
  5712            "datasource": "${DS_TEST-CLUSTER}",
  5713            "description": "The number of rows(events) that are flushed by sink.",
  5714            "fieldConfig": {
  5715              "defaults": {
  5716                "links": []
  5717              },
  5718              "overrides": []
  5719            },
  5720            "fill": 1,
  5721            "fillGradient": 0,
  5722            "gridPos": {
  5723              "h": 4,
  5724              "w": 12,
  5725              "x": 12,
  5726              "y": 20
  5727            },
  5728            "hiddenSeries": false,
  5729            "id": 665,
  5730            "legend": {
  5731              "alignAsTable": true,
  5732              "avg": false,
  5733              "current": true,
  5734              "max": true,
  5735              "min": false,
  5736              "rightSide": false,
  5737              "show": true,
  5738              "sort": "current",
  5739              "sortDesc": true,
  5740              "total": false,
  5741              "values": true
  5742            },
  5743            "lines": true,
  5744            "linewidth": 1,
  5745            "links": [],
  5746            "nullPointMode": "null",
  5747            "options": {
  5748              "alertThreshold": true
  5749            },
  5750            "percentage": false,
  5751            "pluginVersion": "7.5.11",
  5752            "pointradius": 2,
  5753            "points": false,
  5754            "renderer": "flot",
  5755            "seriesOverrides": [],
  5756            "spaceLength": 10,
  5757            "stack": false,
  5758            "steppedLine": false,
  5759            "targets": [
  5760              {
  5761                "exemplar": true,
  5762                "expr": "sum(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, instance)",
  5763                "format": "time_series",
  5764                "interval": "",
  5765                "intervalFactor": 1,
  5766                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}",
  5767                "refId": "A"
  5768              }
  5769            ],
  5770            "thresholds": [],
  5771            "timeFrom": null,
  5772            "timeRegions": [],
  5773            "timeShift": null,
  5774            "title": "Sink flush rows",
  5775            "tooltip": {
  5776              "shared": true,
  5777              "sort": 2,
  5778              "value_type": "individual"
  5779            },
  5780            "type": "graph",
  5781            "xaxis": {
  5782              "buckets": null,
  5783              "mode": "time",
  5784              "name": null,
  5785              "show": true,
  5786              "values": []
  5787            },
  5788            "yaxes": [
  5789              {
  5790                "format": "none",
  5791                "label": null,
  5792                "logBase": 1,
  5793                "max": null,
  5794                "min": null,
  5795                "show": true
  5796              },
  5797              {
  5798                "format": "short",
  5799                "label": null,
  5800                "logBase": 1,
  5801                "max": null,
  5802                "min": null,
  5803                "show": true
  5804              }
  5805            ],
  5806            "yaxis": {
  5807              "align": false,
  5808              "alignLevel": null
  5809            }
  5810          }
  5811        ],
  5812        "title": "Dataflow",
  5813        "type": "row"
  5814      },
  5815      {
  5816        "collapsed": true,
  5817        "datasource": null,
  5818        "gridPos": {
  5819          "h": 1,
  5820          "w": 24,
  5821          "x": 0,
  5822          "y": 4
  5823        },
  5824        "id": 608,
  5825        "panels": [
  5826          {
  5827            "aliasColors": {},
  5828            "bars": false,
  5829            "dashLength": 10,
  5830            "dashes": false,
  5831            "datasource": "${DS_TEST-CLUSTER}",
  5832            "description": "Changefeed memory quota",
  5833            "fieldConfig": {
  5834              "defaults": {},
  5835              "overrides": []
  5836            },
  5837            "fill": 1,
  5838            "fillGradient": 0,
  5839            "gridPos": {
  5840              "h": 8,
  5841              "w": 12,
  5842              "x": 0,
  5843              "y": 5
  5844            },
  5845            "hiddenSeries": false,
  5846            "id": 719,
  5847            "legend": {
  5848              "avg": false,
  5849              "current": false,
  5850              "max": true,
  5851              "min": false,
  5852              "show": true,
  5853              "total": false,
  5854              "values": true
  5855            },
  5856            "lines": true,
  5857            "linewidth": 1,
  5858            "nullPointMode": "null",
  5859            "options": {
  5860              "alertThreshold": true
  5861            },
  5862            "percentage": false,
  5863            "pluginVersion": "7.5.11",
  5864            "pointradius": 2,
  5865            "points": false,
  5866            "renderer": "flot",
  5867            "seriesOverrides": [],
  5868            "spaceLength": 10,
  5869            "stack": false,
  5870            "steppedLine": false,
  5871            "targets": [
  5872              {
  5873                "exemplar": true,
  5874                "expr": "ticdc_sinkmanager_memory_quota{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", component!=\"\", instance=~\"$ticdc_instance\"}",
  5875                "interval": "",
  5876                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-memory-{{component}}-{{type}}",
  5877                "refId": "A"
  5878              },
  5879              {
  5880                "exemplar": true,
  5881                "expr": "ticdc_sinkmanager_redo_event_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}",
  5882                "hide": false,
  5883                "interval": "",
  5884                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-redo-cache",
  5885                "refId": "B"
  5886              }
  5887            ],
  5888            "thresholds": [],
  5889            "timeFrom": null,
  5890            "timeRegions": [],
  5891            "timeShift": null,
  5892            "title": "Memory Quota",
  5893            "tooltip": {
  5894              "shared": true,
  5895              "sort": 0,
  5896              "value_type": "individual"
  5897            },
  5898            "type": "graph",
  5899            "xaxis": {
  5900              "buckets": null,
  5901              "mode": "time",
  5902              "name": null,
  5903              "show": true,
  5904              "values": []
  5905            },
  5906            "yaxes": [
  5907              {
  5908                "format": "bytes",
  5909                "label": null,
  5910                "logBase": 1,
  5911                "max": null,
  5912                "min": null,
  5913                "show": true
  5914              },
  5915              {
  5916                "format": "short",
  5917                "label": null,
  5918                "logBase": 1,
  5919                "max": null,
  5920                "min": null,
  5921                "show": false
  5922              }
  5923            ],
  5924            "yaxis": {
  5925              "align": false,
  5926              "alignLevel": null
  5927            }
  5928          },
  5929          {
  5930            "aliasColors": {},
  5931            "bars": false,
  5932            "dashLength": 10,
  5933            "dashes": false,
  5934            "datasource": "${DS_TEST-CLUSTER}",
  5935            "description": "Ratio of redo event cache hit",
  5936            "fieldConfig": {
  5937              "defaults": {},
  5938              "overrides": []
  5939            },
  5940            "fill": 1,
  5941            "fillGradient": 0,
  5942            "gridPos": {
  5943              "h": 8,
  5944              "w": 12,
  5945              "x": 12,
  5946              "y": 5
  5947            },
  5948            "hiddenSeries": false,
  5949            "id": 721,
  5950            "legend": {
  5951              "avg": false,
  5952              "current": false,
  5953              "max": true,
  5954              "min": false,
  5955              "show": true,
  5956              "total": false,
  5957              "values": true
  5958            },
  5959            "lines": true,
  5960            "linewidth": 1,
  5961            "nullPointMode": "null",
  5962            "options": {
  5963              "alertThreshold": true
  5964            },
  5965            "percentage": false,
  5966            "pluginVersion": "7.5.11",
  5967            "pointradius": 2,
  5968            "points": false,
  5969            "renderer": "flot",
  5970            "seriesOverrides": [],
  5971            "spaceLength": 10,
  5972            "stack": false,
  5973            "steppedLine": false,
  5974            "targets": [
  5975              {
  5976                "exemplar": true,
  5977                "expr": "sum(rate(ticdc_sinkmanager_redo_event_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", type=\"hit\"}[1m])) /\nsum(rate(ticdc_sinkmanager_redo_event_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[1m]))",
  5978                "interval": "",
  5979                "legendFormat": "hit-ratio",
  5980                "queryType": "randomWalk",
  5981                "refId": "A"
  5982              }
  5983            ],
  5984            "thresholds": [],
  5985            "timeFrom": null,
  5986            "timeRegions": [],
  5987            "timeShift": null,
  5988            "title": "Redo Event Cache Hit Ratio",
  5989            "tooltip": {
  5990              "shared": true,
  5991              "sort": 0,
  5992              "value_type": "individual"
  5993            },
  5994            "type": "graph",
  5995            "xaxis": {
  5996              "buckets": null,
  5997              "mode": "time",
  5998              "name": null,
  5999              "show": true,
  6000              "values": []
  6001            },
  6002            "yaxes": [
  6003              {
  6004                "format": "percentunit",
  6005                "label": null,
  6006                "logBase": 1,
  6007                "max": null,
  6008                "min": null,
  6009                "show": true
  6010              },
  6011              {
  6012                "format": "short",
  6013                "label": null,
  6014                "logBase": 1,
  6015                "max": null,
  6016                "min": null,
  6017                "show": true
  6018              }
  6019            ],
  6020            "yaxis": {
  6021              "align": false,
  6022              "alignLevel": null
  6023            }
  6024          },
  6025          {
  6026            "aliasColors": {},
  6027            "bars": false,
  6028            "dashLength": 10,
  6029            "dashes": false,
  6030            "datasource": "${DS_TEST-CLUSTER}",
  6031            "description": "Row count for batch to the downstream sink.",
  6032            "fieldConfig": {
  6033              "defaults": {},
  6034              "overrides": []
  6035            },
  6036            "fill": 1,
  6037            "fillGradient": 0,
  6038            "gridPos": {
  6039              "h": 8,
  6040              "w": 12,
  6041              "x": 0,
  6042              "y": 13
  6043            },
  6044            "hiddenSeries": false,
  6045            "id": 610,
  6046            "legend": {
  6047              "alignAsTable": true,
  6048              "avg": false,
  6049              "current": true,
  6050              "max": false,
  6051              "min": false,
  6052              "rightSide": false,
  6053              "show": true,
  6054              "total": false,
  6055              "values": true
  6056            },
  6057            "lines": true,
  6058            "linewidth": 1,
  6059            "nullPointMode": "null",
  6060            "options": {
  6061              "alertThreshold": true
  6062            },
  6063            "percentage": false,
  6064            "pluginVersion": "7.5.11",
  6065            "pointradius": 2,
  6066            "points": false,
  6067            "renderer": "flot",
  6068            "seriesOverrides": [],
  6069            "spaceLength": 10,
  6070            "stack": false,
  6071            "steppedLine": false,
  6072            "targets": [
  6073              {
  6074                "exemplar": true,
  6075                "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_batch_row_count_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed,instance))",
  6076                "interval": "",
  6077                "legendFormat": "99.9%-{{namespace}}-{{changefeed}}-{{instance}}",
  6078                "queryType": "randomWalk",
  6079                "refId": "A"
  6080              },
  6081              {
  6082                "exemplar": true,
  6083                "expr": "sum(rate(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance) / sum(rate(ticdc_sinkv2_batch_row_count_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)",
  6084                "hide": false,
  6085                "interval": "",
  6086                "legendFormat": "avg-{{namespace}}-{{changefeed}}-{{instance}}",
  6087                "refId": "B"
  6088              }
  6089            ],
  6090            "thresholds": [],
  6091            "timeFrom": null,
  6092            "timeRegions": [],
  6093            "timeShift": null,
  6094            "title": "Output Row Batch Size",
  6095            "tooltip": {
  6096              "shared": true,
  6097              "sort": 0,
  6098              "value_type": "individual"
  6099            },
  6100            "type": "graph",
  6101            "xaxis": {
  6102              "buckets": null,
  6103              "mode": "time",
  6104              "name": null,
  6105              "show": true,
  6106              "values": []
  6107            },
  6108            "yaxes": [
  6109              {
  6110                "format": "short",
  6111                "label": null,
  6112                "logBase": 1,
  6113                "max": null,
  6114                "min": null,
  6115                "show": true
  6116              },
  6117              {
  6118                "format": "short",
  6119                "label": null,
  6120                "logBase": 1,
  6121                "max": null,
  6122                "min": null,
  6123                "show": false
  6124              }
  6125            ],
  6126            "yaxis": {
  6127              "align": false,
  6128              "alignLevel": null
  6129            }
  6130          },
  6131          {
  6132            "aliasColors": {},
  6133            "bars": false,
  6134            "dashLength": 10,
  6135            "dashes": false,
  6136            "datasource": "${DS_TEST-CLUSTER}",
  6137            "description": "Row count for total output rows.",
  6138            "fieldConfig": {
  6139              "defaults": {},
  6140              "overrides": []
  6141            },
  6142            "fill": 1,
  6143            "fillGradient": 0,
  6144            "gridPos": {
  6145              "h": 8,
  6146              "w": 12,
  6147              "x": 12,
  6148              "y": 13
  6149            },
  6150            "hiddenSeries": false,
  6151            "id": 612,
  6152            "legend": {
  6153              "alignAsTable": true,
  6154              "avg": false,
  6155              "current": true,
  6156              "max": false,
  6157              "min": false,
  6158              "rightSide": false,
  6159              "show": true,
  6160              "total": false,
  6161              "values": true
  6162            },
  6163            "lines": true,
  6164            "linewidth": 1,
  6165            "nullPointMode": "null",
  6166            "options": {
  6167              "alertThreshold": true
  6168            },
  6169            "percentage": false,
  6170            "pluginVersion": "7.5.11",
  6171            "pointradius": 2,
  6172            "points": false,
  6173            "renderer": "flot",
  6174            "seriesOverrides": [],
  6175            "spaceLength": 10,
  6176            "stack": false,
  6177            "steppedLine": false,
  6178            "targets": [
  6179              {
  6180                "exemplar": true,
  6181                "expr": "sum(rate(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)",
  6182                "interval": "",
  6183                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}",
  6184                "queryType": "randomWalk",
  6185                "refId": "A"
  6186              }
  6187            ],
  6188            "thresholds": [],
  6189            "timeFrom": null,
  6190            "timeRegions": [],
  6191            "timeShift": null,
  6192            "title": "Output Row Count (per second)",
  6193            "tooltip": {
  6194              "shared": true,
  6195              "sort": 0,
  6196              "value_type": "individual"
  6197            },
  6198            "type": "graph",
  6199            "xaxis": {
  6200              "buckets": null,
  6201              "mode": "time",
  6202              "name": null,
  6203              "show": true,
  6204              "values": []
  6205            },
  6206            "yaxes": [
  6207              {
  6208                "format": "short",
  6209                "label": null,
  6210                "logBase": 1,
  6211                "max": null,
  6212                "min": null,
  6213                "show": true
  6214              },
  6215              {
  6216                "format": "short",
  6217                "label": null,
  6218                "logBase": 1,
  6219                "max": null,
  6220                "min": null,
  6221                "show": false
  6222              }
  6223            ],
  6224            "yaxis": {
  6225              "align": false,
  6226              "alignLevel": null
  6227            }
  6228          },
  6229          {
  6230            "aliasColors": {},
  6231            "bars": false,
  6232            "dashLength": 10,
  6233            "dashes": false,
  6234            "datasource": "${DS_TEST-CLUSTER}",
  6235            "description": "DDL count and executing duration",
  6236            "fieldConfig": {
  6237              "defaults": {},
  6238              "overrides": []
  6239            },
  6240            "fill": 1,
  6241            "fillGradient": 0,
  6242            "gridPos": {
  6243              "h": 8,
  6244              "w": 12,
  6245              "x": 0,
  6246              "y": 21
  6247            },
  6248            "hiddenSeries": false,
  6249            "id": 613,
  6250            "legend": {
  6251              "alignAsTable": true,
  6252              "avg": false,
  6253              "current": true,
  6254              "max": false,
  6255              "min": false,
  6256              "rightSide": false,
  6257              "show": true,
  6258              "total": false,
  6259              "values": true
  6260            },
  6261            "lines": true,
  6262            "linewidth": 1,
  6263            "nullPointMode": "null",
  6264            "options": {
  6265              "alertThreshold": true
  6266            },
  6267            "percentage": false,
  6268            "pluginVersion": "7.5.11",
  6269            "pointradius": 2,
  6270            "points": false,
  6271            "renderer": "flot",
  6272            "seriesOverrides": [
  6273              {
  6274                "alias": "/duration/",
  6275                "yaxis": 2
  6276              }
  6277            ],
  6278            "spaceLength": 10,
  6279            "stack": false,
  6280            "steppedLine": false,
  6281            "targets": [
  6282              {
  6283                "exemplar": true,
  6284                "expr": "sum(rate(ticdc_sink_ddl_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)",
  6285                "interval": "",
  6286                "legendFormat": "count-{{namespace}}-{{changefeed}}-{{instance}}",
  6287                "queryType": "randomWalk",
  6288                "refId": "A"
  6289              },
  6290              {
  6291                "exemplar": true,
  6292                "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_ddl_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed,instance))",
  6293                "hide": false,
  6294                "interval": "",
  6295                "legendFormat": "99.9-duration-{{namespace}}-{{changefeed}}-{{instance}}",
  6296                "refId": "B"
  6297              }
  6298            ],
  6299            "thresholds": [],
  6300            "timeFrom": null,
  6301            "timeRegions": [],
  6302            "timeShift": null,
  6303            "title": "Output DDL Count & Executing Duration",
  6304            "tooltip": {
  6305              "shared": true,
  6306              "sort": 0,
  6307              "value_type": "individual"
  6308            },
  6309            "type": "graph",
  6310            "xaxis": {
  6311              "buckets": null,
  6312              "mode": "time",
  6313              "name": null,
  6314              "show": true,
  6315              "values": []
  6316            },
  6317            "yaxes": [
  6318              {
  6319                "format": "short",
  6320                "label": null,
  6321                "logBase": 1,
  6322                "max": null,
  6323                "min": null,
  6324                "show": true
  6325              },
  6326              {
  6327                "format": "s",
  6328                "label": null,
  6329                "logBase": 1,
  6330                "max": null,
  6331                "min": null,
  6332                "show": true
  6333              }
  6334            ],
  6335            "yaxis": {
  6336              "align": false,
  6337              "alignLevel": null
  6338            }
  6339          },
  6340          {
  6341            "aliasColors": {},
  6342            "bars": false,
  6343            "dashLength": 10,
  6344            "dashes": false,
  6345            "datasource": "${DS_TEST-CLUSTER}",
  6346            "description": "Size of large rows (size >= 2K).",
  6347            "fieldConfig": {
  6348              "defaults": {},
  6349              "overrides": []
  6350            },
  6351            "fill": 1,
  6352            "fillGradient": 0,
  6353            "gridPos": {
  6354              "h": 8,
  6355              "w": 12,
  6356              "x": 12,
  6357              "y": 21
  6358            },
  6359            "hiddenSeries": false,
  6360            "id": 611,
  6361            "legend": {
  6362              "alignAsTable": true,
  6363              "avg": false,
  6364              "current": true,
  6365              "max": false,
  6366              "min": false,
  6367              "rightSide": false,
  6368              "show": true,
  6369              "total": false,
  6370              "values": true
  6371            },
  6372            "lines": true,
  6373            "linewidth": 1,
  6374            "nullPointMode": "null",
  6375            "options": {
  6376              "alertThreshold": true
  6377            },
  6378            "percentage": false,
  6379            "pluginVersion": "7.5.11",
  6380            "pointradius": 2,
  6381            "points": false,
  6382            "renderer": "flot",
  6383            "seriesOverrides": [],
  6384            "spaceLength": 10,
  6385            "stack": false,
  6386            "steppedLine": false,
  6387            "targets": [
  6388              {
  6389                "exemplar": true,
  6390                "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_large_row_size_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed,instance))",
  6391                "interval": "",
  6392                "legendFormat": "99.9%-{{namespace}}-{{changefeed}}-{{instance}}",
  6393                "queryType": "randomWalk",
  6394                "refId": "A"
  6395              },
  6396              {
  6397                "exemplar": true,
  6398                "expr": "sum(rate(ticdc_sink_large_row_size_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance) / sum(rate(ticdc_sink_large_row_size_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)",
  6399                "hide": false,
  6400                "interval": "",
  6401                "legendFormat": "avg-{{namespace}}-{{changefeed}}-{{instance}}",
  6402                "refId": "B"
  6403              }
  6404            ],
  6405            "thresholds": [],
  6406            "timeFrom": null,
  6407            "timeRegions": [],
  6408            "timeShift": null,
  6409            "title": "Large Row (size >= 128K) Size",
  6410            "tooltip": {
  6411              "shared": true,
  6412              "sort": 0,
  6413              "value_type": "individual"
  6414            },
  6415            "type": "graph",
  6416            "xaxis": {
  6417              "buckets": null,
  6418              "mode": "time",
  6419              "name": null,
  6420              "show": true,
  6421              "values": []
  6422            },
  6423            "yaxes": [
  6424              {
  6425                "format": "short",
  6426                "label": null,
  6427                "logBase": 2,
  6428                "max": null,
  6429                "min": null,
  6430                "show": true
  6431              },
  6432              {
  6433                "format": "short",
  6434                "label": null,
  6435                "logBase": 1,
  6436                "max": null,
  6437                "min": null,
  6438                "show": false
  6439              }
  6440            ],
  6441            "yaxis": {
  6442              "align": false,
  6443              "alignLevel": null
  6444            }
  6445          },
  6446          {
  6447            "aliasColors": {},
  6448            "bars": false,
  6449            "dashLength": 10,
  6450            "dashes": false,
  6451            "datasource": "${DS_TEST-CLUSTER}",
  6452            "description": "Count of errors in the last minute.",
  6453            "fieldConfig": {
  6454              "defaults": {},
  6455              "overrides": []
  6456            },
  6457            "fill": 1,
  6458            "fillGradient": 0,
  6459            "gridPos": {
  6460              "h": 8,
  6461              "w": 12,
  6462              "x": 0,
  6463              "y": 29
  6464            },
  6465            "hiddenSeries": false,
  6466            "id": 614,
  6467            "legend": {
  6468              "alignAsTable": true,
  6469              "avg": false,
  6470              "current": true,
  6471              "max": false,
  6472              "min": false,
  6473              "rightSide": false,
  6474              "show": true,
  6475              "total": false,
  6476              "values": true
  6477            },
  6478            "lines": true,
  6479            "linewidth": 1,
  6480            "nullPointMode": "null",
  6481            "options": {
  6482              "alertThreshold": true
  6483            },
  6484            "percentage": false,
  6485            "pluginVersion": "7.5.11",
  6486            "pointradius": 2,
  6487            "points": false,
  6488            "renderer": "flot",
  6489            "seriesOverrides": [],
  6490            "spaceLength": 10,
  6491            "stack": false,
  6492            "steppedLine": false,
  6493            "targets": [
  6494              {
  6495                "exemplar": true,
  6496                "expr": "sum(delta(ticdc_sink_execution_error{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)",
  6497                "interval": "",
  6498                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}",
  6499                "queryType": "randomWalk",
  6500                "refId": "A"
  6501              }
  6502            ],
  6503            "thresholds": [],
  6504            "timeFrom": null,
  6505            "timeRegions": [],
  6506            "timeShift": null,
  6507            "title": "Sink Error Count (1min)",
  6508            "tooltip": {
  6509              "shared": true,
  6510              "sort": 0,
  6511              "value_type": "individual"
  6512            },
  6513            "type": "graph",
  6514            "xaxis": {
  6515              "buckets": null,
  6516              "mode": "time",
  6517              "name": null,
  6518              "show": true,
  6519              "values": []
  6520            },
  6521            "yaxes": [
  6522              {
  6523                "format": "short",
  6524                "label": null,
  6525                "logBase": 1,
  6526                "max": null,
  6527                "min": null,
  6528                "show": true
  6529              },
  6530              {
  6531                "format": "short",
  6532                "label": null,
  6533                "logBase": 1,
  6534                "max": null,
  6535                "min": null,
  6536                "show": false
  6537              }
  6538            ],
  6539            "yaxis": {
  6540              "align": false,
  6541              "alignLevel": null
  6542            }
  6543          }
  6544        ],
  6545        "title": "Sink - General",
  6546        "type": "row"
  6547      },
  6548      {
  6549        "collapsed": true,
  6550        "datasource": null,
  6551        "gridPos": {
  6552          "h": 1,
  6553          "w": 24,
  6554          "x": 0,
  6555          "y": 5
  6556        },
  6557        "id": 616,
  6558        "panels": [
  6559          {
  6560            "aliasColors": {},
  6561            "bars": false,
  6562            "dashLength": 10,
  6563            "dashes": false,
  6564            "datasource": "${DS_TEST-CLUSTER}",
  6565            "description": "Duration of event staying in conflict detector",
  6566            "fieldConfig": {
  6567              "defaults": {},
  6568              "overrides": []
  6569            },
  6570            "fill": 1,
  6571            "fillGradient": 0,
  6572            "gridPos": {
  6573              "h": 8,
  6574              "w": 12,
  6575              "x": 0,
  6576              "y": 26
  6577            },
  6578            "hiddenSeries": false,
  6579            "id": 618,
  6580            "legend": {
  6581              "alignAsTable": true,
  6582              "avg": false,
  6583              "current": true,
  6584              "max": false,
  6585              "min": false,
  6586              "rightSide": false,
  6587              "show": true,
  6588              "total": false,
  6589              "values": true
  6590            },
  6591            "lines": true,
  6592            "linewidth": 1,
  6593            "nullPointMode": "null",
  6594            "options": {
  6595              "alertThreshold": true
  6596            },
  6597            "percentage": false,
  6598            "pluginVersion": "7.5.11",
  6599            "pointradius": 2,
  6600            "points": false,
  6601            "renderer": "flot",
  6602            "seriesOverrides": [],
  6603            "spaceLength": 10,
  6604            "stack": false,
  6605            "steppedLine": false,
  6606            "targets": [
  6607              {
  6608                "exemplar": true,
  6609                "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_conflict_detect_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,changefeed,instance))",
  6610                "interval": "",
  6611                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-detect-P999",
  6612                "queryType": "randomWalk",
  6613                "refId": "C"
  6614              },
  6615              {
  6616                "exemplar": true,
  6617                "expr": "sum(rate(ticdc_sink_txn_conflict_detect_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (changefeed,instance) / \nsum(rate(ticdc_sink_txn_conflict_detect_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (changefeed,instance)",
  6618                "hide": false,
  6619                "interval": "",
  6620                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-detect-avg",
  6621                "refId": "D"
  6622              },
  6623              {
  6624                "exemplar": true,
  6625                "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_queue_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed,instance))",
  6626                "interval": "",
  6627                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-queue-P999",
  6628                "queryType": "randomWalk",
  6629                "refId": "A"
  6630              },
  6631              {
  6632                "exemplar": true,
  6633                "expr": "sum(rate(ticdc_sink_txn_queue_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_txn_queue_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)",
  6634                "hide": false,
  6635                "interval": "",
  6636                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-queue-avg",
  6637                "refId": "B"
  6638              }
  6639            ],
  6640            "thresholds": [],
  6641            "timeFrom": null,
  6642            "timeRegions": [],
  6643            "timeShift": null,
  6644            "title": "Conflict Detect Duration",
  6645            "tooltip": {
  6646              "shared": true,
  6647              "sort": 0,
  6648              "value_type": "individual"
  6649            },
  6650            "type": "graph",
  6651            "xaxis": {
  6652              "buckets": null,
  6653              "mode": "time",
  6654              "name": null,
  6655              "show": true,
  6656              "values": []
  6657            },
  6658            "yaxes": [
  6659              {
  6660                "format": "s",
  6661                "label": null,
  6662                "logBase": 2,
  6663                "max": null,
  6664                "min": null,
  6665                "show": true
  6666              },
  6667              {
  6668                "format": "short",
  6669                "label": null,
  6670                "logBase": 1,
  6671                "max": null,
  6672                "min": null,
  6673                "show": false
  6674              }
  6675            ],
  6676            "yaxis": {
  6677              "align": false,
  6678              "alignLevel": null
  6679            }
  6680          },
  6681          {
  6682            "aliasColors": {},
  6683            "bars": false,
  6684            "dashLength": 10,
  6685            "dashes": false,
  6686            "datasource": "${DS_TEST-CLUSTER}",
  6687            "description": "Full flush (backend flush + callback + conflict detector notify) duration",
  6688            "fieldConfig": {
  6689              "defaults": {},
  6690              "overrides": []
  6691            },
  6692            "fill": 1,
  6693            "fillGradient": 0,
  6694            "gridPos": {
  6695              "h": 8,
  6696              "w": 12,
  6697              "x": 12,
  6698              "y": 26
  6699            },
  6700            "hiddenSeries": false,
  6701            "id": 620,
  6702            "legend": {
  6703              "alignAsTable": true,
  6704              "avg": false,
  6705              "current": true,
  6706              "max": false,
  6707              "min": false,
  6708              "rightSide": false,
  6709              "show": true,
  6710              "total": false,
  6711              "values": true
  6712            },
  6713            "lines": true,
  6714            "linewidth": 1,
  6715            "nullPointMode": "null",
  6716            "options": {
  6717              "alertThreshold": true
  6718            },
  6719            "percentage": false,
  6720            "pluginVersion": "7.5.11",
  6721            "pointradius": 2,
  6722            "points": false,
  6723            "renderer": "flot",
  6724            "seriesOverrides": [],
  6725            "spaceLength": 10,
  6726            "stack": false,
  6727            "steppedLine": false,
  6728            "targets": [
  6729              {
  6730                "exemplar": true,
  6731                "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_worker_flush_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed,instance))",
  6732                "interval": "",
  6733                "legendFormat": "99.9-{{namespace}}-{{changefeed}}-{{instance}}",
  6734                "queryType": "randomWalk",
  6735                "refId": "A"
  6736              },
  6737              {
  6738                "exemplar": true,
  6739                "expr": "sum(rate(ticdc_sink_txn_worker_flush_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_txn_worker_flush_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)",
  6740                "hide": false,
  6741                "interval": "",
  6742                "legendFormat": "avg-{{namespace}}-{{changefeed}}-{{instance}}",
  6743                "refId": "B"
  6744              }
  6745            ],
  6746            "thresholds": [],
  6747            "timeFrom": null,
  6748            "timeRegions": [],
  6749            "timeShift": null,
  6750            "title": "Full Flush Duration",
  6751            "tooltip": {
  6752              "shared": true,
  6753              "sort": 0,
  6754              "value_type": "individual"
  6755            },
  6756            "type": "graph",
  6757            "xaxis": {
  6758              "buckets": null,
  6759              "mode": "time",
  6760              "name": null,
  6761              "show": true,
  6762              "values": []
  6763            },
  6764            "yaxes": [
  6765              {
  6766                "format": "s",
  6767                "label": null,
  6768                "logBase": 2,
  6769                "max": null,
  6770                "min": null,
  6771                "show": true
  6772              },
  6773              {
  6774                "format": "short",
  6775                "label": null,
  6776                "logBase": 1,
  6777                "max": null,
  6778                "min": null,
  6779                "show": false
  6780              }
  6781            ],
  6782            "yaxis": {
  6783              "align": false,
  6784              "alignLevel": null
  6785            }
  6786          },
  6787          {
  6788            "aliasColors": {},
  6789            "bars": false,
  6790            "dashLength": 10,
  6791            "dashes": false,
  6792            "datasource": "${DS_TEST-CLUSTER}",
  6793            "description": "Sink worker busy ratio",
  6794            "fieldConfig": {
  6795              "defaults": {},
  6796              "overrides": []
  6797            },
  6798            "fill": 1,
  6799            "fillGradient": 0,
  6800            "gridPos": {
  6801              "h": 8,
  6802              "w": 12,
  6803              "x": 0,
  6804              "y": 34
  6805            },
  6806            "hiddenSeries": false,
  6807            "id": 619,
  6808            "legend": {
  6809              "alignAsTable": true,
  6810              "avg": false,
  6811              "current": true,
  6812              "max": false,
  6813              "min": false,
  6814              "rightSide": false,
  6815              "show": true,
  6816              "total": false,
  6817              "values": true
  6818            },
  6819            "lines": true,
  6820            "linewidth": 1,
  6821            "nullPointMode": "null",
  6822            "options": {
  6823              "alertThreshold": true
  6824            },
  6825            "percentage": false,
  6826            "pluginVersion": "7.5.11",
  6827            "pointradius": 2,
  6828            "points": false,
  6829            "renderer": "flot",
  6830            "seriesOverrides": [],
  6831            "spaceLength": 10,
  6832            "stack": false,
  6833            "steppedLine": false,
  6834            "targets": [
  6835              {
  6836                "exemplar": true,
  6837                "expr": "sum(rate(ticdc_sink_txn_worker_flush_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance,id) /sum(rate(ticdc_sink_txn_worker_total_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance,id) *100",
  6838                "interval": "",
  6839                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-worker-{{id}}",
  6840                "queryType": "randomWalk",
  6841                "refId": "A"
  6842              }
  6843            ],
  6844            "thresholds": [],
  6845            "timeFrom": null,
  6846            "timeRegions": [],
  6847            "timeShift": null,
  6848            "title": "Worker Busy Ratio",
  6849            "tooltip": {
  6850              "shared": true,
  6851              "sort": 0,
  6852              "value_type": "individual"
  6853            },
  6854            "type": "graph",
  6855            "xaxis": {
  6856              "buckets": null,
  6857              "mode": "time",
  6858              "name": null,
  6859              "show": true,
  6860              "values": []
  6861            },
  6862            "yaxes": [
  6863              {
  6864                "format": "percent",
  6865                "label": null,
  6866                "logBase": 1,
  6867                "max": null,
  6868                "min": null,
  6869                "show": true
  6870              },
  6871              {
  6872                "format": "short",
  6873                "label": null,
  6874                "logBase": 1,
  6875                "max": null,
  6876                "min": null,
  6877                "show": false
  6878              }
  6879            ],
  6880            "yaxis": {
  6881              "align": false,
  6882              "alignLevel": null
  6883            }
  6884          },
  6885          {
  6886            "aliasColors": {},
  6887            "bars": false,
  6888            "dashLength": 10,
  6889            "dashes": false,
  6890            "datasource": "${DS_TEST-CLUSTER}",
  6891            "description": "",
  6892            "fieldConfig": {
  6893              "defaults": {},
  6894              "overrides": []
  6895            },
  6896            "fill": 1,
  6897            "fillGradient": 0,
  6898            "gridPos": {
  6899              "h": 8,
  6900              "w": 12,
  6901              "x": 12,
  6902              "y": 34
  6903            },
  6904            "hiddenSeries": false,
  6905            "id": 634,
  6906            "legend": {
  6907              "alignAsTable": true,
  6908              "avg": false,
  6909              "current": true,
  6910              "max": false,
  6911              "min": false,
  6912              "show": true,
  6913              "total": false,
  6914              "values": true
  6915            },
  6916            "lines": true,
  6917            "linewidth": 1,
  6918            "nullPointMode": "null",
  6919            "options": {
  6920              "alertThreshold": true
  6921            },
  6922            "percentage": false,
  6923            "pluginVersion": "7.5.11",
  6924            "pointradius": 2,
  6925            "points": false,
  6926            "renderer": "flot",
  6927            "seriesOverrides": [],
  6928            "spaceLength": 10,
  6929            "stack": false,
  6930            "steppedLine": false,
  6931            "targets": [
  6932              {
  6933                "exemplar": true,
  6934                "expr": "sum(rate(ticdc_sink_txn_worker_handled_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance,id)",
  6935                "interval": "",
  6936                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{id}}",
  6937                "queryType": "randomWalk",
  6938                "refId": "A"
  6939              }
  6940            ],
  6941            "thresholds": [],
  6942            "timeFrom": null,
  6943            "timeRegions": [],
  6944            "timeShift": null,
  6945            "title": "Worker Input Rows",
  6946            "tooltip": {
  6947              "shared": true,
  6948              "sort": 0,
  6949              "value_type": "individual"
  6950            },
  6951            "type": "graph",
  6952            "xaxis": {
  6953              "buckets": null,
  6954              "mode": "time",
  6955              "name": null,
  6956              "show": true,
  6957              "values": []
  6958            },
  6959            "yaxes": [
  6960              {
  6961                "format": "short",
  6962                "label": null,
  6963                "logBase": 1,
  6964                "max": null,
  6965                "min": null,
  6966                "show": true
  6967              },
  6968              {
  6969                "format": "short",
  6970                "label": null,
  6971                "logBase": 1,
  6972                "max": null,
  6973                "min": null,
  6974                "show": false
  6975              }
  6976            ],
  6977            "yaxis": {
  6978              "align": false,
  6979              "alignLevel": null
  6980            }
  6981          },
  6982          {
  6983            "aliasColors": {},
  6984            "bars": false,
  6985            "dashLength": 10,
  6986            "dashes": false,
  6987            "datasource": "${DS_TEST-CLUSTER}",
  6988            "description": "Distribution of flush transaction duration to backend",
  6989            "fieldConfig": {
  6990              "defaults": {},
  6991              "overrides": []
  6992            },
  6993            "fill": 1,
  6994            "fillGradient": 0,
  6995            "gridPos": {
  6996              "h": 8,
  6997              "w": 12,
  6998              "x": 0,
  6999              "y": 42
  7000            },
  7001            "hiddenSeries": false,
  7002            "id": 635,
  7003            "legend": {
  7004              "alignAsTable": true,
  7005              "avg": false,
  7006              "current": true,
  7007              "max": false,
  7008              "min": false,
  7009              "show": true,
  7010              "total": false,
  7011              "values": true
  7012            },
  7013            "lines": true,
  7014            "linewidth": 1,
  7015            "nullPointMode": "null",
  7016            "options": {
  7017              "alertThreshold": true
  7018            },
  7019            "percentage": false,
  7020            "pluginVersion": "7.5.11",
  7021            "pointradius": 2,
  7022            "points": false,
  7023            "renderer": "flot",
  7024            "seriesOverrides": [],
  7025            "spaceLength": 10,
  7026            "stack": false,
  7027            "steppedLine": false,
  7028            "targets": [
  7029              {
  7030                "exemplar": true,
  7031                "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_sink_dml_batch_commit_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed,instance))",
  7032                "interval": "",
  7033                "legendFormat": "99.9-{{namespace}}-{{changefeed}}-{{instance}}",
  7034                "queryType": "randomWalk",
  7035                "refId": "A"
  7036              },
  7037              {
  7038                "exemplar": true,
  7039                "expr": "sum(rate(ticdc_sink_txn_sink_dml_batch_commit_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_txn_sink_dml_batch_commit_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)",
  7040                "hide": false,
  7041                "interval": "",
  7042                "legendFormat": "avg-{{namespace}}-{{changefeed}}-{{instance}}",
  7043                "refId": "B"
  7044              }
  7045            ],
  7046            "thresholds": [],
  7047            "timeFrom": null,
  7048            "timeRegions": [],
  7049            "timeShift": null,
  7050            "title": "Backend Flush Duration",
  7051            "tooltip": {
  7052              "shared": true,
  7053              "sort": 0,
  7054              "value_type": "individual"
  7055            },
  7056            "type": "graph",
  7057            "xaxis": {
  7058              "buckets": null,
  7059              "mode": "time",
  7060              "name": null,
  7061              "show": true,
  7062              "values": []
  7063            },
  7064            "yaxes": [
  7065              {
  7066                "format": "s",
  7067                "label": null,
  7068                "logBase": 1,
  7069                "max": null,
  7070                "min": null,
  7071                "show": true
  7072              },
  7073              {
  7074                "format": "short",
  7075                "label": null,
  7076                "logBase": 1,
  7077                "max": null,
  7078                "min": null,
  7079                "show": false
  7080              }
  7081            ],
  7082            "yaxis": {
  7083              "align": false,
  7084              "alignLevel": null
  7085            }
  7086          }
  7087        ],
  7088        "title": "Sink - Transaction Sink",
  7089        "type": "row"
  7090      },
  7091      {
  7092        "collapsed": true,
  7093        "datasource": null,
  7094        "gridPos": {
  7095          "h": 1,
  7096          "w": 24,
  7097          "x": 0,
  7098          "y": 6
  7099        },
  7100        "id": 623,
  7101        "panels": [
  7102          {
  7103            "aliasColors": {},
  7104            "bars": false,
  7105            "dashLength": 10,
  7106            "dashes": false,
  7107            "datasource": "${DS_TEST-CLUSTER}",
  7108            "description": "MQ worker send messages to Kafka, this metric record the time cost on send every message.",
  7109            "fieldConfig": {
  7110              "defaults": {},
  7111              "overrides": []
  7112            },
  7113            "fill": 1,
  7114            "fillGradient": 0,
  7115            "gridPos": {
  7116              "h": 7,
  7117              "w": 12,
  7118              "x": 0,
  7119              "y": 7
  7120            },
  7121            "hiddenSeries": false,
  7122            "id": 653,
  7123            "legend": {
  7124              "alignAsTable": true,
  7125              "avg": true,
  7126              "current": true,
  7127              "max": true,
  7128              "min": false,
  7129              "rightSide": false,
  7130              "show": true,
  7131              "total": false,
  7132              "values": true
  7133            },
  7134            "lines": true,
  7135            "linewidth": 1,
  7136            "nullPointMode": "null",
  7137            "options": {
  7138              "alertThreshold": true
  7139            },
  7140            "percentage": false,
  7141            "pluginVersion": "7.5.11",
  7142            "pointradius": 2,
  7143            "points": false,
  7144            "renderer": "flot",
  7145            "seriesOverrides": [],
  7146            "spaceLength": 10,
  7147            "stack": false,
  7148            "steppedLine": false,
  7149            "targets": [
  7150              {
  7151                "exemplar": true,
  7152                "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_worker_send_message_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed,instance))",
  7153                "interval": "",
  7154                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-P999",
  7155                "queryType": "randomWalk",
  7156                "refId": "A"
  7157              },
  7158              {
  7159                "exemplar": true,
  7160                "expr": "sum(rate(ticdc_sink_mq_worker_send_message_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_mq_worker_send_message_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)",
  7161                "hide": false,
  7162                "interval": "",
  7163                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-avg",
  7164                "refId": "B"
  7165              }
  7166            ],
  7167            "thresholds": [],
  7168            "timeFrom": null,
  7169            "timeRegions": [],
  7170            "timeShift": null,
  7171            "title": "Worker Send Message Duration Percentile",
  7172            "tooltip": {
  7173              "shared": true,
  7174              "sort": 0,
  7175              "value_type": "individual"
  7176            },
  7177            "type": "graph",
  7178            "xaxis": {
  7179              "buckets": null,
  7180              "mode": "time",
  7181              "name": null,
  7182              "show": true,
  7183              "values": []
  7184            },
  7185            "yaxes": [
  7186              {
  7187                "format": "s",
  7188                "label": null,
  7189                "logBase": 2,
  7190                "max": null,
  7191                "min": null,
  7192                "show": true
  7193              },
  7194              {
  7195                "format": "short",
  7196                "label": null,
  7197                "logBase": 1,
  7198                "max": null,
  7199                "min": null,
  7200                "show": false
  7201              }
  7202            ],
  7203            "yaxis": {
  7204              "align": false,
  7205              "alignLevel": null
  7206            }
  7207          },
  7208          {
  7209            "aliasColors": {},
  7210            "bars": false,
  7211            "dashLength": 10,
  7212            "dashes": false,
  7213            "datasource": "${DS_TEST-CLUSTER}",
  7214            "description": "Bytes/second written off all brokers.\nvalue = one-minute moving average rate of Bytes per second",
  7215            "fieldConfig": {
  7216              "defaults": {
  7217                "links": []
  7218              },
  7219              "overrides": []
  7220            },
  7221            "fill": 1,
  7222            "fillGradient": 0,
  7223            "gridPos": {
  7224              "h": 7,
  7225              "w": 12,
  7226              "x": 12,
  7227              "y": 7
  7228            },
  7229            "hiddenSeries": false,
  7230            "id": 628,
  7231            "legend": {
  7232              "alignAsTable": true,
  7233              "avg": true,
  7234              "current": true,
  7235              "max": true,
  7236              "min": false,
  7237              "rightSide": false,
  7238              "show": true,
  7239              "total": false,
  7240              "values": true
  7241            },
  7242            "lines": true,
  7243            "linewidth": 1,
  7244            "links": [],
  7245            "nullPointMode": "null",
  7246            "options": {
  7247              "alertThreshold": true
  7248            },
  7249            "paceLength": 10,
  7250            "percentage": false,
  7251            "pluginVersion": "7.5.11",
  7252            "pointradius": 2,
  7253            "points": false,
  7254            "renderer": "flot",
  7255            "seriesOverrides": [],
  7256            "spaceLength": 10,
  7257            "stack": false,
  7258            "steppedLine": false,
  7259            "targets": [
  7260              {
  7261                "exemplar": true,
  7262                "expr": "sum(ticdc_sink_kafka_producer_outgoing_byte_rate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namesapce=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)",
  7263                "format": "time_series",
  7264                "interval": "",
  7265                "intervalFactor": 1,
  7266                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}",
  7267                "refId": "A"
  7268              }
  7269            ],
  7270            "thresholds": [],
  7271            "timeFrom": null,
  7272            "timeRegions": [],
  7273            "timeShift": null,
  7274            "title": "Kafka Outgoing Bytes",
  7275            "tooltip": {
  7276              "shared": true,
  7277              "sort": 0,
  7278              "value_type": "individual"
  7279            },
  7280            "type": "graph",
  7281            "xaxis": {
  7282              "buckets": null,
  7283              "mode": "time",
  7284              "name": null,
  7285              "show": true,
  7286              "values": []
  7287            },
  7288            "yaxes": [
  7289              {
  7290                "format": "bytes",
  7291                "label": null,
  7292                "logBase": 1,
  7293                "max": null,
  7294                "min": "0",
  7295                "show": true
  7296              },
  7297              {
  7298                "format": "short",
  7299                "label": null,
  7300                "logBase": 1,
  7301                "max": null,
  7302                "min": null,
  7303                "show": false
  7304              }
  7305            ],
  7306            "yaxis": {
  7307              "align": false,
  7308              "alignLevel": null
  7309            }
  7310          },
  7311          {
  7312            "aliasColors": {},
  7313            "bars": false,
  7314            "dashLength": 10,
  7315            "dashes": false,
  7316            "datasource": "${DS_TEST-CLUSTER}",
  7317            "description": "The current number of in-flight requests awaiting a response for all brokers.",
  7318            "fieldConfig": {
  7319              "defaults": {
  7320                "links": []
  7321              },
  7322              "overrides": []
  7323            },
  7324            "fill": 1,
  7325            "fillGradient": 0,
  7326            "gridPos": {
  7327              "h": 7,
  7328              "w": 12,
  7329              "x": 0,
  7330              "y": 14
  7331            },
  7332            "hiddenSeries": false,
  7333            "id": 627,
  7334            "legend": {
  7335              "alignAsTable": true,
  7336              "avg": true,
  7337              "current": true,
  7338              "max": true,
  7339              "min": false,
  7340              "rightSide": false,
  7341              "show": true,
  7342              "total": false,
  7343              "values": true
  7344            },
  7345            "lines": true,
  7346            "linewidth": 1,
  7347            "links": [],
  7348            "nullPointMode": "null",
  7349            "options": {
  7350              "alertThreshold": true
  7351            },
  7352            "paceLength": 10,
  7353            "percentage": false,
  7354            "pluginVersion": "7.5.11",
  7355            "pointradius": 2,
  7356            "points": false,
  7357            "renderer": "flot",
  7358            "seriesOverrides": [],
  7359            "spaceLength": 10,
  7360            "stack": false,
  7361            "steppedLine": false,
  7362            "targets": [
  7363              {
  7364                "exemplar": true,
  7365                "expr": "sum(ticdc_sink_kafka_producer_in_flight_requests{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)",
  7366                "format": "time_series",
  7367                "interval": "",
  7368                "intervalFactor": 1,
  7369                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}",
  7370                "refId": "A"
  7371              }
  7372            ],
  7373            "thresholds": [],
  7374            "timeFrom": null,
  7375            "timeRegions": [],
  7376            "timeShift": null,
  7377            "title": "Kafka Inflight Requests",
  7378            "tooltip": {
  7379              "shared": true,
  7380              "sort": 0,
  7381              "value_type": "individual"
  7382            },
  7383            "type": "graph",
  7384            "xaxis": {
  7385              "buckets": null,
  7386              "mode": "time",
  7387              "name": null,
  7388              "show": true,
  7389              "values": []
  7390            },
  7391            "yaxes": [
  7392              {
  7393                "format": "none",
  7394                "label": null,
  7395                "logBase": 1,
  7396                "max": null,
  7397                "min": "0",
  7398                "show": true
  7399              },
  7400              {
  7401                "format": "short",
  7402                "label": null,
  7403                "logBase": 1,
  7404                "max": null,
  7405                "min": null,
  7406                "show": false
  7407              }
  7408            ],
  7409            "yaxis": {
  7410              "align": false,
  7411              "alignLevel": null
  7412            }
  7413          },
  7414          {
  7415            "aliasColors": {},
  7416            "bars": false,
  7417            "dashLength": 10,
  7418            "dashes": false,
  7419            "datasource": "${DS_TEST-CLUSTER}",
  7420            "description": "The request latency in ms for all brokers.\n\nvalue = request latency histogram's mean",
  7421            "fieldConfig": {
  7422              "defaults": {
  7423                "links": []
  7424              },
  7425              "overrides": []
  7426            },
  7427            "fill": 1,
  7428            "fillGradient": 0,
  7429            "gridPos": {
  7430              "h": 7,
  7431              "w": 12,
  7432              "x": 12,
  7433              "y": 14
  7434            },
  7435            "hiddenSeries": false,
  7436            "id": 630,
  7437            "legend": {
  7438              "alignAsTable": true,
  7439              "avg": true,
  7440              "current": true,
  7441              "max": true,
  7442              "min": false,
  7443              "rightSide": false,
  7444              "show": true,
  7445              "total": false,
  7446              "values": true
  7447            },
  7448            "lines": true,
  7449            "linewidth": 1,
  7450            "links": [],
  7451            "nullPointMode": "null",
  7452            "options": {
  7453              "alertThreshold": true
  7454            },
  7455            "paceLength": 10,
  7456            "percentage": false,
  7457            "pluginVersion": "7.5.11",
  7458            "pointradius": 2,
  7459            "points": false,
  7460            "renderer": "flot",
  7461            "seriesOverrides": [],
  7462            "spaceLength": 10,
  7463            "stack": false,
  7464            "steppedLine": false,
  7465            "targets": [
  7466              {
  7467                "exemplar": true,
  7468                "expr": "sum(ticdc_sink_kafka_producer_request_latency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)",
  7469                "format": "time_series",
  7470                "interval": "",
  7471                "intervalFactor": 1,
  7472                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}",
  7473                "refId": "A"
  7474              }
  7475            ],
  7476            "thresholds": [],
  7477            "timeFrom": null,
  7478            "timeRegions": [],
  7479            "timeShift": null,
  7480            "title": "Kafka Request Latency",
  7481            "tooltip": {
  7482              "shared": true,
  7483              "sort": 0,
  7484              "value_type": "individual"
  7485            },
  7486            "type": "graph",
  7487            "xaxis": {
  7488              "buckets": null,
  7489              "mode": "time",
  7490              "name": null,
  7491              "show": true,
  7492              "values": []
  7493            },
  7494            "yaxes": [
  7495              {
  7496                "format": "s",
  7497                "label": null,
  7498                "logBase": 1,
  7499                "max": null,
  7500                "min": "0",
  7501                "show": true
  7502              },
  7503              {
  7504                "format": "short",
  7505                "label": null,
  7506                "logBase": 1,
  7507                "max": null,
  7508                "min": null,
  7509                "show": false
  7510              }
  7511            ],
  7512            "yaxis": {
  7513              "align": false,
  7514              "alignLevel": null
  7515            }
  7516          },
  7517          {
  7518            "aliasColors": {},
  7519            "bars": false,
  7520            "dashLength": 10,
  7521            "dashes": false,
  7522            "datasource": "${DS_TEST-CLUSTER}",
  7523            "description": "Requests/second sent to all brokers.\nvalue = one-minute moving average rate of events per second",
  7524            "fieldConfig": {
  7525              "defaults": {
  7526                "links": []
  7527              },
  7528              "overrides": []
  7529            },
  7530            "fill": 1,
  7531            "fillGradient": 0,
  7532            "gridPos": {
  7533              "h": 7,
  7534              "w": 12,
  7535              "x": 0,
  7536              "y": 21
  7537            },
  7538            "hiddenSeries": false,
  7539            "id": 629,
  7540            "legend": {
  7541              "alignAsTable": true,
  7542              "avg": true,
  7543              "current": true,
  7544              "max": true,
  7545              "min": false,
  7546              "rightSide": false,
  7547              "show": true,
  7548              "total": false,
  7549              "values": true
  7550            },
  7551            "lines": true,
  7552            "linewidth": 1,
  7553            "links": [],
  7554            "nullPointMode": "null",
  7555            "options": {
  7556              "alertThreshold": true
  7557            },
  7558            "paceLength": 10,
  7559            "percentage": false,
  7560            "pluginVersion": "7.5.11",
  7561            "pointradius": 2,
  7562            "points": false,
  7563            "renderer": "flot",
  7564            "seriesOverrides": [],
  7565            "spaceLength": 10,
  7566            "stack": false,
  7567            "steppedLine": false,
  7568            "targets": [
  7569              {
  7570                "exemplar": true,
  7571                "expr": "sum(ticdc_sink_kafka_producer_request_rate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)",
  7572                "format": "time_series",
  7573                "interval": "",
  7574                "intervalFactor": 1,
  7575                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}",
  7576                "refId": "A"
  7577              }
  7578            ],
  7579            "thresholds": [],
  7580            "timeFrom": null,
  7581            "timeRegions": [],
  7582            "timeShift": null,
  7583            "title": "Kafka Request Rate",
  7584            "tooltip": {
  7585              "shared": true,
  7586              "sort": 0,
  7587              "value_type": "individual"
  7588            },
  7589            "type": "graph",
  7590            "xaxis": {
  7591              "buckets": null,
  7592              "mode": "time",
  7593              "name": null,
  7594              "show": true,
  7595              "values": []
  7596            },
  7597            "yaxes": [
  7598              {
  7599                "format": "none",
  7600                "label": null,
  7601                "logBase": 1,
  7602                "max": null,
  7603                "min": "0",
  7604                "show": true
  7605              },
  7606              {
  7607                "format": "short",
  7608                "label": null,
  7609                "logBase": 1,
  7610                "max": null,
  7611                "min": null,
  7612                "show": false
  7613              }
  7614            ],
  7615            "yaxis": {
  7616              "align": false,
  7617              "alignLevel": null
  7618            }
  7619          },
  7620          {
  7621            "aliasColors": {},
  7622            "bars": false,
  7623            "dashLength": 10,
  7624            "dashes": false,
  7625            "datasource": "${DS_TEST-CLUSTER}",
  7626            "description": "Records count per request send to the kafka\nvalue = one-minute moving average of response receive rate",
  7627            "fieldConfig": {
  7628              "defaults": {
  7629                "links": []
  7630              },
  7631              "overrides": []
  7632            },
  7633            "fill": 1,
  7634            "fillGradient": 0,
  7635            "gridPos": {
  7636              "h": 7,
  7637              "w": 12,
  7638              "x": 12,
  7639              "y": 21
  7640            },
  7641            "hiddenSeries": false,
  7642            "id": 631,
  7643            "legend": {
  7644              "alignAsTable": true,
  7645              "avg": true,
  7646              "current": true,
  7647              "max": true,
  7648              "min": false,
  7649              "rightSide": false,
  7650              "show": true,
  7651              "total": false,
  7652              "values": true
  7653            },
  7654            "lines": true,
  7655            "linewidth": 1,
  7656            "links": [],
  7657            "nullPointMode": "null",
  7658            "options": {
  7659              "alertThreshold": true
  7660            },
  7661            "paceLength": 10,
  7662            "percentage": false,
  7663            "pluginVersion": "7.5.11",
  7664            "pointradius": 2,
  7665            "points": false,
  7666            "renderer": "flot",
  7667            "seriesOverrides": [],
  7668            "spaceLength": 10,
  7669            "stack": false,
  7670            "steppedLine": false,
  7671            "targets": [
  7672              {
  7673                "exemplar": true,
  7674                "expr": "sum(ticdc_sink_kafka_producer_records_per_request{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)",
  7675                "format": "time_series",
  7676                "interval": "",
  7677                "intervalFactor": 1,
  7678                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}",
  7679                "refId": "A"
  7680              }
  7681            ],
  7682            "thresholds": [],
  7683            "timeFrom": null,
  7684            "timeRegions": [],
  7685            "timeShift": null,
  7686            "title": "Kafka Records Per Request",
  7687            "tooltip": {
  7688              "shared": true,
  7689              "sort": 0,
  7690              "value_type": "individual"
  7691            },
  7692            "type": "graph",
  7693            "xaxis": {
  7694              "buckets": null,
  7695              "mode": "time",
  7696              "name": null,
  7697              "show": true,
  7698              "values": []
  7699            },
  7700            "yaxes": [
  7701              {
  7702                "format": "none",
  7703                "label": null,
  7704                "logBase": 1,
  7705                "max": null,
  7706                "min": "0",
  7707                "show": true
  7708              },
  7709              {
  7710                "format": "short",
  7711                "label": null,
  7712                "logBase": 1,
  7713                "max": null,
  7714                "min": null,
  7715                "show": false
  7716              }
  7717            ],
  7718            "yaxis": {
  7719              "align": false,
  7720              "alignLevel": null
  7721            }
  7722          },
  7723          {
  7724            "aliasColors": {},
  7725            "bars": false,
  7726            "dashLength": 10,
  7727            "dashes": false,
  7728            "datasource": "${DS_TEST-CLUSTER}",
  7729            "description": "The compression ratio times 100 of record batches for all topics.",
  7730            "fieldConfig": {
  7731              "defaults": {},
  7732              "overrides": []
  7733            },
  7734            "fill": 1,
  7735            "fillGradient": 0,
  7736            "gridPos": {
  7737              "h": 7,
  7738              "w": 12,
  7739              "x": 0,
  7740              "y": 28
  7741            },
  7742            "hiddenSeries": false,
  7743            "id": 626,
  7744            "legend": {
  7745              "alignAsTable": true,
  7746              "avg": true,
  7747              "current": true,
  7748              "max": true,
  7749              "min": false,
  7750              "rightSide": false,
  7751              "show": true,
  7752              "total": false,
  7753              "values": true
  7754            },
  7755            "lines": true,
  7756            "linewidth": 1,
  7757            "links": [],
  7758            "nullPointMode": "null",
  7759            "options": {
  7760              "alertThreshold": true
  7761            },
  7762            "paceLength": 10,
  7763            "percentage": false,
  7764            "pluginVersion": "7.5.11",
  7765            "pointradius": 2,
  7766            "points": false,
  7767            "renderer": "flot",
  7768            "seriesOverrides": [],
  7769            "spaceLength": 10,
  7770            "stack": false,
  7771            "steppedLine": false,
  7772            "targets": [
  7773              {
  7774                "exemplar": true,
  7775                "expr": "sum(ticdc_sink_kafka_producer_compression_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)",
  7776                "format": "time_series",
  7777                "interval": "",
  7778                "intervalFactor": 1,
  7779                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}",
  7780                "refId": "A"
  7781              },
  7782              {
  7783                "exemplar": true,
  7784                "expr": "histogram_quantile(0.99, sum(rate(ticdc_kafka_sink_codec_compression_ratio_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed,instance))",
  7785                "hide": false,
  7786                "interval": "",
  7787                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-Codec-P99",
  7788                "refId": "B"
  7789              }
  7790            ],
  7791            "thresholds": [],
  7792            "timeFrom": null,
  7793            "timeRegions": [],
  7794            "timeShift": null,
  7795            "title": "Kafka Producer Compression Ratio",
  7796            "tooltip": {
  7797              "shared": true,
  7798              "sort": 0,
  7799              "value_type": "individual"
  7800            },
  7801            "type": "graph",
  7802            "xaxis": {
  7803              "buckets": null,
  7804              "mode": "time",
  7805              "name": null,
  7806              "show": true,
  7807              "values": []
  7808            },
  7809            "yaxes": [
  7810              {
  7811                "format": "none",
  7812                "label": null,
  7813                "logBase": 1,
  7814                "max": null,
  7815                "min": "0",
  7816                "show": true
  7817              },
  7818              {
  7819                "format": "short",
  7820                "label": null,
  7821                "logBase": 1,
  7822                "max": null,
  7823                "min": null,
  7824                "show": false
  7825              }
  7826            ],
  7827            "yaxis": {
  7828              "align": false,
  7829              "alignLevel": null
  7830            }
  7831          },
  7832          {
  7833            "aliasColors": {},
  7834            "bars": false,
  7835            "dashLength": 10,
  7836            "dashes": false,
  7837            "datasource": "${DS_TEST-CLUSTER}",
  7838            "description": "",
  7839            "fieldConfig": {
  7840              "defaults": {},
  7841              "overrides": []
  7842            },
  7843            "fill": 1,
  7844            "fillGradient": 0,
  7845            "gridPos": {
  7846              "h": 7,
  7847              "w": 12,
  7848              "x": 12,
  7849              "y": 28
  7850            },
  7851            "hiddenSeries": false,
  7852            "id": 709,
  7853            "legend": {
  7854              "alignAsTable": true,
  7855              "avg": true,
  7856              "current": true,
  7857              "max": true,
  7858              "min": false,
  7859              "rightSide": false,
  7860              "show": true,
  7861              "total": false,
  7862              "values": true
  7863            },
  7864            "lines": true,
  7865            "linewidth": 1,
  7866            "nullPointMode": "null",
  7867            "options": {
  7868              "alertThreshold": true
  7869            },
  7870            "percentage": false,
  7871            "pluginVersion": "7.5.11",
  7872            "pointradius": 2,
  7873            "points": false,
  7874            "renderer": "flot",
  7875            "seriesOverrides": [],
  7876            "spaceLength": 10,
  7877            "stack": false,
  7878            "steppedLine": false,
  7879            "targets": [
  7880              {
  7881                "exemplar": true,
  7882                "expr": "ticdc_sink_encoder_group_input_chan_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}",
  7883                "hide": false,
  7884                "interval": "",
  7885                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{index}}",
  7886                "refId": "A"
  7887              }
  7888            ],
  7889            "thresholds": [],
  7890            "timeFrom": null,
  7891            "timeRegions": [],
  7892            "timeShift": null,
  7893            "title": "encoder group input channel size",
  7894            "tooltip": {
  7895              "shared": true,
  7896              "sort": 0,
  7897              "value_type": "individual"
  7898            },
  7899            "type": "graph",
  7900            "xaxis": {
  7901              "buckets": null,
  7902              "mode": "time",
  7903              "name": null,
  7904              "show": true,
  7905              "values": []
  7906            },
  7907            "yaxes": [
  7908              {
  7909                "format": "none",
  7910                "label": null,
  7911                "logBase": 1,
  7912                "max": null,
  7913                "min": null,
  7914                "show": true
  7915              },
  7916              {
  7917                "format": "none",
  7918                "label": null,
  7919                "logBase": 1,
  7920                "max": null,
  7921                "min": null,
  7922                "show": false
  7923              }
  7924            ],
  7925            "yaxis": {
  7926              "align": false,
  7927              "alignLevel": null
  7928            }
  7929          },
  7930          {
  7931            "aliasColors": {},
  7932            "bars": false,
  7933            "dashLength": 10,
  7934            "dashes": false,
  7935            "datasource": "${DS_TEST-CLUSTER}",
  7936            "description": "",
  7937            "fieldConfig": {
  7938              "defaults": {},
  7939              "overrides": []
  7940            },
  7941            "fill": 1,
  7942            "fillGradient": 0,
  7943            "gridPos": {
  7944              "h": 7,
  7945              "w": 12,
  7946              "x": 0,
  7947              "y": 35
  7948            },
  7949            "hiddenSeries": false,
  7950            "id": 710,
  7951            "legend": {
  7952              "alignAsTable": true,
  7953              "avg": true,
  7954              "current": true,
  7955              "max": true,
  7956              "min": false,
  7957              "rightSide": false,
  7958              "show": true,
  7959              "total": false,
  7960              "values": true
  7961            },
  7962            "lines": true,
  7963            "linewidth": 1,
  7964            "nullPointMode": "null",
  7965            "options": {
  7966              "alertThreshold": true
  7967            },
  7968            "percentage": false,
  7969            "pluginVersion": "7.5.11",
  7970            "pointradius": 2,
  7971            "points": false,
  7972            "renderer": "flot",
  7973            "seriesOverrides": [],
  7974            "spaceLength": 10,
  7975            "stack": false,
  7976            "steppedLine": false,
  7977            "targets": [
  7978              {
  7979                "exemplar": true,
  7980                "expr": "ticdc_sink_encoder_group_output_chan_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}",
  7981                "hide": false,
  7982                "interval": "",
  7983                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}",
  7984                "refId": "A"
  7985              }
  7986            ],
  7987            "thresholds": [],
  7988            "timeFrom": null,
  7989            "timeRegions": [],
  7990            "timeShift": null,
  7991            "title": "encoder group output channel size",
  7992            "tooltip": {
  7993              "shared": true,
  7994              "sort": 0,
  7995              "value_type": "individual"
  7996            },
  7997            "type": "graph",
  7998            "xaxis": {
  7999              "buckets": null,
  8000              "mode": "time",
  8001              "name": null,
  8002              "show": true,
  8003              "values": []
  8004            },
  8005            "yaxes": [
  8006              {
  8007                "format": "none",
  8008                "label": null,
  8009                "logBase": 1,
  8010                "max": null,
  8011                "min": null,
  8012                "show": true
  8013              },
  8014              {
  8015                "format": "none",
  8016                "label": null,
  8017                "logBase": 1,
  8018                "max": null,
  8019                "min": null,
  8020                "show": false
  8021              }
  8022            ],
  8023            "yaxis": {
  8024              "align": false,
  8025              "alignLevel": null
  8026            }
  8027          },
  8028          {
  8029            "aliasColors": {},
  8030            "bars": false,
  8031            "dashLength": 10,
  8032            "dashes": false,
  8033            "datasource": "${DS_TEST-CLUSTER}",
  8034            "description": "MQ worker batch multiple messages into one when using batched encode protocol, this metric record the time cost on batch messages.",
  8035            "fieldConfig": {
  8036              "defaults": {},
  8037              "overrides": []
  8038            },
  8039            "fill": 1,
  8040            "fillGradient": 0,
  8041            "gridPos": {
  8042              "h": 7,
  8043              "w": 12,
  8044              "x": 12,
  8045              "y": 35
  8046            },
  8047            "hiddenSeries": false,
  8048            "id": 711,
  8049            "legend": {
  8050              "alignAsTable": true,
  8051              "avg": true,
  8052              "current": true,
  8053              "max": true,
  8054              "min": false,
  8055              "rightSide": false,
  8056              "show": true,
  8057              "total": false,
  8058              "values": true
  8059            },
  8060            "lines": true,
  8061            "linewidth": 1,
  8062            "nullPointMode": "null",
  8063            "options": {
  8064              "alertThreshold": true
  8065            },
  8066            "percentage": false,
  8067            "pluginVersion": "7.5.11",
  8068            "pointradius": 2,
  8069            "points": false,
  8070            "renderer": "flot",
  8071            "seriesOverrides": [],
  8072            "spaceLength": 10,
  8073            "stack": false,
  8074            "steppedLine": false,
  8075            "targets": [
  8076              {
  8077                "exemplar": true,
  8078                "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_worker_batch_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed,instance))",
  8079                "interval": "",
  8080                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-P999",
  8081                "queryType": "randomWalk",
  8082                "refId": "A"
  8083              },
  8084              {
  8085                "exemplar": true,
  8086                "expr": "sum(rate(ticdc_sink_mq_worker_batch_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_mq_worker_batch_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)",
  8087                "hide": false,
  8088                "interval": "",
  8089                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-avg",
  8090                "refId": "B"
  8091              }
  8092            ],
  8093            "thresholds": [],
  8094            "timeFrom": null,
  8095            "timeRegions": [],
  8096            "timeShift": null,
  8097            "title": "Worker Batch Duration Percentile",
  8098            "tooltip": {
  8099              "shared": true,
  8100              "sort": 0,
  8101              "value_type": "individual"
  8102            },
  8103            "type": "graph",
  8104            "xaxis": {
  8105              "buckets": null,
  8106              "mode": "time",
  8107              "name": null,
  8108              "show": true,
  8109              "values": []
  8110            },
  8111            "yaxes": [
  8112              {
  8113                "format": "s",
  8114                "label": null,
  8115                "logBase": 2,
  8116                "max": null,
  8117                "min": null,
  8118                "show": true
  8119              },
  8120              {
  8121                "format": "short",
  8122                "label": null,
  8123                "logBase": 1,
  8124                "max": null,
  8125                "min": null,
  8126                "show": false
  8127              }
  8128            ],
  8129            "yaxis": {
  8130              "align": false,
  8131              "alignLevel": null
  8132            }
  8133          },
  8134          {
  8135            "aliasColors": {},
  8136            "bars": false,
  8137            "dashLength": 10,
  8138            "dashes": false,
  8139            "datasource": "${DS_TEST-CLUSTER}",
  8140            "description": "MQ worker batch multiple messages into one when using batched encode protocol, this metrics track each batch's size",
  8141            "fieldConfig": {
  8142              "defaults": {},
  8143              "overrides": []
  8144            },
  8145            "fill": 1,
  8146            "fillGradient": 0,
  8147            "gridPos": {
  8148              "h": 7,
  8149              "w": 12,
  8150              "x": 0,
  8151              "y": 42
  8152            },
  8153            "hiddenSeries": false,
  8154            "id": 712,
  8155            "legend": {
  8156              "alignAsTable": true,
  8157              "avg": true,
  8158              "current": true,
  8159              "max": true,
  8160              "min": false,
  8161              "rightSide": false,
  8162              "show": true,
  8163              "total": false,
  8164              "values": true
  8165            },
  8166            "lines": true,
  8167            "linewidth": 1,
  8168            "nullPointMode": "null",
  8169            "options": {
  8170              "alertThreshold": true
  8171            },
  8172            "percentage": false,
  8173            "pluginVersion": "7.5.11",
  8174            "pointradius": 2,
  8175            "points": false,
  8176            "renderer": "flot",
  8177            "seriesOverrides": [],
  8178            "spaceLength": 10,
  8179            "stack": false,
  8180            "steppedLine": false,
  8181            "targets": [
  8182              {
  8183                "exemplar": true,
  8184                "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_worker_batch_size_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed,instance))",
  8185                "hide": false,
  8186                "interval": "",
  8187                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-P999",
  8188                "queryType": "randomWalk",
  8189                "refId": "A"
  8190              },
  8191              {
  8192                "exemplar": true,
  8193                "expr": "sum(rate(ticdc_sink_mq_worker_batch_size_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance) /\nsum(rate(ticdc_sink_mq_worker_batch_size_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)",
  8194                "hide": false,
  8195                "interval": "",
  8196                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-avg",
  8197                "refId": "B"
  8198              }
  8199            ],
  8200            "thresholds": [],
  8201            "timeFrom": null,
  8202            "timeRegions": [],
  8203            "timeShift": null,
  8204            "title": "Worker Batch Size Percentile",
  8205            "tooltip": {
  8206              "shared": true,
  8207              "sort": 0,
  8208              "value_type": "individual"
  8209            },
  8210            "type": "graph",
  8211            "xaxis": {
  8212              "buckets": null,
  8213              "mode": "time",
  8214              "name": null,
  8215              "show": true,
  8216              "values": []
  8217            },
  8218            "yaxes": [
  8219              {
  8220                "format": "none",
  8221                "label": null,
  8222                "logBase": 1,
  8223                "max": null,
  8224                "min": null,
  8225                "show": true
  8226              },
  8227              {
  8228                "format": "short",
  8229                "label": null,
  8230                "logBase": 1,
  8231                "max": null,
  8232                "min": null,
  8233                "show": false
  8234              }
  8235            ],
  8236            "yaxis": {
  8237              "align": false,
  8238              "alignLevel": null
  8239            }
  8240          },
  8241          {
  8242            "aliasColors": {},
  8243            "bars": false,
  8244            "dashLength": 10,
  8245            "dashes": false,
  8246            "datasource": "${DS_TEST-CLUSTER}",
  8247            "description": "Kafka-Go Writer batch messages internally before send to the Kafka cluster.",
  8248            "fieldConfig": {
  8249              "defaults": {},
  8250              "overrides": []
  8251            },
  8252            "fill": 1,
  8253            "fillGradient": 0,
  8254            "gridPos": {
  8255              "h": 7,
  8256              "w": 12,
  8257              "x": 12,
  8258              "y": 42
  8259            },
  8260            "hiddenSeries": false,
  8261            "id": 731,
  8262            "legend": {
  8263              "alignAsTable": true,
  8264              "avg": true,
  8265              "current": true,
  8266              "max": true,
  8267              "min": false,
  8268              "rightSide": false,
  8269              "show": true,
  8270              "total": false,
  8271              "values": true
  8272            },
  8273            "lines": true,
  8274            "linewidth": 1,
  8275            "nullPointMode": "null",
  8276            "options": {
  8277              "alertThreshold": true
  8278            },
  8279            "percentage": false,
  8280            "pluginVersion": "7.5.11",
  8281            "pointradius": 2,
  8282            "points": false,
  8283            "renderer": "flot",
  8284            "seriesOverrides": [],
  8285            "spaceLength": 10,
  8286            "stack": false,
  8287            "steppedLine": false,
  8288            "targets": [
  8289              {
  8290                "exemplar": true,
  8291                "expr": "max(ticdc_sink_kafka_producer_batch_message_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)",
  8292                "hide": false,
  8293                "interval": "",
  8294                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}",
  8295                "refId": "A"
  8296              }
  8297            ],
  8298            "thresholds": [],
  8299            "timeFrom": null,
  8300            "timeRegions": [],
  8301            "timeShift": null,
  8302            "title": "Producer Batch Message Count Percentile (Kafka-Go Only)",
  8303            "tooltip": {
  8304              "shared": true,
  8305              "sort": 0,
  8306              "value_type": "individual"
  8307            },
  8308            "type": "graph",
  8309            "xaxis": {
  8310              "buckets": null,
  8311              "mode": "time",
  8312              "name": null,
  8313              "show": true,
  8314              "values": []
  8315            },
  8316            "yaxes": [
  8317              {
  8318                "format": "none",
  8319                "label": null,
  8320                "logBase": 1,
  8321                "max": null,
  8322                "min": null,
  8323                "show": true
  8324              },
  8325              {
  8326                "format": "short",
  8327                "label": null,
  8328                "logBase": 1,
  8329                "max": null,
  8330                "min": null,
  8331                "show": false
  8332              }
  8333            ],
  8334            "yaxis": {
  8335              "align": false,
  8336              "alignLevel": null
  8337            }
  8338          },
  8339          {
  8340            "aliasColors": {},
  8341            "bars": false,
  8342            "dashLength": 10,
  8343            "dashes": false,
  8344            "datasource": "${DS_TEST-CLUSTER}",
  8345            "description": "MQ worker batch multiple messages into one when using batched encode protocol, this metric record the time cost on batch messages.",
  8346            "fieldConfig": {
  8347              "defaults": {},
  8348              "overrides": []
  8349            },
  8350            "fill": 1,
  8351            "fillGradient": 0,
  8352            "gridPos": {
  8353              "h": 7,
  8354              "w": 12,
  8355              "x": 0,
  8356              "y": 49
  8357            },
  8358            "hiddenSeries": false,
  8359            "id": 733,
  8360            "legend": {
  8361              "alignAsTable": true,
  8362              "avg": true,
  8363              "current": true,
  8364              "max": true,
  8365              "min": false,
  8366              "rightSide": false,
  8367              "show": true,
  8368              "total": false,
  8369              "values": true
  8370            },
  8371            "lines": true,
  8372            "linewidth": 1,
  8373            "nullPointMode": "null",
  8374            "options": {
  8375              "alertThreshold": true
  8376            },
  8377            "percentage": false,
  8378            "pluginVersion": "7.5.11",
  8379            "pointradius": 2,
  8380            "points": false,
  8381            "renderer": "flot",
  8382            "seriesOverrides": [],
  8383            "spaceLength": 10,
  8384            "stack": false,
  8385            "steppedLine": false,
  8386            "targets": [
  8387              {
  8388                "exemplar": true,
  8389                "expr": "max(ticdc_sink_kafka_producer_batch_duration{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)",
  8390                "hide": false,
  8391                "interval": "",
  8392                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}",
  8393                "refId": "A"
  8394              }
  8395            ],
  8396            "thresholds": [],
  8397            "timeFrom": null,
  8398            "timeRegions": [],
  8399            "timeShift": null,
  8400            "title": "Producer Batch Messages Duration Percentile (Kafka-Go Only)",
  8401            "tooltip": {
  8402              "shared": true,
  8403              "sort": 0,
  8404              "value_type": "individual"
  8405            },
  8406            "type": "graph",
  8407            "xaxis": {
  8408              "buckets": null,
  8409              "mode": "time",
  8410              "name": null,
  8411              "show": true,
  8412              "values": []
  8413            },
  8414            "yaxes": [
  8415              {
  8416                "format": "s",
  8417                "label": null,
  8418                "logBase": 1,
  8419                "max": null,
  8420                "min": null,
  8421                "show": true
  8422              },
  8423              {
  8424                "format": "short",
  8425                "label": null,
  8426                "logBase": 1,
  8427                "max": null,
  8428                "min": null,
  8429                "show": false
  8430              }
  8431            ],
  8432            "yaxis": {
  8433              "align": false,
  8434              "alignLevel": null
  8435            }
  8436          },
  8437          {
  8438            "aliasColors": {},
  8439            "bars": false,
  8440            "dashLength": 10,
  8441            "dashes": false,
  8442            "datasource": "${DS_TEST-CLUSTER}",
  8443            "description": "Kafka-Go Writer batch messages internally before send to the Kafka cluster.",
  8444            "fieldConfig": {
  8445              "defaults": {},
  8446              "overrides": []
  8447            },
  8448            "fill": 1,
  8449            "fillGradient": 0,
  8450            "gridPos": {
  8451              "h": 7,
  8452              "w": 12,
  8453              "x": 12,
  8454              "y": 49
  8455            },
  8456            "hiddenSeries": false,
  8457            "id": 735,
  8458            "legend": {
  8459              "alignAsTable": true,
  8460              "avg": true,
  8461              "current": true,
  8462              "max": true,
  8463              "min": false,
  8464              "rightSide": false,
  8465              "show": true,
  8466              "total": false,
  8467              "values": true
  8468            },
  8469            "lines": true,
  8470            "linewidth": 1,
  8471            "nullPointMode": "null",
  8472            "options": {
  8473              "alertThreshold": true
  8474            },
  8475            "percentage": false,
  8476            "pluginVersion": "7.5.11",
  8477            "pointradius": 2,
  8478            "points": false,
  8479            "renderer": "flot",
  8480            "seriesOverrides": [],
  8481            "spaceLength": 10,
  8482            "stack": false,
  8483            "steppedLine": false,
  8484            "targets": [
  8485              {
  8486                "exemplar": true,
  8487                "expr": "max(ticdc_sink_kafka_producer_batch_size{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)",
  8488                "hide": false,
  8489                "interval": "",
  8490                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}",
  8491                "refId": "A"
  8492              }
  8493            ],
  8494            "thresholds": [],
  8495            "timeFrom": null,
  8496            "timeRegions": [],
  8497            "timeShift": null,
  8498            "title": "Producer Batch Bytes Percentile (Kafka-Go Only)",
  8499            "tooltip": {
  8500              "shared": true,
  8501              "sort": 0,
  8502              "value_type": "individual"
  8503            },
  8504            "type": "graph",
  8505            "xaxis": {
  8506              "buckets": null,
  8507              "mode": "time",
  8508              "name": null,
  8509              "show": true,
  8510              "values": []
  8511            },
  8512            "yaxes": [
  8513              {
  8514                "format": "bytes",
  8515                "label": null,
  8516                "logBase": 1,
  8517                "max": null,
  8518                "min": null,
  8519                "show": true
  8520              },
  8521              {
  8522                "format": "short",
  8523                "label": null,
  8524                "logBase": 1,
  8525                "max": null,
  8526                "min": null,
  8527                "show": false
  8528              }
  8529            ],
  8530            "yaxis": {
  8531              "align": false,
  8532              "alignLevel": null
  8533            }
  8534          },
  8535          {
  8536            "aliasColors": {},
  8537            "bars": false,
  8538            "dashLength": 10,
  8539            "dashes": false,
  8540            "datasource": "${DS_TEST-CLUSTER}",
  8541            "description": "Count of errors in the last minute.",
  8542            "fieldConfig": {
  8543              "defaults": {},
  8544              "overrides": []
  8545            },
  8546            "fill": 1,
  8547            "fillGradient": 0,
  8548            "gridPos": {
  8549              "h": 8,
  8550              "w": 12,
  8551              "x": 0,
  8552              "y": 56
  8553            },
  8554            "hiddenSeries": false,
  8555            "id": 739,
  8556            "legend": {
  8557              "alignAsTable": true,
  8558              "avg": false,
  8559              "current": true,
  8560              "max": false,
  8561              "min": false,
  8562              "rightSide": false,
  8563              "show": true,
  8564              "total": false,
  8565              "values": true
  8566            },
  8567            "lines": true,
  8568            "linewidth": 1,
  8569            "nullPointMode": "null",
  8570            "options": {
  8571              "alertThreshold": true
  8572            },
  8573            "percentage": false,
  8574            "pluginVersion": "7.5.11",
  8575            "pointradius": 2,
  8576            "points": false,
  8577            "renderer": "flot",
  8578            "seriesOverrides": [],
  8579            "spaceLength": 10,
  8580            "stack": false,
  8581            "steppedLine": false,
  8582            "targets": [
  8583              {
  8584                "exemplar": true,
  8585                "expr": "sum(rate(ticdc_sink_kafka_producer_retry_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)",
  8586                "interval": "",
  8587                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}",
  8588                "queryType": "randomWalk",
  8589                "refId": "A"
  8590              }
  8591            ],
  8592            "thresholds": [],
  8593            "timeFrom": null,
  8594            "timeRegions": [],
  8595            "timeShift": null,
  8596            "title": "Retry Count (Kafka-Go Only)",
  8597            "tooltip": {
  8598              "shared": true,
  8599              "sort": 0,
  8600              "value_type": "individual"
  8601            },
  8602            "type": "graph",
  8603            "xaxis": {
  8604              "buckets": null,
  8605              "mode": "time",
  8606              "name": null,
  8607              "show": true,
  8608              "values": []
  8609            },
  8610            "yaxes": [
  8611              {
  8612                "format": "none",
  8613                "label": null,
  8614                "logBase": 1,
  8615                "max": null,
  8616                "min": null,
  8617                "show": true
  8618              },
  8619              {
  8620                "format": "short",
  8621                "label": null,
  8622                "logBase": 1,
  8623                "max": null,
  8624                "min": null,
  8625                "show": false
  8626              }
  8627            ],
  8628            "yaxis": {
  8629              "align": false,
  8630              "alignLevel": null
  8631            }
  8632          },
  8633          {
  8634            "aliasColors": {},
  8635            "bars": false,
  8636            "dashLength": 10,
  8637            "dashes": false,
  8638            "datasource": "${DS_TEST-CLUSTER}",
  8639            "description": "Count of errors in the last minute.",
  8640            "fieldConfig": {
  8641              "defaults": {},
  8642              "overrides": []
  8643            },
  8644            "fill": 1,
  8645            "fillGradient": 0,
  8646            "gridPos": {
  8647              "h": 8,
  8648              "w": 12,
  8649              "x": 12,
  8650              "y": 56
  8651            },
  8652            "hiddenSeries": false,
  8653            "id": 737,
  8654            "legend": {
  8655              "alignAsTable": true,
  8656              "avg": false,
  8657              "current": true,
  8658              "max": false,
  8659              "min": false,
  8660              "rightSide": false,
  8661              "show": true,
  8662              "total": false,
  8663              "values": true
  8664            },
  8665            "lines": true,
  8666            "linewidth": 1,
  8667            "nullPointMode": "null",
  8668            "options": {
  8669              "alertThreshold": true
  8670            },
  8671            "percentage": false,
  8672            "pluginVersion": "7.5.11",
  8673            "pointradius": 2,
  8674            "points": false,
  8675            "renderer": "flot",
  8676            "seriesOverrides": [],
  8677            "spaceLength": 10,
  8678            "stack": false,
  8679            "steppedLine": false,
  8680            "targets": [
  8681              {
  8682                "exemplar": true,
  8683                "expr": "sum(rate(ticdc_sink_kafka_producer_err_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)",
  8684                "hide": false,
  8685                "interval": "",
  8686                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}",
  8687                "refId": "B"
  8688              }
  8689            ],
  8690            "thresholds": [],
  8691            "timeFrom": null,
  8692            "timeRegions": [],
  8693            "timeShift": null,
  8694            "title": "Error Count (Kafka-Go Only)",
  8695            "tooltip": {
  8696              "shared": true,
  8697              "sort": 0,
  8698              "value_type": "individual"
  8699            },
  8700            "type": "graph",
  8701            "xaxis": {
  8702              "buckets": null,
  8703              "mode": "time",
  8704              "name": null,
  8705              "show": true,
  8706              "values": []
  8707            },
  8708            "yaxes": [
  8709              {
  8710                "format": "short",
  8711                "label": null,
  8712                "logBase": 1,
  8713                "max": null,
  8714                "min": null,
  8715                "show": true
  8716              },
  8717              {
  8718                "format": "short",
  8719                "label": null,
  8720                "logBase": 1,
  8721                "max": null,
  8722                "min": null,
  8723                "show": false
  8724              }
  8725            ],
  8726            "yaxis": {
  8727              "align": false,
  8728              "alignLevel": null
  8729            }
  8730          },
  8731          {
  8732            "aliasColors": {},
  8733            "bars": false,
  8734            "dashLength": 10,
  8735            "dashes": false,
  8736            "datasource": "${DS_TEST-CLUSTER}",
  8737            "description": "MQ worker send large message to the external storage, this metric record the time cost on send every message.",
  8738            "fieldConfig": {
  8739              "defaults": {},
  8740              "overrides": []
  8741            },
  8742            "fill": 1,
  8743            "fillGradient": 0,
  8744            "gridPos": {
  8745              "h": 7,
  8746              "w": 12,
  8747              "x": 0,
  8748              "y": 64
  8749            },
  8750            "hiddenSeries": false,
  8751            "id": 740,
  8752            "legend": {
  8753              "alignAsTable": true,
  8754              "avg": true,
  8755              "current": true,
  8756              "max": true,
  8757              "min": false,
  8758              "rightSide": false,
  8759              "show": true,
  8760              "total": false,
  8761              "values": true
  8762            },
  8763            "lines": true,
  8764            "linewidth": 1,
  8765            "nullPointMode": "null",
  8766            "options": {
  8767              "alertThreshold": true
  8768            },
  8769            "percentage": false,
  8770            "pluginVersion": "7.5.11",
  8771            "pointradius": 2,
  8772            "points": false,
  8773            "renderer": "flot",
  8774            "seriesOverrides": [],
  8775            "spaceLength": 10,
  8776            "stack": false,
  8777            "steppedLine": false,
  8778            "targets": [
  8779              {
  8780                "exemplar": true,
  8781                "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_claim_check_send_message_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed,instance))",
  8782                "interval": "",
  8783                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-P999",
  8784                "queryType": "randomWalk",
  8785                "refId": "A"
  8786              },
  8787              {
  8788                "exemplar": true,
  8789                "expr": "rate(ticdc_sink_mq_claim_check_send_message_duration_sum[30s]) / rate(ticdc_sink_mq_claim_check_send_message_duration_count[30s])",
  8790                "hide": false,
  8791                "interval": "",
  8792                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-avg",
  8793                "refId": "B"
  8794              }
  8795            ],
  8796            "thresholds": [],
  8797            "timeFrom": null,
  8798            "timeRegions": [],
  8799            "timeShift": null,
  8800            "title": "Claim Check Send Message Duration Percentile",
  8801            "tooltip": {
  8802              "shared": true,
  8803              "sort": 0,
  8804              "value_type": "individual"
  8805            },
  8806            "type": "graph",
  8807            "xaxis": {
  8808              "buckets": null,
  8809              "mode": "time",
  8810              "name": null,
  8811              "show": true,
  8812              "values": []
  8813            },
  8814            "yaxes": [
  8815              {
  8816                "format": "s",
  8817                "label": null,
  8818                "logBase": 2,
  8819                "max": null,
  8820                "min": null,
  8821                "show": true
  8822              },
  8823              {
  8824                "format": "short",
  8825                "label": null,
  8826                "logBase": 1,
  8827                "max": null,
  8828                "min": null,
  8829                "show": false
  8830              }
  8831            ],
  8832            "yaxis": {
  8833              "align": false,
  8834              "alignLevel": null
  8835            }
  8836          },
  8837          {
  8838            "aliasColors": {},
  8839            "bars": false,
  8840            "dashLength": 10,
  8841            "dashes": false,
  8842            "datasource": "${DS_TEST-CLUSTER}",
  8843            "description": "MQ worker send large message to the external storage, this metrics record the message count",
  8844            "fieldConfig": {
  8845              "defaults": {},
  8846              "overrides": []
  8847            },
  8848            "fill": 1,
  8849            "fillGradient": 0,
  8850            "gridPos": {
  8851              "h": 7,
  8852              "w": 12,
  8853              "x": 12,
  8854              "y": 64
  8855            },
  8856            "hiddenSeries": false,
  8857            "id": 741,
  8858            "legend": {
  8859              "alignAsTable": true,
  8860              "avg": true,
  8861              "current": true,
  8862              "max": true,
  8863              "min": false,
  8864              "rightSide": false,
  8865              "show": true,
  8866              "total": false,
  8867              "values": true
  8868            },
  8869            "lines": true,
  8870            "linewidth": 1,
  8871            "nullPointMode": "null",
  8872            "options": {
  8873              "alertThreshold": true
  8874            },
  8875            "percentage": false,
  8876            "pluginVersion": "7.5.11",
  8877            "pointradius": 2,
  8878            "points": false,
  8879            "renderer": "flot",
  8880            "seriesOverrides": [],
  8881            "spaceLength": 10,
  8882            "stack": false,
  8883            "steppedLine": false,
  8884            "targets": [
  8885              {
  8886                "exemplar": true,
  8887                "expr": "sum(rate(ticdc_sink_mq_claim_check_send_message_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)",
  8888                "hide": false,
  8889                "interval": "",
  8890                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}",
  8891                "refId": "B"
  8892              }
  8893            ],
  8894            "thresholds": [],
  8895            "timeFrom": null,
  8896            "timeRegions": [],
  8897            "timeShift": null,
  8898            "title": "Claim Check Send Message Count",
  8899            "tooltip": {
  8900              "shared": true,
  8901              "sort": 0,
  8902              "value_type": "individual"
  8903            },
  8904            "type": "graph",
  8905            "xaxis": {
  8906              "buckets": null,
  8907              "mode": "time",
  8908              "name": null,
  8909              "show": true,
  8910              "values": []
  8911            },
  8912            "yaxes": [
  8913              {
  8914                "format": "short",
  8915                "label": null,
  8916                "logBase": 1,
  8917                "max": null,
  8918                "min": null,
  8919                "show": true
  8920              },
  8921              {
  8922                "format": "short",
  8923                "label": null,
  8924                "logBase": 1,
  8925                "max": null,
  8926                "min": null,
  8927                "show": false
  8928              }
  8929            ],
  8930            "yaxis": {
  8931              "align": false,
  8932              "alignLevel": null
  8933            }
  8934          }
  8935        ],
  8936        "title": "Sink - MQ Sink",
  8937        "type": "row"
  8938      },
  8939      {
  8940        "collapsed": true,
  8941        "datasource": null,
  8942        "gridPos": {
  8943          "h": 1,
  8944          "w": 24,
  8945          "x": 0,
  8946          "y": 7
  8947        },
  8948        "id": 713,
  8949        "panels": [
  8950          {
  8951            "aliasColors": {},
  8952            "bars": false,
  8953            "dashLength": 10,
  8954            "dashes": false,
  8955            "datasource": "${DS_TEST-CLUSTER}",
  8956            "fieldConfig": {
  8957              "defaults": {},
  8958              "overrides": []
  8959            },
  8960            "fill": 1,
  8961            "fillGradient": 0,
  8962            "gridPos": {
  8963              "h": 8,
  8964              "w": 12,
  8965              "x": 0,
  8966              "y": 8
  8967            },
  8968            "hiddenSeries": false,
  8969            "id": 714,
  8970            "legend": {
  8971              "avg": false,
  8972              "current": false,
  8973              "max": false,
  8974              "min": false,
  8975              "show": true,
  8976              "total": false,
  8977              "values": false
  8978            },
  8979            "lines": true,
  8980            "linewidth": 1,
  8981            "nullPointMode": "null",
  8982            "options": {
  8983              "alertThreshold": true
  8984            },
  8985            "percentage": false,
  8986            "pluginVersion": "7.5.11",
  8987            "pointradius": 2,
  8988            "points": false,
  8989            "renderer": "flot",
  8990            "seriesOverrides": [],
  8991            "spaceLength": 10,
  8992            "stack": false,
  8993            "steppedLine": false,
  8994            "targets": [
  8995              {
  8996                "exemplar": true,
  8997                "expr": "sum(rate(ticdc_sink_cloud_storage_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace, changefeed, instance)",
  8998                "hide": false,
  8999                "interval": "",
  9000                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}",
  9001                "queryType": "randomWalk",
  9002                "refId": "A"
  9003              }
  9004            ],
  9005            "thresholds": [],
  9006            "timeFrom": null,
  9007            "timeRegions": [],
  9008            "timeShift": null,
  9009            "title": "Write Bytes/s",
  9010            "tooltip": {
  9011              "shared": true,
  9012              "sort": 0,
  9013              "value_type": "individual"
  9014            },
  9015            "type": "graph",
  9016            "xaxis": {
  9017              "buckets": null,
  9018              "mode": "time",
  9019              "name": null,
  9020              "show": true,
  9021              "values": []
  9022            },
  9023            "yaxes": [
  9024              {
  9025                "format": "short",
  9026                "label": null,
  9027                "logBase": 1,
  9028                "max": null,
  9029                "min": null,
  9030                "show": true
  9031              },
  9032              {
  9033                "format": "short",
  9034                "label": null,
  9035                "logBase": 1,
  9036                "max": null,
  9037                "min": null,
  9038                "show": true
  9039              }
  9040            ],
  9041            "yaxis": {
  9042              "align": false,
  9043              "alignLevel": null
  9044            }
  9045          },
  9046          {
  9047            "aliasColors": {},
  9048            "bars": false,
  9049            "dashLength": 10,
  9050            "dashes": false,
  9051            "datasource": "${DS_TEST-CLUSTER}",
  9052            "fieldConfig": {
  9053              "defaults": {},
  9054              "overrides": []
  9055            },
  9056            "fill": 1,
  9057            "fillGradient": 0,
  9058            "gridPos": {
  9059              "h": 8,
  9060              "w": 12,
  9061              "x": 12,
  9062              "y": 8
  9063            },
  9064            "hiddenSeries": false,
  9065            "id": 715,
  9066            "legend": {
  9067              "avg": false,
  9068              "current": false,
  9069              "max": false,
  9070              "min": false,
  9071              "show": true,
  9072              "total": false,
  9073              "values": false
  9074            },
  9075            "lines": true,
  9076            "linewidth": 1,
  9077            "nullPointMode": "null",
  9078            "options": {
  9079              "alertThreshold": true
  9080            },
  9081            "percentage": false,
  9082            "pluginVersion": "7.5.11",
  9083            "pointradius": 2,
  9084            "points": false,
  9085            "renderer": "flot",
  9086            "seriesOverrides": [],
  9087            "spaceLength": 10,
  9088            "stack": false,
  9089            "steppedLine": false,
  9090            "targets": [
  9091              {
  9092                "exemplar": true,
  9093                "expr": "sum(ticdc_sink_cloud_storage_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, instance)",
  9094                "interval": "",
  9095                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}",
  9096                "queryType": "randomWalk",
  9097                "refId": "A"
  9098              }
  9099            ],
  9100            "thresholds": [],
  9101            "timeFrom": null,
  9102            "timeRegions": [],
  9103            "timeShift": null,
  9104            "title": "File Count",
  9105            "tooltip": {
  9106              "shared": true,
  9107              "sort": 0,
  9108              "value_type": "individual"
  9109            },
  9110            "type": "graph",
  9111            "xaxis": {
  9112              "buckets": null,
  9113              "mode": "time",
  9114              "name": null,
  9115              "show": true,
  9116              "values": []
  9117            },
  9118            "yaxes": [
  9119              {
  9120                "format": "short",
  9121                "label": null,
  9122                "logBase": 1,
  9123                "max": null,
  9124                "min": null,
  9125                "show": true
  9126              },
  9127              {
  9128                "format": "short",
  9129                "label": null,
  9130                "logBase": 1,
  9131                "max": null,
  9132                "min": null,
  9133                "show": true
  9134              }
  9135            ],
  9136            "yaxis": {
  9137              "align": false,
  9138              "alignLevel": null
  9139            }
  9140          },
  9141          {
  9142            "cards": {
  9143              "cardPadding": 0,
  9144              "cardRound": 0
  9145            },
  9146            "color": {
  9147              "cardColor": "#b4ff00",
  9148              "colorScale": "sqrt",
  9149              "colorScheme": "interpolateSpectral",
  9150              "exponent": 0.5,
  9151              "max": null,
  9152              "min": 0,
  9153              "mode": "spectrum"
  9154            },
  9155            "dataFormat": "tsbuckets",
  9156            "datasource": "${DS_TEST-CLUSTER}",
  9157            "description": "The latency distributions of write storage by a cloud storage sink",
  9158            "fieldConfig": {
  9159              "defaults": {},
  9160              "overrides": []
  9161            },
  9162            "gridPos": {
  9163              "h": 8,
  9164              "w": 12,
  9165              "x": 0,
  9166              "y": 16
  9167            },
  9168            "heatmap": {},
  9169            "hideZeroBuckets": true,
  9170            "highlightCards": true,
  9171            "id": 10015,
  9172            "legend": {
  9173              "show": true
  9174            },
  9175            "pluginVersion": "7.5.11",
  9176            "reverseYBuckets": false,
  9177            "targets": [
  9178              {
  9179                "exemplar": true,
  9180                "expr": "max(rate(ticdc_sink_cloud_storage_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)",
  9181                "format": "heatmap",
  9182                "hide": false,
  9183                "interval": "",
  9184                "intervalFactor": 2,
  9185                "legendFormat": "{{le}}",
  9186                "queryType": "randomWalk",
  9187                "refId": "A"
  9188              }
  9189            ],
  9190            "timeFrom": null,
  9191            "timeShift": null,
  9192            "title": "Write duration",
  9193            "tooltip": {
  9194              "show": true,
  9195              "showHistogram": true
  9196            },
  9197            "tooltipDecimals": 1,
  9198            "type": "heatmap",
  9199            "xAxis": {
  9200              "show": true
  9201            },
  9202            "xBucketNumber": null,
  9203            "xBucketSize": null,
  9204            "yAxis": {
  9205              "decimals": 1,
  9206              "format": "s",
  9207              "logBase": 1,
  9208              "max": null,
  9209              "min": null,
  9210              "show": true,
  9211              "splitFactor": null
  9212            },
  9213            "yBucketBound": "upper",
  9214            "yBucketNumber": null,
  9215            "yBucketSize": null
  9216          },
  9217          {
  9218            "cards": {
  9219              "cardPadding": 0,
  9220              "cardRound": 0
  9221            },
  9222            "color": {
  9223              "cardColor": "#b4ff00",
  9224              "colorScale": "sqrt",
  9225              "colorScheme": "interpolateSpectral",
  9226              "exponent": 0.5,
  9227              "max": null,
  9228              "min": 0,
  9229              "mode": "spectrum"
  9230            },
  9231            "dataFormat": "tsbuckets",
  9232            "datasource": "${DS_TEST-CLUSTER}",
  9233            "description": "The latency distributions of flush storage by a cloud storage sink",
  9234            "fieldConfig": {
  9235              "defaults": {},
  9236              "overrides": []
  9237            },
  9238            "gridPos": {
  9239              "h": 8,
  9240              "w": 12,
  9241              "x": 12,
  9242              "y": 16
  9243            },
  9244            "heatmap": {},
  9245            "hideZeroBuckets": true,
  9246            "highlightCards": true,
  9247            "id": 10017,
  9248            "legend": {
  9249              "show": true
  9250            },
  9251            "pluginVersion": "7.5.11",
  9252            "reverseYBuckets": false,
  9253            "targets": [
  9254              {
  9255                "exemplar": true,
  9256                "expr": "max(rate(ticdc_sink_cloud_storage_flush_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)",
  9257                "format": "heatmap",
  9258                "hide": false,
  9259                "interval": "",
  9260                "intervalFactor": 2,
  9261                "legendFormat": "{{le}}",
  9262                "queryType": "randomWalk",
  9263                "refId": "A"
  9264              }
  9265            ],
  9266            "timeFrom": null,
  9267            "timeShift": null,
  9268            "title": "Flush duration",
  9269            "tooltip": {
  9270              "show": true,
  9271              "showHistogram": true
  9272            },
  9273            "tooltipDecimals": 1,
  9274            "type": "heatmap",
  9275            "xAxis": {
  9276              "show": true
  9277            },
  9278            "xBucketNumber": null,
  9279            "xBucketSize": null,
  9280            "yAxis": {
  9281              "decimals": 1,
  9282              "format": "s",
  9283              "logBase": 1,
  9284              "max": null,
  9285              "min": null,
  9286              "show": true,
  9287              "splitFactor": null
  9288            },
  9289            "yBucketBound": "upper",
  9290            "yBucketNumber": null,
  9291            "yBucketSize": null
  9292          },
  9293          {
  9294            "aliasColors": {},
  9295            "bars": false,
  9296            "dashLength": 10,
  9297            "dashes": false,
  9298            "datasource": "${DS_TEST-CLUSTER}",
  9299            "description": "Busy ratio (X ms in 1s) for cloud storage sink dml worker",
  9300            "fieldConfig": {
  9301              "defaults": {},
  9302              "overrides": []
  9303            },
  9304            "fill": 1,
  9305            "fillGradient": 0,
  9306            "gridPos": {
  9307              "h": 8,
  9308              "w": 12,
  9309              "x": 0,
  9310              "y": 24
  9311            },
  9312            "hiddenSeries": false,
  9313            "id": 10019,
  9314            "legend": {
  9315              "avg": true,
  9316              "current": true,
  9317              "max": false,
  9318              "min": false,
  9319              "show": true,
  9320              "total": false,
  9321              "values": true
  9322            },
  9323            "lines": true,
  9324            "linewidth": 1,
  9325            "nullPointMode": "null",
  9326            "options": {
  9327              "alertThreshold": true
  9328            },
  9329            "percentage": false,
  9330            "pluginVersion": "7.5.11",
  9331            "pointradius": 2,
  9332            "points": false,
  9333            "renderer": "flot",
  9334            "seriesOverrides": [],
  9335            "spaceLength": 10,
  9336            "stack": false,
  9337            "steppedLine": false,
  9338            "targets": [
  9339              {
  9340                "exemplar": true,
  9341                "expr": "sum(rate(ticdc_sink_cloud_storage_worker_busy_ratio{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])/10) by (namespace,changefeed,id,instance)",
  9342                "hide": false,
  9343                "interval": "",
  9344                "legendFormat": "{{namespace}}-{{changefeed}}-{{id}}-{{instance}}",
  9345                "queryType": "randomWalk",
  9346                "refId": "A"
  9347              }
  9348            ],
  9349            "thresholds": [],
  9350            "timeFrom": null,
  9351            "timeRegions": [],
  9352            "timeShift": null,
  9353            "title": "Worker Busy Ratio",
  9354            "tooltip": {
  9355              "shared": true,
  9356              "sort": 0,
  9357              "value_type": "individual"
  9358            },
  9359            "type": "graph",
  9360            "xaxis": {
  9361              "buckets": null,
  9362              "mode": "time",
  9363              "name": null,
  9364              "show": true,
  9365              "values": []
  9366            },
  9367            "yaxes": [
  9368              {
  9369                "format": "percent",
  9370                "label": null,
  9371                "logBase": 1,
  9372                "max": null,
  9373                "min": null,
  9374                "show": true
  9375              },
  9376              {
  9377                "format": "short",
  9378                "label": null,
  9379                "logBase": 1,
  9380                "max": null,
  9381                "min": null,
  9382                "show": false
  9383              }
  9384            ],
  9385            "yaxis": {
  9386              "align": false,
  9387              "alignLevel": null
  9388            }
  9389          }
  9390        ],
  9391        "title": "Sink - Cloud Storage Sink",
  9392        "type": "row"
  9393      },
  9394      {
  9395        "collapsed": true,
  9396        "datasource": null,
  9397        "gridPos": {
  9398          "h": 1,
  9399          "w": 24,
  9400          "x": 0,
  9401          "y": 8
  9402        },
  9403        "id": 528,
  9404        "panels": [
  9405          {
  9406            "aliasColors": {},
  9407            "bars": false,
  9408            "dashLength": 10,
  9409            "dashes": false,
  9410            "datasource": "${DS_TEST-CLUSTER}",
  9411            "description": "The lag between changefeed checkpoint ts and the latest ts of upstream TiDB.",
  9412            "fieldConfig": {
  9413              "defaults": {
  9414                "links": []
  9415              },
  9416              "overrides": []
  9417            },
  9418            "fill": 1,
  9419            "fillGradient": 0,
  9420            "gridPos": {
  9421              "h": 4,
  9422              "w": 12,
  9423              "x": 0,
  9424              "y": 9
  9425            },
  9426            "hiddenSeries": false,
  9427            "id": 571,
  9428            "legend": {
  9429              "alignAsTable": false,
  9430              "avg": false,
  9431              "current": false,
  9432              "max": false,
  9433              "min": false,
  9434              "rightSide": false,
  9435              "show": true,
  9436              "total": false,
  9437              "values": false
  9438            },
  9439            "lines": true,
  9440            "linewidth": 1,
  9441            "links": [],
  9442            "nullPointMode": "null",
  9443            "options": {
  9444              "alertThreshold": true
  9445            },
  9446            "paceLength": 10,
  9447            "percentage": false,
  9448            "pluginVersion": "7.5.11",
  9449            "pointradius": 2,
  9450            "points": false,
  9451            "renderer": "flot",
  9452            "seriesOverrides": [],
  9453            "spaceLength": 10,
  9454            "stack": false,
  9455            "steppedLine": false,
  9456            "targets": [
  9457              {
  9458                "exemplar": true,
  9459                "expr": "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)",
  9460                "format": "time_series",
  9461                "interval": "",
  9462                "intervalFactor": 1,
  9463                "legendFormat": "{{namespace}}-{{changefeed}}",
  9464                "refId": "A"
  9465              }
  9466            ],
  9467            "thresholds": [],
  9468            "timeFrom": null,
  9469            "timeRegions": [],
  9470            "timeShift": null,
  9471            "title": "Changefeed checkpoint lag",
  9472            "tooltip": {
  9473              "shared": true,
  9474              "sort": 0,
  9475              "value_type": "individual"
  9476            },
  9477            "type": "graph",
  9478            "xaxis": {
  9479              "buckets": null,
  9480              "mode": "time",
  9481              "name": null,
  9482              "show": true,
  9483              "values": []
  9484            },
  9485            "yaxes": [
  9486              {
  9487                "format": "s",
  9488                "label": null,
  9489                "logBase": 1,
  9490                "max": null,
  9491                "min": "0",
  9492                "show": true
  9493              },
  9494              {
  9495                "format": "short",
  9496                "label": null,
  9497                "logBase": 1,
  9498                "max": null,
  9499                "min": null,
  9500                "show": true
  9501              }
  9502            ],
  9503            "yaxis": {
  9504              "align": false,
  9505              "alignLevel": null
  9506            }
  9507          },
  9508          {
  9509            "aliasColors": {},
  9510            "bars": false,
  9511            "dashLength": 10,
  9512            "dashes": false,
  9513            "datasource": "${DS_TEST-CLUSTER}",
  9514            "description": "The lag between changefeed resolved ts and the latest ts of upstream TiDB.",
  9515            "fieldConfig": {
  9516              "defaults": {
  9517                "links": []
  9518              },
  9519              "overrides": []
  9520            },
  9521            "fill": 1,
  9522            "fillGradient": 0,
  9523            "gridPos": {
  9524              "h": 4,
  9525              "w": 12,
  9526              "x": 12,
  9527              "y": 9
  9528            },
  9529            "hiddenSeries": false,
  9530            "id": 589,
  9531            "legend": {
  9532              "alignAsTable": false,
  9533              "avg": false,
  9534              "current": false,
  9535              "max": false,
  9536              "min": false,
  9537              "rightSide": false,
  9538              "show": true,
  9539              "total": false,
  9540              "values": false
  9541            },
  9542            "lines": true,
  9543            "linewidth": 1,
  9544            "links": [],
  9545            "nullPointMode": "null",
  9546            "options": {
  9547              "alertThreshold": true
  9548            },
  9549            "paceLength": 10,
  9550            "percentage": false,
  9551            "pluginVersion": "7.5.11",
  9552            "pointradius": 2,
  9553            "points": false,
  9554            "renderer": "flot",
  9555            "seriesOverrides": [],
  9556            "spaceLength": 10,
  9557            "stack": false,
  9558            "steppedLine": false,
  9559            "targets": [
  9560              {
  9561                "exemplar": true,
  9562                "expr": "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)",
  9563                "format": "time_series",
  9564                "interval": "",
  9565                "intervalFactor": 1,
  9566                "legendFormat": "{{namespace}}-{{changefeed}}-resolvedts",
  9567                "refId": "C"
  9568              }
  9569            ],
  9570            "thresholds": [],
  9571            "timeFrom": null,
  9572            "timeRegions": [],
  9573            "timeShift": null,
  9574            "title": "Changefeed resolved ts lag",
  9575            "tooltip": {
  9576              "shared": true,
  9577              "sort": 0,
  9578              "value_type": "individual"
  9579            },
  9580            "type": "graph",
  9581            "xaxis": {
  9582              "buckets": null,
  9583              "mode": "time",
  9584              "name": null,
  9585              "show": true,
  9586              "values": []
  9587            },
  9588            "yaxes": [
  9589              {
  9590                "format": "s",
  9591                "label": null,
  9592                "logBase": 1,
  9593                "max": null,
  9594                "min": "0",
  9595                "show": true
  9596              },
  9597              {
  9598                "format": "short",
  9599                "label": null,
  9600                "logBase": 1,
  9601                "max": null,
  9602                "min": null,
  9603                "show": true
  9604              }
  9605            ],
  9606            "yaxis": {
  9607              "align": false,
  9608              "alignLevel": null
  9609            }
  9610          },
  9611          {
  9612            "aliasColors": {},
  9613            "bars": false,
  9614            "dashLength": 10,
  9615            "dashes": false,
  9616            "datasource": "${DS_TEST-CLUSTER}",
  9617            "decimals": 1,
  9618            "description": "The total number of tables in different replication states.\n\n0: ReplicationSetStateUnknown means the replication state is unknown, it should not happen.\n\n1: ReplicationSetStateAbsent means there is no one replicates or prepares it.\n\n2: ReplicationSetStatePrepare means it needs to add a secondary.\n\n3: ReplicationSetStateCommit means it needs to promote secondary to primary.\n\n4: ReplicationSetStateReplicating means there is exactly one capture that is replicating the table.\n\n5: ReplicationSetStateRemoving means all captures need to stop replication eventually.\n\n",
  9619            "editable": true,
  9620            "error": false,
  9621            "fieldConfig": {
  9622              "defaults": {
  9623                "unit": "none"
  9624              },
  9625              "overrides": []
  9626            },
  9627            "fill": 0,
  9628            "fillGradient": 0,
  9629            "grid": {},
  9630            "gridPos": {
  9631              "h": 6,
  9632              "w": 12,
  9633              "x": 0,
  9634              "y": 13
  9635            },
  9636            "hiddenSeries": false,
  9637            "id": 546,
  9638            "legend": {
  9639              "alignAsTable": false,
  9640              "avg": false,
  9641              "current": false,
  9642              "max": false,
  9643              "min": false,
  9644              "rightSide": false,
  9645              "show": true,
  9646              "sideWidth": null,
  9647              "sort": null,
  9648              "sortDesc": null,
  9649              "total": false,
  9650              "values": false
  9651            },
  9652            "lines": true,
  9653            "linewidth": 1,
  9654            "links": [],
  9655            "nullPointMode": "null",
  9656            "options": {
  9657              "alertThreshold": true
  9658            },
  9659            "paceLength": 10,
  9660            "percentage": false,
  9661            "pluginVersion": "7.5.11",
  9662            "pointradius": 1,
  9663            "points": true,
  9664            "renderer": "flot",
  9665            "seriesOverrides": [],
  9666            "spaceLength": 10,
  9667            "stack": false,
  9668            "steppedLine": false,
  9669            "targets": [
  9670              {
  9671                "exemplar": true,
  9672                "expr": "sum(ticdc_scheduler_table_replication_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, state)",
  9673                "format": "time_series",
  9674                "hide": false,
  9675                "interval": "",
  9676                "intervalFactor": 2,
  9677                "legendFormat": "{{namespace}}-{{changefeed}}-{{state}}",
  9678                "refId": "A",
  9679                "step": 10
  9680              }
  9681            ],
  9682            "thresholds": [],
  9683            "timeFrom": null,
  9684            "timeRegions": [],
  9685            "timeShift": null,
  9686            "title": "Table replication state",
  9687            "tooltip": {
  9688              "msResolution": false,
  9689              "shared": true,
  9690              "sort": 0,
  9691              "value_type": "individual"
  9692            },
  9693            "type": "graph",
  9694            "xaxis": {
  9695              "buckets": null,
  9696              "mode": "time",
  9697              "name": null,
  9698              "show": true,
  9699              "values": []
  9700            },
  9701            "yaxes": [
  9702              {
  9703                "format": "none",
  9704                "label": null,
  9705                "logBase": 1,
  9706                "max": null,
  9707                "min": "0",
  9708                "show": true
  9709              },
  9710              {
  9711                "format": "none",
  9712                "label": null,
  9713                "logBase": 1,
  9714                "max": null,
  9715                "min": null,
  9716                "show": false
  9717              }
  9718            ],
  9719            "yaxis": {
  9720              "align": false,
  9721              "alignLevel": null
  9722            }
  9723          },
  9724          {
  9725            "aliasColors": {},
  9726            "bars": false,
  9727            "dashLength": 10,
  9728            "dashes": false,
  9729            "datasource": "${DS_TEST-CLUSTER}",
  9730            "decimals": 1,
  9731            "description": "The total number of accepted schedule tasks.",
  9732            "editable": true,
  9733            "error": false,
  9734            "fieldConfig": {
  9735              "defaults": {
  9736                "unit": "none"
  9737              },
  9738              "overrides": []
  9739            },
  9740            "fill": 0,
  9741            "fillGradient": 0,
  9742            "grid": {},
  9743            "gridPos": {
  9744              "h": 6,
  9745              "w": 12,
  9746              "x": 12,
  9747              "y": 13
  9748            },
  9749            "hiddenSeries": false,
  9750            "id": 549,
  9751            "legend": {
  9752              "alignAsTable": false,
  9753              "avg": false,
  9754              "current": false,
  9755              "max": false,
  9756              "min": false,
  9757              "rightSide": false,
  9758              "show": true,
  9759              "sideWidth": null,
  9760              "sort": null,
  9761              "sortDesc": null,
  9762              "total": false,
  9763              "values": false
  9764            },
  9765            "lines": true,
  9766            "linewidth": 1,
  9767            "links": [],
  9768            "nullPointMode": "null",
  9769            "options": {
  9770              "alertThreshold": true
  9771            },
  9772            "paceLength": 10,
  9773            "percentage": false,
  9774            "pluginVersion": "7.5.11",
  9775            "pointradius": 1,
  9776            "points": true,
  9777            "renderer": "flot",
  9778            "seriesOverrides": [],
  9779            "spaceLength": 10,
  9780            "stack": false,
  9781            "steppedLine": false,
  9782            "targets": [
  9783              {
  9784                "exemplar": true,
  9785                "expr": "sum(ticdc_scheduler_task_accept{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, scheduler, task)",
  9786                "format": "time_series",
  9787                "hide": false,
  9788                "interval": "",
  9789                "intervalFactor": 2,
  9790                "legendFormat": "{{namespace}}-{{changefeed}}-{{task}}",
  9791                "refId": "A",
  9792                "step": 10
  9793              }
  9794            ],
  9795            "thresholds": [],
  9796            "timeFrom": null,
  9797            "timeRegions": [],
  9798            "timeShift": null,
  9799            "title": "Schedule tasks accept",
  9800            "tooltip": {
  9801              "msResolution": false,
  9802              "shared": true,
  9803              "sort": 0,
  9804              "value_type": "individual"
  9805            },
  9806            "type": "graph",
  9807            "xaxis": {
  9808              "buckets": null,
  9809              "mode": "time",
  9810              "name": null,
  9811              "show": true,
  9812              "values": []
  9813            },
  9814            "yaxes": [
  9815              {
  9816                "format": "none",
  9817                "label": null,
  9818                "logBase": 1,
  9819                "max": null,
  9820                "min": "0",
  9821                "show": true
  9822              },
  9823              {
  9824                "format": "none",
  9825                "label": null,
  9826                "logBase": 1,
  9827                "max": null,
  9828                "min": null,
  9829                "show": false
  9830              }
  9831            ],
  9832            "yaxis": {
  9833              "align": false,
  9834              "alignLevel": null
  9835            }
  9836          },
  9837          {
  9838            "aliasColors": {},
  9839            "bars": false,
  9840            "dashLength": 10,
  9841            "dashes": false,
  9842            "datasource": "${DS_TEST-CLUSTER}",
  9843            "decimals": 1,
  9844            "description": "The total number of different schedule tasks.",
  9845            "editable": true,
  9846            "error": false,
  9847            "fieldConfig": {
  9848              "defaults": {
  9849                "unit": "none"
  9850              },
  9851              "overrides": []
  9852            },
  9853            "fill": 0,
  9854            "fillGradient": 0,
  9855            "grid": {},
  9856            "gridPos": {
  9857              "h": 6,
  9858              "w": 12,
  9859              "x": 0,
  9860              "y": 19
  9861            },
  9862            "hiddenSeries": false,
  9863            "id": 548,
  9864            "legend": {
  9865              "alignAsTable": false,
  9866              "avg": false,
  9867              "current": false,
  9868              "max": false,
  9869              "min": false,
  9870              "rightSide": false,
  9871              "show": true,
  9872              "sideWidth": null,
  9873              "sort": null,
  9874              "sortDesc": null,
  9875              "total": false,
  9876              "values": false
  9877            },
  9878            "lines": true,
  9879            "linewidth": 1,
  9880            "links": [],
  9881            "nullPointMode": "null",
  9882            "options": {
  9883              "alertThreshold": true
  9884            },
  9885            "paceLength": 10,
  9886            "percentage": false,
  9887            "pluginVersion": "7.5.11",
  9888            "pointradius": 1,
  9889            "points": true,
  9890            "renderer": "flot",
  9891            "seriesOverrides": [],
  9892            "spaceLength": 10,
  9893            "stack": false,
  9894            "steppedLine": false,
  9895            "targets": [
  9896              {
  9897                "exemplar": true,
  9898                "expr": "sum(ticdc_scheduler_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, scheduler, task)",
  9899                "format": "time_series",
  9900                "hide": false,
  9901                "interval": "",
  9902                "intervalFactor": 2,
  9903                "legendFormat": "{{namespace}}-{{changefeed}}-{{scheduler}}-{{task}}",
  9904                "refId": "A",
  9905                "step": 10
  9906              }
  9907            ],
  9908            "thresholds": [],
  9909            "timeFrom": null,
  9910            "timeRegions": [],
  9911            "timeShift": null,
  9912            "title": "Schedule tasks",
  9913            "tooltip": {
  9914              "msResolution": false,
  9915              "shared": true,
  9916              "sort": 0,
  9917              "value_type": "individual"
  9918            },
  9919            "type": "graph",
  9920            "xaxis": {
  9921              "buckets": null,
  9922              "mode": "time",
  9923              "name": null,
  9924              "show": true,
  9925              "values": []
  9926            },
  9927            "yaxes": [
  9928              {
  9929                "format": "none",
  9930                "label": null,
  9931                "logBase": 1,
  9932                "max": null,
  9933                "min": "0",
  9934                "show": true
  9935              },
  9936              {
  9937                "format": "none",
  9938                "label": null,
  9939                "logBase": 1,
  9940                "max": null,
  9941                "min": null,
  9942                "show": false
  9943              }
  9944            ],
  9945            "yaxis": {
  9946              "align": false,
  9947              "alignLevel": null
  9948            }
  9949          },
  9950          {
  9951            "aliasColors": {},
  9952            "bars": false,
  9953            "dashLength": 10,
  9954            "dashes": false,
  9955            "datasource": "${DS_TEST-CLUSTER}",
  9956            "decimals": 1,
  9957            "description": "The total number of running schedule tasks.",
  9958            "editable": true,
  9959            "error": false,
  9960            "fieldConfig": {
  9961              "defaults": {
  9962                "unit": "none"
  9963              },
  9964              "overrides": []
  9965            },
  9966            "fill": 0,
  9967            "fillGradient": 0,
  9968            "grid": {},
  9969            "gridPos": {
  9970              "h": 6,
  9971              "w": 12,
  9972              "x": 12,
  9973              "y": 19
  9974            },
  9975            "hiddenSeries": false,
  9976            "id": 588,
  9977            "legend": {
  9978              "alignAsTable": false,
  9979              "avg": false,
  9980              "current": false,
  9981              "max": false,
  9982              "min": false,
  9983              "rightSide": false,
  9984              "show": true,
  9985              "sideWidth": null,
  9986              "sort": null,
  9987              "sortDesc": null,
  9988              "total": false,
  9989              "values": false
  9990            },
  9991            "lines": true,
  9992            "linewidth": 1,
  9993            "links": [],
  9994            "nullPointMode": "null",
  9995            "options": {
  9996              "alertThreshold": true
  9997            },
  9998            "paceLength": 10,
  9999            "percentage": false,
 10000            "pluginVersion": "7.5.11",
 10001            "pointradius": 1,
 10002            "points": true,
 10003            "renderer": "flot",
 10004            "seriesOverrides": [],
 10005            "spaceLength": 10,
 10006            "stack": false,
 10007            "steppedLine": false,
 10008            "targets": [
 10009              {
 10010                "exemplar": true,
 10011                "expr": "sum(ticdc_scheduler_task_running{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, scheduler, task)",
 10012                "format": "time_series",
 10013                "hide": false,
 10014                "interval": "",
 10015                "intervalFactor": 2,
 10016                "legendFormat": "{{namespace}}-{{changefeed}}-{{task}}",
 10017                "refId": "A",
 10018                "step": 10
 10019              }
 10020            ],
 10021            "thresholds": [],
 10022            "timeFrom": null,
 10023            "timeRegions": [],
 10024            "timeShift": null,
 10025            "title": "Schedule tasks running",
 10026            "tooltip": {
 10027              "msResolution": false,
 10028              "shared": true,
 10029              "sort": 0,
 10030              "value_type": "individual"
 10031            },
 10032            "type": "graph",
 10033            "xaxis": {
 10034              "buckets": null,
 10035              "mode": "time",
 10036              "name": null,
 10037              "show": true,
 10038              "values": []
 10039            },
 10040            "yaxes": [
 10041              {
 10042                "format": "none",
 10043                "label": null,
 10044                "logBase": 1,
 10045                "max": null,
 10046                "min": "0",
 10047                "show": true
 10048              },
 10049              {
 10050                "format": "none",
 10051                "label": null,
 10052                "logBase": 1,
 10053                "max": null,
 10054                "min": null,
 10055                "show": false
 10056              }
 10057            ],
 10058            "yaxis": {
 10059              "align": false,
 10060              "alignLevel": null
 10061            }
 10062          },
 10063          {
 10064            "aliasColors": {},
 10065            "bars": false,
 10066            "dashLength": 10,
 10067            "dashes": false,
 10068            "datasource": "${DS_TEST-CLUSTER}",
 10069            "description": "The total number of tables",
 10070            "fieldConfig": {
 10071              "defaults": {
 10072                "links": []
 10073              },
 10074              "overrides": []
 10075            },
 10076            "fill": 0,
 10077            "fillGradient": 0,
 10078            "gridPos": {
 10079              "h": 6,
 10080              "w": 12,
 10081              "x": 0,
 10082              "y": 25
 10083            },
 10084            "hiddenSeries": false,
 10085            "id": 545,
 10086            "legend": {
 10087              "alignAsTable": false,
 10088              "avg": false,
 10089              "current": true,
 10090              "max": true,
 10091              "min": false,
 10092              "rightSide": false,
 10093              "show": true,
 10094              "total": false,
 10095              "values": true
 10096            },
 10097            "lines": true,
 10098            "linewidth": 1,
 10099            "links": [],
 10100            "nullPointMode": "null",
 10101            "options": {
 10102              "alertThreshold": false
 10103            },
 10104            "paceLength": 10,
 10105            "percentage": false,
 10106            "pluginVersion": "7.5.11",
 10107            "pointradius": 2,
 10108            "points": true,
 10109            "renderer": "flot",
 10110            "seriesOverrides": [],
 10111            "spaceLength": 10,
 10112            "stack": false,
 10113            "steppedLine": false,
 10114            "targets": [
 10115              {
 10116                "exemplar": true,
 10117                "expr": "sum(ticdc_scheduler_table{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)",
 10118                "format": "time_series",
 10119                "interval": "",
 10120                "intervalFactor": 1,
 10121                "legendFormat": "{{namespace}}-{{changefeed}}",
 10122                "refId": "A"
 10123              }
 10124            ],
 10125            "thresholds": [],
 10126            "timeFrom": null,
 10127            "timeRegions": [],
 10128            "timeShift": null,
 10129            "title": "Table count",
 10130            "tooltip": {
 10131              "shared": true,
 10132              "sort": 0,
 10133              "value_type": "individual"
 10134            },
 10135            "type": "graph",
 10136            "xaxis": {
 10137              "buckets": null,
 10138              "mode": "time",
 10139              "name": null,
 10140              "show": true,
 10141              "values": []
 10142            },
 10143            "yaxes": [
 10144              {
 10145                "format": "short",
 10146                "label": null,
 10147                "logBase": 1,
 10148                "max": null,
 10149                "min": null,
 10150                "show": true
 10151              },
 10152              {
 10153                "format": "short",
 10154                "label": null,
 10155                "logBase": 1,
 10156                "max": null,
 10157                "min": null,
 10158                "show": true
 10159              }
 10160            ],
 10161            "yaxis": {
 10162              "align": false,
 10163              "alignLevel": null
 10164            }
 10165          },
 10166          {
 10167            "aliasColors": {},
 10168            "bars": false,
 10169            "dashLength": 10,
 10170            "dashes": false,
 10171            "datasource": "${DS_TEST-CLUSTER}",
 10172            "decimals": 1,
 10173            "description": "The number of tables in different capture.",
 10174            "editable": true,
 10175            "error": false,
 10176            "fieldConfig": {
 10177              "defaults": {
 10178                "unit": "none"
 10179              },
 10180              "overrides": []
 10181            },
 10182            "fill": 0,
 10183            "fillGradient": 0,
 10184            "grid": {},
 10185            "gridPos": {
 10186              "h": 6,
 10187              "w": 12,
 10188              "x": 12,
 10189              "y": 25
 10190            },
 10191            "hiddenSeries": false,
 10192            "id": 547,
 10193            "legend": {
 10194              "alignAsTable": false,
 10195              "avg": false,
 10196              "current": false,
 10197              "max": false,
 10198              "min": false,
 10199              "rightSide": false,
 10200              "show": true,
 10201              "sideWidth": null,
 10202              "sort": null,
 10203              "sortDesc": null,
 10204              "total": false,
 10205              "values": false
 10206            },
 10207            "lines": true,
 10208            "linewidth": 1,
 10209            "links": [],
 10210            "nullPointMode": "null",
 10211            "options": {
 10212              "alertThreshold": true
 10213            },
 10214            "paceLength": 10,
 10215            "percentage": false,
 10216            "pluginVersion": "7.5.11",
 10217            "pointradius": 1,
 10218            "points": true,
 10219            "renderer": "flot",
 10220            "seriesOverrides": [],
 10221            "spaceLength": 10,
 10222            "stack": false,
 10223            "steppedLine": false,
 10224            "targets": [
 10225              {
 10226                "exemplar": false,
 10227                "expr": "sum(ticdc_scheduler_capture_table{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, addr)",
 10228                "format": "time_series",
 10229                "hide": false,
 10230                "instant": false,
 10231                "interval": "",
 10232                "intervalFactor": 2,
 10233                "legendFormat": "{{namespace}}-{{changefeed}}-{{addr}}",
 10234                "refId": "A",
 10235                "step": 10
 10236              }
 10237            ],
 10238            "thresholds": [],
 10239            "timeFrom": null,
 10240            "timeRegions": [],
 10241            "timeShift": null,
 10242            "title": "Capture tables count",
 10243            "tooltip": {
 10244              "msResolution": false,
 10245              "shared": true,
 10246              "sort": 0,
 10247              "value_type": "individual"
 10248            },
 10249            "type": "graph",
 10250            "xaxis": {
 10251              "buckets": null,
 10252              "mode": "time",
 10253              "name": null,
 10254              "show": true,
 10255              "values": []
 10256            },
 10257            "yaxes": [
 10258              {
 10259                "format": "none",
 10260                "label": null,
 10261                "logBase": 1,
 10262                "max": null,
 10263                "min": "0",
 10264                "show": true
 10265              },
 10266              {
 10267                "format": "none",
 10268                "label": null,
 10269                "logBase": 1,
 10270                "max": null,
 10271                "min": null,
 10272                "show": false
 10273              }
 10274            ],
 10275            "yaxis": {
 10276              "align": false,
 10277              "alignLevel": null
 10278            }
 10279          },
 10280          {
 10281            "aliasColors": {},
 10282            "bars": false,
 10283            "dashLength": 10,
 10284            "dashes": false,
 10285            "datasource": "${DS_TEST-CLUSTER}",
 10286            "decimals": 1,
 10287            "description": "The ID of the slowest table",
 10288            "editable": true,
 10289            "error": false,
 10290            "fieldConfig": {
 10291              "defaults": {},
 10292              "overrides": []
 10293            },
 10294            "fill": 0,
 10295            "fillGradient": 0,
 10296            "grid": {},
 10297            "gridPos": {
 10298              "h": 6,
 10299              "w": 12,
 10300              "x": 0,
 10301              "y": 31
 10302            },
 10303            "hiddenSeries": false,
 10304            "id": 543,
 10305            "legend": {
 10306              "alignAsTable": true,
 10307              "avg": false,
 10308              "current": true,
 10309              "max": false,
 10310              "min": false,
 10311              "rightSide": false,
 10312              "show": true,
 10313              "sideWidth": null,
 10314              "sort": null,
 10315              "sortDesc": null,
 10316              "total": false,
 10317              "values": true
 10318            },
 10319            "lines": false,
 10320            "linewidth": 1,
 10321            "links": [],
 10322            "nullPointMode": "null",
 10323            "options": {
 10324              "alertThreshold": true
 10325            },
 10326            "paceLength": 10,
 10327            "percentage": false,
 10328            "pluginVersion": "7.5.11",
 10329            "pointradius": 1,
 10330            "points": true,
 10331            "renderer": "flot",
 10332            "seriesOverrides": [],
 10333            "spaceLength": 10,
 10334            "stack": false,
 10335            "steppedLine": false,
 10336            "targets": [
 10337              {
 10338                "exemplar": true,
 10339                "expr": "sum(ticdc_scheduler_slow_table_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)",
 10340                "format": "time_series",
 10341                "hide": false,
 10342                "interval": "",
 10343                "intervalFactor": 2,
 10344                "legendFormat": "{{namespace}}-{{changefeed}}",
 10345                "refId": "A",
 10346                "step": 10
 10347              }
 10348            ],
 10349            "thresholds": [],
 10350            "timeFrom": null,
 10351            "timeRegions": [],
 10352            "timeShift": null,
 10353            "title": "Slowest table ID",
 10354            "tooltip": {
 10355              "msResolution": false,
 10356              "shared": true,
 10357              "sort": 0,
 10358              "value_type": "individual"
 10359            },
 10360            "type": "graph",
 10361            "xaxis": {
 10362              "buckets": null,
 10363              "mode": "time",
 10364              "name": null,
 10365              "show": true,
 10366              "values": []
 10367            },
 10368            "yaxes": [
 10369              {
 10370                "format": "none",
 10371                "label": null,
 10372                "logBase": 1,
 10373                "max": null,
 10374                "min": "0",
 10375                "show": true
 10376              },
 10377              {
 10378                "format": "none",
 10379                "label": null,
 10380                "logBase": 1,
 10381                "max": null,
 10382                "min": null,
 10383                "show": false
 10384              }
 10385            ],
 10386            "yaxis": {
 10387              "align": false,
 10388              "alignLevel": null
 10389            }
 10390          },
 10391          {
 10392            "aliasColors": {},
 10393            "bars": false,
 10394            "dashLength": 10,
 10395            "dashes": false,
 10396            "datasource": "${DS_TEST-CLUSTER}",
 10397            "decimals": 1,
 10398            "description": "The state of the slowest table.\n\n0: ReplicationSetStateUnknown means the replication state is unknown, it should not happen.\n\n1: ReplicationSetStateAbsent means there is no one replicates or prepares it.\n\n2: ReplicationSetStatePrepare means it needs to add a secondary.\n\n3: ReplicationSetStateCommit means it needs to promote secondary to primary.\n\n4: ReplicationSetStateReplicating means there is exactly one capture that is replicating the table.\n\n5: ReplicationSetStateRemoving means all captures need to stop replication eventually.\n\n",
 10399            "editable": true,
 10400            "error": false,
 10401            "fieldConfig": {
 10402              "defaults": {
 10403                "unit": "none"
 10404              },
 10405              "overrides": []
 10406            },
 10407            "fill": 0,
 10408            "fillGradient": 0,
 10409            "grid": {},
 10410            "gridPos": {
 10411              "h": 6,
 10412              "w": 12,
 10413              "x": 12,
 10414              "y": 31
 10415            },
 10416            "hiddenSeries": false,
 10417            "id": 544,
 10418            "legend": {
 10419              "alignAsTable": false,
 10420              "avg": false,
 10421              "current": false,
 10422              "max": false,
 10423              "min": false,
 10424              "rightSide": false,
 10425              "show": true,
 10426              "sideWidth": null,
 10427              "sort": null,
 10428              "sortDesc": null,
 10429              "total": false,
 10430              "values": false
 10431            },
 10432            "lines": false,
 10433            "linewidth": 1,
 10434            "links": [],
 10435            "nullPointMode": "null",
 10436            "options": {
 10437              "alertThreshold": true
 10438            },
 10439            "paceLength": 10,
 10440            "percentage": false,
 10441            "pluginVersion": "7.5.11",
 10442            "pointradius": 1,
 10443            "points": true,
 10444            "renderer": "flot",
 10445            "seriesOverrides": [],
 10446            "spaceLength": 10,
 10447            "stack": false,
 10448            "steppedLine": false,
 10449            "targets": [
 10450              {
 10451                "exemplar": true,
 10452                "expr": "sum(ticdc_scheduler_slow_table_replication_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)",
 10453                "format": "time_series",
 10454                "hide": false,
 10455                "interval": "",
 10456                "intervalFactor": 2,
 10457                "legendFormat": "{{namespace}}-{{changefeed}}",
 10458                "refId": "A",
 10459                "step": 10
 10460              }
 10461            ],
 10462            "thresholds": [],
 10463            "timeFrom": null,
 10464            "timeRegions": [],
 10465            "timeShift": null,
 10466            "title": "Slowest table replication state",
 10467            "tooltip": {
 10468              "msResolution": false,
 10469              "shared": true,
 10470              "sort": 0,
 10471              "value_type": "individual"
 10472            },
 10473            "type": "graph",
 10474            "xaxis": {
 10475              "buckets": null,
 10476              "mode": "time",
 10477              "name": null,
 10478              "show": true,
 10479              "values": []
 10480            },
 10481            "yaxes": [
 10482              {
 10483                "format": "none",
 10484                "label": null,
 10485                "logBase": 1,
 10486                "max": null,
 10487                "min": "0",
 10488                "show": true
 10489              },
 10490              {
 10491                "format": "none",
 10492                "label": null,
 10493                "logBase": 1,
 10494                "max": null,
 10495                "min": null,
 10496                "show": false
 10497              }
 10498            ],
 10499            "yaxis": {
 10500              "align": false,
 10501              "alignLevel": null
 10502            }
 10503          },
 10504          {
 10505            "aliasColors": {},
 10506            "bars": true,
 10507            "cacheTimeout": null,
 10508            "dashLength": 10,
 10509            "dashes": false,
 10510            "datasource": "${DS_TEST-CLUSTER}",
 10511            "description": "The checkpoint ts of the slowest table.",
 10512            "fieldConfig": {
 10513              "defaults": {
 10514                "unit": "dateTimeAsIso"
 10515              },
 10516              "overrides": []
 10517            },
 10518            "fill": 0,
 10519            "fillGradient": 0,
 10520            "gridPos": {
 10521              "h": 6,
 10522              "w": 12,
 10523              "x": 0,
 10524              "y": 37
 10525            },
 10526            "hiddenSeries": false,
 10527            "id": 541,
 10528            "legend": {
 10529              "alignAsTable": true,
 10530              "avg": false,
 10531              "current": true,
 10532              "max": false,
 10533              "min": false,
 10534              "rightSide": false,
 10535              "show": true,
 10536              "sideWidth": null,
 10537              "total": false,
 10538              "values": true
 10539            },
 10540            "lines": true,
 10541            "linewidth": 2,
 10542            "links": [],
 10543            "nullPointMode": "null",
 10544            "options": {
 10545              "alertThreshold": true
 10546            },
 10547            "percentage": false,
 10548            "pluginVersion": "7.5.11",
 10549            "pointradius": 2,
 10550            "points": false,
 10551            "renderer": "flot",
 10552            "seriesOverrides": [
 10553              {
 10554                "alias": "/approximate current time.*/",
 10555                "bars": false
 10556              }
 10557            ],
 10558            "spaceLength": 10,
 10559            "stack": false,
 10560            "steppedLine": false,
 10561            "targets": [
 10562              {
 10563                "exemplar": true,
 10564                "expr": "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})",
 10565                "format": "time_series",
 10566                "hide": false,
 10567                "interval": "",
 10568                "intervalFactor": 1,
 10569                "legendFormat": "approximate current time (s)",
 10570                "refId": "A"
 10571              },
 10572              {
 10573                "exemplar": true,
 10574                "expr": "max(ticdc_scheduler_slow_table_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)",
 10575                "format": "time_series",
 10576                "hide": false,
 10577                "interval": "",
 10578                "intervalFactor": 1,
 10579                "legendFormat": "{{namespace}}-{{changefeed}}",
 10580                "refId": "B"
 10581              }
 10582            ],
 10583            "thresholds": [],
 10584            "timeFrom": null,
 10585            "timeRegions": [],
 10586            "timeShift": null,
 10587            "title": "Slowest table checkpoint",
 10588            "tooltip": {
 10589              "shared": true,
 10590              "sort": 0,
 10591              "value_type": "individual"
 10592            },
 10593            "type": "graph",
 10594            "xaxis": {
 10595              "buckets": null,
 10596              "max": null,
 10597              "mode": "time",
 10598              "name": null,
 10599              "show": true,
 10600              "values": []
 10601            },
 10602            "yaxes": [
 10603              {
 10604                "format": "dateTimeAsIso",
 10605                "label": null,
 10606                "logBase": 1,
 10607                "max": null,
 10608                "min": null,
 10609                "show": true
 10610              },
 10611              {
 10612                "format": "short",
 10613                "label": null,
 10614                "logBase": 1,
 10615                "max": null,
 10616                "min": null,
 10617                "show": false
 10618              }
 10619            ],
 10620            "yaxis": {
 10621              "align": false,
 10622              "alignLevel": null
 10623            }
 10624          },
 10625          {
 10626            "aliasColors": {},
 10627            "bars": true,
 10628            "cacheTimeout": null,
 10629            "dashLength": 10,
 10630            "dashes": false,
 10631            "datasource": "${DS_TEST-CLUSTER}",
 10632            "description": "The resolved ts of the slowest table.",
 10633            "fieldConfig": {
 10634              "defaults": {},
 10635              "overrides": []
 10636            },
 10637            "fill": 0,
 10638            "fillGradient": 0,
 10639            "gridPos": {
 10640              "h": 6,
 10641              "w": 12,
 10642              "x": 12,
 10643              "y": 37
 10644            },
 10645            "hiddenSeries": false,
 10646            "id": 590,
 10647            "legend": {
 10648              "alignAsTable": true,
 10649              "avg": false,
 10650              "current": true,
 10651              "max": false,
 10652              "min": false,
 10653              "rightSide": false,
 10654              "show": true,
 10655              "sideWidth": null,
 10656              "total": false,
 10657              "values": true
 10658            },
 10659            "lines": true,
 10660            "linewidth": 2,
 10661            "links": [],
 10662            "nullPointMode": "null",
 10663            "options": {
 10664              "alertThreshold": true
 10665            },
 10666            "percentage": false,
 10667            "pluginVersion": "7.5.11",
 10668            "pointradius": 2,
 10669            "points": false,
 10670            "renderer": "flot",
 10671            "seriesOverrides": [
 10672              {
 10673                "alias": "/approximate current time.*/",
 10674                "bars": false
 10675              }
 10676            ],
 10677            "spaceLength": 10,
 10678            "stack": false,
 10679            "steppedLine": false,
 10680            "targets": [
 10681              {
 10682                "exemplar": true,
 10683                "expr": "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})",
 10684                "format": "time_series",
 10685                "hide": false,
 10686                "interval": "",
 10687                "intervalFactor": 1,
 10688                "legendFormat": "approximate current time (s)",
 10689                "refId": "A"
 10690              },
 10691              {
 10692                "exemplar": true,
 10693                "expr": "max(ticdc_scheduler_slow_table_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)",
 10694                "format": "time_series",
 10695                "hide": false,
 10696                "interval": "",
 10697                "intervalFactor": 1,
 10698                "legendFormat": "{{namespace}}-{{changefeed}}",
 10699                "refId": "B"
 10700              }
 10701            ],
 10702            "thresholds": [],
 10703            "timeFrom": null,
 10704            "timeRegions": [],
 10705            "timeShift": null,
 10706            "title": "Slowest table resolved ts",
 10707            "tooltip": {
 10708              "shared": true,
 10709              "sort": 0,
 10710              "value_type": "individual"
 10711            },
 10712            "type": "graph",
 10713            "xaxis": {
 10714              "buckets": null,
 10715              "max": null,
 10716              "mode": "time",
 10717              "name": null,
 10718              "show": true,
 10719              "values": []
 10720            },
 10721            "yaxes": [
 10722              {
 10723                "format": "dateTimeAsIso",
 10724                "label": null,
 10725                "logBase": 1,
 10726                "max": null,
 10727                "min": null,
 10728                "show": true
 10729              },
 10730              {
 10731                "format": "short",
 10732                "label": null,
 10733                "logBase": 1,
 10734                "max": null,
 10735                "min": null,
 10736                "show": false
 10737              }
 10738            ],
 10739            "yaxis": {
 10740              "align": false,
 10741              "alignLevel": null
 10742            }
 10743          },
 10744          {
 10745            "aliasColors": {},
 10746            "bars": false,
 10747            "dashLength": 10,
 10748            "dashes": false,
 10749            "datasource": "${DS_TEST-CLUSTER}",
 10750            "decimals": 1,
 10751            "description": "The number of incremental scan tasks per minute",
 10752            "fieldConfig": {
 10753              "defaults": {},
 10754              "overrides": []
 10755            },
 10756            "fill": 1,
 10757            "fillGradient": 0,
 10758            "gridPos": {
 10759              "h": 6,
 10760              "w": 12,
 10761              "x": 0,
 10762              "y": 43
 10763            },
 10764            "hiddenSeries": false,
 10765            "id": 560,
 10766            "legend": {
 10767              "alignAsTable": false,
 10768              "avg": false,
 10769              "current": false,
 10770              "hideEmpty": false,
 10771              "hideZero": true,
 10772              "max": false,
 10773              "min": false,
 10774              "rightSide": false,
 10775              "show": true,
 10776              "sideWidth": null,
 10777              "sort": "current",
 10778              "sortDesc": true,
 10779              "total": false,
 10780              "values": false
 10781            },
 10782            "lines": true,
 10783            "linewidth": 1,
 10784            "links": [],
 10785            "nullPointMode": "null",
 10786            "options": {
 10787              "alertThreshold": true
 10788            },
 10789            "paceLength": 10,
 10790            "percentage": false,
 10791            "pluginVersion": "7.5.11",
 10792            "pointradius": 5,
 10793            "points": false,
 10794            "renderer": "flot",
 10795            "seriesOverrides": [],
 10796            "spaceLength": 10,
 10797            "stack": false,
 10798            "steppedLine": false,
 10799            "targets": [
 10800              {
 10801                "exemplar": true,
 10802                "expr": "sum(increase(tikv_cdc_scan_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (type, instance)",
 10803                "format": "time_series",
 10804                "hide": false,
 10805                "interval": "",
 10806                "intervalFactor": 1,
 10807                "legendFormat": "{{instance}}",
 10808                "refId": "A"
 10809              }
 10810            ],
 10811            "thresholds": [],
 10812            "timeFrom": null,
 10813            "timeRegions": [],
 10814            "timeShift": null,
 10815            "title": "TiKV scan tasks/m",
 10816            "tooltip": {
 10817              "shared": true,
 10818              "sort": 0,
 10819              "value_type": "individual"
 10820            },
 10821            "type": "graph",
 10822            "xaxis": {
 10823              "buckets": null,
 10824              "mode": "time",
 10825              "name": null,
 10826              "show": true,
 10827              "values": []
 10828            },
 10829            "yaxes": [
 10830              {
 10831                "format": "none",
 10832                "label": null,
 10833                "logBase": 1,
 10834                "max": null,
 10835                "min": null,
 10836                "show": true
 10837              },
 10838              {
 10839                "format": "none",
 10840                "label": null,
 10841                "logBase": 1,
 10842                "max": null,
 10843                "min": null,
 10844                "show": true
 10845              }
 10846            ],
 10847            "yaxis": {
 10848              "align": false,
 10849              "alignLevel": null
 10850            }
 10851          }
 10852        ],
 10853        "title": "Scheduler",
 10854        "type": "row"
 10855      },
 10856      {
 10857        "collapsed": true,
 10858        "datasource": null,
 10859        "gridPos": {
 10860          "h": 1,
 10861          "w": 24,
 10862          "x": 0,
 10863          "y": 9
 10864        },
 10865        "id": 677,
 10866        "panels": [
 10867          {
 10868            "aliasColors": {},
 10869            "bars": false,
 10870            "dashLength": 10,
 10871            "dashes": false,
 10872            "datasource": "${DS_TEST-CLUSTER}",
 10873            "decimals": 1,
 10874            "description": "The ID of the slowest table",
 10875            "editable": true,
 10876            "error": false,
 10877            "fieldConfig": {
 10878              "defaults": {},
 10879              "overrides": []
 10880            },
 10881            "fill": 0,
 10882            "fillGradient": 0,
 10883            "grid": {},
 10884            "gridPos": {
 10885              "h": 6,
 10886              "w": 12,
 10887              "x": 0,
 10888              "y": 10
 10889            },
 10890            "hiddenSeries": false,
 10891            "id": 689,
 10892            "legend": {
 10893              "alignAsTable": false,
 10894              "avg": false,
 10895              "current": true,
 10896              "max": false,
 10897              "min": false,
 10898              "rightSide": false,
 10899              "show": true,
 10900              "sideWidth": null,
 10901              "sort": null,
 10902              "sortDesc": null,
 10903              "total": false,
 10904              "values": true
 10905            },
 10906            "lines": false,
 10907            "linewidth": 1,
 10908            "links": [],
 10909            "nullPointMode": "null",
 10910            "options": {
 10911              "alertThreshold": true
 10912            },
 10913            "paceLength": 10,
 10914            "percentage": false,
 10915            "pluginVersion": "7.5.11",
 10916            "pointradius": 1,
 10917            "points": true,
 10918            "renderer": "flot",
 10919            "seriesOverrides": [],
 10920            "spaceLength": 10,
 10921            "stack": false,
 10922            "steppedLine": false,
 10923            "targets": [
 10924              {
 10925                "exemplar": true,
 10926                "expr": "sum(ticdc_scheduler_slow_table_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)",
 10927                "format": "time_series",
 10928                "hide": false,
 10929                "interval": "",
 10930                "intervalFactor": 2,
 10931                "legendFormat": "{{namespace}}-{{changefeed}}",
 10932                "refId": "A",
 10933                "step": 10
 10934              }
 10935            ],
 10936            "thresholds": [],
 10937            "timeFrom": null,
 10938            "timeRegions": [],
 10939            "timeShift": null,
 10940            "title": "Slowest table ID",
 10941            "tooltip": {
 10942              "msResolution": false,
 10943              "shared": true,
 10944              "sort": 0,
 10945              "value_type": "individual"
 10946            },
 10947            "type": "graph",
 10948            "xaxis": {
 10949              "buckets": null,
 10950              "mode": "time",
 10951              "name": null,
 10952              "show": true,
 10953              "values": []
 10954            },
 10955            "yaxes": [
 10956              {
 10957                "format": "none",
 10958                "label": null,
 10959                "logBase": 1,
 10960                "max": null,
 10961                "min": "0",
 10962                "show": true
 10963              },
 10964              {
 10965                "format": "none",
 10966                "label": null,
 10967                "logBase": 1,
 10968                "max": null,
 10969                "min": null,
 10970                "show": false
 10971              }
 10972            ],
 10973            "yaxis": {
 10974              "align": false,
 10975              "alignLevel": null
 10976            }
 10977          },
 10978          {
 10979            "aliasColors": {},
 10980            "bars": false,
 10981            "dashLength": 10,
 10982            "dashes": false,
 10983            "datasource": "${DS_TEST-CLUSTER}",
 10984            "decimals": 1,
 10985            "description": "The number of regions captured by the slowest table.",
 10986            "editable": true,
 10987            "error": false,
 10988            "fieldConfig": {
 10989              "defaults": {},
 10990              "overrides": []
 10991            },
 10992            "fill": 0,
 10993            "fillGradient": 0,
 10994            "grid": {},
 10995            "gridPos": {
 10996              "h": 6,
 10997              "w": 12,
 10998              "x": 12,
 10999              "y": 10
 11000            },
 11001            "hiddenSeries": false,
 11002            "id": 691,
 11003            "legend": {
 11004              "alignAsTable": false,
 11005              "avg": false,
 11006              "current": true,
 11007              "max": false,
 11008              "min": false,
 11009              "rightSide": false,
 11010              "show": true,
 11011              "sideWidth": null,
 11012              "sort": null,
 11013              "sortDesc": null,
 11014              "total": false,
 11015              "values": true
 11016            },
 11017            "lines": false,
 11018            "linewidth": 1,
 11019            "links": [],
 11020            "nullPointMode": "null",
 11021            "options": {
 11022              "alertThreshold": true
 11023            },
 11024            "paceLength": 10,
 11025            "percentage": false,
 11026            "pluginVersion": "7.5.11",
 11027            "pointradius": 1,
 11028            "points": true,
 11029            "renderer": "flot",
 11030            "seriesOverrides": [],
 11031            "spaceLength": 10,
 11032            "stack": false,
 11033            "steppedLine": false,
 11034            "targets": [
 11035              {
 11036                "exemplar": true,
 11037                "expr": "sum(ticdc_scheduler_slow_table_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)",
 11038                "format": "time_series",
 11039                "hide": false,
 11040                "interval": "",
 11041                "intervalFactor": 2,
 11042                "legendFormat": "{{namespace}}-{{changefeed}}",
 11043                "refId": "A",
 11044                "step": 10
 11045              }
 11046            ],
 11047            "thresholds": [],
 11048            "timeFrom": null,
 11049            "timeRegions": [],
 11050            "timeShift": null,
 11051            "title": "Slowest table region count",
 11052            "tooltip": {
 11053              "msResolution": false,
 11054              "shared": true,
 11055              "sort": 0,
 11056              "value_type": "individual"
 11057            },
 11058            "type": "graph",
 11059            "xaxis": {
 11060              "buckets": null,
 11061              "mode": "time",
 11062              "name": null,
 11063              "show": true,
 11064              "values": []
 11065            },
 11066            "yaxes": [
 11067              {
 11068                "format": "none",
 11069                "label": null,
 11070                "logBase": 1,
 11071                "max": null,
 11072                "min": "0",
 11073                "show": true
 11074              },
 11075              {
 11076                "format": "none",
 11077                "label": null,
 11078                "logBase": 1,
 11079                "max": null,
 11080                "min": null,
 11081                "show": false
 11082              }
 11083            ],
 11084            "yaxis": {
 11085              "align": false,
 11086              "alignLevel": null
 11087            }
 11088          },
 11089          {
 11090            "aliasColors": {},
 11091            "bars": false,
 11092            "cacheTimeout": null,
 11093            "dashLength": 10,
 11094            "dashes": false,
 11095            "datasource": "${DS_TEST-CLUSTER}",
 11096            "description": "Checkpoint ts of the slowest table each stage.",
 11097            "fieldConfig": {
 11098              "defaults": {
 11099                "unit": "dateTimeAsIso"
 11100              },
 11101              "overrides": []
 11102            },
 11103            "fill": 1,
 11104            "fillGradient": 0,
 11105            "gridPos": {
 11106              "h": 6,
 11107              "w": 12,
 11108              "x": 0,
 11109              "y": 16
 11110            },
 11111            "hiddenSeries": false,
 11112            "id": 693,
 11113            "legend": {
 11114              "avg": false,
 11115              "current": false,
 11116              "max": false,
 11117              "min": false,
 11118              "show": true,
 11119              "total": false,
 11120              "values": false
 11121            },
 11122            "lines": true,
 11123            "linewidth": 1,
 11124            "links": [],
 11125            "nullPointMode": "null",
 11126            "options": {
 11127              "alertThreshold": true
 11128            },
 11129            "percentage": false,
 11130            "pluginVersion": "7.5.11",
 11131            "pointradius": 2,
 11132            "points": false,
 11133            "renderer": "flot",
 11134            "seriesOverrides": [],
 11135            "spaceLength": 10,
 11136            "stack": false,
 11137            "steppedLine": false,
 11138            "targets": [
 11139              {
 11140                "exemplar": true,
 11141                "expr": "max(ticdc_owner_current_pd_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})",
 11142                "format": "time_series",
 11143                "hide": false,
 11144                "interval": "",
 11145                "intervalFactor": 1,
 11146                "legendFormat": "approximate current time (s)",
 11147                "refId": "A"
 11148              },
 11149              {
 11150                "exemplar": true,
 11151                "expr": "sum(ticdc_scheduler_slow_table_stage_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, stage)",
 11152                "format": "time_series",
 11153                "hide": false,
 11154                "interval": "",
 11155                "intervalFactor": 1,
 11156                "legendFormat": "{{namespace}}-{{changefeed}}-{{stage}}",
 11157                "refId": "B"
 11158              }
 11159            ],
 11160            "thresholds": [],
 11161            "timeFrom": null,
 11162            "timeRegions": [],
 11163            "timeShift": null,
 11164            "title": "Slow table checkpoint ts",
 11165            "tooltip": {
 11166              "shared": true,
 11167              "sort": 0,
 11168              "value_type": "individual"
 11169            },
 11170            "type": "graph",
 11171            "xaxis": {
 11172              "buckets": null,
 11173              "mode": "time",
 11174              "name": null,
 11175              "show": true,
 11176              "values": []
 11177            },
 11178            "yaxes": [
 11179              {
 11180                "format": "dateTimeAsIso",
 11181                "label": null,
 11182                "logBase": 1,
 11183                "max": null,
 11184                "min": null,
 11185                "show": true
 11186              },
 11187              {
 11188                "format": "short",
 11189                "label": null,
 11190                "logBase": 1,
 11191                "max": null,
 11192                "min": null,
 11193                "show": true
 11194              }
 11195            ],
 11196            "yaxis": {
 11197              "align": false,
 11198              "alignLevel": null
 11199            }
 11200          },
 11201          {
 11202            "aliasColors": {},
 11203            "bars": false,
 11204            "cacheTimeout": null,
 11205            "dashLength": 10,
 11206            "dashes": false,
 11207            "datasource": "${DS_TEST-CLUSTER}",
 11208            "description": "Resolved ts of the slowest table each stage.",
 11209            "fieldConfig": {
 11210              "defaults": {
 11211                "unit": "dateTimeAsIso"
 11212              },
 11213              "overrides": []
 11214            },
 11215            "fill": 1,
 11216            "fillGradient": 0,
 11217            "gridPos": {
 11218              "h": 6,
 11219              "w": 12,
 11220              "x": 12,
 11221              "y": 16
 11222            },
 11223            "hiddenSeries": false,
 11224            "id": 695,
 11225            "legend": {
 11226              "avg": false,
 11227              "current": false,
 11228              "max": false,
 11229              "min": false,
 11230              "show": true,
 11231              "total": false,
 11232              "values": false
 11233            },
 11234            "lines": true,
 11235            "linewidth": 1,
 11236            "links": [],
 11237            "nullPointMode": "null",
 11238            "options": {
 11239              "alertThreshold": true
 11240            },
 11241            "percentage": false,
 11242            "pluginVersion": "7.5.11",
 11243            "pointradius": 2,
 11244            "points": false,
 11245            "renderer": "flot",
 11246            "seriesOverrides": [],
 11247            "spaceLength": 10,
 11248            "stack": false,
 11249            "steppedLine": false,
 11250            "targets": [
 11251              {
 11252                "exemplar": true,
 11253                "expr": "max(ticdc_owner_current_pd_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})",
 11254                "format": "time_series",
 11255                "hide": false,
 11256                "interval": "",
 11257                "intervalFactor": 1,
 11258                "legendFormat": "approximate current time (s)",
 11259                "refId": "A"
 11260              },
 11261              {
 11262                "exemplar": true,
 11263                "expr": "sum(ticdc_scheduler_slow_table_stage_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, stage)",
 11264                "format": "time_series",
 11265                "hide": false,
 11266                "interval": "",
 11267                "intervalFactor": 1,
 11268                "legendFormat": "{{namespace}}-{{changefeed}}-{{stage}}",
 11269                "refId": "B"
 11270              }
 11271            ],
 11272            "thresholds": [],
 11273            "timeFrom": null,
 11274            "timeRegions": [],
 11275            "timeShift": null,
 11276            "title": "Slow table resolved ts",
 11277            "tooltip": {
 11278              "shared": true,
 11279              "sort": 0,
 11280              "value_type": "individual"
 11281            },
 11282            "type": "graph",
 11283            "xaxis": {
 11284              "buckets": null,
 11285              "mode": "time",
 11286              "name": null,
 11287              "show": true,
 11288              "values": []
 11289            },
 11290            "yaxes": [
 11291              {
 11292                "format": "dateTimeAsIso",
 11293                "label": null,
 11294                "logBase": 1,
 11295                "max": null,
 11296                "min": null,
 11297                "show": true
 11298              },
 11299              {
 11300                "format": "short",
 11301                "label": null,
 11302                "logBase": 1,
 11303                "max": null,
 11304                "min": null,
 11305                "show": true
 11306              }
 11307            ],
 11308            "yaxis": {
 11309              "align": false,
 11310              "alignLevel": null
 11311            }
 11312          },
 11313          {
 11314            "aliasColors": {},
 11315            "bars": false,
 11316            "dashLength": 10,
 11317            "dashes": false,
 11318            "datasource": "${DS_TEST-CLUSTER}",
 11319            "description": "The details of changefeed lag including resolved ts and checkpoint ts in different stages.",
 11320            "fieldConfig": {
 11321              "defaults": {
 11322                "links": []
 11323              },
 11324              "overrides": []
 11325            },
 11326            "fill": 1,
 11327            "fillGradient": 0,
 11328            "gridPos": {
 11329              "h": 6,
 11330              "w": 24,
 11331              "x": 0,
 11332              "y": 22
 11333            },
 11334            "hiddenSeries": false,
 11335            "id": 699,
 11336            "legend": {
 11337              "alignAsTable": false,
 11338              "avg": false,
 11339              "current": true,
 11340              "max": false,
 11341              "min": false,
 11342              "rightSide": false,
 11343              "show": true,
 11344              "total": false,
 11345              "values": true
 11346            },
 11347            "lines": true,
 11348            "linewidth": 1,
 11349            "links": [],
 11350            "nullPointMode": "null",
 11351            "options": {
 11352              "alertThreshold": true
 11353            },
 11354            "paceLength": 10,
 11355            "percentage": false,
 11356            "pluginVersion": "7.5.11",
 11357            "pointradius": 2,
 11358            "points": false,
 11359            "renderer": "flot",
 11360            "seriesOverrides": [],
 11361            "spaceLength": 10,
 11362            "stack": false,
 11363            "steppedLine": false,
 11364            "targets": [
 11365              {
 11366                "exemplar": true,
 11367                "expr": "max(ticdc_scheduler_slow_table_stage_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, stage)",
 11368                "format": "time_series",
 11369                "interval": "",
 11370                "intervalFactor": 1,
 11371                "legendFormat": "{{namespace}}-{{changefeed}}-{{stage}}",
 11372                "refId": "A"
 11373              },
 11374              {
 11375                "exemplar": true,
 11376                "expr": "max(tikv_cdc_min_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000)",
 11377                "hide": false,
 11378                "interval": "",
 11379                "legendFormat": "tikv",
 11380                "refId": "B"
 11381              },
 11382              {
 11383                "exemplar": true,
 11384                "expr": "max(ticdc_scheduler_slow_table_stage_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", stage=\"sink\"}) by (namespace,changefeed)",
 11385                "hide": false,
 11386                "interval": "",
 11387                "legendFormat": "sink-checkpoint",
 11388                "refId": "C"
 11389              }
 11390            ],
 11391            "thresholds": [],
 11392            "timeFrom": null,
 11393            "timeRegions": [],
 11394            "timeShift": null,
 11395            "title": "Slow table lag details",
 11396            "tooltip": {
 11397              "shared": true,
 11398              "sort": 0,
 11399              "value_type": "individual"
 11400            },
 11401            "type": "graph",
 11402            "xaxis": {
 11403              "buckets": null,
 11404              "mode": "time",
 11405              "name": null,
 11406              "show": true,
 11407              "values": []
 11408            },
 11409            "yaxes": [
 11410              {
 11411                "format": "s",
 11412                "label": null,
 11413                "logBase": 1,
 11414                "max": null,
 11415                "min": "0",
 11416                "show": true
 11417              },
 11418              {
 11419                "format": "short",
 11420                "label": null,
 11421                "logBase": 1,
 11422                "max": null,
 11423                "min": null,
 11424                "show": true
 11425              }
 11426            ],
 11427            "yaxis": {
 11428              "align": false,
 11429              "alignLevel": null
 11430            }
 11431          },
 11432          {
 11433            "aliasColors": {},
 11434            "bars": false,
 11435            "dashLength": 10,
 11436            "dashes": false,
 11437            "datasource": "${DS_TEST-CLUSTER}",
 11438            "description": "Lag distribution of the slowest table in different stages.",
 11439            "fieldConfig": {
 11440              "defaults": {},
 11441              "overrides": []
 11442            },
 11443            "fill": 1,
 11444            "fillGradient": 0,
 11445            "gridPos": {
 11446              "h": 6,
 11447              "w": 24,
 11448              "x": 0,
 11449              "y": 28
 11450            },
 11451            "hiddenSeries": false,
 11452            "id": 703,
 11453            "legend": {
 11454              "alignAsTable": false,
 11455              "avg": false,
 11456              "current": true,
 11457              "max": true,
 11458              "min": false,
 11459              "rightSide": false,
 11460              "show": true,
 11461              "total": false,
 11462              "values": true
 11463            },
 11464            "lines": true,
 11465            "linewidth": 1,
 11466            "nullPointMode": "null",
 11467            "options": {
 11468              "alertThreshold": true
 11469            },
 11470            "percentage": false,
 11471            "pluginVersion": "7.5.11",
 11472            "pointradius": 2,
 11473            "points": false,
 11474            "renderer": "flot",
 11475            "seriesOverrides": [],
 11476            "spaceLength": 10,
 11477            "stack": false,
 11478            "steppedLine": false,
 11479            "targets": [
 11480              {
 11481                "exemplar": true,
 11482                "expr": "histogram_quantile(0.99, sum(rate(ticdc_scheduler_slow_table_stage_resolved_ts_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[100s])) by (le,namespace,changefeed,stage))",
 11483                "hide": false,
 11484                "interval": "",
 11485                "legendFormat": "{{namespace}}-{{changefeed}}-{{stage}}-p99",
 11486                "refId": "A"
 11487              },
 11488              {
 11489                "exemplar": true,
 11490                "expr": "histogram_quantile(0.99, sum(rate(ticdc_scheduler_slow_table_stage_checkpoint_ts_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", stage=\"sink\"}[100s])) by (le,namespace,changefeed,stage))",
 11491                "hide": false,
 11492                "interval": "",
 11493                "legendFormat": "{{namespace}}-{{changefeed}}-{{stage}}-checkpoint-p99",
 11494                "refId": "B"
 11495              }
 11496            ],
 11497            "thresholds": [],
 11498            "timeFrom": null,
 11499            "timeRegions": [],
 11500            "timeShift": null,
 11501            "title": "Slow table lag duration percentile",
 11502            "tooltip": {
 11503              "shared": true,
 11504              "sort": 0,
 11505              "value_type": "individual"
 11506            },
 11507            "type": "graph",
 11508            "xaxis": {
 11509              "buckets": null,
 11510              "mode": "time",
 11511              "name": null,
 11512              "show": true,
 11513              "values": []
 11514            },
 11515            "yaxes": [
 11516              {
 11517                "format": "s",
 11518                "label": null,
 11519                "logBase": 1,
 11520                "max": null,
 11521                "min": null,
 11522                "show": true
 11523              },
 11524              {
 11525                "format": "none",
 11526                "label": null,
 11527                "logBase": 1,
 11528                "max": null,
 11529                "min": null,
 11530                "show": true
 11531              }
 11532            ],
 11533            "yaxis": {
 11534              "align": false,
 11535              "alignLevel": null
 11536            }
 11537          }
 11538        ],
 11539        "title": "Slow Table",
 11540        "type": "row"
 11541      },
 11542      {
 11543        "collapsed": true,
 11544        "datasource": null,
 11545        "gridPos": {
 11546          "h": 1,
 11547          "w": 24,
 11548          "x": 0,
 11549          "y": 10
 11550        },
 11551        "id": 269,
 11552        "panels": [
 11553          {
 11554            "aliasColors": {},
 11555            "bars": false,
 11556            "dashLength": 10,
 11557            "dashes": false,
 11558            "datasource": "${DS_TEST-CLUSTER}",
 11559            "fieldConfig": {
 11560              "defaults": {},
 11561              "overrides": []
 11562            },
 11563            "fill": 1,
 11564            "fillGradient": 0,
 11565            "gridPos": {
 11566              "h": 6,
 11567              "w": 8,
 11568              "x": 0,
 11569              "y": 11
 11570            },
 11571            "hiddenSeries": false,
 11572            "id": 271,
 11573            "legend": {
 11574              "avg": false,
 11575              "current": false,
 11576              "max": false,
 11577              "min": false,
 11578              "show": true,
 11579              "total": false,
 11580              "values": false
 11581            },
 11582            "lines": true,
 11583            "linewidth": 1,
 11584            "nullPointMode": "null",
 11585            "options": {
 11586              "alertThreshold": true
 11587            },
 11588            "percentage": false,
 11589            "pluginVersion": "7.5.11",
 11590            "pointradius": 2,
 11591            "points": false,
 11592            "renderer": "flot",
 11593            "seriesOverrides": [],
 11594            "spaceLength": 10,
 11595            "stack": false,
 11596            "steppedLine": false,
 11597            "targets": [
 11598              {
 11599                "expr": "sum(ticdc_sorter_on_disk_data_size_gauge{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)",
 11600                "format": "time_series",
 11601                "intervalFactor": 1,
 11602                "legendFormat": "{{instance}}",
 11603                "refId": "A"
 11604              }
 11605            ],
 11606            "thresholds": [],
 11607            "timeFrom": null,
 11608            "timeRegions": [],
 11609            "timeShift": null,
 11610            "title": "On disk data size",
 11611            "tooltip": {
 11612              "shared": true,
 11613              "sort": 0,
 11614              "value_type": "individual"
 11615            },
 11616            "type": "graph",
 11617            "xaxis": {
 11618              "buckets": null,
 11619              "mode": "time",
 11620              "name": null,
 11621              "show": true,
 11622              "values": []
 11623            },
 11624            "yaxes": [
 11625              {
 11626                "format": "bytes",
 11627                "label": null,
 11628                "logBase": 1,
 11629                "max": null,
 11630                "min": "0",
 11631                "show": true
 11632              },
 11633              {
 11634                "format": "short",
 11635                "label": null,
 11636                "logBase": 1,
 11637                "max": null,
 11638                "min": null,
 11639                "show": true
 11640              }
 11641            ],
 11642            "yaxis": {
 11643              "align": false,
 11644              "alignLevel": null
 11645            }
 11646          },
 11647          {
 11648            "aliasColors": {},
 11649            "bars": false,
 11650            "dashLength": 10,
 11651            "dashes": false,
 11652            "datasource": "${DS_TEST-CLUSTER}",
 11653            "fieldConfig": {
 11654              "defaults": {},
 11655              "overrides": []
 11656            },
 11657            "fill": 1,
 11658            "fillGradient": 0,
 11659            "gridPos": {
 11660              "h": 6,
 11661              "w": 8,
 11662              "x": 8,
 11663              "y": 11
 11664            },
 11665            "hiddenSeries": false,
 11666            "id": 273,
 11667            "legend": {
 11668              "avg": false,
 11669              "current": false,
 11670              "max": false,
 11671              "min": false,
 11672              "show": true,
 11673              "total": false,
 11674              "values": false
 11675            },
 11676            "lines": true,
 11677            "linewidth": 1,
 11678            "links": [],
 11679            "nullPointMode": "null",
 11680            "options": {
 11681              "alertThreshold": true
 11682            },
 11683            "percentage": false,
 11684            "pluginVersion": "7.5.11",
 11685            "pointradius": 2,
 11686            "points": false,
 11687            "renderer": "flot",
 11688            "seriesOverrides": [
 11689              {
 11690                "alias": "/cache-hit.*/",
 11691                "yaxis": 2
 11692              }
 11693            ],
 11694            "spaceLength": 10,
 11695            "stack": false,
 11696            "steppedLine": false,
 11697            "targets": [
 11698              {
 11699                "expr": "sum(ticdc_sorter_in_memory_data_size_gauge{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)",
 11700                "format": "time_series",
 11701                "intervalFactor": 1,
 11702                "legendFormat": "{{instance}}",
 11703                "refId": "A"
 11704              },
 11705              {
 11706                "expr": "sum(ticdc_db_block_cache_access_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", type=\"hit\"}) by (instance) / sum(ticdc_db_block_cache_access_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)",
 11707                "format": "time_series",
 11708                "intervalFactor": 1,
 11709                "legendFormat": "cache-hit-{{instance}}",
 11710                "refId": "B"
 11711              }
 11712            ],
 11713            "thresholds": [],
 11714            "timeFrom": null,
 11715            "timeRegions": [],
 11716            "timeShift": null,
 11717            "title": "In-memory data size",
 11718            "tooltip": {
 11719              "shared": true,
 11720              "sort": 0,
 11721              "value_type": "individual"
 11722            },
 11723            "type": "graph",
 11724            "xaxis": {
 11725              "buckets": null,
 11726              "mode": "time",
 11727              "name": null,
 11728              "show": true,
 11729              "values": []
 11730            },
 11731            "yaxes": [
 11732              {
 11733                "format": "bytes",
 11734                "label": null,
 11735                "logBase": 1,
 11736                "max": null,
 11737                "min": "0",
 11738                "show": true
 11739              },
 11740              {
 11741                "format": "percentunit",
 11742                "label": null,
 11743                "logBase": 1,
 11744                "max": null,
 11745                "min": "0",
 11746                "show": true
 11747              }
 11748            ],
 11749            "yaxis": {
 11750              "align": false,
 11751              "alignLevel": null
 11752            }
 11753          },
 11754          {
 11755            "aliasColors": {},
 11756            "bars": false,
 11757            "dashLength": 10,
 11758            "dashes": false,
 11759            "datasource": "${DS_TEST-CLUSTER}",
 11760            "description": "The count of files of each levels",
 11761            "fieldConfig": {
 11762              "defaults": {},
 11763              "overrides": []
 11764            },
 11765            "fill": 1,
 11766            "fillGradient": 0,
 11767            "gridPos": {
 11768              "h": 6,
 11769              "w": 8,
 11770              "x": 16,
 11771              "y": 11
 11772            },
 11773            "hiddenSeries": false,
 11774            "id": 272,
 11775            "legend": {
 11776              "avg": false,
 11777              "current": true,
 11778              "max": true,
 11779              "min": false,
 11780              "show": true,
 11781              "total": false,
 11782              "values": true
 11783            },
 11784            "lines": true,
 11785            "linewidth": 1,
 11786            "links": [],
 11787            "nullPointMode": "null",
 11788            "options": {
 11789              "alertThreshold": true
 11790            },
 11791            "paceLength": 10,
 11792            "percentage": false,
 11793            "pluginVersion": "7.5.11",
 11794            "pointradius": 2,
 11795            "points": false,
 11796            "renderer": "flot",
 11797            "seriesOverrides": [],
 11798            "spaceLength": 10,
 11799            "stack": false,
 11800            "steppedLine": false,
 11801            "targets": [
 11802              {
 11803                "expr": "sum(ticdc_db_level_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, level)",
 11804                "format": "time_series",
 11805                "intervalFactor": 1,
 11806                "legendFormat": "{{instance}}-{{level}}",
 11807                "refId": "A"
 11808              }
 11809            ],
 11810            "thresholds": [],
 11811            "timeFrom": null,
 11812            "timeRegions": [],
 11813            "timeShift": null,
 11814            "title": "Level files",
 11815            "tooltip": {
 11816              "shared": true,
 11817              "sort": 0,
 11818              "value_type": "individual"
 11819            },
 11820            "type": "graph",
 11821            "xaxis": {
 11822              "buckets": null,
 11823              "mode": "time",
 11824              "name": null,
 11825              "show": true,
 11826              "values": []
 11827            },
 11828            "yaxes": [
 11829              {
 11830                "format": "short",
 11831                "label": null,
 11832                "logBase": 1,
 11833                "max": null,
 11834                "min": null,
 11835                "show": true
 11836              },
 11837              {
 11838                "format": "short",
 11839                "label": null,
 11840                "logBase": 1,
 11841                "max": null,
 11842                "min": null,
 11843                "show": true
 11844              }
 11845            ],
 11846            "yaxis": {
 11847              "align": false,
 11848              "alignLevel": null
 11849            }
 11850          },
 11851          {
 11852            "cards": {
 11853              "cardPadding": 0,
 11854              "cardRound": 0
 11855            },
 11856            "color": {
 11857              "cardColor": "#FF9830",
 11858              "colorScale": "linear",
 11859              "colorScheme": "interpolateSpectral",
 11860              "exponent": 0.5,
 11861              "min": 0,
 11862              "mode": "spectrum"
 11863            },
 11864            "dataFormat": "tsbuckets",
 11865            "datasource": "${DS_TEST-CLUSTER}",
 11866            "description": "The time of sorter write",
 11867            "fieldConfig": {
 11868              "defaults": {},
 11869              "overrides": []
 11870            },
 11871            "gridPos": {
 11872              "h": 6,
 11873              "w": 8,
 11874              "x": 0,
 11875              "y": 17
 11876            },
 11877            "heatmap": {},
 11878            "hideZeroBuckets": true,
 11879            "highlightCards": true,
 11880            "id": 274,
 11881            "legend": {
 11882              "alignAsTable": true,
 11883              "avg": false,
 11884              "current": true,
 11885              "max": true,
 11886              "min": false,
 11887              "rightSide": true,
 11888              "show": true,
 11889              "sort": "current",
 11890              "sortDesc": true,
 11891              "total": false,
 11892              "values": true
 11893            },
 11894            "links": [],
 11895            "maxPerRow": 3,
 11896            "repeatDirection": "h",
 11897            "reverseYBuckets": false,
 11898            "targets": [
 11899              {
 11900                "exemplar": true,
 11901                "expr": "sum(rate(ticdc_sorter_db_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)",
 11902                "format": "heatmap",
 11903                "instant": false,
 11904                "interval": "",
 11905                "intervalFactor": 2,
 11906                "legendFormat": "{{le}}",
 11907                "refId": "A"
 11908              }
 11909            ],
 11910            "title": "Write duration",
 11911            "tooltip": {
 11912              "show": true,
 11913              "showHistogram": true
 11914            },
 11915            "tooltipDecimals": 1,
 11916            "type": "heatmap",
 11917            "xAxis": {
 11918              "show": true
 11919            },
 11920            "xBucketNumber": null,
 11921            "xBucketSize": null,
 11922            "yAxis": {
 11923              "decimals": 1,
 11924              "format": "s",
 11925              "logBase": 1,
 11926              "max": null,
 11927              "min": null,
 11928              "show": true,
 11929              "splitFactor": null
 11930            },
 11931            "yBucketBound": "upper",
 11932            "yBucketNumber": null,
 11933            "yBucketSize": null
 11934          },
 11935          {
 11936            "aliasColors": {},
 11937            "bars": false,
 11938            "dashLength": 10,
 11939            "dashes": false,
 11940            "datasource": "${DS_TEST-CLUSTER}",
 11941            "fieldConfig": {
 11942              "defaults": {},
 11943              "overrides": []
 11944            },
 11945            "fill": 1,
 11946            "fillGradient": 0,
 11947            "gridPos": {
 11948              "h": 6,
 11949              "w": 8,
 11950              "x": 8,
 11951              "y": 17
 11952            },
 11953            "hiddenSeries": false,
 11954            "id": 288,
 11955            "legend": {
 11956              "alignAsTable": true,
 11957              "avg": true,
 11958              "current": true,
 11959              "max": true,
 11960              "min": false,
 11961              "rightSide": false,
 11962              "show": true,
 11963              "total": false,
 11964              "values": true
 11965            },
 11966            "lines": true,
 11967            "linewidth": 1,
 11968            "links": [],
 11969            "nullPointMode": "null",
 11970            "options": {
 11971              "alertThreshold": true
 11972            },
 11973            "percentage": false,
 11974            "pluginVersion": "7.5.11",
 11975            "pointradius": 2,
 11976            "points": false,
 11977            "renderer": "flot",
 11978            "seriesOverrides": [
 11979              {
 11980                "alias": "/.*amplification/",
 11981                "yaxis": 2
 11982              }
 11983            ],
 11984            "spaceLength": 10,
 11985            "stack": false,
 11986            "steppedLine": false,
 11987            "targets": [
 11988              {
 11989                "exemplar": true,
 11990                "expr": "sum(rate(ticdc_sorter_db_write_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)",
 11991                "format": "time_series",
 11992                "hide": false,
 11993                "interval": "",
 11994                "intervalFactor": 1,
 11995                "legendFormat": "{{instance}}-sorter",
 11996                "refId": "A"
 11997              },
 11998              {
 11999                "exemplar": true,
 12000                "expr": "sum(rate(ticdc_sorter_db_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance) / sum(rate(ticdc_sorter_db_write_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)",
 12001                "format": "time_series",
 12002                "hide": false,
 12003                "interval": "",
 12004                "intervalFactor": 1,
 12005                "legendFormat": "{{instance}}-amplification",
 12006                "refId": "B"
 12007              },
 12008              {
 12009                "exemplar": true,
 12010                "expr": "sum(rate(ticdc_sorter_db_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)",
 12011                "format": "time_series",
 12012                "hide": false,
 12013                "interval": "",
 12014                "intervalFactor": 1,
 12015                "legendFormat": "{{instance}}-disk",
 12016                "refId": "C"
 12017              }
 12018            ],
 12019            "thresholds": [],
 12020            "timeFrom": null,
 12021            "timeRegions": [],
 12022            "timeShift": null,
 12023            "title": "Write speed",
 12024            "tooltip": {
 12025              "shared": true,
 12026              "sort": 0,
 12027              "value_type": "individual"
 12028            },
 12029            "type": "graph",
 12030            "xaxis": {
 12031              "buckets": null,
 12032              "mode": "time",
 12033              "name": null,
 12034              "show": true,
 12035              "values": []
 12036            },
 12037            "yaxes": [
 12038              {
 12039                "format": "bytes",
 12040                "label": null,
 12041                "logBase": 1,
 12042                "max": null,
 12043                "min": "0",
 12044                "show": true
 12045              },
 12046              {
 12047                "format": "percentunit",
 12048                "label": null,
 12049                "logBase": 10,
 12050                "max": null,
 12051                "min": null,
 12052                "show": true
 12053              }
 12054            ],
 12055            "yaxis": {
 12056              "align": false,
 12057              "alignLevel": null
 12058            }
 12059          },
 12060          {
 12061            "aliasColors": {},
 12062            "bars": false,
 12063            "dashLength": 10,
 12064            "dashes": false,
 12065            "datasource": "${DS_TEST-CLUSTER}",
 12066            "description": "CPU usage of LevelDB sorter",
 12067            "fieldConfig": {
 12068              "defaults": {
 12069                "links": []
 12070              },
 12071              "overrides": []
 12072            },
 12073            "fill": 1,
 12074            "fillGradient": 0,
 12075            "gridPos": {
 12076              "h": 6,
 12077              "w": 8,
 12078              "x": 16,
 12079              "y": 17
 12080            },
 12081            "hiddenSeries": false,
 12082            "id": 286,
 12083            "legend": {
 12084              "alignAsTable": true,
 12085              "avg": false,
 12086              "current": true,
 12087              "hideEmpty": true,
 12088              "max": true,
 12089              "min": false,
 12090              "rightSide": false,
 12091              "show": true,
 12092              "total": false,
 12093              "values": true
 12094            },
 12095            "lines": true,
 12096            "linewidth": 1,
 12097            "links": [],
 12098            "nullPointMode": "null",
 12099            "options": {
 12100              "alertThreshold": true
 12101            },
 12102            "paceLength": 10,
 12103            "percentage": false,
 12104            "pluginVersion": "7.5.11",
 12105            "pointradius": 2,
 12106            "points": false,
 12107            "renderer": "flot",
 12108            "seriesOverrides": [
 12109              {
 12110                "alias": "/.*sorter-[0-9]+/",
 12111                "yaxis": 2
 12112              }
 12113            ],
 12114            "spaceLength": 10,
 12115            "stack": false,
 12116            "steppedLine": false,
 12117            "targets": [
 12118              {
 12119                "expr": "sum(rate(ticdc_actor_worker_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", name=~\"sorter.*\"}[1m])) by (name, instance)",
 12120                "format": "time_series",
 12121                "intervalFactor": 1,
 12122                "legendFormat": "{{instance}}-{{name}}",
 12123                "refId": "A"
 12124              }
 12125            ],
 12126            "thresholds": [],
 12127            "timeFrom": null,
 12128            "timeRegions": [],
 12129            "timeShift": null,
 12130            "title": "CPU usage",
 12131            "tooltip": {
 12132              "shared": true,
 12133              "sort": 0,
 12134              "value_type": "individual"
 12135            },
 12136            "type": "graph",
 12137            "xaxis": {
 12138              "buckets": null,
 12139              "mode": "time",
 12140              "name": null,
 12141              "show": true,
 12142              "values": []
 12143            },
 12144            "yaxes": [
 12145              {
 12146                "format": "percentunit",
 12147                "label": null,
 12148                "logBase": 1,
 12149                "max": null,
 12150                "min": null,
 12151                "show": true
 12152              },
 12153              {
 12154                "format": "percentunit",
 12155                "label": null,
 12156                "logBase": 1,
 12157                "max": "1.2",
 12158                "min": null,
 12159                "show": true
 12160              }
 12161            ],
 12162            "yaxis": {
 12163              "align": false,
 12164              "alignLevel": null
 12165            }
 12166          },
 12167          {
 12168            "cards": {
 12169              "cardPadding": 0,
 12170              "cardRound": 0
 12171            },
 12172            "color": {
 12173              "cardColor": "#FF9830",
 12174              "colorScale": "linear",
 12175              "colorScheme": "interpolateSpectral",
 12176              "exponent": 0.5,
 12177              "min": 0,
 12178              "mode": "spectrum"
 12179            },
 12180            "dataFormat": "tsbuckets",
 12181            "datasource": "${DS_TEST-CLUSTER}",
 12182            "description": "The time of sorter write",
 12183            "fieldConfig": {
 12184              "defaults": {},
 12185              "overrides": []
 12186            },
 12187            "gridPos": {
 12188              "h": 6,
 12189              "w": 8,
 12190              "x": 0,
 12191              "y": 23
 12192            },
 12193            "heatmap": {},
 12194            "hideZeroBuckets": true,
 12195            "highlightCards": true,
 12196            "id": 277,
 12197            "legend": {
 12198              "alignAsTable": true,
 12199              "avg": false,
 12200              "current": true,
 12201              "max": true,
 12202              "min": false,
 12203              "rightSide": true,
 12204              "show": true,
 12205              "sort": "current",
 12206              "sortDesc": true,
 12207              "total": false,
 12208              "values": true
 12209            },
 12210            "links": [],
 12211            "maxPerRow": 3,
 12212            "repeatDirection": "h",
 12213            "reverseYBuckets": false,
 12214            "targets": [
 12215              {
 12216                "exemplar": true,
 12217                "expr": "sum(rate(ticdc_sorter_db_write_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)",
 12218                "format": "heatmap",
 12219                "instant": false,
 12220                "interval": "",
 12221                "intervalFactor": 2,
 12222                "legendFormat": "{{le}}",
 12223                "refId": "A"
 12224              }
 12225            ],
 12226            "title": "Write bytes",
 12227            "tooltip": {
 12228              "show": true,
 12229              "showHistogram": true
 12230            },
 12231            "tooltipDecimals": 1,
 12232            "type": "heatmap",
 12233            "xAxis": {
 12234              "show": true
 12235            },
 12236            "xBucketNumber": null,
 12237            "xBucketSize": null,
 12238            "yAxis": {
 12239              "decimals": 1,
 12240              "format": "bytes",
 12241              "logBase": 1,
 12242              "max": null,
 12243              "min": null,
 12244              "show": true,
 12245              "splitFactor": null
 12246            },
 12247            "yBucketBound": "upper",
 12248            "yBucketNumber": null,
 12249            "yBucketSize": null
 12250          },
 12251          {
 12252            "aliasColors": {},
 12253            "bars": false,
 12254            "dashLength": 10,
 12255            "dashes": false,
 12256            "datasource": "${DS_TEST-CLUSTER}",
 12257            "fieldConfig": {
 12258              "defaults": {},
 12259              "overrides": []
 12260            },
 12261            "fill": 1,
 12262            "fillGradient": 0,
 12263            "gridPos": {
 12264              "h": 6,
 12265              "w": 8,
 12266              "x": 8,
 12267              "y": 23
 12268            },
 12269            "hiddenSeries": false,
 12270            "id": 279,
 12271            "legend": {
 12272              "alignAsTable": true,
 12273              "avg": true,
 12274              "current": true,
 12275              "max": true,
 12276              "min": false,
 12277              "rightSide": false,
 12278              "show": true,
 12279              "total": false,
 12280              "values": true
 12281            },
 12282            "lines": true,
 12283            "linewidth": 1,
 12284            "links": [],
 12285            "nullPointMode": "null",
 12286            "options": {
 12287              "alertThreshold": true
 12288            },
 12289            "percentage": false,
 12290            "pluginVersion": "7.5.11",
 12291            "pointradius": 2,
 12292            "points": false,
 12293            "renderer": "flot",
 12294            "seriesOverrides": [],
 12295            "spaceLength": 10,
 12296            "stack": false,
 12297            "steppedLine": false,
 12298            "targets": [
 12299              {
 12300                "exemplar": true,
 12301                "expr": "sum(rate(ticdc_sorter_db_write_bytes_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)",
 12302                "format": "time_series",
 12303                "hide": false,
 12304                "interval": "",
 12305                "intervalFactor": 1,
 12306                "legendFormat": "{{instance}}-sorter",
 12307                "refId": "A"
 12308              }
 12309            ],
 12310            "thresholds": [],
 12311            "timeFrom": null,
 12312            "timeRegions": [],
 12313            "timeShift": null,
 12314            "title": "Write OPS",
 12315            "tooltip": {
 12316              "shared": true,
 12317              "sort": 0,
 12318              "value_type": "individual"
 12319            },
 12320            "type": "graph",
 12321            "xaxis": {
 12322              "buckets": null,
 12323              "mode": "time",
 12324              "name": null,
 12325              "show": true,
 12326              "values": []
 12327            },
 12328            "yaxes": [
 12329              {
 12330                "format": "ops",
 12331                "label": null,
 12332                "logBase": 1,
 12333                "max": null,
 12334                "min": "0",
 12335                "show": true
 12336              },
 12337              {
 12338                "format": "percentunit",
 12339                "label": null,
 12340                "logBase": 10,
 12341                "max": null,
 12342                "min": null,
 12343                "show": true
 12344              }
 12345            ],
 12346            "yaxis": {
 12347              "align": false,
 12348              "alignLevel": null
 12349            }
 12350          },
 12351          {
 12352            "aliasColors": {},
 12353            "bars": false,
 12354            "dashLength": 10,
 12355            "dashes": false,
 12356            "datasource": "${DS_TEST-CLUSTER}",
 12357            "description": "The count and duration of write delay",
 12358            "fieldConfig": {
 12359              "defaults": {
 12360                "links": []
 12361              },
 12362              "overrides": []
 12363            },
 12364            "fill": 1,
 12365            "fillGradient": 0,
 12366            "gridPos": {
 12367              "h": 6,
 12368              "w": 8,
 12369              "x": 16,
 12370              "y": 23
 12371            },
 12372            "hiddenSeries": false,
 12373            "id": 275,
 12374            "legend": {
 12375              "alignAsTable": false,
 12376              "avg": false,
 12377              "current": true,
 12378              "max": true,
 12379              "min": false,
 12380              "rightSide": false,
 12381              "show": true,
 12382              "total": false,
 12383              "values": true
 12384            },
 12385            "lines": true,
 12386            "linewidth": 1,
 12387            "links": [],
 12388            "nullPointMode": "null",
 12389            "options": {
 12390              "alertThreshold": true
 12391            },
 12392            "paceLength": 10,
 12393            "percentage": false,
 12394            "pluginVersion": "7.5.11",
 12395            "pointradius": 2,
 12396            "points": false,
 12397            "renderer": "flot",
 12398            "seriesOverrides": [
 12399              {
 12400                "alias": "/.*count.*/",
 12401                "yaxis": 2
 12402              }
 12403            ],
 12404            "spaceLength": 10,
 12405            "stack": false,
 12406            "steppedLine": false,
 12407            "targets": [
 12408              {
 12409                "exemplar": true,
 12410                "expr": "sum(rate(ticdc_db_write_delay_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)",
 12411                "format": "time_series",
 12412                "interval": "",
 12413                "intervalFactor": 1,
 12414                "legendFormat": "{{instance}}-duration",
 12415                "refId": "A"
 12416              },
 12417              {
 12418                "exemplar": true,
 12419                "expr": "sum(rate(ticdc_db_write_delay_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)",
 12420                "format": "time_series",
 12421                "hide": false,
 12422                "interval": "",
 12423                "intervalFactor": 1,
 12424                "legendFormat": "{{instance}}-count",
 12425                "refId": "B"
 12426              }
 12427            ],
 12428            "thresholds": [],
 12429            "timeFrom": null,
 12430            "timeRegions": [],
 12431            "timeShift": null,
 12432            "title": "Write delay",
 12433            "tooltip": {
 12434              "shared": true,
 12435              "sort": 0,
 12436              "value_type": "individual"
 12437            },
 12438            "type": "graph",
 12439            "xaxis": {
 12440              "buckets": null,
 12441              "mode": "time",
 12442              "name": null,
 12443              "show": true,
 12444              "values": []
 12445            },
 12446            "yaxes": [
 12447              {
 12448                "format": "dtdurations",
 12449                "label": null,
 12450                "logBase": 1,
 12451                "max": null,
 12452                "min": null,
 12453                "show": true
 12454              },
 12455              {
 12456                "format": "short",
 12457                "label": null,
 12458                "logBase": 1,
 12459                "max": null,
 12460                "min": null,
 12461                "show": true
 12462              }
 12463            ],
 12464            "yaxis": {
 12465              "align": false,
 12466              "alignLevel": null
 12467            }
 12468          },
 12469          {
 12470            "cards": {
 12471              "cardPadding": 0,
 12472              "cardRound": 0
 12473            },
 12474            "color": {
 12475              "cardColor": "#FF9830",
 12476              "colorScale": "linear",
 12477              "colorScheme": "interpolateSpectral",
 12478              "exponent": 0.5,
 12479              "min": 0,
 12480              "mode": "spectrum"
 12481            },
 12482            "dataFormat": "tsbuckets",
 12483            "datasource": "${DS_TEST-CLUSTER}",
 12484            "description": "The time of sorter iterator read",
 12485            "fieldConfig": {
 12486              "defaults": {},
 12487              "overrides": []
 12488            },
 12489            "gridPos": {
 12490              "h": 6,
 12491              "w": 8,
 12492              "x": 0,
 12493              "y": 29
 12494            },
 12495            "heatmap": {},
 12496            "hideZeroBuckets": true,
 12497            "highlightCards": true,
 12498            "id": 278,
 12499            "legend": {
 12500              "alignAsTable": true,
 12501              "avg": false,
 12502              "current": true,
 12503              "max": true,
 12504              "min": false,
 12505              "rightSide": true,
 12506              "show": true,
 12507              "sort": "current",
 12508              "sortDesc": true,
 12509              "total": false,
 12510              "values": true
 12511            },
 12512            "links": [],
 12513            "maxPerRow": 3,
 12514            "repeatDirection": "h",
 12515            "reverseYBuckets": false,
 12516            "targets": [
 12517              {
 12518                "exemplar": true,
 12519                "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"first\"}[1m])) by (le)",
 12520                "format": "heatmap",
 12521                "instant": false,
 12522                "interval": "",
 12523                "intervalFactor": 2,
 12524                "legendFormat": "{{le}}",
 12525                "refId": "A"
 12526              }
 12527            ],
 12528            "title": "Read duration - First",
 12529            "tooltip": {
 12530              "show": true,
 12531              "showHistogram": true
 12532            },
 12533            "tooltipDecimals": 1,
 12534            "type": "heatmap",
 12535            "xAxis": {
 12536              "show": true
 12537            },
 12538            "xBucketNumber": null,
 12539            "xBucketSize": null,
 12540            "yAxis": {
 12541              "decimals": 1,
 12542              "format": "s",
 12543              "logBase": 1,
 12544              "max": null,
 12545              "min": null,
 12546              "show": true,
 12547              "splitFactor": null
 12548            },
 12549            "yBucketBound": "upper",
 12550            "yBucketNumber": null,
 12551            "yBucketSize": null
 12552          },
 12553          {
 12554            "cards": {
 12555              "cardPadding": 0,
 12556              "cardRound": 0
 12557            },
 12558            "color": {
 12559              "cardColor": "#FF9830",
 12560              "colorScale": "linear",
 12561              "colorScheme": "interpolateSpectral",
 12562              "exponent": 0.5,
 12563              "min": 0,
 12564              "mode": "spectrum"
 12565            },
 12566            "dataFormat": "tsbuckets",
 12567            "datasource": "${DS_TEST-CLUSTER}",
 12568            "description": "The time of sorter iterator read",
 12569            "fieldConfig": {
 12570              "defaults": {},
 12571              "overrides": []
 12572            },
 12573            "gridPos": {
 12574              "h": 6,
 12575              "w": 8,
 12576              "x": 8,
 12577              "y": 29
 12578            },
 12579            "heatmap": {},
 12580            "hideZeroBuckets": true,
 12581            "highlightCards": true,
 12582            "id": 282,
 12583            "legend": {
 12584              "alignAsTable": true,
 12585              "avg": false,
 12586              "current": true,
 12587              "max": true,
 12588              "min": false,
 12589              "rightSide": true,
 12590              "show": true,
 12591              "sort": "current",
 12592              "sortDesc": true,
 12593              "total": false,
 12594              "values": true
 12595            },
 12596            "links": [],
 12597            "maxPerRow": 3,
 12598            "repeatDirection": "h",
 12599            "reverseYBuckets": false,
 12600            "targets": [
 12601              {
 12602                "exemplar": true,
 12603                "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"next\"}[1m])) by (le)",
 12604                "format": "heatmap",
 12605                "instant": false,
 12606                "interval": "",
 12607                "intervalFactor": 2,
 12608                "legendFormat": "{{le}}",
 12609                "refId": "A"
 12610              }
 12611            ],
 12612            "title": "Read duration - Next",
 12613            "tooltip": {
 12614              "show": true,
 12615              "showHistogram": true
 12616            },
 12617            "tooltipDecimals": 1,
 12618            "type": "heatmap",
 12619            "xAxis": {
 12620              "show": true
 12621            },
 12622            "xBucketNumber": null,
 12623            "xBucketSize": null,
 12624            "yAxis": {
 12625              "decimals": 1,
 12626              "format": "s",
 12627              "logBase": 1,
 12628              "max": null,
 12629              "min": null,
 12630              "show": true,
 12631              "splitFactor": null
 12632            },
 12633            "yBucketBound": "upper",
 12634            "yBucketNumber": null,
 12635            "yBucketSize": null
 12636          },
 12637          {
 12638            "cards": {
 12639              "cardPadding": 0,
 12640              "cardRound": 0
 12641            },
 12642            "color": {
 12643              "cardColor": "#FF9830",
 12644              "colorScale": "linear",
 12645              "colorScheme": "interpolateSpectral",
 12646              "exponent": 0.5,
 12647              "min": 0,
 12648              "mode": "spectrum"
 12649            },
 12650            "dataFormat": "tsbuckets",
 12651            "datasource": "${DS_TEST-CLUSTER}",
 12652            "description": "The time of sorter iterator read",
 12653            "fieldConfig": {
 12654              "defaults": {},
 12655              "overrides": []
 12656            },
 12657            "gridPos": {
 12658              "h": 6,
 12659              "w": 8,
 12660              "x": 16,
 12661              "y": 29
 12662            },
 12663            "heatmap": {},
 12664            "hideZeroBuckets": true,
 12665            "highlightCards": true,
 12666            "id": 281,
 12667            "legend": {
 12668              "alignAsTable": true,
 12669              "avg": false,
 12670              "current": true,
 12671              "max": true,
 12672              "min": false,
 12673              "rightSide": true,
 12674              "show": true,
 12675              "sort": "current",
 12676              "sortDesc": true,
 12677              "total": false,
 12678              "values": true
 12679            },
 12680            "links": [],
 12681            "maxPerRow": 3,
 12682            "repeatDirection": "h",
 12683            "reverseYBuckets": false,
 12684            "targets": [
 12685              {
 12686                "exemplar": true,
 12687                "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"release\"}[1m])) by (le)",
 12688                "format": "heatmap",
 12689                "instant": false,
 12690                "interval": "",
 12691                "intervalFactor": 2,
 12692                "legendFormat": "{{le}}",
 12693                "refId": "A"
 12694              }
 12695            ],
 12696            "title": "Read duration - Release",
 12697            "tooltip": {
 12698              "show": true,
 12699              "showHistogram": true
 12700            },
 12701            "tooltipDecimals": 1,
 12702            "type": "heatmap",
 12703            "xAxis": {
 12704              "show": true
 12705            },
 12706            "xBucketNumber": null,
 12707            "xBucketSize": null,
 12708            "yAxis": {
 12709              "decimals": 1,
 12710              "format": "s",
 12711              "logBase": 1,
 12712              "max": null,
 12713              "min": null,
 12714              "show": true,
 12715              "splitFactor": null
 12716            },
 12717            "yBucketBound": "upper",
 12718            "yBucketNumber": null,
 12719            "yBucketSize": null
 12720          },
 12721          {
 12722            "aliasColors": {},
 12723            "bars": false,
 12724            "dashLength": 10,
 12725            "dashes": false,
 12726            "datasource": "${DS_TEST-CLUSTER}",
 12727            "fieldConfig": {
 12728              "defaults": {},
 12729              "overrides": []
 12730            },
 12731            "fill": 1,
 12732            "fillGradient": 0,
 12733            "gridPos": {
 12734              "h": 6,
 12735              "w": 8,
 12736              "x": 0,
 12737              "y": 35
 12738            },
 12739            "hiddenSeries": false,
 12740            "id": 280,
 12741            "legend": {
 12742              "alignAsTable": true,
 12743              "avg": true,
 12744              "current": true,
 12745              "max": true,
 12746              "min": false,
 12747              "rightSide": false,
 12748              "show": true,
 12749              "total": false,
 12750              "values": true
 12751            },
 12752            "lines": true,
 12753            "linewidth": 1,
 12754            "links": [],
 12755            "nullPointMode": "null",
 12756            "options": {
 12757              "alertThreshold": true
 12758            },
 12759            "percentage": false,
 12760            "pluginVersion": "7.5.11",
 12761            "pointradius": 2,
 12762            "points": false,
 12763            "renderer": "flot",
 12764            "seriesOverrides": [],
 12765            "spaceLength": 10,
 12766            "stack": false,
 12767            "steppedLine": false,
 12768            "targets": [
 12769              {
 12770                "exemplar": true,
 12771                "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"first\"}[1m])) by (instance)",
 12772                "format": "time_series",
 12773                "hide": false,
 12774                "interval": "",
 12775                "intervalFactor": 1,
 12776                "legendFormat": "{{instance}}-sorter",
 12777                "refId": "A"
 12778              }
 12779            ],
 12780            "thresholds": [],
 12781            "timeFrom": null,
 12782            "timeRegions": [],
 12783            "timeShift": null,
 12784            "title": "Read OPS - First",
 12785            "tooltip": {
 12786              "shared": true,
 12787              "sort": 0,
 12788              "value_type": "individual"
 12789            },
 12790            "type": "graph",
 12791            "xaxis": {
 12792              "buckets": null,
 12793              "mode": "time",
 12794              "name": null,
 12795              "show": true,
 12796              "values": []
 12797            },
 12798            "yaxes": [
 12799              {
 12800                "format": "ops",
 12801                "label": null,
 12802                "logBase": 1,
 12803                "max": null,
 12804                "min": "0",
 12805                "show": true
 12806              },
 12807              {
 12808                "format": "percentunit",
 12809                "label": null,
 12810                "logBase": 10,
 12811                "max": null,
 12812                "min": null,
 12813                "show": true
 12814              }
 12815            ],
 12816            "yaxis": {
 12817              "align": false,
 12818              "alignLevel": null
 12819            }
 12820          },
 12821          {
 12822            "aliasColors": {},
 12823            "bars": false,
 12824            "dashLength": 10,
 12825            "dashes": false,
 12826            "datasource": "${DS_TEST-CLUSTER}",
 12827            "fieldConfig": {
 12828              "defaults": {},
 12829              "overrides": []
 12830            },
 12831            "fill": 1,
 12832            "fillGradient": 0,
 12833            "gridPos": {
 12834              "h": 6,
 12835              "w": 8,
 12836              "x": 8,
 12837              "y": 35
 12838            },
 12839            "hiddenSeries": false,
 12840            "id": 283,
 12841            "legend": {
 12842              "alignAsTable": true,
 12843              "avg": true,
 12844              "current": true,
 12845              "max": true,
 12846              "min": false,
 12847              "rightSide": false,
 12848              "show": true,
 12849              "total": false,
 12850              "values": true
 12851            },
 12852            "lines": true,
 12853            "linewidth": 1,
 12854            "links": [],
 12855            "nullPointMode": "null",
 12856            "options": {
 12857              "alertThreshold": true
 12858            },
 12859            "percentage": false,
 12860            "pluginVersion": "7.5.11",
 12861            "pointradius": 2,
 12862            "points": false,
 12863            "renderer": "flot",
 12864            "seriesOverrides": [],
 12865            "spaceLength": 10,
 12866            "stack": false,
 12867            "steppedLine": false,
 12868            "targets": [
 12869              {
 12870                "exemplar": true,
 12871                "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"next\"}[1m])) by (instance)",
 12872                "format": "time_series",
 12873                "hide": false,
 12874                "interval": "",
 12875                "intervalFactor": 1,
 12876                "legendFormat": "{{instance}}-sorter",
 12877                "refId": "A"
 12878              }
 12879            ],
 12880            "thresholds": [],
 12881            "timeFrom": null,
 12882            "timeRegions": [],
 12883            "timeShift": null,
 12884            "title": "Read OPS - Next",
 12885            "tooltip": {
 12886              "shared": true,
 12887              "sort": 0,
 12888              "value_type": "individual"
 12889            },
 12890            "type": "graph",
 12891            "xaxis": {
 12892              "buckets": null,
 12893              "mode": "time",
 12894              "name": null,
 12895              "show": true,
 12896              "values": []
 12897            },
 12898            "yaxes": [
 12899              {
 12900                "format": "ops",
 12901                "label": null,
 12902                "logBase": 1,
 12903                "max": null,
 12904                "min": "0",
 12905                "show": true
 12906              },
 12907              {
 12908                "format": "percentunit",
 12909                "label": null,
 12910                "logBase": 10,
 12911                "max": null,
 12912                "min": null,
 12913                "show": true
 12914              }
 12915            ],
 12916            "yaxis": {
 12917              "align": false,
 12918              "alignLevel": null
 12919            }
 12920          },
 12921          {
 12922            "aliasColors": {},
 12923            "bars": false,
 12924            "dashLength": 10,
 12925            "dashes": false,
 12926            "datasource": "${DS_TEST-CLUSTER}",
 12927            "fieldConfig": {
 12928              "defaults": {},
 12929              "overrides": []
 12930            },
 12931            "fill": 1,
 12932            "fillGradient": 0,
 12933            "gridPos": {
 12934              "h": 6,
 12935              "w": 8,
 12936              "x": 16,
 12937              "y": 35
 12938            },
 12939            "hiddenSeries": false,
 12940            "id": 287,
 12941            "legend": {
 12942              "alignAsTable": true,
 12943              "avg": true,
 12944              "current": true,
 12945              "max": true,
 12946              "min": false,
 12947              "rightSide": false,
 12948              "show": true,
 12949              "total": false,
 12950              "values": true
 12951            },
 12952            "lines": true,
 12953            "linewidth": 1,
 12954            "links": [],
 12955            "nullPointMode": "null",
 12956            "options": {
 12957              "alertThreshold": true
 12958            },
 12959            "percentage": false,
 12960            "pluginVersion": "7.5.11",
 12961            "pointradius": 2,
 12962            "points": false,
 12963            "renderer": "flot",
 12964            "seriesOverrides": [],
 12965            "spaceLength": 10,
 12966            "stack": false,
 12967            "steppedLine": false,
 12968            "targets": [
 12969              {
 12970                "exemplar": true,
 12971                "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"release\"}[1m])) by (instance)",
 12972                "format": "time_series",
 12973                "hide": false,
 12974                "interval": "",
 12975                "intervalFactor": 1,
 12976                "legendFormat": "{{instance}}-sorter",
 12977                "refId": "A"
 12978              }
 12979            ],
 12980            "thresholds": [],
 12981            "timeFrom": null,
 12982            "timeRegions": [],
 12983            "timeShift": null,
 12984            "title": "Read OPS - Release",
 12985            "tooltip": {
 12986              "shared": true,
 12987              "sort": 0,
 12988              "value_type": "individual"
 12989            },
 12990            "type": "graph",
 12991            "xaxis": {
 12992              "buckets": null,
 12993              "mode": "time",
 12994              "name": null,
 12995              "show": true,
 12996              "values": []
 12997            },
 12998            "yaxes": [
 12999              {
 13000                "format": "ops",
 13001                "label": null,
 13002                "logBase": 1,
 13003                "max": null,
 13004                "min": "0",
 13005                "show": true
 13006              },
 13007              {
 13008                "format": "percentunit",
 13009                "label": null,
 13010                "logBase": 10,
 13011                "max": null,
 13012                "min": null,
 13013                "show": true
 13014              }
 13015            ],
 13016            "yaxis": {
 13017              "align": false,
 13018              "alignLevel": null
 13019            }
 13020          },
 13021          {
 13022            "cards": {
 13023              "cardPadding": 0,
 13024              "cardRound": 0
 13025            },
 13026            "color": {
 13027              "cardColor": "#FF9830",
 13028              "colorScale": "linear",
 13029              "colorScheme": "interpolateSpectral",
 13030              "exponent": 0.5,
 13031              "min": 0,
 13032              "mode": "spectrum"
 13033            },
 13034            "dataFormat": "tsbuckets",
 13035            "datasource": "${DS_TEST-CLUSTER}",
 13036            "description": "The time of sorter compact",
 13037            "fieldConfig": {
 13038              "defaults": {},
 13039              "overrides": []
 13040            },
 13041            "gridPos": {
 13042              "h": 6,
 13043              "w": 8,
 13044              "x": 0,
 13045              "y": 41
 13046            },
 13047            "heatmap": {},
 13048            "hideZeroBuckets": true,
 13049            "highlightCards": true,
 13050            "id": 285,
 13051            "legend": {
 13052              "alignAsTable": true,
 13053              "avg": false,
 13054              "current": true,
 13055              "max": true,
 13056              "min": false,
 13057              "rightSide": true,
 13058              "show": true,
 13059              "sort": "current",
 13060              "sortDesc": true,
 13061              "total": false,
 13062              "values": true
 13063            },
 13064            "links": [],
 13065            "maxPerRow": 3,
 13066            "repeatDirection": "h",
 13067            "reverseYBuckets": false,
 13068            "targets": [
 13069              {
 13070                "exemplar": true,
 13071                "expr": "sum(rate(ticdc_sorter_db_compact_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)",
 13072                "format": "heatmap",
 13073                "instant": false,
 13074                "interval": "",
 13075                "intervalFactor": 2,
 13076                "legendFormat": "{{le}}",
 13077                "refId": "A"
 13078              }
 13079            ],
 13080            "title": "Compact duration",
 13081            "tooltip": {
 13082              "show": true,
 13083              "showHistogram": true
 13084            },
 13085            "tooltipDecimals": 1,
 13086            "type": "heatmap",
 13087            "xAxis": {
 13088              "show": true
 13089            },
 13090            "xBucketNumber": null,
 13091            "xBucketSize": null,
 13092            "yAxis": {
 13093              "decimals": 1,
 13094              "format": "s",
 13095              "logBase": 1,
 13096              "max": null,
 13097              "min": null,
 13098              "show": true,
 13099              "splitFactor": null
 13100            },
 13101            "yBucketBound": "upper",
 13102            "yBucketNumber": null,
 13103            "yBucketSize": null
 13104          }
 13105        ],
 13106        "title": "DB",
 13107        "type": "row"
 13108      },
 13109      {
 13110        "collapsed": true,
 13111        "datasource": null,
 13112        "gridPos": {
 13113          "h": 1,
 13114          "w": 24,
 13115          "x": 0,
 13116          "y": 11
 13117        },
 13118        "id": 447,
 13119        "panels": [
 13120          {
 13121            "aliasColors": {},
 13122            "bars": false,
 13123            "dashLength": 10,
 13124            "dashes": false,
 13125            "datasource": "${DS_TEST-CLUSTER}",
 13126            "description": "The number of established Eventfeed RPC between TiCDC and TiKV",
 13127            "fieldConfig": {
 13128              "defaults": {
 13129                "links": []
 13130              },
 13131              "overrides": []
 13132            },
 13133            "fill": 1,
 13134            "fillGradient": 0,
 13135            "gridPos": {
 13136              "h": 7,
 13137              "w": 12,
 13138              "x": 0,
 13139              "y": 12
 13140            },
 13141            "hiddenSeries": false,
 13142            "id": 15,
 13143            "legend": {
 13144              "alignAsTable": true,
 13145              "avg": false,
 13146              "current": true,
 13147              "hideEmpty": false,
 13148              "max": false,
 13149              "min": false,
 13150              "rightSide": true,
 13151              "show": true,
 13152              "total": false,
 13153              "values": true
 13154            },
 13155            "lines": true,
 13156            "linewidth": 1,
 13157            "links": [],
 13158            "nullPointMode": "null",
 13159            "options": {
 13160              "alertThreshold": true
 13161            },
 13162            "paceLength": 10,
 13163            "percentage": false,
 13164            "pluginVersion": "7.5.11",
 13165            "pointradius": 2,
 13166            "points": false,
 13167            "renderer": "flot",
 13168            "seriesOverrides": [
 13169              {
 13170                "alias": "/.*-rpc/",
 13171                "yaxis": 2
 13172              }
 13173            ],
 13174            "spaceLength": 10,
 13175            "stack": false,
 13176            "steppedLine": false,
 13177            "targets": [
 13178              {
 13179                "expr": "sum(ticdc_kvclient_event_feed_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (instance)",
 13180                "format": "time_series",
 13181                "intervalFactor": 1,
 13182                "legendFormat": "{{instance}}",
 13183                "refId": "A"
 13184              },
 13185              {
 13186                "expr": "sum(grpc_client_started_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", grpc_method=~\"EventFeed.*\"}) by (instance) - sum(grpc_client_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", grpc_method=~\"EventFeed.*\"}) by (instance)",
 13187                "format": "time_series",
 13188                "intervalFactor": 1,
 13189                "legendFormat": "{{instance}}-rpc",
 13190                "refId": "B"
 13191              },
 13192              {
 13193                "expr": "sum(grpc_client_started_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", grpc_method=~\"EventFeed.*\"}) by (instance)",
 13194                "format": "time_series",
 13195                "hide": false,
 13196                "intervalFactor": 1,
 13197                "legendFormat": "{{instance}}-rpc-started",
 13198                "refId": "C"
 13199              },
 13200              {
 13201                "expr": "sum(grpc_client_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", grpc_method=~\"EventFeed.*\"}) by (instance)",
 13202                "format": "time_series",
 13203                "hide": false,
 13204                "intervalFactor": 1,
 13205                "legendFormat": "{{instance}}-rpc-handled",
 13206                "refId": "D"
 13207              }
 13208            ],
 13209            "thresholds": [],
 13210            "timeFrom": null,
 13211            "timeRegions": [],
 13212            "timeShift": null,
 13213            "title": "Eventfeed count",
 13214            "tooltip": {
 13215              "shared": true,
 13216              "sort": 0,
 13217              "value_type": "individual"
 13218            },
 13219            "type": "graph",
 13220            "xaxis": {
 13221              "buckets": null,
 13222              "mode": "time",
 13223              "name": null,
 13224              "show": true,
 13225              "values": []
 13226            },
 13227            "yaxes": [
 13228              {
 13229                "$$hashKey": "object:359",
 13230                "format": "short",
 13231                "label": null,
 13232                "logBase": 1,
 13233                "max": null,
 13234                "min": null,
 13235                "show": true
 13236              },
 13237              {
 13238                "$$hashKey": "object:360",
 13239                "format": "short",
 13240                "label": null,
 13241                "logBase": 1,
 13242                "max": null,
 13243                "min": null,
 13244                "show": true
 13245              }
 13246            ],
 13247            "yaxis": {
 13248              "align": false,
 13249              "alignLevel": null
 13250            }
 13251          },
 13252          {
 13253            "aliasColors": {},
 13254            "bars": false,
 13255            "dashLength": 10,
 13256            "dashes": false,
 13257            "datasource": "${DS_TEST-CLUSTER}",
 13258            "description": "The number of KV client received events from TiKV per seconds",
 13259            "fieldConfig": {
 13260              "defaults": {
 13261                "links": []
 13262              },
 13263              "overrides": []
 13264            },
 13265            "fill": 1,
 13266            "fillGradient": 0,
 13267            "gridPos": {
 13268              "h": 7,
 13269              "w": 12,
 13270              "x": 12,
 13271              "y": 12
 13272            },
 13273            "hiddenSeries": false,
 13274            "id": 29,
 13275            "legend": {
 13276              "alignAsTable": true,
 13277              "avg": false,
 13278              "current": true,
 13279              "hideEmpty": true,
 13280              "hideZero": true,
 13281              "max": true,
 13282              "min": false,
 13283              "rightSide": false,
 13284              "show": true,
 13285              "total": false,
 13286              "values": true
 13287            },
 13288            "lines": true,
 13289            "linewidth": 1,
 13290            "links": [],
 13291            "nullPointMode": "null",
 13292            "options": {
 13293              "alertThreshold": true
 13294            },
 13295            "paceLength": 10,
 13296            "percentage": false,
 13297            "pluginVersion": "7.5.11",
 13298            "pointradius": 2,
 13299            "points": false,
 13300            "renderer": "flot",
 13301            "seriesOverrides": [],
 13302            "spaceLength": 10,
 13303            "stack": false,
 13304            "steppedLine": false,
 13305            "targets": [
 13306              {
 13307                "exemplar": true,
 13308                "expr": "sum(rate(ticdc_kvclient_pull_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[1m])) by (instance, type, changefeed)",
 13309                "format": "time_series",
 13310                "interval": "",
 13311                "intervalFactor": 1,
 13312                "legendFormat": "{{changefeed}}-{{instance}}-{{type}}",
 13313                "refId": "A"
 13314              }
 13315            ],
 13316            "thresholds": [],
 13317            "timeFrom": null,
 13318            "timeRegions": [],
 13319            "timeShift": null,
 13320            "title": "KV client receive events/s",
 13321            "tooltip": {
 13322              "shared": true,
 13323              "sort": 0,
 13324              "value_type": "individual"
 13325            },
 13326            "type": "graph",
 13327            "xaxis": {
 13328              "buckets": null,
 13329              "mode": "time",
 13330              "name": null,
 13331              "show": true,
 13332              "values": []
 13333            },
 13334            "yaxes": [
 13335              {
 13336                "$$hashKey": "object:242",
 13337                "format": "short",
 13338                "label": null,
 13339                "logBase": 1,
 13340                "max": null,
 13341                "min": null,
 13342                "show": true
 13343              },
 13344              {
 13345                "$$hashKey": "object:243",
 13346                "format": "short",
 13347                "label": null,
 13348                "logBase": 1,
 13349                "max": null,
 13350                "min": null,
 13351                "show": true
 13352              }
 13353            ],
 13354            "yaxis": {
 13355              "align": false,
 13356              "alignLevel": null
 13357            }
 13358          },
 13359          {
 13360            "aliasColors": {},
 13361            "bars": true,
 13362            "dashLength": 10,
 13363            "dashes": false,
 13364            "datasource": "${DS_TEST-CLUSTER}",
 13365            "description": "The number of errors that interrupt Eventfeed RPC",
 13366            "fieldConfig": {
 13367              "defaults": {
 13368                "links": []
 13369              },
 13370              "overrides": []
 13371            },
 13372            "fill": 1,
 13373            "fillGradient": 0,
 13374            "gridPos": {
 13375              "h": 7,
 13376              "w": 12,
 13377              "x": 0,
 13378              "y": 19
 13379            },
 13380            "hiddenSeries": false,
 13381            "id": 28,
 13382            "legend": {
 13383              "alignAsTable": true,
 13384              "avg": false,
 13385              "current": true,
 13386              "hideEmpty": true,
 13387              "hideZero": true,
 13388              "max": true,
 13389              "min": false,
 13390              "rightSide": true,
 13391              "show": true,
 13392              "total": false,
 13393              "values": true
 13394            },
 13395            "lines": true,
 13396            "linewidth": 1,
 13397            "links": [],
 13398            "nullPointMode": "null",
 13399            "options": {
 13400              "alertThreshold": true
 13401            },
 13402            "paceLength": 10,
 13403            "percentage": false,
 13404            "pluginVersion": "7.5.11",
 13405            "pointradius": 2,
 13406            "points": false,
 13407            "renderer": "flot",
 13408            "seriesOverrides": [],
 13409            "spaceLength": 10,
 13410            "stack": false,
 13411            "steppedLine": false,
 13412            "targets": [
 13413              {
 13414                "expr": "sum(increase(ticdc_kvclient_event_feed_error_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (type)",
 13415                "format": "time_series",
 13416                "hide": false,
 13417                "interval": "1m",
 13418                "intervalFactor": 1,
 13419                "legendFormat": "{{type}}",
 13420                "refId": "A"
 13421              },
 13422              {
 13423                "expr": "-sum(increase(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", event=\"create\", type=~\".*leader\"}[1m]))",
 13424                "format": "time_series",
 13425                "hide": false,
 13426                "interval": "1m",
 13427                "intervalFactor": 1,
 13428                "legendFormat": "transfer-leader",
 13429                "refId": "B"
 13430              },
 13431              {
 13432                "expr": "-sum(increase(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", event=\"create\", type=~\".*(peer|region)\"}[1m]))",
 13433                "format": "time_series",
 13434                "interval": "1m",
 13435                "intervalFactor": 1,
 13436                "legendFormat": "move-region",
 13437                "refId": "C"
 13438              }
 13439            ],
 13440            "thresholds": [],
 13441            "timeFrom": null,
 13442            "timeRegions": [],
 13443            "timeShift": null,
 13444            "title": "Eventfeed error/m",
 13445            "tooltip": {
 13446              "shared": true,
 13447              "sort": 0,
 13448              "value_type": "individual"
 13449            },
 13450            "type": "graph",
 13451            "xaxis": {
 13452              "buckets": null,
 13453              "mode": "time",
 13454              "name": null,
 13455              "show": true,
 13456              "values": []
 13457            },
 13458            "yaxes": [
 13459              {
 13460                "format": "short",
 13461                "label": null,
 13462                "logBase": 1,
 13463                "max": null,
 13464                "min": null,
 13465                "show": true
 13466              },
 13467              {
 13468                "format": "short",
 13469                "label": null,
 13470                "logBase": 1,
 13471                "max": null,
 13472                "min": null,
 13473                "show": true
 13474              }
 13475            ],
 13476            "yaxis": {
 13477              "align": false,
 13478              "alignLevel": null
 13479            }
 13480          },
 13481          {
 13482            "aliasColors": {},
 13483            "bars": false,
 13484            "dashLength": 10,
 13485            "dashes": false,
 13486            "datasource": "${DS_TEST-CLUSTER}",
 13487            "description": "The number of KV client dispatched event per second",
 13488            "fieldConfig": {
 13489              "defaults": {
 13490                "links": []
 13491              },
 13492              "overrides": []
 13493            },
 13494            "fill": 1,
 13495            "fillGradient": 0,
 13496            "gridPos": {
 13497              "h": 7,
 13498              "w": 12,
 13499              "x": 12,
 13500              "y": 19
 13501            },
 13502            "hiddenSeries": false,
 13503            "id": 31,
 13504            "legend": {
 13505              "alignAsTable": true,
 13506              "avg": false,
 13507              "current": true,
 13508              "hideEmpty": true,
 13509              "hideZero": true,
 13510              "max": true,
 13511              "min": false,
 13512              "rightSide": true,
 13513              "show": true,
 13514              "total": false,
 13515              "values": true
 13516            },
 13517            "lines": true,
 13518            "linewidth": 1,
 13519            "links": [],
 13520            "nullPointMode": "null",
 13521            "options": {
 13522              "alertThreshold": true
 13523            },
 13524            "paceLength": 10,
 13525            "percentage": false,
 13526            "pluginVersion": "7.5.11",
 13527            "pointradius": 2,
 13528            "points": false,
 13529            "renderer": "flot",
 13530            "seriesOverrides": [
 13531              {
 13532                "alias": "/.*batch-resolved/",
 13533                "yaxis": 2
 13534              }
 13535            ],
 13536            "spaceLength": 10,
 13537            "stack": false,
 13538            "steppedLine": false,
 13539            "targets": [
 13540              {
 13541                "expr": "sum(rate(ticdc_kvclient_send_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[1m])) by (instance, namespace,changefeed, type)",
 13542                "format": "time_series",
 13543                "intervalFactor": 1,
 13544                "legendFormat": "{{instance}}-{{namespace}}-{{changefeed}}-{{type}}",
 13545                "refId": "A"
 13546              },
 13547              {
 13548                "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (instance,namespace, changefeed, table)",
 13549                "format": "time_series",
 13550                "intervalFactor": 1,
 13551                "legendFormat": "{{instance}}-{{namespace}}-{{changefeed}}-batch-resolved",
 13552                "refId": "B"
 13553              }
 13554            ],
 13555            "thresholds": [],
 13556            "timeFrom": null,
 13557            "timeRegions": [],
 13558            "timeShift": null,
 13559            "title": "KV client dispatch events/s",
 13560            "tooltip": {
 13561              "shared": true,
 13562              "sort": 0,
 13563              "value_type": "individual"
 13564            },
 13565            "type": "graph",
 13566            "xaxis": {
 13567              "buckets": null,
 13568              "mode": "time",
 13569              "name": null,
 13570              "show": true,
 13571              "values": []
 13572            },
 13573            "yaxes": [
 13574              {
 13575                "$$hashKey": "object:542",
 13576                "format": "short",
 13577                "label": null,
 13578                "logBase": 1,
 13579                "max": null,
 13580                "min": null,
 13581                "show": true
 13582              },
 13583              {
 13584                "$$hashKey": "object:543",
 13585                "format": "short",
 13586                "label": null,
 13587                "logBase": 1,
 13588                "max": null,
 13589                "min": null,
 13590                "show": true
 13591              }
 13592            ],
 13593            "yaxis": {
 13594              "align": false,
 13595              "alignLevel": null
 13596            }
 13597          },
 13598          {
 13599            "aliasColors": {},
 13600            "bars": false,
 13601            "dashLength": 10,
 13602            "dashes": false,
 13603            "datasource": "${DS_TEST-CLUSTER}",
 13604            "description": "Active stream count of each gRPC connection",
 13605            "fieldConfig": {
 13606              "defaults": {
 13607                "links": []
 13608              },
 13609              "overrides": []
 13610            },
 13611            "fill": 1,
 13612            "fillGradient": 0,
 13613            "gridPos": {
 13614              "h": 7,
 13615              "w": 12,
 13616              "x": 0,
 13617              "y": 26
 13618            },
 13619            "hiddenSeries": false,
 13620            "id": 188,
 13621            "legend": {
 13622              "alignAsTable": true,
 13623              "avg": false,
 13624              "current": true,
 13625              "hideEmpty": true,
 13626              "hideZero": true,
 13627              "max": true,
 13628              "min": false,
 13629              "rightSide": true,
 13630              "show": true,
 13631              "total": false,
 13632              "values": true
 13633            },
 13634            "lines": true,
 13635            "linewidth": 1,
 13636            "links": [],
 13637            "nullPointMode": "null",
 13638            "options": {
 13639              "alertThreshold": true
 13640            },
 13641            "paceLength": 10,
 13642            "percentage": false,
 13643            "pluginVersion": "7.5.11",
 13644            "pointradius": 2,
 13645            "points": false,
 13646            "renderer": "flot",
 13647            "seriesOverrides": [],
 13648            "spaceLength": 10,
 13649            "stack": false,
 13650            "steppedLine": false,
 13651            "targets": [
 13652              {
 13653                "expr": "sum(ticdc_kvclient_grpc_stream_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (store)",
 13654                "format": "time_series",
 13655                "intervalFactor": 1,
 13656                "legendFormat": "{{store}}",
 13657                "refId": "A"
 13658              }
 13659            ],
 13660            "thresholds": [],
 13661            "timeFrom": null,
 13662            "timeRegions": [],
 13663            "timeShift": null,
 13664            "title": "KV client gRPC stream count",
 13665            "tooltip": {
 13666              "shared": true,
 13667              "sort": 0,
 13668              "value_type": "individual"
 13669            },
 13670            "type": "graph",
 13671            "xaxis": {
 13672              "buckets": null,
 13673              "mode": "time",
 13674              "name": null,
 13675              "show": true,
 13676              "values": []
 13677            },
 13678            "yaxes": [
 13679              {
 13680                "format": "short",
 13681                "label": null,
 13682                "logBase": 1,
 13683                "max": null,
 13684                "min": null,
 13685                "show": true
 13686              },
 13687              {
 13688                "format": "short",
 13689                "label": null,
 13690                "logBase": 1,
 13691                "max": null,
 13692                "min": null,
 13693                "show": true
 13694              }
 13695            ],
 13696            "yaxis": {
 13697              "align": false,
 13698              "alignLevel": null
 13699            }
 13700          },
 13701          {
 13702            "aliasColors": {},
 13703            "bars": false,
 13704            "dashLength": 10,
 13705            "dashes": false,
 13706            "datasource": "${DS_TEST-CLUSTER}",
 13707            "decimals": null,
 13708            "description": "Percentiles of Eventfeed message size",
 13709            "fieldConfig": {
 13710              "defaults": {
 13711                "links": []
 13712              },
 13713              "overrides": []
 13714            },
 13715            "fill": 1,
 13716            "fillGradient": 0,
 13717            "gridPos": {
 13718              "h": 7,
 13719              "w": 12,
 13720              "x": 12,
 13721              "y": 26
 13722            },
 13723            "hiddenSeries": false,
 13724            "id": 17,
 13725            "legend": {
 13726              "alignAsTable": true,
 13727              "avg": false,
 13728              "current": true,
 13729              "max": false,
 13730              "min": false,
 13731              "rightSide": true,
 13732              "show": true,
 13733              "total": false,
 13734              "values": true
 13735            },
 13736            "lines": true,
 13737            "linewidth": 1,
 13738            "links": [],
 13739            "nullPointMode": "null",
 13740            "options": {
 13741              "alertThreshold": true
 13742            },
 13743            "paceLength": 10,
 13744            "percentage": false,
 13745            "pluginVersion": "7.5.11",
 13746            "pointradius": 2,
 13747            "points": false,
 13748            "renderer": "flot",
 13749            "seriesOverrides": [],
 13750            "spaceLength": 10,
 13751            "stack": false,
 13752            "steppedLine": false,
 13753            "targets": [
 13754              {
 13755                "expr": "histogram_quantile(0.999, sum(rate(ticdc_kvclient_event_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))",
 13756                "format": "time_series",
 13757                "intervalFactor": 1,
 13758                "legendFormat": "{{instance}}-{{type}}-p999",
 13759                "refId": "A"
 13760              },
 13761              {
 13762                "expr": "histogram_quantile(0.95, sum(rate(ticdc_kvclient_event_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))",
 13763                "format": "time_series",
 13764                "intervalFactor": 1,
 13765                "legendFormat": "{{instance}}-{{type}}-p95",
 13766                "refId": "B"
 13767              }
 13768            ],
 13769            "thresholds": [],
 13770            "timeFrom": null,
 13771            "timeRegions": [],
 13772            "timeShift": null,
 13773            "title": "Event size percentile",
 13774            "tooltip": {
 13775              "shared": true,
 13776              "sort": 0,
 13777              "value_type": "individual"
 13778            },
 13779            "type": "graph",
 13780            "xaxis": {
 13781              "buckets": null,
 13782              "mode": "time",
 13783              "name": null,
 13784              "show": true,
 13785              "values": []
 13786            },
 13787            "yaxes": [
 13788              {
 13789                "format": "bytes",
 13790                "label": null,
 13791                "logBase": 1,
 13792                "max": null,
 13793                "min": null,
 13794                "show": true
 13795              },
 13796              {
 13797                "format": "short",
 13798                "label": null,
 13799                "logBase": 1,
 13800                "max": null,
 13801                "min": null,
 13802                "show": true
 13803              }
 13804            ],
 13805            "yaxis": {
 13806              "align": false,
 13807              "alignLevel": null
 13808            }
 13809          },
 13810          {
 13811            "aliasColors": {},
 13812            "bars": false,
 13813            "dashLength": 10,
 13814            "dashes": false,
 13815            "datasource": "${DS_TEST-CLUSTER}",
 13816            "description": "The number of regions that are being scanned",
 13817            "fieldConfig": {
 13818              "defaults": {
 13819                "links": []
 13820              },
 13821              "overrides": []
 13822            },
 13823            "fill": 1,
 13824            "fillGradient": 0,
 13825            "gridPos": {
 13826              "h": 7,
 13827              "w": 12,
 13828              "x": 0,
 13829              "y": 33
 13830            },
 13831            "hiddenSeries": false,
 13832            "id": 177,
 13833            "legend": {
 13834              "alignAsTable": true,
 13835              "avg": false,
 13836              "current": true,
 13837              "hideEmpty": true,
 13838              "hideZero": true,
 13839              "max": true,
 13840              "min": false,
 13841              "rightSide": true,
 13842              "show": true,
 13843              "total": false,
 13844              "values": true
 13845            },
 13846            "lines": true,
 13847            "linewidth": 1,
 13848            "links": [],
 13849            "nullPointMode": "null",
 13850            "options": {
 13851              "alertThreshold": true
 13852            },
 13853            "paceLength": 10,
 13854            "percentage": false,
 13855            "pluginVersion": "7.5.11",
 13856            "pointradius": 2,
 13857            "points": false,
 13858            "renderer": "flot",
 13859            "seriesOverrides": [],
 13860            "spaceLength": 10,
 13861            "stack": false,
 13862            "steppedLine": false,
 13863            "targets": [
 13864              {
 13865                "exemplar": true,
 13866                "expr": "sum(ticdc_kvclient_region_token{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (instance,namespace, changefeed, store)",
 13867                "format": "time_series",
 13868                "interval": "",
 13869                "intervalFactor": 1,
 13870                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{store}}",
 13871                "refId": "A"
 13872              }
 13873            ],
 13874            "thresholds": [],
 13875            "timeFrom": null,
 13876            "timeRegions": [],
 13877            "timeShift": null,
 13878            "title": "KV client scanning regions",
 13879            "tooltip": {
 13880              "shared": true,
 13881              "sort": 0,
 13882              "value_type": "individual"
 13883            },
 13884            "type": "graph",
 13885            "xaxis": {
 13886              "buckets": null,
 13887              "mode": "time",
 13888              "name": null,
 13889              "show": true,
 13890              "values": []
 13891            },
 13892            "yaxes": [
 13893              {
 13894                "format": "short",
 13895                "label": null,
 13896                "logBase": 1,
 13897                "max": null,
 13898                "min": null,
 13899                "show": true
 13900              },
 13901              {
 13902                "format": "short",
 13903                "label": null,
 13904                "logBase": 1,
 13905                "max": null,
 13906                "min": null,
 13907                "show": true
 13908              }
 13909            ],
 13910            "yaxis": {
 13911              "align": false,
 13912              "alignLevel": null
 13913            }
 13914          },
 13915          {
 13916            "aliasColors": {},
 13917            "bars": false,
 13918            "dashLength": 10,
 13919            "dashes": false,
 13920            "datasource": "${DS_TEST-CLUSTER}",
 13921            "decimals": 1,
 13922            "description": "The count of regoin cache operation initiated by TiCDC",
 13923            "fieldConfig": {
 13924              "defaults": {},
 13925              "overrides": []
 13926            },
 13927            "fill": 1,
 13928            "fillGradient": 0,
 13929            "gridPos": {
 13930              "h": 7,
 13931              "w": 12,
 13932              "x": 12,
 13933              "y": 33
 13934            },
 13935            "hiddenSeries": false,
 13936            "id": 459,
 13937            "legend": {
 13938              "alignAsTable": true,
 13939              "avg": false,
 13940              "current": true,
 13941              "hideEmpty": true,
 13942              "hideZero": true,
 13943              "max": true,
 13944              "min": false,
 13945              "rightSide": true,
 13946              "show": true,
 13947              "sideWidth": null,
 13948              "sort": "current",
 13949              "sortDesc": true,
 13950              "total": false,
 13951              "values": true
 13952            },
 13953            "lines": true,
 13954            "linewidth": 1,
 13955            "links": [],
 13956            "nullPointMode": "null",
 13957            "options": {
 13958              "alertThreshold": true
 13959            },
 13960            "percentage": false,
 13961            "pluginVersion": "7.5.11",
 13962            "pointradius": 5,
 13963            "points": false,
 13964            "renderer": "flot",
 13965            "seriesOverrides": [],
 13966            "spaceLength": 10,
 13967            "stack": false,
 13968            "steppedLine": false,
 13969            "targets": [
 13970              {
 13971                "expr": "sum(rate(ticdc_tikvclient_region_cache_operations_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (type)",
 13972                "format": "time_series",
 13973                "intervalFactor": 2,
 13974                "legendFormat": "{{type}}",
 13975                "metric": "tikv_grpc_msg_duration_seconds_bucket",
 13976                "refId": "A",
 13977                "step": 10
 13978              }
 13979            ],
 13980            "thresholds": [],
 13981            "timeFrom": null,
 13982            "timeRegions": [],
 13983            "timeShift": null,
 13984            "title": "RegionCache operation/s",
 13985            "tooltip": {
 13986              "shared": true,
 13987              "sort": 0,
 13988              "value_type": "individual"
 13989            },
 13990            "type": "graph",
 13991            "xaxis": {
 13992              "buckets": null,
 13993              "mode": "time",
 13994              "name": null,
 13995              "show": true,
 13996              "values": []
 13997            },
 13998            "yaxes": [
 13999              {
 14000                "format": "ops",
 14001                "label": null,
 14002                "logBase": 1,
 14003                "max": null,
 14004                "min": null,
 14005                "show": true
 14006              },
 14007              {
 14008                "format": "short",
 14009                "label": null,
 14010                "logBase": 1,
 14011                "max": null,
 14012                "min": null,
 14013                "show": true
 14014              }
 14015            ],
 14016            "yaxis": {
 14017              "align": false,
 14018              "alignLevel": null
 14019            }
 14020          },
 14021          {
 14022            "cards": {
 14023              "cardPadding": 0,
 14024              "cardRound": 0
 14025            },
 14026            "color": {
 14027              "cardColor": "#FF9830",
 14028              "colorScale": "linear",
 14029              "colorScheme": "interpolateSpectral",
 14030              "exponent": 0.5,
 14031              "min": 0,
 14032              "mode": "spectrum"
 14033            },
 14034            "dataFormat": "tsbuckets",
 14035            "datasource": "${DS_TEST-CLUSTER}",
 14036            "description": "The size of batch resolved regions count",
 14037            "fieldConfig": {
 14038              "defaults": {},
 14039              "overrides": []
 14040            },
 14041            "gridPos": {
 14042              "h": 7,
 14043              "w": 12,
 14044              "x": 0,
 14045              "y": 40
 14046            },
 14047            "heatmap": {},
 14048            "hideZeroBuckets": true,
 14049            "highlightCards": true,
 14050            "id": 97,
 14051            "legend": {
 14052              "alignAsTable": true,
 14053              "avg": false,
 14054              "current": true,
 14055              "max": true,
 14056              "min": false,
 14057              "rightSide": true,
 14058              "show": true,
 14059              "sort": "current",
 14060              "sortDesc": true,
 14061              "total": false,
 14062              "values": true
 14063            },
 14064            "links": [],
 14065            "reverseYBuckets": false,
 14066            "targets": [
 14067              {
 14068                "exemplar": true,
 14069                "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)",
 14070                "format": "heatmap",
 14071                "instant": false,
 14072                "interval": "",
 14073                "intervalFactor": 2,
 14074                "legendFormat": "{{le}}",
 14075                "refId": "A"
 14076              }
 14077            ],
 14078            "title": "KV client batch resolved region count",
 14079            "tooltip": {
 14080              "show": true,
 14081              "showHistogram": true
 14082            },
 14083            "tooltipDecimals": 1,
 14084            "type": "heatmap",
 14085            "xAxis": {
 14086              "show": true
 14087            },
 14088            "xBucketNumber": null,
 14089            "xBucketSize": null,
 14090            "yAxis": {
 14091              "decimals": 1,
 14092              "format": "none",
 14093              "logBase": 1,
 14094              "max": null,
 14095              "min": null,
 14096              "show": true,
 14097              "splitFactor": null
 14098            },
 14099            "yBucketBound": "upper",
 14100            "yBucketNumber": null,
 14101            "yBucketSize": null
 14102          },
 14103          {
 14104            "aliasColors": {},
 14105            "bars": false,
 14106            "dashLength": 10,
 14107            "dashes": false,
 14108            "datasource": "${DS_TEST-CLUSTER}",
 14109            "description": "The number of regions that have not connected to TiKV",
 14110            "fieldConfig": {
 14111              "defaults": {
 14112                "links": []
 14113              },
 14114              "overrides": []
 14115            },
 14116            "fill": 1,
 14117            "fillGradient": 0,
 14118            "gridPos": {
 14119              "h": 7,
 14120              "w": 12,
 14121              "x": 12,
 14122              "y": 40
 14123            },
 14124            "hiddenSeries": false,
 14125            "id": 251,
 14126            "legend": {
 14127              "alignAsTable": true,
 14128              "avg": false,
 14129              "current": true,
 14130              "hideEmpty": true,
 14131              "hideZero": true,
 14132              "max": true,
 14133              "min": false,
 14134              "rightSide": true,
 14135              "show": true,
 14136              "total": false,
 14137              "values": true
 14138            },
 14139            "lines": true,
 14140            "linewidth": 1,
 14141            "links": [],
 14142            "nullPointMode": "null",
 14143            "options": {
 14144              "alertThreshold": true
 14145            },
 14146            "paceLength": 10,
 14147            "percentage": false,
 14148            "pluginVersion": "7.5.11",
 14149            "pointradius": 2,
 14150            "points": false,
 14151            "renderer": "flot",
 14152            "seriesOverrides": [],
 14153            "spaceLength": 10,
 14154            "stack": false,
 14155            "steppedLine": false,
 14156            "targets": [
 14157              {
 14158                "exemplar": true,
 14159                "expr": "sum(ticdc_kvclient_cached_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (instance, namespace,changefeed, store)",
 14160                "format": "time_series",
 14161                "interval": "",
 14162                "intervalFactor": 1,
 14163                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{store}}",
 14164                "refId": "A"
 14165              }
 14166            ],
 14167            "thresholds": [],
 14168            "timeFrom": null,
 14169            "timeRegions": [],
 14170            "timeShift": null,
 14171            "title": "KV client cached regions",
 14172            "tooltip": {
 14173              "shared": true,
 14174              "sort": 0,
 14175              "value_type": "individual"
 14176            },
 14177            "type": "graph",
 14178            "xaxis": {
 14179              "buckets": null,
 14180              "mode": "time",
 14181              "name": null,
 14182              "show": true,
 14183              "values": []
 14184            },
 14185            "yaxes": [
 14186              {
 14187                "format": "short",
 14188                "label": null,
 14189                "logBase": 1,
 14190                "max": null,
 14191                "min": null,
 14192                "show": true
 14193              },
 14194              {
 14195                "format": "short",
 14196                "label": null,
 14197                "logBase": 1,
 14198                "max": null,
 14199                "min": null,
 14200                "show": true
 14201              }
 14202            ],
 14203            "yaxis": {
 14204              "align": false,
 14205              "alignLevel": null
 14206            }
 14207          },
 14208          {
 14209            "aliasColors": {},
 14210            "bars": false,
 14211            "dashLength": 10,
 14212            "dashes": false,
 14213            "datasource": "${DS_TEST-CLUSTER}",
 14214            "decimals": 1,
 14215            "description": "The count of resolve lock RPC initiated by TiCDC",
 14216            "fieldConfig": {
 14217              "defaults": {},
 14218              "overrides": []
 14219            },
 14220            "fill": 1,
 14221            "fillGradient": 0,
 14222            "gridPos": {
 14223              "h": 7,
 14224              "w": 12,
 14225              "x": 0,
 14226              "y": 47
 14227            },
 14228            "hiddenSeries": false,
 14229            "id": 453,
 14230            "legend": {
 14231              "alignAsTable": true,
 14232              "avg": false,
 14233              "current": true,
 14234              "hideEmpty": true,
 14235              "hideZero": true,
 14236              "max": true,
 14237              "min": false,
 14238              "rightSide": true,
 14239              "show": true,
 14240              "sideWidth": null,
 14241              "sort": "current",
 14242              "sortDesc": true,
 14243              "total": false,
 14244              "values": true
 14245            },
 14246            "lines": true,
 14247            "linewidth": 1,
 14248            "links": [],
 14249            "nullPointMode": "null",
 14250            "options": {
 14251              "alertThreshold": true
 14252            },
 14253            "percentage": false,
 14254            "pluginVersion": "7.5.11",
 14255            "pointradius": 5,
 14256            "points": false,
 14257            "renderer": "flot",
 14258            "seriesOverrides": [],
 14259            "spaceLength": 10,
 14260            "stack": false,
 14261            "steppedLine": false,
 14262            "targets": [
 14263              {
 14264                "expr": "sum(delta(ticdc_tikvclient_lock_resolver_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (type)",
 14265                "format": "time_series",
 14266                "intervalFactor": 2,
 14267                "legendFormat": "{{type}}",
 14268                "metric": "tikv_grpc_msg_duration_seconds_bucket",
 14269                "refId": "A",
 14270                "step": 10
 14271              }
 14272            ],
 14273            "thresholds": [],
 14274            "timeFrom": null,
 14275            "timeRegions": [],
 14276            "timeShift": null,
 14277            "title": "ResolveLock count/m",
 14278            "tooltip": {
 14279              "shared": true,
 14280              "sort": 0,
 14281              "value_type": "individual"
 14282            },
 14283            "type": "graph",
 14284            "xaxis": {
 14285              "buckets": null,
 14286              "mode": "time",
 14287              "name": null,
 14288              "show": true,
 14289              "values": []
 14290            },
 14291            "yaxes": [
 14292              {
 14293                "format": "opm",
 14294                "label": null,
 14295                "logBase": 1,
 14296                "max": null,
 14297                "min": null,
 14298                "show": true
 14299              },
 14300              {
 14301                "format": "short",
 14302                "label": null,
 14303                "logBase": 1,
 14304                "max": null,
 14305                "min": null,
 14306                "show": true
 14307              }
 14308            ],
 14309            "yaxis": {
 14310              "align": false,
 14311              "alignLevel": null
 14312            }
 14313          },
 14314          {
 14315            "aliasColors": {},
 14316            "bars": false,
 14317            "dashLength": 10,
 14318            "dashes": false,
 14319            "datasource": "${DS_TEST-CLUSTER}",
 14320            "decimals": 1,
 14321            "description": "",
 14322            "fieldConfig": {
 14323              "defaults": {},
 14324              "overrides": []
 14325            },
 14326            "fill": 1,
 14327            "fillGradient": 0,
 14328            "gridPos": {
 14329              "h": 7,
 14330              "w": 12,
 14331              "x": 12,
 14332              "y": 47
 14333            },
 14334            "hiddenSeries": false,
 14335            "id": 460,
 14336            "legend": {
 14337              "alignAsTable": true,
 14338              "avg": false,
 14339              "current": true,
 14340              "hideEmpty": true,
 14341              "hideZero": true,
 14342              "max": true,
 14343              "min": false,
 14344              "rightSide": true,
 14345              "show": true,
 14346              "sideWidth": null,
 14347              "sort": "current",
 14348              "sortDesc": true,
 14349              "total": false,
 14350              "values": true
 14351            },
 14352            "lines": true,
 14353            "linewidth": 1,
 14354            "links": [],
 14355            "nullPointMode": "null",
 14356            "options": {
 14357              "alertThreshold": true
 14358            },
 14359            "percentage": false,
 14360            "pluginVersion": "7.5.11",
 14361            "pointradius": 5,
 14362            "points": false,
 14363            "renderer": "flot",
 14364            "seriesOverrides": [],
 14365            "spaceLength": 10,
 14366            "stack": false,
 14367            "steppedLine": false,
 14368            "targets": [
 14369              {
 14370                "exemplar": true,
 14371                "expr": "histogram_quantile(0.99999, sum(rate(ticdc_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, type))",
 14372                "format": "time_series",
 14373                "interval": "",
 14374                "intervalFactor": 2,
 14375                "legendFormat": "{{type}}",
 14376                "metric": "tikv_grpc_msg_duration_seconds_bucket",
 14377                "refId": "A",
 14378                "step": 10
 14379              }
 14380            ],
 14381            "thresholds": [],
 14382            "timeFrom": null,
 14383            "timeRegions": [],
 14384            "timeShift": null,
 14385            "title": "Backoff percentiles",
 14386            "tooltip": {
 14387              "shared": true,
 14388              "sort": 0,
 14389              "value_type": "individual"
 14390            },
 14391            "type": "graph",
 14392            "xaxis": {
 14393              "buckets": null,
 14394              "mode": "time",
 14395              "name": null,
 14396              "show": true,
 14397              "values": []
 14398            },
 14399            "yaxes": [
 14400              {
 14401                "format": "ms",
 14402                "label": null,
 14403                "logBase": 1,
 14404                "max": null,
 14405                "min": null,
 14406                "show": true
 14407              },
 14408              {
 14409                "format": "short",
 14410                "label": null,
 14411                "logBase": 1,
 14412                "max": null,
 14413                "min": null,
 14414                "show": false
 14415              }
 14416            ],
 14417            "yaxis": {
 14418              "align": false,
 14419              "alignLevel": null
 14420            }
 14421          },
 14422          {
 14423            "aliasColors": {},
 14424            "bars": false,
 14425            "dashLength": 10,
 14426            "dashes": false,
 14427            "datasource": "${DS_TEST-CLUSTER}",
 14428            "description": "The number of events that puller outputs to sorter \n per second",
 14429            "fieldConfig": {
 14430              "defaults": {
 14431                "links": []
 14432              },
 14433              "overrides": []
 14434            },
 14435            "fill": 1,
 14436            "fillGradient": 0,
 14437            "gridPos": {
 14438              "h": 7,
 14439              "w": 12,
 14440              "x": 0,
 14441              "y": 54
 14442            },
 14443            "hiddenSeries": false,
 14444            "id": 5,
 14445            "legend": {
 14446              "alignAsTable": true,
 14447              "avg": false,
 14448              "current": true,
 14449              "max": true,
 14450              "min": false,
 14451              "rightSide": false,
 14452              "show": true,
 14453              "total": false,
 14454              "values": true
 14455            },
 14456            "lines": true,
 14457            "linewidth": 1,
 14458            "links": [],
 14459            "nullPointMode": "null",
 14460            "options": {
 14461              "alertThreshold": true
 14462            },
 14463            "paceLength": 10,
 14464            "percentage": false,
 14465            "pluginVersion": "7.5.11",
 14466            "pointradius": 2,
 14467            "points": false,
 14468            "renderer": "flot",
 14469            "seriesOverrides": [],
 14470            "spaceLength": 10,
 14471            "stack": false,
 14472            "steppedLine": false,
 14473            "targets": [
 14474              {
 14475                "exemplar": true,
 14476                "expr": "sum (rate(ticdc_puller_txn_collect_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", changefeed!~\".*ddl_puller\"}[1m])) by (instance, type)",
 14477                "format": "time_series",
 14478                "interval": "",
 14479                "intervalFactor": 1,
 14480                "legendFormat": "{{instance}}-{{type}}",
 14481                "refId": "A"
 14482              }
 14483            ],
 14484            "thresholds": [],
 14485            "timeFrom": null,
 14486            "timeRegions": [],
 14487            "timeShift": null,
 14488            "title": "Puller output events/s",
 14489            "tooltip": {
 14490              "shared": true,
 14491              "sort": 0,
 14492              "value_type": "individual"
 14493            },
 14494            "type": "graph",
 14495            "xaxis": {
 14496              "buckets": null,
 14497              "mode": "time",
 14498              "name": null,
 14499              "show": true,
 14500              "values": []
 14501            },
 14502            "yaxes": [
 14503              {
 14504                "format": "none",
 14505                "label": null,
 14506                "logBase": 1,
 14507                "max": null,
 14508                "min": null,
 14509                "show": true
 14510              },
 14511              {
 14512                "format": "short",
 14513                "label": null,
 14514                "logBase": 1,
 14515                "max": null,
 14516                "min": null,
 14517                "show": true
 14518              }
 14519            ],
 14520            "yaxis": {
 14521              "align": false,
 14522              "alignLevel": null
 14523            }
 14524          },
 14525          {
 14526            "aliasColors": {},
 14527            "bars": false,
 14528            "dashLength": 10,
 14529            "dashes": false,
 14530            "datasource": "${DS_TEST-CLUSTER}",
 14531            "description": "Estimate the remaining time for a changefeed initialization (on a specific capture)",
 14532            "fieldConfig": {
 14533              "defaults": {
 14534                "unit": "s"
 14535              },
 14536              "overrides": []
 14537            },
 14538            "fill": 1,
 14539            "fillGradient": 0,
 14540            "gridPos": {
 14541              "h": 7,
 14542              "w": 12,
 14543              "x": 12,
 14544              "y": 54
 14545            },
 14546            "hiddenSeries": false,
 14547            "id": 252,
 14548            "legend": {
 14549              "alignAsTable": true,
 14550              "avg": false,
 14551              "current": true,
 14552              "hideEmpty": true,
 14553              "hideZero": true,
 14554              "max": true,
 14555              "min": false,
 14556              "rightSide": true,
 14557              "show": true,
 14558              "total": false,
 14559              "values": true
 14560            },
 14561            "lines": true,
 14562            "linewidth": 1,
 14563            "links": [],
 14564            "nullPointMode": "null",
 14565            "options": {
 14566              "alertThreshold": true
 14567            },
 14568            "paceLength": 10,
 14569            "percentage": false,
 14570            "pluginVersion": "7.5.11",
 14571            "pointradius": 2,
 14572            "points": false,
 14573            "renderer": "flot",
 14574            "seriesOverrides": [],
 14575            "spaceLength": 10,
 14576            "stack": false,
 14577            "steppedLine": false,
 14578            "targets": [
 14579              {
 14580                "exemplar": true,
 14581                "expr": "abs(sum(ticdc_kvclient_cached_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"} / deriv(ticdc_kvclient_cached_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (instance, namespace,changefeed, store))",
 14582                "format": "time_series",
 14583                "interval": "",
 14584                "intervalFactor": 1,
 14585                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{store}}",
 14586                "refId": "A"
 14587              }
 14588            ],
 14589            "thresholds": [],
 14590            "timeFrom": null,
 14591            "timeRegions": [],
 14592            "timeShift": null,
 14593            "title": "Estimate remaining time for initialization",
 14594            "tooltip": {
 14595              "shared": true,
 14596              "sort": 0,
 14597              "value_type": "individual"
 14598            },
 14599            "type": "graph",
 14600            "xaxis": {
 14601              "buckets": null,
 14602              "mode": "time",
 14603              "name": null,
 14604              "show": true,
 14605              "values": []
 14606            },
 14607            "yaxes": [
 14608              {
 14609                "format": "s",
 14610                "label": "",
 14611                "logBase": 2,
 14612                "max": null,
 14613                "min": null,
 14614                "show": true
 14615              },
 14616              {
 14617                "format": "short",
 14618                "label": null,
 14619                "logBase": 1,
 14620                "max": null,
 14621                "min": null,
 14622                "show": true
 14623              }
 14624            ],
 14625            "yaxis": {
 14626              "align": false,
 14627              "alignLevel": null
 14628            }
 14629          },
 14630          {
 14631            "cards": {
 14632              "cardPadding": 0,
 14633              "cardRound": 0
 14634            },
 14635            "color": {
 14636              "cardColor": "#FF9830",
 14637              "colorScale": "linear",
 14638              "colorScheme": "interpolateSpectral",
 14639              "exponent": 0.5,
 14640              "min": 0,
 14641              "mode": "spectrum"
 14642            },
 14643            "dataFormat": "tsbuckets",
 14644            "datasource": "${DS_TEST-CLUSTER}",
 14645            "description": "Duration of sorting DDL events",
 14646            "fieldConfig": {
 14647              "defaults": {},
 14648              "overrides": []
 14649            },
 14650            "gridPos": {
 14651              "h": 7,
 14652              "w": 12,
 14653              "x": 0,
 14654              "y": 61
 14655            },
 14656            "heatmap": {},
 14657            "hideZeroBuckets": true,
 14658            "highlightCards": true,
 14659            "id": 99,
 14660            "legend": {
 14661              "alignAsTable": true,
 14662              "avg": false,
 14663              "current": true,
 14664              "max": true,
 14665              "min": false,
 14666              "rightSide": true,
 14667              "show": true,
 14668              "sort": "current",
 14669              "sortDesc": true,
 14670              "total": false,
 14671              "values": true
 14672            },
 14673            "links": [],
 14674            "reverseYBuckets": false,
 14675            "targets": [
 14676              {
 14677                "exemplar": true,
 14678                "expr": "sum(rate(ticdc_puller_entry_sorter_sort_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)",
 14679                "format": "heatmap",
 14680                "instant": false,
 14681                "interval": "",
 14682                "intervalFactor": 2,
 14683                "legendFormat": "{{le}}",
 14684                "refId": "A"
 14685              }
 14686            ],
 14687            "title": "DDL sorter sort duration",
 14688            "tooltip": {
 14689              "show": true,
 14690              "showHistogram": true
 14691            },
 14692            "tooltipDecimals": 1,
 14693            "type": "heatmap",
 14694            "xAxis": {
 14695              "show": true
 14696            },
 14697            "xBucketNumber": null,
 14698            "xBucketSize": null,
 14699            "yAxis": {
 14700              "decimals": 1,
 14701              "format": "s",
 14702              "logBase": 1,
 14703              "max": null,
 14704              "min": null,
 14705              "show": true,
 14706              "splitFactor": null
 14707            },
 14708            "yBucketBound": "upper",
 14709            "yBucketNumber": null,
 14710            "yBucketSize": null
 14711          },
 14712          {
 14713            "cards": {
 14714              "cardPadding": 0,
 14715              "cardRound": 0
 14716            },
 14717            "color": {
 14718              "cardColor": "#FF9830",
 14719              "colorScale": "linear",
 14720              "colorScheme": "interpolateSpectral",
 14721              "exponent": 0.5,
 14722              "min": 0,
 14723              "mode": "spectrum"
 14724            },
 14725            "dataFormat": "tsbuckets",
 14726            "datasource": "${DS_TEST-CLUSTER}",
 14727            "description": "Duration of merging sorted events",
 14728            "fieldConfig": {
 14729              "defaults": {},
 14730              "overrides": []
 14731            },
 14732            "gridPos": {
 14733              "h": 7,
 14734              "w": 12,
 14735              "x": 12,
 14736              "y": 61
 14737            },
 14738            "heatmap": {},
 14739            "hideZeroBuckets": true,
 14740            "highlightCards": true,
 14741            "id": 105,
 14742            "legend": {
 14743              "alignAsTable": true,
 14744              "avg": false,
 14745              "current": true,
 14746              "max": true,
 14747              "min": false,
 14748              "rightSide": true,
 14749              "show": true,
 14750              "sort": "current",
 14751              "sortDesc": true,
 14752              "total": false,
 14753              "values": true
 14754            },
 14755            "links": [],
 14756            "reverseYBuckets": false,
 14757            "targets": [
 14758              {
 14759                "expr": "sum(rate(ticdc_puller_entry_sorter_merge_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)",
 14760                "format": "heatmap",
 14761                "instant": false,
 14762                "intervalFactor": 2,
 14763                "legendFormat": "{{le}}",
 14764                "refId": "A"
 14765              }
 14766            ],
 14767            "title": "DDL sorter merge duration",
 14768            "tooltip": {
 14769              "show": true,
 14770              "showHistogram": true
 14771            },
 14772            "tooltipDecimals": 1,
 14773            "type": "heatmap",
 14774            "xAxis": {
 14775              "show": true
 14776            },
 14777            "xBucketNumber": null,
 14778            "xBucketSize": null,
 14779            "yAxis": {
 14780              "decimals": 1,
 14781              "format": "s",
 14782              "logBase": 1,
 14783              "max": null,
 14784              "min": null,
 14785              "show": true,
 14786              "splitFactor": null
 14787            },
 14788            "yBucketBound": "upper",
 14789            "yBucketNumber": null,
 14790            "yBucketSize": null
 14791          },
 14792          {
 14793            "aliasColors": {},
 14794            "bars": false,
 14795            "dashLength": 10,
 14796            "dashes": false,
 14797            "datasource": "${DS_TEST-CLUSTER}",
 14798            "description": "Percentiles of sorting events duration",
 14799            "fieldConfig": {
 14800              "defaults": {
 14801                "links": []
 14802              },
 14803              "overrides": []
 14804            },
 14805            "fill": 1,
 14806            "fillGradient": 0,
 14807            "gridPos": {
 14808              "h": 7,
 14809              "w": 12,
 14810              "x": 0,
 14811              "y": 68
 14812            },
 14813            "hiddenSeries": false,
 14814            "id": 53,
 14815            "legend": {
 14816              "alignAsTable": true,
 14817              "avg": false,
 14818              "current": true,
 14819              "hideEmpty": false,
 14820              "max": true,
 14821              "min": false,
 14822              "rightSide": true,
 14823              "show": true,
 14824              "total": false,
 14825              "values": true
 14826            },
 14827            "lines": true,
 14828            "linewidth": 1,
 14829            "links": [],
 14830            "nullPointMode": "null",
 14831            "options": {
 14832              "alertThreshold": true
 14833            },
 14834            "paceLength": 10,
 14835            "percentage": false,
 14836            "pluginVersion": "7.5.11",
 14837            "pointradius": 2,
 14838            "points": false,
 14839            "renderer": "flot",
 14840            "seriesOverrides": [],
 14841            "spaceLength": 10,
 14842            "stack": false,
 14843            "steppedLine": false,
 14844            "targets": [
 14845              {
 14846                "exemplar": true,
 14847                "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_sort_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))",
 14848                "format": "time_series",
 14849                "interval": "",
 14850                "intervalFactor": 1,
 14851                "legendFormat": "{{instance}}-p999",
 14852                "refId": "A"
 14853              },
 14854              {
 14855                "exemplar": true,
 14856                "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_sort_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))",
 14857                "format": "time_series",
 14858                "hide": false,
 14859                "interval": "",
 14860                "intervalFactor": 1,
 14861                "legendFormat": "{{instance}}-p95",
 14862                "refId": "B"
 14863              }
 14864            ],
 14865            "thresholds": [],
 14866            "timeFrom": null,
 14867            "timeRegions": [],
 14868            "timeShift": null,
 14869            "title": "DDL sorter sort duration percentile",
 14870            "tooltip": {
 14871              "shared": true,
 14872              "sort": 0,
 14873              "value_type": "individual"
 14874            },
 14875            "type": "graph",
 14876            "xaxis": {
 14877              "buckets": null,
 14878              "mode": "time",
 14879              "name": null,
 14880              "show": true,
 14881              "values": []
 14882            },
 14883            "yaxes": [
 14884              {
 14885                "format": "s",
 14886                "label": null,
 14887                "logBase": 2,
 14888                "max": null,
 14889                "min": null,
 14890                "show": true
 14891              },
 14892              {
 14893                "format": "short",
 14894                "label": null,
 14895                "logBase": 1,
 14896                "max": null,
 14897                "min": null,
 14898                "show": true
 14899              }
 14900            ],
 14901            "yaxis": {
 14902              "align": false,
 14903              "alignLevel": null
 14904            }
 14905          },
 14906          {
 14907            "aliasColors": {},
 14908            "bars": false,
 14909            "dashLength": 10,
 14910            "dashes": false,
 14911            "datasource": "${DS_TEST-CLUSTER}",
 14912            "description": "Percentiles of merging sorted events duration",
 14913            "fieldConfig": {
 14914              "defaults": {
 14915                "links": []
 14916              },
 14917              "overrides": []
 14918            },
 14919            "fill": 1,
 14920            "fillGradient": 0,
 14921            "gridPos": {
 14922              "h": 7,
 14923              "w": 12,
 14924              "x": 12,
 14925              "y": 68
 14926            },
 14927            "hiddenSeries": false,
 14928            "id": 106,
 14929            "legend": {
 14930              "alignAsTable": true,
 14931              "avg": false,
 14932              "current": true,
 14933              "hideEmpty": false,
 14934              "max": true,
 14935              "min": false,
 14936              "rightSide": true,
 14937              "show": true,
 14938              "total": false,
 14939              "values": true
 14940            },
 14941            "lines": true,
 14942            "linewidth": 1,
 14943            "links": [],
 14944            "nullPointMode": "null",
 14945            "options": {
 14946              "alertThreshold": true
 14947            },
 14948            "paceLength": 10,
 14949            "percentage": false,
 14950            "pluginVersion": "7.5.11",
 14951            "pointradius": 2,
 14952            "points": false,
 14953            "renderer": "flot",
 14954            "seriesOverrides": [],
 14955            "spaceLength": 10,
 14956            "stack": false,
 14957            "steppedLine": false,
 14958            "targets": [
 14959              {
 14960                "exemplar": true,
 14961                "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_merge_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))",
 14962                "format": "time_series",
 14963                "interval": "",
 14964                "intervalFactor": 1,
 14965                "legendFormat": "{{instance}}-p999",
 14966                "refId": "A"
 14967              },
 14968              {
 14969                "exemplar": true,
 14970                "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_merge_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))",
 14971                "format": "time_series",
 14972                "hide": false,
 14973                "interval": "",
 14974                "intervalFactor": 1,
 14975                "legendFormat": "{{instance}}-p95",
 14976                "refId": "B"
 14977              }
 14978            ],
 14979            "thresholds": [],
 14980            "timeFrom": null,
 14981            "timeRegions": [],
 14982            "timeShift": null,
 14983            "title": "DDL sorter merge duration percentile",
 14984            "tooltip": {
 14985              "shared": true,
 14986              "sort": 0,
 14987              "value_type": "individual"
 14988            },
 14989            "type": "graph",
 14990            "xaxis": {
 14991              "buckets": null,
 14992              "mode": "time",
 14993              "name": null,
 14994              "show": true,
 14995              "values": []
 14996            },
 14997            "yaxes": [
 14998              {
 14999                "format": "s",
 15000                "label": null,
 15001                "logBase": 2,
 15002                "max": null,
 15003                "min": null,
 15004                "show": true
 15005              },
 15006              {
 15007                "format": "short",
 15008                "label": null,
 15009                "logBase": 1,
 15010                "max": null,
 15011                "min": null,
 15012                "show": true
 15013              }
 15014            ],
 15015            "yaxis": {
 15016              "align": false,
 15017              "alignLevel": null
 15018            }
 15019          },
 15020          {
 15021            "aliasColors": {},
 15022            "bars": false,
 15023            "dashLength": 10,
 15024            "dashes": false,
 15025            "datasource": "${DS_TEST-CLUSTER}",
 15026            "description": "batch read events from the sorter, record each batch size ",
 15027            "fieldConfig": {
 15028              "defaults": {},
 15029              "overrides": []
 15030            },
 15031            "fill": 1,
 15032            "fillGradient": 0,
 15033            "gridPos": {
 15034              "h": 7,
 15035              "w": 12,
 15036              "x": 0,
 15037              "y": 75
 15038            },
 15039            "hiddenSeries": false,
 15040            "id": 705,
 15041            "legend": {
 15042              "alignAsTable": true,
 15043              "avg": false,
 15044              "current": true,
 15045              "max": true,
 15046              "min": false,
 15047              "rightSide": false,
 15048              "show": true,
 15049              "total": false,
 15050              "values": true
 15051            },
 15052            "lines": true,
 15053            "linewidth": 1,
 15054            "nullPointMode": "null",
 15055            "options": {
 15056              "alertThreshold": true
 15057            },
 15058            "percentage": false,
 15059            "pluginVersion": "7.5.11",
 15060            "pointradius": 2,
 15061            "points": false,
 15062            "renderer": "flot",
 15063            "seriesOverrides": [],
 15064            "spaceLength": 10,
 15065            "stack": false,
 15066            "steppedLine": false,
 15067            "targets": [
 15068              {
 15069                "exemplar": true,
 15070                "expr": "histogram_quantile(0.999, sum(rate(ticdc_processor_sorter_batch_read_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,instance,changefeed))",
 15071                "hide": false,
 15072                "interval": "",
 15073                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-p999",
 15074                "refId": "A"
 15075              }
 15076            ],
 15077            "thresholds": [],
 15078            "timeFrom": null,
 15079            "timeRegions": [],
 15080            "timeShift": null,
 15081            "title": "batch read sorter size percentile",
 15082            "tooltip": {
 15083              "shared": true,
 15084              "sort": 0,
 15085              "value_type": "individual"
 15086            },
 15087            "type": "graph",
 15088            "xaxis": {
 15089              "buckets": null,
 15090              "mode": "time",
 15091              "name": null,
 15092              "show": true,
 15093              "values": []
 15094            },
 15095            "yaxes": [
 15096              {
 15097                "format": "none",
 15098                "label": null,
 15099                "logBase": 1,
 15100                "max": null,
 15101                "min": null,
 15102                "show": true
 15103              },
 15104              {
 15105                "format": "none",
 15106                "label": null,
 15107                "logBase": 1,
 15108                "max": null,
 15109                "min": null,
 15110                "show": false
 15111              }
 15112            ],
 15113            "yaxis": {
 15114              "align": false,
 15115              "alignLevel": null
 15116            }
 15117          },
 15118          {
 15119            "aliasColors": {},
 15120            "bars": false,
 15121            "dashLength": 10,
 15122            "dashes": false,
 15123            "datasource": "${DS_TEST-CLUSTER}",
 15124            "description": "batch read events from the sorter, record each batch read duration ",
 15125            "fieldConfig": {
 15126              "defaults": {},
 15127              "overrides": []
 15128            },
 15129            "fill": 1,
 15130            "fillGradient": 0,
 15131            "gridPos": {
 15132              "h": 7,
 15133              "w": 12,
 15134              "x": 12,
 15135              "y": 75
 15136            },
 15137            "hiddenSeries": false,
 15138            "id": 706,
 15139            "legend": {
 15140              "alignAsTable": true,
 15141              "avg": false,
 15142              "current": true,
 15143              "max": true,
 15144              "min": false,
 15145              "rightSide": false,
 15146              "show": true,
 15147              "total": false,
 15148              "values": true
 15149            },
 15150            "lines": true,
 15151            "linewidth": 1,
 15152            "nullPointMode": "null",
 15153            "options": {
 15154              "alertThreshold": true
 15155            },
 15156            "percentage": false,
 15157            "pluginVersion": "7.5.11",
 15158            "pointradius": 2,
 15159            "points": false,
 15160            "renderer": "flot",
 15161            "seriesOverrides": [],
 15162            "spaceLength": 10,
 15163            "stack": false,
 15164            "steppedLine": false,
 15165            "targets": [
 15166              {
 15167                "exemplar": true,
 15168                "expr": "histogram_quantile(0.999, sum(rate(ticdc_processor_sorter_batch_read_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le,instance,namespace,changefeed))",
 15169                "hide": false,
 15170                "interval": "",
 15171                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-p999",
 15172                "refId": "A"
 15173              }
 15174            ],
 15175            "thresholds": [],
 15176            "timeFrom": null,
 15177            "timeRegions": [],
 15178            "timeShift": null,
 15179            "title": "batch read sorter duration percentile",
 15180            "tooltip": {
 15181              "shared": true,
 15182              "sort": 0,
 15183              "value_type": "individual"
 15184            },
 15185            "type": "graph",
 15186            "xaxis": {
 15187              "buckets": null,
 15188              "mode": "time",
 15189              "name": null,
 15190              "show": true,
 15191              "values": []
 15192            },
 15193            "yaxes": [
 15194              {
 15195                "format": "s",
 15196                "label": null,
 15197                "logBase": 1,
 15198                "max": null,
 15199                "min": null,
 15200                "show": true
 15201              },
 15202              {
 15203                "format": "none",
 15204                "label": null,
 15205                "logBase": 1,
 15206                "max": null,
 15207                "min": null,
 15208                "show": false
 15209              }
 15210            ],
 15211            "yaxis": {
 15212              "align": false,
 15213              "alignLevel": null
 15214            }
 15215          },
 15216          {
 15217            "aliasColors": {},
 15218            "bars": false,
 15219            "dashLength": 10,
 15220            "dashes": false,
 15221            "datasource": "${DS_TEST-CLUSTER}",
 15222            "description": "",
 15223            "fieldConfig": {
 15224              "defaults": {},
 15225              "overrides": []
 15226            },
 15227            "fill": 1,
 15228            "fillGradient": 0,
 15229            "gridPos": {
 15230              "h": 7,
 15231              "w": 12,
 15232              "x": 0,
 15233              "y": 82
 15234            },
 15235            "hiddenSeries": false,
 15236            "id": 707,
 15237            "legend": {
 15238              "alignAsTable": true,
 15239              "avg": false,
 15240              "current": true,
 15241              "max": true,
 15242              "min": false,
 15243              "rightSide": false,
 15244              "show": true,
 15245              "total": false,
 15246              "values": true
 15247            },
 15248            "lines": true,
 15249            "linewidth": 1,
 15250            "nullPointMode": "null",
 15251            "options": {
 15252              "alertThreshold": true
 15253            },
 15254            "percentage": false,
 15255            "pluginVersion": "7.5.11",
 15256            "pointradius": 2,
 15257            "points": false,
 15258            "renderer": "flot",
 15259            "seriesOverrides": [],
 15260            "spaceLength": 10,
 15261            "stack": false,
 15262            "steppedLine": false,
 15263            "targets": [
 15264              {
 15265                "exemplar": true,
 15266                "expr": "ticdc_mounter_group_input_chan_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}",
 15267                "hide": false,
 15268                "interval": "",
 15269                "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{index}}",
 15270                "refId": "A"
 15271              }
 15272            ],
 15273            "thresholds": [],
 15274            "timeFrom": null,
 15275            "timeRegions": [],
 15276            "timeShift": null,
 15277            "title": "mounter group channel size",
 15278            "tooltip": {
 15279              "shared": true,
 15280              "sort": 0,
 15281              "value_type": "individual"
 15282            },
 15283            "type": "graph",
 15284            "xaxis": {
 15285              "buckets": null,
 15286              "mode": "time",
 15287              "name": null,
 15288              "show": true,
 15289              "values": []
 15290            },
 15291            "yaxes": [
 15292              {
 15293                "format": "none",
 15294                "label": null,
 15295                "logBase": 1,
 15296                "max": null,
 15297                "min": null,
 15298                "show": true
 15299              },
 15300              {
 15301                "format": "none",
 15302                "label": null,
 15303                "logBase": 1,
 15304                "max": null,
 15305                "min": null,
 15306                "show": false
 15307              }
 15308            ],
 15309            "yaxis": {
 15310              "align": false,
 15311              "alignLevel": null
 15312            }
 15313          },
 15314          {
 15315            "aliasColors": {},
 15316            "dashLength": 10,
 15317            "datasource": "${DS_TEST-CLUSTER}",
 15318            "fieldConfig": {
 15319              "defaults": {},
 15320              "overrides": []
 15321            },
 15322            "fill": 1,
 15323            "gridPos": {
 15324              "h": 7,
 15325              "w": 12,
 15326              "x": 12,
 15327              "y": 122
 15328            },
 15329            "id": 10039,
 15330            "legend": {
 15331              "alignAsTable": true,
 15332              "avg": false,
 15333              "current": true,
 15334              "max": true,
 15335              "min": false,
 15336              "rightSide": false,
 15337              "show": true,
 15338              "total": false,
 15339              "values": true
 15340            },
 15341            "lines": true,
 15342            "linewidth": 1,
 15343            "nullPointMode": "null",
 15344            "options": {
 15345              "alertThreshold": true
 15346            },
 15347            "pluginVersion": "7.5.17",
 15348            "pointradius": 2,
 15349            "renderer": "flot",
 15350            "seriesOverrides": [],
 15351            "spaceLength": 10,
 15352            "targets": [
 15353              {
 15354                "expr": "ticdc_kvclient_slow_initialize_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}",
 15355                "legendFormat": "{{instance}}-{{changefeed}}-{{namespace}}",
 15356                "interval": "",
 15357                "exemplar": true,
 15358                "hide": false,
 15359                "refId": "A"
 15360              }
 15361            ],
 15362            "thresholds": [],
 15363            "timeRegions": [],
 15364            "title": "KV client slow initalize region count",
 15365            "tooltip": {
 15366              "shared": true,
 15367              "sort": 0,
 15368              "value_type": "individual"
 15369            },
 15370            "type": "graph",
 15371            "xaxis": {
 15372              "buckets": null,
 15373              "mode": "time",
 15374              "name": null,
 15375              "show": true,
 15376              "values": []
 15377            },
 15378            "yaxes": [
 15379              {
 15380                "format": "none",
 15381                "label": null,
 15382                "logBase": 1,
 15383                "max": null,
 15384                "min": null,
 15385                "show": true,
 15386                "$$hashKey": "object:432"
 15387              },
 15388              {
 15389                "format": "none",
 15390                "label": null,
 15391                "logBase": 1,
 15392                "max": null,
 15393                "min": null,
 15394                "show": false,
 15395                "$$hashKey": "object:433"
 15396              }
 15397            ],
 15398            "yaxis": {
 15399              "align": false,
 15400              "alignLevel": null
 15401            },
 15402            "description": "The count of regions that initialize slow. You can search the log [event feed initializes a region too slow] to get slow region id",
 15403            "bars": false,
 15404            "dashes": false,
 15405            "fillGradient": 0,
 15406            "hiddenSeries": false,
 15407            "percentage": false,
 15408            "points": false,
 15409            "stack": false,
 15410            "steppedLine": false,
 15411            "timeFrom": null,
 15412            "timeShift": null
 15413          }
 15414        ],
 15415        "title": "KVClient",
 15416        "type": "row"
 15417      },
 15418      {
 15419        "collapsed": true,
 15420        "datasource": null,
 15421        "gridPos": {
 15422          "h": 1,
 15423          "w": 24,
 15424          "x": 0,
 15425          "y": 12
 15426        },
 15427        "id": 266,
 15428        "panels": [
 15429          {
 15430            "aliasColors": {},
 15431            "bars": false,
 15432            "dashLength": 10,
 15433            "dashes": false,
 15434            "datasource": "${DS_TEST-CLUSTER}",
 15435            "fieldConfig": {
 15436              "defaults": {},
 15437              "overrides": []
 15438            },
 15439            "fill": 1,
 15440            "fillGradient": 0,
 15441            "gridPos": {
 15442              "h": 7,
 15443              "w": 12,
 15444              "x": 0,
 15445              "y": 13
 15446            },
 15447            "hiddenSeries": false,
 15448            "id": 289,
 15449            "legend": {
 15450              "alignAsTable": true,
 15451              "avg": false,
 15452              "current": true,
 15453              "hideEmpty": true,
 15454              "hideZero": true,
 15455              "max": true,
 15456              "min": false,
 15457              "rightSide": true,
 15458              "show": true,
 15459              "total": false,
 15460              "values": true
 15461            },
 15462            "lines": true,
 15463            "linewidth": 1,
 15464            "links": [],
 15465            "nullPointMode": "null",
 15466            "options": {
 15467              "alertThreshold": true
 15468            },
 15469            "percentage": false,
 15470            "pluginVersion": "7.5.11",
 15471            "pointradius": 2,
 15472            "points": false,
 15473            "renderer": "flot",
 15474            "seriesOverrides": [],
 15475            "spaceLength": 10,
 15476            "stack": false,
 15477            "steppedLine": false,
 15478            "targets": [
 15479              {
 15480                "expr": "etcd_debugging_mvcc_db_total_size_in_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"pd\"}",
 15481                "format": "time_series",
 15482                "hide": false,
 15483                "intervalFactor": 1,
 15484                "legendFormat": "{{instance}}",
 15485                "refId": "A"
 15486              }
 15487            ],
 15488            "thresholds": [],
 15489            "timeFrom": null,
 15490            "timeRegions": [],
 15491            "timeShift": null,
 15492            "title": "Etcd MVCC DB total size",
 15493            "tooltip": {
 15494              "shared": true,
 15495              "sort": 0,
 15496              "value_type": "individual"
 15497            },
 15498            "type": "graph",
 15499            "xaxis": {
 15500              "buckets": null,
 15501              "mode": "time",
 15502              "name": null,
 15503              "show": true,
 15504              "values": []
 15505            },
 15506            "yaxes": [
 15507              {
 15508                "format": "decbytes",
 15509                "label": null,
 15510                "logBase": 1,
 15511                "max": null,
 15512                "min": null,
 15513                "show": true
 15514              },
 15515              {
 15516                "format": "short",
 15517                "label": null,
 15518                "logBase": 1,
 15519                "max": null,
 15520                "min": null,
 15521                "show": false
 15522              }
 15523            ],
 15524            "yaxis": {
 15525              "align": false,
 15526              "alignLevel": null
 15527            }
 15528          },
 15529          {
 15530            "aliasColors": {},
 15531            "bars": false,
 15532            "dashLength": 10,
 15533            "dashes": false,
 15534            "datasource": "${DS_TEST-CLUSTER}",
 15535            "fieldConfig": {
 15536              "defaults": {
 15537                "links": []
 15538              },
 15539              "overrides": []
 15540            },
 15541            "fill": 1,
 15542            "fillGradient": 0,
 15543            "gridPos": {
 15544              "h": 7,
 15545              "w": 12,
 15546              "x": 12,
 15547              "y": 13
 15548            },
 15549            "hiddenSeries": false,
 15550            "id": 114,
 15551            "legend": {
 15552              "alignAsTable": true,
 15553              "avg": false,
 15554              "current": true,
 15555              "hideEmpty": true,
 15556              "max": true,
 15557              "min": false,
 15558              "rightSide": true,
 15559              "show": true,
 15560              "total": false,
 15561              "values": true
 15562            },
 15563            "lines": true,
 15564            "linewidth": 1,
 15565            "links": [],
 15566            "nullPointMode": "null",
 15567            "options": {
 15568              "alertThreshold": true
 15569            },
 15570            "paceLength": 10,
 15571            "percentage": false,
 15572            "pluginVersion": "7.5.11",
 15573            "pointradius": 2,
 15574            "points": false,
 15575            "renderer": "flot",
 15576            "seriesOverrides": [],
 15577            "spaceLength": 10,
 15578            "stack": false,
 15579            "steppedLine": false,
 15580            "targets": [
 15581              {
 15582                "expr": "histogram_quantile(0.999, sum(rate(ticdc_server_etcd_health_check_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le,instance))",
 15583                "format": "time_series",
 15584                "intervalFactor": 1,
 15585                "legendFormat": "p999-{{instance}}",
 15586                "refId": "A"
 15587              },
 15588              {
 15589                "expr": "histogram_quantile(0.99, sum(rate(ticdc_server_etcd_health_check_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le,instance))",
 15590                "format": "time_series",
 15591                "intervalFactor": 1,
 15592                "legendFormat": "p99-{{instance}}",
 15593                "refId": "B"
 15594              },
 15595              {
 15596                "expr": "histogram_quantile(0.95, sum(rate(ticdc_server_etcd_health_check_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le,instance))",
 15597                "format": "time_series",
 15598                "intervalFactor": 1,
 15599                "legendFormat": "p95-{{instance}}",
 15600                "refId": "C"
 15601              }
 15602            ],
 15603            "thresholds": [],
 15604            "timeFrom": null,
 15605            "timeRegions": [],
 15606            "timeShift": null,
 15607            "title": "Etcd health check duration",
 15608            "tooltip": {
 15609              "shared": true,
 15610              "sort": 0,
 15611              "value_type": "individual"
 15612            },
 15613            "type": "graph",
 15614            "xaxis": {
 15615              "buckets": null,
 15616              "mode": "time",
 15617              "name": null,
 15618              "show": true,
 15619              "values": []
 15620            },
 15621            "yaxes": [
 15622              {
 15623                "format": "s",
 15624                "label": null,
 15625                "logBase": 1,
 15626                "max": null,
 15627                "min": null,
 15628                "show": true
 15629              },
 15630              {
 15631                "format": "short",
 15632                "label": null,
 15633                "logBase": 1,
 15634                "max": null,
 15635                "min": null,
 15636                "show": true
 15637              }
 15638            ],
 15639            "yaxis": {
 15640              "align": false,
 15641              "alignLevel": null
 15642            }
 15643          },
 15644          {
 15645            "cards": {
 15646              "cardPadding": 0,
 15647              "cardRound": 0
 15648            },
 15649            "color": {
 15650              "cardColor": "#b4ff00",
 15651              "colorScale": "sqrt",
 15652              "colorScheme": "interpolateSpectral",
 15653              "exponent": 0.5,
 15654              "max": null,
 15655              "min": 1,
 15656              "mode": "spectrum"
 15657            },
 15658            "dataFormat": "tsbuckets",
 15659            "datasource": "${DS_TEST-CLUSTER}",
 15660            "description": "",
 15661            "fieldConfig": {
 15662              "defaults": {},
 15663              "overrides": []
 15664            },
 15665            "gridPos": {
 15666              "h": 7,
 15667              "w": 12,
 15668              "x": 0,
 15669              "y": 20
 15670            },
 15671            "heatmap": {},
 15672            "hideZeroBuckets": true,
 15673            "highlightCards": true,
 15674            "id": 267,
 15675            "legend": {
 15676              "show": true
 15677            },
 15678            "pluginVersion": "6.1.6",
 15679            "reverseYBuckets": false,
 15680            "targets": [
 15681              {
 15682                "exemplar": true,
 15683                "expr": "sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)",
 15684                "format": "heatmap",
 15685                "instant": false,
 15686                "interval": "",
 15687                "intervalFactor": 2,
 15688                "legendFormat": "{{le}}",
 15689                "queryType": "randomWalk",
 15690                "refId": "A"
 15691              }
 15692            ],
 15693            "timeFrom": null,
 15694            "timeShift": null,
 15695            "title": "EtcdWorker tick reactor duration",
 15696            "tooltip": {
 15697              "show": true,
 15698              "showHistogram": true
 15699            },
 15700            "tooltipDecimals": 1,
 15701            "type": "heatmap",
 15702            "xAxis": {
 15703              "show": true
 15704            },
 15705            "xBucketNumber": null,
 15706            "xBucketSize": null,
 15707            "yAxis": {
 15708              "decimals": 1,
 15709              "format": "s",
 15710              "logBase": 1,
 15711              "max": null,
 15712              "min": null,
 15713              "show": true,
 15714              "splitFactor": null
 15715            },
 15716            "yBucketBound": "upper",
 15717            "yBucketNumber": null,
 15718            "yBucketSize": null
 15719          },
 15720          {
 15721            "aliasColors": {},
 15722            "bars": false,
 15723            "dashLength": 10,
 15724            "dashes": false,
 15725            "datasource": "${DS_TEST-CLUSTER}",
 15726            "description": "",
 15727            "fieldConfig": {
 15728              "defaults": {},
 15729              "overrides": []
 15730            },
 15731            "fill": 1,
 15732            "fillGradient": 0,
 15733            "gridPos": {
 15734              "h": 7,
 15735              "w": 12,
 15736              "x": 12,
 15737              "y": 20
 15738            },
 15739            "hiddenSeries": false,
 15740            "id": 264,
 15741            "legend": {
 15742              "avg": false,
 15743              "current": false,
 15744              "max": false,
 15745              "min": false,
 15746              "show": true,
 15747              "total": false,
 15748              "values": false
 15749            },
 15750            "lines": true,
 15751            "linewidth": 1,
 15752            "nullPointMode": "null",
 15753            "options": {
 15754              "alertThreshold": true
 15755            },
 15756            "percentage": false,
 15757            "pluginVersion": "7.5.11",
 15758            "pointradius": 2,
 15759            "points": false,
 15760            "renderer": "flot",
 15761            "seriesOverrides": [],
 15762            "spaceLength": 10,
 15763            "stack": false,
 15764            "steppedLine": false,
 15765            "targets": [
 15766              {
 15767                "exemplar": true,
 15768                "expr": "histogram_quantile(0.999, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))",
 15769                "hide": false,
 15770                "interval": "",
 15771                "legendFormat": "{{instance}}-999",
 15772                "refId": "A"
 15773              },
 15774              {
 15775                "exemplar": true,
 15776                "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))",
 15777                "hide": false,
 15778                "interval": "",
 15779                "legendFormat": "{{instance}}-99",
 15780                "refId": "B"
 15781              }
 15782            ],
 15783            "thresholds": [],
 15784            "timeFrom": null,
 15785            "timeRegions": [],
 15786            "timeShift": null,
 15787            "title": "EtcdWorker tick reactor duration",
 15788            "tooltip": {
 15789              "shared": true,
 15790              "sort": 0,
 15791              "value_type": "individual"
 15792            },
 15793            "type": "graph",
 15794            "xaxis": {
 15795              "buckets": null,
 15796              "mode": "time",
 15797              "name": null,
 15798              "show": true,
 15799              "values": []
 15800            },
 15801            "yaxes": [
 15802              {
 15803                "format": "s",
 15804                "label": null,
 15805                "logBase": 1,
 15806                "max": null,
 15807                "min": null,
 15808                "show": true
 15809              },
 15810              {
 15811                "format": "none",
 15812                "label": null,
 15813                "logBase": 1,
 15814                "max": null,
 15815                "min": null,
 15816                "show": true
 15817              }
 15818            ],
 15819            "yaxis": {
 15820              "align": false,
 15821              "alignLevel": null
 15822            }
 15823          },
 15824          {
 15825            "cards": {
 15826              "cardPadding": 0,
 15827              "cardRound": 0
 15828            },
 15829            "color": {
 15830              "cardColor": "#b4ff00",
 15831              "colorScale": "sqrt",
 15832              "colorScheme": "interpolateSpectral",
 15833              "exponent": 0.5,
 15834              "max": null,
 15835              "min": 1,
 15836              "mode": "spectrum"
 15837            },
 15838            "dataFormat": "tsbuckets",
 15839            "datasource": "${DS_TEST-CLUSTER}",
 15840            "description": "",
 15841            "fieldConfig": {
 15842              "defaults": {},
 15843              "overrides": []
 15844            },
 15845            "gridPos": {
 15846              "h": 7,
 15847              "w": 12,
 15848              "x": 0,
 15849              "y": 27
 15850            },
 15851            "heatmap": {},
 15852            "hideZeroBuckets": true,
 15853            "highlightCards": true,
 15854            "id": 256,
 15855            "legend": {
 15856              "show": true
 15857            },
 15858            "pluginVersion": "6.1.6",
 15859            "reverseYBuckets": false,
 15860            "targets": [
 15861              {
 15862                "exemplar": true,
 15863                "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)",
 15864                "format": "heatmap",
 15865                "instant": false,
 15866                "interval": "",
 15867                "intervalFactor": 2,
 15868                "legendFormat": "{{le}}",
 15869                "queryType": "randomWalk",
 15870                "refId": "A"
 15871              }
 15872            ],
 15873            "timeFrom": null,
 15874            "timeShift": null,
 15875            "title": "EtcdWorker exec etcd txn duration",
 15876            "tooltip": {
 15877              "show": true,
 15878              "showHistogram": true
 15879            },
 15880            "tooltipDecimals": 1,
 15881            "type": "heatmap",
 15882            "xAxis": {
 15883              "show": true
 15884            },
 15885            "xBucketNumber": null,
 15886            "xBucketSize": null,
 15887            "yAxis": {
 15888              "decimals": 1,
 15889              "format": "s",
 15890              "logBase": 1,
 15891              "max": null,
 15892              "min": null,
 15893              "show": true,
 15894              "splitFactor": null
 15895            },
 15896            "yBucketBound": "upper",
 15897            "yBucketNumber": null,
 15898            "yBucketSize": null
 15899          },
 15900          {
 15901            "aliasColors": {},
 15902            "bars": false,
 15903            "dashLength": 10,
 15904            "dashes": false,
 15905            "datasource": "${DS_TEST-CLUSTER}",
 15906            "description": "",
 15907            "fieldConfig": {
 15908              "defaults": {
 15909                "unit": "s"
 15910              },
 15911              "overrides": []
 15912            },
 15913            "fill": 1,
 15914            "fillGradient": 0,
 15915            "gridPos": {
 15916              "h": 7,
 15917              "w": 12,
 15918              "x": 12,
 15919              "y": 27
 15920            },
 15921            "hiddenSeries": false,
 15922            "id": 258,
 15923            "legend": {
 15924              "avg": false,
 15925              "current": false,
 15926              "max": false,
 15927              "min": false,
 15928              "show": true,
 15929              "total": false,
 15930              "values": false
 15931            },
 15932            "lines": true,
 15933            "linewidth": 1,
 15934            "nullPointMode": "null",
 15935            "options": {
 15936              "alertThreshold": true
 15937            },
 15938            "percentage": false,
 15939            "pluginVersion": "7.5.11",
 15940            "pointradius": 2,
 15941            "points": false,
 15942            "renderer": "flot",
 15943            "seriesOverrides": [],
 15944            "spaceLength": 10,
 15945            "stack": false,
 15946            "steppedLine": false,
 15947            "targets": [
 15948              {
 15949                "exemplar": true,
 15950                "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))",
 15951                "format": "time_series",
 15952                "interval": "",
 15953                "intervalFactor": 1,
 15954                "legendFormat": "{{instance}}-p95",
 15955                "queryType": "randomWalk",
 15956                "refId": "A"
 15957              },
 15958              {
 15959                "exemplar": true,
 15960                "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))",
 15961                "format": "time_series",
 15962                "hide": false,
 15963                "interval": "",
 15964                "intervalFactor": 1,
 15965                "legendFormat": "{{instance}}-p99",
 15966                "refId": "B"
 15967              }
 15968            ],
 15969            "thresholds": [],
 15970            "timeFrom": null,
 15971            "timeRegions": [],
 15972            "timeShift": null,
 15973            "title": "EtcdWorker exec etcd txn duration percentile",
 15974            "tooltip": {
 15975              "shared": true,
 15976              "sort": 0,
 15977              "value_type": "individual"
 15978            },
 15979            "type": "graph",
 15980            "xaxis": {
 15981              "buckets": null,
 15982              "mode": "time",
 15983              "name": null,
 15984              "show": true,
 15985              "values": []
 15986            },
 15987            "yaxes": [
 15988              {
 15989                "format": "s",
 15990                "label": null,
 15991                "logBase": 2,
 15992                "max": null,
 15993                "min": null,
 15994                "show": true
 15995              },
 15996              {
 15997                "format": "short",
 15998                "label": null,
 15999                "logBase": 1,
 16000                "max": null,
 16001                "min": null,
 16002                "show": true
 16003              }
 16004            ],
 16005            "yaxis": {
 16006              "align": false,
 16007              "alignLevel": null
 16008            }
 16009          },
 16010          {
 16011            "cards": {
 16012              "cardPadding": null,
 16013              "cardRound": null
 16014            },
 16015            "color": {
 16016              "cardColor": "#b4ff00",
 16017              "colorScale": "sqrt",
 16018              "colorScheme": "interpolatePurples",
 16019              "exponent": 0.5,
 16020              "min": 0,
 16021              "mode": "spectrum"
 16022            },
 16023            "dataFormat": "tsbuckets",
 16024            "datasource": "${DS_TEST-CLUSTER}",
 16025            "description": "",
 16026            "fieldConfig": {
 16027              "defaults": {},
 16028              "overrides": []
 16029            },
 16030            "gridPos": {
 16031              "h": 7,
 16032              "w": 12,
 16033              "x": 0,
 16034              "y": 34
 16035            },
 16036            "heatmap": {},
 16037            "hideZeroBuckets": true,
 16038            "highlightCards": true,
 16039            "id": 254,
 16040            "legend": {
 16041              "show": true
 16042            },
 16043            "pluginVersion": "6.1.6",
 16044            "reverseYBuckets": false,
 16045            "targets": [
 16046              {
 16047                "exemplar": true,
 16048                "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le)",
 16049                "format": "heatmap",
 16050                "instant": false,
 16051                "interval": "",
 16052                "intervalFactor": 1,
 16053                "legendFormat": "{{le}}",
 16054                "queryType": "randomWalk",
 16055                "refId": "A"
 16056              }
 16057            ],
 16058            "title": "EtcdWorker txn size",
 16059            "tooltip": {
 16060              "show": true,
 16061              "showHistogram": true
 16062            },
 16063            "tooltipDecimals": null,
 16064            "type": "heatmap",
 16065            "xAxis": {
 16066              "show": true
 16067            },
 16068            "xBucketNumber": null,
 16069            "xBucketSize": null,
 16070            "yAxis": {
 16071              "decimals": null,
 16072              "format": "decbytes",
 16073              "logBase": 1,
 16074              "max": null,
 16075              "min": null,
 16076              "show": true,
 16077              "splitFactor": null
 16078            },
 16079            "yBucketBound": "auto",
 16080            "yBucketNumber": null,
 16081            "yBucketSize": null
 16082          },
 16083          {
 16084            "aliasColors": {},
 16085            "bars": false,
 16086            "dashLength": 10,
 16087            "dashes": false,
 16088            "datasource": "${DS_TEST-CLUSTER}",
 16089            "description": "",
 16090            "fieldConfig": {
 16091              "defaults": {},
 16092              "overrides": []
 16093            },
 16094            "fill": 1,
 16095            "fillGradient": 0,
 16096            "gridPos": {
 16097              "h": 7,
 16098              "w": 12,
 16099              "x": 12,
 16100              "y": 34
 16101            },
 16102            "hiddenSeries": false,
 16103            "id": 260,
 16104            "legend": {
 16105              "avg": false,
 16106              "current": false,
 16107              "max": false,
 16108              "min": false,
 16109              "show": true,
 16110              "total": false,
 16111              "values": false
 16112            },
 16113            "lines": true,
 16114            "linewidth": 1,
 16115            "nullPointMode": "null",
 16116            "options": {
 16117              "alertThreshold": true
 16118            },
 16119            "percentage": false,
 16120            "pluginVersion": "7.5.11",
 16121            "pointradius": 2,
 16122            "points": false,
 16123            "renderer": "flot",
 16124            "seriesOverrides": [],
 16125            "spaceLength": 10,
 16126            "stack": false,
 16127            "steppedLine": false,
 16128            "targets": [
 16129              {
 16130                "exemplar": true,
 16131                "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))",
 16132                "interval": "",
 16133                "legendFormat": "{{instance}}-p95",
 16134                "queryType": "randomWalk",
 16135                "refId": "A"
 16136              },
 16137              {
 16138                "exemplar": true,
 16139                "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))",
 16140                "hide": false,
 16141                "interval": "",
 16142                "legendFormat": "{{instance}}-p99",
 16143                "refId": "B"
 16144              }
 16145            ],
 16146            "thresholds": [],
 16147            "timeFrom": null,
 16148            "timeRegions": [],
 16149            "timeShift": null,
 16150            "title": "EtcdWorker txn size percentile",
 16151            "tooltip": {
 16152              "shared": true,
 16153              "sort": 0,
 16154              "value_type": "individual"
 16155            },
 16156            "type": "graph",
 16157            "xaxis": {
 16158              "buckets": null,
 16159              "mode": "time",
 16160              "name": null,
 16161              "show": true,
 16162              "values": []
 16163            },
 16164            "yaxes": [
 16165              {
 16166                "format": "bytes",
 16167                "label": null,
 16168                "logBase": 1,
 16169                "max": null,
 16170                "min": null,
 16171                "show": true
 16172              },
 16173              {
 16174                "format": "short",
 16175                "label": null,
 16176                "logBase": 1,
 16177                "max": null,
 16178                "min": null,
 16179                "show": true
 16180              }
 16181            ],
 16182            "yaxis": {
 16183              "align": false,
 16184              "alignLevel": null
 16185            }
 16186          },
 16187          {
 16188            "aliasColors": {},
 16189            "bars": false,
 16190            "dashLength": 10,
 16191            "dashes": false,
 16192            "datasource": "${DS_TEST-CLUSTER}",
 16193            "description": "The time consumed of writing WAL into the persistent storage in .99",
 16194            "editable": true,
 16195            "error": false,
 16196            "fieldConfig": {
 16197              "defaults": {},
 16198              "overrides": []
 16199            },
 16200            "fill": 1,
 16201            "fillGradient": 0,
 16202            "grid": {},
 16203            "gridPos": {
 16204              "h": 7,
 16205              "w": 12,
 16206              "x": 0,
 16207              "y": 41
 16208            },
 16209            "hiddenSeries": false,
 16210            "id": 291,
 16211            "legend": {
 16212              "alignAsTable": true,
 16213              "avg": false,
 16214              "current": true,
 16215              "hideEmpty": true,
 16216              "hideZero": true,
 16217              "max": true,
 16218              "min": false,
 16219              "rightSide": true,
 16220              "show": true,
 16221              "sideWidth": 300,
 16222              "total": false,
 16223              "values": true
 16224            },
 16225            "lines": true,
 16226            "linewidth": 1,
 16227            "links": [],
 16228            "nullPointMode": "null",
 16229            "options": {
 16230              "alertThreshold": true
 16231            },
 16232            "paceLength": 10,
 16233            "percentage": false,
 16234            "pluginVersion": "7.5.11",
 16235            "pointradius": 5,
 16236            "points": false,
 16237            "renderer": "flot",
 16238            "seriesOverrides": [],
 16239            "spaceLength": 10,
 16240            "stack": false,
 16241            "steppedLine": false,
 16242            "targets": [
 16243              {
 16244                "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (instance, le))",
 16245                "format": "time_series",
 16246                "intervalFactor": 2,
 16247                "legendFormat": "{{instance}}",
 16248                "metric": "",
 16249                "refId": "A",
 16250                "step": 4
 16251              }
 16252            ],
 16253            "thresholds": [],
 16254            "timeFrom": null,
 16255            "timeRegions": [],
 16256            "timeShift": null,
 16257            "title": "Etcd 99% WAL fsync duration",
 16258            "tooltip": {
 16259              "msResolution": false,
 16260              "shared": true,
 16261              "sort": 0,
 16262              "value_type": "cumulative"
 16263            },
 16264            "type": "graph",
 16265            "xaxis": {
 16266              "buckets": null,
 16267              "mode": "time",
 16268              "name": null,
 16269              "show": true,
 16270              "values": []
 16271            },
 16272            "yaxes": [
 16273              {
 16274                "format": "s",
 16275                "label": null,
 16276                "logBase": 1,
 16277                "max": null,
 16278                "min": null,
 16279                "show": true
 16280              },
 16281              {
 16282                "format": "short",
 16283                "label": null,
 16284                "logBase": 1,
 16285                "max": null,
 16286                "min": null,
 16287                "show": true
 16288              }
 16289            ],
 16290            "yaxis": {
 16291              "align": false,
 16292              "alignLevel": null
 16293            }
 16294          },
 16295          {
 16296            "aliasColors": {},
 16297            "bars": false,
 16298            "dashLength": 10,
 16299            "dashes": false,
 16300            "datasource": "${DS_TEST-CLUSTER}",
 16301            "description": "The time consumed of handling etcd transactions in .99",
 16302            "editable": true,
 16303            "error": false,
 16304            "fieldConfig": {
 16305              "defaults": {},
 16306              "overrides": []
 16307            },
 16308            "fill": 1,
 16309            "fillGradient": 0,
 16310            "grid": {},
 16311            "gridPos": {
 16312              "h": 7,
 16313              "w": 12,
 16314              "x": 12,
 16315              "y": 41
 16316            },
 16317            "hiddenSeries": false,
 16318            "id": 290,
 16319            "legend": {
 16320              "alignAsTable": true,
 16321              "avg": false,
 16322              "current": true,
 16323              "max": true,
 16324              "min": false,
 16325              "rightSide": true,
 16326              "show": true,
 16327              "sideWidth": 300,
 16328              "total": false,
 16329              "values": true
 16330            },
 16331            "lines": true,
 16332            "linewidth": 1,
 16333            "links": [],
 16334            "nullPointMode": "null",
 16335            "options": {
 16336              "alertThreshold": true
 16337            },
 16338            "paceLength": 10,
 16339            "percentage": false,
 16340            "pluginVersion": "7.5.11",
 16341            "pointradius": 5,
 16342            "points": false,
 16343            "renderer": "flot",
 16344            "seriesOverrides": [],
 16345            "spaceLength": 10,
 16346            "stack": false,
 16347            "steppedLine": false,
 16348            "targets": [
 16349              {
 16350                "expr": "histogram_quantile(0.99, sum(rate(pd_txn_handle_txns_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (instance, result, le))",
 16351                "format": "time_series",
 16352                "intervalFactor": 2,
 16353                "legendFormat": "{{instance}} {{result}}",
 16354                "refId": "A",
 16355                "step": 4
 16356              }
 16357            ],
 16358            "thresholds": [],
 16359            "timeFrom": null,
 16360            "timeRegions": [],
 16361            "timeShift": null,
 16362            "title": "Etcd 99% Handle transactions duration",
 16363            "tooltip": {
 16364              "msResolution": false,
 16365              "shared": true,
 16366              "sort": 0,
 16367              "value_type": "cumulative"
 16368            },
 16369            "type": "graph",
 16370            "xaxis": {
 16371              "buckets": null,
 16372              "mode": "time",
 16373              "name": null,
 16374              "show": true,
 16375              "values": []
 16376            },
 16377            "yaxes": [
 16378              {
 16379                "format": "s",
 16380                "label": null,
 16381                "logBase": 1,
 16382                "max": null,
 16383                "min": null,
 16384                "show": true
 16385              },
 16386              {
 16387                "format": "short",
 16388                "label": null,
 16389                "logBase": 1,
 16390                "max": null,
 16391                "min": null,
 16392                "show": true
 16393              }
 16394            ],
 16395            "yaxis": {
 16396              "align": false,
 16397              "alignLevel": null
 16398            }
 16399          }
 16400        ],
 16401        "title": "Etcd",
 16402        "type": "row"
 16403      },
 16404      {
 16405        "collapsed": true,
 16406        "datasource": null,
 16407        "gridPos": {
 16408          "h": 1,
 16409          "w": 24,
 16410          "x": 0,
 16411          "y": 13
 16412        },
 16413        "id": 58,
 16414        "panels": [
 16415          {
 16416            "aliasColors": {},
 16417            "bars": false,
 16418            "dashLength": 10,
 16419            "dashes": false,
 16420            "datasource": "${DS_TEST-CLUSTER}",
 16421            "decimals": 1,
 16422            "description": "CPU usage of TiKV CDC component",
 16423            "fieldConfig": {
 16424              "defaults": {},
 16425              "overrides": []
 16426            },
 16427            "fill": 1,
 16428            "fillGradient": 0,
 16429            "gridPos": {
 16430              "h": 7,
 16431              "w": 12,
 16432              "x": 0,
 16433              "y": 166
 16434            },
 16435            "hiddenSeries": false,
 16436            "id": 60,
 16437            "legend": {
 16438              "alignAsTable": true,
 16439              "avg": false,
 16440              "current": true,
 16441              "max": true,
 16442              "min": false,
 16443              "rightSide": true,
 16444              "show": true,
 16445              "sideWidth": null,
 16446              "sort": "current",
 16447              "sortDesc": true,
 16448              "total": false,
 16449              "values": true
 16450            },
 16451            "lines": true,
 16452            "linewidth": 1,
 16453            "links": [],
 16454            "nullPointMode": "null",
 16455            "options": {
 16456              "alertThreshold": true
 16457            },
 16458            "paceLength": 10,
 16459            "percentage": false,
 16460            "pluginVersion": "7.5.11",
 16461            "pointradius": 5,
 16462            "points": false,
 16463            "renderer": "flot",
 16464            "seriesOverrides": [
 16465              {
 16466                "alias": "/.*workers/",
 16467                "transform": "negative-Y",
 16468                "yaxis": 2
 16469              },
 16470              {
 16471                "alias": "/.*tso/",
 16472                "fill": 0,
 16473                "pointradius": 1,
 16474                "points": true
 16475              }
 16476            ],
 16477            "spaceLength": 10,
 16478            "stack": false,
 16479            "steppedLine": false,
 16480            "targets": [
 16481              {
 16482                "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"cdc_.*|cdc\"}[1m])) by (instance)",
 16483                "format": "time_series",
 16484                "intervalFactor": 2,
 16485                "legendFormat": "{{instance}}-endpoint",
 16486                "refId": "A",
 16487                "step": 4
 16488              },
 16489              {
 16490                "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"cdcwkr.*\"}[1m])) by (instance)",
 16491                "format": "time_series",
 16492                "intervalFactor": 2,
 16493                "legendFormat": "{{instance}}-workers",
 16494                "refId": "B",
 16495                "step": 4
 16496              },
 16497              {
 16498                "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"tso\"}[1m])) by (instance)",
 16499                "format": "time_series",
 16500                "hide": false,
 16501                "intervalFactor": 2,
 16502                "legendFormat": "{{instance}}-tso",
 16503                "refId": "C",
 16504                "step": 4
 16505              }
 16506            ],
 16507            "thresholds": [],
 16508            "timeFrom": null,
 16509            "timeRegions": [],
 16510            "timeShift": null,
 16511            "title": "CDC CPU",
 16512            "tooltip": {
 16513              "shared": true,
 16514              "sort": 0,
 16515              "value_type": "individual"
 16516            },
 16517            "type": "graph",
 16518            "xaxis": {
 16519              "buckets": null,
 16520              "mode": "time",
 16521              "name": null,
 16522              "show": true,
 16523              "values": []
 16524            },
 16525            "yaxes": [
 16526              {
 16527                "format": "percentunit",
 16528                "label": null,
 16529                "logBase": 1,
 16530                "max": null,
 16531                "min": null,
 16532                "show": true
 16533              },
 16534              {
 16535                "format": "percent",
 16536                "label": null,
 16537                "logBase": 1,
 16538                "max": null,
 16539                "min": null,
 16540                "show": true
 16541              }
 16542            ],
 16543            "yaxis": {
 16544              "align": false,
 16545              "alignLevel": null
 16546            }
 16547          },
 16548          {
 16549            "aliasColors": {},
 16550            "bars": true,
 16551            "dashLength": 10,
 16552            "dashes": false,
 16553            "datasource": "${DS_TEST-CLUSTER}",
 16554            "decimals": 1,
 16555            "description": "Outbound network traffic of TiKV CDC component",
 16556            "editable": true,
 16557            "error": false,
 16558            "fieldConfig": {
 16559              "defaults": {},
 16560              "overrides": []
 16561            },
 16562            "fill": 0,
 16563            "fillGradient": 0,
 16564            "grid": {},
 16565            "gridPos": {
 16566              "h": 7,
 16567              "w": 12,
 16568              "x": 12,
 16569              "y": 166
 16570            },
 16571            "hiddenSeries": false,
 16572            "id": 74,
 16573            "legend": {
 16574              "alignAsTable": true,
 16575              "avg": false,
 16576              "current": true,
 16577              "hideEmpty": false,
 16578              "hideZero": false,
 16579              "max": true,
 16580              "min": false,
 16581              "rightSide": true,
 16582              "show": true,
 16583              "sideWidth": null,
 16584              "sort": "max",
 16585              "sortDesc": true,
 16586              "total": false,
 16587              "values": true
 16588            },
 16589            "lines": false,
 16590            "linewidth": 1,
 16591            "links": [],
 16592            "nullPointMode": "null",
 16593            "options": {
 16594              "alertThreshold": true
 16595            },
 16596            "paceLength": 10,
 16597            "percentage": false,
 16598            "pluginVersion": "7.5.11",
 16599            "pointradius": 5,
 16600            "points": false,
 16601            "renderer": "flot",
 16602            "seriesOverrides": [
 16603              {
 16604                "alias": "/.*resolved_ts/",
 16605                "bars": false,
 16606                "fill": 1,
 16607                "lines": true,
 16608                "linewidth": 2,
 16609                "transform": "negative-Y",
 16610                "yaxis": 2
 16611              }
 16612            ],
 16613            "spaceLength": 10,
 16614            "stack": false,
 16615            "steppedLine": false,
 16616            "targets": [
 16617              {
 16618                "expr": "sum(rate(tikv_cdc_grpc_message_sent_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[30s])) by (instance, type)",
 16619                "format": "time_series",
 16620                "hide": false,
 16621                "intervalFactor": 2,
 16622                "legendFormat": "{{instance}}-{{type}}",
 16623                "refId": "A",
 16624                "step": 10
 16625              }
 16626            ],
 16627            "thresholds": [],
 16628            "timeFrom": null,
 16629            "timeRegions": [],
 16630            "timeShift": null,
 16631            "title": "CDC network traffic",
 16632            "tooltip": {
 16633              "msResolution": false,
 16634              "shared": true,
 16635              "sort": 0,
 16636              "value_type": "individual"
 16637            },
 16638            "type": "graph",
 16639            "xaxis": {
 16640              "buckets": null,
 16641              "mode": "time",
 16642              "name": null,
 16643              "show": true,
 16644              "values": []
 16645            },
 16646            "yaxes": [
 16647              {
 16648                "format": "Bps",
 16649                "label": null,
 16650                "logBase": 1,
 16651                "max": null,
 16652                "min": null,
 16653                "show": true
 16654              },
 16655              {
 16656                "format": "bytes",
 16657                "label": null,
 16658                "logBase": 1,
 16659                "max": null,
 16660                "min": null,
 16661                "show": true
 16662              }
 16663            ],
 16664            "yaxis": {
 16665              "align": false,
 16666              "alignLevel": 0
 16667            }
 16668          },
 16669          {
 16670            "aliasColors": {},
 16671            "bars": false,
 16672            "dashLength": 10,
 16673            "dashes": false,
 16674            "datasource": "${DS_TEST-CLUSTER}",
 16675            "decimals": 1,
 16676            "description": "The count of different kinds of gRPC message",
 16677            "fieldConfig": {
 16678              "defaults": {},
 16679              "overrides": []
 16680            },
 16681            "fill": 1,
 16682            "fillGradient": 0,
 16683            "gridPos": {
 16684              "h": 7,
 16685              "w": 12,
 16686              "x": 0,
 16687              "y": 173
 16688            },
 16689            "hiddenSeries": false,
 16690            "id": 147,
 16691            "legend": {
 16692              "alignAsTable": true,
 16693              "avg": false,
 16694              "current": true,
 16695              "hideEmpty": true,
 16696              "hideZero": true,
 16697              "max": true,
 16698              "min": false,
 16699              "rightSide": true,
 16700              "show": true,
 16701              "sideWidth": null,
 16702              "sort": "current",
 16703              "sortDesc": true,
 16704              "total": false,
 16705              "values": true
 16706            },
 16707            "lines": true,
 16708            "linewidth": 1,
 16709            "links": [],
 16710            "nullPointMode": "null",
 16711            "options": {
 16712              "alertThreshold": true
 16713            },
 16714            "percentage": false,
 16715            "pluginVersion": "7.5.11",
 16716            "pointradius": 5,
 16717            "points": false,
 16718            "renderer": "flot",
 16719            "seriesOverrides": [],
 16720            "spaceLength": 10,
 16721            "stack": false,
 16722            "steppedLine": false,
 16723            "targets": [
 16724              {
 16725                "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type!=\"kv_gc\"}[1m])) by (type)",
 16726                "format": "time_series",
 16727                "intervalFactor": 2,
 16728                "legendFormat": "{{type}}",
 16729                "metric": "tikv_grpc_msg_duration_seconds_bucket",
 16730                "refId": "A",
 16731                "step": 10
 16732              }
 16733            ],
 16734            "thresholds": [],
 16735            "timeFrom": null,
 16736            "timeRegions": [],
 16737            "timeShift": null,
 16738            "title": "gRPC message count",
 16739            "tooltip": {
 16740              "shared": true,
 16741              "sort": 0,
 16742              "value_type": "individual"
 16743            },
 16744            "type": "graph",
 16745            "xaxis": {
 16746              "buckets": null,
 16747              "mode": "time",
 16748              "name": null,
 16749              "show": true,
 16750              "values": []
 16751            },
 16752            "yaxes": [
 16753              {
 16754                "format": "ops",
 16755                "label": null,
 16756                "logBase": 1,
 16757                "max": null,
 16758                "min": null,
 16759                "show": true
 16760              },
 16761              {
 16762                "format": "short",
 16763                "label": null,
 16764                "logBase": 1,
 16765                "max": null,
 16766                "min": null,
 16767                "show": true
 16768              }
 16769            ],
 16770            "yaxis": {
 16771              "align": false,
 16772              "alignLevel": null
 16773            }
 16774          },
 16775          {
 16776            "aliasColors": {},
 16777            "bars": false,
 16778            "dashLength": 10,
 16779            "dashes": false,
 16780            "datasource": "${DS_TEST-CLUSTER}",
 16781            "decimals": 1,
 16782            "description": "The memory usage per TiKV instance",
 16783            "editable": true,
 16784            "error": false,
 16785            "fieldConfig": {
 16786              "defaults": {},
 16787              "overrides": []
 16788            },
 16789            "fill": 0,
 16790            "fillGradient": 0,
 16791            "grid": {},
 16792            "gridPos": {
 16793              "h": 7,
 16794              "w": 12,
 16795              "x": 12,
 16796              "y": 173
 16797            },
 16798            "hiddenSeries": false,
 16799            "id": 194,
 16800            "legend": {
 16801              "alignAsTable": true,
 16802              "avg": false,
 16803              "current": true,
 16804              "max": true,
 16805              "min": false,
 16806              "rightSide": true,
 16807              "show": true,
 16808              "sideWidth": null,
 16809              "sort": "max",
 16810              "sortDesc": true,
 16811              "total": false,
 16812              "values": true
 16813            },
 16814            "lines": true,
 16815            "linewidth": 1,
 16816            "links": [],
 16817            "nullPointMode": "null",
 16818            "options": {
 16819              "alertThreshold": true
 16820            },
 16821            "paceLength": 10,
 16822            "percentage": false,
 16823            "pluginVersion": "7.5.11",
 16824            "pointradius": 5,
 16825            "points": false,
 16826            "renderer": "flot",
 16827            "seriesOverrides": [
 16828              {
 16829                "alias": "/.*-cap-.*/",
 16830                "yaxis": 2
 16831              },
 16832              {
 16833                "alias": "/.*tikv.*/",
 16834                "pointradius": 1,
 16835                "points": true,
 16836                "yaxis": 2
 16837              }
 16838            ],
 16839            "spaceLength": 10,
 16840            "stack": false,
 16841            "steppedLine": false,
 16842            "targets": [
 16843              {
 16844                "expr": "avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=~\"tikv.*\"}) by (instance)",
 16845                "format": "time_series",
 16846                "hide": true,
 16847                "intervalFactor": 2,
 16848                "legendFormat": "tikv-{{instance}}",
 16849                "refId": "A",
 16850                "step": 10
 16851              },
 16852              {
 16853                "expr": "avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=~\"cdc.*\"}) by (instance)",
 16854                "format": "time_series",
 16855                "hide": true,
 16856                "intervalFactor": 2,
 16857                "legendFormat": "cdc-{{instance}}",
 16858                "refId": "B",
 16859                "step": 10
 16860              },
 16861              {
 16862                "expr": "(avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=~\"tikv.*\"}) by (instance)) - (avg(tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", db=\"kv\"}) by(instance))",
 16863                "format": "time_series",
 16864                "hide": false,
 16865                "intervalFactor": 2,
 16866                "legendFormat": "tikv-{{instance}}",
 16867                "refId": "C",
 16868                "step": 10
 16869              },
 16870              {
 16871                "expr": "sum(tikv_cdc_sink_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)",
 16872                "format": "time_series",
 16873                "hide": false,
 16874                "intervalFactor": 2,
 16875                "legendFormat": "sink-{{instance}}",
 16876                "refId": "D",
 16877                "step": 10
 16878              },
 16879              {
 16880                "expr": "sum(tikv_cdc_old_value_cache_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)",
 16881                "format": "time_series",
 16882                "hide": false,
 16883                "intervalFactor": 2,
 16884                "legendFormat": "old-value-{{instance}}",
 16885                "refId": "E",
 16886                "step": 10
 16887              },
 16888              {
 16889                "expr": "sum(tikv_cdc_sink_memory_capacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)",
 16890                "format": "time_series",
 16891                "hide": true,
 16892                "intervalFactor": 2,
 16893                "legendFormat": "sink-cap-{{instance}}",
 16894                "refId": "F",
 16895                "step": 10
 16896              },
 16897              {
 16898                "expr": "sum(tikv_cdc_old_value_cache_memory_quota{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)",
 16899                "format": "time_series",
 16900                "hide": true,
 16901                "intervalFactor": 2,
 16902                "legendFormat": "old-value-cap-{{instance}}",
 16903                "refId": "G",
 16904                "step": 10
 16905              }
 16906            ],
 16907            "thresholds": [],
 16908            "timeFrom": null,
 16909            "timeRegions": [],
 16910            "timeShift": null,
 16911            "title": "CDC memory",
 16912            "tooltip": {
 16913              "msResolution": false,
 16914              "shared": true,
 16915              "sort": 0,
 16916              "value_type": "individual"
 16917            },
 16918            "type": "graph",
 16919            "xaxis": {
 16920              "buckets": null,
 16921              "mode": "time",
 16922              "name": null,
 16923              "show": true,
 16924              "values": []
 16925            },
 16926            "yaxes": [
 16927              {
 16928                "format": "bytes",
 16929                "label": null,
 16930                "logBase": 1,
 16931                "max": null,
 16932                "min": "0",
 16933                "show": true
 16934              },
 16935              {
 16936                "format": "bytes",
 16937                "label": null,
 16938                "logBase": 1,
 16939                "max": null,
 16940                "min": null,
 16941                "show": true
 16942              }
 16943            ],
 16944            "yaxis": {
 16945              "align": false,
 16946              "alignLevel": null
 16947            }
 16948          },
 16949          {
 16950            "aliasColors": {},
 16951            "bars": false,
 16952            "dashLength": 10,
 16953            "dashes": false,
 16954            "datasource": "${DS_TEST-CLUSTER}",
 16955            "decimals": 1,
 16956            "description": "The min resolved ts of each TiKV",
 16957            "editable": true,
 16958            "error": false,
 16959            "fieldConfig": {
 16960              "defaults": {},
 16961              "overrides": []
 16962            },
 16963            "fill": 0,
 16964            "fillGradient": 0,
 16965            "grid": {},
 16966            "gridPos": {
 16967              "h": 7,
 16968              "w": 7,
 16969              "x": 0,
 16970              "y": 180
 16971            },
 16972            "hiddenSeries": false,
 16973            "id": 152,
 16974            "legend": {
 16975              "alignAsTable": true,
 16976              "avg": false,
 16977              "current": true,
 16978              "max": false,
 16979              "min": false,
 16980              "rightSide": false,
 16981              "show": true,
 16982              "sideWidth": null,
 16983              "sort": null,
 16984              "sortDesc": null,
 16985              "total": false,
 16986              "values": true
 16987            },
 16988            "lines": true,
 16989            "linewidth": 1,
 16990            "links": [],
 16991            "nullPointMode": "null",
 16992            "options": {
 16993              "alertThreshold": true
 16994            },
 16995            "paceLength": 10,
 16996            "percentage": false,
 16997            "pluginVersion": "7.5.11",
 16998            "pointradius": 1,
 16999            "points": false,
 17000            "renderer": "flot",
 17001            "seriesOverrides": [
 17002              {
 17003                "alias": "/.*-ts/",
 17004                "lines": true,
 17005                "linewidth": 3,
 17006                "points": false,
 17007                "yaxis": 2
 17008              },
 17009              {
 17010                "alias": "/.*-lag/",
 17011                "bars": true,
 17012                "fill": 1
 17013              }
 17014            ],
 17015            "spaceLength": 10,
 17016            "stack": false,
 17017            "steppedLine": false,
 17018            "targets": [
 17019              {
 17020                "expr": "scalar(max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}))/1000 - avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000) by (instance) > 0",
 17021                "format": "time_series",
 17022                "hide": false,
 17023                "instant": false,
 17024                "intervalFactor": 2,
 17025                "legendFormat": "{{instance}}-min-resolved-lag",
 17026                "refId": "A",
 17027                "step": 10
 17028              },
 17029              {
 17030                "expr": "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})",
 17031                "format": "time_series",
 17032                "hide": false,
 17033                "intervalFactor": 2,
 17034                "legendFormat": "current-ts",
 17035                "refId": "B",
 17036                "step": 10
 17037              },
 17038              {
 17039                "expr": "avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)",
 17040                "format": "time_series",
 17041                "hide": false,
 17042                "intervalFactor": 2,
 17043                "legendFormat": "{{instance}}-min-resolved-ts",
 17044                "refId": "C",
 17045                "step": 10
 17046              }
 17047            ],
 17048            "thresholds": [],
 17049            "timeFrom": null,
 17050            "timeRegions": [],
 17051            "timeShift": null,
 17052            "title": "Min resolved ts",
 17053            "tooltip": {
 17054              "msResolution": false,
 17055              "shared": true,
 17056              "sort": 0,
 17057              "value_type": "individual"
 17058            },
 17059            "type": "graph",
 17060            "xaxis": {
 17061              "buckets": null,
 17062              "mode": "time",
 17063              "name": null,
 17064              "show": true,
 17065              "values": []
 17066            },
 17067            "yaxes": [
 17068              {
 17069                "format": "s",
 17070                "label": null,
 17071                "logBase": 1,
 17072                "max": null,
 17073                "min": "0",
 17074                "show": true
 17075              },
 17076              {
 17077                "format": "none",
 17078                "label": null,
 17079                "logBase": 1,
 17080                "max": null,
 17081                "min": null,
 17082                "show": true
 17083              }
 17084            ],
 17085            "yaxis": {
 17086              "align": false,
 17087              "alignLevel": null
 17088            }
 17089          },
 17090          {
 17091            "aliasColors": {},
 17092            "bars": false,
 17093            "dashLength": 10,
 17094            "dashes": false,
 17095            "datasource": "${DS_TEST-CLUSTER}",
 17096            "decimals": 1,
 17097            "description": "The ID of the min resolved region of each TiKV",
 17098            "editable": true,
 17099            "error": false,
 17100            "fieldConfig": {
 17101              "defaults": {},
 17102              "overrides": []
 17103            },
 17104            "fill": 0,
 17105            "fillGradient": 0,
 17106            "grid": {},
 17107            "gridPos": {
 17108              "h": 7,
 17109              "w": 5,
 17110              "x": 7,
 17111              "y": 180
 17112            },
 17113            "hiddenSeries": false,
 17114            "id": 153,
 17115            "legend": {
 17116              "alignAsTable": true,
 17117              "avg": false,
 17118              "current": true,
 17119              "max": true,
 17120              "min": false,
 17121              "rightSide": false,
 17122              "show": true,
 17123              "sideWidth": null,
 17124              "sort": null,
 17125              "sortDesc": null,
 17126              "total": false,
 17127              "values": true
 17128            },
 17129            "lines": false,
 17130            "linewidth": 1,
 17131            "links": [],
 17132            "nullPointMode": "null",
 17133            "options": {
 17134              "alertThreshold": true
 17135            },
 17136            "paceLength": 10,
 17137            "percentage": false,
 17138            "pluginVersion": "7.5.11",
 17139            "pointradius": 1,
 17140            "points": true,
 17141            "renderer": "flot",
 17142            "seriesOverrides": [],
 17143            "spaceLength": 10,
 17144            "stack": false,
 17145            "steppedLine": false,
 17146            "targets": [
 17147              {
 17148                "exemplar": true,
 17149                "expr": "avg(tikv_cdc_min_resolved_ts_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)",
 17150                "format": "time_series",
 17151                "hide": false,
 17152                "interval": "",
 17153                "intervalFactor": 2,
 17154                "legendFormat": "{{instance}}-min-resolved-region",
 17155                "refId": "A",
 17156                "step": 10
 17157              }
 17158            ],
 17159            "thresholds": [],
 17160            "timeFrom": null,
 17161            "timeRegions": [],
 17162            "timeShift": null,
 17163            "title": "Min resolved Region",
 17164            "tooltip": {
 17165              "msResolution": false,
 17166              "shared": true,
 17167              "sort": 0,
 17168              "value_type": "individual"
 17169            },
 17170            "type": "graph",
 17171            "xaxis": {
 17172              "buckets": null,
 17173              "mode": "time",
 17174              "name": null,
 17175              "show": true,
 17176              "values": []
 17177            },
 17178            "yaxes": [
 17179              {
 17180                "format": "none",
 17181                "label": null,
 17182                "logBase": 1,
 17183                "max": null,
 17184                "min": "0",
 17185                "show": true
 17186              },
 17187              {
 17188                "format": "none",
 17189                "label": null,
 17190                "logBase": 1,
 17191                "max": null,
 17192                "min": null,
 17193                "show": false
 17194              }
 17195            ],
 17196            "yaxis": {
 17197              "align": false,
 17198              "alignLevel": null
 17199            }
 17200          },
 17201          {
 17202            "aliasColors": {},
 17203            "bars": false,
 17204            "dashLength": 10,
 17205            "dashes": false,
 17206            "datasource": "${DS_TEST-CLUSTER}",
 17207            "decimals": 1,
 17208            "description": "",
 17209            "fieldConfig": {
 17210              "defaults": {},
 17211              "overrides": []
 17212            },
 17213            "fill": 1,
 17214            "fillGradient": 0,
 17215            "gridPos": {
 17216              "h": 7,
 17217              "w": 12,
 17218              "x": 12,
 17219              "y": 180
 17220            },
 17221            "hiddenSeries": false,
 17222            "id": 70,
 17223            "legend": {
 17224              "alignAsTable": true,
 17225              "avg": false,
 17226              "current": true,
 17227              "hideEmpty": false,
 17228              "hideZero": true,
 17229              "max": true,
 17230              "min": false,
 17231              "rightSide": true,
 17232              "show": true,
 17233              "sideWidth": null,
 17234              "sort": null,
 17235              "sortDesc": null,
 17236              "total": false,
 17237              "values": true
 17238            },
 17239            "lines": true,
 17240            "linewidth": 1,
 17241            "links": [],
 17242            "nullPointMode": "null",
 17243            "options": {
 17244              "alertThreshold": true
 17245            },
 17246            "paceLength": 10,
 17247            "percentage": false,
 17248            "pluginVersion": "7.5.11",
 17249            "pointradius": 5,
 17250            "points": false,
 17251            "renderer": "flot",
 17252            "seriesOverrides": [],
 17253            "spaceLength": 10,
 17254            "stack": false,
 17255            "steppedLine": false,
 17256            "targets": [
 17257              {
 17258                "expr": "histogram_quantile(0.99999, sum(rate(tikv_cdc_resolved_ts_gap_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance))",
 17259                "format": "time_series",
 17260                "intervalFactor": 1,
 17261                "legendFormat": "{{instance}}-p9999",
 17262                "refId": "A"
 17263              }
 17264            ],
 17265            "thresholds": [],
 17266            "timeFrom": null,
 17267            "timeRegions": [],
 17268            "timeShift": null,
 17269            "title": "Resolved ts lag duration percentile",
 17270            "tooltip": {
 17271              "shared": true,
 17272              "sort": 0,
 17273              "value_type": "individual"
 17274            },
 17275            "type": "graph",
 17276            "xaxis": {
 17277              "buckets": null,
 17278              "mode": "time",
 17279              "name": null,
 17280              "show": true,
 17281              "values": []
 17282            },
 17283            "yaxes": [
 17284              {
 17285                "format": "s",
 17286                "label": null,
 17287                "logBase": 10,
 17288                "max": null,
 17289                "min": null,
 17290                "show": true
 17291              },
 17292              {
 17293                "format": "short",
 17294                "label": null,
 17295                "logBase": 1,
 17296                "max": null,
 17297                "min": null,
 17298                "show": true
 17299              }
 17300            ],
 17301            "yaxis": {
 17302              "align": false,
 17303              "alignLevel": null
 17304            }
 17305          },
 17306          {
 17307            "cards": {
 17308              "cardPadding": 0,
 17309              "cardRound": 0
 17310            },
 17311            "color": {
 17312              "cardColor": "#FF9830",
 17313              "colorScale": "linear",
 17314              "colorScheme": "interpolateSpectral",
 17315              "exponent": 0.5,
 17316              "min": 0,
 17317              "mode": "spectrum"
 17318            },
 17319            "dataFormat": "tsbuckets",
 17320            "datasource": "${DS_TEST-CLUSTER}",
 17321            "description": "The time consumed to CDC incremental scan",
 17322            "fieldConfig": {
 17323              "defaults": {},
 17324              "overrides": []
 17325            },
 17326            "gridPos": {
 17327              "h": 7,
 17328              "w": 12,
 17329              "x": 0,
 17330              "y": 187
 17331            },
 17332            "heatmap": {},
 17333            "hideZeroBuckets": true,
 17334            "highlightCards": true,
 17335            "id": 68,
 17336            "legend": {
 17337              "alignAsTable": true,
 17338              "avg": false,
 17339              "current": true,
 17340              "max": true,
 17341              "min": false,
 17342              "rightSide": true,
 17343              "show": true,
 17344              "sort": "current",
 17345              "sortDesc": true,
 17346              "total": false,
 17347              "values": true
 17348            },
 17349            "links": [],
 17350            "maxPerRow": 3,
 17351            "repeat": null,
 17352            "repeatDirection": "h",
 17353            "reverseYBuckets": false,
 17354            "targets": [
 17355              {
 17356                "expr": "sum(rate(tikv_cdc_scan_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)",
 17357                "format": "heatmap",
 17358                "instant": false,
 17359                "intervalFactor": 2,
 17360                "legendFormat": "{{le}}",
 17361                "refId": "A"
 17362              }
 17363            ],
 17364            "title": "Initial scan duration",
 17365            "tooltip": {
 17366              "show": true,
 17367              "showHistogram": true
 17368            },
 17369            "tooltipDecimals": 1,
 17370            "type": "heatmap",
 17371            "xAxis": {
 17372              "show": true
 17373            },
 17374            "xBucketNumber": null,
 17375            "xBucketSize": null,
 17376            "yAxis": {
 17377              "decimals": 1,
 17378              "format": "s",
 17379              "logBase": 1,
 17380              "max": null,
 17381              "min": null,
 17382              "show": true,
 17383              "splitFactor": null
 17384            },
 17385            "yBucketBound": "upper",
 17386            "yBucketNumber": null,
 17387            "yBucketSize": null
 17388          },
 17389          {
 17390            "aliasColors": {},
 17391            "bars": false,
 17392            "dashLength": 10,
 17393            "dashes": false,
 17394            "datasource": "${DS_TEST-CLUSTER}",
 17395            "decimals": 1,
 17396            "description": "",
 17397            "fieldConfig": {
 17398              "defaults": {},
 17399              "overrides": []
 17400            },
 17401            "fill": 1,
 17402            "fillGradient": 0,
 17403            "gridPos": {
 17404              "h": 7,
 17405              "w": 6,
 17406              "x": 12,
 17407              "y": 187
 17408            },
 17409            "hiddenSeries": false,
 17410            "id": 72,
 17411            "legend": {
 17412              "alignAsTable": true,
 17413              "avg": false,
 17414              "current": true,
 17415              "hideEmpty": false,
 17416              "hideZero": true,
 17417              "max": true,
 17418              "min": false,
 17419              "rightSide": false,
 17420              "show": true,
 17421              "sideWidth": null,
 17422              "sort": "current",
 17423              "sortDesc": true,
 17424              "total": false,
 17425              "values": true
 17426            },
 17427            "lines": true,
 17428            "linewidth": 1,
 17429            "links": [],
 17430            "nullPointMode": "null",
 17431            "options": {
 17432              "alertThreshold": true
 17433            },
 17434            "paceLength": 10,
 17435            "percentage": false,
 17436            "pluginVersion": "7.5.11",
 17437            "pointradius": 5,
 17438            "points": false,
 17439            "renderer": "flot",
 17440            "seriesOverrides": [],
 17441            "spaceLength": 10,
 17442            "stack": false,
 17443            "steppedLine": false,
 17444            "targets": [
 17445              {
 17446                "expr": "histogram_quantile(0.9999, sum(rate(tikv_cdc_scan_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance))",
 17447                "format": "time_series",
 17448                "intervalFactor": 1,
 17449                "legendFormat": "{{instance}}-p9999",
 17450                "refId": "A"
 17451              }
 17452            ],
 17453            "thresholds": [],
 17454            "timeFrom": null,
 17455            "timeRegions": [],
 17456            "timeShift": null,
 17457            "title": "Initial scan duration percentile",
 17458            "tooltip": {
 17459              "shared": true,
 17460              "sort": 0,
 17461              "value_type": "individual"
 17462            },
 17463            "type": "graph",
 17464            "xaxis": {
 17465              "buckets": null,
 17466              "mode": "time",
 17467              "name": null,
 17468              "show": true,
 17469              "values": []
 17470            },
 17471            "yaxes": [
 17472              {
 17473                "format": "s",
 17474                "label": null,
 17475                "logBase": 1,
 17476                "max": null,
 17477                "min": null,
 17478                "show": true
 17479              },
 17480              {
 17481                "format": "short",
 17482                "label": null,
 17483                "logBase": 1,
 17484                "max": null,
 17485                "min": null,
 17486                "show": false
 17487              }
 17488            ],
 17489            "yaxis": {
 17490              "align": false,
 17491              "alignLevel": null
 17492            }
 17493          },
 17494          {
 17495            "aliasColors": {},
 17496            "bars": false,
 17497            "dashLength": 10,
 17498            "dashes": false,
 17499            "datasource": "${DS_TEST-CLUSTER}",
 17500            "decimals": 1,
 17501            "description": "The number of incremental scan task in different status.",
 17502            "fieldConfig": {
 17503              "defaults": {},
 17504              "overrides": []
 17505            },
 17506            "fill": 1,
 17507            "fillGradient": 0,
 17508            "gridPos": {
 17509              "h": 7,
 17510              "w": 6,
 17511              "x": 18,
 17512              "y": 187
 17513            },
 17514            "hiddenSeries": false,
 17515            "id": 140,
 17516            "legend": {
 17517              "alignAsTable": true,
 17518              "avg": false,
 17519              "current": true,
 17520              "hideEmpty": false,
 17521              "hideZero": true,
 17522              "max": true,
 17523              "min": false,
 17524              "rightSide": false,
 17525              "show": true,
 17526              "sideWidth": null,
 17527              "sort": "current",
 17528              "sortDesc": true,
 17529              "total": false,
 17530              "values": true
 17531            },
 17532            "lines": true,
 17533            "linewidth": 1,
 17534            "links": [],
 17535            "nullPointMode": "null",
 17536            "options": {
 17537              "alertThreshold": true
 17538            },
 17539            "paceLength": 10,
 17540            "percentage": false,
 17541            "pluginVersion": "7.5.11",
 17542            "pointradius": 5,
 17543            "points": false,
 17544            "renderer": "flot",
 17545            "seriesOverrides": [
 17546              {
 17547                "alias": "/.*ongoing/",
 17548                "yaxis": 2
 17549              }
 17550            ],
 17551            "spaceLength": 10,
 17552            "stack": false,
 17553            "steppedLine": false,
 17554            "targets": [
 17555              {
 17556                "expr": "sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"ongoing\"}) by (type, instance)",
 17557                "format": "time_series",
 17558                "hide": false,
 17559                "intervalFactor": 1,
 17560                "legendFormat": "{{instance}}-{{type}}",
 17561                "refId": "A"
 17562              },
 17563              {
 17564                "expr": "sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"total\"}) by (instance) - sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=~\"abort|finish\"}) by (instance)",
 17565                "format": "time_series",
 17566                "hide": false,
 17567                "intervalFactor": 1,
 17568                "legendFormat": "{{instance}}-pending",
 17569                "refId": "B"
 17570              }
 17571            ],
 17572            "thresholds": [],
 17573            "timeFrom": null,
 17574            "timeRegions": [],
 17575            "timeShift": null,
 17576            "title": "Initial scan tasks status",
 17577            "tooltip": {
 17578              "shared": true,
 17579              "sort": 0,
 17580              "value_type": "individual"
 17581            },
 17582            "type": "graph",
 17583            "xaxis": {
 17584              "buckets": null,
 17585              "mode": "time",
 17586              "name": null,
 17587              "show": true,
 17588              "values": []
 17589            },
 17590            "yaxes": [
 17591              {
 17592                "format": "none",
 17593                "label": null,
 17594                "logBase": 1,
 17595                "max": null,
 17596                "min": null,
 17597                "show": true
 17598              },
 17599              {
 17600                "format": "none",
 17601                "label": null,
 17602                "logBase": 1,
 17603                "max": null,
 17604                "min": null,
 17605                "show": true
 17606              }
 17607            ],
 17608            "yaxis": {
 17609              "align": false,
 17610              "alignLevel": null
 17611            }
 17612          },
 17613          {
 17614            "aliasColors": {},
 17615            "bars": false,
 17616            "dashLength": 10,
 17617            "dashes": false,
 17618            "datasource": "${DS_TEST-CLUSTER}",
 17619            "decimals": 1,
 17620            "description": "The memory usage per TiKV instance",
 17621            "editable": true,
 17622            "error": false,
 17623            "fieldConfig": {
 17624              "defaults": {},
 17625              "overrides": []
 17626            },
 17627            "fill": 0,
 17628            "fillGradient": 0,
 17629            "grid": {},
 17630            "gridPos": {
 17631              "h": 7,
 17632              "w": 12,
 17633              "x": 0,
 17634              "y": 194
 17635            },
 17636            "hiddenSeries": false,
 17637            "id": 78,
 17638            "legend": {
 17639              "alignAsTable": true,
 17640              "avg": false,
 17641              "current": true,
 17642              "max": true,
 17643              "min": false,
 17644              "rightSide": true,
 17645              "show": true,
 17646              "sideWidth": null,
 17647              "sort": null,
 17648              "sortDesc": null,
 17649              "total": false,
 17650              "values": true
 17651            },
 17652            "lines": true,
 17653            "linewidth": 1,
 17654            "links": [],
 17655            "nullPointMode": "null",
 17656            "options": {
 17657              "alertThreshold": true
 17658            },
 17659            "paceLength": 10,
 17660            "percentage": false,
 17661            "pluginVersion": "7.5.11",
 17662            "pointradius": 5,
 17663            "points": false,
 17664            "renderer": "flot",
 17665            "seriesOverrides": [],
 17666            "spaceLength": 10,
 17667            "stack": false,
 17668            "steppedLine": false,
 17669            "targets": [
 17670              {
 17671                "expr": "avg(tikv_cdc_captured_region_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)",
 17672                "format": "time_series",
 17673                "hide": false,
 17674                "intervalFactor": 2,
 17675                "legendFormat": "tikv-{{instance}}-total",
 17676                "refId": "A",
 17677                "step": 10
 17678              },
 17679              {
 17680                "expr": "sum(tikv_cdc_region_resolve_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance, status)",
 17681                "format": "time_series",
 17682                "hide": false,
 17683                "intervalFactor": 2,
 17684                "legendFormat": "tikv-{{instance}}-{{status}}",
 17685                "refId": "B",
 17686                "step": 10
 17687              }
 17688            ],
 17689            "thresholds": [],
 17690            "timeFrom": null,
 17691            "timeRegions": [],
 17692            "timeShift": null,
 17693            "title": "Captured region count",
 17694            "tooltip": {
 17695              "msResolution": false,
 17696              "shared": true,
 17697              "sort": 0,
 17698              "value_type": "individual"
 17699            },
 17700            "type": "graph",
 17701            "xaxis": {
 17702              "buckets": null,
 17703              "mode": "time",
 17704              "name": null,
 17705              "show": true,
 17706              "values": []
 17707            },
 17708            "yaxes": [
 17709              {
 17710                "format": "none",
 17711                "label": null,
 17712                "logBase": 1,
 17713                "max": null,
 17714                "min": "0",
 17715                "show": true
 17716              },
 17717              {
 17718                "format": "short",
 17719                "label": null,
 17720                "logBase": 1,
 17721                "max": null,
 17722                "min": null,
 17723                "show": true
 17724              }
 17725            ],
 17726            "yaxis": {
 17727              "align": false,
 17728              "alignLevel": null
 17729            }
 17730          },
 17731          {
 17732            "aliasColors": {},
 17733            "bars": false,
 17734            "dashLength": 10,
 17735            "dashes": false,
 17736            "datasource": "${DS_TEST-CLUSTER}",
 17737            "decimals": 1,
 17738            "description": "The speed of TiKV CDC incremental scan",
 17739            "editable": true,
 17740            "error": false,
 17741            "fieldConfig": {
 17742              "defaults": {},
 17743              "overrides": []
 17744            },
 17745            "fill": 0,
 17746            "fillGradient": 0,
 17747            "grid": {},
 17748            "gridPos": {
 17749              "h": 7,
 17750              "w": 6,
 17751              "x": 12,
 17752              "y": 194
 17753            },
 17754            "hiddenSeries": false,
 17755            "id": 76,
 17756            "legend": {
 17757              "alignAsTable": true,
 17758              "avg": false,
 17759              "current": true,
 17760              "hideEmpty": true,
 17761              "hideZero": true,
 17762              "max": true,
 17763              "min": false,
 17764              "rightSide": false,
 17765              "show": true,
 17766              "sideWidth": null,
 17767              "sort": "current",
 17768              "sortDesc": true,
 17769              "total": false,
 17770              "values": true
 17771            },
 17772            "lines": true,
 17773            "linewidth": 1,
 17774            "links": [],
 17775            "nullPointMode": "null",
 17776            "options": {
 17777              "alertThreshold": true
 17778            },
 17779            "paceLength": 10,
 17780            "percentage": false,
 17781            "pluginVersion": "7.5.11",
 17782            "pointradius": 5,
 17783            "points": false,
 17784            "renderer": "flot",
 17785            "seriesOverrides": [],
 17786            "spaceLength": 10,
 17787            "stack": false,
 17788            "steppedLine": false,
 17789            "targets": [
 17790              {
 17791                "expr": "sum(rate(tikv_cdc_scan_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=\"tikv\"}[30s])) by (instance)",
 17792                "format": "time_series",
 17793                "hide": false,
 17794                "intervalFactor": 2,
 17795                "legendFormat": "tikv-{{instance}}",
 17796                "refId": "A",
 17797                "step": 10
 17798              }
 17799            ],
 17800            "thresholds": [],
 17801            "timeFrom": null,
 17802            "timeRegions": [],
 17803            "timeShift": null,
 17804            "title": "CDC scan speed",
 17805            "tooltip": {
 17806              "msResolution": false,
 17807              "shared": true,
 17808              "sort": 0,
 17809              "value_type": "individual"
 17810            },
 17811            "type": "graph",
 17812            "xaxis": {
 17813              "buckets": null,
 17814              "mode": "time",
 17815              "name": null,
 17816              "show": true,
 17817              "values": []
 17818            },
 17819            "yaxes": [
 17820              {
 17821                "format": "bytes",
 17822                "label": null,
 17823                "logBase": 1,
 17824                "max": null,
 17825                "min": "0",
 17826                "show": true
 17827              },
 17828              {
 17829                "format": "short",
 17830                "label": null,
 17831                "logBase": 1,
 17832                "max": null,
 17833                "min": null,
 17834                "show": true
 17835              }
 17836            ],
 17837            "yaxis": {
 17838              "align": false,
 17839              "alignLevel": null
 17840            }
 17841          },
 17842          {
 17843            "aliasColors": {},
 17844            "bars": true,
 17845            "dashLength": 10,
 17846            "dashes": false,
 17847            "datasource": "${DS_TEST-CLUSTER}",
 17848            "decimals": 1,
 17849            "description": "The total bytes of TiKV CDC incremental scan",
 17850            "editable": true,
 17851            "error": false,
 17852            "fieldConfig": {
 17853              "defaults": {},
 17854              "overrides": []
 17855            },
 17856            "fill": 0,
 17857            "fillGradient": 0,
 17858            "grid": {},
 17859            "gridPos": {
 17860              "h": 7,
 17861              "w": 6,
 17862              "x": 18,
 17863              "y": 194
 17864            },
 17865            "hiddenSeries": false,
 17866            "id": 139,
 17867            "legend": {
 17868              "alignAsTable": true,
 17869              "avg": false,
 17870              "current": true,
 17871              "hideEmpty": true,
 17872              "hideZero": true,
 17873              "max": true,
 17874              "min": false,
 17875              "rightSide": false,
 17876              "show": true,
 17877              "sideWidth": null,
 17878              "sort": "current",
 17879              "sortDesc": true,
 17880              "total": false,
 17881              "values": true
 17882            },
 17883            "lines": false,
 17884            "linewidth": 1,
 17885            "links": [],
 17886            "nullPointMode": "null",
 17887            "options": {
 17888              "alertThreshold": true
 17889            },
 17890            "paceLength": 10,
 17891            "percentage": false,
 17892            "pluginVersion": "7.5.11",
 17893            "pointradius": 5,
 17894            "points": false,
 17895            "renderer": "flot",
 17896            "seriesOverrides": [],
 17897            "spaceLength": 10,
 17898            "stack": false,
 17899            "steppedLine": false,
 17900            "targets": [
 17901              {
 17902                "expr": "sum(tikv_cdc_scan_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=\"tikv\"}) by (instance)",
 17903                "format": "time_series",
 17904                "hide": false,
 17905                "intervalFactor": 2,
 17906                "legendFormat": "tikv-{{instance}}",
 17907                "refId": "A",
 17908                "step": 10
 17909              }
 17910            ],
 17911            "thresholds": [],
 17912            "timeFrom": null,
 17913            "timeRegions": [],
 17914            "timeShift": null,
 17915            "title": "CDC total scan bytes",
 17916            "tooltip": {
 17917              "msResolution": false,
 17918              "shared": true,
 17919              "sort": 0,
 17920              "value_type": "individual"
 17921            },
 17922            "type": "graph",
 17923            "xaxis": {
 17924              "buckets": null,
 17925              "mode": "time",
 17926              "name": null,
 17927              "show": true,
 17928              "values": []
 17929            },
 17930            "yaxes": [
 17931              {
 17932                "format": "bytes",
 17933                "label": null,
 17934                "logBase": 1,
 17935                "max": null,
 17936                "min": null,
 17937                "show": true
 17938              },
 17939              {
 17940                "format": "short",
 17941                "label": null,
 17942                "logBase": 1,
 17943                "max": null,
 17944                "min": null,
 17945                "show": true
 17946              }
 17947            ],
 17948            "yaxis": {
 17949              "align": false,
 17950              "alignLevel": null
 17951            }
 17952          },
 17953          {
 17954            "aliasColors": {},
 17955            "bars": false,
 17956            "dashLength": 10,
 17957            "dashes": false,
 17958            "datasource": "${DS_TEST-CLUSTER}",
 17959            "decimals": 1,
 17960            "description": "",
 17961            "fieldConfig": {
 17962              "defaults": {},
 17963              "overrides": []
 17964            },
 17965            "fill": 4,
 17966            "fillGradient": 0,
 17967            "gridPos": {
 17968              "h": 7,
 17969              "w": 12,
 17970              "x": 0,
 17971              "y": 201
 17972            },
 17973            "hiddenSeries": false,
 17974            "id": 143,
 17975            "legend": {
 17976              "alignAsTable": true,
 17977              "avg": false,
 17978              "current": true,
 17979              "hideEmpty": false,
 17980              "hideZero": true,
 17981              "max": false,
 17982              "min": true,
 17983              "rightSide": true,
 17984              "show": true,
 17985              "sideWidth": null,
 17986              "sort": "current",
 17987              "sortDesc": true,
 17988              "total": false,
 17989              "values": true
 17990            },
 17991            "lines": true,
 17992            "linewidth": 1,
 17993            "links": [],
 17994            "nullPointMode": "null",
 17995            "options": {
 17996              "alertThreshold": true
 17997            },
 17998            "paceLength": 10,
 17999            "percentage": false,
 18000            "pluginVersion": "7.5.11",
 18001            "pointradius": 1,
 18002            "points": true,
 18003            "renderer": "flot",
 18004            "seriesOverrides": [
 18005              {
 18006                "alias": "/(access|miss).*/",
 18007                "fill": 0,
 18008                "points": false,
 18009                "yaxis": 2
 18010              }
 18011            ],
 18012            "spaceLength": 10,
 18013            "stack": false,
 18014            "steppedLine": false,
 18015            "targets": [
 18016              {
 18017                "expr": "(sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance) - sum(rate(tikv_cdc_old_value_cache_miss{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)) / sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)",
 18018                "format": "time_series",
 18019                "hide": false,
 18020                "intervalFactor": 1,
 18021                "legendFormat": "hit-rate-{{instance}}",
 18022                "refId": "A"
 18023              },
 18024              {
 18025                "expr": "-sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)",
 18026                "format": "time_series",
 18027                "hide": true,
 18028                "intervalFactor": 1,
 18029                "legendFormat": "access-{{instance}}",
 18030                "refId": "B"
 18031              },
 18032              {
 18033                "expr": "-sum(rate(tikv_cdc_old_value_cache_miss{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)",
 18034                "format": "time_series",
 18035                "hide": false,
 18036                "intervalFactor": 1,
 18037                "legendFormat": "miss-{{instance}}",
 18038                "refId": "C"
 18039              },
 18040              {
 18041                "expr": "-sum(rate(tikv_cdc_old_value_cache_miss_none{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)",
 18042                "format": "time_series",
 18043                "hide": false,
 18044                "intervalFactor": 1,
 18045                "legendFormat": "miss-none-{{instance}}",
 18046                "refId": "D"
 18047              }
 18048            ],
 18049            "thresholds": [],
 18050            "timeFrom": null,
 18051            "timeRegions": [],
 18052            "timeShift": null,
 18053            "title": "Old value cache hit",
 18054            "tooltip": {
 18055              "shared": true,
 18056              "sort": 0,
 18057              "value_type": "individual"
 18058            },
 18059            "type": "graph",
 18060            "xaxis": {
 18061              "buckets": null,
 18062              "mode": "time",
 18063              "name": null,
 18064              "show": true,
 18065              "values": []
 18066            },
 18067            "yaxes": [
 18068              {
 18069                "format": "percentunit",
 18070                "label": null,
 18071                "logBase": 1,
 18072                "max": "1",
 18073                "min": "0",
 18074                "show": true
 18075              },
 18076              {
 18077                "format": "none",
 18078                "label": null,
 18079                "logBase": 1,
 18080                "max": null,
 18081                "min": null,
 18082                "show": true
 18083              }
 18084            ],
 18085            "yaxis": {
 18086              "align": false,
 18087              "alignLevel": null
 18088            }
 18089          },
 18090          {
 18091            "aliasColors": {},
 18092            "bars": false,
 18093            "dashLength": 10,
 18094            "dashes": false,
 18095            "datasource": "${DS_TEST-CLUSTER}",
 18096            "decimals": 1,
 18097            "description": "The total number of cache entries in the old value cache.",
 18098            "fieldConfig": {
 18099              "defaults": {},
 18100              "overrides": []
 18101            },
 18102            "fill": 1,
 18103            "fillGradient": 0,
 18104            "gridPos": {
 18105              "h": 7,
 18106              "w": 12,
 18107              "x": 12,
 18108              "y": 201
 18109            },
 18110            "hiddenSeries": false,
 18111            "id": 145,
 18112            "legend": {
 18113              "alignAsTable": true,
 18114              "avg": false,
 18115              "current": true,
 18116              "hideEmpty": false,
 18117              "hideZero": true,
 18118              "max": true,
 18119              "min": false,
 18120              "rightSide": true,
 18121              "show": true,
 18122              "sideWidth": null,
 18123              "sort": "current",
 18124              "sortDesc": true,
 18125              "total": false,
 18126              "values": true
 18127            },
 18128            "lines": true,
 18129            "linewidth": 1,
 18130            "links": [],
 18131            "nullPointMode": "null",
 18132            "options": {
 18133              "alertThreshold": true
 18134            },
 18135            "paceLength": 10,
 18136            "percentage": false,
 18137            "pluginVersion": "7.5.11",
 18138            "pointradius": 1,
 18139            "points": false,
 18140            "renderer": "flot",
 18141            "seriesOverrides": [
 18142              {
 18143                "alias": "/.*len/",
 18144                "yaxis": 2
 18145              }
 18146            ],
 18147            "spaceLength": 10,
 18148            "stack": false,
 18149            "steppedLine": false,
 18150            "targets": [
 18151              {
 18152                "expr": "sum(tikv_cdc_old_value_cache_length{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)",
 18153                "format": "time_series",
 18154                "hide": false,
 18155                "intervalFactor": 1,
 18156                "legendFormat": "{{instance}}-len",
 18157                "refId": "A"
 18158              },
 18159              {
 18160                "expr": "sum(tikv_cdc_old_value_cache_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance) / sum(tikv_cdc_old_value_cache_length{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)",
 18161                "format": "time_series",
 18162                "hide": false,
 18163                "intervalFactor": 1,
 18164                "legendFormat": "{{instance}}-avg entry bytes",
 18165                "refId": "B"
 18166              },
 18167              {
 18168                "expr": "sum(tikv_cdc_old_value_cache_memory_quota{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)",
 18169                "format": "time_series",
 18170                "hide": false,
 18171                "intervalFactor": 1,
 18172                "legendFormat": "{{instance}}-quota",
 18173                "refId": "C"
 18174              }
 18175            ],
 18176            "thresholds": [],
 18177            "timeFrom": null,
 18178            "timeRegions": [],
 18179            "timeShift": null,
 18180            "title": "Old value cache size",
 18181            "tooltip": {
 18182              "shared": true,
 18183              "sort": 0,
 18184              "value_type": "individual"
 18185            },
 18186            "type": "graph",
 18187            "xaxis": {
 18188              "buckets": null,
 18189              "mode": "time",
 18190              "name": null,
 18191              "show": true,
 18192              "values": []
 18193            },
 18194            "yaxes": [
 18195              {
 18196                "format": "bytes",
 18197                "label": null,
 18198                "logBase": 1,
 18199                "max": null,
 18200                "min": "0",
 18201                "show": true
 18202              },
 18203              {
 18204                "decimals": null,
 18205                "format": "none",
 18206                "label": null,
 18207                "logBase": 1,
 18208                "max": null,
 18209                "min": "0",
 18210                "show": true
 18211              }
 18212            ],
 18213            "yaxis": {
 18214              "align": false,
 18215              "alignLevel": null
 18216            }
 18217          },
 18218          {
 18219            "aliasColors": {},
 18220            "bars": false,
 18221            "dashLength": 10,
 18222            "dashes": false,
 18223            "datasource": "${DS_TEST-CLUSTER}",
 18224            "decimals": 1,
 18225            "description": "",
 18226            "editable": true,
 18227            "error": false,
 18228            "fieldConfig": {
 18229              "defaults": {},
 18230              "overrides": []
 18231            },
 18232            "fill": 0,
 18233            "fillGradient": 0,
 18234            "grid": {},
 18235            "gridPos": {
 18236              "h": 7,
 18237              "w": 12,
 18238              "x": 0,
 18239              "y": 208
 18240            },
 18241            "hiddenSeries": false,
 18242            "id": 141,
 18243            "legend": {
 18244              "alignAsTable": true,
 18245              "avg": false,
 18246              "current": true,
 18247              "hideEmpty": true,
 18248              "hideZero": true,
 18249              "max": true,
 18250              "min": false,
 18251              "rightSide": true,
 18252              "show": true,
 18253              "sideWidth": null,
 18254              "sort": "current",
 18255              "sortDesc": true,
 18256              "total": false,
 18257              "values": true
 18258            },
 18259            "lines": true,
 18260            "linewidth": 1,
 18261            "links": [],
 18262            "nullPointMode": "null",
 18263            "options": {
 18264              "alertThreshold": true
 18265            },
 18266            "paceLength": 10,
 18267            "percentage": false,
 18268            "pluginVersion": "7.5.11",
 18269            "pointradius": 5,
 18270            "points": false,
 18271            "renderer": "flot",
 18272            "seriesOverrides": [],
 18273            "spaceLength": 10,
 18274            "stack": false,
 18275            "steppedLine": false,
 18276            "targets": [
 18277              {
 18278                "expr": "sum(rate(tikv_cdc_old_value_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance, cf, tag)",
 18279                "format": "time_series",
 18280                "hide": false,
 18281                "intervalFactor": 2,
 18282                "legendFormat": "{{instance}}-{{cf}}-{{tag}}",
 18283                "refId": "A",
 18284                "step": 10
 18285              }
 18286            ],
 18287            "thresholds": [],
 18288            "timeFrom": null,
 18289            "timeRegions": [],
 18290            "timeShift": null,
 18291            "title": "Old value seek operation",
 18292            "tooltip": {
 18293              "msResolution": false,
 18294              "shared": true,
 18295              "sort": 0,
 18296              "value_type": "individual"
 18297            },
 18298            "type": "graph",
 18299            "xaxis": {
 18300              "buckets": null,
 18301              "mode": "time",
 18302              "name": null,
 18303              "show": true,
 18304              "values": []
 18305            },
 18306            "yaxes": [
 18307              {
 18308                "format": "ops",
 18309                "label": null,
 18310                "logBase": 1,
 18311                "max": null,
 18312                "min": "0",
 18313                "show": true
 18314              },
 18315              {
 18316                "format": "short",
 18317                "label": null,
 18318                "logBase": 1,
 18319                "max": null,
 18320                "min": null,
 18321                "show": true
 18322              }
 18323            ],
 18324            "yaxis": {
 18325              "align": false,
 18326              "alignLevel": null
 18327            }
 18328          },
 18329          {
 18330            "cards": {
 18331              "cardPadding": 0,
 18332              "cardRound": 0
 18333            },
 18334            "color": {
 18335              "cardColor": "#FF9830",
 18336              "colorScale": "linear",
 18337              "colorScheme": "interpolateSpectral",
 18338              "exponent": 0.5,
 18339              "min": 0,
 18340              "mode": "spectrum"
 18341            },
 18342            "dataFormat": "tsbuckets",
 18343            "datasource": "${DS_TEST-CLUSTER}",
 18344            "description": "The time consumed to get an old value (both from cache and from disk)",
 18345            "fieldConfig": {
 18346              "defaults": {},
 18347              "overrides": []
 18348            },
 18349            "gridPos": {
 18350              "h": 7,
 18351              "w": 6,
 18352              "x": 12,
 18353              "y": 208
 18354            },
 18355            "heatmap": {},
 18356            "hideZeroBuckets": true,
 18357            "highlightCards": true,
 18358            "id": 146,
 18359            "legend": {
 18360              "alignAsTable": true,
 18361              "avg": false,
 18362              "current": true,
 18363              "max": true,
 18364              "min": false,
 18365              "rightSide": true,
 18366              "show": true,
 18367              "sort": "current",
 18368              "sortDesc": true,
 18369              "total": false,
 18370              "values": true
 18371            },
 18372            "links": [],
 18373            "maxPerRow": 3,
 18374            "repeatDirection": "h",
 18375            "reverseYBuckets": false,
 18376            "targets": [
 18377              {
 18378                "expr": "sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)",
 18379                "format": "heatmap",
 18380                "instant": false,
 18381                "intervalFactor": 2,
 18382                "legendFormat": "{{le}}",
 18383                "refId": "A"
 18384              }
 18385            ],
 18386            "title": "Old value seek duration",
 18387            "tooltip": {
 18388              "show": true,
 18389              "showHistogram": true
 18390            },
 18391            "tooltipDecimals": 1,
 18392            "type": "heatmap",
 18393            "xAxis": {
 18394              "show": true
 18395            },
 18396            "xBucketNumber": null,
 18397            "xBucketSize": null,
 18398            "yAxis": {
 18399              "decimals": 1,
 18400              "format": "s",
 18401              "logBase": 1,
 18402              "max": null,
 18403              "min": null,
 18404              "show": true,
 18405              "splitFactor": null
 18406            },
 18407            "yBucketBound": "upper",
 18408            "yBucketNumber": null,
 18409            "yBucketSize": null
 18410          },
 18411          {
 18412            "aliasColors": {},
 18413            "bars": false,
 18414            "dashLength": 10,
 18415            "dashes": false,
 18416            "datasource": "${DS_TEST-CLUSTER}",
 18417            "decimals": 1,
 18418            "description": "",
 18419            "fieldConfig": {
 18420              "defaults": {},
 18421              "overrides": []
 18422            },
 18423            "fill": 1,
 18424            "fillGradient": 0,
 18425            "gridPos": {
 18426              "h": 7,
 18427              "w": 6,
 18428              "x": 18,
 18429              "y": 208
 18430            },
 18431            "hiddenSeries": false,
 18432            "id": 142,
 18433            "legend": {
 18434              "alignAsTable": true,
 18435              "avg": false,
 18436              "current": true,
 18437              "hideEmpty": false,
 18438              "hideZero": true,
 18439              "max": true,
 18440              "min": false,
 18441              "rightSide": false,
 18442              "show": true,
 18443              "sideWidth": null,
 18444              "sort": "current",
 18445              "sortDesc": true,
 18446              "total": false,
 18447              "values": true
 18448            },
 18449            "lines": true,
 18450            "linewidth": 1,
 18451            "links": [],
 18452            "nullPointMode": "null",
 18453            "options": {
 18454              "alertThreshold": true
 18455            },
 18456            "paceLength": 10,
 18457            "percentage": false,
 18458            "pluginVersion": "7.5.11",
 18459            "pointradius": 5,
 18460            "points": false,
 18461            "renderer": "flot",
 18462            "seriesOverrides": [],
 18463            "spaceLength": 10,
 18464            "stack": false,
 18465            "steppedLine": false,
 18466            "targets": [
 18467              {
 18468                "expr": "histogram_quantile(0.99, sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance, tag))",
 18469                "format": "time_series",
 18470                "intervalFactor": 1,
 18471                "legendFormat": "{{instance}}-99%-{{tag}}",
 18472                "refId": "A"
 18473              },
 18474              {
 18475                "expr": "histogram_quantile(0.95, sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance, tag))",
 18476                "format": "time_series",
 18477                "intervalFactor": 1,
 18478                "legendFormat": "{{instance}}-95%-{{tag}}",
 18479                "refId": "B"
 18480              },
 18481              {
 18482                "expr": "sum(rate(tikv_cdc_old_value_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance, tag) / sum(rate(tikv_cdc_old_value_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance, tag)",
 18483                "format": "time_series",
 18484                "intervalFactor": 1,
 18485                "legendFormat": "{{instance}}-avg-{{tag}}",
 18486                "refId": "C"
 18487              }
 18488            ],
 18489            "thresholds": [],
 18490            "timeFrom": null,
 18491            "timeRegions": [],
 18492            "timeShift": null,
 18493            "title": "Old value seek duration",
 18494            "tooltip": {
 18495              "shared": true,
 18496              "sort": 0,
 18497              "value_type": "individual"
 18498            },
 18499            "type": "graph",
 18500            "xaxis": {
 18501              "buckets": null,
 18502              "mode": "time",
 18503              "name": null,
 18504              "show": true,
 18505              "values": []
 18506            },
 18507            "yaxes": [
 18508              {
 18509                "format": "s",
 18510                "label": null,
 18511                "logBase": 1,
 18512                "max": null,
 18513                "min": null,
 18514                "show": true
 18515              },
 18516              {
 18517                "format": "short",
 18518                "label": null,
 18519                "logBase": 1,
 18520                "max": null,
 18521                "min": null,
 18522                "show": false
 18523              }
 18524            ],
 18525            "yaxis": {
 18526              "align": false,
 18527              "alignLevel": null
 18528            }
 18529          }
 18530        ],
 18531        "title": "TiKV",
 18532        "type": "row"
 18533      },
 18534      {
 18535        "collapsed": true,
 18536        "datasource": null,
 18537        "gridPos": {
 18538          "h": 1,
 18539          "w": 24,
 18540          "x": 0,
 18541          "y": 14
 18542        },
 18543        "id": 294,
 18544        "panels": [
 18545          {
 18546            "aliasColors": {},
 18547            "bars": false,
 18548            "dashLength": 10,
 18549            "dashes": false,
 18550            "datasource": "${DS_TEST-CLUSTER}",
 18551            "fieldConfig": {
 18552              "defaults": {},
 18553              "overrides": []
 18554            },
 18555            "fill": 1,
 18556            "fillGradient": 0,
 18557            "gridPos": {
 18558              "h": 7,
 18559              "w": 4,
 18560              "x": 0,
 18561              "y": 15
 18562            },
 18563            "hiddenSeries": false,
 18564            "id": 321,
 18565            "legend": {
 18566              "avg": false,
 18567              "current": false,
 18568              "max": false,
 18569              "min": false,
 18570              "show": true,
 18571              "total": false,
 18572              "values": false
 18573            },
 18574            "lines": true,
 18575            "linewidth": 1,
 18576            "nullPointMode": "null",
 18577            "options": {
 18578              "alertThreshold": true
 18579            },
 18580            "percentage": false,
 18581            "pluginVersion": "7.5.11",
 18582            "pointradius": 2,
 18583            "points": false,
 18584            "renderer": "flot",
 18585            "seriesOverrides": [],
 18586            "spaceLength": 10,
 18587            "stack": false,
 18588            "steppedLine": false,
 18589            "targets": [
 18590              {
 18591                "exemplar": true,
 18592                "expr": "sum without (from) (rate(ticdc_message_server_message_count{instance=~\"$ticdc_instance\"}[30s]))",
 18593                "interval": "",
 18594                "legendFormat": "{{instance}}",
 18595                "queryType": "randomWalk",
 18596                "refId": "A"
 18597              }
 18598            ],
 18599            "thresholds": [],
 18600            "timeFrom": null,
 18601            "timeRegions": [],
 18602            "timeShift": null,
 18603            "title": "Message Receive Rate",
 18604            "tooltip": {
 18605              "shared": true,
 18606              "sort": 0,
 18607              "value_type": "individual"
 18608            },
 18609            "type": "graph",
 18610            "xaxis": {
 18611              "buckets": null,
 18612              "mode": "time",
 18613              "name": null,
 18614              "show": true,
 18615              "values": []
 18616            },
 18617            "yaxes": [
 18618              {
 18619                "format": "short",
 18620                "label": null,
 18621                "logBase": 1,
 18622                "max": null,
 18623                "min": null,
 18624                "show": true
 18625              },
 18626              {
 18627                "format": "short",
 18628                "label": null,
 18629                "logBase": 1,
 18630                "max": null,
 18631                "min": null,
 18632                "show": true
 18633              }
 18634            ],
 18635            "yaxis": {
 18636              "align": false,
 18637              "alignLevel": null
 18638            }
 18639          },
 18640          {
 18641            "aliasColors": {},
 18642            "bars": false,
 18643            "dashLength": 10,
 18644            "dashes": false,
 18645            "datasource": "${DS_TEST-CLUSTER}",
 18646            "fieldConfig": {
 18647              "defaults": {},
 18648              "overrides": []
 18649            },
 18650            "fill": 1,
 18651            "fillGradient": 0,
 18652            "gridPos": {
 18653              "h": 7,
 18654              "w": 4,
 18655              "x": 4,
 18656              "y": 15
 18657            },
 18658            "hiddenSeries": false,
 18659            "id": 323,
 18660            "legend": {
 18661              "avg": false,
 18662              "current": false,
 18663              "max": false,
 18664              "min": false,
 18665              "show": true,
 18666              "total": false,
 18667              "values": false
 18668            },
 18669            "lines": true,
 18670            "linewidth": 1,
 18671            "nullPointMode": "null",
 18672            "options": {
 18673              "alertThreshold": true
 18674            },
 18675            "percentage": false,
 18676            "pluginVersion": "7.5.11",
 18677            "pointradius": 2,
 18678            "points": false,
 18679            "renderer": "flot",
 18680            "seriesOverrides": [],
 18681            "spaceLength": 10,
 18682            "stack": false,
 18683            "steppedLine": false,
 18684            "targets": [
 18685              {
 18686                "exemplar": true,
 18687                "expr": "sum without (to) (rate(ticdc_message_client_message_count{instance=~\"$ticdc_instance\"}[30s]))",
 18688                "interval": "",
 18689                "legendFormat": "{{instance}}",
 18690                "queryType": "randomWalk",
 18691                "refId": "A"
 18692              }
 18693            ],
 18694            "thresholds": [],
 18695            "timeFrom": null,
 18696            "timeRegions": [],
 18697            "timeShift": null,
 18698            "title": "Message Send Rate",
 18699            "tooltip": {
 18700              "shared": true,
 18701              "sort": 0,
 18702              "value_type": "individual"
 18703            },
 18704            "type": "graph",
 18705            "xaxis": {
 18706              "buckets": null,
 18707              "mode": "time",
 18708              "name": null,
 18709              "show": true,
 18710              "values": []
 18711            },
 18712            "yaxes": [
 18713              {
 18714                "format": "short",
 18715                "label": null,
 18716                "logBase": 1,
 18717                "max": null,
 18718                "min": null,
 18719                "show": true
 18720              },
 18721              {
 18722                "format": "short",
 18723                "label": null,
 18724                "logBase": 1,
 18725                "max": null,
 18726                "min": null,
 18727                "show": true
 18728              }
 18729            ],
 18730            "yaxis": {
 18731              "align": false,
 18732              "alignLevel": null
 18733            }
 18734          },
 18735          {
 18736            "aliasColors": {},
 18737            "bars": false,
 18738            "dashLength": 10,
 18739            "dashes": false,
 18740            "datasource": "${DS_TEST-CLUSTER}",
 18741            "fieldConfig": {
 18742              "defaults": {},
 18743              "overrides": []
 18744            },
 18745            "fill": 0,
 18746            "fillGradient": 0,
 18747            "gridPos": {
 18748              "h": 7,
 18749              "w": 4,
 18750              "x": 8,
 18751              "y": 15
 18752            },
 18753            "hiddenSeries": false,
 18754            "id": 352,
 18755            "legend": {
 18756              "avg": false,
 18757              "current": false,
 18758              "hideEmpty": true,
 18759              "hideZero": true,
 18760              "max": false,
 18761              "min": false,
 18762              "show": true,
 18763              "total": false,
 18764              "values": false
 18765            },
 18766            "lines": true,
 18767            "linewidth": 1,
 18768            "nullPointMode": "null",
 18769            "options": {
 18770              "alertThreshold": true
 18771            },
 18772            "percentage": false,
 18773            "pluginVersion": "7.5.11",
 18774            "pointradius": 2,
 18775            "points": false,
 18776            "renderer": "flot",
 18777            "seriesOverrides": [],
 18778            "spaceLength": 10,
 18779            "stack": false,
 18780            "steppedLine": false,
 18781            "targets": [
 18782              {
 18783                "exemplar": true,
 18784                "expr": "histogram_quantile(0.5, sum(rate(ticdc_message_server_message_batch_size_bucket[30s])) by (le))",
 18785                "interval": "",
 18786                "legendFormat": "p50",
 18787                "queryType": "randomWalk",
 18788                "refId": "A"
 18789              },
 18790              {
 18791                "exemplar": true,
 18792                "expr": "histogram_quantile(0.95, sum(rate(ticdc_message_server_message_batch_size_bucket[30s])) by (le))",
 18793                "hide": false,
 18794                "interval": "",
 18795                "legendFormat": "p90",
 18796                "refId": "B"
 18797              },
 18798              {
 18799                "exemplar": true,
 18800                "expr": "histogram_quantile(0.99, sum(rate(ticdc_message_server_message_batch_size_bucket[30s])) by (le))",
 18801                "hide": false,
 18802                "interval": "",
 18803                "legendFormat": "p99",
 18804                "refId": "C"
 18805              }
 18806            ],
 18807            "thresholds": [],
 18808            "timeFrom": null,
 18809            "timeRegions": [],
 18810            "timeShift": null,
 18811            "title": "Message Batch Size",
 18812            "tooltip": {
 18813              "shared": true,
 18814              "sort": 0,
 18815              "value_type": "individual"
 18816            },
 18817            "transformations": [],
 18818            "type": "graph",
 18819            "xaxis": {
 18820              "buckets": null,
 18821              "mode": "time",
 18822              "name": null,
 18823              "show": true,
 18824              "values": []
 18825            },
 18826            "yaxes": [
 18827              {
 18828                "format": "short",
 18829                "label": null,
 18830                "logBase": 1,
 18831                "max": null,
 18832                "min": null,
 18833                "show": true
 18834              },
 18835              {
 18836                "format": "short",
 18837                "label": null,
 18838                "logBase": 1,
 18839                "max": null,
 18840                "min": null,
 18841                "show": true
 18842              }
 18843            ],
 18844            "yaxis": {
 18845              "align": false,
 18846              "alignLevel": null
 18847            }
 18848          },
 18849          {
 18850            "aliasColors": {},
 18851            "bars": false,
 18852            "dashLength": 10,
 18853            "dashes": false,
 18854            "datasource": "${DS_TEST-CLUSTER}",
 18855            "fieldConfig": {
 18856              "defaults": {
 18857                "unit": "short"
 18858              },
 18859              "overrides": []
 18860            },
 18861            "fill": 0,
 18862            "fillGradient": 0,
 18863            "gridPos": {
 18864              "h": 7,
 18865              "w": 6,
 18866              "x": 12,
 18867              "y": 15
 18868            },
 18869            "hiddenSeries": false,
 18870            "id": 354,
 18871            "legend": {
 18872              "avg": false,
 18873              "current": false,
 18874              "max": false,
 18875              "min": false,
 18876              "show": true,
 18877              "total": false,
 18878              "values": false
 18879            },
 18880            "lines": true,
 18881            "linewidth": 1,
 18882            "nullPointMode": "null",
 18883            "options": {
 18884              "alertThreshold": true
 18885            },
 18886            "percentage": false,
 18887            "pluginVersion": "7.5.11",
 18888            "pointradius": 2,
 18889            "points": false,
 18890            "renderer": "flot",
 18891            "seriesOverrides": [],
 18892            "spaceLength": 10,
 18893            "stack": false,
 18894            "steppedLine": false,
 18895            "targets": [
 18896              {
 18897                "exemplar": true,
 18898                "expr": "histogram_quantile(0.5, sum(rate(ticdc_message_server_message_batch_bytes_bucket [30s])) by (le))",
 18899                "hide": false,
 18900                "interval": "",
 18901                "legendFormat": "p50",
 18902                "refId": "C"
 18903              },
 18904              {
 18905                "exemplar": true,
 18906                "expr": "histogram_quantile(0.95, sum(rate(ticdc_message_server_message_batch_bytes_bucket [30s])) by (le))",
 18907                "interval": "",
 18908                "legendFormat": "p95",
 18909                "queryType": "randomWalk",
 18910                "refId": "A"
 18911              },
 18912              {
 18913                "exemplar": true,
 18914                "expr": "histogram_quantile(0.99, sum(rate(ticdc_message_server_message_batch_bytes_bucket [30s])) by (le))",
 18915                "hide": false,
 18916                "interval": "",
 18917                "legendFormat": "p99",
 18918                "refId": "B"
 18919              }
 18920            ],
 18921            "thresholds": [],
 18922            "timeFrom": null,
 18923            "timeRegions": [],
 18924            "timeShift": null,
 18925            "title": "Receive message batch bytes percentiles",
 18926            "tooltip": {
 18927              "shared": true,
 18928              "sort": 0,
 18929              "value_type": "individual"
 18930            },
 18931            "type": "graph",
 18932            "xaxis": {
 18933              "buckets": null,
 18934              "mode": "time",
 18935              "name": null,
 18936              "show": true,
 18937              "values": []
 18938            },
 18939            "yaxes": [
 18940              {
 18941                "format": "short",
 18942                "label": null,
 18943                "logBase": 1,
 18944                "max": null,
 18945                "min": null,
 18946                "show": true
 18947              },
 18948              {
 18949                "format": "short",
 18950                "label": null,
 18951                "logBase": 1,
 18952                "max": null,
 18953                "min": null,
 18954                "show": true
 18955              }
 18956            ],
 18957            "yaxis": {
 18958              "align": false,
 18959              "alignLevel": null
 18960            }
 18961          },
 18962          {
 18963            "aliasColors": {},
 18964            "bars": false,
 18965            "dashLength": 10,
 18966            "dashes": false,
 18967            "datasource": "${DS_TEST-CLUSTER}",
 18968            "fieldConfig": {
 18969              "defaults": {},
 18970              "overrides": []
 18971            },
 18972            "fill": 0,
 18973            "fillGradient": 0,
 18974            "gridPos": {
 18975              "h": 7,
 18976              "w": 6,
 18977              "x": 18,
 18978              "y": 15
 18979            },
 18980            "hiddenSeries": false,
 18981            "id": 356,
 18982            "legend": {
 18983              "avg": false,
 18984              "current": false,
 18985              "max": false,
 18986              "min": false,
 18987              "show": true,
 18988              "total": false,
 18989              "values": false
 18990            },
 18991            "lines": true,
 18992            "linewidth": 1,
 18993            "nullPointMode": "null",
 18994            "options": {
 18995              "alertThreshold": true
 18996            },
 18997            "percentage": false,
 18998            "pluginVersion": "7.5.11",
 18999            "pointradius": 2,
 19000            "points": false,
 19001            "renderer": "flot",
 19002            "seriesOverrides": [],
 19003            "spaceLength": 10,
 19004            "stack": false,
 19005            "steppedLine": false,
 19006            "targets": [
 19007              {
 19008                "exemplar": true,
 19009                "expr": "histogram_quantile(0.5, sum(rate(ticdc_message_server_message_bytes_bucket [30s])) by (le))",
 19010                "interval": "",
 19011                "legendFormat": "p50",
 19012                "queryType": "randomWalk",
 19013                "refId": "A"
 19014              },
 19015              {
 19016                "exemplar": true,
 19017                "expr": "histogram_quantile(0.95, sum(rate(ticdc_message_server_message_bytes_bucket [30s])) by (le))",
 19018                "hide": false,
 19019                "interval": "",
 19020                "legendFormat": "p95",
 19021                "refId": "B"
 19022              },
 19023              {
 19024                "exemplar": true,
 19025                "expr": "histogram_quantile(0.99, sum(rate(ticdc_message_server_message_bytes_bucket [30s])) by (le))",
 19026                "hide": false,
 19027                "interval": "",
 19028                "legendFormat": "p99",
 19029                "refId": "C"
 19030              }
 19031            ],
 19032            "thresholds": [],
 19033            "timeFrom": null,
 19034            "timeRegions": [],
 19035            "timeShift": null,
 19036            "title": "Receive Message Bytes Percentile",
 19037            "tooltip": {
 19038              "shared": true,
 19039              "sort": 0,
 19040              "value_type": "individual"
 19041            },
 19042            "type": "graph",
 19043            "xaxis": {
 19044              "buckets": null,
 19045              "mode": "time",
 19046              "name": null,
 19047              "show": true,
 19048              "values": []
 19049            },
 19050            "yaxes": [
 19051              {
 19052                "format": "short",
 19053                "label": null,
 19054                "logBase": 1,
 19055                "max": null,
 19056                "min": null,
 19057                "show": true
 19058              },
 19059              {
 19060                "format": "short",
 19061                "label": null,
 19062                "logBase": 1,
 19063                "max": null,
 19064                "min": null,
 19065                "show": true
 19066              }
 19067            ],
 19068            "yaxis": {
 19069              "align": false,
 19070              "alignLevel": null
 19071            }
 19072          },
 19073          {
 19074            "datasource": "${DS_TEST-CLUSTER}",
 19075            "fieldConfig": {
 19076              "defaults": {
 19077                "color": {
 19078                  "mode": "thresholds"
 19079                },
 19080                "custom": {
 19081                  "align": "left",
 19082                  "filterable": false
 19083                },
 19084                "decimals": 2,
 19085                "displayName": "",
 19086                "mappings": [],
 19087                "thresholds": {
 19088                  "mode": "absolute",
 19089                  "steps": [
 19090                    {
 19091                      "color": "green",
 19092                      "value": null
 19093                    },
 19094                    {
 19095                      "color": "red",
 19096                      "value": 80
 19097                    }
 19098                  ]
 19099                },
 19100                "unit": "short"
 19101              },
 19102              "overrides": [
 19103                {
 19104                  "matcher": {
 19105                    "id": "byName",
 19106                    "options": "Time"
 19107                  },
 19108                  "properties": [
 19109                    {
 19110                      "id": "displayName",
 19111                      "value": "Time"
 19112                    },
 19113                    {
 19114                      "id": "unit",
 19115                      "value": "time: YYYY-MM-DD HH:mm:ss"
 19116                    },
 19117                    {
 19118                      "id": "custom.align",
 19119                      "value": null
 19120                    }
 19121                  ]
 19122                }
 19123              ]
 19124            },
 19125            "gridPos": {
 19126              "h": 8,
 19127              "w": 20,
 19128              "x": 0,
 19129              "y": 22
 19130            },
 19131            "id": 350,
 19132            "options": {
 19133              "showHeader": true
 19134            },
 19135            "pluginVersion": "7.5.11",
 19136            "targets": [
 19137              {
 19138                "exemplar": true,
 19139                "expr": "ticdc_message_server_cur_stream_count",
 19140                "format": "time_series",
 19141                "instant": true,
 19142                "interval": "",
 19143                "legendFormat": "",
 19144                "queryType": "randomWalk",
 19145                "refId": "A"
 19146              }
 19147            ],
 19148            "title": "Stream Count Between Nodes",
 19149            "transformations": [
 19150              {
 19151                "id": "labelsToFields",
 19152                "options": {
 19153                  "valueLabel": "from"
 19154                }
 19155              },
 19156              {
 19157                "id": "organize",
 19158                "options": {
 19159                  "excludeByName": {
 19160                    "Time": true,
 19161                    "instance": false,
 19162                    "job": true
 19163                  },
 19164                  "indexByName": {},
 19165                  "renameByName": {
 19166                    "instance": "dest\\src"
 19167                  }
 19168                }
 19169              },
 19170              {
 19171                "id": "merge",
 19172                "options": {
 19173                  "reducers": []
 19174                }
 19175              }
 19176            ],
 19177            "type": "table"
 19178          },
 19179          {
 19180            "aliasColors": {},
 19181            "bars": false,
 19182            "dashLength": 10,
 19183            "dashes": false,
 19184            "datasource": "${DS_TEST-CLUSTER}",
 19185            "fieldConfig": {
 19186              "defaults": {},
 19187              "overrides": []
 19188            },
 19189            "fill": 1,
 19190            "fillGradient": 0,
 19191            "gridPos": {
 19192              "h": 8,
 19193              "w": 4,
 19194              "x": 20,
 19195              "y": 22
 19196            },
 19197            "hiddenSeries": false,
 19198            "id": 358,
 19199            "legend": {
 19200              "avg": false,
 19201              "current": false,
 19202              "max": false,
 19203              "min": false,
 19204              "show": true,
 19205              "total": false,
 19206              "values": false
 19207            },
 19208            "lines": true,
 19209            "linewidth": 1,
 19210            "nullPointMode": "null",
 19211            "options": {
 19212              "alertThreshold": true
 19213            },
 19214            "percentage": false,
 19215            "pluginVersion": "7.5.11",
 19216            "pointradius": 2,
 19217            "points": false,
 19218            "renderer": "flot",
 19219            "seriesOverrides": [],
 19220            "spaceLength": 10,
 19221            "stack": false,
 19222            "steppedLine": false,
 19223            "targets": [
 19224              {
 19225                "exemplar": true,
 19226                "expr": "sum(ticdc_message_server_cur_stream_count)",
 19227                "interval": "",
 19228                "legendFormat": "actual",
 19229                "queryType": "randomWalk",
 19230                "refId": "A"
 19231              },
 19232              {
 19233                "exemplar": true,
 19234                "expr": "2 * count(process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}) - 1",
 19235                "hide": false,
 19236                "interval": "",
 19237                "legendFormat": "expected",
 19238                "refId": "B"
 19239              }
 19240            ],
 19241            "thresholds": [],
 19242            "timeFrom": null,
 19243            "timeRegions": [],
 19244            "timeShift": null,
 19245            "title": "Total Stream Count",
 19246            "tooltip": {
 19247              "shared": true,
 19248              "sort": 0,
 19249              "value_type": "individual"
 19250            },
 19251            "type": "graph",
 19252            "xaxis": {
 19253              "buckets": null,
 19254              "mode": "time",
 19255              "name": null,
 19256              "show": true,
 19257              "values": []
 19258            },
 19259            "yaxes": [
 19260              {
 19261                "format": "short",
 19262                "label": null,
 19263                "logBase": 1,
 19264                "max": null,
 19265                "min": null,
 19266                "show": true
 19267              },
 19268              {
 19269                "format": "short",
 19270                "label": null,
 19271                "logBase": 1,
 19272                "max": null,
 19273                "min": null,
 19274                "show": true
 19275              }
 19276            ],
 19277            "yaxis": {
 19278              "align": false,
 19279              "alignLevel": null
 19280            }
 19281          }
 19282        ],
 19283        "title": "Peer Messages",
 19284        "type": "row"
 19285      },
 19286      {
 19287        "collapsed": true,
 19288        "datasource": null,
 19289        "gridPos": {
 19290          "h": 1,
 19291          "w": 24,
 19292          "x": 0,
 19293          "y": 15
 19294        },
 19295        "id": 10021,
 19296        "panels": [
 19297          {
 19298            "cards": {
 19299              "cardPadding": 0,
 19300              "cardRound": 0
 19301            },
 19302            "color": {
 19303              "cardColor": "#b4ff00",
 19304              "colorScale": "sqrt",
 19305              "colorScheme": "interpolateSpectral",
 19306              "exponent": 0.5,
 19307              "min": 0,
 19308              "mode": "spectrum"
 19309            },
 19310            "dataFormat": "tsbuckets",
 19311            "datasource": "${DS_TEST-CLUSTER}",
 19312            "description": "The latency distributions of fsync called by redo writer",
 19313            "fieldConfig": {
 19314              "defaults": {},
 19315              "overrides": []
 19316            },
 19317            "gridPos": {
 19318              "h": 8,
 19319              "w": 12,
 19320              "x": 0,
 19321              "y": 16
 19322            },
 19323            "heatmap": {},
 19324            "hideZeroBuckets": true,
 19325            "highlightCards": true,
 19326            "id": 10023,
 19327            "legend": {
 19328              "show": true
 19329            },
 19330            "pluginVersion": "6.1.6",
 19331            "reverseYBuckets": false,
 19332            "targets": [
 19333              {
 19334                "exemplar": true,
 19335                "expr": "max(rate(ticdc_redo_fsync_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)",
 19336                "format": "heatmap",
 19337                "interval": "",
 19338                "intervalFactor": 2,
 19339                "legendFormat": "{{le}}",
 19340                "queryType": "randomWalk",
 19341                "refId": "A"
 19342              }
 19343            ],
 19344            "timeFrom": null,
 19345            "timeShift": null,
 19346            "title": "Redo fsync duration",
 19347            "tooltip": {
 19348              "show": true,
 19349              "showHistogram": true
 19350            },
 19351            "tooltipDecimals": 1,
 19352            "type": "heatmap",
 19353            "xAxis": {
 19354              "show": true
 19355            },
 19356            "xBucketNumber": null,
 19357            "xBucketSize": null,
 19358            "yAxis": {
 19359              "decimals": 1,
 19360              "format": "s",
 19361              "logBase": 1,
 19362              "max": null,
 19363              "min": null,
 19364              "show": true,
 19365              "splitFactor": null
 19366            },
 19367            "yBucketBound": "upper",
 19368            "yBucketNumber": null,
 19369            "yBucketSize": null
 19370          },
 19371          {
 19372            "cards": {
 19373              "cardPadding": 0,
 19374              "cardRound": 0
 19375            },
 19376            "color": {
 19377              "cardColor": "#b4ff00",
 19378              "colorScale": "sqrt",
 19379              "colorScheme": "interpolateSpectral",
 19380              "exponent": 0.5,
 19381              "min": 0,
 19382              "mode": "spectrum"
 19383            },
 19384            "dataFormat": "tsbuckets",
 19385            "datasource": "${DS_TEST-CLUSTER}",
 19386            "description": "The latency distributions of flushall called by redo writer",
 19387            "fieldConfig": {
 19388              "defaults": {},
 19389              "overrides": []
 19390            },
 19391            "gridPos": {
 19392              "h": 8,
 19393              "w": 12,
 19394              "x": 12,
 19395              "y": 16
 19396            },
 19397            "heatmap": {},
 19398            "hideZeroBuckets": true,
 19399            "highlightCards": true,
 19400            "id": 10025,
 19401            "legend": {
 19402              "show": true
 19403            },
 19404            "pluginVersion": "6.1.6",
 19405            "reverseYBuckets": false,
 19406            "targets": [
 19407              {
 19408                "exemplar": true,
 19409                "expr": "max(rate(ticdc_redo_flushall_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)",
 19410                "format": "heatmap",
 19411                "interval": "",
 19412                "intervalFactor": 2,
 19413                "legendFormat": "{{le}}",
 19414                "queryType": "randomWalk",
 19415                "refId": "A"
 19416              }
 19417            ],
 19418            "timeFrom": null,
 19419            "timeShift": null,
 19420            "title": "Redo flushall duration",
 19421            "tooltip": {
 19422              "show": true,
 19423              "showHistogram": true
 19424            },
 19425            "tooltipDecimals": 1,
 19426            "type": "heatmap",
 19427            "xAxis": {
 19428              "show": true
 19429            },
 19430            "xBucketNumber": null,
 19431            "xBucketSize": null,
 19432            "yAxis": {
 19433              "decimals": 1,
 19434              "format": "s",
 19435              "logBase": 1,
 19436              "max": null,
 19437              "min": null,
 19438              "show": true,
 19439              "splitFactor": null
 19440            },
 19441            "yBucketBound": "upper",
 19442            "yBucketNumber": null,
 19443            "yBucketSize": null
 19444          },
 19445          {
 19446            "cards": {
 19447              "cardPadding": 0,
 19448              "cardRound": 0
 19449            },
 19450            "color": {
 19451              "cardColor": "#b4ff00",
 19452              "colorScale": "sqrt",
 19453              "colorScheme": "interpolateSpectral",
 19454              "exponent": 0.5,
 19455              "min": 0,
 19456              "mode": "spectrum"
 19457            },
 19458            "dataFormat": "tsbuckets",
 19459            "datasource": "${DS_TEST-CLUSTER}",
 19460            "description": "The latency distributions of writeLog called by redoManager",
 19461            "fieldConfig": {
 19462              "defaults": {},
 19463              "overrides": []
 19464            },
 19465            "gridPos": {
 19466              "h": 8,
 19467              "w": 12,
 19468              "x": 0,
 19469              "y": 24
 19470            },
 19471            "heatmap": {},
 19472            "hideZeroBuckets": true,
 19473            "highlightCards": true,
 19474            "id": 10027,
 19475            "legend": {
 19476              "show": true
 19477            },
 19478            "pluginVersion": "6.1.6",
 19479            "reverseYBuckets": false,
 19480            "targets": [
 19481              {
 19482                "exemplar": true,
 19483                "expr": "max(rate(ticdc_redo_write_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)",
 19484                "format": "heatmap",
 19485                "interval": "",
 19486                "intervalFactor": 2,
 19487                "legendFormat": "{{le}}",
 19488                "queryType": "randomWalk",
 19489                "refId": "A"
 19490              }
 19491            ],
 19492            "timeFrom": null,
 19493            "timeShift": null,
 19494            "title": "Redo write log duration",
 19495            "tooltip": {
 19496              "show": true,
 19497              "showHistogram": true
 19498            },
 19499            "tooltipDecimals": 1,
 19500            "type": "heatmap",
 19501            "xAxis": {
 19502              "show": true
 19503            },
 19504            "xBucketNumber": null,
 19505            "xBucketSize": null,
 19506            "yAxis": {
 19507              "decimals": 1,
 19508              "format": "s",
 19509              "logBase": 1,
 19510              "max": null,
 19511              "min": null,
 19512              "show": true,
 19513              "splitFactor": null
 19514            },
 19515            "yBucketBound": "upper",
 19516            "yBucketNumber": null,
 19517            "yBucketSize": null
 19518          },
 19519          {
 19520            "cards": {
 19521              "cardPadding": 0,
 19522              "cardRound": 0
 19523            },
 19524            "color": {
 19525              "cardColor": "#b4ff00",
 19526              "colorScale": "sqrt",
 19527              "colorScheme": "interpolateSpectral",
 19528              "exponent": 0.5,
 19529              "min": 0,
 19530              "mode": "spectrum"
 19531            },
 19532            "dataFormat": "tsbuckets",
 19533            "datasource": "${DS_TEST-CLUSTER}",
 19534            "description": "The latency distributions of flushLog called by redoManager",
 19535            "fieldConfig": {
 19536              "defaults": {},
 19537              "overrides": []
 19538            },
 19539            "gridPos": {
 19540              "h": 8,
 19541              "w": 12,
 19542              "x": 12,
 19543              "y": 24
 19544            },
 19545            "heatmap": {},
 19546            "hideZeroBuckets": true,
 19547            "highlightCards": true,
 19548            "id": 10029,
 19549            "legend": {
 19550              "show": true
 19551            },
 19552            "pluginVersion": "6.1.6",
 19553            "reverseYBuckets": false,
 19554            "targets": [
 19555              {
 19556                "exemplar": true,
 19557                "expr": "max(rate(ticdc_redo_flush_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)",
 19558                "format": "heatmap",
 19559                "interval": "",
 19560                "intervalFactor": 2,
 19561                "legendFormat": "{{le}}",
 19562                "queryType": "randomWalk",
 19563                "refId": "A"
 19564              }
 19565            ],
 19566            "timeFrom": null,
 19567            "timeShift": null,
 19568            "title": "Redo flush log duration",
 19569            "tooltip": {
 19570              "show": true,
 19571              "showHistogram": true
 19572            },
 19573            "tooltipDecimals": 1,
 19574            "type": "heatmap",
 19575            "xAxis": {
 19576              "show": true
 19577            },
 19578            "xBucketNumber": null,
 19579            "xBucketSize": null,
 19580            "yAxis": {
 19581              "decimals": 1,
 19582              "format": "s",
 19583              "logBase": 1,
 19584              "max": null,
 19585              "min": null,
 19586              "show": true,
 19587              "splitFactor": null
 19588            },
 19589            "yBucketBound": "upper",
 19590            "yBucketNumber": null,
 19591            "yBucketSize": null
 19592          },
 19593          {
 19594            "aliasColors": {},
 19595            "bars": false,
 19596            "dashLength": 10,
 19597            "dashes": false,
 19598            "datasource": "${DS_TEST-CLUSTER}",
 19599            "description": "The total count of rows that are processed by redo writer",
 19600            "fieldConfig": {
 19601              "defaults": {},
 19602              "overrides": []
 19603            },
 19604            "fill": 1,
 19605            "fillGradient": 0,
 19606            "gridPos": {
 19607              "h": 8,
 19608              "w": 12,
 19609              "x": 0,
 19610              "y": 32
 19611            },
 19612            "hiddenSeries": false,
 19613            "id": 10031,
 19614            "legend": {
 19615              "alignAsTable": true,
 19616              "avg": true,
 19617              "current": true,
 19618              "max": true,
 19619              "min": false,
 19620              "rightSide": true,
 19621              "show": true,
 19622              "total": false,
 19623              "values": true
 19624            },
 19625            "lines": true,
 19626            "linewidth": 1,
 19627            "nullPointMode": "null",
 19628            "options": {
 19629              "alertThreshold": true
 19630            },
 19631            "percentage": false,
 19632            "pluginVersion": "7.5.11",
 19633            "pointradius": 2,
 19634            "points": false,
 19635            "renderer": "flot",
 19636            "seriesOverrides": [],
 19637            "spaceLength": 10,
 19638            "stack": false,
 19639            "steppedLine": false,
 19640            "targets": [
 19641              {
 19642                "exemplar": true,
 19643                "expr": "sum(rate(ticdc_redo_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (instance)",
 19644                "format": "time_series",
 19645                "interval": "",
 19646                "intervalFactor": 1,
 19647                "legendFormat": "{{instance}}",
 19648                "queryType": "randomWalk",
 19649                "refId": "A"
 19650              },
 19651              {
 19652                "exemplar": true,
 19653                "expr": "sum(rate(ticdc_redo_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (changefeed)",
 19654                "format": "time_series",
 19655                "hide": false,
 19656                "interval": "",
 19657                "intervalFactor": 1,
 19658                "legendFormat": "total",
 19659                "refId": "B"
 19660              }
 19661            ],
 19662            "thresholds": [],
 19663            "timeFrom": null,
 19664            "timeRegions": [],
 19665            "timeShift": null,
 19666            "title": "Redo Write rows/s ",
 19667            "tooltip": {
 19668              "shared": true,
 19669              "sort": 0,
 19670              "value_type": "individual"
 19671            },
 19672            "type": "graph",
 19673            "xaxis": {
 19674              "buckets": null,
 19675              "mode": "time",
 19676              "name": null,
 19677              "show": true,
 19678              "values": []
 19679            },
 19680            "yaxes": [
 19681              {
 19682                "format": "short",
 19683                "label": null,
 19684                "logBase": 1,
 19685                "max": null,
 19686                "min": null,
 19687                "show": true
 19688              },
 19689              {
 19690                "format": "short",
 19691                "label": null,
 19692                "logBase": 1,
 19693                "max": null,
 19694                "min": null,
 19695                "show": false
 19696              }
 19697            ],
 19698            "yaxis": {
 19699              "align": false,
 19700              "alignLevel": null
 19701            }
 19702          },
 19703          {
 19704            "aliasColors": {},
 19705            "bars": false,
 19706            "dashLength": 10,
 19707            "dashes": false,
 19708            "datasource": "${DS_TEST-CLUSTER}",
 19709            "description": "Total number of bytes redo log written",
 19710            "fieldConfig": {
 19711              "defaults": {},
 19712              "overrides": []
 19713            },
 19714            "fill": 1,
 19715            "fillGradient": 0,
 19716            "gridPos": {
 19717              "h": 8,
 19718              "w": 12,
 19719              "x": 12,
 19720              "y": 32
 19721            },
 19722            "hiddenSeries": false,
 19723            "id": 10033,
 19724            "legend": {
 19725              "avg": false,
 19726              "current": false,
 19727              "max": false,
 19728              "min": false,
 19729              "show": true,
 19730              "total": false,
 19731              "values": false
 19732            },
 19733            "lines": true,
 19734            "linewidth": 1,
 19735            "nullPointMode": "null",
 19736            "options": {
 19737              "alertThreshold": true
 19738            },
 19739            "percentage": false,
 19740            "pluginVersion": "7.5.11",
 19741            "pointradius": 2,
 19742            "points": false,
 19743            "renderer": "flot",
 19744            "seriesOverrides": [],
 19745            "spaceLength": 10,
 19746            "stack": false,
 19747            "steppedLine": false,
 19748            "targets": [
 19749              {
 19750                "exemplar": true,
 19751                "expr": "sum(rate(ticdc_redo_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (instance)",
 19752                "interval": "",
 19753                "legendFormat": "{{instance}}",
 19754                "queryType": "randomWalk",
 19755                "refId": "A"
 19756              }
 19757            ],
 19758            "thresholds": [],
 19759            "timeFrom": null,
 19760            "timeRegions": [],
 19761            "timeShift": null,
 19762            "title": "Redo Write bytes/s ",
 19763            "tooltip": {
 19764              "shared": true,
 19765              "sort": 0,
 19766              "value_type": "individual"
 19767            },
 19768            "type": "graph",
 19769            "xaxis": {
 19770              "buckets": null,
 19771              "mode": "time",
 19772              "name": null,
 19773              "show": true,
 19774              "values": []
 19775            },
 19776            "yaxes": [
 19777              {
 19778                "format": "bytes",
 19779                "label": null,
 19780                "logBase": 1,
 19781                "max": null,
 19782                "min": null,
 19783                "show": true
 19784              },
 19785              {
 19786                "format": "short",
 19787                "label": null,
 19788                "logBase": 1,
 19789                "max": null,
 19790                "min": null,
 19791                "show": true
 19792              }
 19793            ],
 19794            "yaxis": {
 19795              "align": false,
 19796              "alignLevel": null
 19797            }
 19798          },
 19799          {
 19800            "aliasColors": {},
 19801            "bars": false,
 19802            "dashLength": 10,
 19803            "dashes": false,
 19804            "datasource": "${DS_TEST-CLUSTER}",
 19805            "description": "Redo bgUpdateLog worker busy ratio",
 19806            "fieldConfig": {
 19807              "defaults": {},
 19808              "overrides": []
 19809            },
 19810            "fill": 1,
 19811            "fillGradient": 0,
 19812            "gridPos": {
 19813              "h": 8,
 19814              "w": 12,
 19815              "x": 0,
 19816              "y": 40
 19817            },
 19818            "hiddenSeries": false,
 19819            "id": 10035,
 19820            "legend": {
 19821              "alignAsTable": true,
 19822              "avg": false,
 19823              "current": true,
 19824              "max": false,
 19825              "min": false,
 19826              "rightSide": false,
 19827              "show": true,
 19828              "total": false,
 19829              "values": true
 19830            },
 19831            "lines": true,
 19832            "linewidth": 1,
 19833            "nullPointMode": "null",
 19834            "options": {
 19835              "alertThreshold": true
 19836            },
 19837            "percentage": false,
 19838            "pluginVersion": "7.5.11",
 19839            "pointradius": 2,
 19840            "points": false,
 19841            "renderer": "flot",
 19842            "seriesOverrides": [],
 19843            "spaceLength": 10,
 19844            "stack": false,
 19845            "steppedLine": false,
 19846            "targets": [
 19847              {
 19848                "exemplar": true,
 19849                "expr": "sum(rate(ticdc_redo_worker_busy_ratio{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])/10) by (changefeed,instance)",
 19850                "interval": "",
 19851                "legendFormat": "{{changefeed}}-{{instance}}",
 19852                "queryType": "randomWalk",
 19853                "refId": "A"
 19854              }
 19855            ],
 19856            "thresholds": [],
 19857            "timeFrom": null,
 19858            "timeRegions": [],
 19859            "timeShift": null,
 19860            "title": "Worker Busy Ratio",
 19861            "tooltip": {
 19862              "shared": true,
 19863              "sort": 0,
 19864              "value_type": "individual"
 19865            },
 19866            "type": "graph",
 19867            "xaxis": {
 19868              "buckets": null,
 19869              "mode": "time",
 19870              "name": null,
 19871              "show": true,
 19872              "values": []
 19873            },
 19874            "yaxes": [
 19875              {
 19876                "format": "percent",
 19877                "label": null,
 19878                "logBase": 1,
 19879                "max": null,
 19880                "min": null,
 19881                "show": true
 19882              },
 19883              {
 19884                "format": "short",
 19885                "label": null,
 19886                "logBase": 1,
 19887                "max": null,
 19888                "min": null,
 19889                "show": false
 19890              }
 19891            ],
 19892            "yaxis": {
 19893              "align": false,
 19894              "alignLevel": null
 19895            }
 19896          }
 19897        ],
 19898        "title": "Redo",
 19899        "type": "row"
 19900      },
 19901      {
 19902        "collapsed": true,
 19903        "datasource": null,
 19904        "gridPos": {
 19905          "h": 1,
 19906          "w": 24,
 19907          "x": 0,
 19908          "y": 16
 19909        },
 19910        "id": 155,
 19911        "panels": [
 19912          {
 19913            "aliasColors": {},
 19914            "bars": true,
 19915            "cacheTimeout": null,
 19916            "dashLength": 10,
 19917            "dashes": false,
 19918            "datasource": "${DS_TEST-CLUSTER}",
 19919            "description": "TiCDC process rss memory usage. TiCDC heap memory size in use ",
 19920            "editable": true,
 19921            "error": false,
 19922            "fill": 0,
 19923            "grid": {},
 19924            "gridPos": {
 19925              "h": 7,
 19926              "w": 12,
 19927              "x": 0,
 19928              "y": 6
 19929            },
 19930            "id": 157,
 19931            "legend": {
 19932              "alignAsTable": false,
 19933              "avg": false,
 19934              "current": false,
 19935              "hideEmpty": true,
 19936              "hideZero": true,
 19937              "max": false,
 19938              "min": false,
 19939              "rightSide": false,
 19940              "show": true,
 19941              "sideWidth": null,
 19942              "total": false,
 19943              "values": false
 19944            },
 19945            "lines": false,
 19946            "linewidth": 1,
 19947            "links": [],
 19948            "maxPerRow": 3,
 19949            "nullPointMode": "null",
 19950            "percentage": false,
 19951            "pluginVersion": "6.1.6",
 19952            "pointradius": 5,
 19953            "points": false,
 19954            "renderer": "flot",
 19955            "repeat": null,
 19956            "repeatDirection": "h",
 19957            "scopedVars": {
 19958              "instance": {
 19959                "selected": false,
 19960                "text": "172.16.5.33:47912",
 19961                "value": "172.16.5.33:47912"
 19962              },
 19963              "runtime_instance": {
 19964                "selected": false,
 19965                "text": "10.2.6.163:8300",
 19966                "value": "10.2.6.163:8300"
 19967              }
 19968            },
 19969            "seriesOverrides": [
 19970              {
 19971                "alias": "alloc-from-os",
 19972                "fill": 3,
 19973                "lines": true,
 19974                "stack": false
 19975              },
 19976              {
 19977                "alias": "gc-threshold",
 19978                "bars": false,
 19979                "color": "#C4162A",
 19980                "lines": true,
 19981                "linewidth": 2,
 19982                "stack": false
 19983              },
 19984              {
 19985                "alias": "gc",
 19986                "bars": false,
 19987                "color": "#C4162A",
 19988                "hideTooltip": true,
 19989                "legend": false,
 19990                "pointradius": 3,
 19991                "points": true,
 19992                "stack": false
 19993              }
 19994            ],
 19995            "spaceLength": 10,
 19996            "stack": true,
 19997            "steppedLine": false,
 19998            "targets": [
 19999              {
 20000                "expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}",
 20001                "format": "time_series",
 20002                "hide": false,
 20003                "intervalFactor": 1,
 20004                "legendFormat": "alloc-from-os",
 20005                "refId": "A"
 20006              },
 20007              {
 20008                "expr": "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + ticdc_server_go_gc{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)",
 20009                "format": "time_series",
 20010                "hide": false,
 20011                "intervalFactor": 1,
 20012                "legendFormat": "estimate-inuse",
 20013                "refId": "H"
 20014              },
 20015              {
 20016                "expr": "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + ticdc_server_go_gc{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)",
 20017                "format": "time_series",
 20018                "hide": false,
 20019                "intervalFactor": 1,
 20020                "legendFormat": "estimate-garbage",
 20021                "refId": "C"
 20022              },
 20023              {
 20024                "expr": "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}",
 20025                "format": "time_series",
 20026                "hide": false,
 20027                "intervalFactor": 1,
 20028                "legendFormat": "reserved-by-go",
 20029                "refId": "B"
 20030              },
 20031              {
 20032                "expr": "go_memstats_stack_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_mspan_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_mcache_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_buck_hash_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_gc_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_other_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}",
 20033                "format": "time_series",
 20034                "hide": false,
 20035                "intervalFactor": 1,
 20036                "legendFormat": "used-by-go",
 20037                "refId": "D"
 20038              },
 20039              {
 20040                "expr": "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}",
 20041                "format": "time_series",
 20042                "hide": false,
 20043                "intervalFactor": 1,
 20044                "legendFormat": "gc-threshold",
 20045                "refId": "E"
 20046              },
 20047              {
 20048                "expr": "(clamp_max(idelta(go_memstats_last_gc_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[1m]), 1) * go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}) > 0",
 20049                "format": "time_series",
 20050                "hide": false,
 20051                "intervalFactor": 1,
 20052                "legendFormat": "gc",
 20053                "refId": "F"
 20054              }
 20055            ],
 20056            "thresholds": [],
 20057            "timeFrom": null,
 20058            "timeRegions": [],
 20059            "timeShift": null,
 20060            "title": "Memory Usage",
 20061            "tooltip": {
 20062              "msResolution": true,
 20063              "shared": true,
 20064              "sort": 0,
 20065              "value_type": "individual"
 20066            },
 20067            "type": "graph",
 20068            "xaxis": {
 20069              "buckets": null,
 20070              "mode": "time",
 20071              "name": null,
 20072              "show": true,
 20073              "values": []
 20074            },
 20075            "yaxes": [
 20076              {
 20077                "format": "bytes",
 20078                "label": "",
 20079                "logBase": 1,
 20080                "max": null,
 20081                "min": "0",
 20082                "show": true
 20083              },
 20084              {
 20085                "format": "short",
 20086                "label": "",
 20087                "logBase": 1,
 20088                "max": null,
 20089                "min": null,
 20090                "show": false
 20091              }
 20092            ],
 20093            "yaxis": {
 20094              "align": false,
 20095              "alignLevel": null
 20096            }
 20097          },
 20098          {
 20099            "aliasColors": {},
 20100            "bars": false,
 20101            "cacheTimeout": null,
 20102            "dashLength": 10,
 20103            "dashes": false,
 20104            "datasource": "${DS_TEST-CLUSTER}",
 20105            "description": "Count of live objects.",
 20106            "editable": true,
 20107            "error": false,
 20108            "fill": 0,
 20109            "grid": {},
 20110            "gridPos": {
 20111              "h": 7,
 20112              "w": 12,
 20113              "x": 12,
 20114              "y": 6
 20115            },
 20116            "id": 158,
 20117            "legend": {
 20118              "alignAsTable": false,
 20119              "avg": false,
 20120              "current": false,
 20121              "hideEmpty": true,
 20122              "hideZero": true,
 20123              "max": false,
 20124              "min": false,
 20125              "rightSide": false,
 20126              "show": true,
 20127              "sideWidth": null,
 20128              "total": false,
 20129              "values": false
 20130            },
 20131            "lines": true,
 20132            "linewidth": 1,
 20133            "links": [],
 20134            "maxPerRow": 3,
 20135            "nullPointMode": "null",
 20136            "percentage": false,
 20137            "pluginVersion": "6.1.6",
 20138            "pointradius": 5,
 20139            "points": false,
 20140            "renderer": "flot",
 20141            "repeat": null,
 20142            "repeatDirection": "h",
 20143            "scopedVars": {
 20144              "instance": {
 20145                "selected": false,
 20146                "text": "172.16.5.33:47912",
 20147                "value": "172.16.5.33:47912"
 20148              },
 20149              "runtime_instance": {
 20150                "selected": false,
 20151                "text": "10.2.6.163:8300",
 20152                "value": "10.2.6.163:8300"
 20153              }
 20154            },
 20155            "seriesOverrides": [],
 20156            "spaceLength": 10,
 20157            "stack": false,
 20158            "steppedLine": false,
 20159            "targets": [
 20160              {
 20161                "expr": "go_memstats_heap_objects{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}",
 20162                "format": "time_series",
 20163                "hide": false,
 20164                "intervalFactor": 1,
 20165                "legendFormat": "objects",
 20166                "refId": "A"
 20167              }
 20168            ],
 20169            "thresholds": [],
 20170            "timeFrom": null,
 20171            "timeRegions": [],
 20172            "timeShift": null,
 20173            "title": "Estimated Live Objects",
 20174            "tooltip": {
 20175              "msResolution": true,
 20176              "shared": true,
 20177              "sort": 0,
 20178              "value_type": "individual"
 20179            },
 20180            "type": "graph",
 20181            "xaxis": {
 20182              "buckets": null,
 20183              "mode": "time",
 20184              "name": null,
 20185              "show": true,
 20186              "values": []
 20187            },
 20188            "yaxes": [
 20189              {
 20190                "format": "short",
 20191                "label": "",
 20192                "logBase": 1,
 20193                "max": null,
 20194                "min": "0",
 20195                "show": true
 20196              },
 20197              {
 20198                "format": "short",
 20199                "label": "",
 20200                "logBase": 1,
 20201                "max": null,
 20202                "min": null,
 20203                "show": false
 20204              }
 20205            ],
 20206            "yaxis": {
 20207              "align": false,
 20208              "alignLevel": null
 20209            }
 20210          },
 20211          {
 20212            "aliasColors": {},
 20213            "bars": false,
 20214            "cacheTimeout": null,
 20215            "dashLength": 10,
 20216            "dashes": false,
 20217            "datasource": "${DS_TEST-CLUSTER}",
 20218            "description": "TiCDC process Go garbage collection STW pause duration",
 20219            "editable": true,
 20220            "error": false,
 20221            "fill": 1,
 20222            "grid": {},
 20223            "gridPos": {
 20224              "h": 7,
 20225              "w": 12,
 20226              "x": 0,
 20227              "y": 13
 20228            },
 20229            "id": 160,
 20230            "legend": {
 20231              "alignAsTable": false,
 20232              "avg": false,
 20233              "current": true,
 20234              "max": false,
 20235              "min": false,
 20236              "rightSide": false,
 20237              "show": true,
 20238              "total": false,
 20239              "values": true
 20240            },
 20241            "lines": true,
 20242            "linewidth": 1,
 20243            "links": [],
 20244            "nullPointMode": "null as zero",
 20245            "percentage": false,
 20246            "pluginVersion": "6.1.6",
 20247            "pointradius": 5,
 20248            "points": false,
 20249            "renderer": "flot",
 20250            "scopedVars": {
 20251              "instance": {
 20252                "selected": false,
 20253                "text": "172.16.5.33:47912",
 20254                "value": "172.16.5.33:47912"
 20255              },
 20256              "runtime_instance": {
 20257                "selected": false,
 20258                "text": "10.2.6.163:8300",
 20259                "value": "10.2.6.163:8300"
 20260              }
 20261            },
 20262            "seriesOverrides": [
 20263              {}
 20264            ],
 20265            "spaceLength": 10,
 20266            "stack": false,
 20267            "steppedLine": false,
 20268            "targets": [
 20269              {
 20270                "expr": "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile=\"0\"}",
 20271                "format": "time_series",
 20272                "hide": false,
 20273                "instant": false,
 20274                "intervalFactor": 1,
 20275                "legendFormat": "min",
 20276                "refId": "A",
 20277                "step": 40
 20278              },
 20279              {
 20280                "expr": "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile!~\"0|1\"}",
 20281                "format": "time_series",
 20282                "instant": false,
 20283                "intervalFactor": 1,
 20284                "legendFormat": "{{quantile}}",
 20285                "refId": "B"
 20286              },
 20287              {
 20288                "expr": "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile=\"1\"}",
 20289                "format": "time_series",
 20290                "instant": false,
 20291                "intervalFactor": 1,
 20292                "legendFormat": "max",
 20293                "refId": "C"
 20294              }
 20295            ],
 20296            "thresholds": [],
 20297            "timeFrom": null,
 20298            "timeRegions": [],
 20299            "timeShift": null,
 20300            "title": "GC STW Duration (last 256 GC cycles)",
 20301            "tooltip": {
 20302              "msResolution": false,
 20303              "shared": true,
 20304              "sort": 1,
 20305              "value_type": "individual"
 20306            },
 20307            "type": "graph",
 20308            "xaxis": {
 20309              "buckets": null,
 20310              "mode": "time",
 20311              "name": null,
 20312              "show": true,
 20313              "values": []
 20314            },
 20315            "yaxes": [
 20316              {
 20317                "format": "s",
 20318                "label": null,
 20319                "logBase": 1,
 20320                "max": null,
 20321                "min": "0",
 20322                "show": true
 20323              },
 20324              {
 20325                "format": "short",
 20326                "label": null,
 20327                "logBase": 1,
 20328                "max": null,
 20329                "min": null,
 20330                "show": false
 20331              }
 20332            ],
 20333            "yaxis": {
 20334              "align": false,
 20335              "alignLevel": null
 20336            }
 20337          },
 20338          {
 20339            "aliasColors": {},
 20340            "bars": false,
 20341            "dashLength": 10,
 20342            "dashes": false,
 20343            "datasource": "${DS_TEST-CLUSTER}",
 20344            "description": "The throughput of Go's memory allocator.",
 20345            "fill": 1,
 20346            "gridPos": {
 20347              "h": 7,
 20348              "w": 12,
 20349              "x": 12,
 20350              "y": 13
 20351            },
 20352            "id": 161,
 20353            "legend": {
 20354              "alignAsTable": false,
 20355              "avg": false,
 20356              "current": true,
 20357              "max": true,
 20358              "min": false,
 20359              "rightSide": false,
 20360              "show": true,
 20361              "total": false,
 20362              "values": true
 20363            },
 20364            "lines": true,
 20365            "linewidth": 1,
 20366            "links": [],
 20367            "nullPointMode": "null",
 20368            "percentage": false,
 20369            "pointradius": 5,
 20370            "points": false,
 20371            "renderer": "flot",
 20372            "scopedVars": {
 20373              "instance": {
 20374                "selected": false,
 20375                "text": "172.16.5.33:47912",
 20376                "value": "172.16.5.33:47912"
 20377              },
 20378              "runtime_instance": {
 20379                "selected": false,
 20380                "text": "10.2.6.163:8300",
 20381                "value": "10.2.6.163:8300"
 20382              }
 20383            },
 20384            "seriesOverrides": [
 20385              {
 20386                "alias": "sweep",
 20387                "transform": "negative-Y"
 20388              },
 20389              {
 20390                "alias": "alloc-ops",
 20391                "yaxis": 2
 20392              },
 20393              {
 20394                "alias": "swepp-ops",
 20395                "transform": "negative-Y",
 20396                "yaxis": 2
 20397              }
 20398            ],
 20399            "spaceLength": 10,
 20400            "stack": false,
 20401            "steppedLine": false,
 20402            "targets": [
 20403              {
 20404                "expr": "irate(go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[30s])",
 20405                "format": "time_series",
 20406                "intervalFactor": 1,
 20407                "legendFormat": "alloc",
 20408                "refId": "A"
 20409              },
 20410              {
 20411                "expr": "irate((go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"})[30s:])",
 20412                "format": "time_series",
 20413                "intervalFactor": 1,
 20414                "legendFormat": "sweep",
 20415                "refId": "B"
 20416              },
 20417              {
 20418                "expr": "irate(go_memstats_mallocs_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[30s])",
 20419                "format": "time_series",
 20420                "intervalFactor": 1,
 20421                "legendFormat": "alloc-ops",
 20422                "refId": "C"
 20423              },
 20424              {
 20425                "expr": "irate(go_memstats_frees_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[30s])",
 20426                "format": "time_series",
 20427                "intervalFactor": 1,
 20428                "legendFormat": "swepp-ops",
 20429                "refId": "D"
 20430              }
 20431            ],
 20432            "thresholds": [],
 20433            "timeFrom": null,
 20434            "timeRegions": [],
 20435            "timeShift": null,
 20436            "title": "Allocator Throughput",
 20437            "tooltip": {
 20438              "shared": true,
 20439              "sort": 0,
 20440              "value_type": "individual"
 20441            },
 20442            "type": "graph",
 20443            "xaxis": {
 20444              "buckets": null,
 20445              "mode": "time",
 20446              "name": null,
 20447              "show": true,
 20448              "values": []
 20449            },
 20450            "yaxes": [
 20451              {
 20452                "format": "Bps",
 20453                "label": null,
 20454                "logBase": 1,
 20455                "max": null,
 20456                "min": null,
 20457                "show": true
 20458              },
 20459              {
 20460                "format": "ops",
 20461                "label": null,
 20462                "logBase": 1,
 20463                "max": null,
 20464                "min": null,
 20465                "show": true
 20466              }
 20467            ],
 20468            "yaxis": {
 20469              "align": true,
 20470              "alignLevel": null
 20471            }
 20472          }
 20473        ],
 20474        "repeat": "runtime_instance",
 20475        "title": "Runtime $runtime_instance",
 20476        "type": "row"
 20477      },
 20478      {
 20479        "collapsed": true,
 20480        "datasource": null,
 20481        "gridPos": {
 20482          "h": 1,
 20483          "w": 24,
 20484          "x": 0,
 20485          "y": 18
 20486        },
 20487        "id": 10000,
 20488        "panels": [
 20489          {
 20490            "aliasColors": {},
 20491            "bars": false,
 20492            "dashLength": 10,
 20493            "dashes": false,
 20494            "datasource": "${DS_TEST-CLUSTER}",
 20495            "fieldConfig": {
 20496              "defaults": {},
 20497              "overrides": []
 20498            },
 20499            "fill": 1,
 20500            "fillGradient": 0,
 20501            "gridPos": {
 20502              "h": 9,
 20503              "w": 12,
 20504              "x": 0,
 20505              "y": 50
 20506            },
 20507            "hiddenSeries": false,
 20508            "id": 10001,
 20509            "legend": {
 20510              "avg": false,
 20511              "current": false,
 20512              "max": false,
 20513              "min": false,
 20514              "show": true,
 20515              "total": false,
 20516              "values": false
 20517            },
 20518            "lines": true,
 20519            "linewidth": 1,
 20520            "nullPointMode": "null as zero",
 20521            "options": {
 20522              "alertThreshold": true
 20523            },
 20524            "percentage": false,
 20525            "pluginVersion": "7.5.11",
 20526            "pointradius": 2,
 20527            "points": false,
 20528            "renderer": "flot",
 20529            "seriesOverrides": [],
 20530            "spaceLength": 10,
 20531            "stack": true,
 20532            "steppedLine": false,
 20533            "targets": [
 20534              {
 20535                "exemplar": true,
 20536                "expr": "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"count\"}[30s])/2",
 20537                "format": "time_series",
 20538                "interval": "",
 20539                "legendFormat": "",
 20540                "queryType": "randomWalk",
 20541                "refId": "A"
 20542              },
 20543              {
 20544                "exemplar": true,
 20545                "expr": "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"count\"}",
 20546                "format": "time_series",
 20547                "interval": "",
 20548                "legendFormat": "",
 20549                "queryType": "randomWalk",
 20550                "refId": "A"
 20551              }
 20552            ],
 20553            "thresholds": [],
 20554            "timeFrom": null,
 20555            "timeRegions": [],
 20556            "timeShift": null,
 20557            "title": "Pulsar published DDL schema count",
 20558            "tooltip": {
 20559              "shared": true,
 20560              "sort": 0,
 20561              "value_type": "individual"
 20562            },
 20563            "type": "graph",
 20564            "xaxis": {
 20565              "buckets": null,
 20566              "mode": "time",
 20567              "name": null,
 20568              "show": true,
 20569              "values": []
 20570            },
 20571            "yaxes": [
 20572              {
 20573                "format": "short",
 20574                "label": null,
 20575                "logBase": 1,
 20576                "max": null,
 20577                "min": null,
 20578                "show": true
 20579              },
 20580              {
 20581                "format": "short",
 20582                "label": null,
 20583                "logBase": 1,
 20584                "max": null,
 20585                "min": null,
 20586                "show": true
 20587              }
 20588            ],
 20589            "yaxis": {
 20590              "align": false,
 20591              "alignLevel": null
 20592            }
 20593          },
 20594          {
 20595            "aliasColors": {},
 20596            "bars": false,
 20597            "dashLength": 10,
 20598            "dashes": false,
 20599            "datasource": "${DS_TEST-CLUSTER}",
 20600            "fieldConfig": {
 20601              "defaults": {},
 20602              "overrides": []
 20603            },
 20604            "fill": 1,
 20605            "fillGradient": 0,
 20606            "gridPos": {
 20607              "h": 9,
 20608              "w": 12,
 20609              "x": 12,
 20610              "y": 50
 20611            },
 20612            "hiddenSeries": false,
 20613            "id": 10002,
 20614            "legend": {
 20615              "avg": false,
 20616              "current": false,
 20617              "max": false,
 20618              "min": false,
 20619              "show": true,
 20620              "total": false,
 20621              "values": false
 20622            },
 20623            "lines": true,
 20624            "linewidth": 1,
 20625            "nullPointMode": "null as zero",
 20626            "options": {
 20627              "alertThreshold": true
 20628            },
 20629            "percentage": false,
 20630            "pluginVersion": "7.5.11",
 20631            "pointradius": 2,
 20632            "points": false,
 20633            "renderer": "flot",
 20634            "seriesOverrides": [],
 20635            "spaceLength": 10,
 20636            "stack": true,
 20637            "steppedLine": false,
 20638            "targets": [
 20639              {
 20640                "exemplar": true,
 20641                "expr": "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}[30s])/2",
 20642                "format": "time_series",
 20643                "interval": "",
 20644                "legendFormat": "",
 20645                "queryType": "randomWalk",
 20646                "refId": "A"
 20647              },
 20648              {
 20649                "exemplar": true,
 20650                "expr": "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}",
 20651                "format": "time_series",
 20652                "interval": "",
 20653                "legendFormat": "",
 20654                "queryType": "randomWalk",
 20655                "refId": "A"
 20656              }
 20657            ],
 20658            "thresholds": [],
 20659            "timeFrom": null,
 20660            "timeRegions": [],
 20661            "timeShift": null,
 20662            "title": "Pulsar published DDL schema success",
 20663            "tooltip": {
 20664              "shared": true,
 20665              "sort": 0,
 20666              "value_type": "individual"
 20667            },
 20668            "type": "graph",
 20669            "xaxis": {
 20670              "buckets": null,
 20671              "mode": "time",
 20672              "name": null,
 20673              "show": true,
 20674              "values": []
 20675            },
 20676            "yaxes": [
 20677              {
 20678                "format": "short",
 20679                "label": null,
 20680                "logBase": 1,
 20681                "max": null,
 20682                "min": null,
 20683                "show": true
 20684              },
 20685              {
 20686                "format": "short",
 20687                "label": null,
 20688                "logBase": 1,
 20689                "max": null,
 20690                "min": null,
 20691                "show": true
 20692              }
 20693            ],
 20694            "yaxis": {
 20695              "align": false,
 20696              "alignLevel": null
 20697            }
 20698          },
 20699          {
 20700            "aliasColors": {},
 20701            "bars": false,
 20702            "dashLength": 10,
 20703            "dashes": false,
 20704            "datasource": "${DS_TEST-CLUSTER}",
 20705            "fieldConfig": {
 20706              "defaults": {},
 20707              "overrides": []
 20708            },
 20709            "fill": 1,
 20710            "fillGradient": 0,
 20711            "gridPos": {
 20712              "h": 9,
 20713              "w": 12,
 20714              "x": 0,
 20715              "y": 59
 20716            },
 20717            "hiddenSeries": false,
 20718            "id": 10003,
 20719            "legend": {
 20720              "avg": false,
 20721              "current": false,
 20722              "max": false,
 20723              "min": false,
 20724              "show": true,
 20725              "total": false,
 20726              "values": false
 20727            },
 20728            "lines": true,
 20729            "linewidth": 1,
 20730            "nullPointMode": "null as zero",
 20731            "options": {
 20732              "alertThreshold": true
 20733            },
 20734            "percentage": false,
 20735            "pluginVersion": "7.5.11",
 20736            "pointradius": 2,
 20737            "points": false,
 20738            "renderer": "flot",
 20739            "seriesOverrides": [],
 20740            "spaceLength": 10,
 20741            "stack": true,
 20742            "steppedLine": false,
 20743            "targets": [
 20744              {
 20745                "exemplar": true,
 20746                "expr": "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\", schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}[30s])/2\r",
 20747                "format": "time_series",
 20748                "interval": "",
 20749                "legendFormat": "",
 20750                "queryType": "randomWalk",
 20751                "refId": "A"
 20752              },
 20753              {
 20754                "exemplar": true,
 20755                "expr": "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\", schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}",
 20756                "format": "time_series",
 20757                "interval": "",
 20758                "legendFormat": "",
 20759                "queryType": "randomWalk",
 20760                "refId": "A"
 20761              }
 20762            ],
 20763            "thresholds": [],
 20764            "timeFrom": null,
 20765            "timeRegions": [],
 20766            "timeShift": null,
 20767            "title": "Pulsar published DDL schema fail",
 20768            "tooltip": {
 20769              "shared": true,
 20770              "sort": 0,
 20771              "value_type": "individual"
 20772            },
 20773            "type": "graph",
 20774            "xaxis": {
 20775              "buckets": null,
 20776              "mode": "time",
 20777              "name": null,
 20778              "show": true,
 20779              "values": []
 20780            },
 20781            "yaxes": [
 20782              {
 20783                "format": "short",
 20784                "label": null,
 20785                "logBase": 1,
 20786                "max": null,
 20787                "min": null,
 20788                "show": true
 20789              },
 20790              {
 20791                "format": "short",
 20792                "label": null,
 20793                "logBase": 1,
 20794                "max": null,
 20795                "min": null,
 20796                "show": true
 20797              }
 20798            ],
 20799            "yaxis": {
 20800              "align": false,
 20801              "alignLevel": null
 20802            }
 20803          },
 20804          {
 20805            "aliasColors": {},
 20806            "bars": false,
 20807            "dashLength": 10,
 20808            "dashes": false,
 20809            "datasource": "${DS_TEST-CLUSTER}",
 20810            "fieldConfig": {
 20811              "defaults": {},
 20812              "overrides": []
 20813            },
 20814            "fill": 1,
 20815            "fillGradient": 0,
 20816            "gridPos": {
 20817              "h": 9,
 20818              "w": 12,
 20819              "x": 12,
 20820              "y": 59
 20821            },
 20822            "hiddenSeries": false,
 20823            "id": 10004,
 20824            "legend": {
 20825              "avg": false,
 20826              "current": false,
 20827              "max": false,
 20828              "min": false,
 20829              "show": true,
 20830              "total": false,
 20831              "values": false
 20832            },
 20833            "lines": true,
 20834            "linewidth": 1,
 20835            "nullPointMode": "null as zero",
 20836            "options": {
 20837              "alertThreshold": true
 20838            },
 20839            "percentage": false,
 20840            "pluginVersion": "7.5.11",
 20841            "pointradius": 2,
 20842            "points": false,
 20843            "renderer": "flot",
 20844            "seriesOverrides": [],
 20845            "spaceLength": 10,
 20846            "stack": true,
 20847            "steppedLine": false,
 20848            "targets": [
 20849              {
 20850                "exemplar": true,
 20851                "expr": "increase(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",\r\nschema=~\"$schema\", topic=~\"$topic\", type=\"count\"}[30s])/2",
 20852                "format": "time_series",
 20853                "interval": "",
 20854                "legendFormat": "",
 20855                "queryType": "randomWalk",
 20856                "refId": "A"
 20857              }
 20858            ],
 20859            "thresholds": [],
 20860            "timeFrom": null,
 20861            "timeRegions": [],
 20862            "timeShift": null,
 20863            "title": "Pulsar published DML schema count",
 20864            "tooltip": {
 20865              "shared": true,
 20866              "sort": 0,
 20867              "value_type": "individual"
 20868            },
 20869            "type": "graph",
 20870            "xaxis": {
 20871              "buckets": null,
 20872              "mode": "time",
 20873              "name": null,
 20874              "show": true,
 20875              "values": []
 20876            },
 20877            "yaxes": [
 20878              {
 20879                "format": "short",
 20880                "label": null,
 20881                "logBase": 1,
 20882                "max": null,
 20883                "min": null,
 20884                "show": true
 20885              },
 20886              {
 20887                "format": "short",
 20888                "label": null,
 20889                "logBase": 1,
 20890                "max": null,
 20891                "min": null,
 20892                "show": true
 20893              }
 20894            ],
 20895            "yaxis": {
 20896              "align": false,
 20897              "alignLevel": null
 20898            }
 20899          },
 20900          {
 20901            "aliasColors": {},
 20902            "bars": false,
 20903            "dashLength": 10,
 20904            "dashes": false,
 20905            "datasource": "${DS_TEST-CLUSTER}",
 20906            "fieldConfig": {
 20907              "defaults": {},
 20908              "overrides": []
 20909            },
 20910            "fill": 1,
 20911            "fillGradient": 0,
 20912            "gridPos": {
 20913              "h": 9,
 20914              "w": 12,
 20915              "x": 0,
 20916              "y": 68
 20917            },
 20918            "hiddenSeries": false,
 20919            "id": 10005,
 20920            "legend": {
 20921              "avg": false,
 20922              "current": false,
 20923              "max": false,
 20924              "min": false,
 20925              "show": true,
 20926              "total": false,
 20927              "values": false
 20928            },
 20929            "lines": true,
 20930            "linewidth": 1,
 20931            "nullPointMode": "null as zero",
 20932            "options": {
 20933              "alertThreshold": true
 20934            },
 20935            "percentage": false,
 20936            "pluginVersion": "7.5.11",
 20937            "pointradius": 2,
 20938            "points": false,
 20939            "renderer": "flot",
 20940            "seriesOverrides": [],
 20941            "spaceLength": 10,
 20942            "stack": true,
 20943            "steppedLine": false,
 20944            "targets": [
 20945              {
 20946                "exemplar": true,
 20947                "expr": "\r\nincrease(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}[30s])/2",
 20948                "format": "time_series",
 20949                "interval": "",
 20950                "legendFormat": "",
 20951                "queryType": "randomWalk",
 20952                "refId": "A"
 20953              }
 20954            ],
 20955            "thresholds": [],
 20956            "timeFrom": null,
 20957            "timeRegions": [],
 20958            "timeShift": null,
 20959            "title": "Pulsar published DML schema success",
 20960            "tooltip": {
 20961              "shared": true,
 20962              "sort": 0,
 20963              "value_type": "individual"
 20964            },
 20965            "type": "graph",
 20966            "xaxis": {
 20967              "buckets": null,
 20968              "mode": "time",
 20969              "name": null,
 20970              "show": true,
 20971              "values": []
 20972            },
 20973            "yaxes": [
 20974              {
 20975                "format": "short",
 20976                "label": null,
 20977                "logBase": 1,
 20978                "max": null,
 20979                "min": null,
 20980                "show": true
 20981              },
 20982              {
 20983                "format": "short",
 20984                "label": null,
 20985                "logBase": 1,
 20986                "max": null,
 20987                "min": null,
 20988                "show": true
 20989              }
 20990            ],
 20991            "yaxis": {
 20992              "align": false,
 20993              "alignLevel": null
 20994            }
 20995          },
 20996          {
 20997            "aliasColors": {},
 20998            "bars": false,
 20999            "dashLength": 10,
 21000            "dashes": false,
 21001            "datasource": "${DS_TEST-CLUSTER}",
 21002            "fieldConfig": {
 21003              "defaults": {},
 21004              "overrides": []
 21005            },
 21006            "fill": 1,
 21007            "fillGradient": 0,
 21008            "gridPos": {
 21009              "h": 9,
 21010              "w": 12,
 21011              "x": 12,
 21012              "y": 68
 21013            },
 21014            "hiddenSeries": false,
 21015            "id": 10006,
 21016            "legend": {
 21017              "avg": false,
 21018              "current": false,
 21019              "max": false,
 21020              "min": false,
 21021              "show": true,
 21022              "total": false,
 21023              "values": false
 21024            },
 21025            "lines": true,
 21026            "linewidth": 1,
 21027            "nullPointMode": "null as zero",
 21028            "options": {
 21029              "alertThreshold": true
 21030            },
 21031            "percentage": false,
 21032            "pluginVersion": "7.5.11",
 21033            "pointradius": 2,
 21034            "points": false,
 21035            "renderer": "flot",
 21036            "seriesOverrides": [],
 21037            "spaceLength": 10,
 21038            "stack": true,
 21039            "steppedLine": false,
 21040            "targets": [
 21041              {
 21042                "exemplar": true,
 21043                "expr": "\r\nincrease(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}[30s])/2",
 21044                "format": "time_series",
 21045                "interval": "",
 21046                "legendFormat": "",
 21047                "queryType": "randomWalk",
 21048                "refId": "A"
 21049              }
 21050            ],
 21051            "thresholds": [],
 21052            "timeFrom": null,
 21053            "timeRegions": [],
 21054            "timeShift": null,
 21055            "title": "Pulsar published DML schema fail",
 21056            "tooltip": {
 21057              "shared": true,
 21058              "sort": 0,
 21059              "value_type": "individual"
 21060            },
 21061            "type": "graph",
 21062            "xaxis": {
 21063              "buckets": null,
 21064              "mode": "time",
 21065              "name": null,
 21066              "show": true,
 21067              "values": []
 21068            },
 21069            "yaxes": [
 21070              {
 21071                "format": "short",
 21072                "label": null,
 21073                "logBase": 1,
 21074                "max": null,
 21075                "min": null,
 21076                "show": true
 21077              },
 21078              {
 21079                "format": "short",
 21080                "label": null,
 21081                "logBase": 1,
 21082                "max": null,
 21083                "min": null,
 21084                "show": true
 21085              }
 21086            ],
 21087            "yaxis": {
 21088              "align": false,
 21089              "alignLevel": null
 21090            }
 21091          },
 21092          {
 21093            "aliasColors": {},
 21094            "bars": false,
 21095            "dashLength": 10,
 21096            "dashes": false,
 21097            "datasource": "${DS_TEST-CLUSTER}",
 21098            "fieldConfig": {
 21099              "defaults": {},
 21100              "overrides": []
 21101            },
 21102            "fill": 1,
 21103            "fillGradient": 0,
 21104            "gridPos": {
 21105              "h": 9,
 21106              "w": 12,
 21107              "x": 0,
 21108              "y": 77
 21109            },
 21110            "hiddenSeries": false,
 21111            "id": 10007,
 21112            "legend": {
 21113              "alignAsTable": true,
 21114              "avg": false,
 21115              "current": true,
 21116              "max": false,
 21117              "min": false,
 21118              "show": true,
 21119              "total": false,
 21120              "values": true
 21121            },
 21122            "lines": true,
 21123            "linewidth": 1,
 21124            "nullPointMode": "null as zero",
 21125            "options": {
 21126              "alertThreshold": true
 21127            },
 21128            "percentage": false,
 21129            "pluginVersion": "7.5.11",
 21130            "pointradius": 2,
 21131            "points": false,
 21132            "renderer": "flot",
 21133            "seriesOverrides": [],
 21134            "spaceLength": 10,
 21135            "stack": true,
 21136            "steppedLine": false,
 21137            "targets": [
 21138              {
 21139                "exemplar": true,
 21140                "expr": "sum(pulsar_client_bytes_published{changefeed=~\"$changefeed\"})  by (changefeed, instance)",
 21141                "format": "time_series",
 21142                "interval": "",
 21143                "legendFormat": "{{changefeed}}-{{instance}}",
 21144                "queryType": "randomWalk",
 21145                "refId": "A"
 21146              }
 21147            ],
 21148            "thresholds": [],
 21149            "timeFrom": null,
 21150            "timeRegions": [],
 21151            "timeShift": null,
 21152            "title": "Pulsar client bytes published ",
 21153            "tooltip": {
 21154              "shared": true,
 21155              "sort": 0,
 21156              "value_type": "individual"
 21157            },
 21158            "type": "graph",
 21159            "xaxis": {
 21160              "buckets": null,
 21161              "mode": "time",
 21162              "name": null,
 21163              "show": true,
 21164              "values": []
 21165            },
 21166            "yaxes": [
 21167              {
 21168                "format": "bytes",
 21169                "label": null,
 21170                "logBase": 1,
 21171                "max": null,
 21172                "min": null,
 21173                "show": true
 21174              },
 21175              {
 21176                "format": "short",
 21177                "label": null,
 21178                "logBase": 1,
 21179                "max": null,
 21180                "min": null,
 21181                "show": true
 21182              }
 21183            ],
 21184            "yaxis": {
 21185              "align": false,
 21186              "alignLevel": null
 21187            }
 21188          },
 21189          {
 21190            "aliasColors": {},
 21191            "bars": false,
 21192            "dashLength": 10,
 21193            "dashes": false,
 21194            "datasource": "${DS_TEST-CLUSTER}",
 21195            "fieldConfig": {
 21196              "defaults": {},
 21197              "overrides": []
 21198            },
 21199            "fill": 1,
 21200            "fillGradient": 0,
 21201            "gridPos": {
 21202              "h": 9,
 21203              "w": 12,
 21204              "x": 12,
 21205              "y": 77
 21206            },
 21207            "hiddenSeries": false,
 21208            "id": 10008,
 21209            "legend": {
 21210              "alignAsTable": false,
 21211              "avg": false,
 21212              "current": true,
 21213              "max": false,
 21214              "min": false,
 21215              "show": true,
 21216              "total": false,
 21217              "values": true
 21218            },
 21219            "lines": true,
 21220            "linewidth": 1,
 21221            "nullPointMode": "null as zero",
 21222            "options": {
 21223              "alertThreshold": true
 21224            },
 21225            "percentage": false,
 21226            "pluginVersion": "7.5.11",
 21227            "pointradius": 2,
 21228            "points": false,
 21229            "renderer": "flot",
 21230            "seriesOverrides": [],
 21231            "spaceLength": 10,
 21232            "stack": true,
 21233            "steppedLine": false,
 21234            "targets": [
 21235              {
 21236                "exemplar": true,
 21237                "expr": "sum(pulsar_client_connections_opened{changefeed=~\"$changefeed\"})  by (changefeed, instance)",
 21238                "format": "time_series",
 21239                "instant": false,
 21240                "interval": "",
 21241                "intervalFactor": 1,
 21242                "legendFormat": "{{changefeed}}-{{instance}}",
 21243                "queryType": "randomWalk",
 21244                "refId": "A"
 21245              }
 21246            ],
 21247            "thresholds": [],
 21248            "timeFrom": null,
 21249            "timeRegions": [],
 21250            "timeShift": null,
 21251            "title": "Pulsar client connections opened",
 21252            "tooltip": {
 21253              "shared": true,
 21254              "sort": 0,
 21255              "value_type": "individual"
 21256            },
 21257            "type": "graph",
 21258            "xaxis": {
 21259              "buckets": null,
 21260              "mode": "time",
 21261              "name": null,
 21262              "show": true,
 21263              "values": []
 21264            },
 21265            "yaxes": [
 21266              {
 21267                "format": "none",
 21268                "label": null,
 21269                "logBase": 1,
 21270                "max": null,
 21271                "min": null,
 21272                "show": true
 21273              },
 21274              {
 21275                "format": "short",
 21276                "label": null,
 21277                "logBase": 1,
 21278                "max": null,
 21279                "min": null,
 21280                "show": true
 21281              }
 21282            ],
 21283            "yaxis": {
 21284              "align": false,
 21285              "alignLevel": null
 21286            }
 21287          },
 21288          {
 21289            "aliasColors": {},
 21290            "bars": false,
 21291            "dashLength": 10,
 21292            "dashes": false,
 21293            "datasource": "${DS_TEST-CLUSTER}",
 21294            "fieldConfig": {
 21295              "defaults": {},
 21296              "overrides": []
 21297            },
 21298            "fill": 1,
 21299            "fillGradient": 0,
 21300            "gridPos": {
 21301              "h": 9,
 21302              "w": 12,
 21303              "x": 0,
 21304              "y": 86
 21305            },
 21306            "hiddenSeries": false,
 21307            "id": 10009,
 21308            "legend": {
 21309              "alignAsTable": false,
 21310              "avg": false,
 21311              "current": true,
 21312              "max": false,
 21313              "min": false,
 21314              "show": true,
 21315              "total": false,
 21316              "values": true
 21317            },
 21318            "lines": true,
 21319            "linewidth": 1,
 21320            "nullPointMode": "null as zero",
 21321            "options": {
 21322              "alertThreshold": true
 21323            },
 21324            "percentage": false,
 21325            "pluginVersion": "7.5.11",
 21326            "pointradius": 2,
 21327            "points": false,
 21328            "renderer": "flot",
 21329            "seriesOverrides": [],
 21330            "spaceLength": 10,
 21331            "stack": true,
 21332            "steppedLine": false,
 21333            "targets": [
 21334              {
 21335                "exemplar": true,
 21336                "expr": "sum(pulsar_client_rpc_count{changefeed=~\"$changefeed\"})  by (changefeed, instance)",
 21337                "format": "time_series",
 21338                "instant": false,
 21339                "interval": "",
 21340                "intervalFactor": 1,
 21341                "legendFormat": "{{changefeed}}-{{instance}}",
 21342                "queryType": "randomWalk",
 21343                "refId": "A"
 21344              }
 21345            ],
 21346            "thresholds": [],
 21347            "timeFrom": null,
 21348            "timeRegions": [],
 21349            "timeShift": null,
 21350            "title": "Pulsar client rpc count",
 21351            "tooltip": {
 21352              "shared": true,
 21353              "sort": 0,
 21354              "value_type": "individual"
 21355            },
 21356            "type": "graph",
 21357            "xaxis": {
 21358              "buckets": null,
 21359              "mode": "time",
 21360              "name": null,
 21361              "show": true,
 21362              "values": []
 21363            },
 21364            "yaxes": [
 21365              {
 21366                "format": "none",
 21367                "label": null,
 21368                "logBase": 1,
 21369                "max": null,
 21370                "min": null,
 21371                "show": true
 21372              },
 21373              {
 21374                "format": "short",
 21375                "label": null,
 21376                "logBase": 1,
 21377                "max": null,
 21378                "min": null,
 21379                "show": true
 21380              }
 21381            ],
 21382            "yaxis": {
 21383              "align": false,
 21384              "alignLevel": null
 21385            }
 21386          },
 21387          {
 21388            "datasource": "${DS_TEST-CLUSTER}",
 21389            "fieldConfig": {
 21390              "defaults": {
 21391                "color": {
 21392                  "mode": "palette-classic"
 21393                },
 21394                "custom": {
 21395                  "axisLabel": "",
 21396                  "axisPlacement": "auto",
 21397                  "barAlignment": 0,
 21398                  "drawStyle": "line",
 21399                  "fillOpacity": 10,
 21400                  "gradientMode": "none",
 21401                  "hideFrom": {
 21402                    "graph": false,
 21403                    "legend": false,
 21404                    "tooltip": false
 21405                  },
 21406                  "lineInterpolation": "linear",
 21407                  "lineWidth": 1,
 21408                  "pointSize": 5,
 21409                  "scaleDistribution": {
 21410                    "type": "linear"
 21411                  },
 21412                  "showPoints": "never",
 21413                  "spanNulls": false
 21414                },
 21415                "mappings": [],
 21416                "thresholds": {
 21417                  "mode": "absolute",
 21418                  "steps": [
 21419                    {
 21420                      "color": "green",
 21421                      "value": null
 21422                    },
 21423                    {
 21424                      "color": "red",
 21425                      "value": 80
 21426                    }
 21427                  ]
 21428                },
 21429                "unit": "s"
 21430              },
 21431              "overrides": []
 21432            },
 21433            "gridPos": {
 21434              "h": 9,
 21435              "w": 12,
 21436              "x": 12,
 21437              "y": 86
 21438            },
 21439            "id": 10010,
 21440            "options": {
 21441              "graph": {},
 21442              "legend": {
 21443                "calcs": [],
 21444                "displayMode": "list",
 21445                "placement": "bottom"
 21446              },
 21447              "tooltipOptions": {
 21448                "mode": "single"
 21449              }
 21450            },
 21451            "pluginVersion": "7.5.11",
 21452            "targets": [
 21453              {
 21454                "exemplar": true,
 21455                "expr": "histogram_quantile(0.999, sum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=~\"$changefeed\"}[1m])) by (le,changefeed,instance))\r",
 21456                "format": "time_series",
 21457                "instant": false,
 21458                "interval": "",
 21459                "intervalFactor": 1,
 21460                "legendFormat": "{{changefeed}}-{{instance}}-P999",
 21461                "queryType": "randomWalk",
 21462                "refId": "A"
 21463              },
 21464              {
 21465                "exemplar": true,
 21466                "expr": "sum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=\"$changefeed\"}[1m])) by (changefeed,instance) / \r\nsum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=\"$changefeed\"}[1m])) by (changefeed,instance)",
 21467                "hide": false,
 21468                "interval": "",
 21469                "legendFormat": "",
 21470                "refId": "B"
 21471              }
 21472            ],
 21473            "timeFrom": null,
 21474            "timeShift": null,
 21475            "title": "Pulsar client producer latency ",
 21476            "type": "timeseries"
 21477          },
 21478          {
 21479            "aliasColors": {},
 21480            "bars": false,
 21481            "dashLength": 10,
 21482            "dashes": false,
 21483            "datasource": "${DS_TEST-CLUSTER}",
 21484            "fieldConfig": {
 21485              "defaults": {
 21486                "unit": "s"
 21487              },
 21488              "overrides": []
 21489            },
 21490            "fill": 1,
 21491            "fillGradient": 0,
 21492            "gridPos": {
 21493              "h": 9,
 21494              "w": 12,
 21495              "x": 0,
 21496              "y": 95
 21497            },
 21498            "hiddenSeries": false,
 21499            "id": 10011,
 21500            "legend": {
 21501              "avg": false,
 21502              "current": false,
 21503              "max": false,
 21504              "min": false,
 21505              "show": true,
 21506              "total": false,
 21507              "values": false
 21508            },
 21509            "lines": true,
 21510            "linewidth": 1,
 21511            "nullPointMode": "null as zero",
 21512            "options": {
 21513              "alertThreshold": true
 21514            },
 21515            "percentage": false,
 21516            "pluginVersion": "7.5.11",
 21517            "pointradius": 2,
 21518            "points": false,
 21519            "renderer": "flot",
 21520            "seriesOverrides": [],
 21521            "spaceLength": 10,
 21522            "stack": false,
 21523            "steppedLine": false,
 21524            "targets": [
 21525              {
 21526                "exemplar": true,
 21527                "expr": "histogram_quantile(0.999, sum(rate(pulsar_client_producer_rpc_latency_seconds_bucket{changefeed=~\"$changefeed\"}[1m])) by (le,changefeed,instance))\r",
 21528                "format": "time_series",
 21529                "instant": false,
 21530                "interval": "",
 21531                "intervalFactor": 1,
 21532                "legendFormat": "{{changefeed}}-{{instance}}",
 21533                "queryType": "randomWalk",
 21534                "refId": "A"
 21535              }
 21536            ],
 21537            "thresholds": [],
 21538            "timeFrom": null,
 21539            "timeRegions": [],
 21540            "timeShift": null,
 21541            "title": "Pulsar client producer rpc latency",
 21542            "tooltip": {
 21543              "shared": true,
 21544              "sort": 0,
 21545              "value_type": "individual"
 21546            },
 21547            "type": "graph",
 21548            "xaxis": {
 21549              "buckets": null,
 21550              "mode": "time",
 21551              "name": null,
 21552              "show": true,
 21553              "values": []
 21554            },
 21555            "yaxes": [
 21556              {
 21557                "format": "s",
 21558                "label": null,
 21559                "logBase": 1,
 21560                "max": null,
 21561                "min": null,
 21562                "show": true
 21563              },
 21564              {
 21565                "format": "short",
 21566                "label": null,
 21567                "logBase": 1,
 21568                "max": null,
 21569                "min": null,
 21570                "show": true
 21571              }
 21572            ],
 21573            "yaxis": {
 21574              "align": false,
 21575              "alignLevel": null
 21576            }
 21577          },
 21578          {
 21579            "aliasColors": {},
 21580            "bars": false,
 21581            "dashLength": 10,
 21582            "dashes": false,
 21583            "datasource": "${DS_TEST-CLUSTER}",
 21584            "description": "",
 21585            "fieldConfig": {
 21586              "defaults": {
 21587                "unit": "none"
 21588              },
 21589              "overrides": []
 21590            },
 21591            "fill": 1,
 21592            "fillGradient": 0,
 21593            "gridPos": {
 21594              "h": 9,
 21595              "w": 12,
 21596              "x": 12,
 21597              "y": 95
 21598            },
 21599            "hiddenSeries": false,
 21600            "id": 10012,
 21601            "legend": {
 21602              "avg": false,
 21603              "current": false,
 21604              "max": false,
 21605              "min": false,
 21606              "show": true,
 21607              "total": false,
 21608              "values": false
 21609            },
 21610            "lines": true,
 21611            "linewidth": 1,
 21612            "nullPointMode": "null as zero",
 21613            "options": {
 21614              "alertThreshold": true
 21615            },
 21616            "percentage": false,
 21617            "pluginVersion": "7.5.11",
 21618            "pointradius": 2,
 21619            "points": false,
 21620            "renderer": "flot",
 21621            "seriesOverrides": [],
 21622            "spaceLength": 10,
 21623            "stack": false,
 21624            "steppedLine": false,
 21625            "targets": [
 21626              {
 21627                "exemplar": true,
 21628                "expr": "sum(pulsar_client_producer_pending_messages{changefeed=~\"$changefeed\"})  by (changefeed, instance)\r",
 21629                "format": "time_series",
 21630                "instant": false,
 21631                "interval": "",
 21632                "intervalFactor": 1,
 21633                "legendFormat": "{{changefeed}}-{{instance}}",
 21634                "queryType": "randomWalk",
 21635                "refId": "A"
 21636              }
 21637            ],
 21638            "thresholds": [],
 21639            "timeFrom": null,
 21640            "timeRegions": [],
 21641            "timeShift": null,
 21642            "title": "Pulsar client producer pending messages",
 21643            "tooltip": {
 21644              "shared": true,
 21645              "sort": 0,
 21646              "value_type": "individual"
 21647            },
 21648            "type": "graph",
 21649            "xaxis": {
 21650              "buckets": null,
 21651              "mode": "time",
 21652              "name": null,
 21653              "show": true,
 21654              "values": []
 21655            },
 21656            "yaxes": [
 21657              {
 21658                "format": "none",
 21659                "label": null,
 21660                "logBase": 1,
 21661                "max": null,
 21662                "min": null,
 21663                "show": true
 21664              },
 21665              {
 21666                "format": "short",
 21667                "label": null,
 21668                "logBase": 1,
 21669                "max": null,
 21670                "min": null,
 21671                "show": true
 21672              }
 21673            ],
 21674            "yaxis": {
 21675              "align": false,
 21676              "alignLevel": null
 21677            }
 21678          },
 21679          {
 21680            "aliasColors": {},
 21681            "bars": false,
 21682            "dashLength": 10,
 21683            "dashes": false,
 21684            "datasource": "${DS_TEST-CLUSTER}",
 21685            "description": "",
 21686            "fieldConfig": {
 21687              "defaults": {
 21688                "unit": "none"
 21689              },
 21690              "overrides": []
 21691            },
 21692            "fill": 1,
 21693            "fillGradient": 0,
 21694            "gridPos": {
 21695              "h": 9,
 21696              "w": 12,
 21697              "x": 12,
 21698              "y": 104
 21699            },
 21700            "hiddenSeries": false,
 21701            "id": 10013,
 21702            "legend": {
 21703              "avg": false,
 21704              "current": false,
 21705              "max": false,
 21706              "min": false,
 21707              "show": true,
 21708              "total": false,
 21709              "values": false
 21710            },
 21711            "lines": true,
 21712            "linewidth": 1,
 21713            "nullPointMode": "null as zero",
 21714            "options": {
 21715              "alertThreshold": true
 21716            },
 21717            "percentage": false,
 21718            "pluginVersion": "7.5.11",
 21719            "pointradius": 2,
 21720            "points": false,
 21721            "renderer": "flot",
 21722            "seriesOverrides": [],
 21723            "spaceLength": 10,
 21724            "stack": false,
 21725            "steppedLine": false,
 21726            "targets": [
 21727              {
 21728                "exemplar": true,
 21729                "expr": "sum(published_message_type_resolved_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\"})  by (changefeed)",
 21730                "format": "time_series",
 21731                "instant": false,
 21732                "interval": "",
 21733                "intervalFactor": 1,
 21734                "legendFormat": "{{changefeed}}-{{topic}}",
 21735                "queryType": "randomWalk",
 21736                "refId": "A"
 21737              }
 21738            ],
 21739            "thresholds": [],
 21740            "timeFrom": null,
 21741            "timeRegions": [],
 21742            "timeShift": null,
 21743            "title": "Pulsar client producer pending messages",
 21744            "tooltip": {
 21745              "shared": true,
 21746              "sort": 0,
 21747              "value_type": "individual"
 21748            },
 21749            "type": "graph",
 21750            "xaxis": {
 21751              "buckets": null,
 21752              "mode": "time",
 21753              "name": null,
 21754              "show": true,
 21755              "values": []
 21756            },
 21757            "yaxes": [
 21758              {
 21759                "format": "none",
 21760                "label": null,
 21761                "logBase": 1,
 21762                "max": null,
 21763                "min": null,
 21764                "show": true
 21765              },
 21766              {
 21767                "format": "short",
 21768                "label": null,
 21769                "logBase": 1,
 21770                "max": null,
 21771                "min": null,
 21772                "show": true
 21773              }
 21774            ],
 21775            "yaxis": {
 21776              "align": false,
 21777              "alignLevel": null
 21778            }
 21779          }
 21780        ],
 21781        "title": "Pulsar Sink",
 21782        "type": "row"
 21783      }
 21784    ],
 21785    "refresh": "1m",
 21786    "schemaVersion": 27,
 21787    "style": "dark",
 21788    "tags": [],
 21789    "templating": {
 21790      "list": [
 21791        {
 21792          "allValue": null,
 21793          "current": {},
 21794          "datasource": "${DS_TEST-CLUSTER}",
 21795          "definition": "",
 21796          "description": null,
 21797          "error": null,
 21798          "hide": 0,
 21799          "includeAll": false,
 21800          "label": "K8s-cluster",
 21801          "multi": false,
 21802          "name": "k8s_cluster",
 21803          "options": [],
 21804          "query": {
 21805            "query": "label_values(go_goroutines, k8s_cluster)",
 21806            "refId": "local-k8s_cluster-Variable-Query"
 21807          },
 21808          "refresh": 2,
 21809          "regex": "",
 21810          "skipUrlSync": false,
 21811          "sort": 1,
 21812          "tagValuesQuery": "",
 21813          "tags": [],
 21814          "tagsQuery": "",
 21815          "type": "query",
 21816          "useTags": false
 21817        },
 21818        {
 21819          "allValue": null,
 21820          "current": {},
 21821          "datasource": "${DS_TEST-CLUSTER}",
 21822          "definition": "",
 21823          "description": null,
 21824          "error": null,
 21825          "hide": 0,
 21826          "includeAll": false,
 21827          "label": "tidb_cluster",
 21828          "multi": false,
 21829          "name": "tidb_cluster",
 21830          "options": [],
 21831          "query": {
 21832            "query": "label_values(go_goroutines{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)",
 21833            "refId": "local-tidb_cluster-Variable-Query"
 21834          },
 21835          "refresh": 2,
 21836          "regex": "",
 21837          "skipUrlSync": false,
 21838          "sort": 1,
 21839          "tagValuesQuery": "",
 21840          "tags": [],
 21841          "tagsQuery": "",
 21842          "type": "query",
 21843          "useTags": false
 21844        },
 21845        {
 21846          "allValue": ".*",
 21847          "current": {},
 21848          "datasource": "${DS_TEST-CLUSTER}",
 21849          "definition": "label_values(ticdc_processor_processor_tick_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, namespace)",
 21850          "description": null,
 21851          "error": null,
 21852          "hide": 0,
 21853          "includeAll": true,
 21854          "label": "Namespace",
 21855          "multi": true,
 21856          "name": "namespace",
 21857          "options": [],
 21858          "query": {
 21859            "query": "label_values(ticdc_processor_processor_tick_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, namespace)",
 21860            "refId": "StandardVariableQuery"
 21861          },
 21862          "refresh": 2,
 21863          "regex": "",
 21864          "skipUrlSync": false,
 21865          "sort": 0,
 21866          "tagValuesQuery": "",
 21867          "tags": [],
 21868          "tagsQuery": "",
 21869          "type": "query",
 21870          "useTags": false
 21871        },
 21872        {
 21873          "allValue": ".*",
 21874          "current": {},
 21875          "datasource": "${DS_TEST-CLUSTER}",
 21876          "definition": "label_values(ticdc_processor_processor_tick_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, changefeed)",
 21877          "description": null,
 21878          "error": null,
 21879          "hide": 0,
 21880          "includeAll": true,
 21881          "label": "Changefeed",
 21882          "multi": true,
 21883          "name": "changefeed",
 21884          "options": [],
 21885          "query": {
 21886            "query": "label_values(ticdc_processor_processor_tick_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, changefeed)",
 21887            "refId": "StandardVariableQuery"
 21888          },
 21889          "refresh": 2,
 21890          "regex": "",
 21891          "skipUrlSync": false,
 21892          "sort": 0,
 21893          "tagValuesQuery": "",
 21894          "tags": [],
 21895          "tagsQuery": "",
 21896          "type": "query",
 21897          "useTags": false
 21898        },
 21899        {
 21900          "allValue": ".*",
 21901          "current": {},
 21902          "datasource": "${DS_TEST-CLUSTER}",
 21903          "definition": "label_values(process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}, instance)",
 21904          "description": null,
 21905          "error": null,
 21906          "hide": 0,
 21907          "includeAll": true,
 21908          "label": "TiCDC",
 21909          "multi": true,
 21910          "name": "ticdc_instance",
 21911          "options": [],
 21912          "query": {
 21913            "query": "label_values(process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}, instance)",
 21914            "refId": "StandardVariableQuery"
 21915          },
 21916          "refresh": 2,
 21917          "regex": "",
 21918          "skipUrlSync": false,
 21919          "sort": 0,
 21920          "tagValuesQuery": "",
 21921          "tags": [],
 21922          "tagsQuery": "",
 21923          "type": "query",
 21924          "useTags": false
 21925        },
 21926        {
 21927          "allValue": ".*",
 21928          "current": {},
 21929          "datasource": "${DS_TEST-CLUSTER}",
 21930          "definition": "label_values(tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, instance)",
 21931          "description": null,
 21932          "error": null,
 21933          "hide": 0,
 21934          "includeAll": true,
 21935          "label": "TiKV",
 21936          "multi": false,
 21937          "name": "tikv_instance",
 21938          "options": [],
 21939          "query": {
 21940            "query": "label_values(tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, instance)",
 21941            "refId": "local-tikv_instance-Variable-Query"
 21942          },
 21943          "refresh": 2,
 21944          "regex": "",
 21945          "skipUrlSync": false,
 21946          "sort": 1,
 21947          "tagValuesQuery": "",
 21948          "tags": [],
 21949          "tagsQuery": "",
 21950          "type": "query",
 21951          "useTags": false
 21952        },
 21953        {
 21954          "allValue": "9999999999",
 21955          "current": {
 21956            "selected": true,
 21957            "text": "All",
 21958            "value": "$__all"
 21959          },
 21960          "description": null,
 21961          "error": null,
 21962          "hide": 0,
 21963          "includeAll": true,
 21964          "label": "Latency spike (s) >",
 21965          "multi": false,
 21966          "name": "spike_threshold",
 21967          "options": [
 21968            {
 21969              "selected": true,
 21970              "text": "All",
 21971              "value": "$__all"
 21972            },
 21973            {
 21974              "selected": false,
 21975              "text": "1",
 21976              "value": "1"
 21977            },
 21978            {
 21979              "selected": false,
 21980              "text": "3",
 21981              "value": "3"
 21982            },
 21983            {
 21984              "selected": false,
 21985              "text": "5",
 21986              "value": "5"
 21987            },
 21988            {
 21989              "selected": false,
 21990              "text": "10",
 21991              "value": "10"
 21992            },
 21993            {
 21994              "selected": false,
 21995              "text": "60",
 21996              "value": "60"
 21997            },
 21998            {
 21999              "selected": false,
 22000              "text": "300",
 22001              "value": "300"
 22002            }
 22003          ],
 22004          "query": "1, 3, 5, 10, 60, 300",
 22005          "queryValue": "",
 22006          "skipUrlSync": false,
 22007          "type": "custom"
 22008        },
 22009        {
 22010          "allValue": "",
 22011          "current": {},
 22012          "datasource": "${DS_TEST-CLUSTER}",
 22013          "definition": "label_values(process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}, instance)",
 22014          "description": null,
 22015          "error": null,
 22016          "hide": 0,
 22017          "includeAll": true,
 22018          "label": "Runtime metrics",
 22019          "multi": false,
 22020          "name": "runtime_instance",
 22021          "options": [],
 22022          "query": {
 22023            "query": "label_values(process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}, instance)",
 22024            "refId": "local-runtime_instance-Variable-Query"
 22025          },
 22026          "refresh": 2,
 22027          "regex": "",
 22028          "skipUrlSync": false,
 22029          "sort": 0,
 22030          "tagValuesQuery": "",
 22031          "tags": [],
 22032          "tagsQuery": "",
 22033          "type": "query",
 22034          "useTags": false
 22035        },
 22036        {
 22037          "allValue": null,
 22038          "current": {},
 22039          "datasource": "${DS_TEST-CLUSTER}",
 22040          "definition": "label_values(ticdc_actor_number_of_workers{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}, name)",
 22041          "description": null,
 22042          "error": null,
 22043          "hide": 0,
 22044          "includeAll": true,
 22045          "label": "Actor",
 22046          "multi": true,
 22047          "name": "actor_name",
 22048          "options": [],
 22049          "query": {
 22050            "query": "label_values(ticdc_actor_number_of_workers{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}, name)",
 22051            "refId": "local-actor_name-Variable-Query"
 22052          },
 22053          "refresh": 2,
 22054          "regex": "",
 22055          "skipUrlSync": false,
 22056          "sort": 0,
 22057          "tagValuesQuery": "",
 22058          "tags": [],
 22059          "tagsQuery": "",
 22060          "type": "query",
 22061          "useTags": false
 22062        }
 22063      ]
 22064    },
 22065    "time": {
 22066      "from": "now-3h",
 22067      "to": "now"
 22068    },
 22069    "timepicker": {
 22070      "refresh_intervals": [
 22071        "5s",
 22072        "10s",
 22073        "30s",
 22074        "1m",
 22075        "5m",
 22076        "15m",
 22077        "30m",
 22078        "1h",
 22079        "2h",
 22080        "1d"
 22081      ],
 22082      "time_options": [
 22083        "5m",
 22084        "15m",
 22085        "1h",
 22086        "6h",
 22087        "12h",
 22088        "24h",
 22089        "2d",
 22090        "7d",
 22091        "30d"
 22092      ]
 22093    },
 22094    "timezone": "browser",
 22095    "title": "${DS_TEST-CLUSTER}-TiCDC",
 22096    "uid": "YiGL8hBZ1",
 22097    "version": 57
 22098  }