github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/metrics/grafana/ticdc.json (about) 1 { 2 "__inputs": [ 3 { 4 "name": "DS_TEST-CLUSTER", 5 "label": "${DS_TEST-CLUSTER}", 6 "description": "", 7 "type": "datasource", 8 "pluginId": "prometheus", 9 "pluginName": "Prometheus" 10 } 11 ], 12 "__requires": [ 13 { 14 "type": "grafana", 15 "id": "grafana", 16 "name": "Grafana", 17 "version": "7.5.11" 18 }, 19 { 20 "type": "panel", 21 "id": "graph", 22 "name": "Graph", 23 "version": "" 24 }, 25 { 26 "type": "panel", 27 "id": "heatmap", 28 "name": "Heatmap", 29 "version": "" 30 }, 31 { 32 "type": "datasource", 33 "id": "prometheus", 34 "name": "Prometheus", 35 "version": "1.0.0" 36 }, 37 { 38 "type": "panel", 39 "id": "table", 40 "name": "Table", 41 "version": "" 42 }, 43 { 44 "type": "panel", 45 "id": "timeseries", 46 "name": "Time series", 47 "version": "" 48 } 49 ], 50 "annotations": { 51 "list": [ 52 { 53 "builtIn": 1, 54 "datasource": "-- Grafana --", 55 "enable": false, 56 "expr": "", 57 "hide": true, 58 "iconColor": "#F2495C", 59 "limit": 100, 60 "name": "", 61 "showIn": 0, 62 "tagKeys": "", 63 "textFormat": "", 64 "titleFormat": "", 65 "type": "dashboard", 66 "useValueForTime": false 67 }, 68 { 69 "datasource": "${DS_TEST-CLUSTER}", 70 "enable": true, 71 "expr": "max(ticdc_owner_checkpoint_ts_lag) by (changefeed, instance) > BOOL $spike_threshold", 72 "hide": true, 73 "iconColor": "#F2495C", 74 "limit": 100, 75 "name": "Latency spike", 76 "showIn": 0, 77 "tagKeys": "changefeed", 78 "tags": [], 79 "titleFormat": "Latency spike", 80 "type": "tags", 81 "useValueForTime": false 82 }, 83 { 84 "datasource": "${DS_TEST-CLUSTER}", 85 "enable": false, 86 "expr": "delta(up{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=~\"tikv|ticdc|tidb|pd\"}[30s]) < BOOL 0", 87 "hide": false, 88 "iconColor": "#FF9830", 89 "limit": 100, 90 "name": "Server down", 91 "showIn": 0, 92 "step": "15s", 93 "tagKeys": "instance,job", 94 "tags": [], 95 "textFormat": "", 96 "titleFormat": "Down", 97 "type": "tags" 98 }, 99 { 100 "datasource": "${DS_TEST-CLUSTER}", 101 "enable": false, 102 "expr": "sum(ALERTS{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", alertstate=\"firing\", alertname=~\"ticdc.*\"}) by (alertname) > BOOL 0", 103 "hide": false, 104 "iconColor": "#B877D9", 105 "limit": 100, 106 "name": "All TiCDC alerts", 107 "showIn": 0, 108 "tagKeys": "alertname", 109 "tags": [], 110 "titleFormat": "Alert Name", 111 "type": "tags" 112 }, 113 { 114 "datasource": "${DS_TEST-CLUSTER}", 115 "enable": false, 116 "expr": "delta(tikv_cdc_region_resolve_status{status=\"resolved\"}[30s]) < BOOL -800", 117 "hide": false, 118 "iconColor": "rgba(255, 96, 96, 1)", 119 "limit": 100, 120 "name": "Resolved region drop", 121 "showIn": 0, 122 "step": "15s", 123 "tagKeys": "instance", 124 "tags": [], 125 "titleFormat": "Resolved region drop", 126 "type": "tags" 127 } 128 ] 129 }, 130 "editable": true, 131 "gnetId": null, 132 "graphTooltip": 1, 133 "id": null, 134 "iteration": 1705029240982, 135 "links": [], 136 "panels": [ 137 { 138 "collapsed": true, 139 "datasource": null, 140 "gridPos": { 141 "h": 1, 142 "w": 24, 143 "x": 0, 144 "y": 0 145 }, 146 "id": 397, 147 "panels": [ 148 { 149 "aliasColors": {}, 150 "bars": false, 151 "dashLength": 10, 152 "dashes": false, 153 "datasource": "${DS_TEST-CLUSTER}", 154 "description": "The lag between changefeed checkpoint ts and the latest ts of upstream TiDB.", 155 "fieldConfig": { 156 "defaults": { 157 "links": [] 158 }, 159 "overrides": [] 160 }, 161 "fill": 1, 162 "fillGradient": 0, 163 "gridPos": { 164 "h": 4, 165 "w": 12, 166 "x": 0, 167 "y": 1 168 }, 169 "hiddenSeries": false, 170 "id": 398, 171 "legend": { 172 "alignAsTable": true, 173 "avg": false, 174 "current": true, 175 "max": false, 176 "min": false, 177 "rightSide": true, 178 "show": false, 179 "total": false, 180 "values": true 181 }, 182 "lines": true, 183 "linewidth": 1, 184 "links": [], 185 "nullPointMode": "null", 186 "options": { 187 "alertThreshold": true 188 }, 189 "paceLength": 10, 190 "percentage": false, 191 "pluginVersion": "7.5.11", 192 "pointradius": 2, 193 "points": false, 194 "renderer": "flot", 195 "seriesOverrides": [], 196 "spaceLength": 10, 197 "stack": false, 198 "steppedLine": false, 199 "targets": [ 200 { 201 "exemplar": true, 202 "expr": "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)", 203 "format": "time_series", 204 "interval": "", 205 "intervalFactor": 1, 206 "legendFormat": "{{namespace}}-{{changefeed}}", 207 "refId": "A" 208 } 209 ], 210 "thresholds": [], 211 "timeFrom": null, 212 "timeRegions": [], 213 "timeShift": null, 214 "title": "Changefeed checkpoint lag", 215 "tooltip": { 216 "shared": true, 217 "sort": 0, 218 "value_type": "individual" 219 }, 220 "type": "graph", 221 "xaxis": { 222 "buckets": null, 223 "mode": "time", 224 "name": null, 225 "show": true, 226 "values": [] 227 }, 228 "yaxes": [ 229 { 230 "format": "s", 231 "label": null, 232 "logBase": 1, 233 "max": null, 234 "min": "0", 235 "show": true 236 }, 237 { 238 "format": "short", 239 "label": null, 240 "logBase": 1, 241 "max": null, 242 "min": null, 243 "show": true 244 } 245 ], 246 "yaxis": { 247 "align": false, 248 "alignLevel": null 249 } 250 }, 251 { 252 "aliasColors": {}, 253 "bars": false, 254 "dashLength": 10, 255 "dashes": false, 256 "datasource": "${DS_TEST-CLUSTER}", 257 "description": "The lag between changefeed resolved ts and the latest ts of upstream TiDB.", 258 "fieldConfig": { 259 "defaults": { 260 "links": [] 261 }, 262 "overrides": [] 263 }, 264 "fill": 1, 265 "fillGradient": 0, 266 "gridPos": { 267 "h": 4, 268 "w": 12, 269 "x": 12, 270 "y": 1 271 }, 272 "hiddenSeries": false, 273 "id": 468, 274 "legend": { 275 "alignAsTable": true, 276 "avg": false, 277 "current": true, 278 "max": false, 279 "min": false, 280 "rightSide": true, 281 "show": false, 282 "total": false, 283 "values": true 284 }, 285 "lines": true, 286 "linewidth": 1, 287 "links": [], 288 "nullPointMode": "null", 289 "options": { 290 "alertThreshold": true 291 }, 292 "paceLength": 10, 293 "percentage": false, 294 "pluginVersion": "7.5.11", 295 "pointradius": 2, 296 "points": false, 297 "renderer": "flot", 298 "seriesOverrides": [], 299 "spaceLength": 10, 300 "stack": false, 301 "steppedLine": false, 302 "targets": [ 303 { 304 "exemplar": true, 305 "expr": "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namesapce=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", 306 "format": "time_series", 307 "interval": "", 308 "intervalFactor": 1, 309 "legendFormat": "{{namespace}}-{{changefeed}}-resolvedts", 310 "refId": "A" 311 } 312 ], 313 "thresholds": [], 314 "timeFrom": null, 315 "timeRegions": [], 316 "timeShift": null, 317 "title": "Changefeed resolved ts lag", 318 "tooltip": { 319 "shared": true, 320 "sort": 0, 321 "value_type": "individual" 322 }, 323 "type": "graph", 324 "xaxis": { 325 "buckets": null, 326 "mode": "time", 327 "name": null, 328 "show": true, 329 "values": [] 330 }, 331 "yaxes": [ 332 { 333 "format": "s", 334 "label": null, 335 "logBase": 1, 336 "max": null, 337 "min": "0", 338 "show": true 339 }, 340 { 341 "format": "short", 342 "label": null, 343 "logBase": 1, 344 "max": null, 345 "min": null, 346 "show": true 347 } 348 ], 349 "yaxis": { 350 "align": false, 351 "alignLevel": null 352 } 353 }, 354 { 355 "aliasColors": {}, 356 "bars": true, 357 "dashLength": 10, 358 "dashes": false, 359 "datasource": "${DS_TEST-CLUSTER}", 360 "description": "The number of errors that interrupt Eventfeed RPC", 361 "fieldConfig": { 362 "defaults": { 363 "links": [] 364 }, 365 "overrides": [] 366 }, 367 "fill": 1, 368 "fillGradient": 0, 369 "gridPos": { 370 "h": 4, 371 "w": 12, 372 "x": 0, 373 "y": 5 374 }, 375 "hiddenSeries": false, 376 "id": 469, 377 "legend": { 378 "alignAsTable": false, 379 "avg": false, 380 "current": false, 381 "hideEmpty": true, 382 "hideZero": true, 383 "max": false, 384 "min": false, 385 "rightSide": false, 386 "show": true, 387 "total": false, 388 "values": false 389 }, 390 "lines": true, 391 "linewidth": 1, 392 "links": [], 393 "nullPointMode": "null", 394 "options": { 395 "alertThreshold": true 396 }, 397 "paceLength": 10, 398 "percentage": false, 399 "pluginVersion": "7.5.11", 400 "pointradius": 2, 401 "points": false, 402 "renderer": "flot", 403 "seriesOverrides": [], 404 "spaceLength": 10, 405 "stack": false, 406 "steppedLine": false, 407 "targets": [ 408 { 409 "expr": "sum(increase(ticdc_kvclient_event_feed_error_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (type)", 410 "format": "time_series", 411 "hide": false, 412 "interval": "1m", 413 "intervalFactor": 1, 414 "legendFormat": "{{type}}", 415 "refId": "A" 416 } 417 ], 418 "thresholds": [], 419 "timeFrom": null, 420 "timeRegions": [], 421 "timeShift": null, 422 "title": "Eventfeed error/m", 423 "tooltip": { 424 "shared": true, 425 "sort": 0, 426 "value_type": "individual" 427 }, 428 "type": "graph", 429 "xaxis": { 430 "buckets": null, 431 "mode": "time", 432 "name": null, 433 "show": true, 434 "values": [] 435 }, 436 "yaxes": [ 437 { 438 "format": "short", 439 "label": null, 440 "logBase": 1, 441 "max": null, 442 "min": null, 443 "show": true 444 }, 445 { 446 "format": "short", 447 "label": null, 448 "logBase": 1, 449 "max": null, 450 "min": null, 451 "show": true 452 } 453 ], 454 "yaxis": { 455 "align": false, 456 "alignLevel": null 457 } 458 }, 459 { 460 "aliasColors": {}, 461 "bars": true, 462 "dashLength": 10, 463 "dashes": false, 464 "datasource": "${DS_TEST-CLUSTER}", 465 "description": "The number of PD scheduling operator.", 466 "fieldConfig": { 467 "defaults": { 468 "links": [] 469 }, 470 "overrides": [] 471 }, 472 "fill": 1, 473 "fillGradient": 0, 474 "gridPos": { 475 "h": 4, 476 "w": 12, 477 "x": 12, 478 "y": 5 479 }, 480 "hiddenSeries": false, 481 "id": 466, 482 "legend": { 483 "alignAsTable": false, 484 "avg": false, 485 "current": false, 486 "hideEmpty": true, 487 "hideZero": true, 488 "max": false, 489 "min": false, 490 "rightSide": false, 491 "show": true, 492 "total": false, 493 "values": false 494 }, 495 "lines": true, 496 "linewidth": 1, 497 "links": [], 498 "nullPointMode": "null", 499 "options": { 500 "alertThreshold": true 501 }, 502 "paceLength": 10, 503 "percentage": false, 504 "pluginVersion": "7.5.11", 505 "pointradius": 2, 506 "points": false, 507 "renderer": "flot", 508 "seriesOverrides": [], 509 "spaceLength": 10, 510 "stack": false, 511 "steppedLine": false, 512 "targets": [ 513 { 514 "expr": "sum(increase(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", event=\"create\"}[1m])) by (type)", 515 "format": "time_series", 516 "hide": false, 517 "interval": "1m", 518 "intervalFactor": 1, 519 "legendFormat": "{{type}}", 520 "refId": "A" 521 } 522 ], 523 "thresholds": [], 524 "timeFrom": null, 525 "timeRegions": [], 526 "timeShift": null, 527 "title": "PD operator/m", 528 "tooltip": { 529 "shared": true, 530 "sort": 0, 531 "value_type": "individual" 532 }, 533 "type": "graph", 534 "xaxis": { 535 "buckets": null, 536 "mode": "time", 537 "name": null, 538 "show": true, 539 "values": [] 540 }, 541 "yaxes": [ 542 { 543 "format": "short", 544 "label": null, 545 "logBase": 1, 546 "max": null, 547 "min": null, 548 "show": true 549 }, 550 { 551 "format": "short", 552 "label": null, 553 "logBase": 1, 554 "max": null, 555 "min": null, 556 "show": true 557 } 558 ], 559 "yaxis": { 560 "align": false, 561 "alignLevel": null 562 } 563 }, 564 { 565 "aliasColors": {}, 566 "bars": false, 567 "dashLength": 10, 568 "dashes": false, 569 "datasource": "${DS_TEST-CLUSTER}", 570 "description": "99.9% of TiDB query durations.", 571 "fieldConfig": { 572 "defaults": {}, 573 "overrides": [] 574 }, 575 "fill": 1, 576 "fillGradient": 0, 577 "gridPos": { 578 "h": 4, 579 "w": 12, 580 "x": 0, 581 "y": 9 582 }, 583 "hiddenSeries": false, 584 "id": 479, 585 "legend": { 586 "alignAsTable": false, 587 "avg": false, 588 "current": false, 589 "hideEmpty": false, 590 "hideZero": false, 591 "max": false, 592 "min": false, 593 "rightSide": false, 594 "show": true, 595 "total": false, 596 "values": false 597 }, 598 "lines": true, 599 "linewidth": 1, 600 "links": [], 601 "nullPointMode": "null as zero", 602 "options": { 603 "alertThreshold": true 604 }, 605 "percentage": false, 606 "pluginVersion": "7.5.11", 607 "pointradius": 5, 608 "points": false, 609 "renderer": "flot", 610 "seriesOverrides": [], 611 "spaceLength": 10, 612 "stack": false, 613 "steppedLine": false, 614 "targets": [ 615 { 616 "expr": "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", sql_type!~\"internal|Use|Show\"}[1m])) by (le, sql_type))", 617 "format": "time_series", 618 "intervalFactor": 2, 619 "legendFormat": "{{sql_type}}", 620 "refId": "B" 621 } 622 ], 623 "thresholds": [], 624 "timeFrom": null, 625 "timeRegions": [], 626 "timeShift": null, 627 "title": "TiDB query duration", 628 "tooltip": { 629 "shared": true, 630 "sort": 0, 631 "value_type": "individual" 632 }, 633 "type": "graph", 634 "xaxis": { 635 "buckets": null, 636 "mode": "time", 637 "name": null, 638 "show": true, 639 "values": [] 640 }, 641 "yaxes": [ 642 { 643 "format": "s", 644 "label": null, 645 "logBase": 1, 646 "max": null, 647 "min": "0", 648 "show": true 649 }, 650 { 651 "format": "short", 652 "label": null, 653 "logBase": 1, 654 "max": null, 655 "min": null, 656 "show": true 657 } 658 ], 659 "yaxis": { 660 "align": false, 661 "alignLevel": null 662 } 663 }, 664 { 665 "aliasColors": {}, 666 "bars": false, 667 "dashLength": 10, 668 "dashes": false, 669 "datasource": "${DS_TEST-CLUSTER}", 670 "decimals": 1, 671 "description": "The min resolved ts lag of each TiKV", 672 "editable": true, 673 "error": false, 674 "fieldConfig": { 675 "defaults": {}, 676 "overrides": [] 677 }, 678 "fill": 0, 679 "fillGradient": 0, 680 "grid": {}, 681 "gridPos": { 682 "h": 4, 683 "w": 12, 684 "x": 12, 685 "y": 9 686 }, 687 "hiddenSeries": false, 688 "id": 473, 689 "legend": { 690 "alignAsTable": true, 691 "avg": false, 692 "current": true, 693 "max": false, 694 "min": false, 695 "rightSide": false, 696 "show": false, 697 "sideWidth": null, 698 "sort": null, 699 "sortDesc": null, 700 "total": false, 701 "values": true 702 }, 703 "lines": true, 704 "linewidth": 1, 705 "links": [], 706 "nullPointMode": "null", 707 "options": { 708 "alertThreshold": true 709 }, 710 "paceLength": 10, 711 "percentage": false, 712 "pluginVersion": "7.5.11", 713 "pointradius": 1, 714 "points": false, 715 "renderer": "flot", 716 "seriesOverrides": [ 717 { 718 "alias": "/.*method/", 719 "lines": true, 720 "linewidth": 3, 721 "points": false, 722 "yaxis": 2 723 }, 724 { 725 "alias": "/.*-lag/", 726 "bars": true, 727 "fill": 1 728 } 729 ], 730 "spaceLength": 10, 731 "stack": false, 732 "steppedLine": false, 733 "targets": [ 734 { 735 "expr": "scalar(max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}))/1000 - avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000) by (instance) > 0", 736 "format": "time_series", 737 "hide": false, 738 "instant": false, 739 "intervalFactor": 2, 740 "legendFormat": "{{instance}}-min-resolved-lag", 741 "refId": "A", 742 "step": 10 743 }, 744 { 745 "expr": "tikv_cdc_resolved_ts_advance_method{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", 746 "format": "time_series", 747 "hide": false, 748 "intervalFactor": 2, 749 "legendFormat": "{{instance}}-resolved_ts_advance_method", 750 "refId": "B", 751 "step": 10 752 } 753 ], 754 "thresholds": [], 755 "timeFrom": null, 756 "timeRegions": [], 757 "timeShift": null, 758 "title": "TiKV min resolved ts lag", 759 "tooltip": { 760 "msResolution": false, 761 "shared": true, 762 "sort": 0, 763 "value_type": "individual" 764 }, 765 "type": "graph", 766 "xaxis": { 767 "buckets": null, 768 "mode": "time", 769 "name": null, 770 "show": true, 771 "values": [] 772 }, 773 "yaxes": [ 774 { 775 "format": "s", 776 "label": null, 777 "logBase": 1, 778 "max": null, 779 "min": "0", 780 "show": true 781 }, 782 { 783 "format": "none", 784 "label": null, 785 "logBase": 1, 786 "max": null, 787 "min": null, 788 "show": true 789 } 790 ], 791 "yaxis": { 792 "align": false, 793 "alignLevel": null 794 } 795 }, 796 { 797 "aliasColors": {}, 798 "bars": false, 799 "dashLength": 10, 800 "dashes": false, 801 "datasource": "${DS_TEST-CLUSTER}", 802 "description": "The number of changed rows that are written to downstream per second", 803 "fieldConfig": { 804 "defaults": { 805 "links": [] 806 }, 807 "overrides": [] 808 }, 809 "fill": 1, 810 "fillGradient": 0, 811 "gridPos": { 812 "h": 4, 813 "w": 12, 814 "x": 0, 815 "y": 13 816 }, 817 "hiddenSeries": false, 818 "id": 486, 819 "legend": { 820 "alignAsTable": false, 821 "avg": false, 822 "current": false, 823 "max": false, 824 "min": false, 825 "rightSide": false, 826 "show": true, 827 "total": false, 828 "values": false 829 }, 830 "lines": true, 831 "linewidth": 1, 832 "links": [], 833 "nullPointMode": "null", 834 "options": { 835 "alertThreshold": true 836 }, 837 "paceLength": 10, 838 "percentage": false, 839 "pluginVersion": "7.5.11", 840 "pointradius": 2, 841 "points": false, 842 "renderer": "flot", 843 "seriesOverrides": [], 844 "spaceLength": 10, 845 "stack": false, 846 "steppedLine": false, 847 "targets": [ 848 { 849 "exemplar": true, 850 "expr": "sum (rate(ticdc_sink_txn_batch_size_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed)", 851 "format": "time_series", 852 "interval": "", 853 "intervalFactor": 1, 854 "legendFormat": "{{namespace}}-{{changefeed}}", 855 "refId": "A" 856 }, 857 { 858 "exemplar": true, 859 "expr": "sum (rate(ticdc_sink_ddl_exec_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed)", 860 "format": "time_series", 861 "interval": "", 862 "intervalFactor": 1, 863 "legendFormat": "{{namespace}}-{{changefeed}}-ddl", 864 "refId": "B" 865 } 866 ], 867 "thresholds": [], 868 "timeFrom": null, 869 "timeRegions": [], 870 "timeShift": null, 871 "title": "Sink write rows/s", 872 "tooltip": { 873 "shared": true, 874 "sort": 0, 875 "value_type": "individual" 876 }, 877 "type": "graph", 878 "xaxis": { 879 "buckets": null, 880 "mode": "time", 881 "name": null, 882 "show": true, 883 "values": [] 884 }, 885 "yaxes": [ 886 { 887 "format": "none", 888 "label": null, 889 "logBase": 1, 890 "max": null, 891 "min": null, 892 "show": true 893 }, 894 { 895 "format": "short", 896 "label": null, 897 "logBase": 1, 898 "max": null, 899 "min": null, 900 "show": true 901 } 902 ], 903 "yaxis": { 904 "align": false, 905 "alignLevel": null 906 } 907 }, 908 { 909 "aliasColors": {}, 910 "bars": false, 911 "dashLength": 10, 912 "dashes": false, 913 "datasource": "${DS_TEST-CLUSTER}", 914 "description": "99.9% of TiCDC sink write durations.", 915 "fieldConfig": { 916 "defaults": {}, 917 "overrides": [] 918 }, 919 "fill": 1, 920 "fillGradient": 0, 921 "gridPos": { 922 "h": 4, 923 "w": 12, 924 "x": 12, 925 "y": 13 926 }, 927 "hiddenSeries": false, 928 "id": 485, 929 "legend": { 930 "alignAsTable": false, 931 "avg": false, 932 "current": false, 933 "hideEmpty": false, 934 "hideZero": false, 935 "max": false, 936 "min": false, 937 "rightSide": false, 938 "show": true, 939 "total": false, 940 "values": false 941 }, 942 "lines": true, 943 "linewidth": 1, 944 "links": [], 945 "nullPointMode": "null as zero", 946 "options": { 947 "alertThreshold": true 948 }, 949 "percentage": false, 950 "pluginVersion": "7.5.11", 951 "pointradius": 5, 952 "points": false, 953 "renderer": "flot", 954 "seriesOverrides": [], 955 "spaceLength": 10, 956 "stack": false, 957 "steppedLine": false, 958 "targets": [ 959 { 960 "exemplar": true, 961 "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_worker_flush_duration{tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[1m])) by (le, namespace,changefeed))", 962 "format": "time_series", 963 "interval": "", 964 "intervalFactor": 2, 965 "legendFormat": "{{namespace}}-{{changefeed}}", 966 "refId": "B" 967 }, 968 { 969 "exemplar": true, 970 "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[1m])) by (le, namespace, changefeed))", 971 "format": "time_series", 972 "interval": "", 973 "intervalFactor": 2, 974 "legendFormat": "{{namespace}}-{{changefeed}}-ddl", 975 "refId": "A" 976 } 977 ], 978 "thresholds": [], 979 "timeFrom": null, 980 "timeRegions": [], 981 "timeShift": null, 982 "title": "Sink write duration", 983 "tooltip": { 984 "shared": true, 985 "sort": 0, 986 "value_type": "individual" 987 }, 988 "type": "graph", 989 "xaxis": { 990 "buckets": null, 991 "mode": "time", 992 "name": null, 993 "show": true, 994 "values": [] 995 }, 996 "yaxes": [ 997 { 998 "format": "s", 999 "label": null, 1000 "logBase": 1, 1001 "max": null, 1002 "min": "0", 1003 "show": true 1004 }, 1005 { 1006 "format": "short", 1007 "label": null, 1008 "logBase": 1, 1009 "max": null, 1010 "min": null, 1011 "show": true 1012 } 1013 ], 1014 "yaxis": { 1015 "align": false, 1016 "alignLevel": null 1017 } 1018 }, 1019 { 1020 "aliasColors": {}, 1021 "bars": false, 1022 "dashLength": 10, 1023 "dashes": false, 1024 "datasource": "${DS_TEST-CLUSTER}", 1025 "decimals": 1, 1026 "description": "The number of incremental scan tasks per minute", 1027 "fieldConfig": { 1028 "defaults": {}, 1029 "overrides": [] 1030 }, 1031 "fill": 1, 1032 "fillGradient": 0, 1033 "gridPos": { 1034 "h": 4, 1035 "w": 12, 1036 "x": 0, 1037 "y": 17 1038 }, 1039 "hiddenSeries": false, 1040 "id": 401, 1041 "legend": { 1042 "alignAsTable": true, 1043 "avg": false, 1044 "current": true, 1045 "hideEmpty": false, 1046 "hideZero": true, 1047 "max": true, 1048 "min": false, 1049 "rightSide": false, 1050 "show": false, 1051 "sideWidth": null, 1052 "sort": "current", 1053 "sortDesc": true, 1054 "total": false, 1055 "values": true 1056 }, 1057 "lines": true, 1058 "linewidth": 1, 1059 "links": [], 1060 "nullPointMode": "null", 1061 "options": { 1062 "alertThreshold": true 1063 }, 1064 "paceLength": 10, 1065 "percentage": false, 1066 "pluginVersion": "7.5.11", 1067 "pointradius": 5, 1068 "points": false, 1069 "renderer": "flot", 1070 "seriesOverrides": [], 1071 "spaceLength": 10, 1072 "stack": false, 1073 "steppedLine": false, 1074 "targets": [ 1075 { 1076 "expr": "sum(increase(tikv_cdc_scan_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (type, instance)", 1077 "format": "time_series", 1078 "hide": false, 1079 "intervalFactor": 1, 1080 "legendFormat": "{{instance}}", 1081 "refId": "A" 1082 } 1083 ], 1084 "thresholds": [], 1085 "timeFrom": null, 1086 "timeRegions": [], 1087 "timeShift": null, 1088 "title": "TiKV scan tasks/m", 1089 "tooltip": { 1090 "shared": true, 1091 "sort": 0, 1092 "value_type": "individual" 1093 }, 1094 "type": "graph", 1095 "xaxis": { 1096 "buckets": null, 1097 "mode": "time", 1098 "name": null, 1099 "show": true, 1100 "values": [] 1101 }, 1102 "yaxes": [ 1103 { 1104 "format": "none", 1105 "label": null, 1106 "logBase": 1, 1107 "max": null, 1108 "min": null, 1109 "show": true 1110 }, 1111 { 1112 "format": "none", 1113 "label": null, 1114 "logBase": 1, 1115 "max": null, 1116 "min": null, 1117 "show": true 1118 } 1119 ], 1120 "yaxis": { 1121 "align": false, 1122 "alignLevel": null 1123 } 1124 }, 1125 { 1126 "aliasColors": {}, 1127 "bars": false, 1128 "dashLength": 10, 1129 "dashes": false, 1130 "datasource": "${DS_TEST-CLUSTER}", 1131 "decimals": 1, 1132 "description": "The total time of incremental scan region takes per minute", 1133 "editable": true, 1134 "error": false, 1135 "fieldConfig": { 1136 "defaults": {}, 1137 "overrides": [] 1138 }, 1139 "fill": 0, 1140 "fillGradient": 0, 1141 "grid": {}, 1142 "gridPos": { 1143 "h": 4, 1144 "w": 12, 1145 "x": 12, 1146 "y": 17 1147 }, 1148 "hiddenSeries": false, 1149 "id": 467, 1150 "legend": { 1151 "alignAsTable": true, 1152 "avg": false, 1153 "current": true, 1154 "max": true, 1155 "min": false, 1156 "rightSide": true, 1157 "show": false, 1158 "sideWidth": null, 1159 "sort": null, 1160 "sortDesc": null, 1161 "total": false, 1162 "values": true 1163 }, 1164 "lines": true, 1165 "linewidth": 1, 1166 "links": [], 1167 "nullPointMode": "null", 1168 "options": { 1169 "alertThreshold": true 1170 }, 1171 "paceLength": 10, 1172 "percentage": false, 1173 "pluginVersion": "7.5.11", 1174 "pointradius": 5, 1175 "points": false, 1176 "renderer": "flot", 1177 "seriesOverrides": [], 1178 "spaceLength": 10, 1179 "stack": false, 1180 "steppedLine": false, 1181 "targets": [ 1182 { 1183 "expr": "sum(increase(tikv_cdc_scan_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)", 1184 "format": "time_series", 1185 "hide": false, 1186 "intervalFactor": 2, 1187 "legendFormat": "{{instance}}", 1188 "refId": "B", 1189 "step": 10 1190 } 1191 ], 1192 "thresholds": [], 1193 "timeFrom": null, 1194 "timeRegions": [], 1195 "timeShift": null, 1196 "title": "TiKV scan region time/m", 1197 "tooltip": { 1198 "msResolution": false, 1199 "shared": true, 1200 "sort": 0, 1201 "value_type": "individual" 1202 }, 1203 "type": "graph", 1204 "xaxis": { 1205 "buckets": null, 1206 "mode": "time", 1207 "name": null, 1208 "show": true, 1209 "values": [] 1210 }, 1211 "yaxes": [ 1212 { 1213 "format": "s", 1214 "label": null, 1215 "logBase": 1, 1216 "max": null, 1217 "min": "0", 1218 "show": true 1219 }, 1220 { 1221 "format": "short", 1222 "label": null, 1223 "logBase": 1, 1224 "max": null, 1225 "min": null, 1226 "show": true 1227 } 1228 ], 1229 "yaxis": { 1230 "align": false, 1231 "alignLevel": null 1232 } 1233 }, 1234 { 1235 "aliasColors": {}, 1236 "bars": false, 1237 "dashLength": 10, 1238 "dashes": false, 1239 "datasource": "${DS_TEST-CLUSTER}", 1240 "decimals": 1, 1241 "description": "The number of leaders on each TiKV instance", 1242 "editable": true, 1243 "error": false, 1244 "fieldConfig": { 1245 "defaults": {}, 1246 "overrides": [] 1247 }, 1248 "fill": 0, 1249 "fillGradient": 0, 1250 "grid": {}, 1251 "gridPos": { 1252 "h": 4, 1253 "w": 12, 1254 "x": 0, 1255 "y": 21 1256 }, 1257 "hiddenSeries": false, 1258 "id": 407, 1259 "legend": { 1260 "alignAsTable": true, 1261 "avg": false, 1262 "current": true, 1263 "max": true, 1264 "min": false, 1265 "rightSide": true, 1266 "show": false, 1267 "sideWidth": null, 1268 "sort": "current", 1269 "sortDesc": true, 1270 "total": false, 1271 "values": true 1272 }, 1273 "lines": true, 1274 "linewidth": 1, 1275 "links": [], 1276 "nullPointMode": "null", 1277 "options": { 1278 "alertThreshold": true 1279 }, 1280 "percentage": false, 1281 "pluginVersion": "7.5.11", 1282 "pointradius": 5, 1283 "points": false, 1284 "renderer": "flot", 1285 "seriesOverrides": [], 1286 "spaceLength": 10, 1287 "stack": false, 1288 "steppedLine": false, 1289 "targets": [ 1290 { 1291 "expr": "sum(delta(tikv_raftstore_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"leader\"}[30s]))", 1292 "format": "time_series", 1293 "hide": false, 1294 "intervalFactor": 2, 1295 "legendFormat": "changed", 1296 "refId": "A", 1297 "step": 10 1298 } 1299 ], 1300 "thresholds": [], 1301 "timeFrom": null, 1302 "timeRegions": [], 1303 "timeShift": null, 1304 "title": "TiKV leader change", 1305 "tooltip": { 1306 "msResolution": false, 1307 "shared": true, 1308 "sort": 0, 1309 "value_type": "cumulative" 1310 }, 1311 "type": "graph", 1312 "xaxis": { 1313 "buckets": null, 1314 "mode": "time", 1315 "name": null, 1316 "show": true, 1317 "values": [] 1318 }, 1319 "yaxes": [ 1320 { 1321 "format": "short", 1322 "label": null, 1323 "logBase": 1, 1324 "max": null, 1325 "min": null, 1326 "show": true 1327 }, 1328 { 1329 "format": "short", 1330 "label": null, 1331 "logBase": 1, 1332 "max": null, 1333 "min": null, 1334 "show": true 1335 } 1336 ], 1337 "yaxis": { 1338 "align": false, 1339 "alignLevel": null 1340 } 1341 }, 1342 { 1343 "aliasColors": {}, 1344 "bars": false, 1345 "dashLength": 10, 1346 "dashes": false, 1347 "datasource": "${DS_TEST-CLUSTER}", 1348 "decimals": 1, 1349 "description": "The number of the processed TiKV admin command", 1350 "editable": true, 1351 "error": false, 1352 "fieldConfig": { 1353 "defaults": {}, 1354 "overrides": [] 1355 }, 1356 "fill": 0, 1357 "fillGradient": 0, 1358 "grid": {}, 1359 "gridPos": { 1360 "h": 4, 1361 "w": 12, 1362 "x": 12, 1363 "y": 21 1364 }, 1365 "hiddenSeries": false, 1366 "id": 470, 1367 "legend": { 1368 "alignAsTable": false, 1369 "avg": false, 1370 "current": false, 1371 "max": false, 1372 "min": false, 1373 "rightSide": false, 1374 "show": true, 1375 "sideWidth": null, 1376 "sort": null, 1377 "sortDesc": null, 1378 "total": false, 1379 "values": false 1380 }, 1381 "lines": true, 1382 "linewidth": 1, 1383 "links": [], 1384 "nullPointMode": "null", 1385 "options": { 1386 "alertThreshold": true 1387 }, 1388 "paceLength": 10, 1389 "percentage": false, 1390 "pluginVersion": "7.5.11", 1391 "pointradius": 5, 1392 "points": false, 1393 "renderer": "flot", 1394 "seriesOverrides": [], 1395 "spaceLength": 10, 1396 "stack": false, 1397 "steppedLine": false, 1398 "targets": [ 1399 { 1400 "expr": "sum(rate(tikv_raftstore_admin_cmd_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", status=\"success\", type!=\"compact\"}[1m])) by (type)", 1401 "format": "time_series", 1402 "hide": false, 1403 "intervalFactor": 2, 1404 "legendFormat": "{{type}}", 1405 "refId": "B", 1406 "step": 10 1407 } 1408 ], 1409 "thresholds": [], 1410 "timeFrom": null, 1411 "timeRegions": [], 1412 "timeShift": null, 1413 "title": "TiKV admin apply/s", 1414 "tooltip": { 1415 "msResolution": false, 1416 "shared": true, 1417 "sort": 0, 1418 "value_type": "individual" 1419 }, 1420 "type": "graph", 1421 "xaxis": { 1422 "buckets": null, 1423 "mode": "time", 1424 "name": null, 1425 "show": true, 1426 "values": [] 1427 }, 1428 "yaxes": [ 1429 { 1430 "format": "none", 1431 "label": null, 1432 "logBase": 1, 1433 "max": null, 1434 "min": "0", 1435 "show": true 1436 }, 1437 { 1438 "format": "short", 1439 "label": null, 1440 "logBase": 1, 1441 "max": null, 1442 "min": null, 1443 "show": true 1444 } 1445 ], 1446 "yaxis": { 1447 "align": false, 1448 "alignLevel": null 1449 } 1450 }, 1451 { 1452 "aliasColors": {}, 1453 "bars": false, 1454 "dashLength": 10, 1455 "dashes": false, 1456 "datasource": "${DS_TEST-CLUSTER}", 1457 "decimals": 1, 1458 "description": "The rate of TiKV advancing resolved ts.", 1459 "fieldConfig": { 1460 "defaults": {}, 1461 "overrides": [] 1462 }, 1463 "fill": 1, 1464 "fillGradient": 0, 1465 "gridPos": { 1466 "h": 4, 1467 "w": 12, 1468 "x": 0, 1469 "y": 25 1470 }, 1471 "hiddenSeries": false, 1472 "id": 471, 1473 "legend": { 1474 "alignAsTable": true, 1475 "avg": false, 1476 "current": true, 1477 "hideEmpty": false, 1478 "hideZero": true, 1479 "max": true, 1480 "min": false, 1481 "rightSide": false, 1482 "show": false, 1483 "sideWidth": null, 1484 "sort": "current", 1485 "sortDesc": true, 1486 "total": false, 1487 "values": true 1488 }, 1489 "lines": true, 1490 "linewidth": 1, 1491 "links": [], 1492 "nullPointMode": "null", 1493 "options": { 1494 "alertThreshold": true 1495 }, 1496 "paceLength": 10, 1497 "percentage": false, 1498 "pluginVersion": "7.5.11", 1499 "pointradius": 5, 1500 "points": false, 1501 "renderer": "flot", 1502 "seriesOverrides": [], 1503 "spaceLength": 10, 1504 "stack": false, 1505 "steppedLine": false, 1506 "targets": [ 1507 { 1508 "expr": "sum(rate(tikv_cdc_resolved_ts_gap_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance) / sum(tikv_cdc_region_resolve_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", status=~\"resolved\"}) by (instance) ", 1509 "format": "time_series", 1510 "hide": false, 1511 "intervalFactor": 1, 1512 "legendFormat": "{{instance}}", 1513 "refId": "A" 1514 } 1515 ], 1516 "thresholds": [], 1517 "timeFrom": null, 1518 "timeRegions": [], 1519 "timeShift": null, 1520 "title": "TiKV advance resovled ts/s", 1521 "tooltip": { 1522 "shared": true, 1523 "sort": 0, 1524 "value_type": "individual" 1525 }, 1526 "type": "graph", 1527 "xaxis": { 1528 "buckets": null, 1529 "mode": "time", 1530 "name": null, 1531 "show": true, 1532 "values": [] 1533 }, 1534 "yaxes": [ 1535 { 1536 "format": "none", 1537 "label": null, 1538 "logBase": 1, 1539 "max": null, 1540 "min": null, 1541 "show": true 1542 }, 1543 { 1544 "format": "none", 1545 "label": null, 1546 "logBase": 1, 1547 "max": null, 1548 "min": null, 1549 "show": true 1550 } 1551 ], 1552 "yaxis": { 1553 "align": false, 1554 "alignLevel": null 1555 } 1556 }, 1557 { 1558 "aliasColors": {}, 1559 "bars": false, 1560 "dashLength": 10, 1561 "dashes": false, 1562 "datasource": "${DS_TEST-CLUSTER}", 1563 "decimals": 1, 1564 "description": "The number of unresolved region per TiKV", 1565 "editable": true, 1566 "error": false, 1567 "fieldConfig": { 1568 "defaults": {}, 1569 "overrides": [] 1570 }, 1571 "fill": 0, 1572 "fillGradient": 0, 1573 "grid": {}, 1574 "gridPos": { 1575 "h": 4, 1576 "w": 12, 1577 "x": 12, 1578 "y": 25 1579 }, 1580 "hiddenSeries": false, 1581 "id": 400, 1582 "legend": { 1583 "alignAsTable": true, 1584 "avg": false, 1585 "current": true, 1586 "max": true, 1587 "min": false, 1588 "rightSide": true, 1589 "show": false, 1590 "sideWidth": null, 1591 "sort": null, 1592 "sortDesc": null, 1593 "total": false, 1594 "values": true 1595 }, 1596 "lines": true, 1597 "linewidth": 1, 1598 "links": [], 1599 "nullPointMode": "null", 1600 "options": { 1601 "alertThreshold": true 1602 }, 1603 "paceLength": 10, 1604 "percentage": false, 1605 "pluginVersion": "7.5.11", 1606 "pointradius": 5, 1607 "points": false, 1608 "renderer": "flot", 1609 "seriesOverrides": [], 1610 "spaceLength": 10, 1611 "stack": false, 1612 "steppedLine": false, 1613 "targets": [ 1614 { 1615 "expr": "sum(tikv_cdc_region_resolve_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", status=\"unresolved\"}) by (status)", 1616 "format": "time_series", 1617 "hide": false, 1618 "intervalFactor": 2, 1619 "legendFormat": "{{status}}", 1620 "refId": "B", 1621 "step": 10 1622 } 1623 ], 1624 "thresholds": [], 1625 "timeFrom": null, 1626 "timeRegions": [], 1627 "timeShift": null, 1628 "title": "TiKV unresolved region count", 1629 "tooltip": { 1630 "msResolution": false, 1631 "shared": true, 1632 "sort": 0, 1633 "value_type": "individual" 1634 }, 1635 "type": "graph", 1636 "xaxis": { 1637 "buckets": null, 1638 "mode": "time", 1639 "name": null, 1640 "show": true, 1641 "values": [] 1642 }, 1643 "yaxes": [ 1644 { 1645 "format": "none", 1646 "label": null, 1647 "logBase": 1, 1648 "max": null, 1649 "min": "0", 1650 "show": true 1651 }, 1652 { 1653 "format": "short", 1654 "label": null, 1655 "logBase": 1, 1656 "max": null, 1657 "min": null, 1658 "show": true 1659 } 1660 ], 1661 "yaxis": { 1662 "align": false, 1663 "alignLevel": null 1664 } 1665 }, 1666 { 1667 "aliasColors": {}, 1668 "bars": false, 1669 "dashLength": 10, 1670 "dashes": false, 1671 "datasource": "${DS_TEST-CLUSTER}", 1672 "decimals": 1, 1673 "description": "99.99% of the number of regions that TiKV checks. ", 1674 "fieldConfig": { 1675 "defaults": {}, 1676 "overrides": [] 1677 }, 1678 "fill": 1, 1679 "fillGradient": 0, 1680 "gridPos": { 1681 "h": 4, 1682 "w": 12, 1683 "x": 0, 1684 "y": 29 1685 }, 1686 "hiddenSeries": false, 1687 "id": 433, 1688 "legend": { 1689 "alignAsTable": true, 1690 "avg": false, 1691 "current": true, 1692 "hideEmpty": false, 1693 "hideZero": true, 1694 "max": true, 1695 "min": false, 1696 "rightSide": false, 1697 "show": false, 1698 "sideWidth": null, 1699 "sort": "current", 1700 "sortDesc": true, 1701 "total": false, 1702 "values": true 1703 }, 1704 "lines": true, 1705 "linewidth": 1, 1706 "links": [], 1707 "nullPointMode": "null", 1708 "options": { 1709 "alertThreshold": true 1710 }, 1711 "paceLength": 10, 1712 "percentage": false, 1713 "pluginVersion": "7.5.11", 1714 "pointradius": 5, 1715 "points": false, 1716 "renderer": "flot", 1717 "seriesOverrides": [], 1718 "spaceLength": 10, 1719 "stack": false, 1720 "steppedLine": false, 1721 "targets": [ 1722 { 1723 "expr": "histogram_quantile(0.9999, sum(rate(tikv_check_leader_request_item_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance))", 1724 "format": "time_series", 1725 "intervalFactor": 1, 1726 "legendFormat": "{{instance}}", 1727 "refId": "A" 1728 } 1729 ], 1730 "thresholds": [], 1731 "timeFrom": null, 1732 "timeRegions": [], 1733 "timeShift": null, 1734 "title": "TiKV check leader region count percentile", 1735 "tooltip": { 1736 "shared": true, 1737 "sort": 0, 1738 "value_type": "individual" 1739 }, 1740 "type": "graph", 1741 "xaxis": { 1742 "buckets": null, 1743 "mode": "time", 1744 "name": null, 1745 "show": true, 1746 "values": [] 1747 }, 1748 "yaxes": [ 1749 { 1750 "format": "none", 1751 "label": null, 1752 "logBase": 1, 1753 "max": null, 1754 "min": null, 1755 "show": true 1756 }, 1757 { 1758 "format": "short", 1759 "label": null, 1760 "logBase": 1, 1761 "max": null, 1762 "min": null, 1763 "show": false 1764 } 1765 ], 1766 "yaxis": { 1767 "align": false, 1768 "alignLevel": null 1769 } 1770 }, 1771 { 1772 "aliasColors": {}, 1773 "bars": false, 1774 "dashLength": 10, 1775 "dashes": false, 1776 "datasource": "${DS_TEST-CLUSTER}", 1777 "description": "The lag between the commit ts of the events and the time when they are flushed to downstream. ", 1778 "fieldConfig": { 1779 "defaults": {}, 1780 "overrides": [] 1781 }, 1782 "fill": 1, 1783 "fillGradient": 0, 1784 "gridPos": { 1785 "h": 4, 1786 "w": 12, 1787 "x": 0, 1788 "y": 37 1789 }, 1790 "hiddenSeries": false, 1791 "id": 727, 1792 "legend": { 1793 "alignAsTable": true, 1794 "avg": false, 1795 "current": true, 1796 "max": true, 1797 "min": false, 1798 "rightSide": true, 1799 "show": true, 1800 "total": false, 1801 "values": true 1802 }, 1803 "lines": true, 1804 "linewidth": 1, 1805 "nullPointMode": "null", 1806 "options": { 1807 "alertThreshold": true 1808 }, 1809 "percentage": false, 1810 "pluginVersion": "7.5.11", 1811 "pointradius": 2, 1812 "points": false, 1813 "renderer": "flot", 1814 "seriesOverrides": [], 1815 "spaceLength": 10, 1816 "stack": false, 1817 "steppedLine": false, 1818 "targets": [ 1819 { 1820 "exemplar": true, 1821 "expr": "histogram_quantile(1.00, sum(rate(ticdc_sink_flush_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", 1822 "interval": "", 1823 "legendFormat": "{{instance}}-{{type}}-max", 1824 "queryType": "randomWalk", 1825 "refId": "A" 1826 }, 1827 { 1828 "exemplar": true, 1829 "expr": "histogram_quantile(0.95, sum(rate(ticdc_sink_flush_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", 1830 "hide": false, 1831 "interval": "", 1832 "legendFormat": "{{instance}}-{{type}}-95", 1833 "refId": "B" 1834 }, 1835 { 1836 "exemplar": true, 1837 "expr": "histogram_quantile(0.8, sum(rate(ticdc_sink_flush_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", 1838 "hide": false, 1839 "interval": "", 1840 "legendFormat": "{{instance}}-{{type}}-80", 1841 "refId": "C" 1842 } 1843 ], 1844 "thresholds": [], 1845 "timeFrom": null, 1846 "timeRegions": [], 1847 "timeShift": null, 1848 "title": "Events Flush Lag", 1849 "tooltip": { 1850 "shared": true, 1851 "sort": 0, 1852 "value_type": "individual" 1853 }, 1854 "type": "graph", 1855 "xaxis": { 1856 "buckets": null, 1857 "mode": "time", 1858 "name": null, 1859 "show": true, 1860 "values": [] 1861 }, 1862 "yaxes": [ 1863 { 1864 "$$hashKey": "object:869", 1865 "format": "s", 1866 "label": null, 1867 "logBase": 1, 1868 "max": null, 1869 "min": "0", 1870 "show": true 1871 }, 1872 { 1873 "$$hashKey": "object:870", 1874 "format": "short", 1875 "label": null, 1876 "logBase": 1, 1877 "max": null, 1878 "min": null, 1879 "show": true 1880 } 1881 ], 1882 "yaxis": { 1883 "align": false, 1884 "alignLevel": null 1885 } 1886 }, 1887 { 1888 "aliasColors": {}, 1889 "bars": false, 1890 "dashLength": 10, 1891 "dashes": false, 1892 "datasource": "${DS_TEST-CLUSTER}", 1893 "decimals": 1, 1894 "description": "The number of failed count of advancing resolved ts.", 1895 "fieldConfig": { 1896 "defaults": {}, 1897 "overrides": [] 1898 }, 1899 "fill": 1, 1900 "fillGradient": 0, 1901 "gridPos": { 1902 "h": 4, 1903 "w": 12, 1904 "x": 12, 1905 "y": 29 1906 }, 1907 "hiddenSeries": false, 1908 "id": 427, 1909 "legend": { 1910 "alignAsTable": true, 1911 "avg": false, 1912 "current": true, 1913 "hideEmpty": false, 1914 "hideZero": true, 1915 "max": true, 1916 "min": false, 1917 "rightSide": false, 1918 "show": false, 1919 "sideWidth": null, 1920 "sort": "current", 1921 "sortDesc": true, 1922 "total": false, 1923 "values": true 1924 }, 1925 "lines": true, 1926 "linewidth": 1, 1927 "links": [], 1928 "nullPointMode": "null", 1929 "options": { 1930 "alertThreshold": true 1931 }, 1932 "paceLength": 10, 1933 "percentage": false, 1934 "pluginVersion": "7.5.11", 1935 "pointradius": 5, 1936 "points": false, 1937 "renderer": "flot", 1938 "seriesOverrides": [], 1939 "spaceLength": 10, 1940 "stack": false, 1941 "steppedLine": false, 1942 "targets": [ 1943 { 1944 "expr": "sum(increase(tikv_resolved_ts_fail_advance_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (reason, instance)", 1945 "format": "time_series", 1946 "hide": false, 1947 "intervalFactor": 1, 1948 "legendFormat": "{{reason}}-{{instance}}", 1949 "refId": "A" 1950 } 1951 ], 1952 "thresholds": [], 1953 "timeFrom": null, 1954 "timeRegions": [], 1955 "timeShift": null, 1956 "title": "TiKV advance resolved ts fail/m", 1957 "tooltip": { 1958 "shared": true, 1959 "sort": 0, 1960 "value_type": "individual" 1961 }, 1962 "type": "graph", 1963 "xaxis": { 1964 "buckets": null, 1965 "mode": "time", 1966 "name": null, 1967 "show": true, 1968 "values": [] 1969 }, 1970 "yaxes": [ 1971 { 1972 "format": "none", 1973 "label": null, 1974 "logBase": 1, 1975 "max": null, 1976 "min": null, 1977 "show": true 1978 }, 1979 { 1980 "format": "none", 1981 "label": null, 1982 "logBase": 1, 1983 "max": null, 1984 "min": null, 1985 "show": true 1986 } 1987 ], 1988 "yaxis": { 1989 "align": false, 1990 "alignLevel": null 1991 } 1992 }, 1993 { 1994 "aliasColors": {}, 1995 "bars": false, 1996 "dashLength": 10, 1997 "dashes": false, 1998 "datasource": "${DS_TEST-CLUSTER}", 1999 "decimals": 1, 2000 "description": "The number of tables rebalance on each TiCDC instance", 2001 "editable": true, 2002 "error": false, 2003 "fieldConfig": { 2004 "defaults": {}, 2005 "overrides": [] 2006 }, 2007 "fill": 0, 2008 "fillGradient": 0, 2009 "grid": {}, 2010 "gridPos": { 2011 "h": 4, 2012 "w": 12, 2013 "x": 0, 2014 "y": 33 2015 }, 2016 "hiddenSeries": false, 2017 "id": 399, 2018 "legend": { 2019 "alignAsTable": true, 2020 "avg": false, 2021 "current": true, 2022 "max": true, 2023 "min": false, 2024 "rightSide": true, 2025 "show": false, 2026 "sideWidth": null, 2027 "sort": "current", 2028 "sortDesc": true, 2029 "total": false, 2030 "values": true 2031 }, 2032 "lines": true, 2033 "linewidth": 1, 2034 "links": [], 2035 "nullPointMode": "null", 2036 "options": { 2037 "alertThreshold": true 2038 }, 2039 "percentage": false, 2040 "pluginVersion": "7.5.11", 2041 "pointradius": 5, 2042 "points": false, 2043 "renderer": "flot", 2044 "seriesOverrides": [], 2045 "spaceLength": 10, 2046 "stack": false, 2047 "steppedLine": false, 2048 "targets": [ 2049 { 2050 "expr": "sum(delta(ticdc_processor_num_of_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[30s])) by (instance)", 2051 "format": "time_series", 2052 "hide": false, 2053 "intervalFactor": 2, 2054 "legendFormat": "{{instance}}", 2055 "refId": "A", 2056 "step": 10 2057 } 2058 ], 2059 "thresholds": [], 2060 "timeFrom": null, 2061 "timeRegions": [], 2062 "timeShift": null, 2063 "title": "Table rebalance", 2064 "tooltip": { 2065 "msResolution": false, 2066 "shared": true, 2067 "sort": 0, 2068 "value_type": "cumulative" 2069 }, 2070 "type": "graph", 2071 "xaxis": { 2072 "buckets": null, 2073 "mode": "time", 2074 "name": null, 2075 "show": true, 2076 "values": [] 2077 }, 2078 "yaxes": [ 2079 { 2080 "format": "short", 2081 "label": null, 2082 "logBase": 1, 2083 "max": null, 2084 "min": null, 2085 "show": true 2086 }, 2087 { 2088 "format": "short", 2089 "label": null, 2090 "logBase": 1, 2091 "max": null, 2092 "min": null, 2093 "show": true 2094 } 2095 ], 2096 "yaxis": { 2097 "align": false, 2098 "alignLevel": null 2099 } 2100 }, 2101 { 2102 "aliasColors": {}, 2103 "bars": false, 2104 "dashLength": 10, 2105 "dashes": false, 2106 "datasource": "${DS_TEST-CLUSTER}", 2107 "decimals": 1, 2108 "description": "99.99% of the duration that TiKV check leader takes. ", 2109 "fieldConfig": { 2110 "defaults": {}, 2111 "overrides": [] 2112 }, 2113 "fill": 1, 2114 "fillGradient": 0, 2115 "gridPos": { 2116 "h": 4, 2117 "w": 12, 2118 "x": 12, 2119 "y": 33 2120 }, 2121 "hiddenSeries": false, 2122 "id": 472, 2123 "legend": { 2124 "alignAsTable": true, 2125 "avg": false, 2126 "current": true, 2127 "hideEmpty": false, 2128 "hideZero": true, 2129 "max": true, 2130 "min": false, 2131 "rightSide": false, 2132 "show": false, 2133 "sideWidth": null, 2134 "sort": "current", 2135 "sortDesc": true, 2136 "total": false, 2137 "values": true 2138 }, 2139 "lines": true, 2140 "linewidth": 1, 2141 "links": [], 2142 "nullPointMode": "null", 2143 "options": { 2144 "alertThreshold": true 2145 }, 2146 "paceLength": 10, 2147 "percentage": false, 2148 "pluginVersion": "7.5.11", 2149 "pointradius": 5, 2150 "points": false, 2151 "renderer": "flot", 2152 "seriesOverrides": [], 2153 "spaceLength": 10, 2154 "stack": false, 2155 "steppedLine": false, 2156 "targets": [ 2157 { 2158 "expr": "histogram_quantile(0.9999, sum(rate(tikv_resolved_ts_tikv_client_init_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le))", 2159 "format": "time_series", 2160 "intervalFactor": 1, 2161 "legendFormat": "new-client", 2162 "refId": "A" 2163 }, 2164 { 2165 "expr": "histogram_quantile(0.9999, sum(rate(tikv_resolved_ts_check_leader_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, type))", 2166 "format": "time_series", 2167 "intervalFactor": 1, 2168 "legendFormat": "{{type}}", 2169 "refId": "B" 2170 } 2171 ], 2172 "thresholds": [], 2173 "timeFrom": null, 2174 "timeRegions": [], 2175 "timeShift": null, 2176 "title": "TiKV check leader duration percentile", 2177 "tooltip": { 2178 "shared": true, 2179 "sort": 0, 2180 "value_type": "individual" 2181 }, 2182 "type": "graph", 2183 "xaxis": { 2184 "buckets": null, 2185 "mode": "time", 2186 "name": null, 2187 "show": true, 2188 "values": [] 2189 }, 2190 "yaxes": [ 2191 { 2192 "format": "s", 2193 "label": null, 2194 "logBase": 1, 2195 "max": null, 2196 "min": null, 2197 "show": true 2198 }, 2199 { 2200 "format": "short", 2201 "label": null, 2202 "logBase": 1, 2203 "max": null, 2204 "min": null, 2205 "show": false 2206 } 2207 ], 2208 "yaxis": { 2209 "align": false, 2210 "alignLevel": null 2211 } 2212 }, 2213 { 2214 "aliasColors": {}, 2215 "dashLength": 10, 2216 "datasource": "${DS_TEST-CLUSTER}", 2217 "decimals": 1, 2218 "description": "The number of regions that take a long time(more than 60s) to scan", 2219 "editable": true, 2220 "fieldConfig": { 2221 "defaults": {}, 2222 "overrides": [] 2223 }, 2224 "grid": {}, 2225 "gridPos": { 2226 "h": 4, 2227 "w": 12, 2228 "x": 12, 2229 "y": 37 2230 }, 2231 "id": 10038, 2232 "legend": { 2233 "alignAsTable": true, 2234 "avg": false, 2235 "current": true, 2236 "max": true, 2237 "min": false, 2238 "rightSide": true, 2239 "show": false, 2240 "sideWidth": null, 2241 "sort": null, 2242 "sortDesc": null, 2243 "total": false, 2244 "values": true 2245 }, 2246 "lines": true, 2247 "linewidth": 1, 2248 "links": [], 2249 "nullPointMode": "null", 2250 "options": { 2251 "alertThreshold": true 2252 }, 2253 "paceLength": 10, 2254 "pluginVersion": "7.5.17", 2255 "pointradius": 5, 2256 "renderer": "flot", 2257 "seriesOverrides": [], 2258 "spaceLength": 10, 2259 "targets": [ 2260 { 2261 "expr": "tikv_cdc_scan_long_duration_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}", 2262 "legendFormat": "{{instance}}", 2263 "interval": "", 2264 "exemplar": true, 2265 "format": "time_series", 2266 "hide": false, 2267 "intervalFactor": 2, 2268 "refId": "B", 2269 "step": 10 2270 } 2271 ], 2272 "thresholds": [], 2273 "timeRegions": [], 2274 "title": "TiKV CDC Incremental Scan Long Duration Region Count", 2275 "tooltip": { 2276 "msResolution": false, 2277 "shared": true, 2278 "sort": 0, 2279 "value_type": "individual" 2280 }, 2281 "type": "graph", 2282 "xaxis": { 2283 "buckets": null, 2284 "mode": "time", 2285 "name": null, 2286 "show": true, 2287 "values": [] 2288 }, 2289 "yaxes": [ 2290 { 2291 "format": "none", 2292 "label": null, 2293 "logBase": 1, 2294 "max": null, 2295 "min": "0", 2296 "show": true, 2297 "$$hashKey": "object:115" 2298 }, 2299 { 2300 "format": "short", 2301 "label": null, 2302 "logBase": 1, 2303 "max": null, 2304 "min": null, 2305 "show": true, 2306 "$$hashKey": "object:116" 2307 } 2308 ], 2309 "yaxis": { 2310 "align": false, 2311 "alignLevel": null 2312 }, 2313 "bars": false, 2314 "dashes": false, 2315 "error": false, 2316 "fill": 0, 2317 "fillGradient": 0, 2318 "hiddenSeries": false, 2319 "percentage": false, 2320 "points": false, 2321 "stack": false, 2322 "steppedLine": false, 2323 "timeFrom": null, 2324 "timeShift": null 2325 } 2326 ], 2327 "title": "Lag analyze", 2328 "type": "row" 2329 }, 2330 { 2331 "collapsed": true, 2332 "datasource": null, 2333 "gridPos": { 2334 "h": 1, 2335 "w": 24, 2336 "x": 0, 2337 "y": 1 2338 }, 2339 "id": 21, 2340 "panels": [ 2341 { 2342 "aliasColors": {}, 2343 "bars": false, 2344 "dashLength": 10, 2345 "dashes": false, 2346 "datasource": "${DS_TEST-CLUSTER}", 2347 "description": "Uptime of TiCDC and TiKV", 2348 "fieldConfig": { 2349 "defaults": { 2350 "links": [] 2351 }, 2352 "overrides": [] 2353 }, 2354 "fill": 1, 2355 "fillGradient": 0, 2356 "gridPos": { 2357 "h": 7, 2358 "w": 8, 2359 "x": 0, 2360 "y": 2 2361 }, 2362 "hiddenSeries": false, 2363 "id": 19, 2364 "legend": { 2365 "alignAsTable": true, 2366 "avg": false, 2367 "current": true, 2368 "max": false, 2369 "min": false, 2370 "rightSide": true, 2371 "show": true, 2372 "total": false, 2373 "values": true 2374 }, 2375 "lines": true, 2376 "linewidth": 1, 2377 "links": [], 2378 "nullPointMode": "null", 2379 "options": { 2380 "alertThreshold": true 2381 }, 2382 "paceLength": 10, 2383 "percentage": false, 2384 "pluginVersion": "7.5.11", 2385 "pointradius": 2, 2386 "points": false, 2387 "renderer": "flot", 2388 "seriesOverrides": [], 2389 "spaceLength": 10, 2390 "stack": false, 2391 "steppedLine": false, 2392 "targets": [ 2393 { 2394 "expr": "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"})", 2395 "format": "time_series", 2396 "intervalFactor": 1, 2397 "legendFormat": "TiCDC-{{instance}}", 2398 "refId": "A" 2399 }, 2400 { 2401 "expr": "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tikv\"})", 2402 "format": "time_series", 2403 "intervalFactor": 1, 2404 "legendFormat": "TiKV-{{instance}}", 2405 "refId": "B" 2406 }, 2407 { 2408 "expr": "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"pd\"})", 2409 "format": "time_series", 2410 "intervalFactor": 1, 2411 "legendFormat": "PD-{{instance}}", 2412 "refId": "C" 2413 } 2414 ], 2415 "thresholds": [], 2416 "timeFrom": null, 2417 "timeRegions": [], 2418 "timeShift": null, 2419 "title": "Uptime", 2420 "tooltip": { 2421 "shared": true, 2422 "sort": 0, 2423 "value_type": "individual" 2424 }, 2425 "type": "graph", 2426 "xaxis": { 2427 "buckets": null, 2428 "mode": "time", 2429 "name": null, 2430 "show": true, 2431 "values": [] 2432 }, 2433 "yaxes": [ 2434 { 2435 "format": "dtdurations", 2436 "label": null, 2437 "logBase": 1, 2438 "max": null, 2439 "min": null, 2440 "show": true 2441 }, 2442 { 2443 "format": "short", 2444 "label": null, 2445 "logBase": 1, 2446 "max": null, 2447 "min": null, 2448 "show": true 2449 } 2450 ], 2451 "yaxis": { 2452 "align": false, 2453 "alignLevel": null 2454 } 2455 }, 2456 { 2457 "aliasColors": {}, 2458 "bars": false, 2459 "dashLength": 10, 2460 "dashes": false, 2461 "datasource": "${DS_TEST-CLUSTER}", 2462 "description": "Goroutine count of TiCDC", 2463 "fieldConfig": { 2464 "defaults": { 2465 "links": [] 2466 }, 2467 "overrides": [] 2468 }, 2469 "fill": 1, 2470 "fillGradient": 0, 2471 "gridPos": { 2472 "h": 7, 2473 "w": 6, 2474 "x": 8, 2475 "y": 2 2476 }, 2477 "hiddenSeries": false, 2478 "id": 26, 2479 "legend": { 2480 "alignAsTable": true, 2481 "avg": false, 2482 "current": true, 2483 "max": true, 2484 "min": false, 2485 "rightSide": false, 2486 "show": true, 2487 "total": false, 2488 "values": true 2489 }, 2490 "lines": true, 2491 "linewidth": 1, 2492 "links": [], 2493 "nullPointMode": "null", 2494 "options": { 2495 "alertThreshold": true 2496 }, 2497 "paceLength": 10, 2498 "percentage": false, 2499 "pluginVersion": "7.5.11", 2500 "pointradius": 2, 2501 "points": false, 2502 "renderer": "flot", 2503 "seriesOverrides": [], 2504 "spaceLength": 10, 2505 "stack": false, 2506 "steppedLine": false, 2507 "targets": [ 2508 { 2509 "expr": " go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}", 2510 "format": "time_series", 2511 "intervalFactor": 1, 2512 "legendFormat": "{{instance}}", 2513 "refId": "A" 2514 }, 2515 { 2516 "expr": "go_threads{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}", 2517 "format": "time_series", 2518 "hide": true, 2519 "intervalFactor": 1, 2520 "legendFormat": "threads-{{instance}}", 2521 "refId": "B" 2522 } 2523 ], 2524 "thresholds": [], 2525 "timeFrom": null, 2526 "timeRegions": [], 2527 "timeShift": null, 2528 "title": "Goroutine count", 2529 "tooltip": { 2530 "shared": true, 2531 "sort": 0, 2532 "value_type": "individual" 2533 }, 2534 "type": "graph", 2535 "xaxis": { 2536 "buckets": null, 2537 "mode": "time", 2538 "name": null, 2539 "show": true, 2540 "values": [] 2541 }, 2542 "yaxes": [ 2543 { 2544 "format": "short", 2545 "label": null, 2546 "logBase": 1, 2547 "max": null, 2548 "min": null, 2549 "show": true 2550 }, 2551 { 2552 "format": "short", 2553 "label": null, 2554 "logBase": 1, 2555 "max": null, 2556 "min": null, 2557 "show": true 2558 } 2559 ], 2560 "yaxis": { 2561 "align": false, 2562 "alignLevel": null 2563 } 2564 }, 2565 { 2566 "aliasColors": {}, 2567 "bars": false, 2568 "dashLength": 10, 2569 "dashes": false, 2570 "datasource": "${DS_TEST-CLUSTER}", 2571 "description": "The count of open FD count of TiCDC", 2572 "fieldConfig": { 2573 "defaults": { 2574 "links": [] 2575 }, 2576 "overrides": [] 2577 }, 2578 "fill": 1, 2579 "fillGradient": 0, 2580 "gridPos": { 2581 "h": 7, 2582 "w": 6, 2583 "x": 14, 2584 "y": 2 2585 }, 2586 "hiddenSeries": false, 2587 "id": 27, 2588 "legend": { 2589 "alignAsTable": true, 2590 "avg": false, 2591 "current": true, 2592 "max": true, 2593 "min": false, 2594 "rightSide": false, 2595 "show": true, 2596 "total": false, 2597 "values": true 2598 }, 2599 "lines": true, 2600 "linewidth": 1, 2601 "links": [], 2602 "nullPointMode": "null", 2603 "options": { 2604 "alertThreshold": true 2605 }, 2606 "paceLength": 10, 2607 "percentage": false, 2608 "pluginVersion": "7.5.11", 2609 "pointradius": 2, 2610 "points": false, 2611 "renderer": "flot", 2612 "seriesOverrides": [], 2613 "spaceLength": 10, 2614 "stack": false, 2615 "steppedLine": false, 2616 "targets": [ 2617 { 2618 "expr": "process_open_fds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}", 2619 "format": "time_series", 2620 "intervalFactor": 1, 2621 "legendFormat": "{{instance}}", 2622 "refId": "A" 2623 } 2624 ], 2625 "thresholds": [], 2626 "timeFrom": null, 2627 "timeRegions": [], 2628 "timeShift": null, 2629 "title": "Open FD count", 2630 "tooltip": { 2631 "shared": true, 2632 "sort": 0, 2633 "value_type": "individual" 2634 }, 2635 "type": "graph", 2636 "xaxis": { 2637 "buckets": null, 2638 "mode": "time", 2639 "name": null, 2640 "show": true, 2641 "values": [] 2642 }, 2643 "yaxes": [ 2644 { 2645 "format": "short", 2646 "label": null, 2647 "logBase": 1, 2648 "max": null, 2649 "min": null, 2650 "show": true 2651 }, 2652 { 2653 "format": "short", 2654 "label": null, 2655 "logBase": 1, 2656 "max": null, 2657 "min": null, 2658 "show": true 2659 } 2660 ], 2661 "yaxis": { 2662 "align": false, 2663 "alignLevel": null 2664 } 2665 }, 2666 { 2667 "datasource": "${DS_TEST-CLUSTER}", 2668 "description": "TiCDC cluster ownership status", 2669 "fieldConfig": { 2670 "defaults": { 2671 "color": { 2672 "mode": "thresholds" 2673 }, 2674 "custom": { 2675 "align": "center", 2676 "filterable": false 2677 }, 2678 "mappings": [], 2679 "thresholds": { 2680 "mode": "absolute", 2681 "steps": [ 2682 { 2683 "color": "green", 2684 "value": null 2685 }, 2686 { 2687 "color": "red", 2688 "value": 80 2689 } 2690 ] 2691 } 2692 }, 2693 "overrides": [ 2694 { 2695 "matcher": { 2696 "id": "byName", 2697 "options": "Field" 2698 }, 2699 "properties": [ 2700 { 2701 "id": "displayName", 2702 "value": "Instance" 2703 }, 2704 { 2705 "id": "unit", 2706 "value": "string" 2707 }, 2708 { 2709 "id": "decimals", 2710 "value": 2 2711 }, 2712 { 2713 "id": "custom.align", 2714 "value": null 2715 }, 2716 { 2717 "id": "thresholds", 2718 "value": { 2719 "mode": "absolute", 2720 "steps": [ 2721 { 2722 "color": "#8AB8FF", 2723 "value": null 2724 }, 2725 { 2726 "color": "#73BF69", 2727 "value": 0 2728 }, 2729 { 2730 "color": "rgba(245, 54, 54, 0.9)", 2731 "value": 2 2732 } 2733 ] 2734 } 2735 } 2736 ] 2737 }, 2738 { 2739 "matcher": { 2740 "id": "byName", 2741 "options": "Last" 2742 }, 2743 "properties": [ 2744 { 2745 "id": "displayName", 2746 "value": "Role" 2747 }, 2748 { 2749 "id": "decimals", 2750 "value": 0 2751 }, 2752 { 2753 "id": "mappings", 2754 "value": [ 2755 { 2756 "from": "0.1", 2757 "id": 1, 2758 "text": "Owner", 2759 "to": "2", 2760 "type": 2, 2761 "value": "1" 2762 }, 2763 { 2764 "from": "0", 2765 "id": 2, 2766 "text": "Processor", 2767 "to": "0.1", 2768 "type": 2 2769 } 2770 ] 2771 } 2772 ] 2773 } 2774 ] 2775 }, 2776 "gridPos": { 2777 "h": 7, 2778 "w": 4, 2779 "x": 20, 2780 "y": 2 2781 }, 2782 "hideTimeOverride": true, 2783 "id": 113, 2784 "links": [], 2785 "options": { 2786 "showHeader": true 2787 }, 2788 "pluginVersion": "7.5.11", 2789 "targets": [ 2790 { 2791 "expr": "rate(ticdc_owner_ownership_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[240s])", 2792 "format": "time_series", 2793 "intervalFactor": 2, 2794 "legendFormat": "{{instance}}", 2795 "refId": "A" 2796 } 2797 ], 2798 "timeFrom": "1s", 2799 "title": "Ownership", 2800 "transformations": [ 2801 { 2802 "id": "reduce", 2803 "options": { 2804 "includeTimeField": false, 2805 "reducers": [ 2806 "last" 2807 ] 2808 } 2809 } 2810 ], 2811 "type": "table" 2812 }, 2813 { 2814 "aliasColors": {}, 2815 "bars": false, 2816 "dashLength": 10, 2817 "dashes": false, 2818 "datasource": "${DS_TEST-CLUSTER}", 2819 "description": "CPU usage of TiCDC", 2820 "fieldConfig": { 2821 "defaults": { 2822 "links": [] 2823 }, 2824 "overrides": [] 2825 }, 2826 "fill": 1, 2827 "fillGradient": 0, 2828 "gridPos": { 2829 "h": 7, 2830 "w": 6, 2831 "x": 0, 2832 "y": 9 2833 }, 2834 "hiddenSeries": false, 2835 "id": 24, 2836 "legend": { 2837 "alignAsTable": true, 2838 "avg": false, 2839 "current": true, 2840 "hideEmpty": true, 2841 "max": true, 2842 "min": false, 2843 "rightSide": false, 2844 "show": true, 2845 "total": false, 2846 "values": true 2847 }, 2848 "lines": true, 2849 "linewidth": 1, 2850 "links": [], 2851 "nullPointMode": "null", 2852 "options": { 2853 "alertThreshold": true 2854 }, 2855 "paceLength": 10, 2856 "percentage": false, 2857 "pluginVersion": "7.5.11", 2858 "pointradius": 2, 2859 "points": false, 2860 "renderer": "flot", 2861 "seriesOverrides": [ 2862 { 2863 "alias": "/.*MaxProcs/", 2864 "fill": 0, 2865 "linewidth": 2, 2866 "yaxis": 2 2867 } 2868 ], 2869 "spaceLength": 10, 2870 "stack": false, 2871 "steppedLine": false, 2872 "targets": [ 2873 { 2874 "expr": "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}[1m])", 2875 "format": "time_series", 2876 "intervalFactor": 1, 2877 "legendFormat": "{{instance}}", 2878 "refId": "A" 2879 }, 2880 { 2881 "expr": "ticdc_server_go_max_procs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}", 2882 "format": "time_series", 2883 "intervalFactor": 1, 2884 "legendFormat": "quota-{{instance}}", 2885 "refId": "B" 2886 } 2887 ], 2888 "thresholds": [], 2889 "timeFrom": null, 2890 "timeRegions": [], 2891 "timeShift": null, 2892 "title": "CPU usage", 2893 "tooltip": { 2894 "shared": true, 2895 "sort": 0, 2896 "value_type": "individual" 2897 }, 2898 "type": "graph", 2899 "xaxis": { 2900 "buckets": null, 2901 "mode": "time", 2902 "name": null, 2903 "show": true, 2904 "values": [] 2905 }, 2906 "yaxes": [ 2907 { 2908 "format": "percentunit", 2909 "label": null, 2910 "logBase": 1, 2911 "max": null, 2912 "min": null, 2913 "show": true 2914 }, 2915 { 2916 "format": "percentunit", 2917 "label": null, 2918 "logBase": 1, 2919 "max": null, 2920 "min": null, 2921 "show": true 2922 } 2923 ], 2924 "yaxis": { 2925 "align": false, 2926 "alignLevel": null 2927 } 2928 }, 2929 { 2930 "aliasColors": {}, 2931 "bars": false, 2932 "dashLength": 10, 2933 "dashes": false, 2934 "datasource": "${DS_TEST-CLUSTER}", 2935 "description": "Memory usage of TiCDC", 2936 "fieldConfig": { 2937 "defaults": { 2938 "links": [] 2939 }, 2940 "overrides": [] 2941 }, 2942 "fill": 1, 2943 "fillGradient": 0, 2944 "gridPos": { 2945 "h": 7, 2946 "w": 6, 2947 "x": 6, 2948 "y": 9 2949 }, 2950 "hiddenSeries": false, 2951 "id": 23, 2952 "legend": { 2953 "alignAsTable": true, 2954 "avg": false, 2955 "current": true, 2956 "hideEmpty": true, 2957 "max": true, 2958 "min": false, 2959 "rightSide": false, 2960 "show": true, 2961 "total": false, 2962 "values": true 2963 }, 2964 "lines": true, 2965 "linewidth": 1, 2966 "links": [], 2967 "nullPointMode": "null", 2968 "options": { 2969 "alertThreshold": true 2970 }, 2971 "paceLength": 10, 2972 "percentage": false, 2973 "pluginVersion": "7.5.11", 2974 "pointradius": 2, 2975 "points": false, 2976 "renderer": "flot", 2977 "seriesOverrides": [], 2978 "spaceLength": 10, 2979 "stack": false, 2980 "steppedLine": false, 2981 "targets": [ 2982 { 2983 "expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}", 2984 "format": "time_series", 2985 "intervalFactor": 1, 2986 "legendFormat": "process-{{instance}}", 2987 "refId": "A" 2988 }, 2989 { 2990 "expr": "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}", 2991 "format": "time_series", 2992 "intervalFactor": 1, 2993 "legendFormat": "heap-{{instance}}", 2994 "refId": "B" 2995 } 2996 ], 2997 "thresholds": [], 2998 "timeFrom": null, 2999 "timeRegions": [], 3000 "timeShift": null, 3001 "title": "Memory usage", 3002 "tooltip": { 3003 "shared": true, 3004 "sort": 0, 3005 "value_type": "individual" 3006 }, 3007 "type": "graph", 3008 "xaxis": { 3009 "buckets": null, 3010 "mode": "time", 3011 "name": null, 3012 "show": true, 3013 "values": [] 3014 }, 3015 "yaxes": [ 3016 { 3017 "format": "bytes", 3018 "label": null, 3019 "logBase": 1, 3020 "max": null, 3021 "min": null, 3022 "show": true 3023 }, 3024 { 3025 "format": "short", 3026 "label": null, 3027 "logBase": 1, 3028 "max": null, 3029 "min": null, 3030 "show": true 3031 } 3032 ], 3033 "yaxis": { 3034 "align": false, 3035 "alignLevel": null 3036 } 3037 }, 3038 { 3039 "aliasColors": {}, 3040 "bars": false, 3041 "dashLength": 10, 3042 "dashes": false, 3043 "datasource": "${DS_TEST-CLUSTER}", 3044 "description": "The history of TiCDC cluster ownership, owner node has a value that is great than 0", 3045 "fieldConfig": { 3046 "defaults": {}, 3047 "overrides": [] 3048 }, 3049 "fill": 1, 3050 "fillGradient": 0, 3051 "gridPos": { 3052 "h": 7, 3053 "w": 6, 3054 "x": 12, 3055 "y": 9 3056 }, 3057 "hiddenSeries": false, 3058 "id": 110, 3059 "legend": { 3060 "alignAsTable": true, 3061 "avg": false, 3062 "current": true, 3063 "max": false, 3064 "min": false, 3065 "rightSide": false, 3066 "show": true, 3067 "total": false, 3068 "values": true 3069 }, 3070 "lines": true, 3071 "linewidth": 1, 3072 "links": [], 3073 "nullPointMode": "null", 3074 "options": { 3075 "alertThreshold": true 3076 }, 3077 "percentage": false, 3078 "pluginVersion": "7.5.11", 3079 "pointradius": 2, 3080 "points": false, 3081 "renderer": "flot", 3082 "seriesOverrides": [], 3083 "spaceLength": 10, 3084 "stack": false, 3085 "steppedLine": true, 3086 "targets": [ 3087 { 3088 "expr": "sum(rate(ticdc_owner_ownership_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[240s])) by (instance) > BOOL 0.0625", 3089 "format": "time_series", 3090 "interval": "30s", 3091 "intervalFactor": 1, 3092 "legendFormat": "{{instance}}", 3093 "refId": "A" 3094 } 3095 ], 3096 "thresholds": [], 3097 "timeFrom": null, 3098 "timeRegions": [], 3099 "timeShift": null, 3100 "title": "Ownership history", 3101 "tooltip": { 3102 "shared": true, 3103 "sort": 0, 3104 "value_type": "individual" 3105 }, 3106 "type": "graph", 3107 "xaxis": { 3108 "buckets": null, 3109 "mode": "time", 3110 "name": null, 3111 "show": true, 3112 "values": [] 3113 }, 3114 "yaxes": [ 3115 { 3116 "format": "short", 3117 "label": null, 3118 "logBase": 1, 3119 "max": null, 3120 "min": null, 3121 "show": true 3122 }, 3123 { 3124 "format": "short", 3125 "label": null, 3126 "logBase": 1, 3127 "max": null, 3128 "min": null, 3129 "show": true 3130 } 3131 ], 3132 "yaxis": { 3133 "align": false, 3134 "alignLevel": null 3135 } 3136 }, 3137 { 3138 "aliasColors": {}, 3139 "bars": false, 3140 "dashLength": 10, 3141 "dashes": false, 3142 "datasource": "${DS_TEST-CLUSTER}", 3143 "description": "The history of PD cluster leadership, leader node has a value that is great than 0", 3144 "fieldConfig": { 3145 "defaults": {}, 3146 "overrides": [] 3147 }, 3148 "fill": 1, 3149 "fillGradient": 0, 3150 "gridPos": { 3151 "h": 7, 3152 "w": 6, 3153 "x": 18, 3154 "y": 9 3155 }, 3156 "hiddenSeries": false, 3157 "id": 293, 3158 "legend": { 3159 "alignAsTable": true, 3160 "avg": false, 3161 "current": true, 3162 "max": false, 3163 "min": false, 3164 "rightSide": false, 3165 "show": true, 3166 "total": false, 3167 "values": true 3168 }, 3169 "lines": true, 3170 "linewidth": 1, 3171 "links": [], 3172 "nullPointMode": "null", 3173 "options": { 3174 "alertThreshold": true 3175 }, 3176 "percentage": false, 3177 "pluginVersion": "7.5.11", 3178 "pointradius": 2, 3179 "points": false, 3180 "renderer": "flot", 3181 "seriesOverrides": [], 3182 "spaceLength": 10, 3183 "stack": false, 3184 "steppedLine": true, 3185 "targets": [ 3186 { 3187 "expr": "pd_tso_role{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", dc=\"global\"} > BOOL 0.5", 3188 "format": "time_series", 3189 "hide": false, 3190 "interval": "30s", 3191 "intervalFactor": 1, 3192 "legendFormat": "PD-{{instance}}", 3193 "refId": "A" 3194 } 3195 ], 3196 "thresholds": [], 3197 "timeFrom": null, 3198 "timeRegions": [], 3199 "timeShift": null, 3200 "title": "PD leader history", 3201 "tooltip": { 3202 "shared": true, 3203 "sort": 0, 3204 "value_type": "individual" 3205 }, 3206 "type": "graph", 3207 "xaxis": { 3208 "buckets": null, 3209 "mode": "time", 3210 "name": null, 3211 "show": true, 3212 "values": [] 3213 }, 3214 "yaxes": [ 3215 { 3216 "format": "short", 3217 "label": null, 3218 "logBase": 1, 3219 "max": null, 3220 "min": null, 3221 "show": true 3222 }, 3223 { 3224 "format": "short", 3225 "label": null, 3226 "logBase": 1, 3227 "max": null, 3228 "min": null, 3229 "show": true 3230 } 3231 ], 3232 "yaxis": { 3233 "align": false, 3234 "alignLevel": null 3235 } 3236 } 3237 ], 3238 "title": "Server", 3239 "type": "row" 3240 }, 3241 { 3242 "collapsed": true, 3243 "datasource": null, 3244 "gridPos": { 3245 "h": 1, 3246 "w": 24, 3247 "x": 0, 3248 "y": 2 3249 }, 3250 "id": 11, 3251 "panels": [ 3252 { 3253 "datasource": "${DS_TEST-CLUSTER}", 3254 "description": "The number of captured table of TiCDC nodes ", 3255 "fieldConfig": { 3256 "defaults": { 3257 "color": { 3258 "mode": "thresholds" 3259 }, 3260 "custom": { 3261 "align": "left", 3262 "filterable": false 3263 }, 3264 "decimals": 2, 3265 "mappings": [], 3266 "thresholds": { 3267 "mode": "absolute", 3268 "steps": [ 3269 { 3270 "color": "green", 3271 "value": null 3272 }, 3273 { 3274 "color": "red", 3275 "value": 80 3276 } 3277 ] 3278 }, 3279 "unit": "short" 3280 }, 3281 "overrides": [ 3282 { 3283 "matcher": { 3284 "id": "byName", 3285 "options": "Field" 3286 }, 3287 "properties": [ 3288 { 3289 "id": "displayName", 3290 "value": "Instance" 3291 }, 3292 { 3293 "id": "unit", 3294 "value": "string" 3295 }, 3296 { 3297 "id": "custom.align", 3298 "value": null 3299 } 3300 ] 3301 }, 3302 { 3303 "matcher": { 3304 "id": "byName", 3305 "options": "Last" 3306 }, 3307 "properties": [ 3308 { 3309 "id": "displayName", 3310 "value": "Count" 3311 } 3312 ] 3313 } 3314 ] 3315 }, 3316 "gridPos": { 3317 "h": 7, 3318 "w": 6, 3319 "x": 0, 3320 "y": 3 3321 }, 3322 "id": 4, 3323 "links": [], 3324 "options": { 3325 "showHeader": true 3326 }, 3327 "pluginVersion": "7.5.11", 3328 "targets": [ 3329 { 3330 "exemplar": true, 3331 "expr": "sum(ticdc_processor_num_of_tables{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}) by (instance)", 3332 "format": "time_series", 3333 "interval": "", 3334 "intervalFactor": 1, 3335 "legendFormat": "{{instance}}", 3336 "refId": "A" 3337 } 3338 ], 3339 "timeFrom": null, 3340 "timeShift": null, 3341 "title": "Changefeed table count", 3342 "transformations": [ 3343 { 3344 "id": "reduce", 3345 "options": { 3346 "includeTimeField": false, 3347 "reducers": [ 3348 "last" 3349 ] 3350 } 3351 } 3352 ], 3353 "type": "table" 3354 }, 3355 { 3356 "aliasColors": {}, 3357 "bars": false, 3358 "dashLength": 10, 3359 "dashes": false, 3360 "datasource": "${DS_TEST-CLUSTER}", 3361 "decimals": 1, 3362 "description": "The number of tables in different capture.", 3363 "editable": true, 3364 "error": false, 3365 "fieldConfig": { 3366 "defaults": { 3367 "unit": "none" 3368 }, 3369 "overrides": [] 3370 }, 3371 "fill": 0, 3372 "fillGradient": 0, 3373 "grid": {}, 3374 "gridPos": { 3375 "h": 7, 3376 "w": 6, 3377 "x": 6, 3378 "y": 3 3379 }, 3380 "hiddenSeries": false, 3381 "id": 642, 3382 "legend": { 3383 "alignAsTable": true, 3384 "avg": false, 3385 "current": true, 3386 "max": false, 3387 "min": false, 3388 "rightSide": false, 3389 "show": true, 3390 "sideWidth": null, 3391 "sort": null, 3392 "sortDesc": null, 3393 "total": false, 3394 "values": true 3395 }, 3396 "lines": true, 3397 "linewidth": 1, 3398 "links": [], 3399 "nullPointMode": "null", 3400 "options": { 3401 "alertThreshold": true 3402 }, 3403 "paceLength": 10, 3404 "percentage": false, 3405 "pluginVersion": "7.5.11", 3406 "pointradius": 1, 3407 "points": true, 3408 "renderer": "flot", 3409 "seriesOverrides": [], 3410 "spaceLength": 10, 3411 "stack": false, 3412 "steppedLine": false, 3413 "targets": [ 3414 { 3415 "exemplar": false, 3416 "expr": "sum(ticdc_scheduler_capture_table{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, addr)", 3417 "format": "time_series", 3418 "hide": false, 3419 "instant": false, 3420 "interval": "", 3421 "intervalFactor": 2, 3422 "legendFormat": "{{namespace}}-{{changefeed}}-{{addr}}", 3423 "refId": "A", 3424 "step": 10 3425 } 3426 ], 3427 "thresholds": [], 3428 "timeFrom": null, 3429 "timeRegions": [], 3430 "timeShift": null, 3431 "title": "Capture tables count", 3432 "tooltip": { 3433 "msResolution": false, 3434 "shared": true, 3435 "sort": 0, 3436 "value_type": "individual" 3437 }, 3438 "type": "graph", 3439 "xaxis": { 3440 "buckets": null, 3441 "mode": "time", 3442 "name": null, 3443 "show": true, 3444 "values": [] 3445 }, 3446 "yaxes": [ 3447 { 3448 "format": "none", 3449 "label": null, 3450 "logBase": 1, 3451 "max": null, 3452 "min": "0", 3453 "show": true 3454 }, 3455 { 3456 "format": "none", 3457 "label": null, 3458 "logBase": 1, 3459 "max": null, 3460 "min": null, 3461 "show": false 3462 } 3463 ], 3464 "yaxis": { 3465 "align": false, 3466 "alignLevel": null 3467 } 3468 }, 3469 { 3470 "aliasColors": {}, 3471 "bars": false, 3472 "dashLength": 10, 3473 "dashes": false, 3474 "datasource": "${DS_TEST-CLUSTER}", 3475 "fieldConfig": { 3476 "defaults": { 3477 "unit": "dateTimeAsIso" 3478 }, 3479 "overrides": [] 3480 }, 3481 "fill": 1, 3482 "fillGradient": 0, 3483 "gridPos": { 3484 "h": 7, 3485 "w": 12, 3486 "x": 12, 3487 "y": 3 3488 }, 3489 "hiddenSeries": false, 3490 "id": 10037, 3491 "legend": { 3492 "avg": false, 3493 "current": true, 3494 "max": false, 3495 "min": false, 3496 "show": true, 3497 "total": false, 3498 "values": true 3499 }, 3500 "lines": true, 3501 "linewidth": 1, 3502 "nullPointMode": "null", 3503 "options": { 3504 "alertThreshold": true 3505 }, 3506 "percentage": false, 3507 "pluginVersion": "7.5.10", 3508 "pointradius": 2, 3509 "points": false, 3510 "renderer": "flot", 3511 "seriesOverrides": [], 3512 "spaceLength": 10, 3513 "stack": false, 3514 "steppedLine": false, 3515 "targets": [ 3516 { 3517 "exemplar": true, 3518 "expr": "max(ticdc_gc_min_service_gc_safepoint{})", 3519 "interval": "", 3520 "legendFormat": "gc time", 3521 "queryType": "randomWalk", 3522 "refId": "A" 3523 }, 3524 { 3525 "exemplar": true, 3526 "expr": "max(ticdc_gc_cdc_gc_safepoint{})", 3527 "hide": false, 3528 "interval": "", 3529 "legendFormat": "cdc service safepoint", 3530 "refId": "B" 3531 } 3532 ], 3533 "thresholds": [], 3534 "timeFrom": null, 3535 "timeRegions": [], 3536 "timeShift": null, 3537 "title": "GC Time", 3538 "tooltip": { 3539 "shared": true, 3540 "sort": 0, 3541 "value_type": "individual" 3542 }, 3543 "type": "graph", 3544 "xaxis": { 3545 "buckets": null, 3546 "mode": "time", 3547 "name": null, 3548 "show": true, 3549 "values": [] 3550 }, 3551 "yaxes": [ 3552 { 3553 "format": "dateTimeAsIso", 3554 "label": null, 3555 "logBase": 1, 3556 "max": null, 3557 "min": null, 3558 "show": true 3559 }, 3560 { 3561 "format": "short", 3562 "label": null, 3563 "logBase": 1, 3564 "max": null, 3565 "min": null, 3566 "show": true 3567 } 3568 ], 3569 "yaxis": { 3570 "align": false, 3571 "alignLevel": null 3572 } 3573 }, 3574 { 3575 "aliasColors": {}, 3576 "bars": true, 3577 "cacheTimeout": null, 3578 "dashLength": 10, 3579 "dashes": false, 3580 "datasource": "${DS_TEST-CLUSTER}", 3581 "description": "The checkpoint ts of changefeeds.", 3582 "fieldConfig": { 3583 "defaults": { 3584 "unit": "dateTimeAsIso" 3585 }, 3586 "overrides": [] 3587 }, 3588 "fill": 0, 3589 "fillGradient": 0, 3590 "gridPos": { 3591 "h": 7, 3592 "w": 12, 3593 "x": 0, 3594 "y": 10 3595 }, 3596 "hiddenSeries": false, 3597 "id": 86, 3598 "legend": { 3599 "alignAsTable": true, 3600 "avg": false, 3601 "current": true, 3602 "max": false, 3603 "min": false, 3604 "rightSide": false, 3605 "show": true, 3606 "sideWidth": null, 3607 "total": false, 3608 "values": true 3609 }, 3610 "lines": true, 3611 "linewidth": 2, 3612 "links": [], 3613 "nullPointMode": "null", 3614 "options": { 3615 "alertThreshold": true 3616 }, 3617 "percentage": false, 3618 "pluginVersion": "7.5.11", 3619 "pointradius": 2, 3620 "points": false, 3621 "renderer": "flot", 3622 "seriesOverrides": [ 3623 { 3624 "alias": "/approximate current time.*/", 3625 "bars": false 3626 } 3627 ], 3628 "spaceLength": 10, 3629 "stack": false, 3630 "steppedLine": false, 3631 "targets": [ 3632 { 3633 "expr": "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", 3634 "format": "time_series", 3635 "hide": false, 3636 "interval": "", 3637 "intervalFactor": 1, 3638 "legendFormat": "approximate current time (s)", 3639 "refId": "A" 3640 }, 3641 { 3642 "exemplar": true, 3643 "expr": "max(ticdc_owner_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", 3644 "format": "time_series", 3645 "hide": false, 3646 "interval": "", 3647 "intervalFactor": 1, 3648 "legendFormat": "{{namespace}}-{{changefeed}}", 3649 "refId": "B" 3650 }, 3651 { 3652 "exemplar": true, 3653 "expr": "max(ticdc_owner_barrier_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", 3654 "hide": false, 3655 "interval": "", 3656 "legendFormat": "{{namespace}}-{{changefeed}}-barrierTs", 3657 "refId": "D" 3658 } 3659 ], 3660 "thresholds": [], 3661 "timeFrom": null, 3662 "timeRegions": [], 3663 "timeShift": null, 3664 "title": "Changefeed checkpoint", 3665 "tooltip": { 3666 "shared": true, 3667 "sort": 0, 3668 "value_type": "individual" 3669 }, 3670 "type": "graph", 3671 "xaxis": { 3672 "buckets": null, 3673 "max": null, 3674 "mode": "time", 3675 "name": null, 3676 "show": true, 3677 "values": [] 3678 }, 3679 "yaxes": [ 3680 { 3681 "format": "dateTimeAsIso", 3682 "label": null, 3683 "logBase": 1, 3684 "max": null, 3685 "min": null, 3686 "show": true 3687 }, 3688 { 3689 "format": "short", 3690 "label": null, 3691 "logBase": 1, 3692 "max": null, 3693 "min": null, 3694 "show": false 3695 } 3696 ], 3697 "yaxis": { 3698 "align": false, 3699 "alignLevel": null 3700 } 3701 }, 3702 { 3703 "aliasColors": {}, 3704 "bars": true, 3705 "cacheTimeout": null, 3706 "dashLength": 10, 3707 "dashes": false, 3708 "datasource": "${DS_TEST-CLUSTER}", 3709 "description": "The resolved ts of changefeeds.", 3710 "fieldConfig": { 3711 "defaults": {}, 3712 "overrides": [] 3713 }, 3714 "fill": 0, 3715 "fillGradient": 0, 3716 "gridPos": { 3717 "h": 7, 3718 "w": 12, 3719 "x": 12, 3720 "y": 10 3721 }, 3722 "hiddenSeries": false, 3723 "id": 512, 3724 "legend": { 3725 "alignAsTable": true, 3726 "avg": false, 3727 "current": true, 3728 "max": false, 3729 "min": false, 3730 "rightSide": false, 3731 "show": true, 3732 "sideWidth": null, 3733 "total": false, 3734 "values": true 3735 }, 3736 "lines": true, 3737 "linewidth": 2, 3738 "links": [], 3739 "nullPointMode": "null", 3740 "options": { 3741 "alertThreshold": true 3742 }, 3743 "percentage": false, 3744 "pluginVersion": "7.5.11", 3745 "pointradius": 2, 3746 "points": false, 3747 "renderer": "flot", 3748 "seriesOverrides": [ 3749 { 3750 "alias": "/approximate current time.*/", 3751 "bars": false 3752 } 3753 ], 3754 "spaceLength": 10, 3755 "stack": false, 3756 "steppedLine": false, 3757 "targets": [ 3758 { 3759 "expr": "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", 3760 "format": "time_series", 3761 "hide": false, 3762 "interval": "", 3763 "intervalFactor": 1, 3764 "legendFormat": "approximate current time (s)", 3765 "refId": "A" 3766 }, 3767 { 3768 "exemplar": true, 3769 "expr": "max(ticdc_owner_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)", 3770 "format": "time_series", 3771 "hide": false, 3772 "interval": "", 3773 "intervalFactor": 1, 3774 "legendFormat": "{{namespace}}-{{changefeed}}-barrier", 3775 "refId": "C" 3776 }, 3777 { 3778 "exemplar": true, 3779 "expr": "max(ticdc_scheduler_slow_table_puller_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)", 3780 "hide": false, 3781 "interval": "", 3782 "legendFormat": "{{namespace}}-{{changefeed}}-puller", 3783 "refId": "B" 3784 } 3785 ], 3786 "thresholds": [], 3787 "timeFrom": null, 3788 "timeRegions": [], 3789 "timeShift": null, 3790 "title": "Changefeed resolved ts", 3791 "tooltip": { 3792 "shared": true, 3793 "sort": 0, 3794 "value_type": "individual" 3795 }, 3796 "type": "graph", 3797 "xaxis": { 3798 "buckets": null, 3799 "max": null, 3800 "mode": "time", 3801 "name": null, 3802 "show": true, 3803 "values": [] 3804 }, 3805 "yaxes": [ 3806 { 3807 "format": "dateTimeAsIso", 3808 "label": null, 3809 "logBase": 1, 3810 "max": null, 3811 "min": null, 3812 "show": true 3813 }, 3814 { 3815 "format": "short", 3816 "label": null, 3817 "logBase": 1, 3818 "max": null, 3819 "min": null, 3820 "show": false 3821 } 3822 ], 3823 "yaxis": { 3824 "align": false, 3825 "alignLevel": null 3826 } 3827 }, 3828 { 3829 "aliasColors": {}, 3830 "bars": false, 3831 "dashLength": 10, 3832 "dashes": false, 3833 "datasource": "${DS_TEST-CLUSTER}", 3834 "description": "The lag between changefeed checkpoint ts and the latest ts of upstream TiDB.", 3835 "fieldConfig": { 3836 "defaults": { 3837 "links": [] 3838 }, 3839 "overrides": [] 3840 }, 3841 "fill": 1, 3842 "fillGradient": 0, 3843 "gridPos": { 3844 "h": 7, 3845 "w": 12, 3846 "x": 0, 3847 "y": 17 3848 }, 3849 "hiddenSeries": false, 3850 "id": 3, 3851 "legend": { 3852 "alignAsTable": true, 3853 "avg": false, 3854 "current": true, 3855 "max": false, 3856 "min": false, 3857 "rightSide": false, 3858 "show": true, 3859 "total": false, 3860 "values": true 3861 }, 3862 "lines": true, 3863 "linewidth": 1, 3864 "links": [], 3865 "nullPointMode": "null", 3866 "options": { 3867 "alertThreshold": true 3868 }, 3869 "paceLength": 10, 3870 "percentage": false, 3871 "pluginVersion": "7.5.11", 3872 "pointradius": 2, 3873 "points": false, 3874 "renderer": "flot", 3875 "seriesOverrides": [], 3876 "spaceLength": 10, 3877 "stack": false, 3878 "steppedLine": false, 3879 "targets": [ 3880 { 3881 "exemplar": true, 3882 "expr": "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)", 3883 "format": "time_series", 3884 "interval": "", 3885 "intervalFactor": 1, 3886 "legendFormat": "{{namespace}}-{{changefeed}}", 3887 "refId": "A" 3888 } 3889 ], 3890 "thresholds": [], 3891 "timeFrom": null, 3892 "timeRegions": [], 3893 "timeShift": null, 3894 "title": "Changefeed checkpoint lag", 3895 "tooltip": { 3896 "shared": true, 3897 "sort": 0, 3898 "value_type": "individual" 3899 }, 3900 "type": "graph", 3901 "xaxis": { 3902 "buckets": null, 3903 "mode": "time", 3904 "name": null, 3905 "show": true, 3906 "values": [] 3907 }, 3908 "yaxes": [ 3909 { 3910 "format": "s", 3911 "label": null, 3912 "logBase": 1, 3913 "max": null, 3914 "min": "0", 3915 "show": true 3916 }, 3917 { 3918 "format": "short", 3919 "label": null, 3920 "logBase": 1, 3921 "max": null, 3922 "min": null, 3923 "show": true 3924 } 3925 ], 3926 "yaxis": { 3927 "align": false, 3928 "alignLevel": null 3929 } 3930 }, 3931 { 3932 "aliasColors": {}, 3933 "bars": false, 3934 "dashLength": 10, 3935 "dashes": false, 3936 "datasource": "${DS_TEST-CLUSTER}", 3937 "description": "The lag between changefeed resolved ts and the latest ts of upstream TiDB.", 3938 "fieldConfig": { 3939 "defaults": { 3940 "links": [] 3941 }, 3942 "overrides": [] 3943 }, 3944 "fill": 1, 3945 "fillGradient": 0, 3946 "gridPos": { 3947 "h": 7, 3948 "w": 12, 3949 "x": 12, 3950 "y": 17 3951 }, 3952 "hiddenSeries": false, 3953 "id": 513, 3954 "legend": { 3955 "alignAsTable": true, 3956 "avg": false, 3957 "current": true, 3958 "max": false, 3959 "min": false, 3960 "rightSide": false, 3961 "show": true, 3962 "total": false, 3963 "values": true 3964 }, 3965 "lines": true, 3966 "linewidth": 1, 3967 "links": [], 3968 "nullPointMode": "null", 3969 "options": { 3970 "alertThreshold": true 3971 }, 3972 "paceLength": 10, 3973 "percentage": false, 3974 "pluginVersion": "7.5.11", 3975 "pointradius": 2, 3976 "points": false, 3977 "renderer": "flot", 3978 "seriesOverrides": [], 3979 "spaceLength": 10, 3980 "stack": false, 3981 "steppedLine": false, 3982 "targets": [ 3983 { 3984 "exemplar": true, 3985 "expr": "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", 3986 "format": "time_series", 3987 "interval": "", 3988 "intervalFactor": 1, 3989 "legendFormat": "{{namespace}}-{{changefeed}}-barrier", 3990 "refId": "C" 3991 }, 3992 { 3993 "exemplar": true, 3994 "expr": "max(ticdc_scheduler_slow_table_puller_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed)", 3995 "hide": false, 3996 "interval": "", 3997 "legendFormat": "{{namespace}}-{{changefeed}}-puller", 3998 "refId": "A" 3999 } 4000 ], 4001 "thresholds": [], 4002 "timeFrom": null, 4003 "timeRegions": [], 4004 "timeShift": null, 4005 "title": "Changefeed resolved ts lag", 4006 "tooltip": { 4007 "shared": true, 4008 "sort": 0, 4009 "value_type": "individual" 4010 }, 4011 "type": "graph", 4012 "xaxis": { 4013 "buckets": null, 4014 "mode": "time", 4015 "name": null, 4016 "show": true, 4017 "values": [] 4018 }, 4019 "yaxes": [ 4020 { 4021 "format": "s", 4022 "label": null, 4023 "logBase": 1, 4024 "max": null, 4025 "min": "0", 4026 "show": true 4027 }, 4028 { 4029 "format": "short", 4030 "label": null, 4031 "logBase": 1, 4032 "max": null, 4033 "min": null, 4034 "show": true 4035 } 4036 ], 4037 "yaxis": { 4038 "align": false, 4039 "alignLevel": null 4040 } 4041 }, 4042 { 4043 "aliasColors": {}, 4044 "bars": false, 4045 "dashLength": 10, 4046 "dashes": false, 4047 "datasource": "${DS_TEST-CLUSTER}", 4048 "description": "The difference between changefeed resolved ts and changefeed checkpointTs.", 4049 "fieldConfig": { 4050 "defaults": { 4051 "links": [] 4052 }, 4053 "overrides": [] 4054 }, 4055 "fill": 1, 4056 "fillGradient": 0, 4057 "gridPos": { 4058 "h": 7, 4059 "w": 12, 4060 "x": 0, 4061 "y": 24 4062 }, 4063 "hiddenSeries": false, 4064 "id": 603, 4065 "legend": { 4066 "alignAsTable": true, 4067 "avg": false, 4068 "current": true, 4069 "max": false, 4070 "min": false, 4071 "rightSide": false, 4072 "show": true, 4073 "total": false, 4074 "values": true 4075 }, 4076 "lines": true, 4077 "linewidth": 1, 4078 "links": [], 4079 "nullPointMode": "null", 4080 "options": { 4081 "alertThreshold": true 4082 }, 4083 "paceLength": 10, 4084 "percentage": false, 4085 "pluginVersion": "7.5.11", 4086 "pointradius": 2, 4087 "points": false, 4088 "renderer": "flot", 4089 "seriesOverrides": [], 4090 "spaceLength": 10, 4091 "stack": false, 4092 "steppedLine": false, 4093 "targets": [ 4094 { 4095 "exemplar": true, 4096 "expr": "(max(ticdc_owner_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed) - max(ticdc_owner_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)) / 1000 > 0", 4097 "format": "time_series", 4098 "interval": "", 4099 "intervalFactor": 1, 4100 "legendFormat": "{{namespace}}-{{changefeed}}", 4101 "refId": "C" 4102 } 4103 ], 4104 "thresholds": [], 4105 "timeFrom": null, 4106 "timeRegions": [], 4107 "timeShift": null, 4108 "title": "Difference of resolvedTs and checkpointTs", 4109 "tooltip": { 4110 "shared": true, 4111 "sort": 0, 4112 "value_type": "individual" 4113 }, 4114 "type": "graph", 4115 "xaxis": { 4116 "buckets": null, 4117 "mode": "time", 4118 "name": null, 4119 "show": true, 4120 "values": [] 4121 }, 4122 "yaxes": [ 4123 { 4124 "format": "s", 4125 "label": null, 4126 "logBase": 1, 4127 "max": null, 4128 "min": "0", 4129 "show": true 4130 }, 4131 { 4132 "format": "short", 4133 "label": null, 4134 "logBase": 1, 4135 "max": null, 4136 "min": null, 4137 "show": true 4138 } 4139 ], 4140 "yaxis": { 4141 "align": false, 4142 "alignLevel": 1 4143 } 4144 }, 4145 { 4146 "aliasColors": {}, 4147 "bars": false, 4148 "dashLength": 10, 4149 "dashes": false, 4150 "datasource": "${DS_TEST-CLUSTER}", 4151 "description": "each changefeed tick periodically, the time cost of each tick", 4152 "fieldConfig": { 4153 "defaults": {}, 4154 "overrides": [] 4155 }, 4156 "fill": 1, 4157 "fillGradient": 0, 4158 "gridPos": { 4159 "h": 7, 4160 "w": 12, 4161 "x": 12, 4162 "y": 24 4163 }, 4164 "hiddenSeries": false, 4165 "id": 597, 4166 "legend": { 4167 "alignAsTable": true, 4168 "avg": false, 4169 "current": true, 4170 "max": true, 4171 "min": false, 4172 "rightSide": false, 4173 "show": true, 4174 "total": false, 4175 "values": true 4176 }, 4177 "lines": true, 4178 "linewidth": 1, 4179 "nullPointMode": "null", 4180 "options": { 4181 "alertThreshold": true 4182 }, 4183 "percentage": false, 4184 "pluginVersion": "7.5.11", 4185 "pointradius": 2, 4186 "points": false, 4187 "renderer": "flot", 4188 "seriesOverrides": [], 4189 "spaceLength": 10, 4190 "stack": false, 4191 "steppedLine": false, 4192 "targets": [ 4193 { 4194 "exemplar": true, 4195 "expr": "histogram_quantile(0.999, sum(rate(ticdc_owner_changefeed_tick_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le,instance,namespace,changefeed))", 4196 "hide": false, 4197 "interval": "", 4198 "legendFormat": "{{namespace}}-{{changefeed}}-p999", 4199 "refId": "A" 4200 }, 4201 { 4202 "exemplar": true, 4203 "expr": "histogram_quantile(0.99, sum(rate(ticdc_processor_processor_tick_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le,instance,namespace,changefeed))", 4204 "hide": false, 4205 "interval": "", 4206 "legendFormat": "{{namespace}}-{{changefeed}}-p99", 4207 "refId": "B" 4208 } 4209 ], 4210 "thresholds": [], 4211 "timeFrom": null, 4212 "timeRegions": [], 4213 "timeShift": null, 4214 "title": "changefeed tick duration percentile", 4215 "tooltip": { 4216 "shared": true, 4217 "sort": 0, 4218 "value_type": "individual" 4219 }, 4220 "type": "graph", 4221 "xaxis": { 4222 "buckets": null, 4223 "mode": "time", 4224 "name": null, 4225 "show": true, 4226 "values": [] 4227 }, 4228 "yaxes": [ 4229 { 4230 "format": "s", 4231 "label": null, 4232 "logBase": 1, 4233 "max": null, 4234 "min": null, 4235 "show": true 4236 }, 4237 { 4238 "format": "none", 4239 "label": null, 4240 "logBase": 1, 4241 "max": null, 4242 "min": null, 4243 "show": true 4244 } 4245 ], 4246 "yaxis": { 4247 "align": false, 4248 "alignLevel": null 4249 } 4250 }, 4251 { 4252 "datasource": "${DS_TEST-CLUSTER}", 4253 "description": "The status of each changefeed.\n\n0: Normal\n\n1 and 6: Warning\n\n2: Failed\n\n3: Stopped\n\n4: Finished\n\n-1: Unknown", 4254 "fieldConfig": { 4255 "defaults": { 4256 "color": { 4257 "mode": "palette-classic" 4258 }, 4259 "custom": { 4260 "axisLabel": "", 4261 "axisPlacement": "auto", 4262 "barAlignment": 0, 4263 "drawStyle": "line", 4264 "fillOpacity": 10, 4265 "gradientMode": "none", 4266 "hideFrom": { 4267 "graph": false, 4268 "legend": false, 4269 "tooltip": false 4270 }, 4271 "lineInterpolation": "stepAfter", 4272 "lineWidth": 1, 4273 "pointSize": 4, 4274 "scaleDistribution": { 4275 "type": "linear" 4276 }, 4277 "showPoints": "always", 4278 "spanNulls": true 4279 }, 4280 "mappings": [ 4281 { 4282 "from": "", 4283 "id": 1, 4284 "text": "Normal", 4285 "to": "", 4286 "type": 1, 4287 "value": "0" 4288 }, 4289 { 4290 "from": "", 4291 "id": 2, 4292 "text": "Warning", 4293 "to": "", 4294 "type": 1, 4295 "value": "1" 4296 }, 4297 { 4298 "from": "", 4299 "id": 3, 4300 "text": "Failed", 4301 "to": "", 4302 "type": 1, 4303 "value": "2" 4304 }, 4305 { 4306 "from": "", 4307 "id": 4, 4308 "text": "Stopped", 4309 "to": "", 4310 "type": 1, 4311 "value": "3" 4312 }, 4313 { 4314 "from": "", 4315 "id": 5, 4316 "text": "Finished", 4317 "to": "", 4318 "type": 1, 4319 "value": "4" 4320 }, 4321 { 4322 "from": "", 4323 "id": 6, 4324 "text": "Unknown", 4325 "to": "", 4326 "type": 1, 4327 "value": "-1" 4328 }, 4329 { 4330 "from": "5", 4331 "id": 7, 4332 "text": "Other", 4333 "to": "10000", 4334 "type": 1, 4335 "value": "5" 4336 }, 4337 { 4338 "from": "6", 4339 "id": 8, 4340 "text": "-", 4341 "to": "1000", 4342 "type": 2 4343 } 4344 ], 4345 "max": 8, 4346 "min": -1, 4347 "noValue": "-", 4348 "thresholds": { 4349 "mode": "absolute", 4350 "steps": [ 4351 { 4352 "color": "green", 4353 "value": null 4354 }, 4355 { 4356 "color": "light-yellow", 4357 "value": -1 4358 }, 4359 { 4360 "color": "green", 4361 "value": 0 4362 }, 4363 { 4364 "color": "semi-dark-red", 4365 "value": 1 4366 }, 4367 { 4368 "color": "dark-red", 4369 "value": 2 4370 }, 4371 { 4372 "color": "dark-yellow", 4373 "value": 3 4374 }, 4375 { 4376 "color": "#6ED0E0", 4377 "value": 4 4378 } 4379 ] 4380 }, 4381 "unit": "short" 4382 }, 4383 "overrides": [] 4384 }, 4385 "gridPos": { 4386 "h": 7, 4387 "w": 12, 4388 "x": 0, 4389 "y": 31 4390 }, 4391 "id": 163, 4392 "links": [], 4393 "options": { 4394 "graph": {}, 4395 "legend": { 4396 "calcs": [ 4397 "lastNotNull" 4398 ], 4399 "displayMode": "table", 4400 "placement": "bottom" 4401 }, 4402 "tooltipOptions": { 4403 "mode": "single" 4404 } 4405 }, 4406 "pluginVersion": "7.5.11", 4407 "targets": [ 4408 { 4409 "exemplar": true, 4410 "expr": "max(ticdc_owner_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", 4411 "format": "time_series", 4412 "instant": false, 4413 "interval": "", 4414 "intervalFactor": 1, 4415 "legendFormat": "{{namespace}}-{{changefeed}}", 4416 "refId": "A" 4417 } 4418 ], 4419 "timeFrom": null, 4420 "timeShift": null, 4421 "title": "The status of changefeeds", 4422 "type": "timeseries" 4423 }, 4424 { 4425 "aliasColors": {}, 4426 "bars": true, 4427 "cacheTimeout": null, 4428 "dashLength": 10, 4429 "dashes": false, 4430 "datasource": "${DS_TEST-CLUSTER}", 4431 "description": "", 4432 "fieldConfig": { 4433 "defaults": {}, 4434 "overrides": [] 4435 }, 4436 "fill": 0, 4437 "fillGradient": 0, 4438 "gridPos": { 4439 "h": 7, 4440 "w": 12, 4441 "x": 12, 4442 "y": 31 4443 }, 4444 "hiddenSeries": false, 4445 "id": 514, 4446 "legend": { 4447 "alignAsTable": true, 4448 "avg": false, 4449 "current": true, 4450 "max": false, 4451 "min": false, 4452 "rightSide": false, 4453 "show": true, 4454 "sideWidth": null, 4455 "total": false, 4456 "values": true 4457 }, 4458 "lines": true, 4459 "linewidth": 2, 4460 "links": [], 4461 "nullPointMode": "null", 4462 "options": { 4463 "alertThreshold": true 4464 }, 4465 "percentage": false, 4466 "pluginVersion": "7.5.11", 4467 "pointradius": 2, 4468 "points": false, 4469 "renderer": "flot", 4470 "seriesOverrides": [ 4471 { 4472 "alias": "/approximate current time.*/", 4473 "bars": false 4474 } 4475 ], 4476 "spaceLength": 10, 4477 "stack": false, 4478 "steppedLine": false, 4479 "targets": [ 4480 { 4481 "expr": "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", 4482 "format": "time_series", 4483 "hide": false, 4484 "interval": "", 4485 "intervalFactor": 1, 4486 "legendFormat": "approximate current time (s)", 4487 "refId": "A" 4488 }, 4489 { 4490 "exemplar": true, 4491 "expr": "max(ticdc_processor_schema_storage_gc_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)", 4492 "format": "time_series", 4493 "hide": false, 4494 "interval": "", 4495 "intervalFactor": 1, 4496 "legendFormat": "{{namespace}}-{{changefeed}}", 4497 "refId": "C" 4498 } 4499 ], 4500 "thresholds": [], 4501 "timeFrom": null, 4502 "timeRegions": [], 4503 "timeShift": null, 4504 "title": "Schema Storage GC progress", 4505 "tooltip": { 4506 "shared": true, 4507 "sort": 0, 4508 "value_type": "individual" 4509 }, 4510 "type": "graph", 4511 "xaxis": { 4512 "buckets": null, 4513 "max": null, 4514 "mode": "time", 4515 "name": null, 4516 "show": true, 4517 "values": [] 4518 }, 4519 "yaxes": [ 4520 { 4521 "format": "dateTimeAsIso", 4522 "label": null, 4523 "logBase": 1, 4524 "max": null, 4525 "min": null, 4526 "show": true 4527 }, 4528 { 4529 "format": "short", 4530 "label": null, 4531 "logBase": 1, 4532 "max": null, 4533 "min": null, 4534 "show": false 4535 } 4536 ], 4537 "yaxis": { 4538 "align": false, 4539 "alignLevel": null 4540 } 4541 }, 4542 { 4543 "aliasColors": {}, 4544 "bars": false, 4545 "dashLength": 10, 4546 "dashes": false, 4547 "datasource": "${DS_TEST-CLUSTER}", 4548 "description": "Request count of etcd operation per second", 4549 "fieldConfig": { 4550 "defaults": { 4551 "links": [] 4552 }, 4553 "overrides": [] 4554 }, 4555 "fill": 1, 4556 "fillGradient": 0, 4557 "gridPos": { 4558 "h": 7, 4559 "w": 12, 4560 "x": 0, 4561 "y": 38 4562 }, 4563 "hiddenSeries": false, 4564 "id": 102, 4565 "legend": { 4566 "alignAsTable": true, 4567 "avg": false, 4568 "current": true, 4569 "hideEmpty": true, 4570 "hideZero": true, 4571 "max": true, 4572 "min": false, 4573 "rightSide": true, 4574 "show": true, 4575 "total": false, 4576 "values": true 4577 }, 4578 "lines": true, 4579 "linewidth": 1, 4580 "links": [], 4581 "nullPointMode": "null", 4582 "options": { 4583 "alertThreshold": true 4584 }, 4585 "paceLength": 10, 4586 "percentage": false, 4587 "pluginVersion": "7.5.11", 4588 "pointradius": 2, 4589 "points": false, 4590 "renderer": "flot", 4591 "seriesOverrides": [], 4592 "spaceLength": 10, 4593 "stack": false, 4594 "steppedLine": false, 4595 "targets": [ 4596 { 4597 "exemplar": true, 4598 "expr": "sum(rate(ticdc_etcd_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance, type)", 4599 "format": "time_series", 4600 "interval": "", 4601 "intervalFactor": 1, 4602 "legendFormat": "{{instance}}-{{type}}", 4603 "refId": "A" 4604 } 4605 ], 4606 "thresholds": [], 4607 "timeFrom": null, 4608 "timeRegions": [], 4609 "timeShift": null, 4610 "title": "PD etcd requests/s", 4611 "tooltip": { 4612 "shared": true, 4613 "sort": 0, 4614 "value_type": "individual" 4615 }, 4616 "type": "graph", 4617 "xaxis": { 4618 "buckets": null, 4619 "mode": "time", 4620 "name": null, 4621 "show": true, 4622 "values": [] 4623 }, 4624 "yaxes": [ 4625 { 4626 "format": "short", 4627 "label": null, 4628 "logBase": 1, 4629 "max": null, 4630 "min": null, 4631 "show": true 4632 }, 4633 { 4634 "format": "short", 4635 "label": null, 4636 "logBase": 1, 4637 "max": null, 4638 "min": null, 4639 "show": true 4640 } 4641 ], 4642 "yaxis": { 4643 "align": false, 4644 "alignLevel": null 4645 } 4646 }, 4647 { 4648 "aliasColors": {}, 4649 "bars": false, 4650 "dashLength": 10, 4651 "dashes": false, 4652 "datasource": "${DS_TEST-CLUSTER}", 4653 "description": "The number of errors that interrupt changefeed per minute ", 4654 "fieldConfig": { 4655 "defaults": {}, 4656 "overrides": [] 4657 }, 4658 "fill": 1, 4659 "fillGradient": 0, 4660 "gridPos": { 4661 "h": 7, 4662 "w": 12, 4663 "x": 12, 4664 "y": 38 4665 }, 4666 "hiddenSeries": false, 4667 "id": 82, 4668 "legend": { 4669 "alignAsTable": true, 4670 "avg": false, 4671 "current": true, 4672 "max": false, 4673 "min": false, 4674 "rightSide": false, 4675 "show": true, 4676 "total": false, 4677 "values": true 4678 }, 4679 "lines": true, 4680 "linewidth": 1, 4681 "links": [], 4682 "nullPointMode": "null", 4683 "options": { 4684 "alertThreshold": true 4685 }, 4686 "percentage": false, 4687 "pluginVersion": "7.5.11", 4688 "pointradius": 2, 4689 "points": false, 4690 "renderer": "flot", 4691 "seriesOverrides": [], 4692 "spaceLength": 10, 4693 "stack": false, 4694 "steppedLine": false, 4695 "targets": [ 4696 { 4697 "exemplar": true, 4698 "expr": "sum(rate(ticdc_processor_exit_with_error_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (instance, namespace,changefeed)", 4699 "hide": false, 4700 "interval": "", 4701 "legendFormat": "{{instance}}-{{namespace}}-{{changefeed}}", 4702 "refId": "B" 4703 } 4704 ], 4705 "thresholds": [], 4706 "timeFrom": null, 4707 "timeRegions": [], 4708 "timeShift": null, 4709 "title": "Exit error count/m", 4710 "tooltip": { 4711 "shared": true, 4712 "sort": 0, 4713 "value_type": "individual" 4714 }, 4715 "type": "graph", 4716 "xaxis": { 4717 "buckets": null, 4718 "mode": "time", 4719 "name": null, 4720 "show": true, 4721 "values": [] 4722 }, 4723 "yaxes": [ 4724 { 4725 "format": "short", 4726 "label": null, 4727 "logBase": 1, 4728 "max": null, 4729 "min": null, 4730 "show": true 4731 }, 4732 { 4733 "format": "short", 4734 "label": null, 4735 "logBase": 1, 4736 "max": null, 4737 "min": null, 4738 "show": true 4739 } 4740 ], 4741 "yaxis": { 4742 "align": false, 4743 "alignLevel": null 4744 } 4745 } 4746 ], 4747 "title": "Changefeed", 4748 "type": "row" 4749 }, 4750 { 4751 "collapsed": true, 4752 "datasource": null, 4753 "gridPos": { 4754 "h": 1, 4755 "w": 24, 4756 "x": 0, 4757 "y": 3 4758 }, 4759 "id": 206, 4760 "panels": [ 4761 { 4762 "aliasColors": {}, 4763 "bars": false, 4764 "dashLength": 10, 4765 "dashes": false, 4766 "datasource": "${DS_TEST-CLUSTER}", 4767 "description": "The number of events that puller outputs to sorter \n per second", 4768 "fieldConfig": { 4769 "defaults": { 4770 "links": [] 4771 }, 4772 "overrides": [] 4773 }, 4774 "fill": 1, 4775 "fillGradient": 0, 4776 "gridPos": { 4777 "h": 4, 4778 "w": 12, 4779 "x": 0, 4780 "y": 4 4781 }, 4782 "hiddenSeries": false, 4783 "id": 218, 4784 "legend": { 4785 "alignAsTable": true, 4786 "avg": false, 4787 "current": true, 4788 "max": true, 4789 "min": false, 4790 "rightSide": false, 4791 "show": true, 4792 "total": false, 4793 "values": true 4794 }, 4795 "lines": true, 4796 "linewidth": 1, 4797 "links": [], 4798 "nullPointMode": "null", 4799 "options": { 4800 "alertThreshold": true 4801 }, 4802 "paceLength": 10, 4803 "percentage": false, 4804 "pluginVersion": "7.5.11", 4805 "pointradius": 2, 4806 "points": false, 4807 "renderer": "flot", 4808 "seriesOverrides": [], 4809 "spaceLength": 10, 4810 "stack": false, 4811 "steppedLine": false, 4812 "targets": [ 4813 { 4814 "exemplar": true, 4815 "expr": "sum (rate(ticdc_puller_txn_collect_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\", changefeed!~\".*ddl_puller\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, type)", 4816 "format": "time_series", 4817 "interval": "", 4818 "intervalFactor": 1, 4819 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{type}}", 4820 "refId": "A" 4821 } 4822 ], 4823 "thresholds": [], 4824 "timeFrom": null, 4825 "timeRegions": [], 4826 "timeShift": null, 4827 "title": "Puller output events/s", 4828 "tooltip": { 4829 "shared": true, 4830 "sort": 0, 4831 "value_type": "individual" 4832 }, 4833 "type": "graph", 4834 "xaxis": { 4835 "buckets": null, 4836 "mode": "time", 4837 "name": null, 4838 "show": true, 4839 "values": [] 4840 }, 4841 "yaxes": [ 4842 { 4843 "$$hashKey": "object:643", 4844 "format": "short", 4845 "label": "", 4846 "logBase": 1, 4847 "max": null, 4848 "min": null, 4849 "show": true 4850 }, 4851 { 4852 "$$hashKey": "object:644", 4853 "format": "short", 4854 "label": null, 4855 "logBase": 1, 4856 "max": null, 4857 "min": null, 4858 "show": false 4859 } 4860 ], 4861 "yaxis": { 4862 "align": false, 4863 "alignLevel": null 4864 } 4865 }, 4866 { 4867 "aliasColors": {}, 4868 "bars": false, 4869 "dashLength": 10, 4870 "dashes": false, 4871 "datasource": "${DS_TEST-CLUSTER}", 4872 "description": "The total number of events that puller outputs", 4873 "fieldConfig": { 4874 "defaults": { 4875 "links": [] 4876 }, 4877 "overrides": [] 4878 }, 4879 "fill": 1, 4880 "fillGradient": 0, 4881 "gridPos": { 4882 "h": 4, 4883 "w": 12, 4884 "x": 12, 4885 "y": 4 4886 }, 4887 "hiddenSeries": false, 4888 "id": 229, 4889 "legend": { 4890 "alignAsTable": true, 4891 "avg": false, 4892 "current": true, 4893 "max": true, 4894 "min": false, 4895 "rightSide": false, 4896 "show": true, 4897 "sort": "max", 4898 "sortDesc": true, 4899 "total": false, 4900 "values": true 4901 }, 4902 "lines": true, 4903 "linewidth": 1, 4904 "links": [], 4905 "nullPointMode": "null", 4906 "options": { 4907 "alertThreshold": true 4908 }, 4909 "paceLength": 10, 4910 "percentage": false, 4911 "pluginVersion": "7.5.11", 4912 "pointradius": 2, 4913 "points": false, 4914 "renderer": "flot", 4915 "seriesOverrides": [], 4916 "spaceLength": 10, 4917 "stack": false, 4918 "steppedLine": false, 4919 "targets": [ 4920 { 4921 "exemplar": true, 4922 "expr": "sum(ticdc_puller_txn_collect_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, type)", 4923 "format": "time_series", 4924 "interval": "", 4925 "intervalFactor": 1, 4926 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{type}}", 4927 "refId": "A" 4928 } 4929 ], 4930 "thresholds": [], 4931 "timeFrom": null, 4932 "timeRegions": [], 4933 "timeShift": null, 4934 "title": "Puller output events", 4935 "tooltip": { 4936 "shared": true, 4937 "sort": 0, 4938 "value_type": "individual" 4939 }, 4940 "type": "graph", 4941 "xaxis": { 4942 "buckets": null, 4943 "mode": "time", 4944 "name": null, 4945 "show": true, 4946 "values": [] 4947 }, 4948 "yaxes": [ 4949 { 4950 "$$hashKey": "object:915", 4951 "format": "none", 4952 "label": null, 4953 "logBase": 1, 4954 "max": null, 4955 "min": null, 4956 "show": true 4957 }, 4958 { 4959 "$$hashKey": "object:916", 4960 "format": "short", 4961 "label": null, 4962 "logBase": 1, 4963 "max": null, 4964 "min": null, 4965 "show": true 4966 } 4967 ], 4968 "yaxis": { 4969 "align": false, 4970 "alignLevel": null 4971 } 4972 }, 4973 { 4974 "aliasColors": {}, 4975 "bars": false, 4976 "dashLength": 10, 4977 "dashes": false, 4978 "datasource": "${DS_TEST-CLUSTER}", 4979 "description": "The number of events that sorter outputs to mounter(or table sink worker) per second", 4980 "fieldConfig": { 4981 "defaults": { 4982 "links": [] 4983 }, 4984 "overrides": [] 4985 }, 4986 "fill": 1, 4987 "fillGradient": 0, 4988 "gridPos": { 4989 "h": 4, 4990 "w": 12, 4991 "x": 0, 4992 "y": 8 4993 }, 4994 "hiddenSeries": false, 4995 "id": 228, 4996 "legend": { 4997 "alignAsTable": true, 4998 "avg": false, 4999 "current": true, 5000 "max": true, 5001 "min": false, 5002 "rightSide": false, 5003 "show": true, 5004 "total": false, 5005 "values": true 5006 }, 5007 "lines": true, 5008 "linewidth": 1, 5009 "links": [], 5010 "nullPointMode": "null", 5011 "options": { 5012 "alertThreshold": true 5013 }, 5014 "paceLength": 10, 5015 "percentage": false, 5016 "pluginVersion": "7.5.11", 5017 "pointradius": 2, 5018 "points": false, 5019 "renderer": "flot", 5020 "seriesOverrides": [], 5021 "spaceLength": 10, 5022 "stack": false, 5023 "steppedLine": false, 5024 "targets": [ 5025 { 5026 "exemplar": true, 5027 "expr": "sum(rate(ticdc_sorter_output_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, type)", 5028 "format": "time_series", 5029 "interval": "", 5030 "intervalFactor": 1, 5031 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{type}}", 5032 "refId": "A" 5033 } 5034 ], 5035 "thresholds": [], 5036 "timeFrom": null, 5037 "timeRegions": [], 5038 "timeShift": null, 5039 "title": "Sorter output events/s", 5040 "tooltip": { 5041 "shared": true, 5042 "sort": 0, 5043 "value_type": "individual" 5044 }, 5045 "type": "graph", 5046 "xaxis": { 5047 "buckets": null, 5048 "mode": "time", 5049 "name": null, 5050 "show": true, 5051 "values": [] 5052 }, 5053 "yaxes": [ 5054 { 5055 "$$hashKey": "object:1089", 5056 "format": "short", 5057 "label": null, 5058 "logBase": 1, 5059 "max": null, 5060 "min": null, 5061 "show": true 5062 }, 5063 { 5064 "$$hashKey": "object:1090", 5065 "format": "short", 5066 "label": null, 5067 "logBase": 1, 5068 "max": null, 5069 "min": null, 5070 "show": false 5071 } 5072 ], 5073 "yaxis": { 5074 "align": false, 5075 "alignLevel": null 5076 } 5077 }, 5078 { 5079 "aliasColors": {}, 5080 "bars": false, 5081 "dashLength": 10, 5082 "dashes": false, 5083 "datasource": "${DS_TEST-CLUSTER}", 5084 "description": "The total number of events that sorter outputs", 5085 "fieldConfig": { 5086 "defaults": { 5087 "links": [] 5088 }, 5089 "overrides": [] 5090 }, 5091 "fill": 1, 5092 "fillGradient": 0, 5093 "gridPos": { 5094 "h": 4, 5095 "w": 12, 5096 "x": 12, 5097 "y": 8 5098 }, 5099 "hiddenSeries": false, 5100 "id": 220, 5101 "legend": { 5102 "alignAsTable": true, 5103 "avg": false, 5104 "current": true, 5105 "max": true, 5106 "min": false, 5107 "rightSide": false, 5108 "show": true, 5109 "total": false, 5110 "values": true 5111 }, 5112 "lines": true, 5113 "linewidth": 1, 5114 "links": [], 5115 "nullPointMode": "null", 5116 "options": { 5117 "alertThreshold": true 5118 }, 5119 "paceLength": 10, 5120 "percentage": false, 5121 "pluginVersion": "7.5.11", 5122 "pointradius": 2, 5123 "points": false, 5124 "renderer": "flot", 5125 "seriesOverrides": [], 5126 "spaceLength": 10, 5127 "stack": false, 5128 "steppedLine": false, 5129 "targets": [ 5130 { 5131 "exemplar": true, 5132 "expr": "sum(ticdc_sorter_output_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace, instance, changefeed, type)", 5133 "format": "time_series", 5134 "interval": "", 5135 "intervalFactor": 1, 5136 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{type}}", 5137 "refId": "A" 5138 } 5139 ], 5140 "thresholds": [], 5141 "timeFrom": null, 5142 "timeRegions": [], 5143 "timeShift": null, 5144 "title": "Sorter output events", 5145 "tooltip": { 5146 "shared": true, 5147 "sort": 0, 5148 "value_type": "individual" 5149 }, 5150 "type": "graph", 5151 "xaxis": { 5152 "buckets": null, 5153 "mode": "time", 5154 "name": null, 5155 "show": true, 5156 "values": [] 5157 }, 5158 "yaxes": [ 5159 { 5160 "$$hashKey": "object:1403", 5161 "format": "none", 5162 "label": null, 5163 "logBase": 1, 5164 "max": null, 5165 "min": null, 5166 "show": true 5167 }, 5168 { 5169 "$$hashKey": "object:1404", 5170 "format": "short", 5171 "label": null, 5172 "logBase": 1, 5173 "max": null, 5174 "min": null, 5175 "show": true 5176 } 5177 ], 5178 "yaxis": { 5179 "align": false, 5180 "alignLevel": null 5181 } 5182 }, 5183 { 5184 "aliasColors": {}, 5185 "bars": false, 5186 "dashLength": 10, 5187 "dashes": false, 5188 "datasource": "${DS_TEST-CLUSTER}", 5189 "description": "The number of events that mounter outputs to sink per second", 5190 "fieldConfig": { 5191 "defaults": { 5192 "links": [] 5193 }, 5194 "overrides": [] 5195 }, 5196 "fill": 1, 5197 "fillGradient": 0, 5198 "gridPos": { 5199 "h": 4, 5200 "w": 12, 5201 "x": 0, 5202 "y": 12 5203 }, 5204 "hiddenSeries": false, 5205 "id": 219, 5206 "legend": { 5207 "alignAsTable": true, 5208 "avg": false, 5209 "current": true, 5210 "max": true, 5211 "min": false, 5212 "rightSide": false, 5213 "show": true, 5214 "total": false, 5215 "values": true 5216 }, 5217 "lines": true, 5218 "linewidth": 1, 5219 "links": [], 5220 "nullPointMode": "null", 5221 "options": { 5222 "alertThreshold": true 5223 }, 5224 "paceLength": 10, 5225 "percentage": false, 5226 "pluginVersion": "7.5.11", 5227 "pointradius": 2, 5228 "points": false, 5229 "renderer": "flot", 5230 "seriesOverrides": [], 5231 "spaceLength": 10, 5232 "stack": false, 5233 "steppedLine": false, 5234 "targets": [ 5235 { 5236 "exemplar": true, 5237 "expr": "sum(rate(ticdc_mounter_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (instance,namespace,changefeed)", 5238 "format": "time_series", 5239 "interval": "", 5240 "intervalFactor": 1, 5241 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 5242 "refId": "A" 5243 } 5244 ], 5245 "thresholds": [], 5246 "timeFrom": null, 5247 "timeRegions": [], 5248 "timeShift": null, 5249 "title": "Mounter output events/s", 5250 "tooltip": { 5251 "shared": true, 5252 "sort": 0, 5253 "value_type": "individual" 5254 }, 5255 "type": "graph", 5256 "xaxis": { 5257 "buckets": null, 5258 "mode": "time", 5259 "name": null, 5260 "show": true, 5261 "values": [] 5262 }, 5263 "yaxes": [ 5264 { 5265 "$$hashKey": "object:1121", 5266 "format": "short", 5267 "label": null, 5268 "logBase": 1, 5269 "max": null, 5270 "min": null, 5271 "show": true 5272 }, 5273 { 5274 "$$hashKey": "object:1122", 5275 "format": "short", 5276 "label": null, 5277 "logBase": 1, 5278 "max": null, 5279 "min": null, 5280 "show": false 5281 } 5282 ], 5283 "yaxis": { 5284 "align": false, 5285 "alignLevel": null 5286 } 5287 }, 5288 { 5289 "aliasColors": {}, 5290 "bars": false, 5291 "dashLength": 10, 5292 "dashes": false, 5293 "datasource": "${DS_TEST-CLUSTER}", 5294 "description": "The total number of events that mounter outputs", 5295 "fieldConfig": { 5296 "defaults": { 5297 "links": [] 5298 }, 5299 "overrides": [] 5300 }, 5301 "fill": 1, 5302 "fillGradient": 0, 5303 "gridPos": { 5304 "h": 4, 5305 "w": 12, 5306 "x": 12, 5307 "y": 12 5308 }, 5309 "hiddenSeries": false, 5310 "id": 224, 5311 "legend": { 5312 "alignAsTable": true, 5313 "avg": false, 5314 "current": true, 5315 "max": true, 5316 "min": false, 5317 "rightSide": false, 5318 "show": true, 5319 "total": false, 5320 "values": true 5321 }, 5322 "lines": true, 5323 "linewidth": 1, 5324 "links": [], 5325 "nullPointMode": "null", 5326 "options": { 5327 "alertThreshold": true 5328 }, 5329 "paceLength": 10, 5330 "percentage": false, 5331 "pluginVersion": "7.5.11", 5332 "pointradius": 2, 5333 "points": false, 5334 "renderer": "flot", 5335 "seriesOverrides": [], 5336 "spaceLength": 10, 5337 "stack": false, 5338 "steppedLine": false, 5339 "targets": [ 5340 { 5341 "exemplar": true, 5342 "expr": "sum(ticdc_mounter_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)", 5343 "format": "time_series", 5344 "interval": "", 5345 "intervalFactor": 1, 5346 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 5347 "refId": "A" 5348 } 5349 ], 5350 "thresholds": [], 5351 "timeFrom": null, 5352 "timeRegions": [], 5353 "timeShift": null, 5354 "title": "Mounter output events", 5355 "tooltip": { 5356 "shared": true, 5357 "sort": 0, 5358 "value_type": "individual" 5359 }, 5360 "type": "graph", 5361 "xaxis": { 5362 "buckets": null, 5363 "mode": "time", 5364 "name": null, 5365 "show": true, 5366 "values": [] 5367 }, 5368 "yaxes": [ 5369 { 5370 "$$hashKey": "object:1568", 5371 "format": "none", 5372 "label": null, 5373 "logBase": 1, 5374 "max": null, 5375 "min": null, 5376 "show": true 5377 }, 5378 { 5379 "$$hashKey": "object:1569", 5380 "format": "short", 5381 "label": null, 5382 "logBase": 1, 5383 "max": null, 5384 "min": null, 5385 "show": true 5386 } 5387 ], 5388 "yaxis": { 5389 "align": false, 5390 "alignLevel": null 5391 } 5392 }, 5393 { 5394 "aliasColors": {}, 5395 "bars": false, 5396 "dashLength": 10, 5397 "dashes": false, 5398 "datasource": "${DS_TEST-CLUSTER}", 5399 "description": "The number of events that table sorter outputs to sink per second", 5400 "fieldConfig": { 5401 "defaults": { 5402 "links": [] 5403 }, 5404 "overrides": [] 5405 }, 5406 "fill": 1, 5407 "fillGradient": 0, 5408 "gridPos": { 5409 "h": 4, 5410 "w": 12, 5411 "x": 0, 5412 "y": 16 5413 }, 5414 "hiddenSeries": false, 5415 "id": 223, 5416 "legend": { 5417 "alignAsTable": true, 5418 "avg": false, 5419 "current": true, 5420 "max": true, 5421 "min": false, 5422 "rightSide": false, 5423 "show": true, 5424 "total": false, 5425 "values": true 5426 }, 5427 "lines": true, 5428 "linewidth": 1, 5429 "links": [], 5430 "nullPointMode": "null", 5431 "options": { 5432 "alertThreshold": true 5433 }, 5434 "paceLength": 10, 5435 "percentage": false, 5436 "pluginVersion": "7.5.11", 5437 "pointradius": 2, 5438 "points": false, 5439 "renderer": "flot", 5440 "seriesOverrides": [], 5441 "spaceLength": 10, 5442 "stack": false, 5443 "steppedLine": false, 5444 "targets": [ 5445 { 5446 "exemplar": true, 5447 "expr": "sum(rate(ticdc_sink_table_sink_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)", 5448 "format": "time_series", 5449 "interval": "", 5450 "intervalFactor": 1, 5451 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 5452 "refId": "A" 5453 } 5454 ], 5455 "thresholds": [], 5456 "timeFrom": null, 5457 "timeRegions": [], 5458 "timeShift": null, 5459 "title": "Table sink output events/s", 5460 "tooltip": { 5461 "shared": true, 5462 "sort": 0, 5463 "value_type": "individual" 5464 }, 5465 "type": "graph", 5466 "xaxis": { 5467 "buckets": null, 5468 "mode": "time", 5469 "name": null, 5470 "show": true, 5471 "values": [] 5472 }, 5473 "yaxes": [ 5474 { 5475 "$$hashKey": "object:1273", 5476 "format": "short", 5477 "label": null, 5478 "logBase": 1, 5479 "max": null, 5480 "min": null, 5481 "show": true 5482 }, 5483 { 5484 "$$hashKey": "object:1274", 5485 "format": "short", 5486 "label": null, 5487 "logBase": 1, 5488 "max": null, 5489 "min": null, 5490 "show": false 5491 } 5492 ], 5493 "yaxis": { 5494 "align": false, 5495 "alignLevel": null 5496 } 5497 }, 5498 { 5499 "aliasColors": {}, 5500 "bars": false, 5501 "dashLength": 10, 5502 "dashes": false, 5503 "datasource": "${DS_TEST-CLUSTER}", 5504 "description": "The total number of events that table sinks emit", 5505 "fieldConfig": { 5506 "defaults": { 5507 "links": [] 5508 }, 5509 "overrides": [] 5510 }, 5511 "fill": 1, 5512 "fillGradient": 0, 5513 "gridPos": { 5514 "h": 4, 5515 "w": 12, 5516 "x": 12, 5517 "y": 16 5518 }, 5519 "hiddenSeries": false, 5520 "id": 221, 5521 "legend": { 5522 "alignAsTable": true, 5523 "avg": false, 5524 "current": true, 5525 "max": true, 5526 "min": false, 5527 "rightSide": false, 5528 "show": true, 5529 "total": false, 5530 "values": true 5531 }, 5532 "lines": true, 5533 "linewidth": 1, 5534 "links": [], 5535 "nullPointMode": "null", 5536 "options": { 5537 "alertThreshold": true 5538 }, 5539 "paceLength": 10, 5540 "percentage": false, 5541 "pluginVersion": "7.5.11", 5542 "pointradius": 2, 5543 "points": false, 5544 "renderer": "flot", 5545 "seriesOverrides": [], 5546 "spaceLength": 10, 5547 "stack": false, 5548 "steppedLine": false, 5549 "targets": [ 5550 { 5551 "exemplar": true, 5552 "expr": "sum(ticdc_sink_table_sink_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)", 5553 "format": "time_series", 5554 "interval": "", 5555 "intervalFactor": 1, 5556 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 5557 "refId": "A" 5558 } 5559 ], 5560 "thresholds": [], 5561 "timeFrom": null, 5562 "timeRegions": [], 5563 "timeShift": null, 5564 "title": "Table sink output events", 5565 "tooltip": { 5566 "shared": true, 5567 "sort": 0, 5568 "value_type": "individual" 5569 }, 5570 "type": "graph", 5571 "xaxis": { 5572 "buckets": null, 5573 "mode": "time", 5574 "name": null, 5575 "show": true, 5576 "values": [] 5577 }, 5578 "yaxes": [ 5579 { 5580 "format": "none", 5581 "label": null, 5582 "logBase": 1, 5583 "max": null, 5584 "min": null, 5585 "show": true 5586 }, 5587 { 5588 "format": "short", 5589 "label": null, 5590 "logBase": 1, 5591 "max": null, 5592 "min": null, 5593 "show": true 5594 } 5595 ], 5596 "yaxis": { 5597 "align": false, 5598 "alignLevel": null 5599 } 5600 }, 5601 { 5602 "aliasColors": {}, 5603 "bars": false, 5604 "dashLength": 10, 5605 "dashes": false, 5606 "datasource": "${DS_TEST-CLUSTER}", 5607 "description": "The number of rows that sink flushes to downstream per second.", 5608 "fieldConfig": { 5609 "defaults": { 5610 "links": [] 5611 }, 5612 "overrides": [] 5613 }, 5614 "fill": 1, 5615 "fillGradient": 0, 5616 "gridPos": { 5617 "h": 4, 5618 "w": 12, 5619 "x": 0, 5620 "y": 20 5621 }, 5622 "hiddenSeries": false, 5623 "id": 664, 5624 "legend": { 5625 "alignAsTable": true, 5626 "avg": false, 5627 "current": true, 5628 "max": true, 5629 "min": false, 5630 "rightSide": false, 5631 "show": true, 5632 "sort": "current", 5633 "sortDesc": true, 5634 "total": false, 5635 "values": true 5636 }, 5637 "lines": true, 5638 "linewidth": 1, 5639 "links": [], 5640 "nullPointMode": "null", 5641 "options": { 5642 "alertThreshold": true 5643 }, 5644 "percentage": false, 5645 "pluginVersion": "7.5.11", 5646 "pointradius": 2, 5647 "points": false, 5648 "renderer": "flot", 5649 "seriesOverrides": [], 5650 "spaceLength": 10, 5651 "stack": false, 5652 "steppedLine": false, 5653 "targets": [ 5654 { 5655 "exemplar": true, 5656 "expr": "sum(rate(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)", 5657 "format": "time_series", 5658 "interval": "", 5659 "intervalFactor": 1, 5660 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 5661 "refId": "A" 5662 } 5663 ], 5664 "thresholds": [], 5665 "timeFrom": null, 5666 "timeRegions": [], 5667 "timeShift": null, 5668 "title": "Sink flush rows/s", 5669 "tooltip": { 5670 "shared": true, 5671 "sort": 2, 5672 "value_type": "individual" 5673 }, 5674 "type": "graph", 5675 "xaxis": { 5676 "buckets": null, 5677 "mode": "time", 5678 "name": null, 5679 "show": true, 5680 "values": [] 5681 }, 5682 "yaxes": [ 5683 { 5684 "$$hashKey": "object:1318", 5685 "format": "short", 5686 "label": null, 5687 "logBase": 1, 5688 "max": null, 5689 "min": null, 5690 "show": true 5691 }, 5692 { 5693 "$$hashKey": "object:1319", 5694 "format": "none", 5695 "label": null, 5696 "logBase": 1, 5697 "max": null, 5698 "min": null, 5699 "show": false 5700 } 5701 ], 5702 "yaxis": { 5703 "align": false, 5704 "alignLevel": null 5705 } 5706 }, 5707 { 5708 "aliasColors": {}, 5709 "bars": false, 5710 "dashLength": 10, 5711 "dashes": false, 5712 "datasource": "${DS_TEST-CLUSTER}", 5713 "description": "The number of rows(events) that are flushed by sink.", 5714 "fieldConfig": { 5715 "defaults": { 5716 "links": [] 5717 }, 5718 "overrides": [] 5719 }, 5720 "fill": 1, 5721 "fillGradient": 0, 5722 "gridPos": { 5723 "h": 4, 5724 "w": 12, 5725 "x": 12, 5726 "y": 20 5727 }, 5728 "hiddenSeries": false, 5729 "id": 665, 5730 "legend": { 5731 "alignAsTable": true, 5732 "avg": false, 5733 "current": true, 5734 "max": true, 5735 "min": false, 5736 "rightSide": false, 5737 "show": true, 5738 "sort": "current", 5739 "sortDesc": true, 5740 "total": false, 5741 "values": true 5742 }, 5743 "lines": true, 5744 "linewidth": 1, 5745 "links": [], 5746 "nullPointMode": "null", 5747 "options": { 5748 "alertThreshold": true 5749 }, 5750 "percentage": false, 5751 "pluginVersion": "7.5.11", 5752 "pointradius": 2, 5753 "points": false, 5754 "renderer": "flot", 5755 "seriesOverrides": [], 5756 "spaceLength": 10, 5757 "stack": false, 5758 "steppedLine": false, 5759 "targets": [ 5760 { 5761 "exemplar": true, 5762 "expr": "sum(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, instance)", 5763 "format": "time_series", 5764 "interval": "", 5765 "intervalFactor": 1, 5766 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 5767 "refId": "A" 5768 } 5769 ], 5770 "thresholds": [], 5771 "timeFrom": null, 5772 "timeRegions": [], 5773 "timeShift": null, 5774 "title": "Sink flush rows", 5775 "tooltip": { 5776 "shared": true, 5777 "sort": 2, 5778 "value_type": "individual" 5779 }, 5780 "type": "graph", 5781 "xaxis": { 5782 "buckets": null, 5783 "mode": "time", 5784 "name": null, 5785 "show": true, 5786 "values": [] 5787 }, 5788 "yaxes": [ 5789 { 5790 "format": "none", 5791 "label": null, 5792 "logBase": 1, 5793 "max": null, 5794 "min": null, 5795 "show": true 5796 }, 5797 { 5798 "format": "short", 5799 "label": null, 5800 "logBase": 1, 5801 "max": null, 5802 "min": null, 5803 "show": true 5804 } 5805 ], 5806 "yaxis": { 5807 "align": false, 5808 "alignLevel": null 5809 } 5810 } 5811 ], 5812 "title": "Dataflow", 5813 "type": "row" 5814 }, 5815 { 5816 "collapsed": true, 5817 "datasource": null, 5818 "gridPos": { 5819 "h": 1, 5820 "w": 24, 5821 "x": 0, 5822 "y": 4 5823 }, 5824 "id": 608, 5825 "panels": [ 5826 { 5827 "aliasColors": {}, 5828 "bars": false, 5829 "dashLength": 10, 5830 "dashes": false, 5831 "datasource": "${DS_TEST-CLUSTER}", 5832 "description": "Changefeed memory quota", 5833 "fieldConfig": { 5834 "defaults": {}, 5835 "overrides": [] 5836 }, 5837 "fill": 1, 5838 "fillGradient": 0, 5839 "gridPos": { 5840 "h": 8, 5841 "w": 12, 5842 "x": 0, 5843 "y": 5 5844 }, 5845 "hiddenSeries": false, 5846 "id": 719, 5847 "legend": { 5848 "avg": false, 5849 "current": false, 5850 "max": true, 5851 "min": false, 5852 "show": true, 5853 "total": false, 5854 "values": true 5855 }, 5856 "lines": true, 5857 "linewidth": 1, 5858 "nullPointMode": "null", 5859 "options": { 5860 "alertThreshold": true 5861 }, 5862 "percentage": false, 5863 "pluginVersion": "7.5.11", 5864 "pointradius": 2, 5865 "points": false, 5866 "renderer": "flot", 5867 "seriesOverrides": [], 5868 "spaceLength": 10, 5869 "stack": false, 5870 "steppedLine": false, 5871 "targets": [ 5872 { 5873 "exemplar": true, 5874 "expr": "ticdc_sinkmanager_memory_quota{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", component!=\"\", instance=~\"$ticdc_instance\"}", 5875 "interval": "", 5876 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-memory-{{component}}-{{type}}", 5877 "refId": "A" 5878 }, 5879 { 5880 "exemplar": true, 5881 "expr": "ticdc_sinkmanager_redo_event_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}", 5882 "hide": false, 5883 "interval": "", 5884 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-redo-cache", 5885 "refId": "B" 5886 } 5887 ], 5888 "thresholds": [], 5889 "timeFrom": null, 5890 "timeRegions": [], 5891 "timeShift": null, 5892 "title": "Memory Quota", 5893 "tooltip": { 5894 "shared": true, 5895 "sort": 0, 5896 "value_type": "individual" 5897 }, 5898 "type": "graph", 5899 "xaxis": { 5900 "buckets": null, 5901 "mode": "time", 5902 "name": null, 5903 "show": true, 5904 "values": [] 5905 }, 5906 "yaxes": [ 5907 { 5908 "format": "bytes", 5909 "label": null, 5910 "logBase": 1, 5911 "max": null, 5912 "min": null, 5913 "show": true 5914 }, 5915 { 5916 "format": "short", 5917 "label": null, 5918 "logBase": 1, 5919 "max": null, 5920 "min": null, 5921 "show": false 5922 } 5923 ], 5924 "yaxis": { 5925 "align": false, 5926 "alignLevel": null 5927 } 5928 }, 5929 { 5930 "aliasColors": {}, 5931 "bars": false, 5932 "dashLength": 10, 5933 "dashes": false, 5934 "datasource": "${DS_TEST-CLUSTER}", 5935 "description": "Ratio of redo event cache hit", 5936 "fieldConfig": { 5937 "defaults": {}, 5938 "overrides": [] 5939 }, 5940 "fill": 1, 5941 "fillGradient": 0, 5942 "gridPos": { 5943 "h": 8, 5944 "w": 12, 5945 "x": 12, 5946 "y": 5 5947 }, 5948 "hiddenSeries": false, 5949 "id": 721, 5950 "legend": { 5951 "avg": false, 5952 "current": false, 5953 "max": true, 5954 "min": false, 5955 "show": true, 5956 "total": false, 5957 "values": true 5958 }, 5959 "lines": true, 5960 "linewidth": 1, 5961 "nullPointMode": "null", 5962 "options": { 5963 "alertThreshold": true 5964 }, 5965 "percentage": false, 5966 "pluginVersion": "7.5.11", 5967 "pointradius": 2, 5968 "points": false, 5969 "renderer": "flot", 5970 "seriesOverrides": [], 5971 "spaceLength": 10, 5972 "stack": false, 5973 "steppedLine": false, 5974 "targets": [ 5975 { 5976 "exemplar": true, 5977 "expr": "sum(rate(ticdc_sinkmanager_redo_event_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", type=\"hit\"}[1m])) /\nsum(rate(ticdc_sinkmanager_redo_event_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[1m]))", 5978 "interval": "", 5979 "legendFormat": "hit-ratio", 5980 "queryType": "randomWalk", 5981 "refId": "A" 5982 } 5983 ], 5984 "thresholds": [], 5985 "timeFrom": null, 5986 "timeRegions": [], 5987 "timeShift": null, 5988 "title": "Redo Event Cache Hit Ratio", 5989 "tooltip": { 5990 "shared": true, 5991 "sort": 0, 5992 "value_type": "individual" 5993 }, 5994 "type": "graph", 5995 "xaxis": { 5996 "buckets": null, 5997 "mode": "time", 5998 "name": null, 5999 "show": true, 6000 "values": [] 6001 }, 6002 "yaxes": [ 6003 { 6004 "format": "percentunit", 6005 "label": null, 6006 "logBase": 1, 6007 "max": null, 6008 "min": null, 6009 "show": true 6010 }, 6011 { 6012 "format": "short", 6013 "label": null, 6014 "logBase": 1, 6015 "max": null, 6016 "min": null, 6017 "show": true 6018 } 6019 ], 6020 "yaxis": { 6021 "align": false, 6022 "alignLevel": null 6023 } 6024 }, 6025 { 6026 "aliasColors": {}, 6027 "bars": false, 6028 "dashLength": 10, 6029 "dashes": false, 6030 "datasource": "${DS_TEST-CLUSTER}", 6031 "description": "Row count for batch to the downstream sink.", 6032 "fieldConfig": { 6033 "defaults": {}, 6034 "overrides": [] 6035 }, 6036 "fill": 1, 6037 "fillGradient": 0, 6038 "gridPos": { 6039 "h": 8, 6040 "w": 12, 6041 "x": 0, 6042 "y": 13 6043 }, 6044 "hiddenSeries": false, 6045 "id": 610, 6046 "legend": { 6047 "alignAsTable": true, 6048 "avg": false, 6049 "current": true, 6050 "max": false, 6051 "min": false, 6052 "rightSide": false, 6053 "show": true, 6054 "total": false, 6055 "values": true 6056 }, 6057 "lines": true, 6058 "linewidth": 1, 6059 "nullPointMode": "null", 6060 "options": { 6061 "alertThreshold": true 6062 }, 6063 "percentage": false, 6064 "pluginVersion": "7.5.11", 6065 "pointradius": 2, 6066 "points": false, 6067 "renderer": "flot", 6068 "seriesOverrides": [], 6069 "spaceLength": 10, 6070 "stack": false, 6071 "steppedLine": false, 6072 "targets": [ 6073 { 6074 "exemplar": true, 6075 "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_batch_row_count_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed,instance))", 6076 "interval": "", 6077 "legendFormat": "99.9%-{{namespace}}-{{changefeed}}-{{instance}}", 6078 "queryType": "randomWalk", 6079 "refId": "A" 6080 }, 6081 { 6082 "exemplar": true, 6083 "expr": "sum(rate(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance) / sum(rate(ticdc_sinkv2_batch_row_count_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)", 6084 "hide": false, 6085 "interval": "", 6086 "legendFormat": "avg-{{namespace}}-{{changefeed}}-{{instance}}", 6087 "refId": "B" 6088 } 6089 ], 6090 "thresholds": [], 6091 "timeFrom": null, 6092 "timeRegions": [], 6093 "timeShift": null, 6094 "title": "Output Row Batch Size", 6095 "tooltip": { 6096 "shared": true, 6097 "sort": 0, 6098 "value_type": "individual" 6099 }, 6100 "type": "graph", 6101 "xaxis": { 6102 "buckets": null, 6103 "mode": "time", 6104 "name": null, 6105 "show": true, 6106 "values": [] 6107 }, 6108 "yaxes": [ 6109 { 6110 "format": "short", 6111 "label": null, 6112 "logBase": 1, 6113 "max": null, 6114 "min": null, 6115 "show": true 6116 }, 6117 { 6118 "format": "short", 6119 "label": null, 6120 "logBase": 1, 6121 "max": null, 6122 "min": null, 6123 "show": false 6124 } 6125 ], 6126 "yaxis": { 6127 "align": false, 6128 "alignLevel": null 6129 } 6130 }, 6131 { 6132 "aliasColors": {}, 6133 "bars": false, 6134 "dashLength": 10, 6135 "dashes": false, 6136 "datasource": "${DS_TEST-CLUSTER}", 6137 "description": "Row count for total output rows.", 6138 "fieldConfig": { 6139 "defaults": {}, 6140 "overrides": [] 6141 }, 6142 "fill": 1, 6143 "fillGradient": 0, 6144 "gridPos": { 6145 "h": 8, 6146 "w": 12, 6147 "x": 12, 6148 "y": 13 6149 }, 6150 "hiddenSeries": false, 6151 "id": 612, 6152 "legend": { 6153 "alignAsTable": true, 6154 "avg": false, 6155 "current": true, 6156 "max": false, 6157 "min": false, 6158 "rightSide": false, 6159 "show": true, 6160 "total": false, 6161 "values": true 6162 }, 6163 "lines": true, 6164 "linewidth": 1, 6165 "nullPointMode": "null", 6166 "options": { 6167 "alertThreshold": true 6168 }, 6169 "percentage": false, 6170 "pluginVersion": "7.5.11", 6171 "pointradius": 2, 6172 "points": false, 6173 "renderer": "flot", 6174 "seriesOverrides": [], 6175 "spaceLength": 10, 6176 "stack": false, 6177 "steppedLine": false, 6178 "targets": [ 6179 { 6180 "exemplar": true, 6181 "expr": "sum(rate(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)", 6182 "interval": "", 6183 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 6184 "queryType": "randomWalk", 6185 "refId": "A" 6186 } 6187 ], 6188 "thresholds": [], 6189 "timeFrom": null, 6190 "timeRegions": [], 6191 "timeShift": null, 6192 "title": "Output Row Count (per second)", 6193 "tooltip": { 6194 "shared": true, 6195 "sort": 0, 6196 "value_type": "individual" 6197 }, 6198 "type": "graph", 6199 "xaxis": { 6200 "buckets": null, 6201 "mode": "time", 6202 "name": null, 6203 "show": true, 6204 "values": [] 6205 }, 6206 "yaxes": [ 6207 { 6208 "format": "short", 6209 "label": null, 6210 "logBase": 1, 6211 "max": null, 6212 "min": null, 6213 "show": true 6214 }, 6215 { 6216 "format": "short", 6217 "label": null, 6218 "logBase": 1, 6219 "max": null, 6220 "min": null, 6221 "show": false 6222 } 6223 ], 6224 "yaxis": { 6225 "align": false, 6226 "alignLevel": null 6227 } 6228 }, 6229 { 6230 "aliasColors": {}, 6231 "bars": false, 6232 "dashLength": 10, 6233 "dashes": false, 6234 "datasource": "${DS_TEST-CLUSTER}", 6235 "description": "DDL count and executing duration", 6236 "fieldConfig": { 6237 "defaults": {}, 6238 "overrides": [] 6239 }, 6240 "fill": 1, 6241 "fillGradient": 0, 6242 "gridPos": { 6243 "h": 8, 6244 "w": 12, 6245 "x": 0, 6246 "y": 21 6247 }, 6248 "hiddenSeries": false, 6249 "id": 613, 6250 "legend": { 6251 "alignAsTable": true, 6252 "avg": false, 6253 "current": true, 6254 "max": false, 6255 "min": false, 6256 "rightSide": false, 6257 "show": true, 6258 "total": false, 6259 "values": true 6260 }, 6261 "lines": true, 6262 "linewidth": 1, 6263 "nullPointMode": "null", 6264 "options": { 6265 "alertThreshold": true 6266 }, 6267 "percentage": false, 6268 "pluginVersion": "7.5.11", 6269 "pointradius": 2, 6270 "points": false, 6271 "renderer": "flot", 6272 "seriesOverrides": [ 6273 { 6274 "alias": "/duration/", 6275 "yaxis": 2 6276 } 6277 ], 6278 "spaceLength": 10, 6279 "stack": false, 6280 "steppedLine": false, 6281 "targets": [ 6282 { 6283 "exemplar": true, 6284 "expr": "sum(rate(ticdc_sink_ddl_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)", 6285 "interval": "", 6286 "legendFormat": "count-{{namespace}}-{{changefeed}}-{{instance}}", 6287 "queryType": "randomWalk", 6288 "refId": "A" 6289 }, 6290 { 6291 "exemplar": true, 6292 "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_ddl_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed,instance))", 6293 "hide": false, 6294 "interval": "", 6295 "legendFormat": "99.9-duration-{{namespace}}-{{changefeed}}-{{instance}}", 6296 "refId": "B" 6297 } 6298 ], 6299 "thresholds": [], 6300 "timeFrom": null, 6301 "timeRegions": [], 6302 "timeShift": null, 6303 "title": "Output DDL Count & Executing Duration", 6304 "tooltip": { 6305 "shared": true, 6306 "sort": 0, 6307 "value_type": "individual" 6308 }, 6309 "type": "graph", 6310 "xaxis": { 6311 "buckets": null, 6312 "mode": "time", 6313 "name": null, 6314 "show": true, 6315 "values": [] 6316 }, 6317 "yaxes": [ 6318 { 6319 "format": "short", 6320 "label": null, 6321 "logBase": 1, 6322 "max": null, 6323 "min": null, 6324 "show": true 6325 }, 6326 { 6327 "format": "s", 6328 "label": null, 6329 "logBase": 1, 6330 "max": null, 6331 "min": null, 6332 "show": true 6333 } 6334 ], 6335 "yaxis": { 6336 "align": false, 6337 "alignLevel": null 6338 } 6339 }, 6340 { 6341 "aliasColors": {}, 6342 "bars": false, 6343 "dashLength": 10, 6344 "dashes": false, 6345 "datasource": "${DS_TEST-CLUSTER}", 6346 "description": "Size of large rows (size >= 2K).", 6347 "fieldConfig": { 6348 "defaults": {}, 6349 "overrides": [] 6350 }, 6351 "fill": 1, 6352 "fillGradient": 0, 6353 "gridPos": { 6354 "h": 8, 6355 "w": 12, 6356 "x": 12, 6357 "y": 21 6358 }, 6359 "hiddenSeries": false, 6360 "id": 611, 6361 "legend": { 6362 "alignAsTable": true, 6363 "avg": false, 6364 "current": true, 6365 "max": false, 6366 "min": false, 6367 "rightSide": false, 6368 "show": true, 6369 "total": false, 6370 "values": true 6371 }, 6372 "lines": true, 6373 "linewidth": 1, 6374 "nullPointMode": "null", 6375 "options": { 6376 "alertThreshold": true 6377 }, 6378 "percentage": false, 6379 "pluginVersion": "7.5.11", 6380 "pointradius": 2, 6381 "points": false, 6382 "renderer": "flot", 6383 "seriesOverrides": [], 6384 "spaceLength": 10, 6385 "stack": false, 6386 "steppedLine": false, 6387 "targets": [ 6388 { 6389 "exemplar": true, 6390 "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_large_row_size_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed,instance))", 6391 "interval": "", 6392 "legendFormat": "99.9%-{{namespace}}-{{changefeed}}-{{instance}}", 6393 "queryType": "randomWalk", 6394 "refId": "A" 6395 }, 6396 { 6397 "exemplar": true, 6398 "expr": "sum(rate(ticdc_sink_large_row_size_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance) / sum(rate(ticdc_sink_large_row_size_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)", 6399 "hide": false, 6400 "interval": "", 6401 "legendFormat": "avg-{{namespace}}-{{changefeed}}-{{instance}}", 6402 "refId": "B" 6403 } 6404 ], 6405 "thresholds": [], 6406 "timeFrom": null, 6407 "timeRegions": [], 6408 "timeShift": null, 6409 "title": "Large Row (size >= 128K) Size", 6410 "tooltip": { 6411 "shared": true, 6412 "sort": 0, 6413 "value_type": "individual" 6414 }, 6415 "type": "graph", 6416 "xaxis": { 6417 "buckets": null, 6418 "mode": "time", 6419 "name": null, 6420 "show": true, 6421 "values": [] 6422 }, 6423 "yaxes": [ 6424 { 6425 "format": "short", 6426 "label": null, 6427 "logBase": 2, 6428 "max": null, 6429 "min": null, 6430 "show": true 6431 }, 6432 { 6433 "format": "short", 6434 "label": null, 6435 "logBase": 1, 6436 "max": null, 6437 "min": null, 6438 "show": false 6439 } 6440 ], 6441 "yaxis": { 6442 "align": false, 6443 "alignLevel": null 6444 } 6445 }, 6446 { 6447 "aliasColors": {}, 6448 "bars": false, 6449 "dashLength": 10, 6450 "dashes": false, 6451 "datasource": "${DS_TEST-CLUSTER}", 6452 "description": "Count of errors in the last minute.", 6453 "fieldConfig": { 6454 "defaults": {}, 6455 "overrides": [] 6456 }, 6457 "fill": 1, 6458 "fillGradient": 0, 6459 "gridPos": { 6460 "h": 8, 6461 "w": 12, 6462 "x": 0, 6463 "y": 29 6464 }, 6465 "hiddenSeries": false, 6466 "id": 614, 6467 "legend": { 6468 "alignAsTable": true, 6469 "avg": false, 6470 "current": true, 6471 "max": false, 6472 "min": false, 6473 "rightSide": false, 6474 "show": true, 6475 "total": false, 6476 "values": true 6477 }, 6478 "lines": true, 6479 "linewidth": 1, 6480 "nullPointMode": "null", 6481 "options": { 6482 "alertThreshold": true 6483 }, 6484 "percentage": false, 6485 "pluginVersion": "7.5.11", 6486 "pointradius": 2, 6487 "points": false, 6488 "renderer": "flot", 6489 "seriesOverrides": [], 6490 "spaceLength": 10, 6491 "stack": false, 6492 "steppedLine": false, 6493 "targets": [ 6494 { 6495 "exemplar": true, 6496 "expr": "sum(delta(ticdc_sink_execution_error{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)", 6497 "interval": "", 6498 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 6499 "queryType": "randomWalk", 6500 "refId": "A" 6501 } 6502 ], 6503 "thresholds": [], 6504 "timeFrom": null, 6505 "timeRegions": [], 6506 "timeShift": null, 6507 "title": "Sink Error Count (1min)", 6508 "tooltip": { 6509 "shared": true, 6510 "sort": 0, 6511 "value_type": "individual" 6512 }, 6513 "type": "graph", 6514 "xaxis": { 6515 "buckets": null, 6516 "mode": "time", 6517 "name": null, 6518 "show": true, 6519 "values": [] 6520 }, 6521 "yaxes": [ 6522 { 6523 "format": "short", 6524 "label": null, 6525 "logBase": 1, 6526 "max": null, 6527 "min": null, 6528 "show": true 6529 }, 6530 { 6531 "format": "short", 6532 "label": null, 6533 "logBase": 1, 6534 "max": null, 6535 "min": null, 6536 "show": false 6537 } 6538 ], 6539 "yaxis": { 6540 "align": false, 6541 "alignLevel": null 6542 } 6543 } 6544 ], 6545 "title": "Sink - General", 6546 "type": "row" 6547 }, 6548 { 6549 "collapsed": true, 6550 "datasource": null, 6551 "gridPos": { 6552 "h": 1, 6553 "w": 24, 6554 "x": 0, 6555 "y": 5 6556 }, 6557 "id": 616, 6558 "panels": [ 6559 { 6560 "aliasColors": {}, 6561 "bars": false, 6562 "dashLength": 10, 6563 "dashes": false, 6564 "datasource": "${DS_TEST-CLUSTER}", 6565 "description": "Duration of event staying in conflict detector", 6566 "fieldConfig": { 6567 "defaults": {}, 6568 "overrides": [] 6569 }, 6570 "fill": 1, 6571 "fillGradient": 0, 6572 "gridPos": { 6573 "h": 8, 6574 "w": 12, 6575 "x": 0, 6576 "y": 26 6577 }, 6578 "hiddenSeries": false, 6579 "id": 618, 6580 "legend": { 6581 "alignAsTable": true, 6582 "avg": false, 6583 "current": true, 6584 "max": false, 6585 "min": false, 6586 "rightSide": false, 6587 "show": true, 6588 "total": false, 6589 "values": true 6590 }, 6591 "lines": true, 6592 "linewidth": 1, 6593 "nullPointMode": "null", 6594 "options": { 6595 "alertThreshold": true 6596 }, 6597 "percentage": false, 6598 "pluginVersion": "7.5.11", 6599 "pointradius": 2, 6600 "points": false, 6601 "renderer": "flot", 6602 "seriesOverrides": [], 6603 "spaceLength": 10, 6604 "stack": false, 6605 "steppedLine": false, 6606 "targets": [ 6607 { 6608 "exemplar": true, 6609 "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_conflict_detect_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,changefeed,instance))", 6610 "interval": "", 6611 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-detect-P999", 6612 "queryType": "randomWalk", 6613 "refId": "C" 6614 }, 6615 { 6616 "exemplar": true, 6617 "expr": "sum(rate(ticdc_sink_txn_conflict_detect_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (changefeed,instance) / \nsum(rate(ticdc_sink_txn_conflict_detect_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (changefeed,instance)", 6618 "hide": false, 6619 "interval": "", 6620 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-detect-avg", 6621 "refId": "D" 6622 }, 6623 { 6624 "exemplar": true, 6625 "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_queue_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed,instance))", 6626 "interval": "", 6627 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-queue-P999", 6628 "queryType": "randomWalk", 6629 "refId": "A" 6630 }, 6631 { 6632 "exemplar": true, 6633 "expr": "sum(rate(ticdc_sink_txn_queue_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_txn_queue_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)", 6634 "hide": false, 6635 "interval": "", 6636 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-queue-avg", 6637 "refId": "B" 6638 } 6639 ], 6640 "thresholds": [], 6641 "timeFrom": null, 6642 "timeRegions": [], 6643 "timeShift": null, 6644 "title": "Conflict Detect Duration", 6645 "tooltip": { 6646 "shared": true, 6647 "sort": 0, 6648 "value_type": "individual" 6649 }, 6650 "type": "graph", 6651 "xaxis": { 6652 "buckets": null, 6653 "mode": "time", 6654 "name": null, 6655 "show": true, 6656 "values": [] 6657 }, 6658 "yaxes": [ 6659 { 6660 "format": "s", 6661 "label": null, 6662 "logBase": 2, 6663 "max": null, 6664 "min": null, 6665 "show": true 6666 }, 6667 { 6668 "format": "short", 6669 "label": null, 6670 "logBase": 1, 6671 "max": null, 6672 "min": null, 6673 "show": false 6674 } 6675 ], 6676 "yaxis": { 6677 "align": false, 6678 "alignLevel": null 6679 } 6680 }, 6681 { 6682 "aliasColors": {}, 6683 "bars": false, 6684 "dashLength": 10, 6685 "dashes": false, 6686 "datasource": "${DS_TEST-CLUSTER}", 6687 "description": "Full flush (backend flush + callback + conflict detector notify) duration", 6688 "fieldConfig": { 6689 "defaults": {}, 6690 "overrides": [] 6691 }, 6692 "fill": 1, 6693 "fillGradient": 0, 6694 "gridPos": { 6695 "h": 8, 6696 "w": 12, 6697 "x": 12, 6698 "y": 26 6699 }, 6700 "hiddenSeries": false, 6701 "id": 620, 6702 "legend": { 6703 "alignAsTable": true, 6704 "avg": false, 6705 "current": true, 6706 "max": false, 6707 "min": false, 6708 "rightSide": false, 6709 "show": true, 6710 "total": false, 6711 "values": true 6712 }, 6713 "lines": true, 6714 "linewidth": 1, 6715 "nullPointMode": "null", 6716 "options": { 6717 "alertThreshold": true 6718 }, 6719 "percentage": false, 6720 "pluginVersion": "7.5.11", 6721 "pointradius": 2, 6722 "points": false, 6723 "renderer": "flot", 6724 "seriesOverrides": [], 6725 "spaceLength": 10, 6726 "stack": false, 6727 "steppedLine": false, 6728 "targets": [ 6729 { 6730 "exemplar": true, 6731 "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_worker_flush_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed,instance))", 6732 "interval": "", 6733 "legendFormat": "99.9-{{namespace}}-{{changefeed}}-{{instance}}", 6734 "queryType": "randomWalk", 6735 "refId": "A" 6736 }, 6737 { 6738 "exemplar": true, 6739 "expr": "sum(rate(ticdc_sink_txn_worker_flush_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_txn_worker_flush_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)", 6740 "hide": false, 6741 "interval": "", 6742 "legendFormat": "avg-{{namespace}}-{{changefeed}}-{{instance}}", 6743 "refId": "B" 6744 } 6745 ], 6746 "thresholds": [], 6747 "timeFrom": null, 6748 "timeRegions": [], 6749 "timeShift": null, 6750 "title": "Full Flush Duration", 6751 "tooltip": { 6752 "shared": true, 6753 "sort": 0, 6754 "value_type": "individual" 6755 }, 6756 "type": "graph", 6757 "xaxis": { 6758 "buckets": null, 6759 "mode": "time", 6760 "name": null, 6761 "show": true, 6762 "values": [] 6763 }, 6764 "yaxes": [ 6765 { 6766 "format": "s", 6767 "label": null, 6768 "logBase": 2, 6769 "max": null, 6770 "min": null, 6771 "show": true 6772 }, 6773 { 6774 "format": "short", 6775 "label": null, 6776 "logBase": 1, 6777 "max": null, 6778 "min": null, 6779 "show": false 6780 } 6781 ], 6782 "yaxis": { 6783 "align": false, 6784 "alignLevel": null 6785 } 6786 }, 6787 { 6788 "aliasColors": {}, 6789 "bars": false, 6790 "dashLength": 10, 6791 "dashes": false, 6792 "datasource": "${DS_TEST-CLUSTER}", 6793 "description": "Sink worker busy ratio", 6794 "fieldConfig": { 6795 "defaults": {}, 6796 "overrides": [] 6797 }, 6798 "fill": 1, 6799 "fillGradient": 0, 6800 "gridPos": { 6801 "h": 8, 6802 "w": 12, 6803 "x": 0, 6804 "y": 34 6805 }, 6806 "hiddenSeries": false, 6807 "id": 619, 6808 "legend": { 6809 "alignAsTable": true, 6810 "avg": false, 6811 "current": true, 6812 "max": false, 6813 "min": false, 6814 "rightSide": false, 6815 "show": true, 6816 "total": false, 6817 "values": true 6818 }, 6819 "lines": true, 6820 "linewidth": 1, 6821 "nullPointMode": "null", 6822 "options": { 6823 "alertThreshold": true 6824 }, 6825 "percentage": false, 6826 "pluginVersion": "7.5.11", 6827 "pointradius": 2, 6828 "points": false, 6829 "renderer": "flot", 6830 "seriesOverrides": [], 6831 "spaceLength": 10, 6832 "stack": false, 6833 "steppedLine": false, 6834 "targets": [ 6835 { 6836 "exemplar": true, 6837 "expr": "sum(rate(ticdc_sink_txn_worker_flush_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance,id) /sum(rate(ticdc_sink_txn_worker_total_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance,id) *100", 6838 "interval": "", 6839 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-worker-{{id}}", 6840 "queryType": "randomWalk", 6841 "refId": "A" 6842 } 6843 ], 6844 "thresholds": [], 6845 "timeFrom": null, 6846 "timeRegions": [], 6847 "timeShift": null, 6848 "title": "Worker Busy Ratio", 6849 "tooltip": { 6850 "shared": true, 6851 "sort": 0, 6852 "value_type": "individual" 6853 }, 6854 "type": "graph", 6855 "xaxis": { 6856 "buckets": null, 6857 "mode": "time", 6858 "name": null, 6859 "show": true, 6860 "values": [] 6861 }, 6862 "yaxes": [ 6863 { 6864 "format": "percent", 6865 "label": null, 6866 "logBase": 1, 6867 "max": null, 6868 "min": null, 6869 "show": true 6870 }, 6871 { 6872 "format": "short", 6873 "label": null, 6874 "logBase": 1, 6875 "max": null, 6876 "min": null, 6877 "show": false 6878 } 6879 ], 6880 "yaxis": { 6881 "align": false, 6882 "alignLevel": null 6883 } 6884 }, 6885 { 6886 "aliasColors": {}, 6887 "bars": false, 6888 "dashLength": 10, 6889 "dashes": false, 6890 "datasource": "${DS_TEST-CLUSTER}", 6891 "description": "", 6892 "fieldConfig": { 6893 "defaults": {}, 6894 "overrides": [] 6895 }, 6896 "fill": 1, 6897 "fillGradient": 0, 6898 "gridPos": { 6899 "h": 8, 6900 "w": 12, 6901 "x": 12, 6902 "y": 34 6903 }, 6904 "hiddenSeries": false, 6905 "id": 634, 6906 "legend": { 6907 "alignAsTable": true, 6908 "avg": false, 6909 "current": true, 6910 "max": false, 6911 "min": false, 6912 "show": true, 6913 "total": false, 6914 "values": true 6915 }, 6916 "lines": true, 6917 "linewidth": 1, 6918 "nullPointMode": "null", 6919 "options": { 6920 "alertThreshold": true 6921 }, 6922 "percentage": false, 6923 "pluginVersion": "7.5.11", 6924 "pointradius": 2, 6925 "points": false, 6926 "renderer": "flot", 6927 "seriesOverrides": [], 6928 "spaceLength": 10, 6929 "stack": false, 6930 "steppedLine": false, 6931 "targets": [ 6932 { 6933 "exemplar": true, 6934 "expr": "sum(rate(ticdc_sink_txn_worker_handled_rows{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance,id)", 6935 "interval": "", 6936 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{id}}", 6937 "queryType": "randomWalk", 6938 "refId": "A" 6939 } 6940 ], 6941 "thresholds": [], 6942 "timeFrom": null, 6943 "timeRegions": [], 6944 "timeShift": null, 6945 "title": "Worker Input Rows", 6946 "tooltip": { 6947 "shared": true, 6948 "sort": 0, 6949 "value_type": "individual" 6950 }, 6951 "type": "graph", 6952 "xaxis": { 6953 "buckets": null, 6954 "mode": "time", 6955 "name": null, 6956 "show": true, 6957 "values": [] 6958 }, 6959 "yaxes": [ 6960 { 6961 "format": "short", 6962 "label": null, 6963 "logBase": 1, 6964 "max": null, 6965 "min": null, 6966 "show": true 6967 }, 6968 { 6969 "format": "short", 6970 "label": null, 6971 "logBase": 1, 6972 "max": null, 6973 "min": null, 6974 "show": false 6975 } 6976 ], 6977 "yaxis": { 6978 "align": false, 6979 "alignLevel": null 6980 } 6981 }, 6982 { 6983 "aliasColors": {}, 6984 "bars": false, 6985 "dashLength": 10, 6986 "dashes": false, 6987 "datasource": "${DS_TEST-CLUSTER}", 6988 "description": "Distribution of flush transaction duration to backend", 6989 "fieldConfig": { 6990 "defaults": {}, 6991 "overrides": [] 6992 }, 6993 "fill": 1, 6994 "fillGradient": 0, 6995 "gridPos": { 6996 "h": 8, 6997 "w": 12, 6998 "x": 0, 6999 "y": 42 7000 }, 7001 "hiddenSeries": false, 7002 "id": 635, 7003 "legend": { 7004 "alignAsTable": true, 7005 "avg": false, 7006 "current": true, 7007 "max": false, 7008 "min": false, 7009 "show": true, 7010 "total": false, 7011 "values": true 7012 }, 7013 "lines": true, 7014 "linewidth": 1, 7015 "nullPointMode": "null", 7016 "options": { 7017 "alertThreshold": true 7018 }, 7019 "percentage": false, 7020 "pluginVersion": "7.5.11", 7021 "pointradius": 2, 7022 "points": false, 7023 "renderer": "flot", 7024 "seriesOverrides": [], 7025 "spaceLength": 10, 7026 "stack": false, 7027 "steppedLine": false, 7028 "targets": [ 7029 { 7030 "exemplar": true, 7031 "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_sink_dml_batch_commit_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed,instance))", 7032 "interval": "", 7033 "legendFormat": "99.9-{{namespace}}-{{changefeed}}-{{instance}}", 7034 "queryType": "randomWalk", 7035 "refId": "A" 7036 }, 7037 { 7038 "exemplar": true, 7039 "expr": "sum(rate(ticdc_sink_txn_sink_dml_batch_commit_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_txn_sink_dml_batch_commit_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)", 7040 "hide": false, 7041 "interval": "", 7042 "legendFormat": "avg-{{namespace}}-{{changefeed}}-{{instance}}", 7043 "refId": "B" 7044 } 7045 ], 7046 "thresholds": [], 7047 "timeFrom": null, 7048 "timeRegions": [], 7049 "timeShift": null, 7050 "title": "Backend Flush Duration", 7051 "tooltip": { 7052 "shared": true, 7053 "sort": 0, 7054 "value_type": "individual" 7055 }, 7056 "type": "graph", 7057 "xaxis": { 7058 "buckets": null, 7059 "mode": "time", 7060 "name": null, 7061 "show": true, 7062 "values": [] 7063 }, 7064 "yaxes": [ 7065 { 7066 "format": "s", 7067 "label": null, 7068 "logBase": 1, 7069 "max": null, 7070 "min": null, 7071 "show": true 7072 }, 7073 { 7074 "format": "short", 7075 "label": null, 7076 "logBase": 1, 7077 "max": null, 7078 "min": null, 7079 "show": false 7080 } 7081 ], 7082 "yaxis": { 7083 "align": false, 7084 "alignLevel": null 7085 } 7086 } 7087 ], 7088 "title": "Sink - Transaction Sink", 7089 "type": "row" 7090 }, 7091 { 7092 "collapsed": true, 7093 "datasource": null, 7094 "gridPos": { 7095 "h": 1, 7096 "w": 24, 7097 "x": 0, 7098 "y": 6 7099 }, 7100 "id": 623, 7101 "panels": [ 7102 { 7103 "aliasColors": {}, 7104 "bars": false, 7105 "dashLength": 10, 7106 "dashes": false, 7107 "datasource": "${DS_TEST-CLUSTER}", 7108 "description": "MQ worker send messages to Kafka, this metric record the time cost on send every message.", 7109 "fieldConfig": { 7110 "defaults": {}, 7111 "overrides": [] 7112 }, 7113 "fill": 1, 7114 "fillGradient": 0, 7115 "gridPos": { 7116 "h": 7, 7117 "w": 12, 7118 "x": 0, 7119 "y": 7 7120 }, 7121 "hiddenSeries": false, 7122 "id": 653, 7123 "legend": { 7124 "alignAsTable": true, 7125 "avg": true, 7126 "current": true, 7127 "max": true, 7128 "min": false, 7129 "rightSide": false, 7130 "show": true, 7131 "total": false, 7132 "values": true 7133 }, 7134 "lines": true, 7135 "linewidth": 1, 7136 "nullPointMode": "null", 7137 "options": { 7138 "alertThreshold": true 7139 }, 7140 "percentage": false, 7141 "pluginVersion": "7.5.11", 7142 "pointradius": 2, 7143 "points": false, 7144 "renderer": "flot", 7145 "seriesOverrides": [], 7146 "spaceLength": 10, 7147 "stack": false, 7148 "steppedLine": false, 7149 "targets": [ 7150 { 7151 "exemplar": true, 7152 "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_worker_send_message_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed,instance))", 7153 "interval": "", 7154 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-P999", 7155 "queryType": "randomWalk", 7156 "refId": "A" 7157 }, 7158 { 7159 "exemplar": true, 7160 "expr": "sum(rate(ticdc_sink_mq_worker_send_message_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_mq_worker_send_message_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)", 7161 "hide": false, 7162 "interval": "", 7163 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-avg", 7164 "refId": "B" 7165 } 7166 ], 7167 "thresholds": [], 7168 "timeFrom": null, 7169 "timeRegions": [], 7170 "timeShift": null, 7171 "title": "Worker Send Message Duration Percentile", 7172 "tooltip": { 7173 "shared": true, 7174 "sort": 0, 7175 "value_type": "individual" 7176 }, 7177 "type": "graph", 7178 "xaxis": { 7179 "buckets": null, 7180 "mode": "time", 7181 "name": null, 7182 "show": true, 7183 "values": [] 7184 }, 7185 "yaxes": [ 7186 { 7187 "format": "s", 7188 "label": null, 7189 "logBase": 2, 7190 "max": null, 7191 "min": null, 7192 "show": true 7193 }, 7194 { 7195 "format": "short", 7196 "label": null, 7197 "logBase": 1, 7198 "max": null, 7199 "min": null, 7200 "show": false 7201 } 7202 ], 7203 "yaxis": { 7204 "align": false, 7205 "alignLevel": null 7206 } 7207 }, 7208 { 7209 "aliasColors": {}, 7210 "bars": false, 7211 "dashLength": 10, 7212 "dashes": false, 7213 "datasource": "${DS_TEST-CLUSTER}", 7214 "description": "Bytes/second written off all brokers.\nvalue = one-minute moving average rate of Bytes per second", 7215 "fieldConfig": { 7216 "defaults": { 7217 "links": [] 7218 }, 7219 "overrides": [] 7220 }, 7221 "fill": 1, 7222 "fillGradient": 0, 7223 "gridPos": { 7224 "h": 7, 7225 "w": 12, 7226 "x": 12, 7227 "y": 7 7228 }, 7229 "hiddenSeries": false, 7230 "id": 628, 7231 "legend": { 7232 "alignAsTable": true, 7233 "avg": true, 7234 "current": true, 7235 "max": true, 7236 "min": false, 7237 "rightSide": false, 7238 "show": true, 7239 "total": false, 7240 "values": true 7241 }, 7242 "lines": true, 7243 "linewidth": 1, 7244 "links": [], 7245 "nullPointMode": "null", 7246 "options": { 7247 "alertThreshold": true 7248 }, 7249 "paceLength": 10, 7250 "percentage": false, 7251 "pluginVersion": "7.5.11", 7252 "pointradius": 2, 7253 "points": false, 7254 "renderer": "flot", 7255 "seriesOverrides": [], 7256 "spaceLength": 10, 7257 "stack": false, 7258 "steppedLine": false, 7259 "targets": [ 7260 { 7261 "exemplar": true, 7262 "expr": "sum(ticdc_sink_kafka_producer_outgoing_byte_rate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namesapce=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)", 7263 "format": "time_series", 7264 "interval": "", 7265 "intervalFactor": 1, 7266 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}", 7267 "refId": "A" 7268 } 7269 ], 7270 "thresholds": [], 7271 "timeFrom": null, 7272 "timeRegions": [], 7273 "timeShift": null, 7274 "title": "Kafka Outgoing Bytes", 7275 "tooltip": { 7276 "shared": true, 7277 "sort": 0, 7278 "value_type": "individual" 7279 }, 7280 "type": "graph", 7281 "xaxis": { 7282 "buckets": null, 7283 "mode": "time", 7284 "name": null, 7285 "show": true, 7286 "values": [] 7287 }, 7288 "yaxes": [ 7289 { 7290 "format": "bytes", 7291 "label": null, 7292 "logBase": 1, 7293 "max": null, 7294 "min": "0", 7295 "show": true 7296 }, 7297 { 7298 "format": "short", 7299 "label": null, 7300 "logBase": 1, 7301 "max": null, 7302 "min": null, 7303 "show": false 7304 } 7305 ], 7306 "yaxis": { 7307 "align": false, 7308 "alignLevel": null 7309 } 7310 }, 7311 { 7312 "aliasColors": {}, 7313 "bars": false, 7314 "dashLength": 10, 7315 "dashes": false, 7316 "datasource": "${DS_TEST-CLUSTER}", 7317 "description": "The current number of in-flight requests awaiting a response for all brokers.", 7318 "fieldConfig": { 7319 "defaults": { 7320 "links": [] 7321 }, 7322 "overrides": [] 7323 }, 7324 "fill": 1, 7325 "fillGradient": 0, 7326 "gridPos": { 7327 "h": 7, 7328 "w": 12, 7329 "x": 0, 7330 "y": 14 7331 }, 7332 "hiddenSeries": false, 7333 "id": 627, 7334 "legend": { 7335 "alignAsTable": true, 7336 "avg": true, 7337 "current": true, 7338 "max": true, 7339 "min": false, 7340 "rightSide": false, 7341 "show": true, 7342 "total": false, 7343 "values": true 7344 }, 7345 "lines": true, 7346 "linewidth": 1, 7347 "links": [], 7348 "nullPointMode": "null", 7349 "options": { 7350 "alertThreshold": true 7351 }, 7352 "paceLength": 10, 7353 "percentage": false, 7354 "pluginVersion": "7.5.11", 7355 "pointradius": 2, 7356 "points": false, 7357 "renderer": "flot", 7358 "seriesOverrides": [], 7359 "spaceLength": 10, 7360 "stack": false, 7361 "steppedLine": false, 7362 "targets": [ 7363 { 7364 "exemplar": true, 7365 "expr": "sum(ticdc_sink_kafka_producer_in_flight_requests{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)", 7366 "format": "time_series", 7367 "interval": "", 7368 "intervalFactor": 1, 7369 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}", 7370 "refId": "A" 7371 } 7372 ], 7373 "thresholds": [], 7374 "timeFrom": null, 7375 "timeRegions": [], 7376 "timeShift": null, 7377 "title": "Kafka Inflight Requests", 7378 "tooltip": { 7379 "shared": true, 7380 "sort": 0, 7381 "value_type": "individual" 7382 }, 7383 "type": "graph", 7384 "xaxis": { 7385 "buckets": null, 7386 "mode": "time", 7387 "name": null, 7388 "show": true, 7389 "values": [] 7390 }, 7391 "yaxes": [ 7392 { 7393 "format": "none", 7394 "label": null, 7395 "logBase": 1, 7396 "max": null, 7397 "min": "0", 7398 "show": true 7399 }, 7400 { 7401 "format": "short", 7402 "label": null, 7403 "logBase": 1, 7404 "max": null, 7405 "min": null, 7406 "show": false 7407 } 7408 ], 7409 "yaxis": { 7410 "align": false, 7411 "alignLevel": null 7412 } 7413 }, 7414 { 7415 "aliasColors": {}, 7416 "bars": false, 7417 "dashLength": 10, 7418 "dashes": false, 7419 "datasource": "${DS_TEST-CLUSTER}", 7420 "description": "The request latency in ms for all brokers.\n\nvalue = request latency histogram's mean", 7421 "fieldConfig": { 7422 "defaults": { 7423 "links": [] 7424 }, 7425 "overrides": [] 7426 }, 7427 "fill": 1, 7428 "fillGradient": 0, 7429 "gridPos": { 7430 "h": 7, 7431 "w": 12, 7432 "x": 12, 7433 "y": 14 7434 }, 7435 "hiddenSeries": false, 7436 "id": 630, 7437 "legend": { 7438 "alignAsTable": true, 7439 "avg": true, 7440 "current": true, 7441 "max": true, 7442 "min": false, 7443 "rightSide": false, 7444 "show": true, 7445 "total": false, 7446 "values": true 7447 }, 7448 "lines": true, 7449 "linewidth": 1, 7450 "links": [], 7451 "nullPointMode": "null", 7452 "options": { 7453 "alertThreshold": true 7454 }, 7455 "paceLength": 10, 7456 "percentage": false, 7457 "pluginVersion": "7.5.11", 7458 "pointradius": 2, 7459 "points": false, 7460 "renderer": "flot", 7461 "seriesOverrides": [], 7462 "spaceLength": 10, 7463 "stack": false, 7464 "steppedLine": false, 7465 "targets": [ 7466 { 7467 "exemplar": true, 7468 "expr": "sum(ticdc_sink_kafka_producer_request_latency{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)", 7469 "format": "time_series", 7470 "interval": "", 7471 "intervalFactor": 1, 7472 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}", 7473 "refId": "A" 7474 } 7475 ], 7476 "thresholds": [], 7477 "timeFrom": null, 7478 "timeRegions": [], 7479 "timeShift": null, 7480 "title": "Kafka Request Latency", 7481 "tooltip": { 7482 "shared": true, 7483 "sort": 0, 7484 "value_type": "individual" 7485 }, 7486 "type": "graph", 7487 "xaxis": { 7488 "buckets": null, 7489 "mode": "time", 7490 "name": null, 7491 "show": true, 7492 "values": [] 7493 }, 7494 "yaxes": [ 7495 { 7496 "format": "s", 7497 "label": null, 7498 "logBase": 1, 7499 "max": null, 7500 "min": "0", 7501 "show": true 7502 }, 7503 { 7504 "format": "short", 7505 "label": null, 7506 "logBase": 1, 7507 "max": null, 7508 "min": null, 7509 "show": false 7510 } 7511 ], 7512 "yaxis": { 7513 "align": false, 7514 "alignLevel": null 7515 } 7516 }, 7517 { 7518 "aliasColors": {}, 7519 "bars": false, 7520 "dashLength": 10, 7521 "dashes": false, 7522 "datasource": "${DS_TEST-CLUSTER}", 7523 "description": "Requests/second sent to all brokers.\nvalue = one-minute moving average rate of events per second", 7524 "fieldConfig": { 7525 "defaults": { 7526 "links": [] 7527 }, 7528 "overrides": [] 7529 }, 7530 "fill": 1, 7531 "fillGradient": 0, 7532 "gridPos": { 7533 "h": 7, 7534 "w": 12, 7535 "x": 0, 7536 "y": 21 7537 }, 7538 "hiddenSeries": false, 7539 "id": 629, 7540 "legend": { 7541 "alignAsTable": true, 7542 "avg": true, 7543 "current": true, 7544 "max": true, 7545 "min": false, 7546 "rightSide": false, 7547 "show": true, 7548 "total": false, 7549 "values": true 7550 }, 7551 "lines": true, 7552 "linewidth": 1, 7553 "links": [], 7554 "nullPointMode": "null", 7555 "options": { 7556 "alertThreshold": true 7557 }, 7558 "paceLength": 10, 7559 "percentage": false, 7560 "pluginVersion": "7.5.11", 7561 "pointradius": 2, 7562 "points": false, 7563 "renderer": "flot", 7564 "seriesOverrides": [], 7565 "spaceLength": 10, 7566 "stack": false, 7567 "steppedLine": false, 7568 "targets": [ 7569 { 7570 "exemplar": true, 7571 "expr": "sum(ticdc_sink_kafka_producer_request_rate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)", 7572 "format": "time_series", 7573 "interval": "", 7574 "intervalFactor": 1, 7575 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}", 7576 "refId": "A" 7577 } 7578 ], 7579 "thresholds": [], 7580 "timeFrom": null, 7581 "timeRegions": [], 7582 "timeShift": null, 7583 "title": "Kafka Request Rate", 7584 "tooltip": { 7585 "shared": true, 7586 "sort": 0, 7587 "value_type": "individual" 7588 }, 7589 "type": "graph", 7590 "xaxis": { 7591 "buckets": null, 7592 "mode": "time", 7593 "name": null, 7594 "show": true, 7595 "values": [] 7596 }, 7597 "yaxes": [ 7598 { 7599 "format": "none", 7600 "label": null, 7601 "logBase": 1, 7602 "max": null, 7603 "min": "0", 7604 "show": true 7605 }, 7606 { 7607 "format": "short", 7608 "label": null, 7609 "logBase": 1, 7610 "max": null, 7611 "min": null, 7612 "show": false 7613 } 7614 ], 7615 "yaxis": { 7616 "align": false, 7617 "alignLevel": null 7618 } 7619 }, 7620 { 7621 "aliasColors": {}, 7622 "bars": false, 7623 "dashLength": 10, 7624 "dashes": false, 7625 "datasource": "${DS_TEST-CLUSTER}", 7626 "description": "Records count per request send to the kafka\nvalue = one-minute moving average of response receive rate", 7627 "fieldConfig": { 7628 "defaults": { 7629 "links": [] 7630 }, 7631 "overrides": [] 7632 }, 7633 "fill": 1, 7634 "fillGradient": 0, 7635 "gridPos": { 7636 "h": 7, 7637 "w": 12, 7638 "x": 12, 7639 "y": 21 7640 }, 7641 "hiddenSeries": false, 7642 "id": 631, 7643 "legend": { 7644 "alignAsTable": true, 7645 "avg": true, 7646 "current": true, 7647 "max": true, 7648 "min": false, 7649 "rightSide": false, 7650 "show": true, 7651 "total": false, 7652 "values": true 7653 }, 7654 "lines": true, 7655 "linewidth": 1, 7656 "links": [], 7657 "nullPointMode": "null", 7658 "options": { 7659 "alertThreshold": true 7660 }, 7661 "paceLength": 10, 7662 "percentage": false, 7663 "pluginVersion": "7.5.11", 7664 "pointradius": 2, 7665 "points": false, 7666 "renderer": "flot", 7667 "seriesOverrides": [], 7668 "spaceLength": 10, 7669 "stack": false, 7670 "steppedLine": false, 7671 "targets": [ 7672 { 7673 "exemplar": true, 7674 "expr": "sum(ticdc_sink_kafka_producer_records_per_request{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)", 7675 "format": "time_series", 7676 "interval": "", 7677 "intervalFactor": 1, 7678 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}", 7679 "refId": "A" 7680 } 7681 ], 7682 "thresholds": [], 7683 "timeFrom": null, 7684 "timeRegions": [], 7685 "timeShift": null, 7686 "title": "Kafka Records Per Request", 7687 "tooltip": { 7688 "shared": true, 7689 "sort": 0, 7690 "value_type": "individual" 7691 }, 7692 "type": "graph", 7693 "xaxis": { 7694 "buckets": null, 7695 "mode": "time", 7696 "name": null, 7697 "show": true, 7698 "values": [] 7699 }, 7700 "yaxes": [ 7701 { 7702 "format": "none", 7703 "label": null, 7704 "logBase": 1, 7705 "max": null, 7706 "min": "0", 7707 "show": true 7708 }, 7709 { 7710 "format": "short", 7711 "label": null, 7712 "logBase": 1, 7713 "max": null, 7714 "min": null, 7715 "show": false 7716 } 7717 ], 7718 "yaxis": { 7719 "align": false, 7720 "alignLevel": null 7721 } 7722 }, 7723 { 7724 "aliasColors": {}, 7725 "bars": false, 7726 "dashLength": 10, 7727 "dashes": false, 7728 "datasource": "${DS_TEST-CLUSTER}", 7729 "description": "The compression ratio times 100 of record batches for all topics.", 7730 "fieldConfig": { 7731 "defaults": {}, 7732 "overrides": [] 7733 }, 7734 "fill": 1, 7735 "fillGradient": 0, 7736 "gridPos": { 7737 "h": 7, 7738 "w": 12, 7739 "x": 0, 7740 "y": 28 7741 }, 7742 "hiddenSeries": false, 7743 "id": 626, 7744 "legend": { 7745 "alignAsTable": true, 7746 "avg": true, 7747 "current": true, 7748 "max": true, 7749 "min": false, 7750 "rightSide": false, 7751 "show": true, 7752 "total": false, 7753 "values": true 7754 }, 7755 "lines": true, 7756 "linewidth": 1, 7757 "links": [], 7758 "nullPointMode": "null", 7759 "options": { 7760 "alertThreshold": true 7761 }, 7762 "paceLength": 10, 7763 "percentage": false, 7764 "pluginVersion": "7.5.11", 7765 "pointradius": 2, 7766 "points": false, 7767 "renderer": "flot", 7768 "seriesOverrides": [], 7769 "spaceLength": 10, 7770 "stack": false, 7771 "steppedLine": false, 7772 "targets": [ 7773 { 7774 "exemplar": true, 7775 "expr": "sum(ticdc_sink_kafka_producer_compression_ratio{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)", 7776 "format": "time_series", 7777 "interval": "", 7778 "intervalFactor": 1, 7779 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 7780 "refId": "A" 7781 }, 7782 { 7783 "exemplar": true, 7784 "expr": "histogram_quantile(0.99, sum(rate(ticdc_kafka_sink_codec_compression_ratio_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed,instance))", 7785 "hide": false, 7786 "interval": "", 7787 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-Codec-P99", 7788 "refId": "B" 7789 } 7790 ], 7791 "thresholds": [], 7792 "timeFrom": null, 7793 "timeRegions": [], 7794 "timeShift": null, 7795 "title": "Kafka Producer Compression Ratio", 7796 "tooltip": { 7797 "shared": true, 7798 "sort": 0, 7799 "value_type": "individual" 7800 }, 7801 "type": "graph", 7802 "xaxis": { 7803 "buckets": null, 7804 "mode": "time", 7805 "name": null, 7806 "show": true, 7807 "values": [] 7808 }, 7809 "yaxes": [ 7810 { 7811 "format": "none", 7812 "label": null, 7813 "logBase": 1, 7814 "max": null, 7815 "min": "0", 7816 "show": true 7817 }, 7818 { 7819 "format": "short", 7820 "label": null, 7821 "logBase": 1, 7822 "max": null, 7823 "min": null, 7824 "show": false 7825 } 7826 ], 7827 "yaxis": { 7828 "align": false, 7829 "alignLevel": null 7830 } 7831 }, 7832 { 7833 "aliasColors": {}, 7834 "bars": false, 7835 "dashLength": 10, 7836 "dashes": false, 7837 "datasource": "${DS_TEST-CLUSTER}", 7838 "description": "", 7839 "fieldConfig": { 7840 "defaults": {}, 7841 "overrides": [] 7842 }, 7843 "fill": 1, 7844 "fillGradient": 0, 7845 "gridPos": { 7846 "h": 7, 7847 "w": 12, 7848 "x": 12, 7849 "y": 28 7850 }, 7851 "hiddenSeries": false, 7852 "id": 709, 7853 "legend": { 7854 "alignAsTable": true, 7855 "avg": true, 7856 "current": true, 7857 "max": true, 7858 "min": false, 7859 "rightSide": false, 7860 "show": true, 7861 "total": false, 7862 "values": true 7863 }, 7864 "lines": true, 7865 "linewidth": 1, 7866 "nullPointMode": "null", 7867 "options": { 7868 "alertThreshold": true 7869 }, 7870 "percentage": false, 7871 "pluginVersion": "7.5.11", 7872 "pointradius": 2, 7873 "points": false, 7874 "renderer": "flot", 7875 "seriesOverrides": [], 7876 "spaceLength": 10, 7877 "stack": false, 7878 "steppedLine": false, 7879 "targets": [ 7880 { 7881 "exemplar": true, 7882 "expr": "ticdc_sink_encoder_group_input_chan_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}", 7883 "hide": false, 7884 "interval": "", 7885 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{index}}", 7886 "refId": "A" 7887 } 7888 ], 7889 "thresholds": [], 7890 "timeFrom": null, 7891 "timeRegions": [], 7892 "timeShift": null, 7893 "title": "encoder group input channel size", 7894 "tooltip": { 7895 "shared": true, 7896 "sort": 0, 7897 "value_type": "individual" 7898 }, 7899 "type": "graph", 7900 "xaxis": { 7901 "buckets": null, 7902 "mode": "time", 7903 "name": null, 7904 "show": true, 7905 "values": [] 7906 }, 7907 "yaxes": [ 7908 { 7909 "format": "none", 7910 "label": null, 7911 "logBase": 1, 7912 "max": null, 7913 "min": null, 7914 "show": true 7915 }, 7916 { 7917 "format": "none", 7918 "label": null, 7919 "logBase": 1, 7920 "max": null, 7921 "min": null, 7922 "show": false 7923 } 7924 ], 7925 "yaxis": { 7926 "align": false, 7927 "alignLevel": null 7928 } 7929 }, 7930 { 7931 "aliasColors": {}, 7932 "bars": false, 7933 "dashLength": 10, 7934 "dashes": false, 7935 "datasource": "${DS_TEST-CLUSTER}", 7936 "description": "", 7937 "fieldConfig": { 7938 "defaults": {}, 7939 "overrides": [] 7940 }, 7941 "fill": 1, 7942 "fillGradient": 0, 7943 "gridPos": { 7944 "h": 7, 7945 "w": 12, 7946 "x": 0, 7947 "y": 35 7948 }, 7949 "hiddenSeries": false, 7950 "id": 710, 7951 "legend": { 7952 "alignAsTable": true, 7953 "avg": true, 7954 "current": true, 7955 "max": true, 7956 "min": false, 7957 "rightSide": false, 7958 "show": true, 7959 "total": false, 7960 "values": true 7961 }, 7962 "lines": true, 7963 "linewidth": 1, 7964 "nullPointMode": "null", 7965 "options": { 7966 "alertThreshold": true 7967 }, 7968 "percentage": false, 7969 "pluginVersion": "7.5.11", 7970 "pointradius": 2, 7971 "points": false, 7972 "renderer": "flot", 7973 "seriesOverrides": [], 7974 "spaceLength": 10, 7975 "stack": false, 7976 "steppedLine": false, 7977 "targets": [ 7978 { 7979 "exemplar": true, 7980 "expr": "ticdc_sink_encoder_group_output_chan_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}", 7981 "hide": false, 7982 "interval": "", 7983 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 7984 "refId": "A" 7985 } 7986 ], 7987 "thresholds": [], 7988 "timeFrom": null, 7989 "timeRegions": [], 7990 "timeShift": null, 7991 "title": "encoder group output channel size", 7992 "tooltip": { 7993 "shared": true, 7994 "sort": 0, 7995 "value_type": "individual" 7996 }, 7997 "type": "graph", 7998 "xaxis": { 7999 "buckets": null, 8000 "mode": "time", 8001 "name": null, 8002 "show": true, 8003 "values": [] 8004 }, 8005 "yaxes": [ 8006 { 8007 "format": "none", 8008 "label": null, 8009 "logBase": 1, 8010 "max": null, 8011 "min": null, 8012 "show": true 8013 }, 8014 { 8015 "format": "none", 8016 "label": null, 8017 "logBase": 1, 8018 "max": null, 8019 "min": null, 8020 "show": false 8021 } 8022 ], 8023 "yaxis": { 8024 "align": false, 8025 "alignLevel": null 8026 } 8027 }, 8028 { 8029 "aliasColors": {}, 8030 "bars": false, 8031 "dashLength": 10, 8032 "dashes": false, 8033 "datasource": "${DS_TEST-CLUSTER}", 8034 "description": "MQ worker batch multiple messages into one when using batched encode protocol, this metric record the time cost on batch messages.", 8035 "fieldConfig": { 8036 "defaults": {}, 8037 "overrides": [] 8038 }, 8039 "fill": 1, 8040 "fillGradient": 0, 8041 "gridPos": { 8042 "h": 7, 8043 "w": 12, 8044 "x": 12, 8045 "y": 35 8046 }, 8047 "hiddenSeries": false, 8048 "id": 711, 8049 "legend": { 8050 "alignAsTable": true, 8051 "avg": true, 8052 "current": true, 8053 "max": true, 8054 "min": false, 8055 "rightSide": false, 8056 "show": true, 8057 "total": false, 8058 "values": true 8059 }, 8060 "lines": true, 8061 "linewidth": 1, 8062 "nullPointMode": "null", 8063 "options": { 8064 "alertThreshold": true 8065 }, 8066 "percentage": false, 8067 "pluginVersion": "7.5.11", 8068 "pointradius": 2, 8069 "points": false, 8070 "renderer": "flot", 8071 "seriesOverrides": [], 8072 "spaceLength": 10, 8073 "stack": false, 8074 "steppedLine": false, 8075 "targets": [ 8076 { 8077 "exemplar": true, 8078 "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_worker_batch_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed,instance))", 8079 "interval": "", 8080 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-P999", 8081 "queryType": "randomWalk", 8082 "refId": "A" 8083 }, 8084 { 8085 "exemplar": true, 8086 "expr": "sum(rate(ticdc_sink_mq_worker_batch_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_mq_worker_batch_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)", 8087 "hide": false, 8088 "interval": "", 8089 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-avg", 8090 "refId": "B" 8091 } 8092 ], 8093 "thresholds": [], 8094 "timeFrom": null, 8095 "timeRegions": [], 8096 "timeShift": null, 8097 "title": "Worker Batch Duration Percentile", 8098 "tooltip": { 8099 "shared": true, 8100 "sort": 0, 8101 "value_type": "individual" 8102 }, 8103 "type": "graph", 8104 "xaxis": { 8105 "buckets": null, 8106 "mode": "time", 8107 "name": null, 8108 "show": true, 8109 "values": [] 8110 }, 8111 "yaxes": [ 8112 { 8113 "format": "s", 8114 "label": null, 8115 "logBase": 2, 8116 "max": null, 8117 "min": null, 8118 "show": true 8119 }, 8120 { 8121 "format": "short", 8122 "label": null, 8123 "logBase": 1, 8124 "max": null, 8125 "min": null, 8126 "show": false 8127 } 8128 ], 8129 "yaxis": { 8130 "align": false, 8131 "alignLevel": null 8132 } 8133 }, 8134 { 8135 "aliasColors": {}, 8136 "bars": false, 8137 "dashLength": 10, 8138 "dashes": false, 8139 "datasource": "${DS_TEST-CLUSTER}", 8140 "description": "MQ worker batch multiple messages into one when using batched encode protocol, this metrics track each batch's size", 8141 "fieldConfig": { 8142 "defaults": {}, 8143 "overrides": [] 8144 }, 8145 "fill": 1, 8146 "fillGradient": 0, 8147 "gridPos": { 8148 "h": 7, 8149 "w": 12, 8150 "x": 0, 8151 "y": 42 8152 }, 8153 "hiddenSeries": false, 8154 "id": 712, 8155 "legend": { 8156 "alignAsTable": true, 8157 "avg": true, 8158 "current": true, 8159 "max": true, 8160 "min": false, 8161 "rightSide": false, 8162 "show": true, 8163 "total": false, 8164 "values": true 8165 }, 8166 "lines": true, 8167 "linewidth": 1, 8168 "nullPointMode": "null", 8169 "options": { 8170 "alertThreshold": true 8171 }, 8172 "percentage": false, 8173 "pluginVersion": "7.5.11", 8174 "pointradius": 2, 8175 "points": false, 8176 "renderer": "flot", 8177 "seriesOverrides": [], 8178 "spaceLength": 10, 8179 "stack": false, 8180 "steppedLine": false, 8181 "targets": [ 8182 { 8183 "exemplar": true, 8184 "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_worker_batch_size_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed,instance))", 8185 "hide": false, 8186 "interval": "", 8187 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-P999", 8188 "queryType": "randomWalk", 8189 "refId": "A" 8190 }, 8191 { 8192 "exemplar": true, 8193 "expr": "sum(rate(ticdc_sink_mq_worker_batch_size_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance) /\nsum(rate(ticdc_sink_mq_worker_batch_size_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed,instance)", 8194 "hide": false, 8195 "interval": "", 8196 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-avg", 8197 "refId": "B" 8198 } 8199 ], 8200 "thresholds": [], 8201 "timeFrom": null, 8202 "timeRegions": [], 8203 "timeShift": null, 8204 "title": "Worker Batch Size Percentile", 8205 "tooltip": { 8206 "shared": true, 8207 "sort": 0, 8208 "value_type": "individual" 8209 }, 8210 "type": "graph", 8211 "xaxis": { 8212 "buckets": null, 8213 "mode": "time", 8214 "name": null, 8215 "show": true, 8216 "values": [] 8217 }, 8218 "yaxes": [ 8219 { 8220 "format": "none", 8221 "label": null, 8222 "logBase": 1, 8223 "max": null, 8224 "min": null, 8225 "show": true 8226 }, 8227 { 8228 "format": "short", 8229 "label": null, 8230 "logBase": 1, 8231 "max": null, 8232 "min": null, 8233 "show": false 8234 } 8235 ], 8236 "yaxis": { 8237 "align": false, 8238 "alignLevel": null 8239 } 8240 }, 8241 { 8242 "aliasColors": {}, 8243 "bars": false, 8244 "dashLength": 10, 8245 "dashes": false, 8246 "datasource": "${DS_TEST-CLUSTER}", 8247 "description": "Kafka-Go Writer batch messages internally before send to the Kafka cluster.", 8248 "fieldConfig": { 8249 "defaults": {}, 8250 "overrides": [] 8251 }, 8252 "fill": 1, 8253 "fillGradient": 0, 8254 "gridPos": { 8255 "h": 7, 8256 "w": 12, 8257 "x": 12, 8258 "y": 42 8259 }, 8260 "hiddenSeries": false, 8261 "id": 731, 8262 "legend": { 8263 "alignAsTable": true, 8264 "avg": true, 8265 "current": true, 8266 "max": true, 8267 "min": false, 8268 "rightSide": false, 8269 "show": true, 8270 "total": false, 8271 "values": true 8272 }, 8273 "lines": true, 8274 "linewidth": 1, 8275 "nullPointMode": "null", 8276 "options": { 8277 "alertThreshold": true 8278 }, 8279 "percentage": false, 8280 "pluginVersion": "7.5.11", 8281 "pointradius": 2, 8282 "points": false, 8283 "renderer": "flot", 8284 "seriesOverrides": [], 8285 "spaceLength": 10, 8286 "stack": false, 8287 "steppedLine": false, 8288 "targets": [ 8289 { 8290 "exemplar": true, 8291 "expr": "max(ticdc_sink_kafka_producer_batch_message_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)", 8292 "hide": false, 8293 "interval": "", 8294 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 8295 "refId": "A" 8296 } 8297 ], 8298 "thresholds": [], 8299 "timeFrom": null, 8300 "timeRegions": [], 8301 "timeShift": null, 8302 "title": "Producer Batch Message Count Percentile (Kafka-Go Only)", 8303 "tooltip": { 8304 "shared": true, 8305 "sort": 0, 8306 "value_type": "individual" 8307 }, 8308 "type": "graph", 8309 "xaxis": { 8310 "buckets": null, 8311 "mode": "time", 8312 "name": null, 8313 "show": true, 8314 "values": [] 8315 }, 8316 "yaxes": [ 8317 { 8318 "format": "none", 8319 "label": null, 8320 "logBase": 1, 8321 "max": null, 8322 "min": null, 8323 "show": true 8324 }, 8325 { 8326 "format": "short", 8327 "label": null, 8328 "logBase": 1, 8329 "max": null, 8330 "min": null, 8331 "show": false 8332 } 8333 ], 8334 "yaxis": { 8335 "align": false, 8336 "alignLevel": null 8337 } 8338 }, 8339 { 8340 "aliasColors": {}, 8341 "bars": false, 8342 "dashLength": 10, 8343 "dashes": false, 8344 "datasource": "${DS_TEST-CLUSTER}", 8345 "description": "MQ worker batch multiple messages into one when using batched encode protocol, this metric record the time cost on batch messages.", 8346 "fieldConfig": { 8347 "defaults": {}, 8348 "overrides": [] 8349 }, 8350 "fill": 1, 8351 "fillGradient": 0, 8352 "gridPos": { 8353 "h": 7, 8354 "w": 12, 8355 "x": 0, 8356 "y": 49 8357 }, 8358 "hiddenSeries": false, 8359 "id": 733, 8360 "legend": { 8361 "alignAsTable": true, 8362 "avg": true, 8363 "current": true, 8364 "max": true, 8365 "min": false, 8366 "rightSide": false, 8367 "show": true, 8368 "total": false, 8369 "values": true 8370 }, 8371 "lines": true, 8372 "linewidth": 1, 8373 "nullPointMode": "null", 8374 "options": { 8375 "alertThreshold": true 8376 }, 8377 "percentage": false, 8378 "pluginVersion": "7.5.11", 8379 "pointradius": 2, 8380 "points": false, 8381 "renderer": "flot", 8382 "seriesOverrides": [], 8383 "spaceLength": 10, 8384 "stack": false, 8385 "steppedLine": false, 8386 "targets": [ 8387 { 8388 "exemplar": true, 8389 "expr": "max(ticdc_sink_kafka_producer_batch_duration{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)", 8390 "hide": false, 8391 "interval": "", 8392 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 8393 "refId": "A" 8394 } 8395 ], 8396 "thresholds": [], 8397 "timeFrom": null, 8398 "timeRegions": [], 8399 "timeShift": null, 8400 "title": "Producer Batch Messages Duration Percentile (Kafka-Go Only)", 8401 "tooltip": { 8402 "shared": true, 8403 "sort": 0, 8404 "value_type": "individual" 8405 }, 8406 "type": "graph", 8407 "xaxis": { 8408 "buckets": null, 8409 "mode": "time", 8410 "name": null, 8411 "show": true, 8412 "values": [] 8413 }, 8414 "yaxes": [ 8415 { 8416 "format": "s", 8417 "label": null, 8418 "logBase": 1, 8419 "max": null, 8420 "min": null, 8421 "show": true 8422 }, 8423 { 8424 "format": "short", 8425 "label": null, 8426 "logBase": 1, 8427 "max": null, 8428 "min": null, 8429 "show": false 8430 } 8431 ], 8432 "yaxis": { 8433 "align": false, 8434 "alignLevel": null 8435 } 8436 }, 8437 { 8438 "aliasColors": {}, 8439 "bars": false, 8440 "dashLength": 10, 8441 "dashes": false, 8442 "datasource": "${DS_TEST-CLUSTER}", 8443 "description": "Kafka-Go Writer batch messages internally before send to the Kafka cluster.", 8444 "fieldConfig": { 8445 "defaults": {}, 8446 "overrides": [] 8447 }, 8448 "fill": 1, 8449 "fillGradient": 0, 8450 "gridPos": { 8451 "h": 7, 8452 "w": 12, 8453 "x": 12, 8454 "y": 49 8455 }, 8456 "hiddenSeries": false, 8457 "id": 735, 8458 "legend": { 8459 "alignAsTable": true, 8460 "avg": true, 8461 "current": true, 8462 "max": true, 8463 "min": false, 8464 "rightSide": false, 8465 "show": true, 8466 "total": false, 8467 "values": true 8468 }, 8469 "lines": true, 8470 "linewidth": 1, 8471 "nullPointMode": "null", 8472 "options": { 8473 "alertThreshold": true 8474 }, 8475 "percentage": false, 8476 "pluginVersion": "7.5.11", 8477 "pointradius": 2, 8478 "points": false, 8479 "renderer": "flot", 8480 "seriesOverrides": [], 8481 "spaceLength": 10, 8482 "stack": false, 8483 "steppedLine": false, 8484 "targets": [ 8485 { 8486 "exemplar": true, 8487 "expr": "max(ticdc_sink_kafka_producer_batch_size{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)", 8488 "hide": false, 8489 "interval": "", 8490 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 8491 "refId": "A" 8492 } 8493 ], 8494 "thresholds": [], 8495 "timeFrom": null, 8496 "timeRegions": [], 8497 "timeShift": null, 8498 "title": "Producer Batch Bytes Percentile (Kafka-Go Only)", 8499 "tooltip": { 8500 "shared": true, 8501 "sort": 0, 8502 "value_type": "individual" 8503 }, 8504 "type": "graph", 8505 "xaxis": { 8506 "buckets": null, 8507 "mode": "time", 8508 "name": null, 8509 "show": true, 8510 "values": [] 8511 }, 8512 "yaxes": [ 8513 { 8514 "format": "bytes", 8515 "label": null, 8516 "logBase": 1, 8517 "max": null, 8518 "min": null, 8519 "show": true 8520 }, 8521 { 8522 "format": "short", 8523 "label": null, 8524 "logBase": 1, 8525 "max": null, 8526 "min": null, 8527 "show": false 8528 } 8529 ], 8530 "yaxis": { 8531 "align": false, 8532 "alignLevel": null 8533 } 8534 }, 8535 { 8536 "aliasColors": {}, 8537 "bars": false, 8538 "dashLength": 10, 8539 "dashes": false, 8540 "datasource": "${DS_TEST-CLUSTER}", 8541 "description": "Count of errors in the last minute.", 8542 "fieldConfig": { 8543 "defaults": {}, 8544 "overrides": [] 8545 }, 8546 "fill": 1, 8547 "fillGradient": 0, 8548 "gridPos": { 8549 "h": 8, 8550 "w": 12, 8551 "x": 0, 8552 "y": 56 8553 }, 8554 "hiddenSeries": false, 8555 "id": 739, 8556 "legend": { 8557 "alignAsTable": true, 8558 "avg": false, 8559 "current": true, 8560 "max": false, 8561 "min": false, 8562 "rightSide": false, 8563 "show": true, 8564 "total": false, 8565 "values": true 8566 }, 8567 "lines": true, 8568 "linewidth": 1, 8569 "nullPointMode": "null", 8570 "options": { 8571 "alertThreshold": true 8572 }, 8573 "percentage": false, 8574 "pluginVersion": "7.5.11", 8575 "pointradius": 2, 8576 "points": false, 8577 "renderer": "flot", 8578 "seriesOverrides": [], 8579 "spaceLength": 10, 8580 "stack": false, 8581 "steppedLine": false, 8582 "targets": [ 8583 { 8584 "exemplar": true, 8585 "expr": "sum(rate(ticdc_sink_kafka_producer_retry_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)", 8586 "interval": "", 8587 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 8588 "queryType": "randomWalk", 8589 "refId": "A" 8590 } 8591 ], 8592 "thresholds": [], 8593 "timeFrom": null, 8594 "timeRegions": [], 8595 "timeShift": null, 8596 "title": "Retry Count (Kafka-Go Only)", 8597 "tooltip": { 8598 "shared": true, 8599 "sort": 0, 8600 "value_type": "individual" 8601 }, 8602 "type": "graph", 8603 "xaxis": { 8604 "buckets": null, 8605 "mode": "time", 8606 "name": null, 8607 "show": true, 8608 "values": [] 8609 }, 8610 "yaxes": [ 8611 { 8612 "format": "none", 8613 "label": null, 8614 "logBase": 1, 8615 "max": null, 8616 "min": null, 8617 "show": true 8618 }, 8619 { 8620 "format": "short", 8621 "label": null, 8622 "logBase": 1, 8623 "max": null, 8624 "min": null, 8625 "show": false 8626 } 8627 ], 8628 "yaxis": { 8629 "align": false, 8630 "alignLevel": null 8631 } 8632 }, 8633 { 8634 "aliasColors": {}, 8635 "bars": false, 8636 "dashLength": 10, 8637 "dashes": false, 8638 "datasource": "${DS_TEST-CLUSTER}", 8639 "description": "Count of errors in the last minute.", 8640 "fieldConfig": { 8641 "defaults": {}, 8642 "overrides": [] 8643 }, 8644 "fill": 1, 8645 "fillGradient": 0, 8646 "gridPos": { 8647 "h": 8, 8648 "w": 12, 8649 "x": 12, 8650 "y": 56 8651 }, 8652 "hiddenSeries": false, 8653 "id": 737, 8654 "legend": { 8655 "alignAsTable": true, 8656 "avg": false, 8657 "current": true, 8658 "max": false, 8659 "min": false, 8660 "rightSide": false, 8661 "show": true, 8662 "total": false, 8663 "values": true 8664 }, 8665 "lines": true, 8666 "linewidth": 1, 8667 "nullPointMode": "null", 8668 "options": { 8669 "alertThreshold": true 8670 }, 8671 "percentage": false, 8672 "pluginVersion": "7.5.11", 8673 "pointradius": 2, 8674 "points": false, 8675 "renderer": "flot", 8676 "seriesOverrides": [], 8677 "spaceLength": 10, 8678 "stack": false, 8679 "steppedLine": false, 8680 "targets": [ 8681 { 8682 "exemplar": true, 8683 "expr": "sum(rate(ticdc_sink_kafka_producer_err_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)", 8684 "hide": false, 8685 "interval": "", 8686 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 8687 "refId": "B" 8688 } 8689 ], 8690 "thresholds": [], 8691 "timeFrom": null, 8692 "timeRegions": [], 8693 "timeShift": null, 8694 "title": "Error Count (Kafka-Go Only)", 8695 "tooltip": { 8696 "shared": true, 8697 "sort": 0, 8698 "value_type": "individual" 8699 }, 8700 "type": "graph", 8701 "xaxis": { 8702 "buckets": null, 8703 "mode": "time", 8704 "name": null, 8705 "show": true, 8706 "values": [] 8707 }, 8708 "yaxes": [ 8709 { 8710 "format": "short", 8711 "label": null, 8712 "logBase": 1, 8713 "max": null, 8714 "min": null, 8715 "show": true 8716 }, 8717 { 8718 "format": "short", 8719 "label": null, 8720 "logBase": 1, 8721 "max": null, 8722 "min": null, 8723 "show": false 8724 } 8725 ], 8726 "yaxis": { 8727 "align": false, 8728 "alignLevel": null 8729 } 8730 }, 8731 { 8732 "aliasColors": {}, 8733 "bars": false, 8734 "dashLength": 10, 8735 "dashes": false, 8736 "datasource": "${DS_TEST-CLUSTER}", 8737 "description": "MQ worker send large message to the external storage, this metric record the time cost on send every message.", 8738 "fieldConfig": { 8739 "defaults": {}, 8740 "overrides": [] 8741 }, 8742 "fill": 1, 8743 "fillGradient": 0, 8744 "gridPos": { 8745 "h": 7, 8746 "w": 12, 8747 "x": 0, 8748 "y": 64 8749 }, 8750 "hiddenSeries": false, 8751 "id": 740, 8752 "legend": { 8753 "alignAsTable": true, 8754 "avg": true, 8755 "current": true, 8756 "max": true, 8757 "min": false, 8758 "rightSide": false, 8759 "show": true, 8760 "total": false, 8761 "values": true 8762 }, 8763 "lines": true, 8764 "linewidth": 1, 8765 "nullPointMode": "null", 8766 "options": { 8767 "alertThreshold": true 8768 }, 8769 "percentage": false, 8770 "pluginVersion": "7.5.11", 8771 "pointradius": 2, 8772 "points": false, 8773 "renderer": "flot", 8774 "seriesOverrides": [], 8775 "spaceLength": 10, 8776 "stack": false, 8777 "steppedLine": false, 8778 "targets": [ 8779 { 8780 "exemplar": true, 8781 "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_claim_check_send_message_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,changefeed,instance))", 8782 "interval": "", 8783 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-P999", 8784 "queryType": "randomWalk", 8785 "refId": "A" 8786 }, 8787 { 8788 "exemplar": true, 8789 "expr": "rate(ticdc_sink_mq_claim_check_send_message_duration_sum[30s]) / rate(ticdc_sink_mq_claim_check_send_message_duration_count[30s])", 8790 "hide": false, 8791 "interval": "", 8792 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-avg", 8793 "refId": "B" 8794 } 8795 ], 8796 "thresholds": [], 8797 "timeFrom": null, 8798 "timeRegions": [], 8799 "timeShift": null, 8800 "title": "Claim Check Send Message Duration Percentile", 8801 "tooltip": { 8802 "shared": true, 8803 "sort": 0, 8804 "value_type": "individual" 8805 }, 8806 "type": "graph", 8807 "xaxis": { 8808 "buckets": null, 8809 "mode": "time", 8810 "name": null, 8811 "show": true, 8812 "values": [] 8813 }, 8814 "yaxes": [ 8815 { 8816 "format": "s", 8817 "label": null, 8818 "logBase": 2, 8819 "max": null, 8820 "min": null, 8821 "show": true 8822 }, 8823 { 8824 "format": "short", 8825 "label": null, 8826 "logBase": 1, 8827 "max": null, 8828 "min": null, 8829 "show": false 8830 } 8831 ], 8832 "yaxis": { 8833 "align": false, 8834 "alignLevel": null 8835 } 8836 }, 8837 { 8838 "aliasColors": {}, 8839 "bars": false, 8840 "dashLength": 10, 8841 "dashes": false, 8842 "datasource": "${DS_TEST-CLUSTER}", 8843 "description": "MQ worker send large message to the external storage, this metrics record the message count", 8844 "fieldConfig": { 8845 "defaults": {}, 8846 "overrides": [] 8847 }, 8848 "fill": 1, 8849 "fillGradient": 0, 8850 "gridPos": { 8851 "h": 7, 8852 "w": 12, 8853 "x": 12, 8854 "y": 64 8855 }, 8856 "hiddenSeries": false, 8857 "id": 741, 8858 "legend": { 8859 "alignAsTable": true, 8860 "avg": true, 8861 "current": true, 8862 "max": true, 8863 "min": false, 8864 "rightSide": false, 8865 "show": true, 8866 "total": false, 8867 "values": true 8868 }, 8869 "lines": true, 8870 "linewidth": 1, 8871 "nullPointMode": "null", 8872 "options": { 8873 "alertThreshold": true 8874 }, 8875 "percentage": false, 8876 "pluginVersion": "7.5.11", 8877 "pointradius": 2, 8878 "points": false, 8879 "renderer": "flot", 8880 "seriesOverrides": [], 8881 "spaceLength": 10, 8882 "stack": false, 8883 "steppedLine": false, 8884 "targets": [ 8885 { 8886 "exemplar": true, 8887 "expr": "sum(rate(ticdc_sink_mq_claim_check_send_message_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)", 8888 "hide": false, 8889 "interval": "", 8890 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 8891 "refId": "B" 8892 } 8893 ], 8894 "thresholds": [], 8895 "timeFrom": null, 8896 "timeRegions": [], 8897 "timeShift": null, 8898 "title": "Claim Check Send Message Count", 8899 "tooltip": { 8900 "shared": true, 8901 "sort": 0, 8902 "value_type": "individual" 8903 }, 8904 "type": "graph", 8905 "xaxis": { 8906 "buckets": null, 8907 "mode": "time", 8908 "name": null, 8909 "show": true, 8910 "values": [] 8911 }, 8912 "yaxes": [ 8913 { 8914 "format": "short", 8915 "label": null, 8916 "logBase": 1, 8917 "max": null, 8918 "min": null, 8919 "show": true 8920 }, 8921 { 8922 "format": "short", 8923 "label": null, 8924 "logBase": 1, 8925 "max": null, 8926 "min": null, 8927 "show": false 8928 } 8929 ], 8930 "yaxis": { 8931 "align": false, 8932 "alignLevel": null 8933 } 8934 } 8935 ], 8936 "title": "Sink - MQ Sink", 8937 "type": "row" 8938 }, 8939 { 8940 "collapsed": true, 8941 "datasource": null, 8942 "gridPos": { 8943 "h": 1, 8944 "w": 24, 8945 "x": 0, 8946 "y": 7 8947 }, 8948 "id": 713, 8949 "panels": [ 8950 { 8951 "aliasColors": {}, 8952 "bars": false, 8953 "dashLength": 10, 8954 "dashes": false, 8955 "datasource": "${DS_TEST-CLUSTER}", 8956 "fieldConfig": { 8957 "defaults": {}, 8958 "overrides": [] 8959 }, 8960 "fill": 1, 8961 "fillGradient": 0, 8962 "gridPos": { 8963 "h": 8, 8964 "w": 12, 8965 "x": 0, 8966 "y": 8 8967 }, 8968 "hiddenSeries": false, 8969 "id": 714, 8970 "legend": { 8971 "avg": false, 8972 "current": false, 8973 "max": false, 8974 "min": false, 8975 "show": true, 8976 "total": false, 8977 "values": false 8978 }, 8979 "lines": true, 8980 "linewidth": 1, 8981 "nullPointMode": "null", 8982 "options": { 8983 "alertThreshold": true 8984 }, 8985 "percentage": false, 8986 "pluginVersion": "7.5.11", 8987 "pointradius": 2, 8988 "points": false, 8989 "renderer": "flot", 8990 "seriesOverrides": [], 8991 "spaceLength": 10, 8992 "stack": false, 8993 "steppedLine": false, 8994 "targets": [ 8995 { 8996 "exemplar": true, 8997 "expr": "sum(rate(ticdc_sink_cloud_storage_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace, changefeed, instance)", 8998 "hide": false, 8999 "interval": "", 9000 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 9001 "queryType": "randomWalk", 9002 "refId": "A" 9003 } 9004 ], 9005 "thresholds": [], 9006 "timeFrom": null, 9007 "timeRegions": [], 9008 "timeShift": null, 9009 "title": "Write Bytes/s", 9010 "tooltip": { 9011 "shared": true, 9012 "sort": 0, 9013 "value_type": "individual" 9014 }, 9015 "type": "graph", 9016 "xaxis": { 9017 "buckets": null, 9018 "mode": "time", 9019 "name": null, 9020 "show": true, 9021 "values": [] 9022 }, 9023 "yaxes": [ 9024 { 9025 "format": "short", 9026 "label": null, 9027 "logBase": 1, 9028 "max": null, 9029 "min": null, 9030 "show": true 9031 }, 9032 { 9033 "format": "short", 9034 "label": null, 9035 "logBase": 1, 9036 "max": null, 9037 "min": null, 9038 "show": true 9039 } 9040 ], 9041 "yaxis": { 9042 "align": false, 9043 "alignLevel": null 9044 } 9045 }, 9046 { 9047 "aliasColors": {}, 9048 "bars": false, 9049 "dashLength": 10, 9050 "dashes": false, 9051 "datasource": "${DS_TEST-CLUSTER}", 9052 "fieldConfig": { 9053 "defaults": {}, 9054 "overrides": [] 9055 }, 9056 "fill": 1, 9057 "fillGradient": 0, 9058 "gridPos": { 9059 "h": 8, 9060 "w": 12, 9061 "x": 12, 9062 "y": 8 9063 }, 9064 "hiddenSeries": false, 9065 "id": 715, 9066 "legend": { 9067 "avg": false, 9068 "current": false, 9069 "max": false, 9070 "min": false, 9071 "show": true, 9072 "total": false, 9073 "values": false 9074 }, 9075 "lines": true, 9076 "linewidth": 1, 9077 "nullPointMode": "null", 9078 "options": { 9079 "alertThreshold": true 9080 }, 9081 "percentage": false, 9082 "pluginVersion": "7.5.11", 9083 "pointradius": 2, 9084 "points": false, 9085 "renderer": "flot", 9086 "seriesOverrides": [], 9087 "spaceLength": 10, 9088 "stack": false, 9089 "steppedLine": false, 9090 "targets": [ 9091 { 9092 "exemplar": true, 9093 "expr": "sum(ticdc_sink_cloud_storage_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, instance)", 9094 "interval": "", 9095 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 9096 "queryType": "randomWalk", 9097 "refId": "A" 9098 } 9099 ], 9100 "thresholds": [], 9101 "timeFrom": null, 9102 "timeRegions": [], 9103 "timeShift": null, 9104 "title": "File Count", 9105 "tooltip": { 9106 "shared": true, 9107 "sort": 0, 9108 "value_type": "individual" 9109 }, 9110 "type": "graph", 9111 "xaxis": { 9112 "buckets": null, 9113 "mode": "time", 9114 "name": null, 9115 "show": true, 9116 "values": [] 9117 }, 9118 "yaxes": [ 9119 { 9120 "format": "short", 9121 "label": null, 9122 "logBase": 1, 9123 "max": null, 9124 "min": null, 9125 "show": true 9126 }, 9127 { 9128 "format": "short", 9129 "label": null, 9130 "logBase": 1, 9131 "max": null, 9132 "min": null, 9133 "show": true 9134 } 9135 ], 9136 "yaxis": { 9137 "align": false, 9138 "alignLevel": null 9139 } 9140 }, 9141 { 9142 "cards": { 9143 "cardPadding": 0, 9144 "cardRound": 0 9145 }, 9146 "color": { 9147 "cardColor": "#b4ff00", 9148 "colorScale": "sqrt", 9149 "colorScheme": "interpolateSpectral", 9150 "exponent": 0.5, 9151 "max": null, 9152 "min": 0, 9153 "mode": "spectrum" 9154 }, 9155 "dataFormat": "tsbuckets", 9156 "datasource": "${DS_TEST-CLUSTER}", 9157 "description": "The latency distributions of write storage by a cloud storage sink", 9158 "fieldConfig": { 9159 "defaults": {}, 9160 "overrides": [] 9161 }, 9162 "gridPos": { 9163 "h": 8, 9164 "w": 12, 9165 "x": 0, 9166 "y": 16 9167 }, 9168 "heatmap": {}, 9169 "hideZeroBuckets": true, 9170 "highlightCards": true, 9171 "id": 10015, 9172 "legend": { 9173 "show": true 9174 }, 9175 "pluginVersion": "7.5.11", 9176 "reverseYBuckets": false, 9177 "targets": [ 9178 { 9179 "exemplar": true, 9180 "expr": "max(rate(ticdc_sink_cloud_storage_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)", 9181 "format": "heatmap", 9182 "hide": false, 9183 "interval": "", 9184 "intervalFactor": 2, 9185 "legendFormat": "{{le}}", 9186 "queryType": "randomWalk", 9187 "refId": "A" 9188 } 9189 ], 9190 "timeFrom": null, 9191 "timeShift": null, 9192 "title": "Write duration", 9193 "tooltip": { 9194 "show": true, 9195 "showHistogram": true 9196 }, 9197 "tooltipDecimals": 1, 9198 "type": "heatmap", 9199 "xAxis": { 9200 "show": true 9201 }, 9202 "xBucketNumber": null, 9203 "xBucketSize": null, 9204 "yAxis": { 9205 "decimals": 1, 9206 "format": "s", 9207 "logBase": 1, 9208 "max": null, 9209 "min": null, 9210 "show": true, 9211 "splitFactor": null 9212 }, 9213 "yBucketBound": "upper", 9214 "yBucketNumber": null, 9215 "yBucketSize": null 9216 }, 9217 { 9218 "cards": { 9219 "cardPadding": 0, 9220 "cardRound": 0 9221 }, 9222 "color": { 9223 "cardColor": "#b4ff00", 9224 "colorScale": "sqrt", 9225 "colorScheme": "interpolateSpectral", 9226 "exponent": 0.5, 9227 "max": null, 9228 "min": 0, 9229 "mode": "spectrum" 9230 }, 9231 "dataFormat": "tsbuckets", 9232 "datasource": "${DS_TEST-CLUSTER}", 9233 "description": "The latency distributions of flush storage by a cloud storage sink", 9234 "fieldConfig": { 9235 "defaults": {}, 9236 "overrides": [] 9237 }, 9238 "gridPos": { 9239 "h": 8, 9240 "w": 12, 9241 "x": 12, 9242 "y": 16 9243 }, 9244 "heatmap": {}, 9245 "hideZeroBuckets": true, 9246 "highlightCards": true, 9247 "id": 10017, 9248 "legend": { 9249 "show": true 9250 }, 9251 "pluginVersion": "7.5.11", 9252 "reverseYBuckets": false, 9253 "targets": [ 9254 { 9255 "exemplar": true, 9256 "expr": "max(rate(ticdc_sink_cloud_storage_flush_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)", 9257 "format": "heatmap", 9258 "hide": false, 9259 "interval": "", 9260 "intervalFactor": 2, 9261 "legendFormat": "{{le}}", 9262 "queryType": "randomWalk", 9263 "refId": "A" 9264 } 9265 ], 9266 "timeFrom": null, 9267 "timeShift": null, 9268 "title": "Flush duration", 9269 "tooltip": { 9270 "show": true, 9271 "showHistogram": true 9272 }, 9273 "tooltipDecimals": 1, 9274 "type": "heatmap", 9275 "xAxis": { 9276 "show": true 9277 }, 9278 "xBucketNumber": null, 9279 "xBucketSize": null, 9280 "yAxis": { 9281 "decimals": 1, 9282 "format": "s", 9283 "logBase": 1, 9284 "max": null, 9285 "min": null, 9286 "show": true, 9287 "splitFactor": null 9288 }, 9289 "yBucketBound": "upper", 9290 "yBucketNumber": null, 9291 "yBucketSize": null 9292 }, 9293 { 9294 "aliasColors": {}, 9295 "bars": false, 9296 "dashLength": 10, 9297 "dashes": false, 9298 "datasource": "${DS_TEST-CLUSTER}", 9299 "description": "Busy ratio (X ms in 1s) for cloud storage sink dml worker", 9300 "fieldConfig": { 9301 "defaults": {}, 9302 "overrides": [] 9303 }, 9304 "fill": 1, 9305 "fillGradient": 0, 9306 "gridPos": { 9307 "h": 8, 9308 "w": 12, 9309 "x": 0, 9310 "y": 24 9311 }, 9312 "hiddenSeries": false, 9313 "id": 10019, 9314 "legend": { 9315 "avg": true, 9316 "current": true, 9317 "max": false, 9318 "min": false, 9319 "show": true, 9320 "total": false, 9321 "values": true 9322 }, 9323 "lines": true, 9324 "linewidth": 1, 9325 "nullPointMode": "null", 9326 "options": { 9327 "alertThreshold": true 9328 }, 9329 "percentage": false, 9330 "pluginVersion": "7.5.11", 9331 "pointradius": 2, 9332 "points": false, 9333 "renderer": "flot", 9334 "seriesOverrides": [], 9335 "spaceLength": 10, 9336 "stack": false, 9337 "steppedLine": false, 9338 "targets": [ 9339 { 9340 "exemplar": true, 9341 "expr": "sum(rate(ticdc_sink_cloud_storage_worker_busy_ratio{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])/10) by (namespace,changefeed,id,instance)", 9342 "hide": false, 9343 "interval": "", 9344 "legendFormat": "{{namespace}}-{{changefeed}}-{{id}}-{{instance}}", 9345 "queryType": "randomWalk", 9346 "refId": "A" 9347 } 9348 ], 9349 "thresholds": [], 9350 "timeFrom": null, 9351 "timeRegions": [], 9352 "timeShift": null, 9353 "title": "Worker Busy Ratio", 9354 "tooltip": { 9355 "shared": true, 9356 "sort": 0, 9357 "value_type": "individual" 9358 }, 9359 "type": "graph", 9360 "xaxis": { 9361 "buckets": null, 9362 "mode": "time", 9363 "name": null, 9364 "show": true, 9365 "values": [] 9366 }, 9367 "yaxes": [ 9368 { 9369 "format": "percent", 9370 "label": null, 9371 "logBase": 1, 9372 "max": null, 9373 "min": null, 9374 "show": true 9375 }, 9376 { 9377 "format": "short", 9378 "label": null, 9379 "logBase": 1, 9380 "max": null, 9381 "min": null, 9382 "show": false 9383 } 9384 ], 9385 "yaxis": { 9386 "align": false, 9387 "alignLevel": null 9388 } 9389 } 9390 ], 9391 "title": "Sink - Cloud Storage Sink", 9392 "type": "row" 9393 }, 9394 { 9395 "collapsed": true, 9396 "datasource": null, 9397 "gridPos": { 9398 "h": 1, 9399 "w": 24, 9400 "x": 0, 9401 "y": 8 9402 }, 9403 "id": 528, 9404 "panels": [ 9405 { 9406 "aliasColors": {}, 9407 "bars": false, 9408 "dashLength": 10, 9409 "dashes": false, 9410 "datasource": "${DS_TEST-CLUSTER}", 9411 "description": "The lag between changefeed checkpoint ts and the latest ts of upstream TiDB.", 9412 "fieldConfig": { 9413 "defaults": { 9414 "links": [] 9415 }, 9416 "overrides": [] 9417 }, 9418 "fill": 1, 9419 "fillGradient": 0, 9420 "gridPos": { 9421 "h": 4, 9422 "w": 12, 9423 "x": 0, 9424 "y": 9 9425 }, 9426 "hiddenSeries": false, 9427 "id": 571, 9428 "legend": { 9429 "alignAsTable": false, 9430 "avg": false, 9431 "current": false, 9432 "max": false, 9433 "min": false, 9434 "rightSide": false, 9435 "show": true, 9436 "total": false, 9437 "values": false 9438 }, 9439 "lines": true, 9440 "linewidth": 1, 9441 "links": [], 9442 "nullPointMode": "null", 9443 "options": { 9444 "alertThreshold": true 9445 }, 9446 "paceLength": 10, 9447 "percentage": false, 9448 "pluginVersion": "7.5.11", 9449 "pointradius": 2, 9450 "points": false, 9451 "renderer": "flot", 9452 "seriesOverrides": [], 9453 "spaceLength": 10, 9454 "stack": false, 9455 "steppedLine": false, 9456 "targets": [ 9457 { 9458 "exemplar": true, 9459 "expr": "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)", 9460 "format": "time_series", 9461 "interval": "", 9462 "intervalFactor": 1, 9463 "legendFormat": "{{namespace}}-{{changefeed}}", 9464 "refId": "A" 9465 } 9466 ], 9467 "thresholds": [], 9468 "timeFrom": null, 9469 "timeRegions": [], 9470 "timeShift": null, 9471 "title": "Changefeed checkpoint lag", 9472 "tooltip": { 9473 "shared": true, 9474 "sort": 0, 9475 "value_type": "individual" 9476 }, 9477 "type": "graph", 9478 "xaxis": { 9479 "buckets": null, 9480 "mode": "time", 9481 "name": null, 9482 "show": true, 9483 "values": [] 9484 }, 9485 "yaxes": [ 9486 { 9487 "format": "s", 9488 "label": null, 9489 "logBase": 1, 9490 "max": null, 9491 "min": "0", 9492 "show": true 9493 }, 9494 { 9495 "format": "short", 9496 "label": null, 9497 "logBase": 1, 9498 "max": null, 9499 "min": null, 9500 "show": true 9501 } 9502 ], 9503 "yaxis": { 9504 "align": false, 9505 "alignLevel": null 9506 } 9507 }, 9508 { 9509 "aliasColors": {}, 9510 "bars": false, 9511 "dashLength": 10, 9512 "dashes": false, 9513 "datasource": "${DS_TEST-CLUSTER}", 9514 "description": "The lag between changefeed resolved ts and the latest ts of upstream TiDB.", 9515 "fieldConfig": { 9516 "defaults": { 9517 "links": [] 9518 }, 9519 "overrides": [] 9520 }, 9521 "fill": 1, 9522 "fillGradient": 0, 9523 "gridPos": { 9524 "h": 4, 9525 "w": 12, 9526 "x": 12, 9527 "y": 9 9528 }, 9529 "hiddenSeries": false, 9530 "id": 589, 9531 "legend": { 9532 "alignAsTable": false, 9533 "avg": false, 9534 "current": false, 9535 "max": false, 9536 "min": false, 9537 "rightSide": false, 9538 "show": true, 9539 "total": false, 9540 "values": false 9541 }, 9542 "lines": true, 9543 "linewidth": 1, 9544 "links": [], 9545 "nullPointMode": "null", 9546 "options": { 9547 "alertThreshold": true 9548 }, 9549 "paceLength": 10, 9550 "percentage": false, 9551 "pluginVersion": "7.5.11", 9552 "pointradius": 2, 9553 "points": false, 9554 "renderer": "flot", 9555 "seriesOverrides": [], 9556 "spaceLength": 10, 9557 "stack": false, 9558 "steppedLine": false, 9559 "targets": [ 9560 { 9561 "exemplar": true, 9562 "expr": "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)", 9563 "format": "time_series", 9564 "interval": "", 9565 "intervalFactor": 1, 9566 "legendFormat": "{{namespace}}-{{changefeed}}-resolvedts", 9567 "refId": "C" 9568 } 9569 ], 9570 "thresholds": [], 9571 "timeFrom": null, 9572 "timeRegions": [], 9573 "timeShift": null, 9574 "title": "Changefeed resolved ts lag", 9575 "tooltip": { 9576 "shared": true, 9577 "sort": 0, 9578 "value_type": "individual" 9579 }, 9580 "type": "graph", 9581 "xaxis": { 9582 "buckets": null, 9583 "mode": "time", 9584 "name": null, 9585 "show": true, 9586 "values": [] 9587 }, 9588 "yaxes": [ 9589 { 9590 "format": "s", 9591 "label": null, 9592 "logBase": 1, 9593 "max": null, 9594 "min": "0", 9595 "show": true 9596 }, 9597 { 9598 "format": "short", 9599 "label": null, 9600 "logBase": 1, 9601 "max": null, 9602 "min": null, 9603 "show": true 9604 } 9605 ], 9606 "yaxis": { 9607 "align": false, 9608 "alignLevel": null 9609 } 9610 }, 9611 { 9612 "aliasColors": {}, 9613 "bars": false, 9614 "dashLength": 10, 9615 "dashes": false, 9616 "datasource": "${DS_TEST-CLUSTER}", 9617 "decimals": 1, 9618 "description": "The total number of tables in different replication states.\n\n0: ReplicationSetStateUnknown means the replication state is unknown, it should not happen.\n\n1: ReplicationSetStateAbsent means there is no one replicates or prepares it.\n\n2: ReplicationSetStatePrepare means it needs to add a secondary.\n\n3: ReplicationSetStateCommit means it needs to promote secondary to primary.\n\n4: ReplicationSetStateReplicating means there is exactly one capture that is replicating the table.\n\n5: ReplicationSetStateRemoving means all captures need to stop replication eventually.\n\n", 9619 "editable": true, 9620 "error": false, 9621 "fieldConfig": { 9622 "defaults": { 9623 "unit": "none" 9624 }, 9625 "overrides": [] 9626 }, 9627 "fill": 0, 9628 "fillGradient": 0, 9629 "grid": {}, 9630 "gridPos": { 9631 "h": 6, 9632 "w": 12, 9633 "x": 0, 9634 "y": 13 9635 }, 9636 "hiddenSeries": false, 9637 "id": 546, 9638 "legend": { 9639 "alignAsTable": false, 9640 "avg": false, 9641 "current": false, 9642 "max": false, 9643 "min": false, 9644 "rightSide": false, 9645 "show": true, 9646 "sideWidth": null, 9647 "sort": null, 9648 "sortDesc": null, 9649 "total": false, 9650 "values": false 9651 }, 9652 "lines": true, 9653 "linewidth": 1, 9654 "links": [], 9655 "nullPointMode": "null", 9656 "options": { 9657 "alertThreshold": true 9658 }, 9659 "paceLength": 10, 9660 "percentage": false, 9661 "pluginVersion": "7.5.11", 9662 "pointradius": 1, 9663 "points": true, 9664 "renderer": "flot", 9665 "seriesOverrides": [], 9666 "spaceLength": 10, 9667 "stack": false, 9668 "steppedLine": false, 9669 "targets": [ 9670 { 9671 "exemplar": true, 9672 "expr": "sum(ticdc_scheduler_table_replication_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, state)", 9673 "format": "time_series", 9674 "hide": false, 9675 "interval": "", 9676 "intervalFactor": 2, 9677 "legendFormat": "{{namespace}}-{{changefeed}}-{{state}}", 9678 "refId": "A", 9679 "step": 10 9680 } 9681 ], 9682 "thresholds": [], 9683 "timeFrom": null, 9684 "timeRegions": [], 9685 "timeShift": null, 9686 "title": "Table replication state", 9687 "tooltip": { 9688 "msResolution": false, 9689 "shared": true, 9690 "sort": 0, 9691 "value_type": "individual" 9692 }, 9693 "type": "graph", 9694 "xaxis": { 9695 "buckets": null, 9696 "mode": "time", 9697 "name": null, 9698 "show": true, 9699 "values": [] 9700 }, 9701 "yaxes": [ 9702 { 9703 "format": "none", 9704 "label": null, 9705 "logBase": 1, 9706 "max": null, 9707 "min": "0", 9708 "show": true 9709 }, 9710 { 9711 "format": "none", 9712 "label": null, 9713 "logBase": 1, 9714 "max": null, 9715 "min": null, 9716 "show": false 9717 } 9718 ], 9719 "yaxis": { 9720 "align": false, 9721 "alignLevel": null 9722 } 9723 }, 9724 { 9725 "aliasColors": {}, 9726 "bars": false, 9727 "dashLength": 10, 9728 "dashes": false, 9729 "datasource": "${DS_TEST-CLUSTER}", 9730 "decimals": 1, 9731 "description": "The total number of accepted schedule tasks.", 9732 "editable": true, 9733 "error": false, 9734 "fieldConfig": { 9735 "defaults": { 9736 "unit": "none" 9737 }, 9738 "overrides": [] 9739 }, 9740 "fill": 0, 9741 "fillGradient": 0, 9742 "grid": {}, 9743 "gridPos": { 9744 "h": 6, 9745 "w": 12, 9746 "x": 12, 9747 "y": 13 9748 }, 9749 "hiddenSeries": false, 9750 "id": 549, 9751 "legend": { 9752 "alignAsTable": false, 9753 "avg": false, 9754 "current": false, 9755 "max": false, 9756 "min": false, 9757 "rightSide": false, 9758 "show": true, 9759 "sideWidth": null, 9760 "sort": null, 9761 "sortDesc": null, 9762 "total": false, 9763 "values": false 9764 }, 9765 "lines": true, 9766 "linewidth": 1, 9767 "links": [], 9768 "nullPointMode": "null", 9769 "options": { 9770 "alertThreshold": true 9771 }, 9772 "paceLength": 10, 9773 "percentage": false, 9774 "pluginVersion": "7.5.11", 9775 "pointradius": 1, 9776 "points": true, 9777 "renderer": "flot", 9778 "seriesOverrides": [], 9779 "spaceLength": 10, 9780 "stack": false, 9781 "steppedLine": false, 9782 "targets": [ 9783 { 9784 "exemplar": true, 9785 "expr": "sum(ticdc_scheduler_task_accept{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, scheduler, task)", 9786 "format": "time_series", 9787 "hide": false, 9788 "interval": "", 9789 "intervalFactor": 2, 9790 "legendFormat": "{{namespace}}-{{changefeed}}-{{task}}", 9791 "refId": "A", 9792 "step": 10 9793 } 9794 ], 9795 "thresholds": [], 9796 "timeFrom": null, 9797 "timeRegions": [], 9798 "timeShift": null, 9799 "title": "Schedule tasks accept", 9800 "tooltip": { 9801 "msResolution": false, 9802 "shared": true, 9803 "sort": 0, 9804 "value_type": "individual" 9805 }, 9806 "type": "graph", 9807 "xaxis": { 9808 "buckets": null, 9809 "mode": "time", 9810 "name": null, 9811 "show": true, 9812 "values": [] 9813 }, 9814 "yaxes": [ 9815 { 9816 "format": "none", 9817 "label": null, 9818 "logBase": 1, 9819 "max": null, 9820 "min": "0", 9821 "show": true 9822 }, 9823 { 9824 "format": "none", 9825 "label": null, 9826 "logBase": 1, 9827 "max": null, 9828 "min": null, 9829 "show": false 9830 } 9831 ], 9832 "yaxis": { 9833 "align": false, 9834 "alignLevel": null 9835 } 9836 }, 9837 { 9838 "aliasColors": {}, 9839 "bars": false, 9840 "dashLength": 10, 9841 "dashes": false, 9842 "datasource": "${DS_TEST-CLUSTER}", 9843 "decimals": 1, 9844 "description": "The total number of different schedule tasks.", 9845 "editable": true, 9846 "error": false, 9847 "fieldConfig": { 9848 "defaults": { 9849 "unit": "none" 9850 }, 9851 "overrides": [] 9852 }, 9853 "fill": 0, 9854 "fillGradient": 0, 9855 "grid": {}, 9856 "gridPos": { 9857 "h": 6, 9858 "w": 12, 9859 "x": 0, 9860 "y": 19 9861 }, 9862 "hiddenSeries": false, 9863 "id": 548, 9864 "legend": { 9865 "alignAsTable": false, 9866 "avg": false, 9867 "current": false, 9868 "max": false, 9869 "min": false, 9870 "rightSide": false, 9871 "show": true, 9872 "sideWidth": null, 9873 "sort": null, 9874 "sortDesc": null, 9875 "total": false, 9876 "values": false 9877 }, 9878 "lines": true, 9879 "linewidth": 1, 9880 "links": [], 9881 "nullPointMode": "null", 9882 "options": { 9883 "alertThreshold": true 9884 }, 9885 "paceLength": 10, 9886 "percentage": false, 9887 "pluginVersion": "7.5.11", 9888 "pointradius": 1, 9889 "points": true, 9890 "renderer": "flot", 9891 "seriesOverrides": [], 9892 "spaceLength": 10, 9893 "stack": false, 9894 "steppedLine": false, 9895 "targets": [ 9896 { 9897 "exemplar": true, 9898 "expr": "sum(ticdc_scheduler_task{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, scheduler, task)", 9899 "format": "time_series", 9900 "hide": false, 9901 "interval": "", 9902 "intervalFactor": 2, 9903 "legendFormat": "{{namespace}}-{{changefeed}}-{{scheduler}}-{{task}}", 9904 "refId": "A", 9905 "step": 10 9906 } 9907 ], 9908 "thresholds": [], 9909 "timeFrom": null, 9910 "timeRegions": [], 9911 "timeShift": null, 9912 "title": "Schedule tasks", 9913 "tooltip": { 9914 "msResolution": false, 9915 "shared": true, 9916 "sort": 0, 9917 "value_type": "individual" 9918 }, 9919 "type": "graph", 9920 "xaxis": { 9921 "buckets": null, 9922 "mode": "time", 9923 "name": null, 9924 "show": true, 9925 "values": [] 9926 }, 9927 "yaxes": [ 9928 { 9929 "format": "none", 9930 "label": null, 9931 "logBase": 1, 9932 "max": null, 9933 "min": "0", 9934 "show": true 9935 }, 9936 { 9937 "format": "none", 9938 "label": null, 9939 "logBase": 1, 9940 "max": null, 9941 "min": null, 9942 "show": false 9943 } 9944 ], 9945 "yaxis": { 9946 "align": false, 9947 "alignLevel": null 9948 } 9949 }, 9950 { 9951 "aliasColors": {}, 9952 "bars": false, 9953 "dashLength": 10, 9954 "dashes": false, 9955 "datasource": "${DS_TEST-CLUSTER}", 9956 "decimals": 1, 9957 "description": "The total number of running schedule tasks.", 9958 "editable": true, 9959 "error": false, 9960 "fieldConfig": { 9961 "defaults": { 9962 "unit": "none" 9963 }, 9964 "overrides": [] 9965 }, 9966 "fill": 0, 9967 "fillGradient": 0, 9968 "grid": {}, 9969 "gridPos": { 9970 "h": 6, 9971 "w": 12, 9972 "x": 12, 9973 "y": 19 9974 }, 9975 "hiddenSeries": false, 9976 "id": 588, 9977 "legend": { 9978 "alignAsTable": false, 9979 "avg": false, 9980 "current": false, 9981 "max": false, 9982 "min": false, 9983 "rightSide": false, 9984 "show": true, 9985 "sideWidth": null, 9986 "sort": null, 9987 "sortDesc": null, 9988 "total": false, 9989 "values": false 9990 }, 9991 "lines": true, 9992 "linewidth": 1, 9993 "links": [], 9994 "nullPointMode": "null", 9995 "options": { 9996 "alertThreshold": true 9997 }, 9998 "paceLength": 10, 9999 "percentage": false, 10000 "pluginVersion": "7.5.11", 10001 "pointradius": 1, 10002 "points": true, 10003 "renderer": "flot", 10004 "seriesOverrides": [], 10005 "spaceLength": 10, 10006 "stack": false, 10007 "steppedLine": false, 10008 "targets": [ 10009 { 10010 "exemplar": true, 10011 "expr": "sum(ticdc_scheduler_task_running{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, scheduler, task)", 10012 "format": "time_series", 10013 "hide": false, 10014 "interval": "", 10015 "intervalFactor": 2, 10016 "legendFormat": "{{namespace}}-{{changefeed}}-{{task}}", 10017 "refId": "A", 10018 "step": 10 10019 } 10020 ], 10021 "thresholds": [], 10022 "timeFrom": null, 10023 "timeRegions": [], 10024 "timeShift": null, 10025 "title": "Schedule tasks running", 10026 "tooltip": { 10027 "msResolution": false, 10028 "shared": true, 10029 "sort": 0, 10030 "value_type": "individual" 10031 }, 10032 "type": "graph", 10033 "xaxis": { 10034 "buckets": null, 10035 "mode": "time", 10036 "name": null, 10037 "show": true, 10038 "values": [] 10039 }, 10040 "yaxes": [ 10041 { 10042 "format": "none", 10043 "label": null, 10044 "logBase": 1, 10045 "max": null, 10046 "min": "0", 10047 "show": true 10048 }, 10049 { 10050 "format": "none", 10051 "label": null, 10052 "logBase": 1, 10053 "max": null, 10054 "min": null, 10055 "show": false 10056 } 10057 ], 10058 "yaxis": { 10059 "align": false, 10060 "alignLevel": null 10061 } 10062 }, 10063 { 10064 "aliasColors": {}, 10065 "bars": false, 10066 "dashLength": 10, 10067 "dashes": false, 10068 "datasource": "${DS_TEST-CLUSTER}", 10069 "description": "The total number of tables", 10070 "fieldConfig": { 10071 "defaults": { 10072 "links": [] 10073 }, 10074 "overrides": [] 10075 }, 10076 "fill": 0, 10077 "fillGradient": 0, 10078 "gridPos": { 10079 "h": 6, 10080 "w": 12, 10081 "x": 0, 10082 "y": 25 10083 }, 10084 "hiddenSeries": false, 10085 "id": 545, 10086 "legend": { 10087 "alignAsTable": false, 10088 "avg": false, 10089 "current": true, 10090 "max": true, 10091 "min": false, 10092 "rightSide": false, 10093 "show": true, 10094 "total": false, 10095 "values": true 10096 }, 10097 "lines": true, 10098 "linewidth": 1, 10099 "links": [], 10100 "nullPointMode": "null", 10101 "options": { 10102 "alertThreshold": false 10103 }, 10104 "paceLength": 10, 10105 "percentage": false, 10106 "pluginVersion": "7.5.11", 10107 "pointradius": 2, 10108 "points": true, 10109 "renderer": "flot", 10110 "seriesOverrides": [], 10111 "spaceLength": 10, 10112 "stack": false, 10113 "steppedLine": false, 10114 "targets": [ 10115 { 10116 "exemplar": true, 10117 "expr": "sum(ticdc_scheduler_table{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)", 10118 "format": "time_series", 10119 "interval": "", 10120 "intervalFactor": 1, 10121 "legendFormat": "{{namespace}}-{{changefeed}}", 10122 "refId": "A" 10123 } 10124 ], 10125 "thresholds": [], 10126 "timeFrom": null, 10127 "timeRegions": [], 10128 "timeShift": null, 10129 "title": "Table count", 10130 "tooltip": { 10131 "shared": true, 10132 "sort": 0, 10133 "value_type": "individual" 10134 }, 10135 "type": "graph", 10136 "xaxis": { 10137 "buckets": null, 10138 "mode": "time", 10139 "name": null, 10140 "show": true, 10141 "values": [] 10142 }, 10143 "yaxes": [ 10144 { 10145 "format": "short", 10146 "label": null, 10147 "logBase": 1, 10148 "max": null, 10149 "min": null, 10150 "show": true 10151 }, 10152 { 10153 "format": "short", 10154 "label": null, 10155 "logBase": 1, 10156 "max": null, 10157 "min": null, 10158 "show": true 10159 } 10160 ], 10161 "yaxis": { 10162 "align": false, 10163 "alignLevel": null 10164 } 10165 }, 10166 { 10167 "aliasColors": {}, 10168 "bars": false, 10169 "dashLength": 10, 10170 "dashes": false, 10171 "datasource": "${DS_TEST-CLUSTER}", 10172 "decimals": 1, 10173 "description": "The number of tables in different capture.", 10174 "editable": true, 10175 "error": false, 10176 "fieldConfig": { 10177 "defaults": { 10178 "unit": "none" 10179 }, 10180 "overrides": [] 10181 }, 10182 "fill": 0, 10183 "fillGradient": 0, 10184 "grid": {}, 10185 "gridPos": { 10186 "h": 6, 10187 "w": 12, 10188 "x": 12, 10189 "y": 25 10190 }, 10191 "hiddenSeries": false, 10192 "id": 547, 10193 "legend": { 10194 "alignAsTable": false, 10195 "avg": false, 10196 "current": false, 10197 "max": false, 10198 "min": false, 10199 "rightSide": false, 10200 "show": true, 10201 "sideWidth": null, 10202 "sort": null, 10203 "sortDesc": null, 10204 "total": false, 10205 "values": false 10206 }, 10207 "lines": true, 10208 "linewidth": 1, 10209 "links": [], 10210 "nullPointMode": "null", 10211 "options": { 10212 "alertThreshold": true 10213 }, 10214 "paceLength": 10, 10215 "percentage": false, 10216 "pluginVersion": "7.5.11", 10217 "pointradius": 1, 10218 "points": true, 10219 "renderer": "flot", 10220 "seriesOverrides": [], 10221 "spaceLength": 10, 10222 "stack": false, 10223 "steppedLine": false, 10224 "targets": [ 10225 { 10226 "exemplar": false, 10227 "expr": "sum(ticdc_scheduler_capture_table{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, addr)", 10228 "format": "time_series", 10229 "hide": false, 10230 "instant": false, 10231 "interval": "", 10232 "intervalFactor": 2, 10233 "legendFormat": "{{namespace}}-{{changefeed}}-{{addr}}", 10234 "refId": "A", 10235 "step": 10 10236 } 10237 ], 10238 "thresholds": [], 10239 "timeFrom": null, 10240 "timeRegions": [], 10241 "timeShift": null, 10242 "title": "Capture tables count", 10243 "tooltip": { 10244 "msResolution": false, 10245 "shared": true, 10246 "sort": 0, 10247 "value_type": "individual" 10248 }, 10249 "type": "graph", 10250 "xaxis": { 10251 "buckets": null, 10252 "mode": "time", 10253 "name": null, 10254 "show": true, 10255 "values": [] 10256 }, 10257 "yaxes": [ 10258 { 10259 "format": "none", 10260 "label": null, 10261 "logBase": 1, 10262 "max": null, 10263 "min": "0", 10264 "show": true 10265 }, 10266 { 10267 "format": "none", 10268 "label": null, 10269 "logBase": 1, 10270 "max": null, 10271 "min": null, 10272 "show": false 10273 } 10274 ], 10275 "yaxis": { 10276 "align": false, 10277 "alignLevel": null 10278 } 10279 }, 10280 { 10281 "aliasColors": {}, 10282 "bars": false, 10283 "dashLength": 10, 10284 "dashes": false, 10285 "datasource": "${DS_TEST-CLUSTER}", 10286 "decimals": 1, 10287 "description": "The ID of the slowest table", 10288 "editable": true, 10289 "error": false, 10290 "fieldConfig": { 10291 "defaults": {}, 10292 "overrides": [] 10293 }, 10294 "fill": 0, 10295 "fillGradient": 0, 10296 "grid": {}, 10297 "gridPos": { 10298 "h": 6, 10299 "w": 12, 10300 "x": 0, 10301 "y": 31 10302 }, 10303 "hiddenSeries": false, 10304 "id": 543, 10305 "legend": { 10306 "alignAsTable": true, 10307 "avg": false, 10308 "current": true, 10309 "max": false, 10310 "min": false, 10311 "rightSide": false, 10312 "show": true, 10313 "sideWidth": null, 10314 "sort": null, 10315 "sortDesc": null, 10316 "total": false, 10317 "values": true 10318 }, 10319 "lines": false, 10320 "linewidth": 1, 10321 "links": [], 10322 "nullPointMode": "null", 10323 "options": { 10324 "alertThreshold": true 10325 }, 10326 "paceLength": 10, 10327 "percentage": false, 10328 "pluginVersion": "7.5.11", 10329 "pointradius": 1, 10330 "points": true, 10331 "renderer": "flot", 10332 "seriesOverrides": [], 10333 "spaceLength": 10, 10334 "stack": false, 10335 "steppedLine": false, 10336 "targets": [ 10337 { 10338 "exemplar": true, 10339 "expr": "sum(ticdc_scheduler_slow_table_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", 10340 "format": "time_series", 10341 "hide": false, 10342 "interval": "", 10343 "intervalFactor": 2, 10344 "legendFormat": "{{namespace}}-{{changefeed}}", 10345 "refId": "A", 10346 "step": 10 10347 } 10348 ], 10349 "thresholds": [], 10350 "timeFrom": null, 10351 "timeRegions": [], 10352 "timeShift": null, 10353 "title": "Slowest table ID", 10354 "tooltip": { 10355 "msResolution": false, 10356 "shared": true, 10357 "sort": 0, 10358 "value_type": "individual" 10359 }, 10360 "type": "graph", 10361 "xaxis": { 10362 "buckets": null, 10363 "mode": "time", 10364 "name": null, 10365 "show": true, 10366 "values": [] 10367 }, 10368 "yaxes": [ 10369 { 10370 "format": "none", 10371 "label": null, 10372 "logBase": 1, 10373 "max": null, 10374 "min": "0", 10375 "show": true 10376 }, 10377 { 10378 "format": "none", 10379 "label": null, 10380 "logBase": 1, 10381 "max": null, 10382 "min": null, 10383 "show": false 10384 } 10385 ], 10386 "yaxis": { 10387 "align": false, 10388 "alignLevel": null 10389 } 10390 }, 10391 { 10392 "aliasColors": {}, 10393 "bars": false, 10394 "dashLength": 10, 10395 "dashes": false, 10396 "datasource": "${DS_TEST-CLUSTER}", 10397 "decimals": 1, 10398 "description": "The state of the slowest table.\n\n0: ReplicationSetStateUnknown means the replication state is unknown, it should not happen.\n\n1: ReplicationSetStateAbsent means there is no one replicates or prepares it.\n\n2: ReplicationSetStatePrepare means it needs to add a secondary.\n\n3: ReplicationSetStateCommit means it needs to promote secondary to primary.\n\n4: ReplicationSetStateReplicating means there is exactly one capture that is replicating the table.\n\n5: ReplicationSetStateRemoving means all captures need to stop replication eventually.\n\n", 10399 "editable": true, 10400 "error": false, 10401 "fieldConfig": { 10402 "defaults": { 10403 "unit": "none" 10404 }, 10405 "overrides": [] 10406 }, 10407 "fill": 0, 10408 "fillGradient": 0, 10409 "grid": {}, 10410 "gridPos": { 10411 "h": 6, 10412 "w": 12, 10413 "x": 12, 10414 "y": 31 10415 }, 10416 "hiddenSeries": false, 10417 "id": 544, 10418 "legend": { 10419 "alignAsTable": false, 10420 "avg": false, 10421 "current": false, 10422 "max": false, 10423 "min": false, 10424 "rightSide": false, 10425 "show": true, 10426 "sideWidth": null, 10427 "sort": null, 10428 "sortDesc": null, 10429 "total": false, 10430 "values": false 10431 }, 10432 "lines": false, 10433 "linewidth": 1, 10434 "links": [], 10435 "nullPointMode": "null", 10436 "options": { 10437 "alertThreshold": true 10438 }, 10439 "paceLength": 10, 10440 "percentage": false, 10441 "pluginVersion": "7.5.11", 10442 "pointradius": 1, 10443 "points": true, 10444 "renderer": "flot", 10445 "seriesOverrides": [], 10446 "spaceLength": 10, 10447 "stack": false, 10448 "steppedLine": false, 10449 "targets": [ 10450 { 10451 "exemplar": true, 10452 "expr": "sum(ticdc_scheduler_slow_table_replication_state{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)", 10453 "format": "time_series", 10454 "hide": false, 10455 "interval": "", 10456 "intervalFactor": 2, 10457 "legendFormat": "{{namespace}}-{{changefeed}}", 10458 "refId": "A", 10459 "step": 10 10460 } 10461 ], 10462 "thresholds": [], 10463 "timeFrom": null, 10464 "timeRegions": [], 10465 "timeShift": null, 10466 "title": "Slowest table replication state", 10467 "tooltip": { 10468 "msResolution": false, 10469 "shared": true, 10470 "sort": 0, 10471 "value_type": "individual" 10472 }, 10473 "type": "graph", 10474 "xaxis": { 10475 "buckets": null, 10476 "mode": "time", 10477 "name": null, 10478 "show": true, 10479 "values": [] 10480 }, 10481 "yaxes": [ 10482 { 10483 "format": "none", 10484 "label": null, 10485 "logBase": 1, 10486 "max": null, 10487 "min": "0", 10488 "show": true 10489 }, 10490 { 10491 "format": "none", 10492 "label": null, 10493 "logBase": 1, 10494 "max": null, 10495 "min": null, 10496 "show": false 10497 } 10498 ], 10499 "yaxis": { 10500 "align": false, 10501 "alignLevel": null 10502 } 10503 }, 10504 { 10505 "aliasColors": {}, 10506 "bars": true, 10507 "cacheTimeout": null, 10508 "dashLength": 10, 10509 "dashes": false, 10510 "datasource": "${DS_TEST-CLUSTER}", 10511 "description": "The checkpoint ts of the slowest table.", 10512 "fieldConfig": { 10513 "defaults": { 10514 "unit": "dateTimeAsIso" 10515 }, 10516 "overrides": [] 10517 }, 10518 "fill": 0, 10519 "fillGradient": 0, 10520 "gridPos": { 10521 "h": 6, 10522 "w": 12, 10523 "x": 0, 10524 "y": 37 10525 }, 10526 "hiddenSeries": false, 10527 "id": 541, 10528 "legend": { 10529 "alignAsTable": true, 10530 "avg": false, 10531 "current": true, 10532 "max": false, 10533 "min": false, 10534 "rightSide": false, 10535 "show": true, 10536 "sideWidth": null, 10537 "total": false, 10538 "values": true 10539 }, 10540 "lines": true, 10541 "linewidth": 2, 10542 "links": [], 10543 "nullPointMode": "null", 10544 "options": { 10545 "alertThreshold": true 10546 }, 10547 "percentage": false, 10548 "pluginVersion": "7.5.11", 10549 "pointradius": 2, 10550 "points": false, 10551 "renderer": "flot", 10552 "seriesOverrides": [ 10553 { 10554 "alias": "/approximate current time.*/", 10555 "bars": false 10556 } 10557 ], 10558 "spaceLength": 10, 10559 "stack": false, 10560 "steppedLine": false, 10561 "targets": [ 10562 { 10563 "exemplar": true, 10564 "expr": "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", 10565 "format": "time_series", 10566 "hide": false, 10567 "interval": "", 10568 "intervalFactor": 1, 10569 "legendFormat": "approximate current time (s)", 10570 "refId": "A" 10571 }, 10572 { 10573 "exemplar": true, 10574 "expr": "max(ticdc_scheduler_slow_table_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", 10575 "format": "time_series", 10576 "hide": false, 10577 "interval": "", 10578 "intervalFactor": 1, 10579 "legendFormat": "{{namespace}}-{{changefeed}}", 10580 "refId": "B" 10581 } 10582 ], 10583 "thresholds": [], 10584 "timeFrom": null, 10585 "timeRegions": [], 10586 "timeShift": null, 10587 "title": "Slowest table checkpoint", 10588 "tooltip": { 10589 "shared": true, 10590 "sort": 0, 10591 "value_type": "individual" 10592 }, 10593 "type": "graph", 10594 "xaxis": { 10595 "buckets": null, 10596 "max": null, 10597 "mode": "time", 10598 "name": null, 10599 "show": true, 10600 "values": [] 10601 }, 10602 "yaxes": [ 10603 { 10604 "format": "dateTimeAsIso", 10605 "label": null, 10606 "logBase": 1, 10607 "max": null, 10608 "min": null, 10609 "show": true 10610 }, 10611 { 10612 "format": "short", 10613 "label": null, 10614 "logBase": 1, 10615 "max": null, 10616 "min": null, 10617 "show": false 10618 } 10619 ], 10620 "yaxis": { 10621 "align": false, 10622 "alignLevel": null 10623 } 10624 }, 10625 { 10626 "aliasColors": {}, 10627 "bars": true, 10628 "cacheTimeout": null, 10629 "dashLength": 10, 10630 "dashes": false, 10631 "datasource": "${DS_TEST-CLUSTER}", 10632 "description": "The resolved ts of the slowest table.", 10633 "fieldConfig": { 10634 "defaults": {}, 10635 "overrides": [] 10636 }, 10637 "fill": 0, 10638 "fillGradient": 0, 10639 "gridPos": { 10640 "h": 6, 10641 "w": 12, 10642 "x": 12, 10643 "y": 37 10644 }, 10645 "hiddenSeries": false, 10646 "id": 590, 10647 "legend": { 10648 "alignAsTable": true, 10649 "avg": false, 10650 "current": true, 10651 "max": false, 10652 "min": false, 10653 "rightSide": false, 10654 "show": true, 10655 "sideWidth": null, 10656 "total": false, 10657 "values": true 10658 }, 10659 "lines": true, 10660 "linewidth": 2, 10661 "links": [], 10662 "nullPointMode": "null", 10663 "options": { 10664 "alertThreshold": true 10665 }, 10666 "percentage": false, 10667 "pluginVersion": "7.5.11", 10668 "pointradius": 2, 10669 "points": false, 10670 "renderer": "flot", 10671 "seriesOverrides": [ 10672 { 10673 "alias": "/approximate current time.*/", 10674 "bars": false 10675 } 10676 ], 10677 "spaceLength": 10, 10678 "stack": false, 10679 "steppedLine": false, 10680 "targets": [ 10681 { 10682 "exemplar": true, 10683 "expr": "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", 10684 "format": "time_series", 10685 "hide": false, 10686 "interval": "", 10687 "intervalFactor": 1, 10688 "legendFormat": "approximate current time (s)", 10689 "refId": "A" 10690 }, 10691 { 10692 "exemplar": true, 10693 "expr": "max(ticdc_scheduler_slow_table_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", 10694 "format": "time_series", 10695 "hide": false, 10696 "interval": "", 10697 "intervalFactor": 1, 10698 "legendFormat": "{{namespace}}-{{changefeed}}", 10699 "refId": "B" 10700 } 10701 ], 10702 "thresholds": [], 10703 "timeFrom": null, 10704 "timeRegions": [], 10705 "timeShift": null, 10706 "title": "Slowest table resolved ts", 10707 "tooltip": { 10708 "shared": true, 10709 "sort": 0, 10710 "value_type": "individual" 10711 }, 10712 "type": "graph", 10713 "xaxis": { 10714 "buckets": null, 10715 "max": null, 10716 "mode": "time", 10717 "name": null, 10718 "show": true, 10719 "values": [] 10720 }, 10721 "yaxes": [ 10722 { 10723 "format": "dateTimeAsIso", 10724 "label": null, 10725 "logBase": 1, 10726 "max": null, 10727 "min": null, 10728 "show": true 10729 }, 10730 { 10731 "format": "short", 10732 "label": null, 10733 "logBase": 1, 10734 "max": null, 10735 "min": null, 10736 "show": false 10737 } 10738 ], 10739 "yaxis": { 10740 "align": false, 10741 "alignLevel": null 10742 } 10743 }, 10744 { 10745 "aliasColors": {}, 10746 "bars": false, 10747 "dashLength": 10, 10748 "dashes": false, 10749 "datasource": "${DS_TEST-CLUSTER}", 10750 "decimals": 1, 10751 "description": "The number of incremental scan tasks per minute", 10752 "fieldConfig": { 10753 "defaults": {}, 10754 "overrides": [] 10755 }, 10756 "fill": 1, 10757 "fillGradient": 0, 10758 "gridPos": { 10759 "h": 6, 10760 "w": 12, 10761 "x": 0, 10762 "y": 43 10763 }, 10764 "hiddenSeries": false, 10765 "id": 560, 10766 "legend": { 10767 "alignAsTable": false, 10768 "avg": false, 10769 "current": false, 10770 "hideEmpty": false, 10771 "hideZero": true, 10772 "max": false, 10773 "min": false, 10774 "rightSide": false, 10775 "show": true, 10776 "sideWidth": null, 10777 "sort": "current", 10778 "sortDesc": true, 10779 "total": false, 10780 "values": false 10781 }, 10782 "lines": true, 10783 "linewidth": 1, 10784 "links": [], 10785 "nullPointMode": "null", 10786 "options": { 10787 "alertThreshold": true 10788 }, 10789 "paceLength": 10, 10790 "percentage": false, 10791 "pluginVersion": "7.5.11", 10792 "pointradius": 5, 10793 "points": false, 10794 "renderer": "flot", 10795 "seriesOverrides": [], 10796 "spaceLength": 10, 10797 "stack": false, 10798 "steppedLine": false, 10799 "targets": [ 10800 { 10801 "exemplar": true, 10802 "expr": "sum(increase(tikv_cdc_scan_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (type, instance)", 10803 "format": "time_series", 10804 "hide": false, 10805 "interval": "", 10806 "intervalFactor": 1, 10807 "legendFormat": "{{instance}}", 10808 "refId": "A" 10809 } 10810 ], 10811 "thresholds": [], 10812 "timeFrom": null, 10813 "timeRegions": [], 10814 "timeShift": null, 10815 "title": "TiKV scan tasks/m", 10816 "tooltip": { 10817 "shared": true, 10818 "sort": 0, 10819 "value_type": "individual" 10820 }, 10821 "type": "graph", 10822 "xaxis": { 10823 "buckets": null, 10824 "mode": "time", 10825 "name": null, 10826 "show": true, 10827 "values": [] 10828 }, 10829 "yaxes": [ 10830 { 10831 "format": "none", 10832 "label": null, 10833 "logBase": 1, 10834 "max": null, 10835 "min": null, 10836 "show": true 10837 }, 10838 { 10839 "format": "none", 10840 "label": null, 10841 "logBase": 1, 10842 "max": null, 10843 "min": null, 10844 "show": true 10845 } 10846 ], 10847 "yaxis": { 10848 "align": false, 10849 "alignLevel": null 10850 } 10851 } 10852 ], 10853 "title": "Scheduler", 10854 "type": "row" 10855 }, 10856 { 10857 "collapsed": true, 10858 "datasource": null, 10859 "gridPos": { 10860 "h": 1, 10861 "w": 24, 10862 "x": 0, 10863 "y": 9 10864 }, 10865 "id": 677, 10866 "panels": [ 10867 { 10868 "aliasColors": {}, 10869 "bars": false, 10870 "dashLength": 10, 10871 "dashes": false, 10872 "datasource": "${DS_TEST-CLUSTER}", 10873 "decimals": 1, 10874 "description": "The ID of the slowest table", 10875 "editable": true, 10876 "error": false, 10877 "fieldConfig": { 10878 "defaults": {}, 10879 "overrides": [] 10880 }, 10881 "fill": 0, 10882 "fillGradient": 0, 10883 "grid": {}, 10884 "gridPos": { 10885 "h": 6, 10886 "w": 12, 10887 "x": 0, 10888 "y": 10 10889 }, 10890 "hiddenSeries": false, 10891 "id": 689, 10892 "legend": { 10893 "alignAsTable": false, 10894 "avg": false, 10895 "current": true, 10896 "max": false, 10897 "min": false, 10898 "rightSide": false, 10899 "show": true, 10900 "sideWidth": null, 10901 "sort": null, 10902 "sortDesc": null, 10903 "total": false, 10904 "values": true 10905 }, 10906 "lines": false, 10907 "linewidth": 1, 10908 "links": [], 10909 "nullPointMode": "null", 10910 "options": { 10911 "alertThreshold": true 10912 }, 10913 "paceLength": 10, 10914 "percentage": false, 10915 "pluginVersion": "7.5.11", 10916 "pointradius": 1, 10917 "points": true, 10918 "renderer": "flot", 10919 "seriesOverrides": [], 10920 "spaceLength": 10, 10921 "stack": false, 10922 "steppedLine": false, 10923 "targets": [ 10924 { 10925 "exemplar": true, 10926 "expr": "sum(ticdc_scheduler_slow_table_id{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", 10927 "format": "time_series", 10928 "hide": false, 10929 "interval": "", 10930 "intervalFactor": 2, 10931 "legendFormat": "{{namespace}}-{{changefeed}}", 10932 "refId": "A", 10933 "step": 10 10934 } 10935 ], 10936 "thresholds": [], 10937 "timeFrom": null, 10938 "timeRegions": [], 10939 "timeShift": null, 10940 "title": "Slowest table ID", 10941 "tooltip": { 10942 "msResolution": false, 10943 "shared": true, 10944 "sort": 0, 10945 "value_type": "individual" 10946 }, 10947 "type": "graph", 10948 "xaxis": { 10949 "buckets": null, 10950 "mode": "time", 10951 "name": null, 10952 "show": true, 10953 "values": [] 10954 }, 10955 "yaxes": [ 10956 { 10957 "format": "none", 10958 "label": null, 10959 "logBase": 1, 10960 "max": null, 10961 "min": "0", 10962 "show": true 10963 }, 10964 { 10965 "format": "none", 10966 "label": null, 10967 "logBase": 1, 10968 "max": null, 10969 "min": null, 10970 "show": false 10971 } 10972 ], 10973 "yaxis": { 10974 "align": false, 10975 "alignLevel": null 10976 } 10977 }, 10978 { 10979 "aliasColors": {}, 10980 "bars": false, 10981 "dashLength": 10, 10982 "dashes": false, 10983 "datasource": "${DS_TEST-CLUSTER}", 10984 "decimals": 1, 10985 "description": "The number of regions captured by the slowest table.", 10986 "editable": true, 10987 "error": false, 10988 "fieldConfig": { 10989 "defaults": {}, 10990 "overrides": [] 10991 }, 10992 "fill": 0, 10993 "fillGradient": 0, 10994 "grid": {}, 10995 "gridPos": { 10996 "h": 6, 10997 "w": 12, 10998 "x": 12, 10999 "y": 10 11000 }, 11001 "hiddenSeries": false, 11002 "id": 691, 11003 "legend": { 11004 "alignAsTable": false, 11005 "avg": false, 11006 "current": true, 11007 "max": false, 11008 "min": false, 11009 "rightSide": false, 11010 "show": true, 11011 "sideWidth": null, 11012 "sort": null, 11013 "sortDesc": null, 11014 "total": false, 11015 "values": true 11016 }, 11017 "lines": false, 11018 "linewidth": 1, 11019 "links": [], 11020 "nullPointMode": "null", 11021 "options": { 11022 "alertThreshold": true 11023 }, 11024 "paceLength": 10, 11025 "percentage": false, 11026 "pluginVersion": "7.5.11", 11027 "pointradius": 1, 11028 "points": true, 11029 "renderer": "flot", 11030 "seriesOverrides": [], 11031 "spaceLength": 10, 11032 "stack": false, 11033 "steppedLine": false, 11034 "targets": [ 11035 { 11036 "exemplar": true, 11037 "expr": "sum(ticdc_scheduler_slow_table_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed)", 11038 "format": "time_series", 11039 "hide": false, 11040 "interval": "", 11041 "intervalFactor": 2, 11042 "legendFormat": "{{namespace}}-{{changefeed}}", 11043 "refId": "A", 11044 "step": 10 11045 } 11046 ], 11047 "thresholds": [], 11048 "timeFrom": null, 11049 "timeRegions": [], 11050 "timeShift": null, 11051 "title": "Slowest table region count", 11052 "tooltip": { 11053 "msResolution": false, 11054 "shared": true, 11055 "sort": 0, 11056 "value_type": "individual" 11057 }, 11058 "type": "graph", 11059 "xaxis": { 11060 "buckets": null, 11061 "mode": "time", 11062 "name": null, 11063 "show": true, 11064 "values": [] 11065 }, 11066 "yaxes": [ 11067 { 11068 "format": "none", 11069 "label": null, 11070 "logBase": 1, 11071 "max": null, 11072 "min": "0", 11073 "show": true 11074 }, 11075 { 11076 "format": "none", 11077 "label": null, 11078 "logBase": 1, 11079 "max": null, 11080 "min": null, 11081 "show": false 11082 } 11083 ], 11084 "yaxis": { 11085 "align": false, 11086 "alignLevel": null 11087 } 11088 }, 11089 { 11090 "aliasColors": {}, 11091 "bars": false, 11092 "cacheTimeout": null, 11093 "dashLength": 10, 11094 "dashes": false, 11095 "datasource": "${DS_TEST-CLUSTER}", 11096 "description": "Checkpoint ts of the slowest table each stage.", 11097 "fieldConfig": { 11098 "defaults": { 11099 "unit": "dateTimeAsIso" 11100 }, 11101 "overrides": [] 11102 }, 11103 "fill": 1, 11104 "fillGradient": 0, 11105 "gridPos": { 11106 "h": 6, 11107 "w": 12, 11108 "x": 0, 11109 "y": 16 11110 }, 11111 "hiddenSeries": false, 11112 "id": 693, 11113 "legend": { 11114 "avg": false, 11115 "current": false, 11116 "max": false, 11117 "min": false, 11118 "show": true, 11119 "total": false, 11120 "values": false 11121 }, 11122 "lines": true, 11123 "linewidth": 1, 11124 "links": [], 11125 "nullPointMode": "null", 11126 "options": { 11127 "alertThreshold": true 11128 }, 11129 "percentage": false, 11130 "pluginVersion": "7.5.11", 11131 "pointradius": 2, 11132 "points": false, 11133 "renderer": "flot", 11134 "seriesOverrides": [], 11135 "spaceLength": 10, 11136 "stack": false, 11137 "steppedLine": false, 11138 "targets": [ 11139 { 11140 "exemplar": true, 11141 "expr": "max(ticdc_owner_current_pd_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", 11142 "format": "time_series", 11143 "hide": false, 11144 "interval": "", 11145 "intervalFactor": 1, 11146 "legendFormat": "approximate current time (s)", 11147 "refId": "A" 11148 }, 11149 { 11150 "exemplar": true, 11151 "expr": "sum(ticdc_scheduler_slow_table_stage_checkpoint_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, stage)", 11152 "format": "time_series", 11153 "hide": false, 11154 "interval": "", 11155 "intervalFactor": 1, 11156 "legendFormat": "{{namespace}}-{{changefeed}}-{{stage}}", 11157 "refId": "B" 11158 } 11159 ], 11160 "thresholds": [], 11161 "timeFrom": null, 11162 "timeRegions": [], 11163 "timeShift": null, 11164 "title": "Slow table checkpoint ts", 11165 "tooltip": { 11166 "shared": true, 11167 "sort": 0, 11168 "value_type": "individual" 11169 }, 11170 "type": "graph", 11171 "xaxis": { 11172 "buckets": null, 11173 "mode": "time", 11174 "name": null, 11175 "show": true, 11176 "values": [] 11177 }, 11178 "yaxes": [ 11179 { 11180 "format": "dateTimeAsIso", 11181 "label": null, 11182 "logBase": 1, 11183 "max": null, 11184 "min": null, 11185 "show": true 11186 }, 11187 { 11188 "format": "short", 11189 "label": null, 11190 "logBase": 1, 11191 "max": null, 11192 "min": null, 11193 "show": true 11194 } 11195 ], 11196 "yaxis": { 11197 "align": false, 11198 "alignLevel": null 11199 } 11200 }, 11201 { 11202 "aliasColors": {}, 11203 "bars": false, 11204 "cacheTimeout": null, 11205 "dashLength": 10, 11206 "dashes": false, 11207 "datasource": "${DS_TEST-CLUSTER}", 11208 "description": "Resolved ts of the slowest table each stage.", 11209 "fieldConfig": { 11210 "defaults": { 11211 "unit": "dateTimeAsIso" 11212 }, 11213 "overrides": [] 11214 }, 11215 "fill": 1, 11216 "fillGradient": 0, 11217 "gridPos": { 11218 "h": 6, 11219 "w": 12, 11220 "x": 12, 11221 "y": 16 11222 }, 11223 "hiddenSeries": false, 11224 "id": 695, 11225 "legend": { 11226 "avg": false, 11227 "current": false, 11228 "max": false, 11229 "min": false, 11230 "show": true, 11231 "total": false, 11232 "values": false 11233 }, 11234 "lines": true, 11235 "linewidth": 1, 11236 "links": [], 11237 "nullPointMode": "null", 11238 "options": { 11239 "alertThreshold": true 11240 }, 11241 "percentage": false, 11242 "pluginVersion": "7.5.11", 11243 "pointradius": 2, 11244 "points": false, 11245 "renderer": "flot", 11246 "seriesOverrides": [], 11247 "spaceLength": 10, 11248 "stack": false, 11249 "steppedLine": false, 11250 "targets": [ 11251 { 11252 "exemplar": true, 11253 "expr": "max(ticdc_owner_current_pd_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", 11254 "format": "time_series", 11255 "hide": false, 11256 "interval": "", 11257 "intervalFactor": 1, 11258 "legendFormat": "approximate current time (s)", 11259 "refId": "A" 11260 }, 11261 { 11262 "exemplar": true, 11263 "expr": "sum(ticdc_scheduler_slow_table_stage_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, stage)", 11264 "format": "time_series", 11265 "hide": false, 11266 "interval": "", 11267 "intervalFactor": 1, 11268 "legendFormat": "{{namespace}}-{{changefeed}}-{{stage}}", 11269 "refId": "B" 11270 } 11271 ], 11272 "thresholds": [], 11273 "timeFrom": null, 11274 "timeRegions": [], 11275 "timeShift": null, 11276 "title": "Slow table resolved ts", 11277 "tooltip": { 11278 "shared": true, 11279 "sort": 0, 11280 "value_type": "individual" 11281 }, 11282 "type": "graph", 11283 "xaxis": { 11284 "buckets": null, 11285 "mode": "time", 11286 "name": null, 11287 "show": true, 11288 "values": [] 11289 }, 11290 "yaxes": [ 11291 { 11292 "format": "dateTimeAsIso", 11293 "label": null, 11294 "logBase": 1, 11295 "max": null, 11296 "min": null, 11297 "show": true 11298 }, 11299 { 11300 "format": "short", 11301 "label": null, 11302 "logBase": 1, 11303 "max": null, 11304 "min": null, 11305 "show": true 11306 } 11307 ], 11308 "yaxis": { 11309 "align": false, 11310 "alignLevel": null 11311 } 11312 }, 11313 { 11314 "aliasColors": {}, 11315 "bars": false, 11316 "dashLength": 10, 11317 "dashes": false, 11318 "datasource": "${DS_TEST-CLUSTER}", 11319 "description": "The details of changefeed lag including resolved ts and checkpoint ts in different stages.", 11320 "fieldConfig": { 11321 "defaults": { 11322 "links": [] 11323 }, 11324 "overrides": [] 11325 }, 11326 "fill": 1, 11327 "fillGradient": 0, 11328 "gridPos": { 11329 "h": 6, 11330 "w": 24, 11331 "x": 0, 11332 "y": 22 11333 }, 11334 "hiddenSeries": false, 11335 "id": 699, 11336 "legend": { 11337 "alignAsTable": false, 11338 "avg": false, 11339 "current": true, 11340 "max": false, 11341 "min": false, 11342 "rightSide": false, 11343 "show": true, 11344 "total": false, 11345 "values": true 11346 }, 11347 "lines": true, 11348 "linewidth": 1, 11349 "links": [], 11350 "nullPointMode": "null", 11351 "options": { 11352 "alertThreshold": true 11353 }, 11354 "paceLength": 10, 11355 "percentage": false, 11356 "pluginVersion": "7.5.11", 11357 "pointradius": 2, 11358 "points": false, 11359 "renderer": "flot", 11360 "seriesOverrides": [], 11361 "spaceLength": 10, 11362 "stack": false, 11363 "steppedLine": false, 11364 "targets": [ 11365 { 11366 "exemplar": true, 11367 "expr": "max(ticdc_scheduler_slow_table_stage_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\"}) by (namespace,changefeed, stage)", 11368 "format": "time_series", 11369 "interval": "", 11370 "intervalFactor": 1, 11371 "legendFormat": "{{namespace}}-{{changefeed}}-{{stage}}", 11372 "refId": "A" 11373 }, 11374 { 11375 "exemplar": true, 11376 "expr": "max(tikv_cdc_min_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000)", 11377 "hide": false, 11378 "interval": "", 11379 "legendFormat": "tikv", 11380 "refId": "B" 11381 }, 11382 { 11383 "exemplar": true, 11384 "expr": "max(ticdc_scheduler_slow_table_stage_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", stage=\"sink\"}) by (namespace,changefeed)", 11385 "hide": false, 11386 "interval": "", 11387 "legendFormat": "sink-checkpoint", 11388 "refId": "C" 11389 } 11390 ], 11391 "thresholds": [], 11392 "timeFrom": null, 11393 "timeRegions": [], 11394 "timeShift": null, 11395 "title": "Slow table lag details", 11396 "tooltip": { 11397 "shared": true, 11398 "sort": 0, 11399 "value_type": "individual" 11400 }, 11401 "type": "graph", 11402 "xaxis": { 11403 "buckets": null, 11404 "mode": "time", 11405 "name": null, 11406 "show": true, 11407 "values": [] 11408 }, 11409 "yaxes": [ 11410 { 11411 "format": "s", 11412 "label": null, 11413 "logBase": 1, 11414 "max": null, 11415 "min": "0", 11416 "show": true 11417 }, 11418 { 11419 "format": "short", 11420 "label": null, 11421 "logBase": 1, 11422 "max": null, 11423 "min": null, 11424 "show": true 11425 } 11426 ], 11427 "yaxis": { 11428 "align": false, 11429 "alignLevel": null 11430 } 11431 }, 11432 { 11433 "aliasColors": {}, 11434 "bars": false, 11435 "dashLength": 10, 11436 "dashes": false, 11437 "datasource": "${DS_TEST-CLUSTER}", 11438 "description": "Lag distribution of the slowest table in different stages.", 11439 "fieldConfig": { 11440 "defaults": {}, 11441 "overrides": [] 11442 }, 11443 "fill": 1, 11444 "fillGradient": 0, 11445 "gridPos": { 11446 "h": 6, 11447 "w": 24, 11448 "x": 0, 11449 "y": 28 11450 }, 11451 "hiddenSeries": false, 11452 "id": 703, 11453 "legend": { 11454 "alignAsTable": false, 11455 "avg": false, 11456 "current": true, 11457 "max": true, 11458 "min": false, 11459 "rightSide": false, 11460 "show": true, 11461 "total": false, 11462 "values": true 11463 }, 11464 "lines": true, 11465 "linewidth": 1, 11466 "nullPointMode": "null", 11467 "options": { 11468 "alertThreshold": true 11469 }, 11470 "percentage": false, 11471 "pluginVersion": "7.5.11", 11472 "pointradius": 2, 11473 "points": false, 11474 "renderer": "flot", 11475 "seriesOverrides": [], 11476 "spaceLength": 10, 11477 "stack": false, 11478 "steppedLine": false, 11479 "targets": [ 11480 { 11481 "exemplar": true, 11482 "expr": "histogram_quantile(0.99, sum(rate(ticdc_scheduler_slow_table_stage_resolved_ts_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[100s])) by (le,namespace,changefeed,stage))", 11483 "hide": false, 11484 "interval": "", 11485 "legendFormat": "{{namespace}}-{{changefeed}}-{{stage}}-p99", 11486 "refId": "A" 11487 }, 11488 { 11489 "exemplar": true, 11490 "expr": "histogram_quantile(0.99, sum(rate(ticdc_scheduler_slow_table_stage_checkpoint_ts_lag_histogram_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", stage=\"sink\"}[100s])) by (le,namespace,changefeed,stage))", 11491 "hide": false, 11492 "interval": "", 11493 "legendFormat": "{{namespace}}-{{changefeed}}-{{stage}}-checkpoint-p99", 11494 "refId": "B" 11495 } 11496 ], 11497 "thresholds": [], 11498 "timeFrom": null, 11499 "timeRegions": [], 11500 "timeShift": null, 11501 "title": "Slow table lag duration percentile", 11502 "tooltip": { 11503 "shared": true, 11504 "sort": 0, 11505 "value_type": "individual" 11506 }, 11507 "type": "graph", 11508 "xaxis": { 11509 "buckets": null, 11510 "mode": "time", 11511 "name": null, 11512 "show": true, 11513 "values": [] 11514 }, 11515 "yaxes": [ 11516 { 11517 "format": "s", 11518 "label": null, 11519 "logBase": 1, 11520 "max": null, 11521 "min": null, 11522 "show": true 11523 }, 11524 { 11525 "format": "none", 11526 "label": null, 11527 "logBase": 1, 11528 "max": null, 11529 "min": null, 11530 "show": true 11531 } 11532 ], 11533 "yaxis": { 11534 "align": false, 11535 "alignLevel": null 11536 } 11537 } 11538 ], 11539 "title": "Slow Table", 11540 "type": "row" 11541 }, 11542 { 11543 "collapsed": true, 11544 "datasource": null, 11545 "gridPos": { 11546 "h": 1, 11547 "w": 24, 11548 "x": 0, 11549 "y": 10 11550 }, 11551 "id": 269, 11552 "panels": [ 11553 { 11554 "aliasColors": {}, 11555 "bars": false, 11556 "dashLength": 10, 11557 "dashes": false, 11558 "datasource": "${DS_TEST-CLUSTER}", 11559 "fieldConfig": { 11560 "defaults": {}, 11561 "overrides": [] 11562 }, 11563 "fill": 1, 11564 "fillGradient": 0, 11565 "gridPos": { 11566 "h": 6, 11567 "w": 8, 11568 "x": 0, 11569 "y": 11 11570 }, 11571 "hiddenSeries": false, 11572 "id": 271, 11573 "legend": { 11574 "avg": false, 11575 "current": false, 11576 "max": false, 11577 "min": false, 11578 "show": true, 11579 "total": false, 11580 "values": false 11581 }, 11582 "lines": true, 11583 "linewidth": 1, 11584 "nullPointMode": "null", 11585 "options": { 11586 "alertThreshold": true 11587 }, 11588 "percentage": false, 11589 "pluginVersion": "7.5.11", 11590 "pointradius": 2, 11591 "points": false, 11592 "renderer": "flot", 11593 "seriesOverrides": [], 11594 "spaceLength": 10, 11595 "stack": false, 11596 "steppedLine": false, 11597 "targets": [ 11598 { 11599 "expr": "sum(ticdc_sorter_on_disk_data_size_gauge{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)", 11600 "format": "time_series", 11601 "intervalFactor": 1, 11602 "legendFormat": "{{instance}}", 11603 "refId": "A" 11604 } 11605 ], 11606 "thresholds": [], 11607 "timeFrom": null, 11608 "timeRegions": [], 11609 "timeShift": null, 11610 "title": "On disk data size", 11611 "tooltip": { 11612 "shared": true, 11613 "sort": 0, 11614 "value_type": "individual" 11615 }, 11616 "type": "graph", 11617 "xaxis": { 11618 "buckets": null, 11619 "mode": "time", 11620 "name": null, 11621 "show": true, 11622 "values": [] 11623 }, 11624 "yaxes": [ 11625 { 11626 "format": "bytes", 11627 "label": null, 11628 "logBase": 1, 11629 "max": null, 11630 "min": "0", 11631 "show": true 11632 }, 11633 { 11634 "format": "short", 11635 "label": null, 11636 "logBase": 1, 11637 "max": null, 11638 "min": null, 11639 "show": true 11640 } 11641 ], 11642 "yaxis": { 11643 "align": false, 11644 "alignLevel": null 11645 } 11646 }, 11647 { 11648 "aliasColors": {}, 11649 "bars": false, 11650 "dashLength": 10, 11651 "dashes": false, 11652 "datasource": "${DS_TEST-CLUSTER}", 11653 "fieldConfig": { 11654 "defaults": {}, 11655 "overrides": [] 11656 }, 11657 "fill": 1, 11658 "fillGradient": 0, 11659 "gridPos": { 11660 "h": 6, 11661 "w": 8, 11662 "x": 8, 11663 "y": 11 11664 }, 11665 "hiddenSeries": false, 11666 "id": 273, 11667 "legend": { 11668 "avg": false, 11669 "current": false, 11670 "max": false, 11671 "min": false, 11672 "show": true, 11673 "total": false, 11674 "values": false 11675 }, 11676 "lines": true, 11677 "linewidth": 1, 11678 "links": [], 11679 "nullPointMode": "null", 11680 "options": { 11681 "alertThreshold": true 11682 }, 11683 "percentage": false, 11684 "pluginVersion": "7.5.11", 11685 "pointradius": 2, 11686 "points": false, 11687 "renderer": "flot", 11688 "seriesOverrides": [ 11689 { 11690 "alias": "/cache-hit.*/", 11691 "yaxis": 2 11692 } 11693 ], 11694 "spaceLength": 10, 11695 "stack": false, 11696 "steppedLine": false, 11697 "targets": [ 11698 { 11699 "expr": "sum(ticdc_sorter_in_memory_data_size_gauge{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)", 11700 "format": "time_series", 11701 "intervalFactor": 1, 11702 "legendFormat": "{{instance}}", 11703 "refId": "A" 11704 }, 11705 { 11706 "expr": "sum(ticdc_db_block_cache_access_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", type=\"hit\"}) by (instance) / sum(ticdc_db_block_cache_access_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance)", 11707 "format": "time_series", 11708 "intervalFactor": 1, 11709 "legendFormat": "cache-hit-{{instance}}", 11710 "refId": "B" 11711 } 11712 ], 11713 "thresholds": [], 11714 "timeFrom": null, 11715 "timeRegions": [], 11716 "timeShift": null, 11717 "title": "In-memory data size", 11718 "tooltip": { 11719 "shared": true, 11720 "sort": 0, 11721 "value_type": "individual" 11722 }, 11723 "type": "graph", 11724 "xaxis": { 11725 "buckets": null, 11726 "mode": "time", 11727 "name": null, 11728 "show": true, 11729 "values": [] 11730 }, 11731 "yaxes": [ 11732 { 11733 "format": "bytes", 11734 "label": null, 11735 "logBase": 1, 11736 "max": null, 11737 "min": "0", 11738 "show": true 11739 }, 11740 { 11741 "format": "percentunit", 11742 "label": null, 11743 "logBase": 1, 11744 "max": null, 11745 "min": "0", 11746 "show": true 11747 } 11748 ], 11749 "yaxis": { 11750 "align": false, 11751 "alignLevel": null 11752 } 11753 }, 11754 { 11755 "aliasColors": {}, 11756 "bars": false, 11757 "dashLength": 10, 11758 "dashes": false, 11759 "datasource": "${DS_TEST-CLUSTER}", 11760 "description": "The count of files of each levels", 11761 "fieldConfig": { 11762 "defaults": {}, 11763 "overrides": [] 11764 }, 11765 "fill": 1, 11766 "fillGradient": 0, 11767 "gridPos": { 11768 "h": 6, 11769 "w": 8, 11770 "x": 16, 11771 "y": 11 11772 }, 11773 "hiddenSeries": false, 11774 "id": 272, 11775 "legend": { 11776 "avg": false, 11777 "current": true, 11778 "max": true, 11779 "min": false, 11780 "show": true, 11781 "total": false, 11782 "values": true 11783 }, 11784 "lines": true, 11785 "linewidth": 1, 11786 "links": [], 11787 "nullPointMode": "null", 11788 "options": { 11789 "alertThreshold": true 11790 }, 11791 "paceLength": 10, 11792 "percentage": false, 11793 "pluginVersion": "7.5.11", 11794 "pointradius": 2, 11795 "points": false, 11796 "renderer": "flot", 11797 "seriesOverrides": [], 11798 "spaceLength": 10, 11799 "stack": false, 11800 "steppedLine": false, 11801 "targets": [ 11802 { 11803 "expr": "sum(ticdc_db_level_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}) by (instance, level)", 11804 "format": "time_series", 11805 "intervalFactor": 1, 11806 "legendFormat": "{{instance}}-{{level}}", 11807 "refId": "A" 11808 } 11809 ], 11810 "thresholds": [], 11811 "timeFrom": null, 11812 "timeRegions": [], 11813 "timeShift": null, 11814 "title": "Level files", 11815 "tooltip": { 11816 "shared": true, 11817 "sort": 0, 11818 "value_type": "individual" 11819 }, 11820 "type": "graph", 11821 "xaxis": { 11822 "buckets": null, 11823 "mode": "time", 11824 "name": null, 11825 "show": true, 11826 "values": [] 11827 }, 11828 "yaxes": [ 11829 { 11830 "format": "short", 11831 "label": null, 11832 "logBase": 1, 11833 "max": null, 11834 "min": null, 11835 "show": true 11836 }, 11837 { 11838 "format": "short", 11839 "label": null, 11840 "logBase": 1, 11841 "max": null, 11842 "min": null, 11843 "show": true 11844 } 11845 ], 11846 "yaxis": { 11847 "align": false, 11848 "alignLevel": null 11849 } 11850 }, 11851 { 11852 "cards": { 11853 "cardPadding": 0, 11854 "cardRound": 0 11855 }, 11856 "color": { 11857 "cardColor": "#FF9830", 11858 "colorScale": "linear", 11859 "colorScheme": "interpolateSpectral", 11860 "exponent": 0.5, 11861 "min": 0, 11862 "mode": "spectrum" 11863 }, 11864 "dataFormat": "tsbuckets", 11865 "datasource": "${DS_TEST-CLUSTER}", 11866 "description": "The time of sorter write", 11867 "fieldConfig": { 11868 "defaults": {}, 11869 "overrides": [] 11870 }, 11871 "gridPos": { 11872 "h": 6, 11873 "w": 8, 11874 "x": 0, 11875 "y": 17 11876 }, 11877 "heatmap": {}, 11878 "hideZeroBuckets": true, 11879 "highlightCards": true, 11880 "id": 274, 11881 "legend": { 11882 "alignAsTable": true, 11883 "avg": false, 11884 "current": true, 11885 "max": true, 11886 "min": false, 11887 "rightSide": true, 11888 "show": true, 11889 "sort": "current", 11890 "sortDesc": true, 11891 "total": false, 11892 "values": true 11893 }, 11894 "links": [], 11895 "maxPerRow": 3, 11896 "repeatDirection": "h", 11897 "reverseYBuckets": false, 11898 "targets": [ 11899 { 11900 "exemplar": true, 11901 "expr": "sum(rate(ticdc_sorter_db_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", 11902 "format": "heatmap", 11903 "instant": false, 11904 "interval": "", 11905 "intervalFactor": 2, 11906 "legendFormat": "{{le}}", 11907 "refId": "A" 11908 } 11909 ], 11910 "title": "Write duration", 11911 "tooltip": { 11912 "show": true, 11913 "showHistogram": true 11914 }, 11915 "tooltipDecimals": 1, 11916 "type": "heatmap", 11917 "xAxis": { 11918 "show": true 11919 }, 11920 "xBucketNumber": null, 11921 "xBucketSize": null, 11922 "yAxis": { 11923 "decimals": 1, 11924 "format": "s", 11925 "logBase": 1, 11926 "max": null, 11927 "min": null, 11928 "show": true, 11929 "splitFactor": null 11930 }, 11931 "yBucketBound": "upper", 11932 "yBucketNumber": null, 11933 "yBucketSize": null 11934 }, 11935 { 11936 "aliasColors": {}, 11937 "bars": false, 11938 "dashLength": 10, 11939 "dashes": false, 11940 "datasource": "${DS_TEST-CLUSTER}", 11941 "fieldConfig": { 11942 "defaults": {}, 11943 "overrides": [] 11944 }, 11945 "fill": 1, 11946 "fillGradient": 0, 11947 "gridPos": { 11948 "h": 6, 11949 "w": 8, 11950 "x": 8, 11951 "y": 17 11952 }, 11953 "hiddenSeries": false, 11954 "id": 288, 11955 "legend": { 11956 "alignAsTable": true, 11957 "avg": true, 11958 "current": true, 11959 "max": true, 11960 "min": false, 11961 "rightSide": false, 11962 "show": true, 11963 "total": false, 11964 "values": true 11965 }, 11966 "lines": true, 11967 "linewidth": 1, 11968 "links": [], 11969 "nullPointMode": "null", 11970 "options": { 11971 "alertThreshold": true 11972 }, 11973 "percentage": false, 11974 "pluginVersion": "7.5.11", 11975 "pointradius": 2, 11976 "points": false, 11977 "renderer": "flot", 11978 "seriesOverrides": [ 11979 { 11980 "alias": "/.*amplification/", 11981 "yaxis": 2 11982 } 11983 ], 11984 "spaceLength": 10, 11985 "stack": false, 11986 "steppedLine": false, 11987 "targets": [ 11988 { 11989 "exemplar": true, 11990 "expr": "sum(rate(ticdc_sorter_db_write_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", 11991 "format": "time_series", 11992 "hide": false, 11993 "interval": "", 11994 "intervalFactor": 1, 11995 "legendFormat": "{{instance}}-sorter", 11996 "refId": "A" 11997 }, 11998 { 11999 "exemplar": true, 12000 "expr": "sum(rate(ticdc_sorter_db_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance) / sum(rate(ticdc_sorter_db_write_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", 12001 "format": "time_series", 12002 "hide": false, 12003 "interval": "", 12004 "intervalFactor": 1, 12005 "legendFormat": "{{instance}}-amplification", 12006 "refId": "B" 12007 }, 12008 { 12009 "exemplar": true, 12010 "expr": "sum(rate(ticdc_sorter_db_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", 12011 "format": "time_series", 12012 "hide": false, 12013 "interval": "", 12014 "intervalFactor": 1, 12015 "legendFormat": "{{instance}}-disk", 12016 "refId": "C" 12017 } 12018 ], 12019 "thresholds": [], 12020 "timeFrom": null, 12021 "timeRegions": [], 12022 "timeShift": null, 12023 "title": "Write speed", 12024 "tooltip": { 12025 "shared": true, 12026 "sort": 0, 12027 "value_type": "individual" 12028 }, 12029 "type": "graph", 12030 "xaxis": { 12031 "buckets": null, 12032 "mode": "time", 12033 "name": null, 12034 "show": true, 12035 "values": [] 12036 }, 12037 "yaxes": [ 12038 { 12039 "format": "bytes", 12040 "label": null, 12041 "logBase": 1, 12042 "max": null, 12043 "min": "0", 12044 "show": true 12045 }, 12046 { 12047 "format": "percentunit", 12048 "label": null, 12049 "logBase": 10, 12050 "max": null, 12051 "min": null, 12052 "show": true 12053 } 12054 ], 12055 "yaxis": { 12056 "align": false, 12057 "alignLevel": null 12058 } 12059 }, 12060 { 12061 "aliasColors": {}, 12062 "bars": false, 12063 "dashLength": 10, 12064 "dashes": false, 12065 "datasource": "${DS_TEST-CLUSTER}", 12066 "description": "CPU usage of LevelDB sorter", 12067 "fieldConfig": { 12068 "defaults": { 12069 "links": [] 12070 }, 12071 "overrides": [] 12072 }, 12073 "fill": 1, 12074 "fillGradient": 0, 12075 "gridPos": { 12076 "h": 6, 12077 "w": 8, 12078 "x": 16, 12079 "y": 17 12080 }, 12081 "hiddenSeries": false, 12082 "id": 286, 12083 "legend": { 12084 "alignAsTable": true, 12085 "avg": false, 12086 "current": true, 12087 "hideEmpty": true, 12088 "max": true, 12089 "min": false, 12090 "rightSide": false, 12091 "show": true, 12092 "total": false, 12093 "values": true 12094 }, 12095 "lines": true, 12096 "linewidth": 1, 12097 "links": [], 12098 "nullPointMode": "null", 12099 "options": { 12100 "alertThreshold": true 12101 }, 12102 "paceLength": 10, 12103 "percentage": false, 12104 "pluginVersion": "7.5.11", 12105 "pointradius": 2, 12106 "points": false, 12107 "renderer": "flot", 12108 "seriesOverrides": [ 12109 { 12110 "alias": "/.*sorter-[0-9]+/", 12111 "yaxis": 2 12112 } 12113 ], 12114 "spaceLength": 10, 12115 "stack": false, 12116 "steppedLine": false, 12117 "targets": [ 12118 { 12119 "expr": "sum(rate(ticdc_actor_worker_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", name=~\"sorter.*\"}[1m])) by (name, instance)", 12120 "format": "time_series", 12121 "intervalFactor": 1, 12122 "legendFormat": "{{instance}}-{{name}}", 12123 "refId": "A" 12124 } 12125 ], 12126 "thresholds": [], 12127 "timeFrom": null, 12128 "timeRegions": [], 12129 "timeShift": null, 12130 "title": "CPU usage", 12131 "tooltip": { 12132 "shared": true, 12133 "sort": 0, 12134 "value_type": "individual" 12135 }, 12136 "type": "graph", 12137 "xaxis": { 12138 "buckets": null, 12139 "mode": "time", 12140 "name": null, 12141 "show": true, 12142 "values": [] 12143 }, 12144 "yaxes": [ 12145 { 12146 "format": "percentunit", 12147 "label": null, 12148 "logBase": 1, 12149 "max": null, 12150 "min": null, 12151 "show": true 12152 }, 12153 { 12154 "format": "percentunit", 12155 "label": null, 12156 "logBase": 1, 12157 "max": "1.2", 12158 "min": null, 12159 "show": true 12160 } 12161 ], 12162 "yaxis": { 12163 "align": false, 12164 "alignLevel": null 12165 } 12166 }, 12167 { 12168 "cards": { 12169 "cardPadding": 0, 12170 "cardRound": 0 12171 }, 12172 "color": { 12173 "cardColor": "#FF9830", 12174 "colorScale": "linear", 12175 "colorScheme": "interpolateSpectral", 12176 "exponent": 0.5, 12177 "min": 0, 12178 "mode": "spectrum" 12179 }, 12180 "dataFormat": "tsbuckets", 12181 "datasource": "${DS_TEST-CLUSTER}", 12182 "description": "The time of sorter write", 12183 "fieldConfig": { 12184 "defaults": {}, 12185 "overrides": [] 12186 }, 12187 "gridPos": { 12188 "h": 6, 12189 "w": 8, 12190 "x": 0, 12191 "y": 23 12192 }, 12193 "heatmap": {}, 12194 "hideZeroBuckets": true, 12195 "highlightCards": true, 12196 "id": 277, 12197 "legend": { 12198 "alignAsTable": true, 12199 "avg": false, 12200 "current": true, 12201 "max": true, 12202 "min": false, 12203 "rightSide": true, 12204 "show": true, 12205 "sort": "current", 12206 "sortDesc": true, 12207 "total": false, 12208 "values": true 12209 }, 12210 "links": [], 12211 "maxPerRow": 3, 12212 "repeatDirection": "h", 12213 "reverseYBuckets": false, 12214 "targets": [ 12215 { 12216 "exemplar": true, 12217 "expr": "sum(rate(ticdc_sorter_db_write_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", 12218 "format": "heatmap", 12219 "instant": false, 12220 "interval": "", 12221 "intervalFactor": 2, 12222 "legendFormat": "{{le}}", 12223 "refId": "A" 12224 } 12225 ], 12226 "title": "Write bytes", 12227 "tooltip": { 12228 "show": true, 12229 "showHistogram": true 12230 }, 12231 "tooltipDecimals": 1, 12232 "type": "heatmap", 12233 "xAxis": { 12234 "show": true 12235 }, 12236 "xBucketNumber": null, 12237 "xBucketSize": null, 12238 "yAxis": { 12239 "decimals": 1, 12240 "format": "bytes", 12241 "logBase": 1, 12242 "max": null, 12243 "min": null, 12244 "show": true, 12245 "splitFactor": null 12246 }, 12247 "yBucketBound": "upper", 12248 "yBucketNumber": null, 12249 "yBucketSize": null 12250 }, 12251 { 12252 "aliasColors": {}, 12253 "bars": false, 12254 "dashLength": 10, 12255 "dashes": false, 12256 "datasource": "${DS_TEST-CLUSTER}", 12257 "fieldConfig": { 12258 "defaults": {}, 12259 "overrides": [] 12260 }, 12261 "fill": 1, 12262 "fillGradient": 0, 12263 "gridPos": { 12264 "h": 6, 12265 "w": 8, 12266 "x": 8, 12267 "y": 23 12268 }, 12269 "hiddenSeries": false, 12270 "id": 279, 12271 "legend": { 12272 "alignAsTable": true, 12273 "avg": true, 12274 "current": true, 12275 "max": true, 12276 "min": false, 12277 "rightSide": false, 12278 "show": true, 12279 "total": false, 12280 "values": true 12281 }, 12282 "lines": true, 12283 "linewidth": 1, 12284 "links": [], 12285 "nullPointMode": "null", 12286 "options": { 12287 "alertThreshold": true 12288 }, 12289 "percentage": false, 12290 "pluginVersion": "7.5.11", 12291 "pointradius": 2, 12292 "points": false, 12293 "renderer": "flot", 12294 "seriesOverrides": [], 12295 "spaceLength": 10, 12296 "stack": false, 12297 "steppedLine": false, 12298 "targets": [ 12299 { 12300 "exemplar": true, 12301 "expr": "sum(rate(ticdc_sorter_db_write_bytes_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", 12302 "format": "time_series", 12303 "hide": false, 12304 "interval": "", 12305 "intervalFactor": 1, 12306 "legendFormat": "{{instance}}-sorter", 12307 "refId": "A" 12308 } 12309 ], 12310 "thresholds": [], 12311 "timeFrom": null, 12312 "timeRegions": [], 12313 "timeShift": null, 12314 "title": "Write OPS", 12315 "tooltip": { 12316 "shared": true, 12317 "sort": 0, 12318 "value_type": "individual" 12319 }, 12320 "type": "graph", 12321 "xaxis": { 12322 "buckets": null, 12323 "mode": "time", 12324 "name": null, 12325 "show": true, 12326 "values": [] 12327 }, 12328 "yaxes": [ 12329 { 12330 "format": "ops", 12331 "label": null, 12332 "logBase": 1, 12333 "max": null, 12334 "min": "0", 12335 "show": true 12336 }, 12337 { 12338 "format": "percentunit", 12339 "label": null, 12340 "logBase": 10, 12341 "max": null, 12342 "min": null, 12343 "show": true 12344 } 12345 ], 12346 "yaxis": { 12347 "align": false, 12348 "alignLevel": null 12349 } 12350 }, 12351 { 12352 "aliasColors": {}, 12353 "bars": false, 12354 "dashLength": 10, 12355 "dashes": false, 12356 "datasource": "${DS_TEST-CLUSTER}", 12357 "description": "The count and duration of write delay", 12358 "fieldConfig": { 12359 "defaults": { 12360 "links": [] 12361 }, 12362 "overrides": [] 12363 }, 12364 "fill": 1, 12365 "fillGradient": 0, 12366 "gridPos": { 12367 "h": 6, 12368 "w": 8, 12369 "x": 16, 12370 "y": 23 12371 }, 12372 "hiddenSeries": false, 12373 "id": 275, 12374 "legend": { 12375 "alignAsTable": false, 12376 "avg": false, 12377 "current": true, 12378 "max": true, 12379 "min": false, 12380 "rightSide": false, 12381 "show": true, 12382 "total": false, 12383 "values": true 12384 }, 12385 "lines": true, 12386 "linewidth": 1, 12387 "links": [], 12388 "nullPointMode": "null", 12389 "options": { 12390 "alertThreshold": true 12391 }, 12392 "paceLength": 10, 12393 "percentage": false, 12394 "pluginVersion": "7.5.11", 12395 "pointradius": 2, 12396 "points": false, 12397 "renderer": "flot", 12398 "seriesOverrides": [ 12399 { 12400 "alias": "/.*count.*/", 12401 "yaxis": 2 12402 } 12403 ], 12404 "spaceLength": 10, 12405 "stack": false, 12406 "steppedLine": false, 12407 "targets": [ 12408 { 12409 "exemplar": true, 12410 "expr": "sum(rate(ticdc_db_write_delay_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", 12411 "format": "time_series", 12412 "interval": "", 12413 "intervalFactor": 1, 12414 "legendFormat": "{{instance}}-duration", 12415 "refId": "A" 12416 }, 12417 { 12418 "exemplar": true, 12419 "expr": "sum(rate(ticdc_db_write_delay_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (instance)", 12420 "format": "time_series", 12421 "hide": false, 12422 "interval": "", 12423 "intervalFactor": 1, 12424 "legendFormat": "{{instance}}-count", 12425 "refId": "B" 12426 } 12427 ], 12428 "thresholds": [], 12429 "timeFrom": null, 12430 "timeRegions": [], 12431 "timeShift": null, 12432 "title": "Write delay", 12433 "tooltip": { 12434 "shared": true, 12435 "sort": 0, 12436 "value_type": "individual" 12437 }, 12438 "type": "graph", 12439 "xaxis": { 12440 "buckets": null, 12441 "mode": "time", 12442 "name": null, 12443 "show": true, 12444 "values": [] 12445 }, 12446 "yaxes": [ 12447 { 12448 "format": "dtdurations", 12449 "label": null, 12450 "logBase": 1, 12451 "max": null, 12452 "min": null, 12453 "show": true 12454 }, 12455 { 12456 "format": "short", 12457 "label": null, 12458 "logBase": 1, 12459 "max": null, 12460 "min": null, 12461 "show": true 12462 } 12463 ], 12464 "yaxis": { 12465 "align": false, 12466 "alignLevel": null 12467 } 12468 }, 12469 { 12470 "cards": { 12471 "cardPadding": 0, 12472 "cardRound": 0 12473 }, 12474 "color": { 12475 "cardColor": "#FF9830", 12476 "colorScale": "linear", 12477 "colorScheme": "interpolateSpectral", 12478 "exponent": 0.5, 12479 "min": 0, 12480 "mode": "spectrum" 12481 }, 12482 "dataFormat": "tsbuckets", 12483 "datasource": "${DS_TEST-CLUSTER}", 12484 "description": "The time of sorter iterator read", 12485 "fieldConfig": { 12486 "defaults": {}, 12487 "overrides": [] 12488 }, 12489 "gridPos": { 12490 "h": 6, 12491 "w": 8, 12492 "x": 0, 12493 "y": 29 12494 }, 12495 "heatmap": {}, 12496 "hideZeroBuckets": true, 12497 "highlightCards": true, 12498 "id": 278, 12499 "legend": { 12500 "alignAsTable": true, 12501 "avg": false, 12502 "current": true, 12503 "max": true, 12504 "min": false, 12505 "rightSide": true, 12506 "show": true, 12507 "sort": "current", 12508 "sortDesc": true, 12509 "total": false, 12510 "values": true 12511 }, 12512 "links": [], 12513 "maxPerRow": 3, 12514 "repeatDirection": "h", 12515 "reverseYBuckets": false, 12516 "targets": [ 12517 { 12518 "exemplar": true, 12519 "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"first\"}[1m])) by (le)", 12520 "format": "heatmap", 12521 "instant": false, 12522 "interval": "", 12523 "intervalFactor": 2, 12524 "legendFormat": "{{le}}", 12525 "refId": "A" 12526 } 12527 ], 12528 "title": "Read duration - First", 12529 "tooltip": { 12530 "show": true, 12531 "showHistogram": true 12532 }, 12533 "tooltipDecimals": 1, 12534 "type": "heatmap", 12535 "xAxis": { 12536 "show": true 12537 }, 12538 "xBucketNumber": null, 12539 "xBucketSize": null, 12540 "yAxis": { 12541 "decimals": 1, 12542 "format": "s", 12543 "logBase": 1, 12544 "max": null, 12545 "min": null, 12546 "show": true, 12547 "splitFactor": null 12548 }, 12549 "yBucketBound": "upper", 12550 "yBucketNumber": null, 12551 "yBucketSize": null 12552 }, 12553 { 12554 "cards": { 12555 "cardPadding": 0, 12556 "cardRound": 0 12557 }, 12558 "color": { 12559 "cardColor": "#FF9830", 12560 "colorScale": "linear", 12561 "colorScheme": "interpolateSpectral", 12562 "exponent": 0.5, 12563 "min": 0, 12564 "mode": "spectrum" 12565 }, 12566 "dataFormat": "tsbuckets", 12567 "datasource": "${DS_TEST-CLUSTER}", 12568 "description": "The time of sorter iterator read", 12569 "fieldConfig": { 12570 "defaults": {}, 12571 "overrides": [] 12572 }, 12573 "gridPos": { 12574 "h": 6, 12575 "w": 8, 12576 "x": 8, 12577 "y": 29 12578 }, 12579 "heatmap": {}, 12580 "hideZeroBuckets": true, 12581 "highlightCards": true, 12582 "id": 282, 12583 "legend": { 12584 "alignAsTable": true, 12585 "avg": false, 12586 "current": true, 12587 "max": true, 12588 "min": false, 12589 "rightSide": true, 12590 "show": true, 12591 "sort": "current", 12592 "sortDesc": true, 12593 "total": false, 12594 "values": true 12595 }, 12596 "links": [], 12597 "maxPerRow": 3, 12598 "repeatDirection": "h", 12599 "reverseYBuckets": false, 12600 "targets": [ 12601 { 12602 "exemplar": true, 12603 "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"next\"}[1m])) by (le)", 12604 "format": "heatmap", 12605 "instant": false, 12606 "interval": "", 12607 "intervalFactor": 2, 12608 "legendFormat": "{{le}}", 12609 "refId": "A" 12610 } 12611 ], 12612 "title": "Read duration - Next", 12613 "tooltip": { 12614 "show": true, 12615 "showHistogram": true 12616 }, 12617 "tooltipDecimals": 1, 12618 "type": "heatmap", 12619 "xAxis": { 12620 "show": true 12621 }, 12622 "xBucketNumber": null, 12623 "xBucketSize": null, 12624 "yAxis": { 12625 "decimals": 1, 12626 "format": "s", 12627 "logBase": 1, 12628 "max": null, 12629 "min": null, 12630 "show": true, 12631 "splitFactor": null 12632 }, 12633 "yBucketBound": "upper", 12634 "yBucketNumber": null, 12635 "yBucketSize": null 12636 }, 12637 { 12638 "cards": { 12639 "cardPadding": 0, 12640 "cardRound": 0 12641 }, 12642 "color": { 12643 "cardColor": "#FF9830", 12644 "colorScale": "linear", 12645 "colorScheme": "interpolateSpectral", 12646 "exponent": 0.5, 12647 "min": 0, 12648 "mode": "spectrum" 12649 }, 12650 "dataFormat": "tsbuckets", 12651 "datasource": "${DS_TEST-CLUSTER}", 12652 "description": "The time of sorter iterator read", 12653 "fieldConfig": { 12654 "defaults": {}, 12655 "overrides": [] 12656 }, 12657 "gridPos": { 12658 "h": 6, 12659 "w": 8, 12660 "x": 16, 12661 "y": 29 12662 }, 12663 "heatmap": {}, 12664 "hideZeroBuckets": true, 12665 "highlightCards": true, 12666 "id": 281, 12667 "legend": { 12668 "alignAsTable": true, 12669 "avg": false, 12670 "current": true, 12671 "max": true, 12672 "min": false, 12673 "rightSide": true, 12674 "show": true, 12675 "sort": "current", 12676 "sortDesc": true, 12677 "total": false, 12678 "values": true 12679 }, 12680 "links": [], 12681 "maxPerRow": 3, 12682 "repeatDirection": "h", 12683 "reverseYBuckets": false, 12684 "targets": [ 12685 { 12686 "exemplar": true, 12687 "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"release\"}[1m])) by (le)", 12688 "format": "heatmap", 12689 "instant": false, 12690 "interval": "", 12691 "intervalFactor": 2, 12692 "legendFormat": "{{le}}", 12693 "refId": "A" 12694 } 12695 ], 12696 "title": "Read duration - Release", 12697 "tooltip": { 12698 "show": true, 12699 "showHistogram": true 12700 }, 12701 "tooltipDecimals": 1, 12702 "type": "heatmap", 12703 "xAxis": { 12704 "show": true 12705 }, 12706 "xBucketNumber": null, 12707 "xBucketSize": null, 12708 "yAxis": { 12709 "decimals": 1, 12710 "format": "s", 12711 "logBase": 1, 12712 "max": null, 12713 "min": null, 12714 "show": true, 12715 "splitFactor": null 12716 }, 12717 "yBucketBound": "upper", 12718 "yBucketNumber": null, 12719 "yBucketSize": null 12720 }, 12721 { 12722 "aliasColors": {}, 12723 "bars": false, 12724 "dashLength": 10, 12725 "dashes": false, 12726 "datasource": "${DS_TEST-CLUSTER}", 12727 "fieldConfig": { 12728 "defaults": {}, 12729 "overrides": [] 12730 }, 12731 "fill": 1, 12732 "fillGradient": 0, 12733 "gridPos": { 12734 "h": 6, 12735 "w": 8, 12736 "x": 0, 12737 "y": 35 12738 }, 12739 "hiddenSeries": false, 12740 "id": 280, 12741 "legend": { 12742 "alignAsTable": true, 12743 "avg": true, 12744 "current": true, 12745 "max": true, 12746 "min": false, 12747 "rightSide": false, 12748 "show": true, 12749 "total": false, 12750 "values": true 12751 }, 12752 "lines": true, 12753 "linewidth": 1, 12754 "links": [], 12755 "nullPointMode": "null", 12756 "options": { 12757 "alertThreshold": true 12758 }, 12759 "percentage": false, 12760 "pluginVersion": "7.5.11", 12761 "pointradius": 2, 12762 "points": false, 12763 "renderer": "flot", 12764 "seriesOverrides": [], 12765 "spaceLength": 10, 12766 "stack": false, 12767 "steppedLine": false, 12768 "targets": [ 12769 { 12770 "exemplar": true, 12771 "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"first\"}[1m])) by (instance)", 12772 "format": "time_series", 12773 "hide": false, 12774 "interval": "", 12775 "intervalFactor": 1, 12776 "legendFormat": "{{instance}}-sorter", 12777 "refId": "A" 12778 } 12779 ], 12780 "thresholds": [], 12781 "timeFrom": null, 12782 "timeRegions": [], 12783 "timeShift": null, 12784 "title": "Read OPS - First", 12785 "tooltip": { 12786 "shared": true, 12787 "sort": 0, 12788 "value_type": "individual" 12789 }, 12790 "type": "graph", 12791 "xaxis": { 12792 "buckets": null, 12793 "mode": "time", 12794 "name": null, 12795 "show": true, 12796 "values": [] 12797 }, 12798 "yaxes": [ 12799 { 12800 "format": "ops", 12801 "label": null, 12802 "logBase": 1, 12803 "max": null, 12804 "min": "0", 12805 "show": true 12806 }, 12807 { 12808 "format": "percentunit", 12809 "label": null, 12810 "logBase": 10, 12811 "max": null, 12812 "min": null, 12813 "show": true 12814 } 12815 ], 12816 "yaxis": { 12817 "align": false, 12818 "alignLevel": null 12819 } 12820 }, 12821 { 12822 "aliasColors": {}, 12823 "bars": false, 12824 "dashLength": 10, 12825 "dashes": false, 12826 "datasource": "${DS_TEST-CLUSTER}", 12827 "fieldConfig": { 12828 "defaults": {}, 12829 "overrides": [] 12830 }, 12831 "fill": 1, 12832 "fillGradient": 0, 12833 "gridPos": { 12834 "h": 6, 12835 "w": 8, 12836 "x": 8, 12837 "y": 35 12838 }, 12839 "hiddenSeries": false, 12840 "id": 283, 12841 "legend": { 12842 "alignAsTable": true, 12843 "avg": true, 12844 "current": true, 12845 "max": true, 12846 "min": false, 12847 "rightSide": false, 12848 "show": true, 12849 "total": false, 12850 "values": true 12851 }, 12852 "lines": true, 12853 "linewidth": 1, 12854 "links": [], 12855 "nullPointMode": "null", 12856 "options": { 12857 "alertThreshold": true 12858 }, 12859 "percentage": false, 12860 "pluginVersion": "7.5.11", 12861 "pointradius": 2, 12862 "points": false, 12863 "renderer": "flot", 12864 "seriesOverrides": [], 12865 "spaceLength": 10, 12866 "stack": false, 12867 "steppedLine": false, 12868 "targets": [ 12869 { 12870 "exemplar": true, 12871 "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"next\"}[1m])) by (instance)", 12872 "format": "time_series", 12873 "hide": false, 12874 "interval": "", 12875 "intervalFactor": 1, 12876 "legendFormat": "{{instance}}-sorter", 12877 "refId": "A" 12878 } 12879 ], 12880 "thresholds": [], 12881 "timeFrom": null, 12882 "timeRegions": [], 12883 "timeShift": null, 12884 "title": "Read OPS - Next", 12885 "tooltip": { 12886 "shared": true, 12887 "sort": 0, 12888 "value_type": "individual" 12889 }, 12890 "type": "graph", 12891 "xaxis": { 12892 "buckets": null, 12893 "mode": "time", 12894 "name": null, 12895 "show": true, 12896 "values": [] 12897 }, 12898 "yaxes": [ 12899 { 12900 "format": "ops", 12901 "label": null, 12902 "logBase": 1, 12903 "max": null, 12904 "min": "0", 12905 "show": true 12906 }, 12907 { 12908 "format": "percentunit", 12909 "label": null, 12910 "logBase": 10, 12911 "max": null, 12912 "min": null, 12913 "show": true 12914 } 12915 ], 12916 "yaxis": { 12917 "align": false, 12918 "alignLevel": null 12919 } 12920 }, 12921 { 12922 "aliasColors": {}, 12923 "bars": false, 12924 "dashLength": 10, 12925 "dashes": false, 12926 "datasource": "${DS_TEST-CLUSTER}", 12927 "fieldConfig": { 12928 "defaults": {}, 12929 "overrides": [] 12930 }, 12931 "fill": 1, 12932 "fillGradient": 0, 12933 "gridPos": { 12934 "h": 6, 12935 "w": 8, 12936 "x": 16, 12937 "y": 35 12938 }, 12939 "hiddenSeries": false, 12940 "id": 287, 12941 "legend": { 12942 "alignAsTable": true, 12943 "avg": true, 12944 "current": true, 12945 "max": true, 12946 "min": false, 12947 "rightSide": false, 12948 "show": true, 12949 "total": false, 12950 "values": true 12951 }, 12952 "lines": true, 12953 "linewidth": 1, 12954 "links": [], 12955 "nullPointMode": "null", 12956 "options": { 12957 "alertThreshold": true 12958 }, 12959 "percentage": false, 12960 "pluginVersion": "7.5.11", 12961 "pointradius": 2, 12962 "points": false, 12963 "renderer": "flot", 12964 "seriesOverrides": [], 12965 "spaceLength": 10, 12966 "stack": false, 12967 "steppedLine": false, 12968 "targets": [ 12969 { 12970 "exemplar": true, 12971 "expr": "sum(rate(ticdc_sorter_db_iter_read_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\", call=\"release\"}[1m])) by (instance)", 12972 "format": "time_series", 12973 "hide": false, 12974 "interval": "", 12975 "intervalFactor": 1, 12976 "legendFormat": "{{instance}}-sorter", 12977 "refId": "A" 12978 } 12979 ], 12980 "thresholds": [], 12981 "timeFrom": null, 12982 "timeRegions": [], 12983 "timeShift": null, 12984 "title": "Read OPS - Release", 12985 "tooltip": { 12986 "shared": true, 12987 "sort": 0, 12988 "value_type": "individual" 12989 }, 12990 "type": "graph", 12991 "xaxis": { 12992 "buckets": null, 12993 "mode": "time", 12994 "name": null, 12995 "show": true, 12996 "values": [] 12997 }, 12998 "yaxes": [ 12999 { 13000 "format": "ops", 13001 "label": null, 13002 "logBase": 1, 13003 "max": null, 13004 "min": "0", 13005 "show": true 13006 }, 13007 { 13008 "format": "percentunit", 13009 "label": null, 13010 "logBase": 10, 13011 "max": null, 13012 "min": null, 13013 "show": true 13014 } 13015 ], 13016 "yaxis": { 13017 "align": false, 13018 "alignLevel": null 13019 } 13020 }, 13021 { 13022 "cards": { 13023 "cardPadding": 0, 13024 "cardRound": 0 13025 }, 13026 "color": { 13027 "cardColor": "#FF9830", 13028 "colorScale": "linear", 13029 "colorScheme": "interpolateSpectral", 13030 "exponent": 0.5, 13031 "min": 0, 13032 "mode": "spectrum" 13033 }, 13034 "dataFormat": "tsbuckets", 13035 "datasource": "${DS_TEST-CLUSTER}", 13036 "description": "The time of sorter compact", 13037 "fieldConfig": { 13038 "defaults": {}, 13039 "overrides": [] 13040 }, 13041 "gridPos": { 13042 "h": 6, 13043 "w": 8, 13044 "x": 0, 13045 "y": 41 13046 }, 13047 "heatmap": {}, 13048 "hideZeroBuckets": true, 13049 "highlightCards": true, 13050 "id": 285, 13051 "legend": { 13052 "alignAsTable": true, 13053 "avg": false, 13054 "current": true, 13055 "max": true, 13056 "min": false, 13057 "rightSide": true, 13058 "show": true, 13059 "sort": "current", 13060 "sortDesc": true, 13061 "total": false, 13062 "values": true 13063 }, 13064 "links": [], 13065 "maxPerRow": 3, 13066 "repeatDirection": "h", 13067 "reverseYBuckets": false, 13068 "targets": [ 13069 { 13070 "exemplar": true, 13071 "expr": "sum(rate(ticdc_sorter_db_compact_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", 13072 "format": "heatmap", 13073 "instant": false, 13074 "interval": "", 13075 "intervalFactor": 2, 13076 "legendFormat": "{{le}}", 13077 "refId": "A" 13078 } 13079 ], 13080 "title": "Compact duration", 13081 "tooltip": { 13082 "show": true, 13083 "showHistogram": true 13084 }, 13085 "tooltipDecimals": 1, 13086 "type": "heatmap", 13087 "xAxis": { 13088 "show": true 13089 }, 13090 "xBucketNumber": null, 13091 "xBucketSize": null, 13092 "yAxis": { 13093 "decimals": 1, 13094 "format": "s", 13095 "logBase": 1, 13096 "max": null, 13097 "min": null, 13098 "show": true, 13099 "splitFactor": null 13100 }, 13101 "yBucketBound": "upper", 13102 "yBucketNumber": null, 13103 "yBucketSize": null 13104 } 13105 ], 13106 "title": "DB", 13107 "type": "row" 13108 }, 13109 { 13110 "collapsed": true, 13111 "datasource": null, 13112 "gridPos": { 13113 "h": 1, 13114 "w": 24, 13115 "x": 0, 13116 "y": 11 13117 }, 13118 "id": 447, 13119 "panels": [ 13120 { 13121 "aliasColors": {}, 13122 "bars": false, 13123 "dashLength": 10, 13124 "dashes": false, 13125 "datasource": "${DS_TEST-CLUSTER}", 13126 "description": "The number of established Eventfeed RPC between TiCDC and TiKV", 13127 "fieldConfig": { 13128 "defaults": { 13129 "links": [] 13130 }, 13131 "overrides": [] 13132 }, 13133 "fill": 1, 13134 "fillGradient": 0, 13135 "gridPos": { 13136 "h": 7, 13137 "w": 12, 13138 "x": 0, 13139 "y": 12 13140 }, 13141 "hiddenSeries": false, 13142 "id": 15, 13143 "legend": { 13144 "alignAsTable": true, 13145 "avg": false, 13146 "current": true, 13147 "hideEmpty": false, 13148 "max": false, 13149 "min": false, 13150 "rightSide": true, 13151 "show": true, 13152 "total": false, 13153 "values": true 13154 }, 13155 "lines": true, 13156 "linewidth": 1, 13157 "links": [], 13158 "nullPointMode": "null", 13159 "options": { 13160 "alertThreshold": true 13161 }, 13162 "paceLength": 10, 13163 "percentage": false, 13164 "pluginVersion": "7.5.11", 13165 "pointradius": 2, 13166 "points": false, 13167 "renderer": "flot", 13168 "seriesOverrides": [ 13169 { 13170 "alias": "/.*-rpc/", 13171 "yaxis": 2 13172 } 13173 ], 13174 "spaceLength": 10, 13175 "stack": false, 13176 "steppedLine": false, 13177 "targets": [ 13178 { 13179 "expr": "sum(ticdc_kvclient_event_feed_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (instance)", 13180 "format": "time_series", 13181 "intervalFactor": 1, 13182 "legendFormat": "{{instance}}", 13183 "refId": "A" 13184 }, 13185 { 13186 "expr": "sum(grpc_client_started_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", grpc_method=~\"EventFeed.*\"}) by (instance) - sum(grpc_client_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", grpc_method=~\"EventFeed.*\"}) by (instance)", 13187 "format": "time_series", 13188 "intervalFactor": 1, 13189 "legendFormat": "{{instance}}-rpc", 13190 "refId": "B" 13191 }, 13192 { 13193 "expr": "sum(grpc_client_started_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", grpc_method=~\"EventFeed.*\"}) by (instance)", 13194 "format": "time_series", 13195 "hide": false, 13196 "intervalFactor": 1, 13197 "legendFormat": "{{instance}}-rpc-started", 13198 "refId": "C" 13199 }, 13200 { 13201 "expr": "sum(grpc_client_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", grpc_method=~\"EventFeed.*\"}) by (instance)", 13202 "format": "time_series", 13203 "hide": false, 13204 "intervalFactor": 1, 13205 "legendFormat": "{{instance}}-rpc-handled", 13206 "refId": "D" 13207 } 13208 ], 13209 "thresholds": [], 13210 "timeFrom": null, 13211 "timeRegions": [], 13212 "timeShift": null, 13213 "title": "Eventfeed count", 13214 "tooltip": { 13215 "shared": true, 13216 "sort": 0, 13217 "value_type": "individual" 13218 }, 13219 "type": "graph", 13220 "xaxis": { 13221 "buckets": null, 13222 "mode": "time", 13223 "name": null, 13224 "show": true, 13225 "values": [] 13226 }, 13227 "yaxes": [ 13228 { 13229 "$$hashKey": "object:359", 13230 "format": "short", 13231 "label": null, 13232 "logBase": 1, 13233 "max": null, 13234 "min": null, 13235 "show": true 13236 }, 13237 { 13238 "$$hashKey": "object:360", 13239 "format": "short", 13240 "label": null, 13241 "logBase": 1, 13242 "max": null, 13243 "min": null, 13244 "show": true 13245 } 13246 ], 13247 "yaxis": { 13248 "align": false, 13249 "alignLevel": null 13250 } 13251 }, 13252 { 13253 "aliasColors": {}, 13254 "bars": false, 13255 "dashLength": 10, 13256 "dashes": false, 13257 "datasource": "${DS_TEST-CLUSTER}", 13258 "description": "The number of KV client received events from TiKV per seconds", 13259 "fieldConfig": { 13260 "defaults": { 13261 "links": [] 13262 }, 13263 "overrides": [] 13264 }, 13265 "fill": 1, 13266 "fillGradient": 0, 13267 "gridPos": { 13268 "h": 7, 13269 "w": 12, 13270 "x": 12, 13271 "y": 12 13272 }, 13273 "hiddenSeries": false, 13274 "id": 29, 13275 "legend": { 13276 "alignAsTable": true, 13277 "avg": false, 13278 "current": true, 13279 "hideEmpty": true, 13280 "hideZero": true, 13281 "max": true, 13282 "min": false, 13283 "rightSide": false, 13284 "show": true, 13285 "total": false, 13286 "values": true 13287 }, 13288 "lines": true, 13289 "linewidth": 1, 13290 "links": [], 13291 "nullPointMode": "null", 13292 "options": { 13293 "alertThreshold": true 13294 }, 13295 "paceLength": 10, 13296 "percentage": false, 13297 "pluginVersion": "7.5.11", 13298 "pointradius": 2, 13299 "points": false, 13300 "renderer": "flot", 13301 "seriesOverrides": [], 13302 "spaceLength": 10, 13303 "stack": false, 13304 "steppedLine": false, 13305 "targets": [ 13306 { 13307 "exemplar": true, 13308 "expr": "sum(rate(ticdc_kvclient_pull_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[1m])) by (instance, type, changefeed)", 13309 "format": "time_series", 13310 "interval": "", 13311 "intervalFactor": 1, 13312 "legendFormat": "{{changefeed}}-{{instance}}-{{type}}", 13313 "refId": "A" 13314 } 13315 ], 13316 "thresholds": [], 13317 "timeFrom": null, 13318 "timeRegions": [], 13319 "timeShift": null, 13320 "title": "KV client receive events/s", 13321 "tooltip": { 13322 "shared": true, 13323 "sort": 0, 13324 "value_type": "individual" 13325 }, 13326 "type": "graph", 13327 "xaxis": { 13328 "buckets": null, 13329 "mode": "time", 13330 "name": null, 13331 "show": true, 13332 "values": [] 13333 }, 13334 "yaxes": [ 13335 { 13336 "$$hashKey": "object:242", 13337 "format": "short", 13338 "label": null, 13339 "logBase": 1, 13340 "max": null, 13341 "min": null, 13342 "show": true 13343 }, 13344 { 13345 "$$hashKey": "object:243", 13346 "format": "short", 13347 "label": null, 13348 "logBase": 1, 13349 "max": null, 13350 "min": null, 13351 "show": true 13352 } 13353 ], 13354 "yaxis": { 13355 "align": false, 13356 "alignLevel": null 13357 } 13358 }, 13359 { 13360 "aliasColors": {}, 13361 "bars": true, 13362 "dashLength": 10, 13363 "dashes": false, 13364 "datasource": "${DS_TEST-CLUSTER}", 13365 "description": "The number of errors that interrupt Eventfeed RPC", 13366 "fieldConfig": { 13367 "defaults": { 13368 "links": [] 13369 }, 13370 "overrides": [] 13371 }, 13372 "fill": 1, 13373 "fillGradient": 0, 13374 "gridPos": { 13375 "h": 7, 13376 "w": 12, 13377 "x": 0, 13378 "y": 19 13379 }, 13380 "hiddenSeries": false, 13381 "id": 28, 13382 "legend": { 13383 "alignAsTable": true, 13384 "avg": false, 13385 "current": true, 13386 "hideEmpty": true, 13387 "hideZero": true, 13388 "max": true, 13389 "min": false, 13390 "rightSide": true, 13391 "show": true, 13392 "total": false, 13393 "values": true 13394 }, 13395 "lines": true, 13396 "linewidth": 1, 13397 "links": [], 13398 "nullPointMode": "null", 13399 "options": { 13400 "alertThreshold": true 13401 }, 13402 "paceLength": 10, 13403 "percentage": false, 13404 "pluginVersion": "7.5.11", 13405 "pointradius": 2, 13406 "points": false, 13407 "renderer": "flot", 13408 "seriesOverrides": [], 13409 "spaceLength": 10, 13410 "stack": false, 13411 "steppedLine": false, 13412 "targets": [ 13413 { 13414 "expr": "sum(increase(ticdc_kvclient_event_feed_error_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (type)", 13415 "format": "time_series", 13416 "hide": false, 13417 "interval": "1m", 13418 "intervalFactor": 1, 13419 "legendFormat": "{{type}}", 13420 "refId": "A" 13421 }, 13422 { 13423 "expr": "-sum(increase(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", event=\"create\", type=~\".*leader\"}[1m]))", 13424 "format": "time_series", 13425 "hide": false, 13426 "interval": "1m", 13427 "intervalFactor": 1, 13428 "legendFormat": "transfer-leader", 13429 "refId": "B" 13430 }, 13431 { 13432 "expr": "-sum(increase(pd_schedule_operators_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", event=\"create\", type=~\".*(peer|region)\"}[1m]))", 13433 "format": "time_series", 13434 "interval": "1m", 13435 "intervalFactor": 1, 13436 "legendFormat": "move-region", 13437 "refId": "C" 13438 } 13439 ], 13440 "thresholds": [], 13441 "timeFrom": null, 13442 "timeRegions": [], 13443 "timeShift": null, 13444 "title": "Eventfeed error/m", 13445 "tooltip": { 13446 "shared": true, 13447 "sort": 0, 13448 "value_type": "individual" 13449 }, 13450 "type": "graph", 13451 "xaxis": { 13452 "buckets": null, 13453 "mode": "time", 13454 "name": null, 13455 "show": true, 13456 "values": [] 13457 }, 13458 "yaxes": [ 13459 { 13460 "format": "short", 13461 "label": null, 13462 "logBase": 1, 13463 "max": null, 13464 "min": null, 13465 "show": true 13466 }, 13467 { 13468 "format": "short", 13469 "label": null, 13470 "logBase": 1, 13471 "max": null, 13472 "min": null, 13473 "show": true 13474 } 13475 ], 13476 "yaxis": { 13477 "align": false, 13478 "alignLevel": null 13479 } 13480 }, 13481 { 13482 "aliasColors": {}, 13483 "bars": false, 13484 "dashLength": 10, 13485 "dashes": false, 13486 "datasource": "${DS_TEST-CLUSTER}", 13487 "description": "The number of KV client dispatched event per second", 13488 "fieldConfig": { 13489 "defaults": { 13490 "links": [] 13491 }, 13492 "overrides": [] 13493 }, 13494 "fill": 1, 13495 "fillGradient": 0, 13496 "gridPos": { 13497 "h": 7, 13498 "w": 12, 13499 "x": 12, 13500 "y": 19 13501 }, 13502 "hiddenSeries": false, 13503 "id": 31, 13504 "legend": { 13505 "alignAsTable": true, 13506 "avg": false, 13507 "current": true, 13508 "hideEmpty": true, 13509 "hideZero": true, 13510 "max": true, 13511 "min": false, 13512 "rightSide": true, 13513 "show": true, 13514 "total": false, 13515 "values": true 13516 }, 13517 "lines": true, 13518 "linewidth": 1, 13519 "links": [], 13520 "nullPointMode": "null", 13521 "options": { 13522 "alertThreshold": true 13523 }, 13524 "paceLength": 10, 13525 "percentage": false, 13526 "pluginVersion": "7.5.11", 13527 "pointradius": 2, 13528 "points": false, 13529 "renderer": "flot", 13530 "seriesOverrides": [ 13531 { 13532 "alias": "/.*batch-resolved/", 13533 "yaxis": 2 13534 } 13535 ], 13536 "spaceLength": 10, 13537 "stack": false, 13538 "steppedLine": false, 13539 "targets": [ 13540 { 13541 "expr": "sum(rate(ticdc_kvclient_send_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[1m])) by (instance, namespace,changefeed, type)", 13542 "format": "time_series", 13543 "intervalFactor": 1, 13544 "legendFormat": "{{instance}}-{{namespace}}-{{changefeed}}-{{type}}", 13545 "refId": "A" 13546 }, 13547 { 13548 "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (instance,namespace, changefeed, table)", 13549 "format": "time_series", 13550 "intervalFactor": 1, 13551 "legendFormat": "{{instance}}-{{namespace}}-{{changefeed}}-batch-resolved", 13552 "refId": "B" 13553 } 13554 ], 13555 "thresholds": [], 13556 "timeFrom": null, 13557 "timeRegions": [], 13558 "timeShift": null, 13559 "title": "KV client dispatch events/s", 13560 "tooltip": { 13561 "shared": true, 13562 "sort": 0, 13563 "value_type": "individual" 13564 }, 13565 "type": "graph", 13566 "xaxis": { 13567 "buckets": null, 13568 "mode": "time", 13569 "name": null, 13570 "show": true, 13571 "values": [] 13572 }, 13573 "yaxes": [ 13574 { 13575 "$$hashKey": "object:542", 13576 "format": "short", 13577 "label": null, 13578 "logBase": 1, 13579 "max": null, 13580 "min": null, 13581 "show": true 13582 }, 13583 { 13584 "$$hashKey": "object:543", 13585 "format": "short", 13586 "label": null, 13587 "logBase": 1, 13588 "max": null, 13589 "min": null, 13590 "show": true 13591 } 13592 ], 13593 "yaxis": { 13594 "align": false, 13595 "alignLevel": null 13596 } 13597 }, 13598 { 13599 "aliasColors": {}, 13600 "bars": false, 13601 "dashLength": 10, 13602 "dashes": false, 13603 "datasource": "${DS_TEST-CLUSTER}", 13604 "description": "Active stream count of each gRPC connection", 13605 "fieldConfig": { 13606 "defaults": { 13607 "links": [] 13608 }, 13609 "overrides": [] 13610 }, 13611 "fill": 1, 13612 "fillGradient": 0, 13613 "gridPos": { 13614 "h": 7, 13615 "w": 12, 13616 "x": 0, 13617 "y": 26 13618 }, 13619 "hiddenSeries": false, 13620 "id": 188, 13621 "legend": { 13622 "alignAsTable": true, 13623 "avg": false, 13624 "current": true, 13625 "hideEmpty": true, 13626 "hideZero": true, 13627 "max": true, 13628 "min": false, 13629 "rightSide": true, 13630 "show": true, 13631 "total": false, 13632 "values": true 13633 }, 13634 "lines": true, 13635 "linewidth": 1, 13636 "links": [], 13637 "nullPointMode": "null", 13638 "options": { 13639 "alertThreshold": true 13640 }, 13641 "paceLength": 10, 13642 "percentage": false, 13643 "pluginVersion": "7.5.11", 13644 "pointradius": 2, 13645 "points": false, 13646 "renderer": "flot", 13647 "seriesOverrides": [], 13648 "spaceLength": 10, 13649 "stack": false, 13650 "steppedLine": false, 13651 "targets": [ 13652 { 13653 "expr": "sum(ticdc_kvclient_grpc_stream_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (store)", 13654 "format": "time_series", 13655 "intervalFactor": 1, 13656 "legendFormat": "{{store}}", 13657 "refId": "A" 13658 } 13659 ], 13660 "thresholds": [], 13661 "timeFrom": null, 13662 "timeRegions": [], 13663 "timeShift": null, 13664 "title": "KV client gRPC stream count", 13665 "tooltip": { 13666 "shared": true, 13667 "sort": 0, 13668 "value_type": "individual" 13669 }, 13670 "type": "graph", 13671 "xaxis": { 13672 "buckets": null, 13673 "mode": "time", 13674 "name": null, 13675 "show": true, 13676 "values": [] 13677 }, 13678 "yaxes": [ 13679 { 13680 "format": "short", 13681 "label": null, 13682 "logBase": 1, 13683 "max": null, 13684 "min": null, 13685 "show": true 13686 }, 13687 { 13688 "format": "short", 13689 "label": null, 13690 "logBase": 1, 13691 "max": null, 13692 "min": null, 13693 "show": true 13694 } 13695 ], 13696 "yaxis": { 13697 "align": false, 13698 "alignLevel": null 13699 } 13700 }, 13701 { 13702 "aliasColors": {}, 13703 "bars": false, 13704 "dashLength": 10, 13705 "dashes": false, 13706 "datasource": "${DS_TEST-CLUSTER}", 13707 "decimals": null, 13708 "description": "Percentiles of Eventfeed message size", 13709 "fieldConfig": { 13710 "defaults": { 13711 "links": [] 13712 }, 13713 "overrides": [] 13714 }, 13715 "fill": 1, 13716 "fillGradient": 0, 13717 "gridPos": { 13718 "h": 7, 13719 "w": 12, 13720 "x": 12, 13721 "y": 26 13722 }, 13723 "hiddenSeries": false, 13724 "id": 17, 13725 "legend": { 13726 "alignAsTable": true, 13727 "avg": false, 13728 "current": true, 13729 "max": false, 13730 "min": false, 13731 "rightSide": true, 13732 "show": true, 13733 "total": false, 13734 "values": true 13735 }, 13736 "lines": true, 13737 "linewidth": 1, 13738 "links": [], 13739 "nullPointMode": "null", 13740 "options": { 13741 "alertThreshold": true 13742 }, 13743 "paceLength": 10, 13744 "percentage": false, 13745 "pluginVersion": "7.5.11", 13746 "pointradius": 2, 13747 "points": false, 13748 "renderer": "flot", 13749 "seriesOverrides": [], 13750 "spaceLength": 10, 13751 "stack": false, 13752 "steppedLine": false, 13753 "targets": [ 13754 { 13755 "expr": "histogram_quantile(0.999, sum(rate(ticdc_kvclient_event_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", 13756 "format": "time_series", 13757 "intervalFactor": 1, 13758 "legendFormat": "{{instance}}-{{type}}-p999", 13759 "refId": "A" 13760 }, 13761 { 13762 "expr": "histogram_quantile(0.95, sum(rate(ticdc_kvclient_event_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", 13763 "format": "time_series", 13764 "intervalFactor": 1, 13765 "legendFormat": "{{instance}}-{{type}}-p95", 13766 "refId": "B" 13767 } 13768 ], 13769 "thresholds": [], 13770 "timeFrom": null, 13771 "timeRegions": [], 13772 "timeShift": null, 13773 "title": "Event size percentile", 13774 "tooltip": { 13775 "shared": true, 13776 "sort": 0, 13777 "value_type": "individual" 13778 }, 13779 "type": "graph", 13780 "xaxis": { 13781 "buckets": null, 13782 "mode": "time", 13783 "name": null, 13784 "show": true, 13785 "values": [] 13786 }, 13787 "yaxes": [ 13788 { 13789 "format": "bytes", 13790 "label": null, 13791 "logBase": 1, 13792 "max": null, 13793 "min": null, 13794 "show": true 13795 }, 13796 { 13797 "format": "short", 13798 "label": null, 13799 "logBase": 1, 13800 "max": null, 13801 "min": null, 13802 "show": true 13803 } 13804 ], 13805 "yaxis": { 13806 "align": false, 13807 "alignLevel": null 13808 } 13809 }, 13810 { 13811 "aliasColors": {}, 13812 "bars": false, 13813 "dashLength": 10, 13814 "dashes": false, 13815 "datasource": "${DS_TEST-CLUSTER}", 13816 "description": "The number of regions that are being scanned", 13817 "fieldConfig": { 13818 "defaults": { 13819 "links": [] 13820 }, 13821 "overrides": [] 13822 }, 13823 "fill": 1, 13824 "fillGradient": 0, 13825 "gridPos": { 13826 "h": 7, 13827 "w": 12, 13828 "x": 0, 13829 "y": 33 13830 }, 13831 "hiddenSeries": false, 13832 "id": 177, 13833 "legend": { 13834 "alignAsTable": true, 13835 "avg": false, 13836 "current": true, 13837 "hideEmpty": true, 13838 "hideZero": true, 13839 "max": true, 13840 "min": false, 13841 "rightSide": true, 13842 "show": true, 13843 "total": false, 13844 "values": true 13845 }, 13846 "lines": true, 13847 "linewidth": 1, 13848 "links": [], 13849 "nullPointMode": "null", 13850 "options": { 13851 "alertThreshold": true 13852 }, 13853 "paceLength": 10, 13854 "percentage": false, 13855 "pluginVersion": "7.5.11", 13856 "pointradius": 2, 13857 "points": false, 13858 "renderer": "flot", 13859 "seriesOverrides": [], 13860 "spaceLength": 10, 13861 "stack": false, 13862 "steppedLine": false, 13863 "targets": [ 13864 { 13865 "exemplar": true, 13866 "expr": "sum(ticdc_kvclient_region_token{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (instance,namespace, changefeed, store)", 13867 "format": "time_series", 13868 "interval": "", 13869 "intervalFactor": 1, 13870 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{store}}", 13871 "refId": "A" 13872 } 13873 ], 13874 "thresholds": [], 13875 "timeFrom": null, 13876 "timeRegions": [], 13877 "timeShift": null, 13878 "title": "KV client scanning regions", 13879 "tooltip": { 13880 "shared": true, 13881 "sort": 0, 13882 "value_type": "individual" 13883 }, 13884 "type": "graph", 13885 "xaxis": { 13886 "buckets": null, 13887 "mode": "time", 13888 "name": null, 13889 "show": true, 13890 "values": [] 13891 }, 13892 "yaxes": [ 13893 { 13894 "format": "short", 13895 "label": null, 13896 "logBase": 1, 13897 "max": null, 13898 "min": null, 13899 "show": true 13900 }, 13901 { 13902 "format": "short", 13903 "label": null, 13904 "logBase": 1, 13905 "max": null, 13906 "min": null, 13907 "show": true 13908 } 13909 ], 13910 "yaxis": { 13911 "align": false, 13912 "alignLevel": null 13913 } 13914 }, 13915 { 13916 "aliasColors": {}, 13917 "bars": false, 13918 "dashLength": 10, 13919 "dashes": false, 13920 "datasource": "${DS_TEST-CLUSTER}", 13921 "decimals": 1, 13922 "description": "The count of regoin cache operation initiated by TiCDC", 13923 "fieldConfig": { 13924 "defaults": {}, 13925 "overrides": [] 13926 }, 13927 "fill": 1, 13928 "fillGradient": 0, 13929 "gridPos": { 13930 "h": 7, 13931 "w": 12, 13932 "x": 12, 13933 "y": 33 13934 }, 13935 "hiddenSeries": false, 13936 "id": 459, 13937 "legend": { 13938 "alignAsTable": true, 13939 "avg": false, 13940 "current": true, 13941 "hideEmpty": true, 13942 "hideZero": true, 13943 "max": true, 13944 "min": false, 13945 "rightSide": true, 13946 "show": true, 13947 "sideWidth": null, 13948 "sort": "current", 13949 "sortDesc": true, 13950 "total": false, 13951 "values": true 13952 }, 13953 "lines": true, 13954 "linewidth": 1, 13955 "links": [], 13956 "nullPointMode": "null", 13957 "options": { 13958 "alertThreshold": true 13959 }, 13960 "percentage": false, 13961 "pluginVersion": "7.5.11", 13962 "pointradius": 5, 13963 "points": false, 13964 "renderer": "flot", 13965 "seriesOverrides": [], 13966 "spaceLength": 10, 13967 "stack": false, 13968 "steppedLine": false, 13969 "targets": [ 13970 { 13971 "expr": "sum(rate(ticdc_tikvclient_region_cache_operations_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (type)", 13972 "format": "time_series", 13973 "intervalFactor": 2, 13974 "legendFormat": "{{type}}", 13975 "metric": "tikv_grpc_msg_duration_seconds_bucket", 13976 "refId": "A", 13977 "step": 10 13978 } 13979 ], 13980 "thresholds": [], 13981 "timeFrom": null, 13982 "timeRegions": [], 13983 "timeShift": null, 13984 "title": "RegionCache operation/s", 13985 "tooltip": { 13986 "shared": true, 13987 "sort": 0, 13988 "value_type": "individual" 13989 }, 13990 "type": "graph", 13991 "xaxis": { 13992 "buckets": null, 13993 "mode": "time", 13994 "name": null, 13995 "show": true, 13996 "values": [] 13997 }, 13998 "yaxes": [ 13999 { 14000 "format": "ops", 14001 "label": null, 14002 "logBase": 1, 14003 "max": null, 14004 "min": null, 14005 "show": true 14006 }, 14007 { 14008 "format": "short", 14009 "label": null, 14010 "logBase": 1, 14011 "max": null, 14012 "min": null, 14013 "show": true 14014 } 14015 ], 14016 "yaxis": { 14017 "align": false, 14018 "alignLevel": null 14019 } 14020 }, 14021 { 14022 "cards": { 14023 "cardPadding": 0, 14024 "cardRound": 0 14025 }, 14026 "color": { 14027 "cardColor": "#FF9830", 14028 "colorScale": "linear", 14029 "colorScheme": "interpolateSpectral", 14030 "exponent": 0.5, 14031 "min": 0, 14032 "mode": "spectrum" 14033 }, 14034 "dataFormat": "tsbuckets", 14035 "datasource": "${DS_TEST-CLUSTER}", 14036 "description": "The size of batch resolved regions count", 14037 "fieldConfig": { 14038 "defaults": {}, 14039 "overrides": [] 14040 }, 14041 "gridPos": { 14042 "h": 7, 14043 "w": 12, 14044 "x": 0, 14045 "y": 40 14046 }, 14047 "heatmap": {}, 14048 "hideZeroBuckets": true, 14049 "highlightCards": true, 14050 "id": 97, 14051 "legend": { 14052 "alignAsTable": true, 14053 "avg": false, 14054 "current": true, 14055 "max": true, 14056 "min": false, 14057 "rightSide": true, 14058 "show": true, 14059 "sort": "current", 14060 "sortDesc": true, 14061 "total": false, 14062 "values": true 14063 }, 14064 "links": [], 14065 "reverseYBuckets": false, 14066 "targets": [ 14067 { 14068 "exemplar": true, 14069 "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", 14070 "format": "heatmap", 14071 "instant": false, 14072 "interval": "", 14073 "intervalFactor": 2, 14074 "legendFormat": "{{le}}", 14075 "refId": "A" 14076 } 14077 ], 14078 "title": "KV client batch resolved region count", 14079 "tooltip": { 14080 "show": true, 14081 "showHistogram": true 14082 }, 14083 "tooltipDecimals": 1, 14084 "type": "heatmap", 14085 "xAxis": { 14086 "show": true 14087 }, 14088 "xBucketNumber": null, 14089 "xBucketSize": null, 14090 "yAxis": { 14091 "decimals": 1, 14092 "format": "none", 14093 "logBase": 1, 14094 "max": null, 14095 "min": null, 14096 "show": true, 14097 "splitFactor": null 14098 }, 14099 "yBucketBound": "upper", 14100 "yBucketNumber": null, 14101 "yBucketSize": null 14102 }, 14103 { 14104 "aliasColors": {}, 14105 "bars": false, 14106 "dashLength": 10, 14107 "dashes": false, 14108 "datasource": "${DS_TEST-CLUSTER}", 14109 "description": "The number of regions that have not connected to TiKV", 14110 "fieldConfig": { 14111 "defaults": { 14112 "links": [] 14113 }, 14114 "overrides": [] 14115 }, 14116 "fill": 1, 14117 "fillGradient": 0, 14118 "gridPos": { 14119 "h": 7, 14120 "w": 12, 14121 "x": 12, 14122 "y": 40 14123 }, 14124 "hiddenSeries": false, 14125 "id": 251, 14126 "legend": { 14127 "alignAsTable": true, 14128 "avg": false, 14129 "current": true, 14130 "hideEmpty": true, 14131 "hideZero": true, 14132 "max": true, 14133 "min": false, 14134 "rightSide": true, 14135 "show": true, 14136 "total": false, 14137 "values": true 14138 }, 14139 "lines": true, 14140 "linewidth": 1, 14141 "links": [], 14142 "nullPointMode": "null", 14143 "options": { 14144 "alertThreshold": true 14145 }, 14146 "paceLength": 10, 14147 "percentage": false, 14148 "pluginVersion": "7.5.11", 14149 "pointradius": 2, 14150 "points": false, 14151 "renderer": "flot", 14152 "seriesOverrides": [], 14153 "spaceLength": 10, 14154 "stack": false, 14155 "steppedLine": false, 14156 "targets": [ 14157 { 14158 "exemplar": true, 14159 "expr": "sum(ticdc_kvclient_cached_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (instance, namespace,changefeed, store)", 14160 "format": "time_series", 14161 "interval": "", 14162 "intervalFactor": 1, 14163 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{store}}", 14164 "refId": "A" 14165 } 14166 ], 14167 "thresholds": [], 14168 "timeFrom": null, 14169 "timeRegions": [], 14170 "timeShift": null, 14171 "title": "KV client cached regions", 14172 "tooltip": { 14173 "shared": true, 14174 "sort": 0, 14175 "value_type": "individual" 14176 }, 14177 "type": "graph", 14178 "xaxis": { 14179 "buckets": null, 14180 "mode": "time", 14181 "name": null, 14182 "show": true, 14183 "values": [] 14184 }, 14185 "yaxes": [ 14186 { 14187 "format": "short", 14188 "label": null, 14189 "logBase": 1, 14190 "max": null, 14191 "min": null, 14192 "show": true 14193 }, 14194 { 14195 "format": "short", 14196 "label": null, 14197 "logBase": 1, 14198 "max": null, 14199 "min": null, 14200 "show": true 14201 } 14202 ], 14203 "yaxis": { 14204 "align": false, 14205 "alignLevel": null 14206 } 14207 }, 14208 { 14209 "aliasColors": {}, 14210 "bars": false, 14211 "dashLength": 10, 14212 "dashes": false, 14213 "datasource": "${DS_TEST-CLUSTER}", 14214 "decimals": 1, 14215 "description": "The count of resolve lock RPC initiated by TiCDC", 14216 "fieldConfig": { 14217 "defaults": {}, 14218 "overrides": [] 14219 }, 14220 "fill": 1, 14221 "fillGradient": 0, 14222 "gridPos": { 14223 "h": 7, 14224 "w": 12, 14225 "x": 0, 14226 "y": 47 14227 }, 14228 "hiddenSeries": false, 14229 "id": 453, 14230 "legend": { 14231 "alignAsTable": true, 14232 "avg": false, 14233 "current": true, 14234 "hideEmpty": true, 14235 "hideZero": true, 14236 "max": true, 14237 "min": false, 14238 "rightSide": true, 14239 "show": true, 14240 "sideWidth": null, 14241 "sort": "current", 14242 "sortDesc": true, 14243 "total": false, 14244 "values": true 14245 }, 14246 "lines": true, 14247 "linewidth": 1, 14248 "links": [], 14249 "nullPointMode": "null", 14250 "options": { 14251 "alertThreshold": true 14252 }, 14253 "percentage": false, 14254 "pluginVersion": "7.5.11", 14255 "pointradius": 5, 14256 "points": false, 14257 "renderer": "flot", 14258 "seriesOverrides": [], 14259 "spaceLength": 10, 14260 "stack": false, 14261 "steppedLine": false, 14262 "targets": [ 14263 { 14264 "expr": "sum(delta(ticdc_tikvclient_lock_resolver_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (type)", 14265 "format": "time_series", 14266 "intervalFactor": 2, 14267 "legendFormat": "{{type}}", 14268 "metric": "tikv_grpc_msg_duration_seconds_bucket", 14269 "refId": "A", 14270 "step": 10 14271 } 14272 ], 14273 "thresholds": [], 14274 "timeFrom": null, 14275 "timeRegions": [], 14276 "timeShift": null, 14277 "title": "ResolveLock count/m", 14278 "tooltip": { 14279 "shared": true, 14280 "sort": 0, 14281 "value_type": "individual" 14282 }, 14283 "type": "graph", 14284 "xaxis": { 14285 "buckets": null, 14286 "mode": "time", 14287 "name": null, 14288 "show": true, 14289 "values": [] 14290 }, 14291 "yaxes": [ 14292 { 14293 "format": "opm", 14294 "label": null, 14295 "logBase": 1, 14296 "max": null, 14297 "min": null, 14298 "show": true 14299 }, 14300 { 14301 "format": "short", 14302 "label": null, 14303 "logBase": 1, 14304 "max": null, 14305 "min": null, 14306 "show": true 14307 } 14308 ], 14309 "yaxis": { 14310 "align": false, 14311 "alignLevel": null 14312 } 14313 }, 14314 { 14315 "aliasColors": {}, 14316 "bars": false, 14317 "dashLength": 10, 14318 "dashes": false, 14319 "datasource": "${DS_TEST-CLUSTER}", 14320 "decimals": 1, 14321 "description": "", 14322 "fieldConfig": { 14323 "defaults": {}, 14324 "overrides": [] 14325 }, 14326 "fill": 1, 14327 "fillGradient": 0, 14328 "gridPos": { 14329 "h": 7, 14330 "w": 12, 14331 "x": 12, 14332 "y": 47 14333 }, 14334 "hiddenSeries": false, 14335 "id": 460, 14336 "legend": { 14337 "alignAsTable": true, 14338 "avg": false, 14339 "current": true, 14340 "hideEmpty": true, 14341 "hideZero": true, 14342 "max": true, 14343 "min": false, 14344 "rightSide": true, 14345 "show": true, 14346 "sideWidth": null, 14347 "sort": "current", 14348 "sortDesc": true, 14349 "total": false, 14350 "values": true 14351 }, 14352 "lines": true, 14353 "linewidth": 1, 14354 "links": [], 14355 "nullPointMode": "null", 14356 "options": { 14357 "alertThreshold": true 14358 }, 14359 "percentage": false, 14360 "pluginVersion": "7.5.11", 14361 "pointradius": 5, 14362 "points": false, 14363 "renderer": "flot", 14364 "seriesOverrides": [], 14365 "spaceLength": 10, 14366 "stack": false, 14367 "steppedLine": false, 14368 "targets": [ 14369 { 14370 "exemplar": true, 14371 "expr": "histogram_quantile(0.99999, sum(rate(ticdc_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, type))", 14372 "format": "time_series", 14373 "interval": "", 14374 "intervalFactor": 2, 14375 "legendFormat": "{{type}}", 14376 "metric": "tikv_grpc_msg_duration_seconds_bucket", 14377 "refId": "A", 14378 "step": 10 14379 } 14380 ], 14381 "thresholds": [], 14382 "timeFrom": null, 14383 "timeRegions": [], 14384 "timeShift": null, 14385 "title": "Backoff percentiles", 14386 "tooltip": { 14387 "shared": true, 14388 "sort": 0, 14389 "value_type": "individual" 14390 }, 14391 "type": "graph", 14392 "xaxis": { 14393 "buckets": null, 14394 "mode": "time", 14395 "name": null, 14396 "show": true, 14397 "values": [] 14398 }, 14399 "yaxes": [ 14400 { 14401 "format": "ms", 14402 "label": null, 14403 "logBase": 1, 14404 "max": null, 14405 "min": null, 14406 "show": true 14407 }, 14408 { 14409 "format": "short", 14410 "label": null, 14411 "logBase": 1, 14412 "max": null, 14413 "min": null, 14414 "show": false 14415 } 14416 ], 14417 "yaxis": { 14418 "align": false, 14419 "alignLevel": null 14420 } 14421 }, 14422 { 14423 "aliasColors": {}, 14424 "bars": false, 14425 "dashLength": 10, 14426 "dashes": false, 14427 "datasource": "${DS_TEST-CLUSTER}", 14428 "description": "The number of events that puller outputs to sorter \n per second", 14429 "fieldConfig": { 14430 "defaults": { 14431 "links": [] 14432 }, 14433 "overrides": [] 14434 }, 14435 "fill": 1, 14436 "fillGradient": 0, 14437 "gridPos": { 14438 "h": 7, 14439 "w": 12, 14440 "x": 0, 14441 "y": 54 14442 }, 14443 "hiddenSeries": false, 14444 "id": 5, 14445 "legend": { 14446 "alignAsTable": true, 14447 "avg": false, 14448 "current": true, 14449 "max": true, 14450 "min": false, 14451 "rightSide": false, 14452 "show": true, 14453 "total": false, 14454 "values": true 14455 }, 14456 "lines": true, 14457 "linewidth": 1, 14458 "links": [], 14459 "nullPointMode": "null", 14460 "options": { 14461 "alertThreshold": true 14462 }, 14463 "paceLength": 10, 14464 "percentage": false, 14465 "pluginVersion": "7.5.11", 14466 "pointradius": 2, 14467 "points": false, 14468 "renderer": "flot", 14469 "seriesOverrides": [], 14470 "spaceLength": 10, 14471 "stack": false, 14472 "steppedLine": false, 14473 "targets": [ 14474 { 14475 "exemplar": true, 14476 "expr": "sum (rate(ticdc_puller_txn_collect_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", changefeed!~\".*ddl_puller\"}[1m])) by (instance, type)", 14477 "format": "time_series", 14478 "interval": "", 14479 "intervalFactor": 1, 14480 "legendFormat": "{{instance}}-{{type}}", 14481 "refId": "A" 14482 } 14483 ], 14484 "thresholds": [], 14485 "timeFrom": null, 14486 "timeRegions": [], 14487 "timeShift": null, 14488 "title": "Puller output events/s", 14489 "tooltip": { 14490 "shared": true, 14491 "sort": 0, 14492 "value_type": "individual" 14493 }, 14494 "type": "graph", 14495 "xaxis": { 14496 "buckets": null, 14497 "mode": "time", 14498 "name": null, 14499 "show": true, 14500 "values": [] 14501 }, 14502 "yaxes": [ 14503 { 14504 "format": "none", 14505 "label": null, 14506 "logBase": 1, 14507 "max": null, 14508 "min": null, 14509 "show": true 14510 }, 14511 { 14512 "format": "short", 14513 "label": null, 14514 "logBase": 1, 14515 "max": null, 14516 "min": null, 14517 "show": true 14518 } 14519 ], 14520 "yaxis": { 14521 "align": false, 14522 "alignLevel": null 14523 } 14524 }, 14525 { 14526 "aliasColors": {}, 14527 "bars": false, 14528 "dashLength": 10, 14529 "dashes": false, 14530 "datasource": "${DS_TEST-CLUSTER}", 14531 "description": "Estimate the remaining time for a changefeed initialization (on a specific capture)", 14532 "fieldConfig": { 14533 "defaults": { 14534 "unit": "s" 14535 }, 14536 "overrides": [] 14537 }, 14538 "fill": 1, 14539 "fillGradient": 0, 14540 "gridPos": { 14541 "h": 7, 14542 "w": 12, 14543 "x": 12, 14544 "y": 54 14545 }, 14546 "hiddenSeries": false, 14547 "id": 252, 14548 "legend": { 14549 "alignAsTable": true, 14550 "avg": false, 14551 "current": true, 14552 "hideEmpty": true, 14553 "hideZero": true, 14554 "max": true, 14555 "min": false, 14556 "rightSide": true, 14557 "show": true, 14558 "total": false, 14559 "values": true 14560 }, 14561 "lines": true, 14562 "linewidth": 1, 14563 "links": [], 14564 "nullPointMode": "null", 14565 "options": { 14566 "alertThreshold": true 14567 }, 14568 "paceLength": 10, 14569 "percentage": false, 14570 "pluginVersion": "7.5.11", 14571 "pointradius": 2, 14572 "points": false, 14573 "renderer": "flot", 14574 "seriesOverrides": [], 14575 "spaceLength": 10, 14576 "stack": false, 14577 "steppedLine": false, 14578 "targets": [ 14579 { 14580 "exemplar": true, 14581 "expr": "abs(sum(ticdc_kvclient_cached_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"} / deriv(ticdc_kvclient_cached_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (instance, namespace,changefeed, store))", 14582 "format": "time_series", 14583 "interval": "", 14584 "intervalFactor": 1, 14585 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{store}}", 14586 "refId": "A" 14587 } 14588 ], 14589 "thresholds": [], 14590 "timeFrom": null, 14591 "timeRegions": [], 14592 "timeShift": null, 14593 "title": "Estimate remaining time for initialization", 14594 "tooltip": { 14595 "shared": true, 14596 "sort": 0, 14597 "value_type": "individual" 14598 }, 14599 "type": "graph", 14600 "xaxis": { 14601 "buckets": null, 14602 "mode": "time", 14603 "name": null, 14604 "show": true, 14605 "values": [] 14606 }, 14607 "yaxes": [ 14608 { 14609 "format": "s", 14610 "label": "", 14611 "logBase": 2, 14612 "max": null, 14613 "min": null, 14614 "show": true 14615 }, 14616 { 14617 "format": "short", 14618 "label": null, 14619 "logBase": 1, 14620 "max": null, 14621 "min": null, 14622 "show": true 14623 } 14624 ], 14625 "yaxis": { 14626 "align": false, 14627 "alignLevel": null 14628 } 14629 }, 14630 { 14631 "cards": { 14632 "cardPadding": 0, 14633 "cardRound": 0 14634 }, 14635 "color": { 14636 "cardColor": "#FF9830", 14637 "colorScale": "linear", 14638 "colorScheme": "interpolateSpectral", 14639 "exponent": 0.5, 14640 "min": 0, 14641 "mode": "spectrum" 14642 }, 14643 "dataFormat": "tsbuckets", 14644 "datasource": "${DS_TEST-CLUSTER}", 14645 "description": "Duration of sorting DDL events", 14646 "fieldConfig": { 14647 "defaults": {}, 14648 "overrides": [] 14649 }, 14650 "gridPos": { 14651 "h": 7, 14652 "w": 12, 14653 "x": 0, 14654 "y": 61 14655 }, 14656 "heatmap": {}, 14657 "hideZeroBuckets": true, 14658 "highlightCards": true, 14659 "id": 99, 14660 "legend": { 14661 "alignAsTable": true, 14662 "avg": false, 14663 "current": true, 14664 "max": true, 14665 "min": false, 14666 "rightSide": true, 14667 "show": true, 14668 "sort": "current", 14669 "sortDesc": true, 14670 "total": false, 14671 "values": true 14672 }, 14673 "links": [], 14674 "reverseYBuckets": false, 14675 "targets": [ 14676 { 14677 "exemplar": true, 14678 "expr": "sum(rate(ticdc_puller_entry_sorter_sort_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", 14679 "format": "heatmap", 14680 "instant": false, 14681 "interval": "", 14682 "intervalFactor": 2, 14683 "legendFormat": "{{le}}", 14684 "refId": "A" 14685 } 14686 ], 14687 "title": "DDL sorter sort duration", 14688 "tooltip": { 14689 "show": true, 14690 "showHistogram": true 14691 }, 14692 "tooltipDecimals": 1, 14693 "type": "heatmap", 14694 "xAxis": { 14695 "show": true 14696 }, 14697 "xBucketNumber": null, 14698 "xBucketSize": null, 14699 "yAxis": { 14700 "decimals": 1, 14701 "format": "s", 14702 "logBase": 1, 14703 "max": null, 14704 "min": null, 14705 "show": true, 14706 "splitFactor": null 14707 }, 14708 "yBucketBound": "upper", 14709 "yBucketNumber": null, 14710 "yBucketSize": null 14711 }, 14712 { 14713 "cards": { 14714 "cardPadding": 0, 14715 "cardRound": 0 14716 }, 14717 "color": { 14718 "cardColor": "#FF9830", 14719 "colorScale": "linear", 14720 "colorScheme": "interpolateSpectral", 14721 "exponent": 0.5, 14722 "min": 0, 14723 "mode": "spectrum" 14724 }, 14725 "dataFormat": "tsbuckets", 14726 "datasource": "${DS_TEST-CLUSTER}", 14727 "description": "Duration of merging sorted events", 14728 "fieldConfig": { 14729 "defaults": {}, 14730 "overrides": [] 14731 }, 14732 "gridPos": { 14733 "h": 7, 14734 "w": 12, 14735 "x": 12, 14736 "y": 61 14737 }, 14738 "heatmap": {}, 14739 "hideZeroBuckets": true, 14740 "highlightCards": true, 14741 "id": 105, 14742 "legend": { 14743 "alignAsTable": true, 14744 "avg": false, 14745 "current": true, 14746 "max": true, 14747 "min": false, 14748 "rightSide": true, 14749 "show": true, 14750 "sort": "current", 14751 "sortDesc": true, 14752 "total": false, 14753 "values": true 14754 }, 14755 "links": [], 14756 "reverseYBuckets": false, 14757 "targets": [ 14758 { 14759 "expr": "sum(rate(ticdc_puller_entry_sorter_merge_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", 14760 "format": "heatmap", 14761 "instant": false, 14762 "intervalFactor": 2, 14763 "legendFormat": "{{le}}", 14764 "refId": "A" 14765 } 14766 ], 14767 "title": "DDL sorter merge duration", 14768 "tooltip": { 14769 "show": true, 14770 "showHistogram": true 14771 }, 14772 "tooltipDecimals": 1, 14773 "type": "heatmap", 14774 "xAxis": { 14775 "show": true 14776 }, 14777 "xBucketNumber": null, 14778 "xBucketSize": null, 14779 "yAxis": { 14780 "decimals": 1, 14781 "format": "s", 14782 "logBase": 1, 14783 "max": null, 14784 "min": null, 14785 "show": true, 14786 "splitFactor": null 14787 }, 14788 "yBucketBound": "upper", 14789 "yBucketNumber": null, 14790 "yBucketSize": null 14791 }, 14792 { 14793 "aliasColors": {}, 14794 "bars": false, 14795 "dashLength": 10, 14796 "dashes": false, 14797 "datasource": "${DS_TEST-CLUSTER}", 14798 "description": "Percentiles of sorting events duration", 14799 "fieldConfig": { 14800 "defaults": { 14801 "links": [] 14802 }, 14803 "overrides": [] 14804 }, 14805 "fill": 1, 14806 "fillGradient": 0, 14807 "gridPos": { 14808 "h": 7, 14809 "w": 12, 14810 "x": 0, 14811 "y": 68 14812 }, 14813 "hiddenSeries": false, 14814 "id": 53, 14815 "legend": { 14816 "alignAsTable": true, 14817 "avg": false, 14818 "current": true, 14819 "hideEmpty": false, 14820 "max": true, 14821 "min": false, 14822 "rightSide": true, 14823 "show": true, 14824 "total": false, 14825 "values": true 14826 }, 14827 "lines": true, 14828 "linewidth": 1, 14829 "links": [], 14830 "nullPointMode": "null", 14831 "options": { 14832 "alertThreshold": true 14833 }, 14834 "paceLength": 10, 14835 "percentage": false, 14836 "pluginVersion": "7.5.11", 14837 "pointradius": 2, 14838 "points": false, 14839 "renderer": "flot", 14840 "seriesOverrides": [], 14841 "spaceLength": 10, 14842 "stack": false, 14843 "steppedLine": false, 14844 "targets": [ 14845 { 14846 "exemplar": true, 14847 "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_sort_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", 14848 "format": "time_series", 14849 "interval": "", 14850 "intervalFactor": 1, 14851 "legendFormat": "{{instance}}-p999", 14852 "refId": "A" 14853 }, 14854 { 14855 "exemplar": true, 14856 "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_sort_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", 14857 "format": "time_series", 14858 "hide": false, 14859 "interval": "", 14860 "intervalFactor": 1, 14861 "legendFormat": "{{instance}}-p95", 14862 "refId": "B" 14863 } 14864 ], 14865 "thresholds": [], 14866 "timeFrom": null, 14867 "timeRegions": [], 14868 "timeShift": null, 14869 "title": "DDL sorter sort duration percentile", 14870 "tooltip": { 14871 "shared": true, 14872 "sort": 0, 14873 "value_type": "individual" 14874 }, 14875 "type": "graph", 14876 "xaxis": { 14877 "buckets": null, 14878 "mode": "time", 14879 "name": null, 14880 "show": true, 14881 "values": [] 14882 }, 14883 "yaxes": [ 14884 { 14885 "format": "s", 14886 "label": null, 14887 "logBase": 2, 14888 "max": null, 14889 "min": null, 14890 "show": true 14891 }, 14892 { 14893 "format": "short", 14894 "label": null, 14895 "logBase": 1, 14896 "max": null, 14897 "min": null, 14898 "show": true 14899 } 14900 ], 14901 "yaxis": { 14902 "align": false, 14903 "alignLevel": null 14904 } 14905 }, 14906 { 14907 "aliasColors": {}, 14908 "bars": false, 14909 "dashLength": 10, 14910 "dashes": false, 14911 "datasource": "${DS_TEST-CLUSTER}", 14912 "description": "Percentiles of merging sorted events duration", 14913 "fieldConfig": { 14914 "defaults": { 14915 "links": [] 14916 }, 14917 "overrides": [] 14918 }, 14919 "fill": 1, 14920 "fillGradient": 0, 14921 "gridPos": { 14922 "h": 7, 14923 "w": 12, 14924 "x": 12, 14925 "y": 68 14926 }, 14927 "hiddenSeries": false, 14928 "id": 106, 14929 "legend": { 14930 "alignAsTable": true, 14931 "avg": false, 14932 "current": true, 14933 "hideEmpty": false, 14934 "max": true, 14935 "min": false, 14936 "rightSide": true, 14937 "show": true, 14938 "total": false, 14939 "values": true 14940 }, 14941 "lines": true, 14942 "linewidth": 1, 14943 "links": [], 14944 "nullPointMode": "null", 14945 "options": { 14946 "alertThreshold": true 14947 }, 14948 "paceLength": 10, 14949 "percentage": false, 14950 "pluginVersion": "7.5.11", 14951 "pointradius": 2, 14952 "points": false, 14953 "renderer": "flot", 14954 "seriesOverrides": [], 14955 "spaceLength": 10, 14956 "stack": false, 14957 "steppedLine": false, 14958 "targets": [ 14959 { 14960 "exemplar": true, 14961 "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_merge_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", 14962 "format": "time_series", 14963 "interval": "", 14964 "intervalFactor": 1, 14965 "legendFormat": "{{instance}}-p999", 14966 "refId": "A" 14967 }, 14968 { 14969 "exemplar": true, 14970 "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_merge_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", 14971 "format": "time_series", 14972 "hide": false, 14973 "interval": "", 14974 "intervalFactor": 1, 14975 "legendFormat": "{{instance}}-p95", 14976 "refId": "B" 14977 } 14978 ], 14979 "thresholds": [], 14980 "timeFrom": null, 14981 "timeRegions": [], 14982 "timeShift": null, 14983 "title": "DDL sorter merge duration percentile", 14984 "tooltip": { 14985 "shared": true, 14986 "sort": 0, 14987 "value_type": "individual" 14988 }, 14989 "type": "graph", 14990 "xaxis": { 14991 "buckets": null, 14992 "mode": "time", 14993 "name": null, 14994 "show": true, 14995 "values": [] 14996 }, 14997 "yaxes": [ 14998 { 14999 "format": "s", 15000 "label": null, 15001 "logBase": 2, 15002 "max": null, 15003 "min": null, 15004 "show": true 15005 }, 15006 { 15007 "format": "short", 15008 "label": null, 15009 "logBase": 1, 15010 "max": null, 15011 "min": null, 15012 "show": true 15013 } 15014 ], 15015 "yaxis": { 15016 "align": false, 15017 "alignLevel": null 15018 } 15019 }, 15020 { 15021 "aliasColors": {}, 15022 "bars": false, 15023 "dashLength": 10, 15024 "dashes": false, 15025 "datasource": "${DS_TEST-CLUSTER}", 15026 "description": "batch read events from the sorter, record each batch size ", 15027 "fieldConfig": { 15028 "defaults": {}, 15029 "overrides": [] 15030 }, 15031 "fill": 1, 15032 "fillGradient": 0, 15033 "gridPos": { 15034 "h": 7, 15035 "w": 12, 15036 "x": 0, 15037 "y": 75 15038 }, 15039 "hiddenSeries": false, 15040 "id": 705, 15041 "legend": { 15042 "alignAsTable": true, 15043 "avg": false, 15044 "current": true, 15045 "max": true, 15046 "min": false, 15047 "rightSide": false, 15048 "show": true, 15049 "total": false, 15050 "values": true 15051 }, 15052 "lines": true, 15053 "linewidth": 1, 15054 "nullPointMode": "null", 15055 "options": { 15056 "alertThreshold": true 15057 }, 15058 "percentage": false, 15059 "pluginVersion": "7.5.11", 15060 "pointradius": 2, 15061 "points": false, 15062 "renderer": "flot", 15063 "seriesOverrides": [], 15064 "spaceLength": 10, 15065 "stack": false, 15066 "steppedLine": false, 15067 "targets": [ 15068 { 15069 "exemplar": true, 15070 "expr": "histogram_quantile(0.999, sum(rate(ticdc_processor_sorter_batch_read_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le,namespace,instance,changefeed))", 15071 "hide": false, 15072 "interval": "", 15073 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-p999", 15074 "refId": "A" 15075 } 15076 ], 15077 "thresholds": [], 15078 "timeFrom": null, 15079 "timeRegions": [], 15080 "timeShift": null, 15081 "title": "batch read sorter size percentile", 15082 "tooltip": { 15083 "shared": true, 15084 "sort": 0, 15085 "value_type": "individual" 15086 }, 15087 "type": "graph", 15088 "xaxis": { 15089 "buckets": null, 15090 "mode": "time", 15091 "name": null, 15092 "show": true, 15093 "values": [] 15094 }, 15095 "yaxes": [ 15096 { 15097 "format": "none", 15098 "label": null, 15099 "logBase": 1, 15100 "max": null, 15101 "min": null, 15102 "show": true 15103 }, 15104 { 15105 "format": "none", 15106 "label": null, 15107 "logBase": 1, 15108 "max": null, 15109 "min": null, 15110 "show": false 15111 } 15112 ], 15113 "yaxis": { 15114 "align": false, 15115 "alignLevel": null 15116 } 15117 }, 15118 { 15119 "aliasColors": {}, 15120 "bars": false, 15121 "dashLength": 10, 15122 "dashes": false, 15123 "datasource": "${DS_TEST-CLUSTER}", 15124 "description": "batch read events from the sorter, record each batch read duration ", 15125 "fieldConfig": { 15126 "defaults": {}, 15127 "overrides": [] 15128 }, 15129 "fill": 1, 15130 "fillGradient": 0, 15131 "gridPos": { 15132 "h": 7, 15133 "w": 12, 15134 "x": 12, 15135 "y": 75 15136 }, 15137 "hiddenSeries": false, 15138 "id": 706, 15139 "legend": { 15140 "alignAsTable": true, 15141 "avg": false, 15142 "current": true, 15143 "max": true, 15144 "min": false, 15145 "rightSide": false, 15146 "show": true, 15147 "total": false, 15148 "values": true 15149 }, 15150 "lines": true, 15151 "linewidth": 1, 15152 "nullPointMode": "null", 15153 "options": { 15154 "alertThreshold": true 15155 }, 15156 "percentage": false, 15157 "pluginVersion": "7.5.11", 15158 "pointradius": 2, 15159 "points": false, 15160 "renderer": "flot", 15161 "seriesOverrides": [], 15162 "spaceLength": 10, 15163 "stack": false, 15164 "steppedLine": false, 15165 "targets": [ 15166 { 15167 "exemplar": true, 15168 "expr": "histogram_quantile(0.999, sum(rate(ticdc_processor_sorter_batch_read_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le,instance,namespace,changefeed))", 15169 "hide": false, 15170 "interval": "", 15171 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-p999", 15172 "refId": "A" 15173 } 15174 ], 15175 "thresholds": [], 15176 "timeFrom": null, 15177 "timeRegions": [], 15178 "timeShift": null, 15179 "title": "batch read sorter duration percentile", 15180 "tooltip": { 15181 "shared": true, 15182 "sort": 0, 15183 "value_type": "individual" 15184 }, 15185 "type": "graph", 15186 "xaxis": { 15187 "buckets": null, 15188 "mode": "time", 15189 "name": null, 15190 "show": true, 15191 "values": [] 15192 }, 15193 "yaxes": [ 15194 { 15195 "format": "s", 15196 "label": null, 15197 "logBase": 1, 15198 "max": null, 15199 "min": null, 15200 "show": true 15201 }, 15202 { 15203 "format": "none", 15204 "label": null, 15205 "logBase": 1, 15206 "max": null, 15207 "min": null, 15208 "show": false 15209 } 15210 ], 15211 "yaxis": { 15212 "align": false, 15213 "alignLevel": null 15214 } 15215 }, 15216 { 15217 "aliasColors": {}, 15218 "bars": false, 15219 "dashLength": 10, 15220 "dashes": false, 15221 "datasource": "${DS_TEST-CLUSTER}", 15222 "description": "", 15223 "fieldConfig": { 15224 "defaults": {}, 15225 "overrides": [] 15226 }, 15227 "fill": 1, 15228 "fillGradient": 0, 15229 "gridPos": { 15230 "h": 7, 15231 "w": 12, 15232 "x": 0, 15233 "y": 82 15234 }, 15235 "hiddenSeries": false, 15236 "id": 707, 15237 "legend": { 15238 "alignAsTable": true, 15239 "avg": false, 15240 "current": true, 15241 "max": true, 15242 "min": false, 15243 "rightSide": false, 15244 "show": true, 15245 "total": false, 15246 "values": true 15247 }, 15248 "lines": true, 15249 "linewidth": 1, 15250 "nullPointMode": "null", 15251 "options": { 15252 "alertThreshold": true 15253 }, 15254 "percentage": false, 15255 "pluginVersion": "7.5.11", 15256 "pointradius": 2, 15257 "points": false, 15258 "renderer": "flot", 15259 "seriesOverrides": [], 15260 "spaceLength": 10, 15261 "stack": false, 15262 "steppedLine": false, 15263 "targets": [ 15264 { 15265 "exemplar": true, 15266 "expr": "ticdc_mounter_group_input_chan_size{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}", 15267 "hide": false, 15268 "interval": "", 15269 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{index}}", 15270 "refId": "A" 15271 } 15272 ], 15273 "thresholds": [], 15274 "timeFrom": null, 15275 "timeRegions": [], 15276 "timeShift": null, 15277 "title": "mounter group channel size", 15278 "tooltip": { 15279 "shared": true, 15280 "sort": 0, 15281 "value_type": "individual" 15282 }, 15283 "type": "graph", 15284 "xaxis": { 15285 "buckets": null, 15286 "mode": "time", 15287 "name": null, 15288 "show": true, 15289 "values": [] 15290 }, 15291 "yaxes": [ 15292 { 15293 "format": "none", 15294 "label": null, 15295 "logBase": 1, 15296 "max": null, 15297 "min": null, 15298 "show": true 15299 }, 15300 { 15301 "format": "none", 15302 "label": null, 15303 "logBase": 1, 15304 "max": null, 15305 "min": null, 15306 "show": false 15307 } 15308 ], 15309 "yaxis": { 15310 "align": false, 15311 "alignLevel": null 15312 } 15313 }, 15314 { 15315 "aliasColors": {}, 15316 "dashLength": 10, 15317 "datasource": "${DS_TEST-CLUSTER}", 15318 "fieldConfig": { 15319 "defaults": {}, 15320 "overrides": [] 15321 }, 15322 "fill": 1, 15323 "gridPos": { 15324 "h": 7, 15325 "w": 12, 15326 "x": 12, 15327 "y": 122 15328 }, 15329 "id": 10039, 15330 "legend": { 15331 "alignAsTable": true, 15332 "avg": false, 15333 "current": true, 15334 "max": true, 15335 "min": false, 15336 "rightSide": false, 15337 "show": true, 15338 "total": false, 15339 "values": true 15340 }, 15341 "lines": true, 15342 "linewidth": 1, 15343 "nullPointMode": "null", 15344 "options": { 15345 "alertThreshold": true 15346 }, 15347 "pluginVersion": "7.5.17", 15348 "pointradius": 2, 15349 "renderer": "flot", 15350 "seriesOverrides": [], 15351 "spaceLength": 10, 15352 "targets": [ 15353 { 15354 "expr": "ticdc_kvclient_slow_initialize_region_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}", 15355 "legendFormat": "{{instance}}-{{changefeed}}-{{namespace}}", 15356 "interval": "", 15357 "exemplar": true, 15358 "hide": false, 15359 "refId": "A" 15360 } 15361 ], 15362 "thresholds": [], 15363 "timeRegions": [], 15364 "title": "KV client slow initalize region count", 15365 "tooltip": { 15366 "shared": true, 15367 "sort": 0, 15368 "value_type": "individual" 15369 }, 15370 "type": "graph", 15371 "xaxis": { 15372 "buckets": null, 15373 "mode": "time", 15374 "name": null, 15375 "show": true, 15376 "values": [] 15377 }, 15378 "yaxes": [ 15379 { 15380 "format": "none", 15381 "label": null, 15382 "logBase": 1, 15383 "max": null, 15384 "min": null, 15385 "show": true, 15386 "$$hashKey": "object:432" 15387 }, 15388 { 15389 "format": "none", 15390 "label": null, 15391 "logBase": 1, 15392 "max": null, 15393 "min": null, 15394 "show": false, 15395 "$$hashKey": "object:433" 15396 } 15397 ], 15398 "yaxis": { 15399 "align": false, 15400 "alignLevel": null 15401 }, 15402 "description": "The count of regions that initialize slow. You can search the log [event feed initializes a region too slow] to get slow region id", 15403 "bars": false, 15404 "dashes": false, 15405 "fillGradient": 0, 15406 "hiddenSeries": false, 15407 "percentage": false, 15408 "points": false, 15409 "stack": false, 15410 "steppedLine": false, 15411 "timeFrom": null, 15412 "timeShift": null 15413 } 15414 ], 15415 "title": "KVClient", 15416 "type": "row" 15417 }, 15418 { 15419 "collapsed": true, 15420 "datasource": null, 15421 "gridPos": { 15422 "h": 1, 15423 "w": 24, 15424 "x": 0, 15425 "y": 12 15426 }, 15427 "id": 266, 15428 "panels": [ 15429 { 15430 "aliasColors": {}, 15431 "bars": false, 15432 "dashLength": 10, 15433 "dashes": false, 15434 "datasource": "${DS_TEST-CLUSTER}", 15435 "fieldConfig": { 15436 "defaults": {}, 15437 "overrides": [] 15438 }, 15439 "fill": 1, 15440 "fillGradient": 0, 15441 "gridPos": { 15442 "h": 7, 15443 "w": 12, 15444 "x": 0, 15445 "y": 13 15446 }, 15447 "hiddenSeries": false, 15448 "id": 289, 15449 "legend": { 15450 "alignAsTable": true, 15451 "avg": false, 15452 "current": true, 15453 "hideEmpty": true, 15454 "hideZero": true, 15455 "max": true, 15456 "min": false, 15457 "rightSide": true, 15458 "show": true, 15459 "total": false, 15460 "values": true 15461 }, 15462 "lines": true, 15463 "linewidth": 1, 15464 "links": [], 15465 "nullPointMode": "null", 15466 "options": { 15467 "alertThreshold": true 15468 }, 15469 "percentage": false, 15470 "pluginVersion": "7.5.11", 15471 "pointradius": 2, 15472 "points": false, 15473 "renderer": "flot", 15474 "seriesOverrides": [], 15475 "spaceLength": 10, 15476 "stack": false, 15477 "steppedLine": false, 15478 "targets": [ 15479 { 15480 "expr": "etcd_debugging_mvcc_db_total_size_in_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"pd\"}", 15481 "format": "time_series", 15482 "hide": false, 15483 "intervalFactor": 1, 15484 "legendFormat": "{{instance}}", 15485 "refId": "A" 15486 } 15487 ], 15488 "thresholds": [], 15489 "timeFrom": null, 15490 "timeRegions": [], 15491 "timeShift": null, 15492 "title": "Etcd MVCC DB total size", 15493 "tooltip": { 15494 "shared": true, 15495 "sort": 0, 15496 "value_type": "individual" 15497 }, 15498 "type": "graph", 15499 "xaxis": { 15500 "buckets": null, 15501 "mode": "time", 15502 "name": null, 15503 "show": true, 15504 "values": [] 15505 }, 15506 "yaxes": [ 15507 { 15508 "format": "decbytes", 15509 "label": null, 15510 "logBase": 1, 15511 "max": null, 15512 "min": null, 15513 "show": true 15514 }, 15515 { 15516 "format": "short", 15517 "label": null, 15518 "logBase": 1, 15519 "max": null, 15520 "min": null, 15521 "show": false 15522 } 15523 ], 15524 "yaxis": { 15525 "align": false, 15526 "alignLevel": null 15527 } 15528 }, 15529 { 15530 "aliasColors": {}, 15531 "bars": false, 15532 "dashLength": 10, 15533 "dashes": false, 15534 "datasource": "${DS_TEST-CLUSTER}", 15535 "fieldConfig": { 15536 "defaults": { 15537 "links": [] 15538 }, 15539 "overrides": [] 15540 }, 15541 "fill": 1, 15542 "fillGradient": 0, 15543 "gridPos": { 15544 "h": 7, 15545 "w": 12, 15546 "x": 12, 15547 "y": 13 15548 }, 15549 "hiddenSeries": false, 15550 "id": 114, 15551 "legend": { 15552 "alignAsTable": true, 15553 "avg": false, 15554 "current": true, 15555 "hideEmpty": true, 15556 "max": true, 15557 "min": false, 15558 "rightSide": true, 15559 "show": true, 15560 "total": false, 15561 "values": true 15562 }, 15563 "lines": true, 15564 "linewidth": 1, 15565 "links": [], 15566 "nullPointMode": "null", 15567 "options": { 15568 "alertThreshold": true 15569 }, 15570 "paceLength": 10, 15571 "percentage": false, 15572 "pluginVersion": "7.5.11", 15573 "pointradius": 2, 15574 "points": false, 15575 "renderer": "flot", 15576 "seriesOverrides": [], 15577 "spaceLength": 10, 15578 "stack": false, 15579 "steppedLine": false, 15580 "targets": [ 15581 { 15582 "expr": "histogram_quantile(0.999, sum(rate(ticdc_server_etcd_health_check_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le,instance))", 15583 "format": "time_series", 15584 "intervalFactor": 1, 15585 "legendFormat": "p999-{{instance}}", 15586 "refId": "A" 15587 }, 15588 { 15589 "expr": "histogram_quantile(0.99, sum(rate(ticdc_server_etcd_health_check_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le,instance))", 15590 "format": "time_series", 15591 "intervalFactor": 1, 15592 "legendFormat": "p99-{{instance}}", 15593 "refId": "B" 15594 }, 15595 { 15596 "expr": "histogram_quantile(0.95, sum(rate(ticdc_server_etcd_health_check_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le,instance))", 15597 "format": "time_series", 15598 "intervalFactor": 1, 15599 "legendFormat": "p95-{{instance}}", 15600 "refId": "C" 15601 } 15602 ], 15603 "thresholds": [], 15604 "timeFrom": null, 15605 "timeRegions": [], 15606 "timeShift": null, 15607 "title": "Etcd health check duration", 15608 "tooltip": { 15609 "shared": true, 15610 "sort": 0, 15611 "value_type": "individual" 15612 }, 15613 "type": "graph", 15614 "xaxis": { 15615 "buckets": null, 15616 "mode": "time", 15617 "name": null, 15618 "show": true, 15619 "values": [] 15620 }, 15621 "yaxes": [ 15622 { 15623 "format": "s", 15624 "label": null, 15625 "logBase": 1, 15626 "max": null, 15627 "min": null, 15628 "show": true 15629 }, 15630 { 15631 "format": "short", 15632 "label": null, 15633 "logBase": 1, 15634 "max": null, 15635 "min": null, 15636 "show": true 15637 } 15638 ], 15639 "yaxis": { 15640 "align": false, 15641 "alignLevel": null 15642 } 15643 }, 15644 { 15645 "cards": { 15646 "cardPadding": 0, 15647 "cardRound": 0 15648 }, 15649 "color": { 15650 "cardColor": "#b4ff00", 15651 "colorScale": "sqrt", 15652 "colorScheme": "interpolateSpectral", 15653 "exponent": 0.5, 15654 "max": null, 15655 "min": 1, 15656 "mode": "spectrum" 15657 }, 15658 "dataFormat": "tsbuckets", 15659 "datasource": "${DS_TEST-CLUSTER}", 15660 "description": "", 15661 "fieldConfig": { 15662 "defaults": {}, 15663 "overrides": [] 15664 }, 15665 "gridPos": { 15666 "h": 7, 15667 "w": 12, 15668 "x": 0, 15669 "y": 20 15670 }, 15671 "heatmap": {}, 15672 "hideZeroBuckets": true, 15673 "highlightCards": true, 15674 "id": 267, 15675 "legend": { 15676 "show": true 15677 }, 15678 "pluginVersion": "6.1.6", 15679 "reverseYBuckets": false, 15680 "targets": [ 15681 { 15682 "exemplar": true, 15683 "expr": "sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", 15684 "format": "heatmap", 15685 "instant": false, 15686 "interval": "", 15687 "intervalFactor": 2, 15688 "legendFormat": "{{le}}", 15689 "queryType": "randomWalk", 15690 "refId": "A" 15691 } 15692 ], 15693 "timeFrom": null, 15694 "timeShift": null, 15695 "title": "EtcdWorker tick reactor duration", 15696 "tooltip": { 15697 "show": true, 15698 "showHistogram": true 15699 }, 15700 "tooltipDecimals": 1, 15701 "type": "heatmap", 15702 "xAxis": { 15703 "show": true 15704 }, 15705 "xBucketNumber": null, 15706 "xBucketSize": null, 15707 "yAxis": { 15708 "decimals": 1, 15709 "format": "s", 15710 "logBase": 1, 15711 "max": null, 15712 "min": null, 15713 "show": true, 15714 "splitFactor": null 15715 }, 15716 "yBucketBound": "upper", 15717 "yBucketNumber": null, 15718 "yBucketSize": null 15719 }, 15720 { 15721 "aliasColors": {}, 15722 "bars": false, 15723 "dashLength": 10, 15724 "dashes": false, 15725 "datasource": "${DS_TEST-CLUSTER}", 15726 "description": "", 15727 "fieldConfig": { 15728 "defaults": {}, 15729 "overrides": [] 15730 }, 15731 "fill": 1, 15732 "fillGradient": 0, 15733 "gridPos": { 15734 "h": 7, 15735 "w": 12, 15736 "x": 12, 15737 "y": 20 15738 }, 15739 "hiddenSeries": false, 15740 "id": 264, 15741 "legend": { 15742 "avg": false, 15743 "current": false, 15744 "max": false, 15745 "min": false, 15746 "show": true, 15747 "total": false, 15748 "values": false 15749 }, 15750 "lines": true, 15751 "linewidth": 1, 15752 "nullPointMode": "null", 15753 "options": { 15754 "alertThreshold": true 15755 }, 15756 "percentage": false, 15757 "pluginVersion": "7.5.11", 15758 "pointradius": 2, 15759 "points": false, 15760 "renderer": "flot", 15761 "seriesOverrides": [], 15762 "spaceLength": 10, 15763 "stack": false, 15764 "steppedLine": false, 15765 "targets": [ 15766 { 15767 "exemplar": true, 15768 "expr": "histogram_quantile(0.999, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", 15769 "hide": false, 15770 "interval": "", 15771 "legendFormat": "{{instance}}-999", 15772 "refId": "A" 15773 }, 15774 { 15775 "exemplar": true, 15776 "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_tick_reactor_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", 15777 "hide": false, 15778 "interval": "", 15779 "legendFormat": "{{instance}}-99", 15780 "refId": "B" 15781 } 15782 ], 15783 "thresholds": [], 15784 "timeFrom": null, 15785 "timeRegions": [], 15786 "timeShift": null, 15787 "title": "EtcdWorker tick reactor duration", 15788 "tooltip": { 15789 "shared": true, 15790 "sort": 0, 15791 "value_type": "individual" 15792 }, 15793 "type": "graph", 15794 "xaxis": { 15795 "buckets": null, 15796 "mode": "time", 15797 "name": null, 15798 "show": true, 15799 "values": [] 15800 }, 15801 "yaxes": [ 15802 { 15803 "format": "s", 15804 "label": null, 15805 "logBase": 1, 15806 "max": null, 15807 "min": null, 15808 "show": true 15809 }, 15810 { 15811 "format": "none", 15812 "label": null, 15813 "logBase": 1, 15814 "max": null, 15815 "min": null, 15816 "show": true 15817 } 15818 ], 15819 "yaxis": { 15820 "align": false, 15821 "alignLevel": null 15822 } 15823 }, 15824 { 15825 "cards": { 15826 "cardPadding": 0, 15827 "cardRound": 0 15828 }, 15829 "color": { 15830 "cardColor": "#b4ff00", 15831 "colorScale": "sqrt", 15832 "colorScheme": "interpolateSpectral", 15833 "exponent": 0.5, 15834 "max": null, 15835 "min": 1, 15836 "mode": "spectrum" 15837 }, 15838 "dataFormat": "tsbuckets", 15839 "datasource": "${DS_TEST-CLUSTER}", 15840 "description": "", 15841 "fieldConfig": { 15842 "defaults": {}, 15843 "overrides": [] 15844 }, 15845 "gridPos": { 15846 "h": 7, 15847 "w": 12, 15848 "x": 0, 15849 "y": 27 15850 }, 15851 "heatmap": {}, 15852 "hideZeroBuckets": true, 15853 "highlightCards": true, 15854 "id": 256, 15855 "legend": { 15856 "show": true 15857 }, 15858 "pluginVersion": "6.1.6", 15859 "reverseYBuckets": false, 15860 "targets": [ 15861 { 15862 "exemplar": true, 15863 "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le)", 15864 "format": "heatmap", 15865 "instant": false, 15866 "interval": "", 15867 "intervalFactor": 2, 15868 "legendFormat": "{{le}}", 15869 "queryType": "randomWalk", 15870 "refId": "A" 15871 } 15872 ], 15873 "timeFrom": null, 15874 "timeShift": null, 15875 "title": "EtcdWorker exec etcd txn duration", 15876 "tooltip": { 15877 "show": true, 15878 "showHistogram": true 15879 }, 15880 "tooltipDecimals": 1, 15881 "type": "heatmap", 15882 "xAxis": { 15883 "show": true 15884 }, 15885 "xBucketNumber": null, 15886 "xBucketSize": null, 15887 "yAxis": { 15888 "decimals": 1, 15889 "format": "s", 15890 "logBase": 1, 15891 "max": null, 15892 "min": null, 15893 "show": true, 15894 "splitFactor": null 15895 }, 15896 "yBucketBound": "upper", 15897 "yBucketNumber": null, 15898 "yBucketSize": null 15899 }, 15900 { 15901 "aliasColors": {}, 15902 "bars": false, 15903 "dashLength": 10, 15904 "dashes": false, 15905 "datasource": "${DS_TEST-CLUSTER}", 15906 "description": "", 15907 "fieldConfig": { 15908 "defaults": { 15909 "unit": "s" 15910 }, 15911 "overrides": [] 15912 }, 15913 "fill": 1, 15914 "fillGradient": 0, 15915 "gridPos": { 15916 "h": 7, 15917 "w": 12, 15918 "x": 12, 15919 "y": 27 15920 }, 15921 "hiddenSeries": false, 15922 "id": 258, 15923 "legend": { 15924 "avg": false, 15925 "current": false, 15926 "max": false, 15927 "min": false, 15928 "show": true, 15929 "total": false, 15930 "values": false 15931 }, 15932 "lines": true, 15933 "linewidth": 1, 15934 "nullPointMode": "null", 15935 "options": { 15936 "alertThreshold": true 15937 }, 15938 "percentage": false, 15939 "pluginVersion": "7.5.11", 15940 "pointradius": 2, 15941 "points": false, 15942 "renderer": "flot", 15943 "seriesOverrides": [], 15944 "spaceLength": 10, 15945 "stack": false, 15946 "steppedLine": false, 15947 "targets": [ 15948 { 15949 "exemplar": true, 15950 "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", 15951 "format": "time_series", 15952 "interval": "", 15953 "intervalFactor": 1, 15954 "legendFormat": "{{instance}}-p95", 15955 "queryType": "randomWalk", 15956 "refId": "A" 15957 }, 15958 { 15959 "exemplar": true, 15960 "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_exec_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", 15961 "format": "time_series", 15962 "hide": false, 15963 "interval": "", 15964 "intervalFactor": 1, 15965 "legendFormat": "{{instance}}-p99", 15966 "refId": "B" 15967 } 15968 ], 15969 "thresholds": [], 15970 "timeFrom": null, 15971 "timeRegions": [], 15972 "timeShift": null, 15973 "title": "EtcdWorker exec etcd txn duration percentile", 15974 "tooltip": { 15975 "shared": true, 15976 "sort": 0, 15977 "value_type": "individual" 15978 }, 15979 "type": "graph", 15980 "xaxis": { 15981 "buckets": null, 15982 "mode": "time", 15983 "name": null, 15984 "show": true, 15985 "values": [] 15986 }, 15987 "yaxes": [ 15988 { 15989 "format": "s", 15990 "label": null, 15991 "logBase": 2, 15992 "max": null, 15993 "min": null, 15994 "show": true 15995 }, 15996 { 15997 "format": "short", 15998 "label": null, 15999 "logBase": 1, 16000 "max": null, 16001 "min": null, 16002 "show": true 16003 } 16004 ], 16005 "yaxis": { 16006 "align": false, 16007 "alignLevel": null 16008 } 16009 }, 16010 { 16011 "cards": { 16012 "cardPadding": null, 16013 "cardRound": null 16014 }, 16015 "color": { 16016 "cardColor": "#b4ff00", 16017 "colorScale": "sqrt", 16018 "colorScheme": "interpolatePurples", 16019 "exponent": 0.5, 16020 "min": 0, 16021 "mode": "spectrum" 16022 }, 16023 "dataFormat": "tsbuckets", 16024 "datasource": "${DS_TEST-CLUSTER}", 16025 "description": "", 16026 "fieldConfig": { 16027 "defaults": {}, 16028 "overrides": [] 16029 }, 16030 "gridPos": { 16031 "h": 7, 16032 "w": 12, 16033 "x": 0, 16034 "y": 34 16035 }, 16036 "heatmap": {}, 16037 "hideZeroBuckets": true, 16038 "highlightCards": true, 16039 "id": 254, 16040 "legend": { 16041 "show": true 16042 }, 16043 "pluginVersion": "6.1.6", 16044 "reverseYBuckets": false, 16045 "targets": [ 16046 { 16047 "exemplar": true, 16048 "expr": "sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le)", 16049 "format": "heatmap", 16050 "instant": false, 16051 "interval": "", 16052 "intervalFactor": 1, 16053 "legendFormat": "{{le}}", 16054 "queryType": "randomWalk", 16055 "refId": "A" 16056 } 16057 ], 16058 "title": "EtcdWorker txn size", 16059 "tooltip": { 16060 "show": true, 16061 "showHistogram": true 16062 }, 16063 "tooltipDecimals": null, 16064 "type": "heatmap", 16065 "xAxis": { 16066 "show": true 16067 }, 16068 "xBucketNumber": null, 16069 "xBucketSize": null, 16070 "yAxis": { 16071 "decimals": null, 16072 "format": "decbytes", 16073 "logBase": 1, 16074 "max": null, 16075 "min": null, 16076 "show": true, 16077 "splitFactor": null 16078 }, 16079 "yBucketBound": "auto", 16080 "yBucketNumber": null, 16081 "yBucketSize": null 16082 }, 16083 { 16084 "aliasColors": {}, 16085 "bars": false, 16086 "dashLength": 10, 16087 "dashes": false, 16088 "datasource": "${DS_TEST-CLUSTER}", 16089 "description": "", 16090 "fieldConfig": { 16091 "defaults": {}, 16092 "overrides": [] 16093 }, 16094 "fill": 1, 16095 "fillGradient": 0, 16096 "gridPos": { 16097 "h": 7, 16098 "w": 12, 16099 "x": 12, 16100 "y": 34 16101 }, 16102 "hiddenSeries": false, 16103 "id": 260, 16104 "legend": { 16105 "avg": false, 16106 "current": false, 16107 "max": false, 16108 "min": false, 16109 "show": true, 16110 "total": false, 16111 "values": false 16112 }, 16113 "lines": true, 16114 "linewidth": 1, 16115 "nullPointMode": "null", 16116 "options": { 16117 "alertThreshold": true 16118 }, 16119 "percentage": false, 16120 "pluginVersion": "7.5.11", 16121 "pointradius": 2, 16122 "points": false, 16123 "renderer": "flot", 16124 "seriesOverrides": [], 16125 "spaceLength": 10, 16126 "stack": false, 16127 "steppedLine": false, 16128 "targets": [ 16129 { 16130 "exemplar": true, 16131 "expr": "histogram_quantile(0.95, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", 16132 "interval": "", 16133 "legendFormat": "{{instance}}-p95", 16134 "queryType": "randomWalk", 16135 "refId": "A" 16136 }, 16137 { 16138 "exemplar": true, 16139 "expr": "histogram_quantile(0.99, sum(rate(ticdc_etcd_worker_etcd_txn_size_bytes_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$ticdc_instance\"}[1m])) by (le, instance))", 16140 "hide": false, 16141 "interval": "", 16142 "legendFormat": "{{instance}}-p99", 16143 "refId": "B" 16144 } 16145 ], 16146 "thresholds": [], 16147 "timeFrom": null, 16148 "timeRegions": [], 16149 "timeShift": null, 16150 "title": "EtcdWorker txn size percentile", 16151 "tooltip": { 16152 "shared": true, 16153 "sort": 0, 16154 "value_type": "individual" 16155 }, 16156 "type": "graph", 16157 "xaxis": { 16158 "buckets": null, 16159 "mode": "time", 16160 "name": null, 16161 "show": true, 16162 "values": [] 16163 }, 16164 "yaxes": [ 16165 { 16166 "format": "bytes", 16167 "label": null, 16168 "logBase": 1, 16169 "max": null, 16170 "min": null, 16171 "show": true 16172 }, 16173 { 16174 "format": "short", 16175 "label": null, 16176 "logBase": 1, 16177 "max": null, 16178 "min": null, 16179 "show": true 16180 } 16181 ], 16182 "yaxis": { 16183 "align": false, 16184 "alignLevel": null 16185 } 16186 }, 16187 { 16188 "aliasColors": {}, 16189 "bars": false, 16190 "dashLength": 10, 16191 "dashes": false, 16192 "datasource": "${DS_TEST-CLUSTER}", 16193 "description": "The time consumed of writing WAL into the persistent storage in .99", 16194 "editable": true, 16195 "error": false, 16196 "fieldConfig": { 16197 "defaults": {}, 16198 "overrides": [] 16199 }, 16200 "fill": 1, 16201 "fillGradient": 0, 16202 "grid": {}, 16203 "gridPos": { 16204 "h": 7, 16205 "w": 12, 16206 "x": 0, 16207 "y": 41 16208 }, 16209 "hiddenSeries": false, 16210 "id": 291, 16211 "legend": { 16212 "alignAsTable": true, 16213 "avg": false, 16214 "current": true, 16215 "hideEmpty": true, 16216 "hideZero": true, 16217 "max": true, 16218 "min": false, 16219 "rightSide": true, 16220 "show": true, 16221 "sideWidth": 300, 16222 "total": false, 16223 "values": true 16224 }, 16225 "lines": true, 16226 "linewidth": 1, 16227 "links": [], 16228 "nullPointMode": "null", 16229 "options": { 16230 "alertThreshold": true 16231 }, 16232 "paceLength": 10, 16233 "percentage": false, 16234 "pluginVersion": "7.5.11", 16235 "pointradius": 5, 16236 "points": false, 16237 "renderer": "flot", 16238 "seriesOverrides": [], 16239 "spaceLength": 10, 16240 "stack": false, 16241 "steppedLine": false, 16242 "targets": [ 16243 { 16244 "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (instance, le))", 16245 "format": "time_series", 16246 "intervalFactor": 2, 16247 "legendFormat": "{{instance}}", 16248 "metric": "", 16249 "refId": "A", 16250 "step": 4 16251 } 16252 ], 16253 "thresholds": [], 16254 "timeFrom": null, 16255 "timeRegions": [], 16256 "timeShift": null, 16257 "title": "Etcd 99% WAL fsync duration", 16258 "tooltip": { 16259 "msResolution": false, 16260 "shared": true, 16261 "sort": 0, 16262 "value_type": "cumulative" 16263 }, 16264 "type": "graph", 16265 "xaxis": { 16266 "buckets": null, 16267 "mode": "time", 16268 "name": null, 16269 "show": true, 16270 "values": [] 16271 }, 16272 "yaxes": [ 16273 { 16274 "format": "s", 16275 "label": null, 16276 "logBase": 1, 16277 "max": null, 16278 "min": null, 16279 "show": true 16280 }, 16281 { 16282 "format": "short", 16283 "label": null, 16284 "logBase": 1, 16285 "max": null, 16286 "min": null, 16287 "show": true 16288 } 16289 ], 16290 "yaxis": { 16291 "align": false, 16292 "alignLevel": null 16293 } 16294 }, 16295 { 16296 "aliasColors": {}, 16297 "bars": false, 16298 "dashLength": 10, 16299 "dashes": false, 16300 "datasource": "${DS_TEST-CLUSTER}", 16301 "description": "The time consumed of handling etcd transactions in .99", 16302 "editable": true, 16303 "error": false, 16304 "fieldConfig": { 16305 "defaults": {}, 16306 "overrides": [] 16307 }, 16308 "fill": 1, 16309 "fillGradient": 0, 16310 "grid": {}, 16311 "gridPos": { 16312 "h": 7, 16313 "w": 12, 16314 "x": 12, 16315 "y": 41 16316 }, 16317 "hiddenSeries": false, 16318 "id": 290, 16319 "legend": { 16320 "alignAsTable": true, 16321 "avg": false, 16322 "current": true, 16323 "max": true, 16324 "min": false, 16325 "rightSide": true, 16326 "show": true, 16327 "sideWidth": 300, 16328 "total": false, 16329 "values": true 16330 }, 16331 "lines": true, 16332 "linewidth": 1, 16333 "links": [], 16334 "nullPointMode": "null", 16335 "options": { 16336 "alertThreshold": true 16337 }, 16338 "paceLength": 10, 16339 "percentage": false, 16340 "pluginVersion": "7.5.11", 16341 "pointradius": 5, 16342 "points": false, 16343 "renderer": "flot", 16344 "seriesOverrides": [], 16345 "spaceLength": 10, 16346 "stack": false, 16347 "steppedLine": false, 16348 "targets": [ 16349 { 16350 "expr": "histogram_quantile(0.99, sum(rate(pd_txn_handle_txns_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (instance, result, le))", 16351 "format": "time_series", 16352 "intervalFactor": 2, 16353 "legendFormat": "{{instance}} {{result}}", 16354 "refId": "A", 16355 "step": 4 16356 } 16357 ], 16358 "thresholds": [], 16359 "timeFrom": null, 16360 "timeRegions": [], 16361 "timeShift": null, 16362 "title": "Etcd 99% Handle transactions duration", 16363 "tooltip": { 16364 "msResolution": false, 16365 "shared": true, 16366 "sort": 0, 16367 "value_type": "cumulative" 16368 }, 16369 "type": "graph", 16370 "xaxis": { 16371 "buckets": null, 16372 "mode": "time", 16373 "name": null, 16374 "show": true, 16375 "values": [] 16376 }, 16377 "yaxes": [ 16378 { 16379 "format": "s", 16380 "label": null, 16381 "logBase": 1, 16382 "max": null, 16383 "min": null, 16384 "show": true 16385 }, 16386 { 16387 "format": "short", 16388 "label": null, 16389 "logBase": 1, 16390 "max": null, 16391 "min": null, 16392 "show": true 16393 } 16394 ], 16395 "yaxis": { 16396 "align": false, 16397 "alignLevel": null 16398 } 16399 } 16400 ], 16401 "title": "Etcd", 16402 "type": "row" 16403 }, 16404 { 16405 "collapsed": true, 16406 "datasource": null, 16407 "gridPos": { 16408 "h": 1, 16409 "w": 24, 16410 "x": 0, 16411 "y": 13 16412 }, 16413 "id": 58, 16414 "panels": [ 16415 { 16416 "aliasColors": {}, 16417 "bars": false, 16418 "dashLength": 10, 16419 "dashes": false, 16420 "datasource": "${DS_TEST-CLUSTER}", 16421 "decimals": 1, 16422 "description": "CPU usage of TiKV CDC component", 16423 "fieldConfig": { 16424 "defaults": {}, 16425 "overrides": [] 16426 }, 16427 "fill": 1, 16428 "fillGradient": 0, 16429 "gridPos": { 16430 "h": 7, 16431 "w": 12, 16432 "x": 0, 16433 "y": 166 16434 }, 16435 "hiddenSeries": false, 16436 "id": 60, 16437 "legend": { 16438 "alignAsTable": true, 16439 "avg": false, 16440 "current": true, 16441 "max": true, 16442 "min": false, 16443 "rightSide": true, 16444 "show": true, 16445 "sideWidth": null, 16446 "sort": "current", 16447 "sortDesc": true, 16448 "total": false, 16449 "values": true 16450 }, 16451 "lines": true, 16452 "linewidth": 1, 16453 "links": [], 16454 "nullPointMode": "null", 16455 "options": { 16456 "alertThreshold": true 16457 }, 16458 "paceLength": 10, 16459 "percentage": false, 16460 "pluginVersion": "7.5.11", 16461 "pointradius": 5, 16462 "points": false, 16463 "renderer": "flot", 16464 "seriesOverrides": [ 16465 { 16466 "alias": "/.*workers/", 16467 "transform": "negative-Y", 16468 "yaxis": 2 16469 }, 16470 { 16471 "alias": "/.*tso/", 16472 "fill": 0, 16473 "pointradius": 1, 16474 "points": true 16475 } 16476 ], 16477 "spaceLength": 10, 16478 "stack": false, 16479 "steppedLine": false, 16480 "targets": [ 16481 { 16482 "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"cdc_.*|cdc\"}[1m])) by (instance)", 16483 "format": "time_series", 16484 "intervalFactor": 2, 16485 "legendFormat": "{{instance}}-endpoint", 16486 "refId": "A", 16487 "step": 4 16488 }, 16489 { 16490 "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"cdcwkr.*\"}[1m])) by (instance)", 16491 "format": "time_series", 16492 "intervalFactor": 2, 16493 "legendFormat": "{{instance}}-workers", 16494 "refId": "B", 16495 "step": 4 16496 }, 16497 { 16498 "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"tso\"}[1m])) by (instance)", 16499 "format": "time_series", 16500 "hide": false, 16501 "intervalFactor": 2, 16502 "legendFormat": "{{instance}}-tso", 16503 "refId": "C", 16504 "step": 4 16505 } 16506 ], 16507 "thresholds": [], 16508 "timeFrom": null, 16509 "timeRegions": [], 16510 "timeShift": null, 16511 "title": "CDC CPU", 16512 "tooltip": { 16513 "shared": true, 16514 "sort": 0, 16515 "value_type": "individual" 16516 }, 16517 "type": "graph", 16518 "xaxis": { 16519 "buckets": null, 16520 "mode": "time", 16521 "name": null, 16522 "show": true, 16523 "values": [] 16524 }, 16525 "yaxes": [ 16526 { 16527 "format": "percentunit", 16528 "label": null, 16529 "logBase": 1, 16530 "max": null, 16531 "min": null, 16532 "show": true 16533 }, 16534 { 16535 "format": "percent", 16536 "label": null, 16537 "logBase": 1, 16538 "max": null, 16539 "min": null, 16540 "show": true 16541 } 16542 ], 16543 "yaxis": { 16544 "align": false, 16545 "alignLevel": null 16546 } 16547 }, 16548 { 16549 "aliasColors": {}, 16550 "bars": true, 16551 "dashLength": 10, 16552 "dashes": false, 16553 "datasource": "${DS_TEST-CLUSTER}", 16554 "decimals": 1, 16555 "description": "Outbound network traffic of TiKV CDC component", 16556 "editable": true, 16557 "error": false, 16558 "fieldConfig": { 16559 "defaults": {}, 16560 "overrides": [] 16561 }, 16562 "fill": 0, 16563 "fillGradient": 0, 16564 "grid": {}, 16565 "gridPos": { 16566 "h": 7, 16567 "w": 12, 16568 "x": 12, 16569 "y": 166 16570 }, 16571 "hiddenSeries": false, 16572 "id": 74, 16573 "legend": { 16574 "alignAsTable": true, 16575 "avg": false, 16576 "current": true, 16577 "hideEmpty": false, 16578 "hideZero": false, 16579 "max": true, 16580 "min": false, 16581 "rightSide": true, 16582 "show": true, 16583 "sideWidth": null, 16584 "sort": "max", 16585 "sortDesc": true, 16586 "total": false, 16587 "values": true 16588 }, 16589 "lines": false, 16590 "linewidth": 1, 16591 "links": [], 16592 "nullPointMode": "null", 16593 "options": { 16594 "alertThreshold": true 16595 }, 16596 "paceLength": 10, 16597 "percentage": false, 16598 "pluginVersion": "7.5.11", 16599 "pointradius": 5, 16600 "points": false, 16601 "renderer": "flot", 16602 "seriesOverrides": [ 16603 { 16604 "alias": "/.*resolved_ts/", 16605 "bars": false, 16606 "fill": 1, 16607 "lines": true, 16608 "linewidth": 2, 16609 "transform": "negative-Y", 16610 "yaxis": 2 16611 } 16612 ], 16613 "spaceLength": 10, 16614 "stack": false, 16615 "steppedLine": false, 16616 "targets": [ 16617 { 16618 "expr": "sum(rate(tikv_cdc_grpc_message_sent_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[30s])) by (instance, type)", 16619 "format": "time_series", 16620 "hide": false, 16621 "intervalFactor": 2, 16622 "legendFormat": "{{instance}}-{{type}}", 16623 "refId": "A", 16624 "step": 10 16625 } 16626 ], 16627 "thresholds": [], 16628 "timeFrom": null, 16629 "timeRegions": [], 16630 "timeShift": null, 16631 "title": "CDC network traffic", 16632 "tooltip": { 16633 "msResolution": false, 16634 "shared": true, 16635 "sort": 0, 16636 "value_type": "individual" 16637 }, 16638 "type": "graph", 16639 "xaxis": { 16640 "buckets": null, 16641 "mode": "time", 16642 "name": null, 16643 "show": true, 16644 "values": [] 16645 }, 16646 "yaxes": [ 16647 { 16648 "format": "Bps", 16649 "label": null, 16650 "logBase": 1, 16651 "max": null, 16652 "min": null, 16653 "show": true 16654 }, 16655 { 16656 "format": "bytes", 16657 "label": null, 16658 "logBase": 1, 16659 "max": null, 16660 "min": null, 16661 "show": true 16662 } 16663 ], 16664 "yaxis": { 16665 "align": false, 16666 "alignLevel": 0 16667 } 16668 }, 16669 { 16670 "aliasColors": {}, 16671 "bars": false, 16672 "dashLength": 10, 16673 "dashes": false, 16674 "datasource": "${DS_TEST-CLUSTER}", 16675 "decimals": 1, 16676 "description": "The count of different kinds of gRPC message", 16677 "fieldConfig": { 16678 "defaults": {}, 16679 "overrides": [] 16680 }, 16681 "fill": 1, 16682 "fillGradient": 0, 16683 "gridPos": { 16684 "h": 7, 16685 "w": 12, 16686 "x": 0, 16687 "y": 173 16688 }, 16689 "hiddenSeries": false, 16690 "id": 147, 16691 "legend": { 16692 "alignAsTable": true, 16693 "avg": false, 16694 "current": true, 16695 "hideEmpty": true, 16696 "hideZero": true, 16697 "max": true, 16698 "min": false, 16699 "rightSide": true, 16700 "show": true, 16701 "sideWidth": null, 16702 "sort": "current", 16703 "sortDesc": true, 16704 "total": false, 16705 "values": true 16706 }, 16707 "lines": true, 16708 "linewidth": 1, 16709 "links": [], 16710 "nullPointMode": "null", 16711 "options": { 16712 "alertThreshold": true 16713 }, 16714 "percentage": false, 16715 "pluginVersion": "7.5.11", 16716 "pointradius": 5, 16717 "points": false, 16718 "renderer": "flot", 16719 "seriesOverrides": [], 16720 "spaceLength": 10, 16721 "stack": false, 16722 "steppedLine": false, 16723 "targets": [ 16724 { 16725 "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type!=\"kv_gc\"}[1m])) by (type)", 16726 "format": "time_series", 16727 "intervalFactor": 2, 16728 "legendFormat": "{{type}}", 16729 "metric": "tikv_grpc_msg_duration_seconds_bucket", 16730 "refId": "A", 16731 "step": 10 16732 } 16733 ], 16734 "thresholds": [], 16735 "timeFrom": null, 16736 "timeRegions": [], 16737 "timeShift": null, 16738 "title": "gRPC message count", 16739 "tooltip": { 16740 "shared": true, 16741 "sort": 0, 16742 "value_type": "individual" 16743 }, 16744 "type": "graph", 16745 "xaxis": { 16746 "buckets": null, 16747 "mode": "time", 16748 "name": null, 16749 "show": true, 16750 "values": [] 16751 }, 16752 "yaxes": [ 16753 { 16754 "format": "ops", 16755 "label": null, 16756 "logBase": 1, 16757 "max": null, 16758 "min": null, 16759 "show": true 16760 }, 16761 { 16762 "format": "short", 16763 "label": null, 16764 "logBase": 1, 16765 "max": null, 16766 "min": null, 16767 "show": true 16768 } 16769 ], 16770 "yaxis": { 16771 "align": false, 16772 "alignLevel": null 16773 } 16774 }, 16775 { 16776 "aliasColors": {}, 16777 "bars": false, 16778 "dashLength": 10, 16779 "dashes": false, 16780 "datasource": "${DS_TEST-CLUSTER}", 16781 "decimals": 1, 16782 "description": "The memory usage per TiKV instance", 16783 "editable": true, 16784 "error": false, 16785 "fieldConfig": { 16786 "defaults": {}, 16787 "overrides": [] 16788 }, 16789 "fill": 0, 16790 "fillGradient": 0, 16791 "grid": {}, 16792 "gridPos": { 16793 "h": 7, 16794 "w": 12, 16795 "x": 12, 16796 "y": 173 16797 }, 16798 "hiddenSeries": false, 16799 "id": 194, 16800 "legend": { 16801 "alignAsTable": true, 16802 "avg": false, 16803 "current": true, 16804 "max": true, 16805 "min": false, 16806 "rightSide": true, 16807 "show": true, 16808 "sideWidth": null, 16809 "sort": "max", 16810 "sortDesc": true, 16811 "total": false, 16812 "values": true 16813 }, 16814 "lines": true, 16815 "linewidth": 1, 16816 "links": [], 16817 "nullPointMode": "null", 16818 "options": { 16819 "alertThreshold": true 16820 }, 16821 "paceLength": 10, 16822 "percentage": false, 16823 "pluginVersion": "7.5.11", 16824 "pointradius": 5, 16825 "points": false, 16826 "renderer": "flot", 16827 "seriesOverrides": [ 16828 { 16829 "alias": "/.*-cap-.*/", 16830 "yaxis": 2 16831 }, 16832 { 16833 "alias": "/.*tikv.*/", 16834 "pointradius": 1, 16835 "points": true, 16836 "yaxis": 2 16837 } 16838 ], 16839 "spaceLength": 10, 16840 "stack": false, 16841 "steppedLine": false, 16842 "targets": [ 16843 { 16844 "expr": "avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=~\"tikv.*\"}) by (instance)", 16845 "format": "time_series", 16846 "hide": true, 16847 "intervalFactor": 2, 16848 "legendFormat": "tikv-{{instance}}", 16849 "refId": "A", 16850 "step": 10 16851 }, 16852 { 16853 "expr": "avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=~\"cdc.*\"}) by (instance)", 16854 "format": "time_series", 16855 "hide": true, 16856 "intervalFactor": 2, 16857 "legendFormat": "cdc-{{instance}}", 16858 "refId": "B", 16859 "step": 10 16860 }, 16861 { 16862 "expr": "(avg(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=~\"tikv.*\"}) by (instance)) - (avg(tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", db=\"kv\"}) by(instance))", 16863 "format": "time_series", 16864 "hide": false, 16865 "intervalFactor": 2, 16866 "legendFormat": "tikv-{{instance}}", 16867 "refId": "C", 16868 "step": 10 16869 }, 16870 { 16871 "expr": "sum(tikv_cdc_sink_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", 16872 "format": "time_series", 16873 "hide": false, 16874 "intervalFactor": 2, 16875 "legendFormat": "sink-{{instance}}", 16876 "refId": "D", 16877 "step": 10 16878 }, 16879 { 16880 "expr": "sum(tikv_cdc_old_value_cache_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", 16881 "format": "time_series", 16882 "hide": false, 16883 "intervalFactor": 2, 16884 "legendFormat": "old-value-{{instance}}", 16885 "refId": "E", 16886 "step": 10 16887 }, 16888 { 16889 "expr": "sum(tikv_cdc_sink_memory_capacity{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", 16890 "format": "time_series", 16891 "hide": true, 16892 "intervalFactor": 2, 16893 "legendFormat": "sink-cap-{{instance}}", 16894 "refId": "F", 16895 "step": 10 16896 }, 16897 { 16898 "expr": "sum(tikv_cdc_old_value_cache_memory_quota{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", 16899 "format": "time_series", 16900 "hide": true, 16901 "intervalFactor": 2, 16902 "legendFormat": "old-value-cap-{{instance}}", 16903 "refId": "G", 16904 "step": 10 16905 } 16906 ], 16907 "thresholds": [], 16908 "timeFrom": null, 16909 "timeRegions": [], 16910 "timeShift": null, 16911 "title": "CDC memory", 16912 "tooltip": { 16913 "msResolution": false, 16914 "shared": true, 16915 "sort": 0, 16916 "value_type": "individual" 16917 }, 16918 "type": "graph", 16919 "xaxis": { 16920 "buckets": null, 16921 "mode": "time", 16922 "name": null, 16923 "show": true, 16924 "values": [] 16925 }, 16926 "yaxes": [ 16927 { 16928 "format": "bytes", 16929 "label": null, 16930 "logBase": 1, 16931 "max": null, 16932 "min": "0", 16933 "show": true 16934 }, 16935 { 16936 "format": "bytes", 16937 "label": null, 16938 "logBase": 1, 16939 "max": null, 16940 "min": null, 16941 "show": true 16942 } 16943 ], 16944 "yaxis": { 16945 "align": false, 16946 "alignLevel": null 16947 } 16948 }, 16949 { 16950 "aliasColors": {}, 16951 "bars": false, 16952 "dashLength": 10, 16953 "dashes": false, 16954 "datasource": "${DS_TEST-CLUSTER}", 16955 "decimals": 1, 16956 "description": "The min resolved ts of each TiKV", 16957 "editable": true, 16958 "error": false, 16959 "fieldConfig": { 16960 "defaults": {}, 16961 "overrides": [] 16962 }, 16963 "fill": 0, 16964 "fillGradient": 0, 16965 "grid": {}, 16966 "gridPos": { 16967 "h": 7, 16968 "w": 7, 16969 "x": 0, 16970 "y": 180 16971 }, 16972 "hiddenSeries": false, 16973 "id": 152, 16974 "legend": { 16975 "alignAsTable": true, 16976 "avg": false, 16977 "current": true, 16978 "max": false, 16979 "min": false, 16980 "rightSide": false, 16981 "show": true, 16982 "sideWidth": null, 16983 "sort": null, 16984 "sortDesc": null, 16985 "total": false, 16986 "values": true 16987 }, 16988 "lines": true, 16989 "linewidth": 1, 16990 "links": [], 16991 "nullPointMode": "null", 16992 "options": { 16993 "alertThreshold": true 16994 }, 16995 "paceLength": 10, 16996 "percentage": false, 16997 "pluginVersion": "7.5.11", 16998 "pointradius": 1, 16999 "points": false, 17000 "renderer": "flot", 17001 "seriesOverrides": [ 17002 { 17003 "alias": "/.*-ts/", 17004 "lines": true, 17005 "linewidth": 3, 17006 "points": false, 17007 "yaxis": 2 17008 }, 17009 { 17010 "alias": "/.*-lag/", 17011 "bars": true, 17012 "fill": 1 17013 } 17014 ], 17015 "spaceLength": 10, 17016 "stack": false, 17017 "steppedLine": false, 17018 "targets": [ 17019 { 17020 "expr": "scalar(max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}))/1000 - avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000) by (instance) > 0", 17021 "format": "time_series", 17022 "hide": false, 17023 "instant": false, 17024 "intervalFactor": 2, 17025 "legendFormat": "{{instance}}-min-resolved-lag", 17026 "refId": "A", 17027 "step": 10 17028 }, 17029 { 17030 "expr": "max(pd_cluster_tso{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", 17031 "format": "time_series", 17032 "hide": false, 17033 "intervalFactor": 2, 17034 "legendFormat": "current-ts", 17035 "refId": "B", 17036 "step": 10 17037 }, 17038 { 17039 "expr": "avg(tikv_cdc_min_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", 17040 "format": "time_series", 17041 "hide": false, 17042 "intervalFactor": 2, 17043 "legendFormat": "{{instance}}-min-resolved-ts", 17044 "refId": "C", 17045 "step": 10 17046 } 17047 ], 17048 "thresholds": [], 17049 "timeFrom": null, 17050 "timeRegions": [], 17051 "timeShift": null, 17052 "title": "Min resolved ts", 17053 "tooltip": { 17054 "msResolution": false, 17055 "shared": true, 17056 "sort": 0, 17057 "value_type": "individual" 17058 }, 17059 "type": "graph", 17060 "xaxis": { 17061 "buckets": null, 17062 "mode": "time", 17063 "name": null, 17064 "show": true, 17065 "values": [] 17066 }, 17067 "yaxes": [ 17068 { 17069 "format": "s", 17070 "label": null, 17071 "logBase": 1, 17072 "max": null, 17073 "min": "0", 17074 "show": true 17075 }, 17076 { 17077 "format": "none", 17078 "label": null, 17079 "logBase": 1, 17080 "max": null, 17081 "min": null, 17082 "show": true 17083 } 17084 ], 17085 "yaxis": { 17086 "align": false, 17087 "alignLevel": null 17088 } 17089 }, 17090 { 17091 "aliasColors": {}, 17092 "bars": false, 17093 "dashLength": 10, 17094 "dashes": false, 17095 "datasource": "${DS_TEST-CLUSTER}", 17096 "decimals": 1, 17097 "description": "The ID of the min resolved region of each TiKV", 17098 "editable": true, 17099 "error": false, 17100 "fieldConfig": { 17101 "defaults": {}, 17102 "overrides": [] 17103 }, 17104 "fill": 0, 17105 "fillGradient": 0, 17106 "grid": {}, 17107 "gridPos": { 17108 "h": 7, 17109 "w": 5, 17110 "x": 7, 17111 "y": 180 17112 }, 17113 "hiddenSeries": false, 17114 "id": 153, 17115 "legend": { 17116 "alignAsTable": true, 17117 "avg": false, 17118 "current": true, 17119 "max": true, 17120 "min": false, 17121 "rightSide": false, 17122 "show": true, 17123 "sideWidth": null, 17124 "sort": null, 17125 "sortDesc": null, 17126 "total": false, 17127 "values": true 17128 }, 17129 "lines": false, 17130 "linewidth": 1, 17131 "links": [], 17132 "nullPointMode": "null", 17133 "options": { 17134 "alertThreshold": true 17135 }, 17136 "paceLength": 10, 17137 "percentage": false, 17138 "pluginVersion": "7.5.11", 17139 "pointradius": 1, 17140 "points": true, 17141 "renderer": "flot", 17142 "seriesOverrides": [], 17143 "spaceLength": 10, 17144 "stack": false, 17145 "steppedLine": false, 17146 "targets": [ 17147 { 17148 "exemplar": true, 17149 "expr": "avg(tikv_cdc_min_resolved_ts_region{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", 17150 "format": "time_series", 17151 "hide": false, 17152 "interval": "", 17153 "intervalFactor": 2, 17154 "legendFormat": "{{instance}}-min-resolved-region", 17155 "refId": "A", 17156 "step": 10 17157 } 17158 ], 17159 "thresholds": [], 17160 "timeFrom": null, 17161 "timeRegions": [], 17162 "timeShift": null, 17163 "title": "Min resolved Region", 17164 "tooltip": { 17165 "msResolution": false, 17166 "shared": true, 17167 "sort": 0, 17168 "value_type": "individual" 17169 }, 17170 "type": "graph", 17171 "xaxis": { 17172 "buckets": null, 17173 "mode": "time", 17174 "name": null, 17175 "show": true, 17176 "values": [] 17177 }, 17178 "yaxes": [ 17179 { 17180 "format": "none", 17181 "label": null, 17182 "logBase": 1, 17183 "max": null, 17184 "min": "0", 17185 "show": true 17186 }, 17187 { 17188 "format": "none", 17189 "label": null, 17190 "logBase": 1, 17191 "max": null, 17192 "min": null, 17193 "show": false 17194 } 17195 ], 17196 "yaxis": { 17197 "align": false, 17198 "alignLevel": null 17199 } 17200 }, 17201 { 17202 "aliasColors": {}, 17203 "bars": false, 17204 "dashLength": 10, 17205 "dashes": false, 17206 "datasource": "${DS_TEST-CLUSTER}", 17207 "decimals": 1, 17208 "description": "", 17209 "fieldConfig": { 17210 "defaults": {}, 17211 "overrides": [] 17212 }, 17213 "fill": 1, 17214 "fillGradient": 0, 17215 "gridPos": { 17216 "h": 7, 17217 "w": 12, 17218 "x": 12, 17219 "y": 180 17220 }, 17221 "hiddenSeries": false, 17222 "id": 70, 17223 "legend": { 17224 "alignAsTable": true, 17225 "avg": false, 17226 "current": true, 17227 "hideEmpty": false, 17228 "hideZero": true, 17229 "max": true, 17230 "min": false, 17231 "rightSide": true, 17232 "show": true, 17233 "sideWidth": null, 17234 "sort": null, 17235 "sortDesc": null, 17236 "total": false, 17237 "values": true 17238 }, 17239 "lines": true, 17240 "linewidth": 1, 17241 "links": [], 17242 "nullPointMode": "null", 17243 "options": { 17244 "alertThreshold": true 17245 }, 17246 "paceLength": 10, 17247 "percentage": false, 17248 "pluginVersion": "7.5.11", 17249 "pointradius": 5, 17250 "points": false, 17251 "renderer": "flot", 17252 "seriesOverrides": [], 17253 "spaceLength": 10, 17254 "stack": false, 17255 "steppedLine": false, 17256 "targets": [ 17257 { 17258 "expr": "histogram_quantile(0.99999, sum(rate(tikv_cdc_resolved_ts_gap_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance))", 17259 "format": "time_series", 17260 "intervalFactor": 1, 17261 "legendFormat": "{{instance}}-p9999", 17262 "refId": "A" 17263 } 17264 ], 17265 "thresholds": [], 17266 "timeFrom": null, 17267 "timeRegions": [], 17268 "timeShift": null, 17269 "title": "Resolved ts lag duration percentile", 17270 "tooltip": { 17271 "shared": true, 17272 "sort": 0, 17273 "value_type": "individual" 17274 }, 17275 "type": "graph", 17276 "xaxis": { 17277 "buckets": null, 17278 "mode": "time", 17279 "name": null, 17280 "show": true, 17281 "values": [] 17282 }, 17283 "yaxes": [ 17284 { 17285 "format": "s", 17286 "label": null, 17287 "logBase": 10, 17288 "max": null, 17289 "min": null, 17290 "show": true 17291 }, 17292 { 17293 "format": "short", 17294 "label": null, 17295 "logBase": 1, 17296 "max": null, 17297 "min": null, 17298 "show": true 17299 } 17300 ], 17301 "yaxis": { 17302 "align": false, 17303 "alignLevel": null 17304 } 17305 }, 17306 { 17307 "cards": { 17308 "cardPadding": 0, 17309 "cardRound": 0 17310 }, 17311 "color": { 17312 "cardColor": "#FF9830", 17313 "colorScale": "linear", 17314 "colorScheme": "interpolateSpectral", 17315 "exponent": 0.5, 17316 "min": 0, 17317 "mode": "spectrum" 17318 }, 17319 "dataFormat": "tsbuckets", 17320 "datasource": "${DS_TEST-CLUSTER}", 17321 "description": "The time consumed to CDC incremental scan", 17322 "fieldConfig": { 17323 "defaults": {}, 17324 "overrides": [] 17325 }, 17326 "gridPos": { 17327 "h": 7, 17328 "w": 12, 17329 "x": 0, 17330 "y": 187 17331 }, 17332 "heatmap": {}, 17333 "hideZeroBuckets": true, 17334 "highlightCards": true, 17335 "id": 68, 17336 "legend": { 17337 "alignAsTable": true, 17338 "avg": false, 17339 "current": true, 17340 "max": true, 17341 "min": false, 17342 "rightSide": true, 17343 "show": true, 17344 "sort": "current", 17345 "sortDesc": true, 17346 "total": false, 17347 "values": true 17348 }, 17349 "links": [], 17350 "maxPerRow": 3, 17351 "repeat": null, 17352 "repeatDirection": "h", 17353 "reverseYBuckets": false, 17354 "targets": [ 17355 { 17356 "expr": "sum(rate(tikv_cdc_scan_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)", 17357 "format": "heatmap", 17358 "instant": false, 17359 "intervalFactor": 2, 17360 "legendFormat": "{{le}}", 17361 "refId": "A" 17362 } 17363 ], 17364 "title": "Initial scan duration", 17365 "tooltip": { 17366 "show": true, 17367 "showHistogram": true 17368 }, 17369 "tooltipDecimals": 1, 17370 "type": "heatmap", 17371 "xAxis": { 17372 "show": true 17373 }, 17374 "xBucketNumber": null, 17375 "xBucketSize": null, 17376 "yAxis": { 17377 "decimals": 1, 17378 "format": "s", 17379 "logBase": 1, 17380 "max": null, 17381 "min": null, 17382 "show": true, 17383 "splitFactor": null 17384 }, 17385 "yBucketBound": "upper", 17386 "yBucketNumber": null, 17387 "yBucketSize": null 17388 }, 17389 { 17390 "aliasColors": {}, 17391 "bars": false, 17392 "dashLength": 10, 17393 "dashes": false, 17394 "datasource": "${DS_TEST-CLUSTER}", 17395 "decimals": 1, 17396 "description": "", 17397 "fieldConfig": { 17398 "defaults": {}, 17399 "overrides": [] 17400 }, 17401 "fill": 1, 17402 "fillGradient": 0, 17403 "gridPos": { 17404 "h": 7, 17405 "w": 6, 17406 "x": 12, 17407 "y": 187 17408 }, 17409 "hiddenSeries": false, 17410 "id": 72, 17411 "legend": { 17412 "alignAsTable": true, 17413 "avg": false, 17414 "current": true, 17415 "hideEmpty": false, 17416 "hideZero": true, 17417 "max": true, 17418 "min": false, 17419 "rightSide": false, 17420 "show": true, 17421 "sideWidth": null, 17422 "sort": "current", 17423 "sortDesc": true, 17424 "total": false, 17425 "values": true 17426 }, 17427 "lines": true, 17428 "linewidth": 1, 17429 "links": [], 17430 "nullPointMode": "null", 17431 "options": { 17432 "alertThreshold": true 17433 }, 17434 "paceLength": 10, 17435 "percentage": false, 17436 "pluginVersion": "7.5.11", 17437 "pointradius": 5, 17438 "points": false, 17439 "renderer": "flot", 17440 "seriesOverrides": [], 17441 "spaceLength": 10, 17442 "stack": false, 17443 "steppedLine": false, 17444 "targets": [ 17445 { 17446 "expr": "histogram_quantile(0.9999, sum(rate(tikv_cdc_scan_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance))", 17447 "format": "time_series", 17448 "intervalFactor": 1, 17449 "legendFormat": "{{instance}}-p9999", 17450 "refId": "A" 17451 } 17452 ], 17453 "thresholds": [], 17454 "timeFrom": null, 17455 "timeRegions": [], 17456 "timeShift": null, 17457 "title": "Initial scan duration percentile", 17458 "tooltip": { 17459 "shared": true, 17460 "sort": 0, 17461 "value_type": "individual" 17462 }, 17463 "type": "graph", 17464 "xaxis": { 17465 "buckets": null, 17466 "mode": "time", 17467 "name": null, 17468 "show": true, 17469 "values": [] 17470 }, 17471 "yaxes": [ 17472 { 17473 "format": "s", 17474 "label": null, 17475 "logBase": 1, 17476 "max": null, 17477 "min": null, 17478 "show": true 17479 }, 17480 { 17481 "format": "short", 17482 "label": null, 17483 "logBase": 1, 17484 "max": null, 17485 "min": null, 17486 "show": false 17487 } 17488 ], 17489 "yaxis": { 17490 "align": false, 17491 "alignLevel": null 17492 } 17493 }, 17494 { 17495 "aliasColors": {}, 17496 "bars": false, 17497 "dashLength": 10, 17498 "dashes": false, 17499 "datasource": "${DS_TEST-CLUSTER}", 17500 "decimals": 1, 17501 "description": "The number of incremental scan task in different status.", 17502 "fieldConfig": { 17503 "defaults": {}, 17504 "overrides": [] 17505 }, 17506 "fill": 1, 17507 "fillGradient": 0, 17508 "gridPos": { 17509 "h": 7, 17510 "w": 6, 17511 "x": 18, 17512 "y": 187 17513 }, 17514 "hiddenSeries": false, 17515 "id": 140, 17516 "legend": { 17517 "alignAsTable": true, 17518 "avg": false, 17519 "current": true, 17520 "hideEmpty": false, 17521 "hideZero": true, 17522 "max": true, 17523 "min": false, 17524 "rightSide": false, 17525 "show": true, 17526 "sideWidth": null, 17527 "sort": "current", 17528 "sortDesc": true, 17529 "total": false, 17530 "values": true 17531 }, 17532 "lines": true, 17533 "linewidth": 1, 17534 "links": [], 17535 "nullPointMode": "null", 17536 "options": { 17537 "alertThreshold": true 17538 }, 17539 "paceLength": 10, 17540 "percentage": false, 17541 "pluginVersion": "7.5.11", 17542 "pointradius": 5, 17543 "points": false, 17544 "renderer": "flot", 17545 "seriesOverrides": [ 17546 { 17547 "alias": "/.*ongoing/", 17548 "yaxis": 2 17549 } 17550 ], 17551 "spaceLength": 10, 17552 "stack": false, 17553 "steppedLine": false, 17554 "targets": [ 17555 { 17556 "expr": "sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"ongoing\"}) by (type, instance)", 17557 "format": "time_series", 17558 "hide": false, 17559 "intervalFactor": 1, 17560 "legendFormat": "{{instance}}-{{type}}", 17561 "refId": "A" 17562 }, 17563 { 17564 "expr": "sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"total\"}) by (instance) - sum(tikv_cdc_scan_tasks{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=~\"abort|finish\"}) by (instance)", 17565 "format": "time_series", 17566 "hide": false, 17567 "intervalFactor": 1, 17568 "legendFormat": "{{instance}}-pending", 17569 "refId": "B" 17570 } 17571 ], 17572 "thresholds": [], 17573 "timeFrom": null, 17574 "timeRegions": [], 17575 "timeShift": null, 17576 "title": "Initial scan tasks status", 17577 "tooltip": { 17578 "shared": true, 17579 "sort": 0, 17580 "value_type": "individual" 17581 }, 17582 "type": "graph", 17583 "xaxis": { 17584 "buckets": null, 17585 "mode": "time", 17586 "name": null, 17587 "show": true, 17588 "values": [] 17589 }, 17590 "yaxes": [ 17591 { 17592 "format": "none", 17593 "label": null, 17594 "logBase": 1, 17595 "max": null, 17596 "min": null, 17597 "show": true 17598 }, 17599 { 17600 "format": "none", 17601 "label": null, 17602 "logBase": 1, 17603 "max": null, 17604 "min": null, 17605 "show": true 17606 } 17607 ], 17608 "yaxis": { 17609 "align": false, 17610 "alignLevel": null 17611 } 17612 }, 17613 { 17614 "aliasColors": {}, 17615 "bars": false, 17616 "dashLength": 10, 17617 "dashes": false, 17618 "datasource": "${DS_TEST-CLUSTER}", 17619 "decimals": 1, 17620 "description": "The memory usage per TiKV instance", 17621 "editable": true, 17622 "error": false, 17623 "fieldConfig": { 17624 "defaults": {}, 17625 "overrides": [] 17626 }, 17627 "fill": 0, 17628 "fillGradient": 0, 17629 "grid": {}, 17630 "gridPos": { 17631 "h": 7, 17632 "w": 12, 17633 "x": 0, 17634 "y": 194 17635 }, 17636 "hiddenSeries": false, 17637 "id": 78, 17638 "legend": { 17639 "alignAsTable": true, 17640 "avg": false, 17641 "current": true, 17642 "max": true, 17643 "min": false, 17644 "rightSide": true, 17645 "show": true, 17646 "sideWidth": null, 17647 "sort": null, 17648 "sortDesc": null, 17649 "total": false, 17650 "values": true 17651 }, 17652 "lines": true, 17653 "linewidth": 1, 17654 "links": [], 17655 "nullPointMode": "null", 17656 "options": { 17657 "alertThreshold": true 17658 }, 17659 "paceLength": 10, 17660 "percentage": false, 17661 "pluginVersion": "7.5.11", 17662 "pointradius": 5, 17663 "points": false, 17664 "renderer": "flot", 17665 "seriesOverrides": [], 17666 "spaceLength": 10, 17667 "stack": false, 17668 "steppedLine": false, 17669 "targets": [ 17670 { 17671 "expr": "avg(tikv_cdc_captured_region_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", 17672 "format": "time_series", 17673 "hide": false, 17674 "intervalFactor": 2, 17675 "legendFormat": "tikv-{{instance}}-total", 17676 "refId": "A", 17677 "step": 10 17678 }, 17679 { 17680 "expr": "sum(tikv_cdc_region_resolve_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance, status)", 17681 "format": "time_series", 17682 "hide": false, 17683 "intervalFactor": 2, 17684 "legendFormat": "tikv-{{instance}}-{{status}}", 17685 "refId": "B", 17686 "step": 10 17687 } 17688 ], 17689 "thresholds": [], 17690 "timeFrom": null, 17691 "timeRegions": [], 17692 "timeShift": null, 17693 "title": "Captured region count", 17694 "tooltip": { 17695 "msResolution": false, 17696 "shared": true, 17697 "sort": 0, 17698 "value_type": "individual" 17699 }, 17700 "type": "graph", 17701 "xaxis": { 17702 "buckets": null, 17703 "mode": "time", 17704 "name": null, 17705 "show": true, 17706 "values": [] 17707 }, 17708 "yaxes": [ 17709 { 17710 "format": "none", 17711 "label": null, 17712 "logBase": 1, 17713 "max": null, 17714 "min": "0", 17715 "show": true 17716 }, 17717 { 17718 "format": "short", 17719 "label": null, 17720 "logBase": 1, 17721 "max": null, 17722 "min": null, 17723 "show": true 17724 } 17725 ], 17726 "yaxis": { 17727 "align": false, 17728 "alignLevel": null 17729 } 17730 }, 17731 { 17732 "aliasColors": {}, 17733 "bars": false, 17734 "dashLength": 10, 17735 "dashes": false, 17736 "datasource": "${DS_TEST-CLUSTER}", 17737 "decimals": 1, 17738 "description": "The speed of TiKV CDC incremental scan", 17739 "editable": true, 17740 "error": false, 17741 "fieldConfig": { 17742 "defaults": {}, 17743 "overrides": [] 17744 }, 17745 "fill": 0, 17746 "fillGradient": 0, 17747 "grid": {}, 17748 "gridPos": { 17749 "h": 7, 17750 "w": 6, 17751 "x": 12, 17752 "y": 194 17753 }, 17754 "hiddenSeries": false, 17755 "id": 76, 17756 "legend": { 17757 "alignAsTable": true, 17758 "avg": false, 17759 "current": true, 17760 "hideEmpty": true, 17761 "hideZero": true, 17762 "max": true, 17763 "min": false, 17764 "rightSide": false, 17765 "show": true, 17766 "sideWidth": null, 17767 "sort": "current", 17768 "sortDesc": true, 17769 "total": false, 17770 "values": true 17771 }, 17772 "lines": true, 17773 "linewidth": 1, 17774 "links": [], 17775 "nullPointMode": "null", 17776 "options": { 17777 "alertThreshold": true 17778 }, 17779 "paceLength": 10, 17780 "percentage": false, 17781 "pluginVersion": "7.5.11", 17782 "pointradius": 5, 17783 "points": false, 17784 "renderer": "flot", 17785 "seriesOverrides": [], 17786 "spaceLength": 10, 17787 "stack": false, 17788 "steppedLine": false, 17789 "targets": [ 17790 { 17791 "expr": "sum(rate(tikv_cdc_scan_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=\"tikv\"}[30s])) by (instance)", 17792 "format": "time_series", 17793 "hide": false, 17794 "intervalFactor": 2, 17795 "legendFormat": "tikv-{{instance}}", 17796 "refId": "A", 17797 "step": 10 17798 } 17799 ], 17800 "thresholds": [], 17801 "timeFrom": null, 17802 "timeRegions": [], 17803 "timeShift": null, 17804 "title": "CDC scan speed", 17805 "tooltip": { 17806 "msResolution": false, 17807 "shared": true, 17808 "sort": 0, 17809 "value_type": "individual" 17810 }, 17811 "type": "graph", 17812 "xaxis": { 17813 "buckets": null, 17814 "mode": "time", 17815 "name": null, 17816 "show": true, 17817 "values": [] 17818 }, 17819 "yaxes": [ 17820 { 17821 "format": "bytes", 17822 "label": null, 17823 "logBase": 1, 17824 "max": null, 17825 "min": "0", 17826 "show": true 17827 }, 17828 { 17829 "format": "short", 17830 "label": null, 17831 "logBase": 1, 17832 "max": null, 17833 "min": null, 17834 "show": true 17835 } 17836 ], 17837 "yaxis": { 17838 "align": false, 17839 "alignLevel": null 17840 } 17841 }, 17842 { 17843 "aliasColors": {}, 17844 "bars": true, 17845 "dashLength": 10, 17846 "dashes": false, 17847 "datasource": "${DS_TEST-CLUSTER}", 17848 "decimals": 1, 17849 "description": "The total bytes of TiKV CDC incremental scan", 17850 "editable": true, 17851 "error": false, 17852 "fieldConfig": { 17853 "defaults": {}, 17854 "overrides": [] 17855 }, 17856 "fill": 0, 17857 "fillGradient": 0, 17858 "grid": {}, 17859 "gridPos": { 17860 "h": 7, 17861 "w": 6, 17862 "x": 18, 17863 "y": 194 17864 }, 17865 "hiddenSeries": false, 17866 "id": 139, 17867 "legend": { 17868 "alignAsTable": true, 17869 "avg": false, 17870 "current": true, 17871 "hideEmpty": true, 17872 "hideZero": true, 17873 "max": true, 17874 "min": false, 17875 "rightSide": false, 17876 "show": true, 17877 "sideWidth": null, 17878 "sort": "current", 17879 "sortDesc": true, 17880 "total": false, 17881 "values": true 17882 }, 17883 "lines": false, 17884 "linewidth": 1, 17885 "links": [], 17886 "nullPointMode": "null", 17887 "options": { 17888 "alertThreshold": true 17889 }, 17890 "paceLength": 10, 17891 "percentage": false, 17892 "pluginVersion": "7.5.11", 17893 "pointradius": 5, 17894 "points": false, 17895 "renderer": "flot", 17896 "seriesOverrides": [], 17897 "spaceLength": 10, 17898 "stack": false, 17899 "steppedLine": false, 17900 "targets": [ 17901 { 17902 "expr": "sum(tikv_cdc_scan_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=\"tikv\"}) by (instance)", 17903 "format": "time_series", 17904 "hide": false, 17905 "intervalFactor": 2, 17906 "legendFormat": "tikv-{{instance}}", 17907 "refId": "A", 17908 "step": 10 17909 } 17910 ], 17911 "thresholds": [], 17912 "timeFrom": null, 17913 "timeRegions": [], 17914 "timeShift": null, 17915 "title": "CDC total scan bytes", 17916 "tooltip": { 17917 "msResolution": false, 17918 "shared": true, 17919 "sort": 0, 17920 "value_type": "individual" 17921 }, 17922 "type": "graph", 17923 "xaxis": { 17924 "buckets": null, 17925 "mode": "time", 17926 "name": null, 17927 "show": true, 17928 "values": [] 17929 }, 17930 "yaxes": [ 17931 { 17932 "format": "bytes", 17933 "label": null, 17934 "logBase": 1, 17935 "max": null, 17936 "min": null, 17937 "show": true 17938 }, 17939 { 17940 "format": "short", 17941 "label": null, 17942 "logBase": 1, 17943 "max": null, 17944 "min": null, 17945 "show": true 17946 } 17947 ], 17948 "yaxis": { 17949 "align": false, 17950 "alignLevel": null 17951 } 17952 }, 17953 { 17954 "aliasColors": {}, 17955 "bars": false, 17956 "dashLength": 10, 17957 "dashes": false, 17958 "datasource": "${DS_TEST-CLUSTER}", 17959 "decimals": 1, 17960 "description": "", 17961 "fieldConfig": { 17962 "defaults": {}, 17963 "overrides": [] 17964 }, 17965 "fill": 4, 17966 "fillGradient": 0, 17967 "gridPos": { 17968 "h": 7, 17969 "w": 12, 17970 "x": 0, 17971 "y": 201 17972 }, 17973 "hiddenSeries": false, 17974 "id": 143, 17975 "legend": { 17976 "alignAsTable": true, 17977 "avg": false, 17978 "current": true, 17979 "hideEmpty": false, 17980 "hideZero": true, 17981 "max": false, 17982 "min": true, 17983 "rightSide": true, 17984 "show": true, 17985 "sideWidth": null, 17986 "sort": "current", 17987 "sortDesc": true, 17988 "total": false, 17989 "values": true 17990 }, 17991 "lines": true, 17992 "linewidth": 1, 17993 "links": [], 17994 "nullPointMode": "null", 17995 "options": { 17996 "alertThreshold": true 17997 }, 17998 "paceLength": 10, 17999 "percentage": false, 18000 "pluginVersion": "7.5.11", 18001 "pointradius": 1, 18002 "points": true, 18003 "renderer": "flot", 18004 "seriesOverrides": [ 18005 { 18006 "alias": "/(access|miss).*/", 18007 "fill": 0, 18008 "points": false, 18009 "yaxis": 2 18010 } 18011 ], 18012 "spaceLength": 10, 18013 "stack": false, 18014 "steppedLine": false, 18015 "targets": [ 18016 { 18017 "expr": "(sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance) - sum(rate(tikv_cdc_old_value_cache_miss{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)) / sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)", 18018 "format": "time_series", 18019 "hide": false, 18020 "intervalFactor": 1, 18021 "legendFormat": "hit-rate-{{instance}}", 18022 "refId": "A" 18023 }, 18024 { 18025 "expr": "-sum(rate(tikv_cdc_old_value_cache_access{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)", 18026 "format": "time_series", 18027 "hide": true, 18028 "intervalFactor": 1, 18029 "legendFormat": "access-{{instance}}", 18030 "refId": "B" 18031 }, 18032 { 18033 "expr": "-sum(rate(tikv_cdc_old_value_cache_miss{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)", 18034 "format": "time_series", 18035 "hide": false, 18036 "intervalFactor": 1, 18037 "legendFormat": "miss-{{instance}}", 18038 "refId": "C" 18039 }, 18040 { 18041 "expr": "-sum(rate(tikv_cdc_old_value_cache_miss_none{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)", 18042 "format": "time_series", 18043 "hide": false, 18044 "intervalFactor": 1, 18045 "legendFormat": "miss-none-{{instance}}", 18046 "refId": "D" 18047 } 18048 ], 18049 "thresholds": [], 18050 "timeFrom": null, 18051 "timeRegions": [], 18052 "timeShift": null, 18053 "title": "Old value cache hit", 18054 "tooltip": { 18055 "shared": true, 18056 "sort": 0, 18057 "value_type": "individual" 18058 }, 18059 "type": "graph", 18060 "xaxis": { 18061 "buckets": null, 18062 "mode": "time", 18063 "name": null, 18064 "show": true, 18065 "values": [] 18066 }, 18067 "yaxes": [ 18068 { 18069 "format": "percentunit", 18070 "label": null, 18071 "logBase": 1, 18072 "max": "1", 18073 "min": "0", 18074 "show": true 18075 }, 18076 { 18077 "format": "none", 18078 "label": null, 18079 "logBase": 1, 18080 "max": null, 18081 "min": null, 18082 "show": true 18083 } 18084 ], 18085 "yaxis": { 18086 "align": false, 18087 "alignLevel": null 18088 } 18089 }, 18090 { 18091 "aliasColors": {}, 18092 "bars": false, 18093 "dashLength": 10, 18094 "dashes": false, 18095 "datasource": "${DS_TEST-CLUSTER}", 18096 "decimals": 1, 18097 "description": "The total number of cache entries in the old value cache.", 18098 "fieldConfig": { 18099 "defaults": {}, 18100 "overrides": [] 18101 }, 18102 "fill": 1, 18103 "fillGradient": 0, 18104 "gridPos": { 18105 "h": 7, 18106 "w": 12, 18107 "x": 12, 18108 "y": 201 18109 }, 18110 "hiddenSeries": false, 18111 "id": 145, 18112 "legend": { 18113 "alignAsTable": true, 18114 "avg": false, 18115 "current": true, 18116 "hideEmpty": false, 18117 "hideZero": true, 18118 "max": true, 18119 "min": false, 18120 "rightSide": true, 18121 "show": true, 18122 "sideWidth": null, 18123 "sort": "current", 18124 "sortDesc": true, 18125 "total": false, 18126 "values": true 18127 }, 18128 "lines": true, 18129 "linewidth": 1, 18130 "links": [], 18131 "nullPointMode": "null", 18132 "options": { 18133 "alertThreshold": true 18134 }, 18135 "paceLength": 10, 18136 "percentage": false, 18137 "pluginVersion": "7.5.11", 18138 "pointradius": 1, 18139 "points": false, 18140 "renderer": "flot", 18141 "seriesOverrides": [ 18142 { 18143 "alias": "/.*len/", 18144 "yaxis": 2 18145 } 18146 ], 18147 "spaceLength": 10, 18148 "stack": false, 18149 "steppedLine": false, 18150 "targets": [ 18151 { 18152 "expr": "sum(tikv_cdc_old_value_cache_length{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", 18153 "format": "time_series", 18154 "hide": false, 18155 "intervalFactor": 1, 18156 "legendFormat": "{{instance}}-len", 18157 "refId": "A" 18158 }, 18159 { 18160 "expr": "sum(tikv_cdc_old_value_cache_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance) / sum(tikv_cdc_old_value_cache_length{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", 18161 "format": "time_series", 18162 "hide": false, 18163 "intervalFactor": 1, 18164 "legendFormat": "{{instance}}-avg entry bytes", 18165 "refId": "B" 18166 }, 18167 { 18168 "expr": "sum(tikv_cdc_old_value_cache_memory_quota{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", 18169 "format": "time_series", 18170 "hide": false, 18171 "intervalFactor": 1, 18172 "legendFormat": "{{instance}}-quota", 18173 "refId": "C" 18174 } 18175 ], 18176 "thresholds": [], 18177 "timeFrom": null, 18178 "timeRegions": [], 18179 "timeShift": null, 18180 "title": "Old value cache size", 18181 "tooltip": { 18182 "shared": true, 18183 "sort": 0, 18184 "value_type": "individual" 18185 }, 18186 "type": "graph", 18187 "xaxis": { 18188 "buckets": null, 18189 "mode": "time", 18190 "name": null, 18191 "show": true, 18192 "values": [] 18193 }, 18194 "yaxes": [ 18195 { 18196 "format": "bytes", 18197 "label": null, 18198 "logBase": 1, 18199 "max": null, 18200 "min": "0", 18201 "show": true 18202 }, 18203 { 18204 "decimals": null, 18205 "format": "none", 18206 "label": null, 18207 "logBase": 1, 18208 "max": null, 18209 "min": "0", 18210 "show": true 18211 } 18212 ], 18213 "yaxis": { 18214 "align": false, 18215 "alignLevel": null 18216 } 18217 }, 18218 { 18219 "aliasColors": {}, 18220 "bars": false, 18221 "dashLength": 10, 18222 "dashes": false, 18223 "datasource": "${DS_TEST-CLUSTER}", 18224 "decimals": 1, 18225 "description": "", 18226 "editable": true, 18227 "error": false, 18228 "fieldConfig": { 18229 "defaults": {}, 18230 "overrides": [] 18231 }, 18232 "fill": 0, 18233 "fillGradient": 0, 18234 "grid": {}, 18235 "gridPos": { 18236 "h": 7, 18237 "w": 12, 18238 "x": 0, 18239 "y": 208 18240 }, 18241 "hiddenSeries": false, 18242 "id": 141, 18243 "legend": { 18244 "alignAsTable": true, 18245 "avg": false, 18246 "current": true, 18247 "hideEmpty": true, 18248 "hideZero": true, 18249 "max": true, 18250 "min": false, 18251 "rightSide": true, 18252 "show": true, 18253 "sideWidth": null, 18254 "sort": "current", 18255 "sortDesc": true, 18256 "total": false, 18257 "values": true 18258 }, 18259 "lines": true, 18260 "linewidth": 1, 18261 "links": [], 18262 "nullPointMode": "null", 18263 "options": { 18264 "alertThreshold": true 18265 }, 18266 "paceLength": 10, 18267 "percentage": false, 18268 "pluginVersion": "7.5.11", 18269 "pointradius": 5, 18270 "points": false, 18271 "renderer": "flot", 18272 "seriesOverrides": [], 18273 "spaceLength": 10, 18274 "stack": false, 18275 "steppedLine": false, 18276 "targets": [ 18277 { 18278 "expr": "sum(rate(tikv_cdc_old_value_scan_details{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance, cf, tag)", 18279 "format": "time_series", 18280 "hide": false, 18281 "intervalFactor": 2, 18282 "legendFormat": "{{instance}}-{{cf}}-{{tag}}", 18283 "refId": "A", 18284 "step": 10 18285 } 18286 ], 18287 "thresholds": [], 18288 "timeFrom": null, 18289 "timeRegions": [], 18290 "timeShift": null, 18291 "title": "Old value seek operation", 18292 "tooltip": { 18293 "msResolution": false, 18294 "shared": true, 18295 "sort": 0, 18296 "value_type": "individual" 18297 }, 18298 "type": "graph", 18299 "xaxis": { 18300 "buckets": null, 18301 "mode": "time", 18302 "name": null, 18303 "show": true, 18304 "values": [] 18305 }, 18306 "yaxes": [ 18307 { 18308 "format": "ops", 18309 "label": null, 18310 "logBase": 1, 18311 "max": null, 18312 "min": "0", 18313 "show": true 18314 }, 18315 { 18316 "format": "short", 18317 "label": null, 18318 "logBase": 1, 18319 "max": null, 18320 "min": null, 18321 "show": true 18322 } 18323 ], 18324 "yaxis": { 18325 "align": false, 18326 "alignLevel": null 18327 } 18328 }, 18329 { 18330 "cards": { 18331 "cardPadding": 0, 18332 "cardRound": 0 18333 }, 18334 "color": { 18335 "cardColor": "#FF9830", 18336 "colorScale": "linear", 18337 "colorScheme": "interpolateSpectral", 18338 "exponent": 0.5, 18339 "min": 0, 18340 "mode": "spectrum" 18341 }, 18342 "dataFormat": "tsbuckets", 18343 "datasource": "${DS_TEST-CLUSTER}", 18344 "description": "The time consumed to get an old value (both from cache and from disk)", 18345 "fieldConfig": { 18346 "defaults": {}, 18347 "overrides": [] 18348 }, 18349 "gridPos": { 18350 "h": 7, 18351 "w": 6, 18352 "x": 12, 18353 "y": 208 18354 }, 18355 "heatmap": {}, 18356 "hideZeroBuckets": true, 18357 "highlightCards": true, 18358 "id": 146, 18359 "legend": { 18360 "alignAsTable": true, 18361 "avg": false, 18362 "current": true, 18363 "max": true, 18364 "min": false, 18365 "rightSide": true, 18366 "show": true, 18367 "sort": "current", 18368 "sortDesc": true, 18369 "total": false, 18370 "values": true 18371 }, 18372 "links": [], 18373 "maxPerRow": 3, 18374 "repeatDirection": "h", 18375 "reverseYBuckets": false, 18376 "targets": [ 18377 { 18378 "expr": "sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)", 18379 "format": "heatmap", 18380 "instant": false, 18381 "intervalFactor": 2, 18382 "legendFormat": "{{le}}", 18383 "refId": "A" 18384 } 18385 ], 18386 "title": "Old value seek duration", 18387 "tooltip": { 18388 "show": true, 18389 "showHistogram": true 18390 }, 18391 "tooltipDecimals": 1, 18392 "type": "heatmap", 18393 "xAxis": { 18394 "show": true 18395 }, 18396 "xBucketNumber": null, 18397 "xBucketSize": null, 18398 "yAxis": { 18399 "decimals": 1, 18400 "format": "s", 18401 "logBase": 1, 18402 "max": null, 18403 "min": null, 18404 "show": true, 18405 "splitFactor": null 18406 }, 18407 "yBucketBound": "upper", 18408 "yBucketNumber": null, 18409 "yBucketSize": null 18410 }, 18411 { 18412 "aliasColors": {}, 18413 "bars": false, 18414 "dashLength": 10, 18415 "dashes": false, 18416 "datasource": "${DS_TEST-CLUSTER}", 18417 "decimals": 1, 18418 "description": "", 18419 "fieldConfig": { 18420 "defaults": {}, 18421 "overrides": [] 18422 }, 18423 "fill": 1, 18424 "fillGradient": 0, 18425 "gridPos": { 18426 "h": 7, 18427 "w": 6, 18428 "x": 18, 18429 "y": 208 18430 }, 18431 "hiddenSeries": false, 18432 "id": 142, 18433 "legend": { 18434 "alignAsTable": true, 18435 "avg": false, 18436 "current": true, 18437 "hideEmpty": false, 18438 "hideZero": true, 18439 "max": true, 18440 "min": false, 18441 "rightSide": false, 18442 "show": true, 18443 "sideWidth": null, 18444 "sort": "current", 18445 "sortDesc": true, 18446 "total": false, 18447 "values": true 18448 }, 18449 "lines": true, 18450 "linewidth": 1, 18451 "links": [], 18452 "nullPointMode": "null", 18453 "options": { 18454 "alertThreshold": true 18455 }, 18456 "paceLength": 10, 18457 "percentage": false, 18458 "pluginVersion": "7.5.11", 18459 "pointradius": 5, 18460 "points": false, 18461 "renderer": "flot", 18462 "seriesOverrides": [], 18463 "spaceLength": 10, 18464 "stack": false, 18465 "steppedLine": false, 18466 "targets": [ 18467 { 18468 "expr": "histogram_quantile(0.99, sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance, tag))", 18469 "format": "time_series", 18470 "intervalFactor": 1, 18471 "legendFormat": "{{instance}}-99%-{{tag}}", 18472 "refId": "A" 18473 }, 18474 { 18475 "expr": "histogram_quantile(0.95, sum(rate(tikv_cdc_old_value_duration_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance, tag))", 18476 "format": "time_series", 18477 "intervalFactor": 1, 18478 "legendFormat": "{{instance}}-95%-{{tag}}", 18479 "refId": "B" 18480 }, 18481 { 18482 "expr": "sum(rate(tikv_cdc_old_value_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance, tag) / sum(rate(tikv_cdc_old_value_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance, tag)", 18483 "format": "time_series", 18484 "intervalFactor": 1, 18485 "legendFormat": "{{instance}}-avg-{{tag}}", 18486 "refId": "C" 18487 } 18488 ], 18489 "thresholds": [], 18490 "timeFrom": null, 18491 "timeRegions": [], 18492 "timeShift": null, 18493 "title": "Old value seek duration", 18494 "tooltip": { 18495 "shared": true, 18496 "sort": 0, 18497 "value_type": "individual" 18498 }, 18499 "type": "graph", 18500 "xaxis": { 18501 "buckets": null, 18502 "mode": "time", 18503 "name": null, 18504 "show": true, 18505 "values": [] 18506 }, 18507 "yaxes": [ 18508 { 18509 "format": "s", 18510 "label": null, 18511 "logBase": 1, 18512 "max": null, 18513 "min": null, 18514 "show": true 18515 }, 18516 { 18517 "format": "short", 18518 "label": null, 18519 "logBase": 1, 18520 "max": null, 18521 "min": null, 18522 "show": false 18523 } 18524 ], 18525 "yaxis": { 18526 "align": false, 18527 "alignLevel": null 18528 } 18529 } 18530 ], 18531 "title": "TiKV", 18532 "type": "row" 18533 }, 18534 { 18535 "collapsed": true, 18536 "datasource": null, 18537 "gridPos": { 18538 "h": 1, 18539 "w": 24, 18540 "x": 0, 18541 "y": 14 18542 }, 18543 "id": 294, 18544 "panels": [ 18545 { 18546 "aliasColors": {}, 18547 "bars": false, 18548 "dashLength": 10, 18549 "dashes": false, 18550 "datasource": "${DS_TEST-CLUSTER}", 18551 "fieldConfig": { 18552 "defaults": {}, 18553 "overrides": [] 18554 }, 18555 "fill": 1, 18556 "fillGradient": 0, 18557 "gridPos": { 18558 "h": 7, 18559 "w": 4, 18560 "x": 0, 18561 "y": 15 18562 }, 18563 "hiddenSeries": false, 18564 "id": 321, 18565 "legend": { 18566 "avg": false, 18567 "current": false, 18568 "max": false, 18569 "min": false, 18570 "show": true, 18571 "total": false, 18572 "values": false 18573 }, 18574 "lines": true, 18575 "linewidth": 1, 18576 "nullPointMode": "null", 18577 "options": { 18578 "alertThreshold": true 18579 }, 18580 "percentage": false, 18581 "pluginVersion": "7.5.11", 18582 "pointradius": 2, 18583 "points": false, 18584 "renderer": "flot", 18585 "seriesOverrides": [], 18586 "spaceLength": 10, 18587 "stack": false, 18588 "steppedLine": false, 18589 "targets": [ 18590 { 18591 "exemplar": true, 18592 "expr": "sum without (from) (rate(ticdc_message_server_message_count{instance=~\"$ticdc_instance\"}[30s]))", 18593 "interval": "", 18594 "legendFormat": "{{instance}}", 18595 "queryType": "randomWalk", 18596 "refId": "A" 18597 } 18598 ], 18599 "thresholds": [], 18600 "timeFrom": null, 18601 "timeRegions": [], 18602 "timeShift": null, 18603 "title": "Message Receive Rate", 18604 "tooltip": { 18605 "shared": true, 18606 "sort": 0, 18607 "value_type": "individual" 18608 }, 18609 "type": "graph", 18610 "xaxis": { 18611 "buckets": null, 18612 "mode": "time", 18613 "name": null, 18614 "show": true, 18615 "values": [] 18616 }, 18617 "yaxes": [ 18618 { 18619 "format": "short", 18620 "label": null, 18621 "logBase": 1, 18622 "max": null, 18623 "min": null, 18624 "show": true 18625 }, 18626 { 18627 "format": "short", 18628 "label": null, 18629 "logBase": 1, 18630 "max": null, 18631 "min": null, 18632 "show": true 18633 } 18634 ], 18635 "yaxis": { 18636 "align": false, 18637 "alignLevel": null 18638 } 18639 }, 18640 { 18641 "aliasColors": {}, 18642 "bars": false, 18643 "dashLength": 10, 18644 "dashes": false, 18645 "datasource": "${DS_TEST-CLUSTER}", 18646 "fieldConfig": { 18647 "defaults": {}, 18648 "overrides": [] 18649 }, 18650 "fill": 1, 18651 "fillGradient": 0, 18652 "gridPos": { 18653 "h": 7, 18654 "w": 4, 18655 "x": 4, 18656 "y": 15 18657 }, 18658 "hiddenSeries": false, 18659 "id": 323, 18660 "legend": { 18661 "avg": false, 18662 "current": false, 18663 "max": false, 18664 "min": false, 18665 "show": true, 18666 "total": false, 18667 "values": false 18668 }, 18669 "lines": true, 18670 "linewidth": 1, 18671 "nullPointMode": "null", 18672 "options": { 18673 "alertThreshold": true 18674 }, 18675 "percentage": false, 18676 "pluginVersion": "7.5.11", 18677 "pointradius": 2, 18678 "points": false, 18679 "renderer": "flot", 18680 "seriesOverrides": [], 18681 "spaceLength": 10, 18682 "stack": false, 18683 "steppedLine": false, 18684 "targets": [ 18685 { 18686 "exemplar": true, 18687 "expr": "sum without (to) (rate(ticdc_message_client_message_count{instance=~\"$ticdc_instance\"}[30s]))", 18688 "interval": "", 18689 "legendFormat": "{{instance}}", 18690 "queryType": "randomWalk", 18691 "refId": "A" 18692 } 18693 ], 18694 "thresholds": [], 18695 "timeFrom": null, 18696 "timeRegions": [], 18697 "timeShift": null, 18698 "title": "Message Send Rate", 18699 "tooltip": { 18700 "shared": true, 18701 "sort": 0, 18702 "value_type": "individual" 18703 }, 18704 "type": "graph", 18705 "xaxis": { 18706 "buckets": null, 18707 "mode": "time", 18708 "name": null, 18709 "show": true, 18710 "values": [] 18711 }, 18712 "yaxes": [ 18713 { 18714 "format": "short", 18715 "label": null, 18716 "logBase": 1, 18717 "max": null, 18718 "min": null, 18719 "show": true 18720 }, 18721 { 18722 "format": "short", 18723 "label": null, 18724 "logBase": 1, 18725 "max": null, 18726 "min": null, 18727 "show": true 18728 } 18729 ], 18730 "yaxis": { 18731 "align": false, 18732 "alignLevel": null 18733 } 18734 }, 18735 { 18736 "aliasColors": {}, 18737 "bars": false, 18738 "dashLength": 10, 18739 "dashes": false, 18740 "datasource": "${DS_TEST-CLUSTER}", 18741 "fieldConfig": { 18742 "defaults": {}, 18743 "overrides": [] 18744 }, 18745 "fill": 0, 18746 "fillGradient": 0, 18747 "gridPos": { 18748 "h": 7, 18749 "w": 4, 18750 "x": 8, 18751 "y": 15 18752 }, 18753 "hiddenSeries": false, 18754 "id": 352, 18755 "legend": { 18756 "avg": false, 18757 "current": false, 18758 "hideEmpty": true, 18759 "hideZero": true, 18760 "max": false, 18761 "min": false, 18762 "show": true, 18763 "total": false, 18764 "values": false 18765 }, 18766 "lines": true, 18767 "linewidth": 1, 18768 "nullPointMode": "null", 18769 "options": { 18770 "alertThreshold": true 18771 }, 18772 "percentage": false, 18773 "pluginVersion": "7.5.11", 18774 "pointradius": 2, 18775 "points": false, 18776 "renderer": "flot", 18777 "seriesOverrides": [], 18778 "spaceLength": 10, 18779 "stack": false, 18780 "steppedLine": false, 18781 "targets": [ 18782 { 18783 "exemplar": true, 18784 "expr": "histogram_quantile(0.5, sum(rate(ticdc_message_server_message_batch_size_bucket[30s])) by (le))", 18785 "interval": "", 18786 "legendFormat": "p50", 18787 "queryType": "randomWalk", 18788 "refId": "A" 18789 }, 18790 { 18791 "exemplar": true, 18792 "expr": "histogram_quantile(0.95, sum(rate(ticdc_message_server_message_batch_size_bucket[30s])) by (le))", 18793 "hide": false, 18794 "interval": "", 18795 "legendFormat": "p90", 18796 "refId": "B" 18797 }, 18798 { 18799 "exemplar": true, 18800 "expr": "histogram_quantile(0.99, sum(rate(ticdc_message_server_message_batch_size_bucket[30s])) by (le))", 18801 "hide": false, 18802 "interval": "", 18803 "legendFormat": "p99", 18804 "refId": "C" 18805 } 18806 ], 18807 "thresholds": [], 18808 "timeFrom": null, 18809 "timeRegions": [], 18810 "timeShift": null, 18811 "title": "Message Batch Size", 18812 "tooltip": { 18813 "shared": true, 18814 "sort": 0, 18815 "value_type": "individual" 18816 }, 18817 "transformations": [], 18818 "type": "graph", 18819 "xaxis": { 18820 "buckets": null, 18821 "mode": "time", 18822 "name": null, 18823 "show": true, 18824 "values": [] 18825 }, 18826 "yaxes": [ 18827 { 18828 "format": "short", 18829 "label": null, 18830 "logBase": 1, 18831 "max": null, 18832 "min": null, 18833 "show": true 18834 }, 18835 { 18836 "format": "short", 18837 "label": null, 18838 "logBase": 1, 18839 "max": null, 18840 "min": null, 18841 "show": true 18842 } 18843 ], 18844 "yaxis": { 18845 "align": false, 18846 "alignLevel": null 18847 } 18848 }, 18849 { 18850 "aliasColors": {}, 18851 "bars": false, 18852 "dashLength": 10, 18853 "dashes": false, 18854 "datasource": "${DS_TEST-CLUSTER}", 18855 "fieldConfig": { 18856 "defaults": { 18857 "unit": "short" 18858 }, 18859 "overrides": [] 18860 }, 18861 "fill": 0, 18862 "fillGradient": 0, 18863 "gridPos": { 18864 "h": 7, 18865 "w": 6, 18866 "x": 12, 18867 "y": 15 18868 }, 18869 "hiddenSeries": false, 18870 "id": 354, 18871 "legend": { 18872 "avg": false, 18873 "current": false, 18874 "max": false, 18875 "min": false, 18876 "show": true, 18877 "total": false, 18878 "values": false 18879 }, 18880 "lines": true, 18881 "linewidth": 1, 18882 "nullPointMode": "null", 18883 "options": { 18884 "alertThreshold": true 18885 }, 18886 "percentage": false, 18887 "pluginVersion": "7.5.11", 18888 "pointradius": 2, 18889 "points": false, 18890 "renderer": "flot", 18891 "seriesOverrides": [], 18892 "spaceLength": 10, 18893 "stack": false, 18894 "steppedLine": false, 18895 "targets": [ 18896 { 18897 "exemplar": true, 18898 "expr": "histogram_quantile(0.5, sum(rate(ticdc_message_server_message_batch_bytes_bucket [30s])) by (le))", 18899 "hide": false, 18900 "interval": "", 18901 "legendFormat": "p50", 18902 "refId": "C" 18903 }, 18904 { 18905 "exemplar": true, 18906 "expr": "histogram_quantile(0.95, sum(rate(ticdc_message_server_message_batch_bytes_bucket [30s])) by (le))", 18907 "interval": "", 18908 "legendFormat": "p95", 18909 "queryType": "randomWalk", 18910 "refId": "A" 18911 }, 18912 { 18913 "exemplar": true, 18914 "expr": "histogram_quantile(0.99, sum(rate(ticdc_message_server_message_batch_bytes_bucket [30s])) by (le))", 18915 "hide": false, 18916 "interval": "", 18917 "legendFormat": "p99", 18918 "refId": "B" 18919 } 18920 ], 18921 "thresholds": [], 18922 "timeFrom": null, 18923 "timeRegions": [], 18924 "timeShift": null, 18925 "title": "Receive message batch bytes percentiles", 18926 "tooltip": { 18927 "shared": true, 18928 "sort": 0, 18929 "value_type": "individual" 18930 }, 18931 "type": "graph", 18932 "xaxis": { 18933 "buckets": null, 18934 "mode": "time", 18935 "name": null, 18936 "show": true, 18937 "values": [] 18938 }, 18939 "yaxes": [ 18940 { 18941 "format": "short", 18942 "label": null, 18943 "logBase": 1, 18944 "max": null, 18945 "min": null, 18946 "show": true 18947 }, 18948 { 18949 "format": "short", 18950 "label": null, 18951 "logBase": 1, 18952 "max": null, 18953 "min": null, 18954 "show": true 18955 } 18956 ], 18957 "yaxis": { 18958 "align": false, 18959 "alignLevel": null 18960 } 18961 }, 18962 { 18963 "aliasColors": {}, 18964 "bars": false, 18965 "dashLength": 10, 18966 "dashes": false, 18967 "datasource": "${DS_TEST-CLUSTER}", 18968 "fieldConfig": { 18969 "defaults": {}, 18970 "overrides": [] 18971 }, 18972 "fill": 0, 18973 "fillGradient": 0, 18974 "gridPos": { 18975 "h": 7, 18976 "w": 6, 18977 "x": 18, 18978 "y": 15 18979 }, 18980 "hiddenSeries": false, 18981 "id": 356, 18982 "legend": { 18983 "avg": false, 18984 "current": false, 18985 "max": false, 18986 "min": false, 18987 "show": true, 18988 "total": false, 18989 "values": false 18990 }, 18991 "lines": true, 18992 "linewidth": 1, 18993 "nullPointMode": "null", 18994 "options": { 18995 "alertThreshold": true 18996 }, 18997 "percentage": false, 18998 "pluginVersion": "7.5.11", 18999 "pointradius": 2, 19000 "points": false, 19001 "renderer": "flot", 19002 "seriesOverrides": [], 19003 "spaceLength": 10, 19004 "stack": false, 19005 "steppedLine": false, 19006 "targets": [ 19007 { 19008 "exemplar": true, 19009 "expr": "histogram_quantile(0.5, sum(rate(ticdc_message_server_message_bytes_bucket [30s])) by (le))", 19010 "interval": "", 19011 "legendFormat": "p50", 19012 "queryType": "randomWalk", 19013 "refId": "A" 19014 }, 19015 { 19016 "exemplar": true, 19017 "expr": "histogram_quantile(0.95, sum(rate(ticdc_message_server_message_bytes_bucket [30s])) by (le))", 19018 "hide": false, 19019 "interval": "", 19020 "legendFormat": "p95", 19021 "refId": "B" 19022 }, 19023 { 19024 "exemplar": true, 19025 "expr": "histogram_quantile(0.99, sum(rate(ticdc_message_server_message_bytes_bucket [30s])) by (le))", 19026 "hide": false, 19027 "interval": "", 19028 "legendFormat": "p99", 19029 "refId": "C" 19030 } 19031 ], 19032 "thresholds": [], 19033 "timeFrom": null, 19034 "timeRegions": [], 19035 "timeShift": null, 19036 "title": "Receive Message Bytes Percentile", 19037 "tooltip": { 19038 "shared": true, 19039 "sort": 0, 19040 "value_type": "individual" 19041 }, 19042 "type": "graph", 19043 "xaxis": { 19044 "buckets": null, 19045 "mode": "time", 19046 "name": null, 19047 "show": true, 19048 "values": [] 19049 }, 19050 "yaxes": [ 19051 { 19052 "format": "short", 19053 "label": null, 19054 "logBase": 1, 19055 "max": null, 19056 "min": null, 19057 "show": true 19058 }, 19059 { 19060 "format": "short", 19061 "label": null, 19062 "logBase": 1, 19063 "max": null, 19064 "min": null, 19065 "show": true 19066 } 19067 ], 19068 "yaxis": { 19069 "align": false, 19070 "alignLevel": null 19071 } 19072 }, 19073 { 19074 "datasource": "${DS_TEST-CLUSTER}", 19075 "fieldConfig": { 19076 "defaults": { 19077 "color": { 19078 "mode": "thresholds" 19079 }, 19080 "custom": { 19081 "align": "left", 19082 "filterable": false 19083 }, 19084 "decimals": 2, 19085 "displayName": "", 19086 "mappings": [], 19087 "thresholds": { 19088 "mode": "absolute", 19089 "steps": [ 19090 { 19091 "color": "green", 19092 "value": null 19093 }, 19094 { 19095 "color": "red", 19096 "value": 80 19097 } 19098 ] 19099 }, 19100 "unit": "short" 19101 }, 19102 "overrides": [ 19103 { 19104 "matcher": { 19105 "id": "byName", 19106 "options": "Time" 19107 }, 19108 "properties": [ 19109 { 19110 "id": "displayName", 19111 "value": "Time" 19112 }, 19113 { 19114 "id": "unit", 19115 "value": "time: YYYY-MM-DD HH:mm:ss" 19116 }, 19117 { 19118 "id": "custom.align", 19119 "value": null 19120 } 19121 ] 19122 } 19123 ] 19124 }, 19125 "gridPos": { 19126 "h": 8, 19127 "w": 20, 19128 "x": 0, 19129 "y": 22 19130 }, 19131 "id": 350, 19132 "options": { 19133 "showHeader": true 19134 }, 19135 "pluginVersion": "7.5.11", 19136 "targets": [ 19137 { 19138 "exemplar": true, 19139 "expr": "ticdc_message_server_cur_stream_count", 19140 "format": "time_series", 19141 "instant": true, 19142 "interval": "", 19143 "legendFormat": "", 19144 "queryType": "randomWalk", 19145 "refId": "A" 19146 } 19147 ], 19148 "title": "Stream Count Between Nodes", 19149 "transformations": [ 19150 { 19151 "id": "labelsToFields", 19152 "options": { 19153 "valueLabel": "from" 19154 } 19155 }, 19156 { 19157 "id": "organize", 19158 "options": { 19159 "excludeByName": { 19160 "Time": true, 19161 "instance": false, 19162 "job": true 19163 }, 19164 "indexByName": {}, 19165 "renameByName": { 19166 "instance": "dest\\src" 19167 } 19168 } 19169 }, 19170 { 19171 "id": "merge", 19172 "options": { 19173 "reducers": [] 19174 } 19175 } 19176 ], 19177 "type": "table" 19178 }, 19179 { 19180 "aliasColors": {}, 19181 "bars": false, 19182 "dashLength": 10, 19183 "dashes": false, 19184 "datasource": "${DS_TEST-CLUSTER}", 19185 "fieldConfig": { 19186 "defaults": {}, 19187 "overrides": [] 19188 }, 19189 "fill": 1, 19190 "fillGradient": 0, 19191 "gridPos": { 19192 "h": 8, 19193 "w": 4, 19194 "x": 20, 19195 "y": 22 19196 }, 19197 "hiddenSeries": false, 19198 "id": 358, 19199 "legend": { 19200 "avg": false, 19201 "current": false, 19202 "max": false, 19203 "min": false, 19204 "show": true, 19205 "total": false, 19206 "values": false 19207 }, 19208 "lines": true, 19209 "linewidth": 1, 19210 "nullPointMode": "null", 19211 "options": { 19212 "alertThreshold": true 19213 }, 19214 "percentage": false, 19215 "pluginVersion": "7.5.11", 19216 "pointradius": 2, 19217 "points": false, 19218 "renderer": "flot", 19219 "seriesOverrides": [], 19220 "spaceLength": 10, 19221 "stack": false, 19222 "steppedLine": false, 19223 "targets": [ 19224 { 19225 "exemplar": true, 19226 "expr": "sum(ticdc_message_server_cur_stream_count)", 19227 "interval": "", 19228 "legendFormat": "actual", 19229 "queryType": "randomWalk", 19230 "refId": "A" 19231 }, 19232 { 19233 "exemplar": true, 19234 "expr": "2 * count(process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}) - 1", 19235 "hide": false, 19236 "interval": "", 19237 "legendFormat": "expected", 19238 "refId": "B" 19239 } 19240 ], 19241 "thresholds": [], 19242 "timeFrom": null, 19243 "timeRegions": [], 19244 "timeShift": null, 19245 "title": "Total Stream Count", 19246 "tooltip": { 19247 "shared": true, 19248 "sort": 0, 19249 "value_type": "individual" 19250 }, 19251 "type": "graph", 19252 "xaxis": { 19253 "buckets": null, 19254 "mode": "time", 19255 "name": null, 19256 "show": true, 19257 "values": [] 19258 }, 19259 "yaxes": [ 19260 { 19261 "format": "short", 19262 "label": null, 19263 "logBase": 1, 19264 "max": null, 19265 "min": null, 19266 "show": true 19267 }, 19268 { 19269 "format": "short", 19270 "label": null, 19271 "logBase": 1, 19272 "max": null, 19273 "min": null, 19274 "show": true 19275 } 19276 ], 19277 "yaxis": { 19278 "align": false, 19279 "alignLevel": null 19280 } 19281 } 19282 ], 19283 "title": "Peer Messages", 19284 "type": "row" 19285 }, 19286 { 19287 "collapsed": true, 19288 "datasource": null, 19289 "gridPos": { 19290 "h": 1, 19291 "w": 24, 19292 "x": 0, 19293 "y": 15 19294 }, 19295 "id": 10021, 19296 "panels": [ 19297 { 19298 "cards": { 19299 "cardPadding": 0, 19300 "cardRound": 0 19301 }, 19302 "color": { 19303 "cardColor": "#b4ff00", 19304 "colorScale": "sqrt", 19305 "colorScheme": "interpolateSpectral", 19306 "exponent": 0.5, 19307 "min": 0, 19308 "mode": "spectrum" 19309 }, 19310 "dataFormat": "tsbuckets", 19311 "datasource": "${DS_TEST-CLUSTER}", 19312 "description": "The latency distributions of fsync called by redo writer", 19313 "fieldConfig": { 19314 "defaults": {}, 19315 "overrides": [] 19316 }, 19317 "gridPos": { 19318 "h": 8, 19319 "w": 12, 19320 "x": 0, 19321 "y": 16 19322 }, 19323 "heatmap": {}, 19324 "hideZeroBuckets": true, 19325 "highlightCards": true, 19326 "id": 10023, 19327 "legend": { 19328 "show": true 19329 }, 19330 "pluginVersion": "6.1.6", 19331 "reverseYBuckets": false, 19332 "targets": [ 19333 { 19334 "exemplar": true, 19335 "expr": "max(rate(ticdc_redo_fsync_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)", 19336 "format": "heatmap", 19337 "interval": "", 19338 "intervalFactor": 2, 19339 "legendFormat": "{{le}}", 19340 "queryType": "randomWalk", 19341 "refId": "A" 19342 } 19343 ], 19344 "timeFrom": null, 19345 "timeShift": null, 19346 "title": "Redo fsync duration", 19347 "tooltip": { 19348 "show": true, 19349 "showHistogram": true 19350 }, 19351 "tooltipDecimals": 1, 19352 "type": "heatmap", 19353 "xAxis": { 19354 "show": true 19355 }, 19356 "xBucketNumber": null, 19357 "xBucketSize": null, 19358 "yAxis": { 19359 "decimals": 1, 19360 "format": "s", 19361 "logBase": 1, 19362 "max": null, 19363 "min": null, 19364 "show": true, 19365 "splitFactor": null 19366 }, 19367 "yBucketBound": "upper", 19368 "yBucketNumber": null, 19369 "yBucketSize": null 19370 }, 19371 { 19372 "cards": { 19373 "cardPadding": 0, 19374 "cardRound": 0 19375 }, 19376 "color": { 19377 "cardColor": "#b4ff00", 19378 "colorScale": "sqrt", 19379 "colorScheme": "interpolateSpectral", 19380 "exponent": 0.5, 19381 "min": 0, 19382 "mode": "spectrum" 19383 }, 19384 "dataFormat": "tsbuckets", 19385 "datasource": "${DS_TEST-CLUSTER}", 19386 "description": "The latency distributions of flushall called by redo writer", 19387 "fieldConfig": { 19388 "defaults": {}, 19389 "overrides": [] 19390 }, 19391 "gridPos": { 19392 "h": 8, 19393 "w": 12, 19394 "x": 12, 19395 "y": 16 19396 }, 19397 "heatmap": {}, 19398 "hideZeroBuckets": true, 19399 "highlightCards": true, 19400 "id": 10025, 19401 "legend": { 19402 "show": true 19403 }, 19404 "pluginVersion": "6.1.6", 19405 "reverseYBuckets": false, 19406 "targets": [ 19407 { 19408 "exemplar": true, 19409 "expr": "max(rate(ticdc_redo_flushall_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)", 19410 "format": "heatmap", 19411 "interval": "", 19412 "intervalFactor": 2, 19413 "legendFormat": "{{le}}", 19414 "queryType": "randomWalk", 19415 "refId": "A" 19416 } 19417 ], 19418 "timeFrom": null, 19419 "timeShift": null, 19420 "title": "Redo flushall duration", 19421 "tooltip": { 19422 "show": true, 19423 "showHistogram": true 19424 }, 19425 "tooltipDecimals": 1, 19426 "type": "heatmap", 19427 "xAxis": { 19428 "show": true 19429 }, 19430 "xBucketNumber": null, 19431 "xBucketSize": null, 19432 "yAxis": { 19433 "decimals": 1, 19434 "format": "s", 19435 "logBase": 1, 19436 "max": null, 19437 "min": null, 19438 "show": true, 19439 "splitFactor": null 19440 }, 19441 "yBucketBound": "upper", 19442 "yBucketNumber": null, 19443 "yBucketSize": null 19444 }, 19445 { 19446 "cards": { 19447 "cardPadding": 0, 19448 "cardRound": 0 19449 }, 19450 "color": { 19451 "cardColor": "#b4ff00", 19452 "colorScale": "sqrt", 19453 "colorScheme": "interpolateSpectral", 19454 "exponent": 0.5, 19455 "min": 0, 19456 "mode": "spectrum" 19457 }, 19458 "dataFormat": "tsbuckets", 19459 "datasource": "${DS_TEST-CLUSTER}", 19460 "description": "The latency distributions of writeLog called by redoManager", 19461 "fieldConfig": { 19462 "defaults": {}, 19463 "overrides": [] 19464 }, 19465 "gridPos": { 19466 "h": 8, 19467 "w": 12, 19468 "x": 0, 19469 "y": 24 19470 }, 19471 "heatmap": {}, 19472 "hideZeroBuckets": true, 19473 "highlightCards": true, 19474 "id": 10027, 19475 "legend": { 19476 "show": true 19477 }, 19478 "pluginVersion": "6.1.6", 19479 "reverseYBuckets": false, 19480 "targets": [ 19481 { 19482 "exemplar": true, 19483 "expr": "max(rate(ticdc_redo_write_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)", 19484 "format": "heatmap", 19485 "interval": "", 19486 "intervalFactor": 2, 19487 "legendFormat": "{{le}}", 19488 "queryType": "randomWalk", 19489 "refId": "A" 19490 } 19491 ], 19492 "timeFrom": null, 19493 "timeShift": null, 19494 "title": "Redo write log duration", 19495 "tooltip": { 19496 "show": true, 19497 "showHistogram": true 19498 }, 19499 "tooltipDecimals": 1, 19500 "type": "heatmap", 19501 "xAxis": { 19502 "show": true 19503 }, 19504 "xBucketNumber": null, 19505 "xBucketSize": null, 19506 "yAxis": { 19507 "decimals": 1, 19508 "format": "s", 19509 "logBase": 1, 19510 "max": null, 19511 "min": null, 19512 "show": true, 19513 "splitFactor": null 19514 }, 19515 "yBucketBound": "upper", 19516 "yBucketNumber": null, 19517 "yBucketSize": null 19518 }, 19519 { 19520 "cards": { 19521 "cardPadding": 0, 19522 "cardRound": 0 19523 }, 19524 "color": { 19525 "cardColor": "#b4ff00", 19526 "colorScale": "sqrt", 19527 "colorScheme": "interpolateSpectral", 19528 "exponent": 0.5, 19529 "min": 0, 19530 "mode": "spectrum" 19531 }, 19532 "dataFormat": "tsbuckets", 19533 "datasource": "${DS_TEST-CLUSTER}", 19534 "description": "The latency distributions of flushLog called by redoManager", 19535 "fieldConfig": { 19536 "defaults": {}, 19537 "overrides": [] 19538 }, 19539 "gridPos": { 19540 "h": 8, 19541 "w": 12, 19542 "x": 12, 19543 "y": 24 19544 }, 19545 "heatmap": {}, 19546 "hideZeroBuckets": true, 19547 "highlightCards": true, 19548 "id": 10029, 19549 "legend": { 19550 "show": true 19551 }, 19552 "pluginVersion": "6.1.6", 19553 "reverseYBuckets": false, 19554 "targets": [ 19555 { 19556 "exemplar": true, 19557 "expr": "max(rate(ticdc_redo_flush_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)", 19558 "format": "heatmap", 19559 "interval": "", 19560 "intervalFactor": 2, 19561 "legendFormat": "{{le}}", 19562 "queryType": "randomWalk", 19563 "refId": "A" 19564 } 19565 ], 19566 "timeFrom": null, 19567 "timeShift": null, 19568 "title": "Redo flush log duration", 19569 "tooltip": { 19570 "show": true, 19571 "showHistogram": true 19572 }, 19573 "tooltipDecimals": 1, 19574 "type": "heatmap", 19575 "xAxis": { 19576 "show": true 19577 }, 19578 "xBucketNumber": null, 19579 "xBucketSize": null, 19580 "yAxis": { 19581 "decimals": 1, 19582 "format": "s", 19583 "logBase": 1, 19584 "max": null, 19585 "min": null, 19586 "show": true, 19587 "splitFactor": null 19588 }, 19589 "yBucketBound": "upper", 19590 "yBucketNumber": null, 19591 "yBucketSize": null 19592 }, 19593 { 19594 "aliasColors": {}, 19595 "bars": false, 19596 "dashLength": 10, 19597 "dashes": false, 19598 "datasource": "${DS_TEST-CLUSTER}", 19599 "description": "The total count of rows that are processed by redo writer", 19600 "fieldConfig": { 19601 "defaults": {}, 19602 "overrides": [] 19603 }, 19604 "fill": 1, 19605 "fillGradient": 0, 19606 "gridPos": { 19607 "h": 8, 19608 "w": 12, 19609 "x": 0, 19610 "y": 32 19611 }, 19612 "hiddenSeries": false, 19613 "id": 10031, 19614 "legend": { 19615 "alignAsTable": true, 19616 "avg": true, 19617 "current": true, 19618 "max": true, 19619 "min": false, 19620 "rightSide": true, 19621 "show": true, 19622 "total": false, 19623 "values": true 19624 }, 19625 "lines": true, 19626 "linewidth": 1, 19627 "nullPointMode": "null", 19628 "options": { 19629 "alertThreshold": true 19630 }, 19631 "percentage": false, 19632 "pluginVersion": "7.5.11", 19633 "pointradius": 2, 19634 "points": false, 19635 "renderer": "flot", 19636 "seriesOverrides": [], 19637 "spaceLength": 10, 19638 "stack": false, 19639 "steppedLine": false, 19640 "targets": [ 19641 { 19642 "exemplar": true, 19643 "expr": "sum(rate(ticdc_redo_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (instance)", 19644 "format": "time_series", 19645 "interval": "", 19646 "intervalFactor": 1, 19647 "legendFormat": "{{instance}}", 19648 "queryType": "randomWalk", 19649 "refId": "A" 19650 }, 19651 { 19652 "exemplar": true, 19653 "expr": "sum(rate(ticdc_redo_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (changefeed)", 19654 "format": "time_series", 19655 "hide": false, 19656 "interval": "", 19657 "intervalFactor": 1, 19658 "legendFormat": "total", 19659 "refId": "B" 19660 } 19661 ], 19662 "thresholds": [], 19663 "timeFrom": null, 19664 "timeRegions": [], 19665 "timeShift": null, 19666 "title": "Redo Write rows/s ", 19667 "tooltip": { 19668 "shared": true, 19669 "sort": 0, 19670 "value_type": "individual" 19671 }, 19672 "type": "graph", 19673 "xaxis": { 19674 "buckets": null, 19675 "mode": "time", 19676 "name": null, 19677 "show": true, 19678 "values": [] 19679 }, 19680 "yaxes": [ 19681 { 19682 "format": "short", 19683 "label": null, 19684 "logBase": 1, 19685 "max": null, 19686 "min": null, 19687 "show": true 19688 }, 19689 { 19690 "format": "short", 19691 "label": null, 19692 "logBase": 1, 19693 "max": null, 19694 "min": null, 19695 "show": false 19696 } 19697 ], 19698 "yaxis": { 19699 "align": false, 19700 "alignLevel": null 19701 } 19702 }, 19703 { 19704 "aliasColors": {}, 19705 "bars": false, 19706 "dashLength": 10, 19707 "dashes": false, 19708 "datasource": "${DS_TEST-CLUSTER}", 19709 "description": "Total number of bytes redo log written", 19710 "fieldConfig": { 19711 "defaults": {}, 19712 "overrides": [] 19713 }, 19714 "fill": 1, 19715 "fillGradient": 0, 19716 "gridPos": { 19717 "h": 8, 19718 "w": 12, 19719 "x": 12, 19720 "y": 32 19721 }, 19722 "hiddenSeries": false, 19723 "id": 10033, 19724 "legend": { 19725 "avg": false, 19726 "current": false, 19727 "max": false, 19728 "min": false, 19729 "show": true, 19730 "total": false, 19731 "values": false 19732 }, 19733 "lines": true, 19734 "linewidth": 1, 19735 "nullPointMode": "null", 19736 "options": { 19737 "alertThreshold": true 19738 }, 19739 "percentage": false, 19740 "pluginVersion": "7.5.11", 19741 "pointradius": 2, 19742 "points": false, 19743 "renderer": "flot", 19744 "seriesOverrides": [], 19745 "spaceLength": 10, 19746 "stack": false, 19747 "steppedLine": false, 19748 "targets": [ 19749 { 19750 "exemplar": true, 19751 "expr": "sum(rate(ticdc_redo_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (instance)", 19752 "interval": "", 19753 "legendFormat": "{{instance}}", 19754 "queryType": "randomWalk", 19755 "refId": "A" 19756 } 19757 ], 19758 "thresholds": [], 19759 "timeFrom": null, 19760 "timeRegions": [], 19761 "timeShift": null, 19762 "title": "Redo Write bytes/s ", 19763 "tooltip": { 19764 "shared": true, 19765 "sort": 0, 19766 "value_type": "individual" 19767 }, 19768 "type": "graph", 19769 "xaxis": { 19770 "buckets": null, 19771 "mode": "time", 19772 "name": null, 19773 "show": true, 19774 "values": [] 19775 }, 19776 "yaxes": [ 19777 { 19778 "format": "bytes", 19779 "label": null, 19780 "logBase": 1, 19781 "max": null, 19782 "min": null, 19783 "show": true 19784 }, 19785 { 19786 "format": "short", 19787 "label": null, 19788 "logBase": 1, 19789 "max": null, 19790 "min": null, 19791 "show": true 19792 } 19793 ], 19794 "yaxis": { 19795 "align": false, 19796 "alignLevel": null 19797 } 19798 }, 19799 { 19800 "aliasColors": {}, 19801 "bars": false, 19802 "dashLength": 10, 19803 "dashes": false, 19804 "datasource": "${DS_TEST-CLUSTER}", 19805 "description": "Redo bgUpdateLog worker busy ratio", 19806 "fieldConfig": { 19807 "defaults": {}, 19808 "overrides": [] 19809 }, 19810 "fill": 1, 19811 "fillGradient": 0, 19812 "gridPos": { 19813 "h": 8, 19814 "w": 12, 19815 "x": 0, 19816 "y": 40 19817 }, 19818 "hiddenSeries": false, 19819 "id": 10035, 19820 "legend": { 19821 "alignAsTable": true, 19822 "avg": false, 19823 "current": true, 19824 "max": false, 19825 "min": false, 19826 "rightSide": false, 19827 "show": true, 19828 "total": false, 19829 "values": true 19830 }, 19831 "lines": true, 19832 "linewidth": 1, 19833 "nullPointMode": "null", 19834 "options": { 19835 "alertThreshold": true 19836 }, 19837 "percentage": false, 19838 "pluginVersion": "7.5.11", 19839 "pointradius": 2, 19840 "points": false, 19841 "renderer": "flot", 19842 "seriesOverrides": [], 19843 "spaceLength": 10, 19844 "stack": false, 19845 "steppedLine": false, 19846 "targets": [ 19847 { 19848 "exemplar": true, 19849 "expr": "sum(rate(ticdc_redo_worker_busy_ratio{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])/10) by (changefeed,instance)", 19850 "interval": "", 19851 "legendFormat": "{{changefeed}}-{{instance}}", 19852 "queryType": "randomWalk", 19853 "refId": "A" 19854 } 19855 ], 19856 "thresholds": [], 19857 "timeFrom": null, 19858 "timeRegions": [], 19859 "timeShift": null, 19860 "title": "Worker Busy Ratio", 19861 "tooltip": { 19862 "shared": true, 19863 "sort": 0, 19864 "value_type": "individual" 19865 }, 19866 "type": "graph", 19867 "xaxis": { 19868 "buckets": null, 19869 "mode": "time", 19870 "name": null, 19871 "show": true, 19872 "values": [] 19873 }, 19874 "yaxes": [ 19875 { 19876 "format": "percent", 19877 "label": null, 19878 "logBase": 1, 19879 "max": null, 19880 "min": null, 19881 "show": true 19882 }, 19883 { 19884 "format": "short", 19885 "label": null, 19886 "logBase": 1, 19887 "max": null, 19888 "min": null, 19889 "show": false 19890 } 19891 ], 19892 "yaxis": { 19893 "align": false, 19894 "alignLevel": null 19895 } 19896 } 19897 ], 19898 "title": "Redo", 19899 "type": "row" 19900 }, 19901 { 19902 "collapsed": true, 19903 "datasource": null, 19904 "gridPos": { 19905 "h": 1, 19906 "w": 24, 19907 "x": 0, 19908 "y": 16 19909 }, 19910 "id": 155, 19911 "panels": [ 19912 { 19913 "aliasColors": {}, 19914 "bars": true, 19915 "cacheTimeout": null, 19916 "dashLength": 10, 19917 "dashes": false, 19918 "datasource": "${DS_TEST-CLUSTER}", 19919 "description": "TiCDC process rss memory usage. TiCDC heap memory size in use ", 19920 "editable": true, 19921 "error": false, 19922 "fill": 0, 19923 "grid": {}, 19924 "gridPos": { 19925 "h": 7, 19926 "w": 12, 19927 "x": 0, 19928 "y": 6 19929 }, 19930 "id": 157, 19931 "legend": { 19932 "alignAsTable": false, 19933 "avg": false, 19934 "current": false, 19935 "hideEmpty": true, 19936 "hideZero": true, 19937 "max": false, 19938 "min": false, 19939 "rightSide": false, 19940 "show": true, 19941 "sideWidth": null, 19942 "total": false, 19943 "values": false 19944 }, 19945 "lines": false, 19946 "linewidth": 1, 19947 "links": [], 19948 "maxPerRow": 3, 19949 "nullPointMode": "null", 19950 "percentage": false, 19951 "pluginVersion": "6.1.6", 19952 "pointradius": 5, 19953 "points": false, 19954 "renderer": "flot", 19955 "repeat": null, 19956 "repeatDirection": "h", 19957 "scopedVars": { 19958 "instance": { 19959 "selected": false, 19960 "text": "172.16.5.33:47912", 19961 "value": "172.16.5.33:47912" 19962 }, 19963 "runtime_instance": { 19964 "selected": false, 19965 "text": "10.2.6.163:8300", 19966 "value": "10.2.6.163:8300" 19967 } 19968 }, 19969 "seriesOverrides": [ 19970 { 19971 "alias": "alloc-from-os", 19972 "fill": 3, 19973 "lines": true, 19974 "stack": false 19975 }, 19976 { 19977 "alias": "gc-threshold", 19978 "bars": false, 19979 "color": "#C4162A", 19980 "lines": true, 19981 "linewidth": 2, 19982 "stack": false 19983 }, 19984 { 19985 "alias": "gc", 19986 "bars": false, 19987 "color": "#C4162A", 19988 "hideTooltip": true, 19989 "legend": false, 19990 "pointradius": 3, 19991 "points": true, 19992 "stack": false 19993 } 19994 ], 19995 "spaceLength": 10, 19996 "stack": true, 19997 "steppedLine": false, 19998 "targets": [ 19999 { 20000 "expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", 20001 "format": "time_series", 20002 "hide": false, 20003 "intervalFactor": 1, 20004 "legendFormat": "alloc-from-os", 20005 "refId": "A" 20006 }, 20007 { 20008 "expr": "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + ticdc_server_go_gc{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)", 20009 "format": "time_series", 20010 "hide": false, 20011 "intervalFactor": 1, 20012 "legendFormat": "estimate-inuse", 20013 "refId": "H" 20014 }, 20015 { 20016 "expr": "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + ticdc_server_go_gc{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)", 20017 "format": "time_series", 20018 "hide": false, 20019 "intervalFactor": 1, 20020 "legendFormat": "estimate-garbage", 20021 "refId": "C" 20022 }, 20023 { 20024 "expr": "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", 20025 "format": "time_series", 20026 "hide": false, 20027 "intervalFactor": 1, 20028 "legendFormat": "reserved-by-go", 20029 "refId": "B" 20030 }, 20031 { 20032 "expr": "go_memstats_stack_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_mspan_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_mcache_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_buck_hash_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_gc_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_other_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", 20033 "format": "time_series", 20034 "hide": false, 20035 "intervalFactor": 1, 20036 "legendFormat": "used-by-go", 20037 "refId": "D" 20038 }, 20039 { 20040 "expr": "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", 20041 "format": "time_series", 20042 "hide": false, 20043 "intervalFactor": 1, 20044 "legendFormat": "gc-threshold", 20045 "refId": "E" 20046 }, 20047 { 20048 "expr": "(clamp_max(idelta(go_memstats_last_gc_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[1m]), 1) * go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}) > 0", 20049 "format": "time_series", 20050 "hide": false, 20051 "intervalFactor": 1, 20052 "legendFormat": "gc", 20053 "refId": "F" 20054 } 20055 ], 20056 "thresholds": [], 20057 "timeFrom": null, 20058 "timeRegions": [], 20059 "timeShift": null, 20060 "title": "Memory Usage", 20061 "tooltip": { 20062 "msResolution": true, 20063 "shared": true, 20064 "sort": 0, 20065 "value_type": "individual" 20066 }, 20067 "type": "graph", 20068 "xaxis": { 20069 "buckets": null, 20070 "mode": "time", 20071 "name": null, 20072 "show": true, 20073 "values": [] 20074 }, 20075 "yaxes": [ 20076 { 20077 "format": "bytes", 20078 "label": "", 20079 "logBase": 1, 20080 "max": null, 20081 "min": "0", 20082 "show": true 20083 }, 20084 { 20085 "format": "short", 20086 "label": "", 20087 "logBase": 1, 20088 "max": null, 20089 "min": null, 20090 "show": false 20091 } 20092 ], 20093 "yaxis": { 20094 "align": false, 20095 "alignLevel": null 20096 } 20097 }, 20098 { 20099 "aliasColors": {}, 20100 "bars": false, 20101 "cacheTimeout": null, 20102 "dashLength": 10, 20103 "dashes": false, 20104 "datasource": "${DS_TEST-CLUSTER}", 20105 "description": "Count of live objects.", 20106 "editable": true, 20107 "error": false, 20108 "fill": 0, 20109 "grid": {}, 20110 "gridPos": { 20111 "h": 7, 20112 "w": 12, 20113 "x": 12, 20114 "y": 6 20115 }, 20116 "id": 158, 20117 "legend": { 20118 "alignAsTable": false, 20119 "avg": false, 20120 "current": false, 20121 "hideEmpty": true, 20122 "hideZero": true, 20123 "max": false, 20124 "min": false, 20125 "rightSide": false, 20126 "show": true, 20127 "sideWidth": null, 20128 "total": false, 20129 "values": false 20130 }, 20131 "lines": true, 20132 "linewidth": 1, 20133 "links": [], 20134 "maxPerRow": 3, 20135 "nullPointMode": "null", 20136 "percentage": false, 20137 "pluginVersion": "6.1.6", 20138 "pointradius": 5, 20139 "points": false, 20140 "renderer": "flot", 20141 "repeat": null, 20142 "repeatDirection": "h", 20143 "scopedVars": { 20144 "instance": { 20145 "selected": false, 20146 "text": "172.16.5.33:47912", 20147 "value": "172.16.5.33:47912" 20148 }, 20149 "runtime_instance": { 20150 "selected": false, 20151 "text": "10.2.6.163:8300", 20152 "value": "10.2.6.163:8300" 20153 } 20154 }, 20155 "seriesOverrides": [], 20156 "spaceLength": 10, 20157 "stack": false, 20158 "steppedLine": false, 20159 "targets": [ 20160 { 20161 "expr": "go_memstats_heap_objects{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", 20162 "format": "time_series", 20163 "hide": false, 20164 "intervalFactor": 1, 20165 "legendFormat": "objects", 20166 "refId": "A" 20167 } 20168 ], 20169 "thresholds": [], 20170 "timeFrom": null, 20171 "timeRegions": [], 20172 "timeShift": null, 20173 "title": "Estimated Live Objects", 20174 "tooltip": { 20175 "msResolution": true, 20176 "shared": true, 20177 "sort": 0, 20178 "value_type": "individual" 20179 }, 20180 "type": "graph", 20181 "xaxis": { 20182 "buckets": null, 20183 "mode": "time", 20184 "name": null, 20185 "show": true, 20186 "values": [] 20187 }, 20188 "yaxes": [ 20189 { 20190 "format": "short", 20191 "label": "", 20192 "logBase": 1, 20193 "max": null, 20194 "min": "0", 20195 "show": true 20196 }, 20197 { 20198 "format": "short", 20199 "label": "", 20200 "logBase": 1, 20201 "max": null, 20202 "min": null, 20203 "show": false 20204 } 20205 ], 20206 "yaxis": { 20207 "align": false, 20208 "alignLevel": null 20209 } 20210 }, 20211 { 20212 "aliasColors": {}, 20213 "bars": false, 20214 "cacheTimeout": null, 20215 "dashLength": 10, 20216 "dashes": false, 20217 "datasource": "${DS_TEST-CLUSTER}", 20218 "description": "TiCDC process Go garbage collection STW pause duration", 20219 "editable": true, 20220 "error": false, 20221 "fill": 1, 20222 "grid": {}, 20223 "gridPos": { 20224 "h": 7, 20225 "w": 12, 20226 "x": 0, 20227 "y": 13 20228 }, 20229 "id": 160, 20230 "legend": { 20231 "alignAsTable": false, 20232 "avg": false, 20233 "current": true, 20234 "max": false, 20235 "min": false, 20236 "rightSide": false, 20237 "show": true, 20238 "total": false, 20239 "values": true 20240 }, 20241 "lines": true, 20242 "linewidth": 1, 20243 "links": [], 20244 "nullPointMode": "null as zero", 20245 "percentage": false, 20246 "pluginVersion": "6.1.6", 20247 "pointradius": 5, 20248 "points": false, 20249 "renderer": "flot", 20250 "scopedVars": { 20251 "instance": { 20252 "selected": false, 20253 "text": "172.16.5.33:47912", 20254 "value": "172.16.5.33:47912" 20255 }, 20256 "runtime_instance": { 20257 "selected": false, 20258 "text": "10.2.6.163:8300", 20259 "value": "10.2.6.163:8300" 20260 } 20261 }, 20262 "seriesOverrides": [ 20263 {} 20264 ], 20265 "spaceLength": 10, 20266 "stack": false, 20267 "steppedLine": false, 20268 "targets": [ 20269 { 20270 "expr": "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile=\"0\"}", 20271 "format": "time_series", 20272 "hide": false, 20273 "instant": false, 20274 "intervalFactor": 1, 20275 "legendFormat": "min", 20276 "refId": "A", 20277 "step": 40 20278 }, 20279 { 20280 "expr": "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile!~\"0|1\"}", 20281 "format": "time_series", 20282 "instant": false, 20283 "intervalFactor": 1, 20284 "legendFormat": "{{quantile}}", 20285 "refId": "B" 20286 }, 20287 { 20288 "expr": "go_gc_duration_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile=\"1\"}", 20289 "format": "time_series", 20290 "instant": false, 20291 "intervalFactor": 1, 20292 "legendFormat": "max", 20293 "refId": "C" 20294 } 20295 ], 20296 "thresholds": [], 20297 "timeFrom": null, 20298 "timeRegions": [], 20299 "timeShift": null, 20300 "title": "GC STW Duration (last 256 GC cycles)", 20301 "tooltip": { 20302 "msResolution": false, 20303 "shared": true, 20304 "sort": 1, 20305 "value_type": "individual" 20306 }, 20307 "type": "graph", 20308 "xaxis": { 20309 "buckets": null, 20310 "mode": "time", 20311 "name": null, 20312 "show": true, 20313 "values": [] 20314 }, 20315 "yaxes": [ 20316 { 20317 "format": "s", 20318 "label": null, 20319 "logBase": 1, 20320 "max": null, 20321 "min": "0", 20322 "show": true 20323 }, 20324 { 20325 "format": "short", 20326 "label": null, 20327 "logBase": 1, 20328 "max": null, 20329 "min": null, 20330 "show": false 20331 } 20332 ], 20333 "yaxis": { 20334 "align": false, 20335 "alignLevel": null 20336 } 20337 }, 20338 { 20339 "aliasColors": {}, 20340 "bars": false, 20341 "dashLength": 10, 20342 "dashes": false, 20343 "datasource": "${DS_TEST-CLUSTER}", 20344 "description": "The throughput of Go's memory allocator.", 20345 "fill": 1, 20346 "gridPos": { 20347 "h": 7, 20348 "w": 12, 20349 "x": 12, 20350 "y": 13 20351 }, 20352 "id": 161, 20353 "legend": { 20354 "alignAsTable": false, 20355 "avg": false, 20356 "current": true, 20357 "max": true, 20358 "min": false, 20359 "rightSide": false, 20360 "show": true, 20361 "total": false, 20362 "values": true 20363 }, 20364 "lines": true, 20365 "linewidth": 1, 20366 "links": [], 20367 "nullPointMode": "null", 20368 "percentage": false, 20369 "pointradius": 5, 20370 "points": false, 20371 "renderer": "flot", 20372 "scopedVars": { 20373 "instance": { 20374 "selected": false, 20375 "text": "172.16.5.33:47912", 20376 "value": "172.16.5.33:47912" 20377 }, 20378 "runtime_instance": { 20379 "selected": false, 20380 "text": "10.2.6.163:8300", 20381 "value": "10.2.6.163:8300" 20382 } 20383 }, 20384 "seriesOverrides": [ 20385 { 20386 "alias": "sweep", 20387 "transform": "negative-Y" 20388 }, 20389 { 20390 "alias": "alloc-ops", 20391 "yaxis": 2 20392 }, 20393 { 20394 "alias": "swepp-ops", 20395 "transform": "negative-Y", 20396 "yaxis": 2 20397 } 20398 ], 20399 "spaceLength": 10, 20400 "stack": false, 20401 "steppedLine": false, 20402 "targets": [ 20403 { 20404 "expr": "irate(go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[30s])", 20405 "format": "time_series", 20406 "intervalFactor": 1, 20407 "legendFormat": "alloc", 20408 "refId": "A" 20409 }, 20410 { 20411 "expr": "irate((go_memstats_alloc_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"})[30s:])", 20412 "format": "time_series", 20413 "intervalFactor": 1, 20414 "legendFormat": "sweep", 20415 "refId": "B" 20416 }, 20417 { 20418 "expr": "irate(go_memstats_mallocs_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[30s])", 20419 "format": "time_series", 20420 "intervalFactor": 1, 20421 "legendFormat": "alloc-ops", 20422 "refId": "C" 20423 }, 20424 { 20425 "expr": "irate(go_memstats_frees_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[30s])", 20426 "format": "time_series", 20427 "intervalFactor": 1, 20428 "legendFormat": "swepp-ops", 20429 "refId": "D" 20430 } 20431 ], 20432 "thresholds": [], 20433 "timeFrom": null, 20434 "timeRegions": [], 20435 "timeShift": null, 20436 "title": "Allocator Throughput", 20437 "tooltip": { 20438 "shared": true, 20439 "sort": 0, 20440 "value_type": "individual" 20441 }, 20442 "type": "graph", 20443 "xaxis": { 20444 "buckets": null, 20445 "mode": "time", 20446 "name": null, 20447 "show": true, 20448 "values": [] 20449 }, 20450 "yaxes": [ 20451 { 20452 "format": "Bps", 20453 "label": null, 20454 "logBase": 1, 20455 "max": null, 20456 "min": null, 20457 "show": true 20458 }, 20459 { 20460 "format": "ops", 20461 "label": null, 20462 "logBase": 1, 20463 "max": null, 20464 "min": null, 20465 "show": true 20466 } 20467 ], 20468 "yaxis": { 20469 "align": true, 20470 "alignLevel": null 20471 } 20472 } 20473 ], 20474 "repeat": "runtime_instance", 20475 "title": "Runtime $runtime_instance", 20476 "type": "row" 20477 }, 20478 { 20479 "collapsed": true, 20480 "datasource": null, 20481 "gridPos": { 20482 "h": 1, 20483 "w": 24, 20484 "x": 0, 20485 "y": 18 20486 }, 20487 "id": 10000, 20488 "panels": [ 20489 { 20490 "aliasColors": {}, 20491 "bars": false, 20492 "dashLength": 10, 20493 "dashes": false, 20494 "datasource": "${DS_TEST-CLUSTER}", 20495 "fieldConfig": { 20496 "defaults": {}, 20497 "overrides": [] 20498 }, 20499 "fill": 1, 20500 "fillGradient": 0, 20501 "gridPos": { 20502 "h": 9, 20503 "w": 12, 20504 "x": 0, 20505 "y": 50 20506 }, 20507 "hiddenSeries": false, 20508 "id": 10001, 20509 "legend": { 20510 "avg": false, 20511 "current": false, 20512 "max": false, 20513 "min": false, 20514 "show": true, 20515 "total": false, 20516 "values": false 20517 }, 20518 "lines": true, 20519 "linewidth": 1, 20520 "nullPointMode": "null as zero", 20521 "options": { 20522 "alertThreshold": true 20523 }, 20524 "percentage": false, 20525 "pluginVersion": "7.5.11", 20526 "pointradius": 2, 20527 "points": false, 20528 "renderer": "flot", 20529 "seriesOverrides": [], 20530 "spaceLength": 10, 20531 "stack": true, 20532 "steppedLine": false, 20533 "targets": [ 20534 { 20535 "exemplar": true, 20536 "expr": "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"count\"}[30s])/2", 20537 "format": "time_series", 20538 "interval": "", 20539 "legendFormat": "", 20540 "queryType": "randomWalk", 20541 "refId": "A" 20542 }, 20543 { 20544 "exemplar": true, 20545 "expr": "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"count\"}", 20546 "format": "time_series", 20547 "interval": "", 20548 "legendFormat": "", 20549 "queryType": "randomWalk", 20550 "refId": "A" 20551 } 20552 ], 20553 "thresholds": [], 20554 "timeFrom": null, 20555 "timeRegions": [], 20556 "timeShift": null, 20557 "title": "Pulsar published DDL schema count", 20558 "tooltip": { 20559 "shared": true, 20560 "sort": 0, 20561 "value_type": "individual" 20562 }, 20563 "type": "graph", 20564 "xaxis": { 20565 "buckets": null, 20566 "mode": "time", 20567 "name": null, 20568 "show": true, 20569 "values": [] 20570 }, 20571 "yaxes": [ 20572 { 20573 "format": "short", 20574 "label": null, 20575 "logBase": 1, 20576 "max": null, 20577 "min": null, 20578 "show": true 20579 }, 20580 { 20581 "format": "short", 20582 "label": null, 20583 "logBase": 1, 20584 "max": null, 20585 "min": null, 20586 "show": true 20587 } 20588 ], 20589 "yaxis": { 20590 "align": false, 20591 "alignLevel": null 20592 } 20593 }, 20594 { 20595 "aliasColors": {}, 20596 "bars": false, 20597 "dashLength": 10, 20598 "dashes": false, 20599 "datasource": "${DS_TEST-CLUSTER}", 20600 "fieldConfig": { 20601 "defaults": {}, 20602 "overrides": [] 20603 }, 20604 "fill": 1, 20605 "fillGradient": 0, 20606 "gridPos": { 20607 "h": 9, 20608 "w": 12, 20609 "x": 12, 20610 "y": 50 20611 }, 20612 "hiddenSeries": false, 20613 "id": 10002, 20614 "legend": { 20615 "avg": false, 20616 "current": false, 20617 "max": false, 20618 "min": false, 20619 "show": true, 20620 "total": false, 20621 "values": false 20622 }, 20623 "lines": true, 20624 "linewidth": 1, 20625 "nullPointMode": "null as zero", 20626 "options": { 20627 "alertThreshold": true 20628 }, 20629 "percentage": false, 20630 "pluginVersion": "7.5.11", 20631 "pointradius": 2, 20632 "points": false, 20633 "renderer": "flot", 20634 "seriesOverrides": [], 20635 "spaceLength": 10, 20636 "stack": true, 20637 "steppedLine": false, 20638 "targets": [ 20639 { 20640 "exemplar": true, 20641 "expr": "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}[30s])/2", 20642 "format": "time_series", 20643 "interval": "", 20644 "legendFormat": "", 20645 "queryType": "randomWalk", 20646 "refId": "A" 20647 }, 20648 { 20649 "exemplar": true, 20650 "expr": "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}", 20651 "format": "time_series", 20652 "interval": "", 20653 "legendFormat": "", 20654 "queryType": "randomWalk", 20655 "refId": "A" 20656 } 20657 ], 20658 "thresholds": [], 20659 "timeFrom": null, 20660 "timeRegions": [], 20661 "timeShift": null, 20662 "title": "Pulsar published DDL schema success", 20663 "tooltip": { 20664 "shared": true, 20665 "sort": 0, 20666 "value_type": "individual" 20667 }, 20668 "type": "graph", 20669 "xaxis": { 20670 "buckets": null, 20671 "mode": "time", 20672 "name": null, 20673 "show": true, 20674 "values": [] 20675 }, 20676 "yaxes": [ 20677 { 20678 "format": "short", 20679 "label": null, 20680 "logBase": 1, 20681 "max": null, 20682 "min": null, 20683 "show": true 20684 }, 20685 { 20686 "format": "short", 20687 "label": null, 20688 "logBase": 1, 20689 "max": null, 20690 "min": null, 20691 "show": true 20692 } 20693 ], 20694 "yaxis": { 20695 "align": false, 20696 "alignLevel": null 20697 } 20698 }, 20699 { 20700 "aliasColors": {}, 20701 "bars": false, 20702 "dashLength": 10, 20703 "dashes": false, 20704 "datasource": "${DS_TEST-CLUSTER}", 20705 "fieldConfig": { 20706 "defaults": {}, 20707 "overrides": [] 20708 }, 20709 "fill": 1, 20710 "fillGradient": 0, 20711 "gridPos": { 20712 "h": 9, 20713 "w": 12, 20714 "x": 0, 20715 "y": 59 20716 }, 20717 "hiddenSeries": false, 20718 "id": 10003, 20719 "legend": { 20720 "avg": false, 20721 "current": false, 20722 "max": false, 20723 "min": false, 20724 "show": true, 20725 "total": false, 20726 "values": false 20727 }, 20728 "lines": true, 20729 "linewidth": 1, 20730 "nullPointMode": "null as zero", 20731 "options": { 20732 "alertThreshold": true 20733 }, 20734 "percentage": false, 20735 "pluginVersion": "7.5.11", 20736 "pointradius": 2, 20737 "points": false, 20738 "renderer": "flot", 20739 "seriesOverrides": [], 20740 "spaceLength": 10, 20741 "stack": true, 20742 "steppedLine": false, 20743 "targets": [ 20744 { 20745 "exemplar": true, 20746 "expr": "increase(ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\", schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}[30s])/2\r", 20747 "format": "time_series", 20748 "interval": "", 20749 "legendFormat": "", 20750 "queryType": "randomWalk", 20751 "refId": "A" 20752 }, 20753 { 20754 "exemplar": true, 20755 "expr": "ticdc_pulsar_published_DDL_schema_table_count{changefeed=~\"$changefeed\", schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}", 20756 "format": "time_series", 20757 "interval": "", 20758 "legendFormat": "", 20759 "queryType": "randomWalk", 20760 "refId": "A" 20761 } 20762 ], 20763 "thresholds": [], 20764 "timeFrom": null, 20765 "timeRegions": [], 20766 "timeShift": null, 20767 "title": "Pulsar published DDL schema fail", 20768 "tooltip": { 20769 "shared": true, 20770 "sort": 0, 20771 "value_type": "individual" 20772 }, 20773 "type": "graph", 20774 "xaxis": { 20775 "buckets": null, 20776 "mode": "time", 20777 "name": null, 20778 "show": true, 20779 "values": [] 20780 }, 20781 "yaxes": [ 20782 { 20783 "format": "short", 20784 "label": null, 20785 "logBase": 1, 20786 "max": null, 20787 "min": null, 20788 "show": true 20789 }, 20790 { 20791 "format": "short", 20792 "label": null, 20793 "logBase": 1, 20794 "max": null, 20795 "min": null, 20796 "show": true 20797 } 20798 ], 20799 "yaxis": { 20800 "align": false, 20801 "alignLevel": null 20802 } 20803 }, 20804 { 20805 "aliasColors": {}, 20806 "bars": false, 20807 "dashLength": 10, 20808 "dashes": false, 20809 "datasource": "${DS_TEST-CLUSTER}", 20810 "fieldConfig": { 20811 "defaults": {}, 20812 "overrides": [] 20813 }, 20814 "fill": 1, 20815 "fillGradient": 0, 20816 "gridPos": { 20817 "h": 9, 20818 "w": 12, 20819 "x": 12, 20820 "y": 59 20821 }, 20822 "hiddenSeries": false, 20823 "id": 10004, 20824 "legend": { 20825 "avg": false, 20826 "current": false, 20827 "max": false, 20828 "min": false, 20829 "show": true, 20830 "total": false, 20831 "values": false 20832 }, 20833 "lines": true, 20834 "linewidth": 1, 20835 "nullPointMode": "null as zero", 20836 "options": { 20837 "alertThreshold": true 20838 }, 20839 "percentage": false, 20840 "pluginVersion": "7.5.11", 20841 "pointradius": 2, 20842 "points": false, 20843 "renderer": "flot", 20844 "seriesOverrides": [], 20845 "spaceLength": 10, 20846 "stack": true, 20847 "steppedLine": false, 20848 "targets": [ 20849 { 20850 "exemplar": true, 20851 "expr": "increase(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",\r\nschema=~\"$schema\", topic=~\"$topic\", type=\"count\"}[30s])/2", 20852 "format": "time_series", 20853 "interval": "", 20854 "legendFormat": "", 20855 "queryType": "randomWalk", 20856 "refId": "A" 20857 } 20858 ], 20859 "thresholds": [], 20860 "timeFrom": null, 20861 "timeRegions": [], 20862 "timeShift": null, 20863 "title": "Pulsar published DML schema count", 20864 "tooltip": { 20865 "shared": true, 20866 "sort": 0, 20867 "value_type": "individual" 20868 }, 20869 "type": "graph", 20870 "xaxis": { 20871 "buckets": null, 20872 "mode": "time", 20873 "name": null, 20874 "show": true, 20875 "values": [] 20876 }, 20877 "yaxes": [ 20878 { 20879 "format": "short", 20880 "label": null, 20881 "logBase": 1, 20882 "max": null, 20883 "min": null, 20884 "show": true 20885 }, 20886 { 20887 "format": "short", 20888 "label": null, 20889 "logBase": 1, 20890 "max": null, 20891 "min": null, 20892 "show": true 20893 } 20894 ], 20895 "yaxis": { 20896 "align": false, 20897 "alignLevel": null 20898 } 20899 }, 20900 { 20901 "aliasColors": {}, 20902 "bars": false, 20903 "dashLength": 10, 20904 "dashes": false, 20905 "datasource": "${DS_TEST-CLUSTER}", 20906 "fieldConfig": { 20907 "defaults": {}, 20908 "overrides": [] 20909 }, 20910 "fill": 1, 20911 "fillGradient": 0, 20912 "gridPos": { 20913 "h": 9, 20914 "w": 12, 20915 "x": 0, 20916 "y": 68 20917 }, 20918 "hiddenSeries": false, 20919 "id": 10005, 20920 "legend": { 20921 "avg": false, 20922 "current": false, 20923 "max": false, 20924 "min": false, 20925 "show": true, 20926 "total": false, 20927 "values": false 20928 }, 20929 "lines": true, 20930 "linewidth": 1, 20931 "nullPointMode": "null as zero", 20932 "options": { 20933 "alertThreshold": true 20934 }, 20935 "percentage": false, 20936 "pluginVersion": "7.5.11", 20937 "pointradius": 2, 20938 "points": false, 20939 "renderer": "flot", 20940 "seriesOverrides": [], 20941 "spaceLength": 10, 20942 "stack": true, 20943 "steppedLine": false, 20944 "targets": [ 20945 { 20946 "exemplar": true, 20947 "expr": "\r\nincrease(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"success\"}[30s])/2", 20948 "format": "time_series", 20949 "interval": "", 20950 "legendFormat": "", 20951 "queryType": "randomWalk", 20952 "refId": "A" 20953 } 20954 ], 20955 "thresholds": [], 20956 "timeFrom": null, 20957 "timeRegions": [], 20958 "timeShift": null, 20959 "title": "Pulsar published DML schema success", 20960 "tooltip": { 20961 "shared": true, 20962 "sort": 0, 20963 "value_type": "individual" 20964 }, 20965 "type": "graph", 20966 "xaxis": { 20967 "buckets": null, 20968 "mode": "time", 20969 "name": null, 20970 "show": true, 20971 "values": [] 20972 }, 20973 "yaxes": [ 20974 { 20975 "format": "short", 20976 "label": null, 20977 "logBase": 1, 20978 "max": null, 20979 "min": null, 20980 "show": true 20981 }, 20982 { 20983 "format": "short", 20984 "label": null, 20985 "logBase": 1, 20986 "max": null, 20987 "min": null, 20988 "show": true 20989 } 20990 ], 20991 "yaxis": { 20992 "align": false, 20993 "alignLevel": null 20994 } 20995 }, 20996 { 20997 "aliasColors": {}, 20998 "bars": false, 20999 "dashLength": 10, 21000 "dashes": false, 21001 "datasource": "${DS_TEST-CLUSTER}", 21002 "fieldConfig": { 21003 "defaults": {}, 21004 "overrides": [] 21005 }, 21006 "fill": 1, 21007 "fillGradient": 0, 21008 "gridPos": { 21009 "h": 9, 21010 "w": 12, 21011 "x": 12, 21012 "y": 68 21013 }, 21014 "hiddenSeries": false, 21015 "id": 10006, 21016 "legend": { 21017 "avg": false, 21018 "current": false, 21019 "max": false, 21020 "min": false, 21021 "show": true, 21022 "total": false, 21023 "values": false 21024 }, 21025 "lines": true, 21026 "linewidth": 1, 21027 "nullPointMode": "null as zero", 21028 "options": { 21029 "alertThreshold": true 21030 }, 21031 "percentage": false, 21032 "pluginVersion": "7.5.11", 21033 "pointradius": 2, 21034 "points": false, 21035 "renderer": "flot", 21036 "seriesOverrides": [], 21037 "spaceLength": 10, 21038 "stack": true, 21039 "steppedLine": false, 21040 "targets": [ 21041 { 21042 "exemplar": true, 21043 "expr": "\r\nincrease(ticdc_pulsar_published_DML_schema_table_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\", type=\"fail\"}[30s])/2", 21044 "format": "time_series", 21045 "interval": "", 21046 "legendFormat": "", 21047 "queryType": "randomWalk", 21048 "refId": "A" 21049 } 21050 ], 21051 "thresholds": [], 21052 "timeFrom": null, 21053 "timeRegions": [], 21054 "timeShift": null, 21055 "title": "Pulsar published DML schema fail", 21056 "tooltip": { 21057 "shared": true, 21058 "sort": 0, 21059 "value_type": "individual" 21060 }, 21061 "type": "graph", 21062 "xaxis": { 21063 "buckets": null, 21064 "mode": "time", 21065 "name": null, 21066 "show": true, 21067 "values": [] 21068 }, 21069 "yaxes": [ 21070 { 21071 "format": "short", 21072 "label": null, 21073 "logBase": 1, 21074 "max": null, 21075 "min": null, 21076 "show": true 21077 }, 21078 { 21079 "format": "short", 21080 "label": null, 21081 "logBase": 1, 21082 "max": null, 21083 "min": null, 21084 "show": true 21085 } 21086 ], 21087 "yaxis": { 21088 "align": false, 21089 "alignLevel": null 21090 } 21091 }, 21092 { 21093 "aliasColors": {}, 21094 "bars": false, 21095 "dashLength": 10, 21096 "dashes": false, 21097 "datasource": "${DS_TEST-CLUSTER}", 21098 "fieldConfig": { 21099 "defaults": {}, 21100 "overrides": [] 21101 }, 21102 "fill": 1, 21103 "fillGradient": 0, 21104 "gridPos": { 21105 "h": 9, 21106 "w": 12, 21107 "x": 0, 21108 "y": 77 21109 }, 21110 "hiddenSeries": false, 21111 "id": 10007, 21112 "legend": { 21113 "alignAsTable": true, 21114 "avg": false, 21115 "current": true, 21116 "max": false, 21117 "min": false, 21118 "show": true, 21119 "total": false, 21120 "values": true 21121 }, 21122 "lines": true, 21123 "linewidth": 1, 21124 "nullPointMode": "null as zero", 21125 "options": { 21126 "alertThreshold": true 21127 }, 21128 "percentage": false, 21129 "pluginVersion": "7.5.11", 21130 "pointradius": 2, 21131 "points": false, 21132 "renderer": "flot", 21133 "seriesOverrides": [], 21134 "spaceLength": 10, 21135 "stack": true, 21136 "steppedLine": false, 21137 "targets": [ 21138 { 21139 "exemplar": true, 21140 "expr": "sum(pulsar_client_bytes_published{changefeed=~\"$changefeed\"}) by (changefeed, instance)", 21141 "format": "time_series", 21142 "interval": "", 21143 "legendFormat": "{{changefeed}}-{{instance}}", 21144 "queryType": "randomWalk", 21145 "refId": "A" 21146 } 21147 ], 21148 "thresholds": [], 21149 "timeFrom": null, 21150 "timeRegions": [], 21151 "timeShift": null, 21152 "title": "Pulsar client bytes published ", 21153 "tooltip": { 21154 "shared": true, 21155 "sort": 0, 21156 "value_type": "individual" 21157 }, 21158 "type": "graph", 21159 "xaxis": { 21160 "buckets": null, 21161 "mode": "time", 21162 "name": null, 21163 "show": true, 21164 "values": [] 21165 }, 21166 "yaxes": [ 21167 { 21168 "format": "bytes", 21169 "label": null, 21170 "logBase": 1, 21171 "max": null, 21172 "min": null, 21173 "show": true 21174 }, 21175 { 21176 "format": "short", 21177 "label": null, 21178 "logBase": 1, 21179 "max": null, 21180 "min": null, 21181 "show": true 21182 } 21183 ], 21184 "yaxis": { 21185 "align": false, 21186 "alignLevel": null 21187 } 21188 }, 21189 { 21190 "aliasColors": {}, 21191 "bars": false, 21192 "dashLength": 10, 21193 "dashes": false, 21194 "datasource": "${DS_TEST-CLUSTER}", 21195 "fieldConfig": { 21196 "defaults": {}, 21197 "overrides": [] 21198 }, 21199 "fill": 1, 21200 "fillGradient": 0, 21201 "gridPos": { 21202 "h": 9, 21203 "w": 12, 21204 "x": 12, 21205 "y": 77 21206 }, 21207 "hiddenSeries": false, 21208 "id": 10008, 21209 "legend": { 21210 "alignAsTable": false, 21211 "avg": false, 21212 "current": true, 21213 "max": false, 21214 "min": false, 21215 "show": true, 21216 "total": false, 21217 "values": true 21218 }, 21219 "lines": true, 21220 "linewidth": 1, 21221 "nullPointMode": "null as zero", 21222 "options": { 21223 "alertThreshold": true 21224 }, 21225 "percentage": false, 21226 "pluginVersion": "7.5.11", 21227 "pointradius": 2, 21228 "points": false, 21229 "renderer": "flot", 21230 "seriesOverrides": [], 21231 "spaceLength": 10, 21232 "stack": true, 21233 "steppedLine": false, 21234 "targets": [ 21235 { 21236 "exemplar": true, 21237 "expr": "sum(pulsar_client_connections_opened{changefeed=~\"$changefeed\"}) by (changefeed, instance)", 21238 "format": "time_series", 21239 "instant": false, 21240 "interval": "", 21241 "intervalFactor": 1, 21242 "legendFormat": "{{changefeed}}-{{instance}}", 21243 "queryType": "randomWalk", 21244 "refId": "A" 21245 } 21246 ], 21247 "thresholds": [], 21248 "timeFrom": null, 21249 "timeRegions": [], 21250 "timeShift": null, 21251 "title": "Pulsar client connections opened", 21252 "tooltip": { 21253 "shared": true, 21254 "sort": 0, 21255 "value_type": "individual" 21256 }, 21257 "type": "graph", 21258 "xaxis": { 21259 "buckets": null, 21260 "mode": "time", 21261 "name": null, 21262 "show": true, 21263 "values": [] 21264 }, 21265 "yaxes": [ 21266 { 21267 "format": "none", 21268 "label": null, 21269 "logBase": 1, 21270 "max": null, 21271 "min": null, 21272 "show": true 21273 }, 21274 { 21275 "format": "short", 21276 "label": null, 21277 "logBase": 1, 21278 "max": null, 21279 "min": null, 21280 "show": true 21281 } 21282 ], 21283 "yaxis": { 21284 "align": false, 21285 "alignLevel": null 21286 } 21287 }, 21288 { 21289 "aliasColors": {}, 21290 "bars": false, 21291 "dashLength": 10, 21292 "dashes": false, 21293 "datasource": "${DS_TEST-CLUSTER}", 21294 "fieldConfig": { 21295 "defaults": {}, 21296 "overrides": [] 21297 }, 21298 "fill": 1, 21299 "fillGradient": 0, 21300 "gridPos": { 21301 "h": 9, 21302 "w": 12, 21303 "x": 0, 21304 "y": 86 21305 }, 21306 "hiddenSeries": false, 21307 "id": 10009, 21308 "legend": { 21309 "alignAsTable": false, 21310 "avg": false, 21311 "current": true, 21312 "max": false, 21313 "min": false, 21314 "show": true, 21315 "total": false, 21316 "values": true 21317 }, 21318 "lines": true, 21319 "linewidth": 1, 21320 "nullPointMode": "null as zero", 21321 "options": { 21322 "alertThreshold": true 21323 }, 21324 "percentage": false, 21325 "pluginVersion": "7.5.11", 21326 "pointradius": 2, 21327 "points": false, 21328 "renderer": "flot", 21329 "seriesOverrides": [], 21330 "spaceLength": 10, 21331 "stack": true, 21332 "steppedLine": false, 21333 "targets": [ 21334 { 21335 "exemplar": true, 21336 "expr": "sum(pulsar_client_rpc_count{changefeed=~\"$changefeed\"}) by (changefeed, instance)", 21337 "format": "time_series", 21338 "instant": false, 21339 "interval": "", 21340 "intervalFactor": 1, 21341 "legendFormat": "{{changefeed}}-{{instance}}", 21342 "queryType": "randomWalk", 21343 "refId": "A" 21344 } 21345 ], 21346 "thresholds": [], 21347 "timeFrom": null, 21348 "timeRegions": [], 21349 "timeShift": null, 21350 "title": "Pulsar client rpc count", 21351 "tooltip": { 21352 "shared": true, 21353 "sort": 0, 21354 "value_type": "individual" 21355 }, 21356 "type": "graph", 21357 "xaxis": { 21358 "buckets": null, 21359 "mode": "time", 21360 "name": null, 21361 "show": true, 21362 "values": [] 21363 }, 21364 "yaxes": [ 21365 { 21366 "format": "none", 21367 "label": null, 21368 "logBase": 1, 21369 "max": null, 21370 "min": null, 21371 "show": true 21372 }, 21373 { 21374 "format": "short", 21375 "label": null, 21376 "logBase": 1, 21377 "max": null, 21378 "min": null, 21379 "show": true 21380 } 21381 ], 21382 "yaxis": { 21383 "align": false, 21384 "alignLevel": null 21385 } 21386 }, 21387 { 21388 "datasource": "${DS_TEST-CLUSTER}", 21389 "fieldConfig": { 21390 "defaults": { 21391 "color": { 21392 "mode": "palette-classic" 21393 }, 21394 "custom": { 21395 "axisLabel": "", 21396 "axisPlacement": "auto", 21397 "barAlignment": 0, 21398 "drawStyle": "line", 21399 "fillOpacity": 10, 21400 "gradientMode": "none", 21401 "hideFrom": { 21402 "graph": false, 21403 "legend": false, 21404 "tooltip": false 21405 }, 21406 "lineInterpolation": "linear", 21407 "lineWidth": 1, 21408 "pointSize": 5, 21409 "scaleDistribution": { 21410 "type": "linear" 21411 }, 21412 "showPoints": "never", 21413 "spanNulls": false 21414 }, 21415 "mappings": [], 21416 "thresholds": { 21417 "mode": "absolute", 21418 "steps": [ 21419 { 21420 "color": "green", 21421 "value": null 21422 }, 21423 { 21424 "color": "red", 21425 "value": 80 21426 } 21427 ] 21428 }, 21429 "unit": "s" 21430 }, 21431 "overrides": [] 21432 }, 21433 "gridPos": { 21434 "h": 9, 21435 "w": 12, 21436 "x": 12, 21437 "y": 86 21438 }, 21439 "id": 10010, 21440 "options": { 21441 "graph": {}, 21442 "legend": { 21443 "calcs": [], 21444 "displayMode": "list", 21445 "placement": "bottom" 21446 }, 21447 "tooltipOptions": { 21448 "mode": "single" 21449 } 21450 }, 21451 "pluginVersion": "7.5.11", 21452 "targets": [ 21453 { 21454 "exemplar": true, 21455 "expr": "histogram_quantile(0.999, sum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=~\"$changefeed\"}[1m])) by (le,changefeed,instance))\r", 21456 "format": "time_series", 21457 "instant": false, 21458 "interval": "", 21459 "intervalFactor": 1, 21460 "legendFormat": "{{changefeed}}-{{instance}}-P999", 21461 "queryType": "randomWalk", 21462 "refId": "A" 21463 }, 21464 { 21465 "exemplar": true, 21466 "expr": "sum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=\"$changefeed\"}[1m])) by (changefeed,instance) / \r\nsum(rate(pulsar_client_producer_latency_seconds_bucket{changefeed=\"$changefeed\"}[1m])) by (changefeed,instance)", 21467 "hide": false, 21468 "interval": "", 21469 "legendFormat": "", 21470 "refId": "B" 21471 } 21472 ], 21473 "timeFrom": null, 21474 "timeShift": null, 21475 "title": "Pulsar client producer latency ", 21476 "type": "timeseries" 21477 }, 21478 { 21479 "aliasColors": {}, 21480 "bars": false, 21481 "dashLength": 10, 21482 "dashes": false, 21483 "datasource": "${DS_TEST-CLUSTER}", 21484 "fieldConfig": { 21485 "defaults": { 21486 "unit": "s" 21487 }, 21488 "overrides": [] 21489 }, 21490 "fill": 1, 21491 "fillGradient": 0, 21492 "gridPos": { 21493 "h": 9, 21494 "w": 12, 21495 "x": 0, 21496 "y": 95 21497 }, 21498 "hiddenSeries": false, 21499 "id": 10011, 21500 "legend": { 21501 "avg": false, 21502 "current": false, 21503 "max": false, 21504 "min": false, 21505 "show": true, 21506 "total": false, 21507 "values": false 21508 }, 21509 "lines": true, 21510 "linewidth": 1, 21511 "nullPointMode": "null as zero", 21512 "options": { 21513 "alertThreshold": true 21514 }, 21515 "percentage": false, 21516 "pluginVersion": "7.5.11", 21517 "pointradius": 2, 21518 "points": false, 21519 "renderer": "flot", 21520 "seriesOverrides": [], 21521 "spaceLength": 10, 21522 "stack": false, 21523 "steppedLine": false, 21524 "targets": [ 21525 { 21526 "exemplar": true, 21527 "expr": "histogram_quantile(0.999, sum(rate(pulsar_client_producer_rpc_latency_seconds_bucket{changefeed=~\"$changefeed\"}[1m])) by (le,changefeed,instance))\r", 21528 "format": "time_series", 21529 "instant": false, 21530 "interval": "", 21531 "intervalFactor": 1, 21532 "legendFormat": "{{changefeed}}-{{instance}}", 21533 "queryType": "randomWalk", 21534 "refId": "A" 21535 } 21536 ], 21537 "thresholds": [], 21538 "timeFrom": null, 21539 "timeRegions": [], 21540 "timeShift": null, 21541 "title": "Pulsar client producer rpc latency", 21542 "tooltip": { 21543 "shared": true, 21544 "sort": 0, 21545 "value_type": "individual" 21546 }, 21547 "type": "graph", 21548 "xaxis": { 21549 "buckets": null, 21550 "mode": "time", 21551 "name": null, 21552 "show": true, 21553 "values": [] 21554 }, 21555 "yaxes": [ 21556 { 21557 "format": "s", 21558 "label": null, 21559 "logBase": 1, 21560 "max": null, 21561 "min": null, 21562 "show": true 21563 }, 21564 { 21565 "format": "short", 21566 "label": null, 21567 "logBase": 1, 21568 "max": null, 21569 "min": null, 21570 "show": true 21571 } 21572 ], 21573 "yaxis": { 21574 "align": false, 21575 "alignLevel": null 21576 } 21577 }, 21578 { 21579 "aliasColors": {}, 21580 "bars": false, 21581 "dashLength": 10, 21582 "dashes": false, 21583 "datasource": "${DS_TEST-CLUSTER}", 21584 "description": "", 21585 "fieldConfig": { 21586 "defaults": { 21587 "unit": "none" 21588 }, 21589 "overrides": [] 21590 }, 21591 "fill": 1, 21592 "fillGradient": 0, 21593 "gridPos": { 21594 "h": 9, 21595 "w": 12, 21596 "x": 12, 21597 "y": 95 21598 }, 21599 "hiddenSeries": false, 21600 "id": 10012, 21601 "legend": { 21602 "avg": false, 21603 "current": false, 21604 "max": false, 21605 "min": false, 21606 "show": true, 21607 "total": false, 21608 "values": false 21609 }, 21610 "lines": true, 21611 "linewidth": 1, 21612 "nullPointMode": "null as zero", 21613 "options": { 21614 "alertThreshold": true 21615 }, 21616 "percentage": false, 21617 "pluginVersion": "7.5.11", 21618 "pointradius": 2, 21619 "points": false, 21620 "renderer": "flot", 21621 "seriesOverrides": [], 21622 "spaceLength": 10, 21623 "stack": false, 21624 "steppedLine": false, 21625 "targets": [ 21626 { 21627 "exemplar": true, 21628 "expr": "sum(pulsar_client_producer_pending_messages{changefeed=~\"$changefeed\"}) by (changefeed, instance)\r", 21629 "format": "time_series", 21630 "instant": false, 21631 "interval": "", 21632 "intervalFactor": 1, 21633 "legendFormat": "{{changefeed}}-{{instance}}", 21634 "queryType": "randomWalk", 21635 "refId": "A" 21636 } 21637 ], 21638 "thresholds": [], 21639 "timeFrom": null, 21640 "timeRegions": [], 21641 "timeShift": null, 21642 "title": "Pulsar client producer pending messages", 21643 "tooltip": { 21644 "shared": true, 21645 "sort": 0, 21646 "value_type": "individual" 21647 }, 21648 "type": "graph", 21649 "xaxis": { 21650 "buckets": null, 21651 "mode": "time", 21652 "name": null, 21653 "show": true, 21654 "values": [] 21655 }, 21656 "yaxes": [ 21657 { 21658 "format": "none", 21659 "label": null, 21660 "logBase": 1, 21661 "max": null, 21662 "min": null, 21663 "show": true 21664 }, 21665 { 21666 "format": "short", 21667 "label": null, 21668 "logBase": 1, 21669 "max": null, 21670 "min": null, 21671 "show": true 21672 } 21673 ], 21674 "yaxis": { 21675 "align": false, 21676 "alignLevel": null 21677 } 21678 }, 21679 { 21680 "aliasColors": {}, 21681 "bars": false, 21682 "dashLength": 10, 21683 "dashes": false, 21684 "datasource": "${DS_TEST-CLUSTER}", 21685 "description": "", 21686 "fieldConfig": { 21687 "defaults": { 21688 "unit": "none" 21689 }, 21690 "overrides": [] 21691 }, 21692 "fill": 1, 21693 "fillGradient": 0, 21694 "gridPos": { 21695 "h": 9, 21696 "w": 12, 21697 "x": 12, 21698 "y": 104 21699 }, 21700 "hiddenSeries": false, 21701 "id": 10013, 21702 "legend": { 21703 "avg": false, 21704 "current": false, 21705 "max": false, 21706 "min": false, 21707 "show": true, 21708 "total": false, 21709 "values": false 21710 }, 21711 "lines": true, 21712 "linewidth": 1, 21713 "nullPointMode": "null as zero", 21714 "options": { 21715 "alertThreshold": true 21716 }, 21717 "percentage": false, 21718 "pluginVersion": "7.5.11", 21719 "pointradius": 2, 21720 "points": false, 21721 "renderer": "flot", 21722 "seriesOverrides": [], 21723 "spaceLength": 10, 21724 "stack": false, 21725 "steppedLine": false, 21726 "targets": [ 21727 { 21728 "exemplar": true, 21729 "expr": "sum(published_message_type_resolved_count{changefeed=~\"$changefeed\",schema=~\"$schema\", topic=~\"$topic\"}) by (changefeed)", 21730 "format": "time_series", 21731 "instant": false, 21732 "interval": "", 21733 "intervalFactor": 1, 21734 "legendFormat": "{{changefeed}}-{{topic}}", 21735 "queryType": "randomWalk", 21736 "refId": "A" 21737 } 21738 ], 21739 "thresholds": [], 21740 "timeFrom": null, 21741 "timeRegions": [], 21742 "timeShift": null, 21743 "title": "Pulsar client producer pending messages", 21744 "tooltip": { 21745 "shared": true, 21746 "sort": 0, 21747 "value_type": "individual" 21748 }, 21749 "type": "graph", 21750 "xaxis": { 21751 "buckets": null, 21752 "mode": "time", 21753 "name": null, 21754 "show": true, 21755 "values": [] 21756 }, 21757 "yaxes": [ 21758 { 21759 "format": "none", 21760 "label": null, 21761 "logBase": 1, 21762 "max": null, 21763 "min": null, 21764 "show": true 21765 }, 21766 { 21767 "format": "short", 21768 "label": null, 21769 "logBase": 1, 21770 "max": null, 21771 "min": null, 21772 "show": true 21773 } 21774 ], 21775 "yaxis": { 21776 "align": false, 21777 "alignLevel": null 21778 } 21779 } 21780 ], 21781 "title": "Pulsar Sink", 21782 "type": "row" 21783 } 21784 ], 21785 "refresh": "1m", 21786 "schemaVersion": 27, 21787 "style": "dark", 21788 "tags": [], 21789 "templating": { 21790 "list": [ 21791 { 21792 "allValue": null, 21793 "current": {}, 21794 "datasource": "${DS_TEST-CLUSTER}", 21795 "definition": "", 21796 "description": null, 21797 "error": null, 21798 "hide": 0, 21799 "includeAll": false, 21800 "label": "K8s-cluster", 21801 "multi": false, 21802 "name": "k8s_cluster", 21803 "options": [], 21804 "query": { 21805 "query": "label_values(go_goroutines, k8s_cluster)", 21806 "refId": "local-k8s_cluster-Variable-Query" 21807 }, 21808 "refresh": 2, 21809 "regex": "", 21810 "skipUrlSync": false, 21811 "sort": 1, 21812 "tagValuesQuery": "", 21813 "tags": [], 21814 "tagsQuery": "", 21815 "type": "query", 21816 "useTags": false 21817 }, 21818 { 21819 "allValue": null, 21820 "current": {}, 21821 "datasource": "${DS_TEST-CLUSTER}", 21822 "definition": "", 21823 "description": null, 21824 "error": null, 21825 "hide": 0, 21826 "includeAll": false, 21827 "label": "tidb_cluster", 21828 "multi": false, 21829 "name": "tidb_cluster", 21830 "options": [], 21831 "query": { 21832 "query": "label_values(go_goroutines{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)", 21833 "refId": "local-tidb_cluster-Variable-Query" 21834 }, 21835 "refresh": 2, 21836 "regex": "", 21837 "skipUrlSync": false, 21838 "sort": 1, 21839 "tagValuesQuery": "", 21840 "tags": [], 21841 "tagsQuery": "", 21842 "type": "query", 21843 "useTags": false 21844 }, 21845 { 21846 "allValue": ".*", 21847 "current": {}, 21848 "datasource": "${DS_TEST-CLUSTER}", 21849 "definition": "label_values(ticdc_processor_processor_tick_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, namespace)", 21850 "description": null, 21851 "error": null, 21852 "hide": 0, 21853 "includeAll": true, 21854 "label": "Namespace", 21855 "multi": true, 21856 "name": "namespace", 21857 "options": [], 21858 "query": { 21859 "query": "label_values(ticdc_processor_processor_tick_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, namespace)", 21860 "refId": "StandardVariableQuery" 21861 }, 21862 "refresh": 2, 21863 "regex": "", 21864 "skipUrlSync": false, 21865 "sort": 0, 21866 "tagValuesQuery": "", 21867 "tags": [], 21868 "tagsQuery": "", 21869 "type": "query", 21870 "useTags": false 21871 }, 21872 { 21873 "allValue": ".*", 21874 "current": {}, 21875 "datasource": "${DS_TEST-CLUSTER}", 21876 "definition": "label_values(ticdc_processor_processor_tick_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, changefeed)", 21877 "description": null, 21878 "error": null, 21879 "hide": 0, 21880 "includeAll": true, 21881 "label": "Changefeed", 21882 "multi": true, 21883 "name": "changefeed", 21884 "options": [], 21885 "query": { 21886 "query": "label_values(ticdc_processor_processor_tick_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, changefeed)", 21887 "refId": "StandardVariableQuery" 21888 }, 21889 "refresh": 2, 21890 "regex": "", 21891 "skipUrlSync": false, 21892 "sort": 0, 21893 "tagValuesQuery": "", 21894 "tags": [], 21895 "tagsQuery": "", 21896 "type": "query", 21897 "useTags": false 21898 }, 21899 { 21900 "allValue": ".*", 21901 "current": {}, 21902 "datasource": "${DS_TEST-CLUSTER}", 21903 "definition": "label_values(process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}, instance)", 21904 "description": null, 21905 "error": null, 21906 "hide": 0, 21907 "includeAll": true, 21908 "label": "TiCDC", 21909 "multi": true, 21910 "name": "ticdc_instance", 21911 "options": [], 21912 "query": { 21913 "query": "label_values(process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}, instance)", 21914 "refId": "StandardVariableQuery" 21915 }, 21916 "refresh": 2, 21917 "regex": "", 21918 "skipUrlSync": false, 21919 "sort": 0, 21920 "tagValuesQuery": "", 21921 "tags": [], 21922 "tagsQuery": "", 21923 "type": "query", 21924 "useTags": false 21925 }, 21926 { 21927 "allValue": ".*", 21928 "current": {}, 21929 "datasource": "${DS_TEST-CLUSTER}", 21930 "definition": "label_values(tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, instance)", 21931 "description": null, 21932 "error": null, 21933 "hide": 0, 21934 "includeAll": true, 21935 "label": "TiKV", 21936 "multi": false, 21937 "name": "tikv_instance", 21938 "options": [], 21939 "query": { 21940 "query": "label_values(tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, instance)", 21941 "refId": "local-tikv_instance-Variable-Query" 21942 }, 21943 "refresh": 2, 21944 "regex": "", 21945 "skipUrlSync": false, 21946 "sort": 1, 21947 "tagValuesQuery": "", 21948 "tags": [], 21949 "tagsQuery": "", 21950 "type": "query", 21951 "useTags": false 21952 }, 21953 { 21954 "allValue": "9999999999", 21955 "current": { 21956 "selected": true, 21957 "text": "All", 21958 "value": "$__all" 21959 }, 21960 "description": null, 21961 "error": null, 21962 "hide": 0, 21963 "includeAll": true, 21964 "label": "Latency spike (s) >", 21965 "multi": false, 21966 "name": "spike_threshold", 21967 "options": [ 21968 { 21969 "selected": true, 21970 "text": "All", 21971 "value": "$__all" 21972 }, 21973 { 21974 "selected": false, 21975 "text": "1", 21976 "value": "1" 21977 }, 21978 { 21979 "selected": false, 21980 "text": "3", 21981 "value": "3" 21982 }, 21983 { 21984 "selected": false, 21985 "text": "5", 21986 "value": "5" 21987 }, 21988 { 21989 "selected": false, 21990 "text": "10", 21991 "value": "10" 21992 }, 21993 { 21994 "selected": false, 21995 "text": "60", 21996 "value": "60" 21997 }, 21998 { 21999 "selected": false, 22000 "text": "300", 22001 "value": "300" 22002 } 22003 ], 22004 "query": "1, 3, 5, 10, 60, 300", 22005 "queryValue": "", 22006 "skipUrlSync": false, 22007 "type": "custom" 22008 }, 22009 { 22010 "allValue": "", 22011 "current": {}, 22012 "datasource": "${DS_TEST-CLUSTER}", 22013 "definition": "label_values(process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}, instance)", 22014 "description": null, 22015 "error": null, 22016 "hide": 0, 22017 "includeAll": true, 22018 "label": "Runtime metrics", 22019 "multi": false, 22020 "name": "runtime_instance", 22021 "options": [], 22022 "query": { 22023 "query": "label_values(process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}, instance)", 22024 "refId": "local-runtime_instance-Variable-Query" 22025 }, 22026 "refresh": 2, 22027 "regex": "", 22028 "skipUrlSync": false, 22029 "sort": 0, 22030 "tagValuesQuery": "", 22031 "tags": [], 22032 "tagsQuery": "", 22033 "type": "query", 22034 "useTags": false 22035 }, 22036 { 22037 "allValue": null, 22038 "current": {}, 22039 "datasource": "${DS_TEST-CLUSTER}", 22040 "definition": "label_values(ticdc_actor_number_of_workers{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}, name)", 22041 "description": null, 22042 "error": null, 22043 "hide": 0, 22044 "includeAll": true, 22045 "label": "Actor", 22046 "multi": true, 22047 "name": "actor_name", 22048 "options": [], 22049 "query": { 22050 "query": "label_values(ticdc_actor_number_of_workers{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}, name)", 22051 "refId": "local-actor_name-Variable-Query" 22052 }, 22053 "refresh": 2, 22054 "regex": "", 22055 "skipUrlSync": false, 22056 "sort": 0, 22057 "tagValuesQuery": "", 22058 "tags": [], 22059 "tagsQuery": "", 22060 "type": "query", 22061 "useTags": false 22062 } 22063 ] 22064 }, 22065 "time": { 22066 "from": "now-3h", 22067 "to": "now" 22068 }, 22069 "timepicker": { 22070 "refresh_intervals": [ 22071 "5s", 22072 "10s", 22073 "30s", 22074 "1m", 22075 "5m", 22076 "15m", 22077 "30m", 22078 "1h", 22079 "2h", 22080 "1d" 22081 ], 22082 "time_options": [ 22083 "5m", 22084 "15m", 22085 "1h", 22086 "6h", 22087 "12h", 22088 "24h", 22089 "2d", 22090 "7d", 22091 "30d" 22092 ] 22093 }, 22094 "timezone": "browser", 22095 "title": "${DS_TEST-CLUSTER}-TiCDC", 22096 "uid": "YiGL8hBZ1", 22097 "version": 57 22098 }