github.com/pingcap/ticdc@v0.0.0-20220526033649-485a10ef2652/metrics/grafana/ticdc.json (about) 1 { 2 "__inputs": [ 3 { 4 "name": "DS_TEST-CLUSTER", 5 "label": "test-cluster", 6 "description": "", 7 "type": "datasource", 8 "pluginId": "prometheus", 9 "pluginName": "Prometheus" 10 } 11 ], 12 "__requires": [ 13 { 14 "type": "grafana", 15 "id": "grafana", 16 "name": "Grafana", 17 "version": "6.1.6" 18 }, 19 { 20 "type": "panel", 21 "id": "graph", 22 "name": "Graph", 23 "version": "" 24 }, 25 { 26 "type": "panel", 27 "id": "heatmap", 28 "name": "Heatmap", 29 "version": "" 30 }, 31 { 32 "type": "datasource", 33 "id": "prometheus", 34 "name": "Prometheus", 35 "version": "1.0.0" 36 }, 37 { 38 "type": "panel", 39 "id": "table", 40 "name": "Table", 41 "version": "" 42 } 43 ], 44 "annotations": { 45 "list": [ 46 { 47 "builtIn": 1, 48 "datasource": "-- Grafana --", 49 "enable": false, 50 "expr": "", 51 "hide": true, 52 "iconColor": "#F2495C", 53 "limit": 100, 54 "name": "", 55 "showIn": 0, 56 "tagKeys": "", 57 "textFormat": "", 58 "titleFormat": "", 59 "type": "dashboard", 60 "useValueForTime": false 61 }, 62 { 63 "datasource": "${DS_TEST-CLUSTER}", 64 "enable": true, 65 "expr": "max(ticdc_processor_checkpoint_ts_lag) by (changefeed, capture) > BOOL $spike_threshold", 66 "hide": true, 67 "iconColor": "#F2495C", 68 "limit": 100, 69 "name": "Latency spike", 70 "showIn": 0, 71 "tagKeys": "changefeed", 72 "tags": [], 73 "titleFormat": "Latency spike", 74 "type": "tags", 75 "useValueForTime": false 76 }, 77 { 78 "datasource": "${DS_TEST-CLUSTER}", 79 "enable": false, 80 "expr": "delta(up{tidb_cluster=\"$tidb_cluster\", job=~\"tikv|ticdc\"}[30s]) < BOOL 0", 81 "hide": false, 82 "iconColor": "#FF9830", 83 "limit": 100, 84 "name": "Server down", 85 "showIn": 0, 86 "step": "15s", 87 "tagKeys": "instance,job", 88 "tags": [], 89 "textFormat": "", 90 "titleFormat": "Down", 91 "type": "tags" 92 }, 93 { 94 "datasource": "${DS_TEST-CLUSTER}", 95 "enable": false, 96 "expr": "sum(ALERTS{tidb_cluster=\"$tidb_cluster\", alertstate=\"firing\", alertname=~\"ticdc.*\"}) by (alertname) > BOOL 0", 97 "hide": false, 98 "iconColor": "#B877D9", 99 "limit": 100, 100 "name": "All TiCDC alerts", 101 "showIn": 0, 102 "tagKeys": "alertname", 103 "tags": [], 104 "titleFormat": "Alert Name", 105 "type": "tags" 106 }, 107 { 108 "datasource": "${DS_TEST-CLUSTER}", 109 "enable": false, 110 "expr": "delta(tikv_cdc_region_resolve_status{status=\"resolved\"}[30s]) < BOOL -800", 111 "hide": false, 112 "iconColor": "rgba(255, 96, 96, 1)", 113 "limit": 100, 114 "name": "Resolved region drop", 115 "showIn": 0, 116 "step": "15s", 117 "tagKeys": "instance", 118 "tags": [], 119 "titleFormat": "Resolved region drop", 120 "type": "tags" 121 } 122 ] 123 }, 124 "editable": true, 125 "gnetId": null, 126 "graphTooltip": 1, 127 "id": null, 128 "iteration": 1630305196418, 129 "links": [], 130 "panels": [ 131 { 132 "collapsed": true, 133 "gridPos": { 134 "h": 1, 135 "w": 24, 136 "x": 0, 137 "y": 0 138 }, 139 "id": 21, 140 "panels": [ 141 { 142 "aliasColors": {}, 143 "bars": false, 144 "dashLength": 10, 145 "dashes": false, 146 "datasource": "${DS_TEST-CLUSTER}", 147 "description": "Uptime of TiCDC and TiKV", 148 "fill": 1, 149 "fillGradient": 0, 150 "gridPos": { 151 "h": 7, 152 "w": 8, 153 "x": 0, 154 "y": 1 155 }, 156 "hiddenSeries": false, 157 "id": 19, 158 "legend": { 159 "alignAsTable": true, 160 "avg": false, 161 "current": true, 162 "max": false, 163 "min": false, 164 "rightSide": true, 165 "show": true, 166 "total": false, 167 "values": true 168 }, 169 "lines": true, 170 "linewidth": 1, 171 "links": [], 172 "nullPointMode": "null", 173 "options": { 174 "dataLinks": [] 175 }, 176 "paceLength": 10, 177 "percentage": false, 178 "pointradius": 2, 179 "points": false, 180 "renderer": "flot", 181 "seriesOverrides": [], 182 "spaceLength": 10, 183 "stack": false, 184 "steppedLine": false, 185 "targets": [ 186 { 187 "expr": "(time() - process_start_time_seconds{tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"})", 188 "format": "time_series", 189 "intervalFactor": 1, 190 "legendFormat": "TiCDC - {{instance}}", 191 "refId": "A" 192 }, 193 { 194 "expr": "(time() - process_start_time_seconds{tidb_cluster=\"$tidb_cluster\", job=\"tikv\"})", 195 "format": "time_series", 196 "intervalFactor": 1, 197 "legendFormat": "TiKV - {{instance}}", 198 "refId": "B" 199 } 200 ], 201 "thresholds": [], 202 "timeFrom": null, 203 "timeRegions": [], 204 "timeShift": null, 205 "title": "Uptime", 206 "tooltip": { 207 "shared": true, 208 "sort": 0, 209 "value_type": "individual" 210 }, 211 "type": "graph", 212 "xaxis": { 213 "buckets": null, 214 "mode": "time", 215 "name": null, 216 "show": true, 217 "values": [] 218 }, 219 "yaxes": [ 220 { 221 "format": "dtdurations", 222 "label": null, 223 "logBase": 1, 224 "max": null, 225 "min": null, 226 "show": true 227 }, 228 { 229 "format": "short", 230 "label": null, 231 "logBase": 1, 232 "max": null, 233 "min": null, 234 "show": true 235 } 236 ], 237 "yaxis": { 238 "align": false, 239 "alignLevel": null 240 } 241 }, 242 { 243 "aliasColors": {}, 244 "bars": false, 245 "dashLength": 10, 246 "dashes": false, 247 "datasource": "${DS_TEST-CLUSTER}", 248 "description": "Goroutine count of TiCDC", 249 "fill": 1, 250 "fillGradient": 0, 251 "gridPos": { 252 "h": 7, 253 "w": 6, 254 "x": 8, 255 "y": 1 256 }, 257 "hiddenSeries": false, 258 "id": 26, 259 "legend": { 260 "alignAsTable": true, 261 "avg": false, 262 "current": true, 263 "max": true, 264 "min": false, 265 "rightSide": false, 266 "show": true, 267 "total": false, 268 "values": true 269 }, 270 "lines": true, 271 "linewidth": 1, 272 "links": [], 273 "nullPointMode": "null", 274 "options": { 275 "dataLinks": [] 276 }, 277 "paceLength": 10, 278 "percentage": false, 279 "pointradius": 2, 280 "points": false, 281 "renderer": "flot", 282 "seriesOverrides": [], 283 "spaceLength": 10, 284 "stack": false, 285 "steppedLine": false, 286 "targets": [ 287 { 288 "expr": " go_goroutines{tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}", 289 "format": "time_series", 290 "intervalFactor": 1, 291 "legendFormat": "{{instance}}", 292 "refId": "A" 293 }, 294 { 295 "expr": "go_threads{tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}", 296 "format": "time_series", 297 "hide": true, 298 "intervalFactor": 1, 299 "legendFormat": "threads-{{instance}}", 300 "refId": "B" 301 } 302 ], 303 "thresholds": [], 304 "timeFrom": null, 305 "timeRegions": [], 306 "timeShift": null, 307 "title": "Goroutine count", 308 "tooltip": { 309 "shared": true, 310 "sort": 0, 311 "value_type": "individual" 312 }, 313 "type": "graph", 314 "xaxis": { 315 "buckets": null, 316 "mode": "time", 317 "name": null, 318 "show": true, 319 "values": [] 320 }, 321 "yaxes": [ 322 { 323 "format": "short", 324 "label": null, 325 "logBase": 1, 326 "max": null, 327 "min": null, 328 "show": true 329 }, 330 { 331 "format": "short", 332 "label": null, 333 "logBase": 1, 334 "max": null, 335 "min": null, 336 "show": true 337 } 338 ], 339 "yaxis": { 340 "align": false, 341 "alignLevel": null 342 } 343 }, 344 { 345 "aliasColors": {}, 346 "bars": false, 347 "dashLength": 10, 348 "dashes": false, 349 "datasource": "${DS_TEST-CLUSTER}", 350 "description": "The count of open FD count of TiCDC", 351 "fill": 1, 352 "fillGradient": 0, 353 "gridPos": { 354 "h": 7, 355 "w": 6, 356 "x": 14, 357 "y": 1 358 }, 359 "hiddenSeries": false, 360 "id": 27, 361 "legend": { 362 "alignAsTable": true, 363 "avg": false, 364 "current": true, 365 "max": true, 366 "min": false, 367 "rightSide": false, 368 "show": true, 369 "total": false, 370 "values": true 371 }, 372 "lines": true, 373 "linewidth": 1, 374 "links": [], 375 "nullPointMode": "null", 376 "options": { 377 "dataLinks": [] 378 }, 379 "paceLength": 10, 380 "percentage": false, 381 "pointradius": 2, 382 "points": false, 383 "renderer": "flot", 384 "seriesOverrides": [], 385 "spaceLength": 10, 386 "stack": false, 387 "steppedLine": false, 388 "targets": [ 389 { 390 "expr": "process_open_fds{tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}", 391 "format": "time_series", 392 "intervalFactor": 1, 393 "legendFormat": "{{instance}}", 394 "refId": "A" 395 } 396 ], 397 "thresholds": [], 398 "timeFrom": null, 399 "timeRegions": [], 400 "timeShift": null, 401 "title": "Open FD count", 402 "tooltip": { 403 "shared": true, 404 "sort": 0, 405 "value_type": "individual" 406 }, 407 "type": "graph", 408 "xaxis": { 409 "buckets": null, 410 "mode": "time", 411 "name": null, 412 "show": true, 413 "values": [] 414 }, 415 "yaxes": [ 416 { 417 "format": "short", 418 "label": null, 419 "logBase": 1, 420 "max": null, 421 "min": null, 422 "show": true 423 }, 424 { 425 "format": "short", 426 "label": null, 427 "logBase": 1, 428 "max": null, 429 "min": null, 430 "show": true 431 } 432 ], 433 "yaxis": { 434 "align": false, 435 "alignLevel": null 436 } 437 }, 438 { 439 "columns": [ 440 { 441 "text": "Current", 442 "value": "current" 443 } 444 ], 445 "datasource": "${DS_TEST-CLUSTER}", 446 "description": "TiCDC cluster ownership status", 447 "fontSize": "100%", 448 "gridPos": { 449 "h": 7, 450 "w": 4, 451 "x": 20, 452 "y": 1 453 }, 454 "hideTimeOverride": true, 455 "id": 113, 456 "links": [], 457 "pageSize": null, 458 "scroll": true, 459 "showHeader": true, 460 "sort": { 461 "col": 2, 462 "desc": false 463 }, 464 "styles": [ 465 { 466 "alias": "Instance", 467 "colorMode": null, 468 "colors": [ 469 "#8AB8FF", 470 "#73BF69", 471 "rgba(245, 54, 54, 0.9)" 472 ], 473 "dateFormat": "YYYY-MM-DD HH:mm:ss", 474 "decimals": 2, 475 "link": false, 476 "mappingType": 2, 477 "pattern": "Current", 478 "preserveFormat": false, 479 "rangeMaps": [ 480 { 481 "from": "0.1", 482 "text": "Owner", 483 "to": "2" 484 }, 485 { 486 "from": "0", 487 "text": "Worker", 488 "to": "0.1" 489 } 490 ], 491 "sanitize": false, 492 "thresholds": [ 493 "0.1", 494 "2" 495 ], 496 "type": "string", 497 "unit": "short", 498 "valueMaps": [ 499 { 500 "text": "Owner", 501 "value": "1" 502 } 503 ] 504 } 505 ], 506 "targets": [ 507 { 508 "expr": "rate(ticdc_owner_ownership_counter{tidb_cluster=\"$tidb_cluster\"}[30s])", 509 "format": "time_series", 510 "intervalFactor": 2, 511 "legendFormat": "{{instance}}", 512 "refId": "A" 513 } 514 ], 515 "timeFrom": "1s", 516 "title": "Ownership", 517 "transform": "timeseries_aggregations", 518 "type": "table" 519 }, 520 { 521 "aliasColors": {}, 522 "bars": false, 523 "dashLength": 10, 524 "dashes": false, 525 "datasource": "${DS_TEST-CLUSTER}", 526 "description": "CPU usage of TiCDC", 527 "fill": 1, 528 "fillGradient": 0, 529 "gridPos": { 530 "h": 7, 531 "w": 6, 532 "x": 0, 533 "y": 8 534 }, 535 "hiddenSeries": false, 536 "id": 24, 537 "legend": { 538 "alignAsTable": true, 539 "avg": false, 540 "current": true, 541 "hideEmpty": true, 542 "max": true, 543 "min": false, 544 "rightSide": false, 545 "show": true, 546 "total": false, 547 "values": true 548 }, 549 "lines": true, 550 "linewidth": 1, 551 "links": [], 552 "nullPointMode": "null", 553 "options": { 554 "dataLinks": [] 555 }, 556 "paceLength": 10, 557 "percentage": false, 558 "pointradius": 2, 559 "points": false, 560 "renderer": "flot", 561 "seriesOverrides": [], 562 "spaceLength": 10, 563 "stack": false, 564 "steppedLine": false, 565 "targets": [ 566 { 567 "expr": "rate(process_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}[1m])", 568 "format": "time_series", 569 "intervalFactor": 1, 570 "legendFormat": "{{instance}}", 571 "refId": "A" 572 } 573 ], 574 "thresholds": [], 575 "timeFrom": null, 576 "timeRegions": [], 577 "timeShift": null, 578 "title": "CPU usage", 579 "tooltip": { 580 "shared": true, 581 "sort": 0, 582 "value_type": "individual" 583 }, 584 "type": "graph", 585 "xaxis": { 586 "buckets": null, 587 "mode": "time", 588 "name": null, 589 "show": true, 590 "values": [] 591 }, 592 "yaxes": [ 593 { 594 "format": "percentunit", 595 "label": null, 596 "logBase": 1, 597 "max": null, 598 "min": null, 599 "show": true 600 }, 601 { 602 "format": "short", 603 "label": null, 604 "logBase": 1, 605 "max": null, 606 "min": null, 607 "show": true 608 } 609 ], 610 "yaxis": { 611 "align": false, 612 "alignLevel": null 613 } 614 }, 615 { 616 "aliasColors": {}, 617 "bars": false, 618 "dashLength": 10, 619 "dashes": false, 620 "datasource": "${DS_TEST-CLUSTER}", 621 "description": "Memory usage of TiCDC", 622 "fill": 1, 623 "fillGradient": 0, 624 "gridPos": { 625 "h": 7, 626 "w": 6, 627 "x": 6, 628 "y": 8 629 }, 630 "hiddenSeries": false, 631 "id": 23, 632 "legend": { 633 "alignAsTable": true, 634 "avg": false, 635 "current": true, 636 "hideEmpty": true, 637 "max": true, 638 "min": false, 639 "rightSide": false, 640 "show": true, 641 "total": false, 642 "values": true 643 }, 644 "lines": true, 645 "linewidth": 1, 646 "links": [], 647 "nullPointMode": "null", 648 "options": { 649 "dataLinks": [] 650 }, 651 "paceLength": 10, 652 "percentage": false, 653 "pointradius": 2, 654 "points": false, 655 "renderer": "flot", 656 "seriesOverrides": [], 657 "spaceLength": 10, 658 "stack": false, 659 "steppedLine": false, 660 "targets": [ 661 { 662 "expr": "process_resident_memory_bytes{tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}", 663 "format": "time_series", 664 "intervalFactor": 1, 665 "legendFormat": "process-{{instance}}", 666 "refId": "A" 667 }, 668 { 669 "expr": "go_memstats_heap_alloc_bytes{tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}", 670 "format": "time_series", 671 "intervalFactor": 1, 672 "legendFormat": "heap-{{instance}}", 673 "refId": "B" 674 } 675 ], 676 "thresholds": [], 677 "timeFrom": null, 678 "timeRegions": [], 679 "timeShift": null, 680 "title": "Memory usage", 681 "tooltip": { 682 "shared": true, 683 "sort": 0, 684 "value_type": "individual" 685 }, 686 "type": "graph", 687 "xaxis": { 688 "buckets": null, 689 "mode": "time", 690 "name": null, 691 "show": true, 692 "values": [] 693 }, 694 "yaxes": [ 695 { 696 "format": "bytes", 697 "label": null, 698 "logBase": 1, 699 "max": null, 700 "min": null, 701 "show": true 702 }, 703 { 704 "format": "short", 705 "label": null, 706 "logBase": 1, 707 "max": null, 708 "min": null, 709 "show": true 710 } 711 ], 712 "yaxis": { 713 "align": false, 714 "alignLevel": null 715 } 716 }, 717 { 718 "aliasColors": {}, 719 "bars": false, 720 "dashLength": 10, 721 "dashes": false, 722 "datasource": "${DS_TEST-CLUSTER}", 723 "description": "The history of TiCDC cluster ownership, owner node has a value that is great than 0", 724 "fill": 1, 725 "gridPos": { 726 "h": 7, 727 "w": 6, 728 "x": 12, 729 "y": 8 730 }, 731 "id": 110, 732 "legend": { 733 "alignAsTable": true, 734 "avg": false, 735 "current": true, 736 "max": false, 737 "min": false, 738 "rightSide": false, 739 "show": true, 740 "total": false, 741 "values": true 742 }, 743 "lines": true, 744 "linewidth": 1, 745 "links": [], 746 "nullPointMode": "null", 747 "percentage": false, 748 "pointradius": 2, 749 "points": false, 750 "renderer": "flot", 751 "seriesOverrides": [], 752 "spaceLength": 10, 753 "stack": false, 754 "steppedLine": true, 755 "targets": [ 756 { 757 "expr": "sum(rate(ticdc_owner_ownership_counter{tidb_cluster=\"$tidb_cluster\"}[30s])) by (instance) > BOOL 0.5", 758 "format": "time_series", 759 "interval": "30s", 760 "intervalFactor": 1, 761 "legendFormat": "{{instance}}", 762 "refId": "A" 763 } 764 ], 765 "thresholds": [], 766 "timeFrom": null, 767 "timeRegions": [], 768 "timeShift": null, 769 "title": "Ownership history", 770 "tooltip": { 771 "shared": true, 772 "sort": 0, 773 "value_type": "individual" 774 }, 775 "type": "graph", 776 "xaxis": { 777 "buckets": null, 778 "mode": "time", 779 "name": null, 780 "show": true, 781 "values": [] 782 }, 783 "yaxes": [ 784 { 785 "format": "short", 786 "label": null, 787 "logBase": 1, 788 "max": null, 789 "min": null, 790 "show": true 791 }, 792 { 793 "format": "short", 794 "label": null, 795 "logBase": 1, 796 "max": null, 797 "min": null, 798 "show": true 799 } 800 ], 801 "yaxis": { 802 "align": false, 803 "alignLevel": null 804 } 805 }, 806 { 807 "aliasColors": {}, 808 "bars": false, 809 "dashLength": 10, 810 "dashes": false, 811 "datasource": "${DS_TEST-CLUSTER}", 812 "fill": 1, 813 "fillGradient": 0, 814 "gridPos": { 815 "h": 7, 816 "w": 6, 817 "x": 18, 818 "y": 8 819 }, 820 "hiddenSeries": false, 821 "id": 114, 822 "legend": { 823 "alignAsTable": true, 824 "avg": false, 825 "current": true, 826 "hideEmpty": true, 827 "max": true, 828 "min": false, 829 "rightSide": true, 830 "show": true, 831 "total": false, 832 "values": true 833 }, 834 "lines": true, 835 "linewidth": 1, 836 "links": [], 837 "nullPointMode": "null", 838 "options": { 839 "dataLinks": [] 840 }, 841 "paceLength": 10, 842 "percentage": false, 843 "pointradius": 2, 844 "points": false, 845 "renderer": "flot", 846 "seriesOverrides": [], 847 "spaceLength": 10, 848 "stack": false, 849 "steppedLine": false, 850 "targets": [ 851 { 852 "expr": "histogram_quantile(0.999, sum(rate(ticdc_server_etcd_health_check_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,instance))", 853 "format": "time_series", 854 "intervalFactor": 1, 855 "legendFormat": "p999-{{instance}}", 856 "refId": "A" 857 }, 858 { 859 "expr": "histogram_quantile(0.99, sum(rate(ticdc_server_etcd_health_check_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,instance))", 860 "format": "time_series", 861 "intervalFactor": 1, 862 "legendFormat": "p99-{{instance}}", 863 "refId": "B" 864 }, 865 { 866 "expr": "histogram_quantile(0.95, sum(rate(ticdc_server_etcd_health_check_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le,instance))", 867 "format": "time_series", 868 "intervalFactor": 1, 869 "legendFormat": "p95-{{instance}}", 870 "refId": "C" 871 } 872 ], 873 "thresholds": [], 874 "timeFrom": null, 875 "timeRegions": [], 876 "timeShift": null, 877 "title": "Etcd health check duration", 878 "tooltip": { 879 "shared": true, 880 "sort": 0, 881 "value_type": "individual" 882 }, 883 "type": "graph", 884 "xaxis": { 885 "buckets": null, 886 "mode": "time", 887 "name": null, 888 "show": true, 889 "values": [] 890 }, 891 "yaxes": [ 892 { 893 "format": "s", 894 "label": null, 895 "logBase": 1, 896 "max": null, 897 "min": null, 898 "show": true 899 }, 900 { 901 "format": "short", 902 "label": null, 903 "logBase": 1, 904 "max": null, 905 "min": null, 906 "show": true 907 } 908 ], 909 "yaxis": { 910 "align": false, 911 "alignLevel": null 912 } 913 } 914 ], 915 "title": "Server", 916 "type": "row" 917 }, 918 { 919 "collapsed": true, 920 "gridPos": { 921 "h": 1, 922 "w": 24, 923 "x": 0, 924 "y": 1 925 }, 926 "id": 11, 927 "panels": [ 928 { 929 "columns": [ 930 { 931 "text": "Current", 932 "value": "current" 933 } 934 ], 935 "datasource": "${DS_TEST-CLUSTER}", 936 "description": "The number of captured table of TiCDC nodes ", 937 "fontSize": "100%", 938 "gridPos": { 939 "h": 5, 940 "w": 7, 941 "x": 0, 942 "y": 2 943 }, 944 "id": 4, 945 "links": [], 946 "pageSize": null, 947 "scroll": true, 948 "showHeader": true, 949 "sort": { 950 "col": 0, 951 "desc": true 952 }, 953 "styles": [ 954 { 955 "alias": "Time", 956 "dateFormat": "YYYY-MM-DD HH:mm:ss", 957 "pattern": "Time", 958 "type": "date" 959 }, 960 { 961 "alias": "", 962 "colorMode": null, 963 "colors": [ 964 "rgba(245, 54, 54, 0.9)", 965 "rgba(237, 129, 40, 0.89)", 966 "rgba(50, 172, 45, 0.97)" 967 ], 968 "decimals": 2, 969 "pattern": "/.*/", 970 "thresholds": [], 971 "type": "number", 972 "unit": "short" 973 } 974 ], 975 "targets": [ 976 { 977 "expr": "sum(ticdc_processor_num_of_tables{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", 978 "format": "time_series", 979 "interval": "", 980 "intervalFactor": 1, 981 "legendFormat": "{{capture}}", 982 "refId": "A" 983 } 984 ], 985 "timeFrom": null, 986 "timeShift": null, 987 "title": "Changefeed table count", 988 "transform": "timeseries_aggregations", 989 "type": "table" 990 }, 991 { 992 "columns": [ 993 { 994 "text": "Current", 995 "value": "current" 996 } 997 ], 998 "datasource": "${DS_TEST-CLUSTER}", 999 "description": "Internal resolved ts of TiCDC nodes", 1000 "fontSize": "100%", 1001 "gridPos": { 1002 "h": 10, 1003 "w": 7, 1004 "x": 7, 1005 "y": 2 1006 }, 1007 "id": 90, 1008 "links": [], 1009 "pageSize": null, 1010 "scroll": true, 1011 "showHeader": true, 1012 "sort": { 1013 "col": 0, 1014 "desc": true 1015 }, 1016 "styles": [ 1017 { 1018 "alias": "table", 1019 "colorMode": null, 1020 "colors": [ 1021 "rgba(245, 54, 54, 0.9)", 1022 "rgba(237, 129, 40, 0.89)", 1023 "rgba(50, 172, 45, 0.97)" 1024 ], 1025 "dateFormat": "YYYY-MM-DD HH:mm:ss", 1026 "decimals": 2, 1027 "mappingType": 1, 1028 "pattern": "Metric", 1029 "thresholds": [], 1030 "type": "string", 1031 "unit": "short" 1032 }, 1033 { 1034 "alias": "resolved ts", 1035 "colorMode": null, 1036 "colors": [ 1037 "rgba(245, 54, 54, 0.9)", 1038 "rgba(237, 129, 40, 0.89)", 1039 "rgba(50, 172, 45, 0.97)" 1040 ], 1041 "dateFormat": "MM-DD HH:mm:ss.SSS", 1042 "decimals": 2, 1043 "pattern": "Current", 1044 "thresholds": [], 1045 "type": "date", 1046 "unit": "short" 1047 } 1048 ], 1049 "targets": [ 1050 { 1051 "expr": "max(ticdc_processor_resolved_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed)", 1052 "format": "time_series", 1053 "interval": "", 1054 "intervalFactor": 1, 1055 "legendFormat": "{{capture}}--{{changefeed}}", 1056 "refId": "A" 1057 }, 1058 { 1059 "expr": "max(ticdc_processor_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed) > 0", 1060 "format": "time_series", 1061 "interval": "", 1062 "intervalFactor": 1, 1063 "legendFormat": "checkpoint-{{capture}}--{{changefeed}}", 1064 "refId": "B" 1065 } 1066 ], 1067 "timeFrom": null, 1068 "timeShift": null, 1069 "title": "Processor resolved ts", 1070 "transform": "timeseries_aggregations", 1071 "type": "table" 1072 }, 1073 { 1074 "columns": [ 1075 { 1076 "text": "Current", 1077 "value": "current" 1078 } 1079 ], 1080 "datasource": "${DS_TEST-CLUSTER}", 1081 "description": "Internal resolved ts of captured tables", 1082 "fontSize": "100%", 1083 "gridPos": { 1084 "h": 10, 1085 "w": 10, 1086 "x": 14, 1087 "y": 2 1088 }, 1089 "id": 30, 1090 "links": [], 1091 "pageSize": null, 1092 "scroll": true, 1093 "showHeader": true, 1094 "sort": { 1095 "col": 0, 1096 "desc": true 1097 }, 1098 "styles": [ 1099 { 1100 "alias": "table", 1101 "colorMode": null, 1102 "colors": [ 1103 "rgba(245, 54, 54, 0.9)", 1104 "rgba(237, 129, 40, 0.89)", 1105 "rgba(50, 172, 45, 0.97)" 1106 ], 1107 "dateFormat": "YYYY-MM-DD HH:mm:ss", 1108 "decimals": 2, 1109 "mappingType": 1, 1110 "pattern": "Metric", 1111 "thresholds": [], 1112 "type": "string", 1113 "unit": "short" 1114 }, 1115 { 1116 "alias": "resolved ts", 1117 "colorMode": null, 1118 "colors": [ 1119 "rgba(245, 54, 54, 0.9)", 1120 "rgba(237, 129, 40, 0.89)", 1121 "rgba(50, 172, 45, 0.97)" 1122 ], 1123 "dateFormat": "MM-DD HH:mm:ss.SSS", 1124 "decimals": 2, 1125 "pattern": "Current", 1126 "thresholds": [], 1127 "type": "date", 1128 "unit": "short" 1129 } 1130 ], 1131 "targets": [ 1132 { 1133 "expr": "max(ticdc_processor_table_resolved_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed,table)", 1134 "format": "time_series", 1135 "instant": true, 1136 "interval": "", 1137 "intervalFactor": 1, 1138 "legendFormat": "{{capture}}-{{changefeed}}-{{table}}", 1139 "refId": "A" 1140 }, 1141 { 1142 "expr": "max(ticdc_processor_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed,table) > 0", 1143 "format": "time_series", 1144 "interval": "", 1145 "intervalFactor": 1, 1146 "legendFormat": "checkpoint-{{changefeed}}", 1147 "refId": "B" 1148 } 1149 ], 1150 "timeFrom": null, 1151 "timeShift": null, 1152 "title": "Table resolved ts", 1153 "transform": "timeseries_aggregations", 1154 "type": "table" 1155 }, 1156 { 1157 "columns": [ 1158 { 1159 "text": "Current", 1160 "value": "current" 1161 } 1162 ], 1163 "datasource": "${DS_TEST-CLUSTER}", 1164 "description": "The number of replicated tables maintained in owner", 1165 "fontSize": "100%", 1166 "gridPos": { 1167 "h": 5, 1168 "w": 7, 1169 "x": 0, 1170 "y": 7 1171 }, 1172 "id": 138, 1173 "links": [], 1174 "pageSize": null, 1175 "scroll": true, 1176 "showHeader": true, 1177 "sort": { 1178 "col": null, 1179 "desc": false 1180 }, 1181 "styles": [ 1182 { 1183 "alias": "Time", 1184 "align": "auto", 1185 "dateFormat": "YYYY-MM-DD HH:mm:ss", 1186 "pattern": "Time", 1187 "type": "date" 1188 }, 1189 { 1190 "alias": "", 1191 "align": "auto", 1192 "colorMode": null, 1193 "colors": [ 1194 "rgba(245, 54, 54, 0.9)", 1195 "rgba(237, 129, 40, 0.89)", 1196 "rgba(50, 172, 45, 0.97)" 1197 ], 1198 "decimals": 2, 1199 "pattern": "/.*/", 1200 "thresholds": [], 1201 "type": "number", 1202 "unit": "short" 1203 } 1204 ], 1205 "targets": [ 1206 { 1207 "expr": "sum(ticdc_owner_maintain_table_num{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\",type=\"total\"}) by (capture)", 1208 "format": "time_series", 1209 "interval": "", 1210 "intervalFactor": 1, 1211 "legendFormat": "{{capture}}-total", 1212 "refId": "A" 1213 }, 1214 { 1215 "expr": "sum(ticdc_owner_maintain_table_num{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\",type=\"wip\"}) by (capture)", 1216 "format": "time_series", 1217 "interval": "", 1218 "intervalFactor": 1, 1219 "legendFormat": "{{capture}}-wip", 1220 "refId": "B" 1221 } 1222 ], 1223 "timeFrom": null, 1224 "timeShift": null, 1225 "title": "Table count maintained by owner", 1226 "transform": "timeseries_aggregations", 1227 "type": "table" 1228 }, 1229 { 1230 "aliasColors": {}, 1231 "bars": true, 1232 "cacheTimeout": null, 1233 "dashLength": 10, 1234 "dashes": false, 1235 "datasource": "${DS_TEST-CLUSTER}", 1236 "description": "The checkpoint ts of changefeeds.", 1237 "fill": 0, 1238 "gridPos": { 1239 "h": 7, 1240 "w": 9, 1241 "x": 0, 1242 "y": 12 1243 }, 1244 "id": 86, 1245 "legend": { 1246 "alignAsTable": true, 1247 "avg": false, 1248 "current": true, 1249 "max": false, 1250 "min": false, 1251 "rightSide": false, 1252 "show": true, 1253 "sideWidth": null, 1254 "total": false, 1255 "values": true 1256 }, 1257 "lines": true, 1258 "linewidth": 2, 1259 "links": [], 1260 "nullPointMode": "null", 1261 "percentage": false, 1262 "pointradius": 2, 1263 "points": false, 1264 "renderer": "flot", 1265 "seriesOverrides": [ 1266 { 1267 "alias": "/approximate current time.*/", 1268 "bars": false 1269 } 1270 ], 1271 "spaceLength": 10, 1272 "stack": false, 1273 "steppedLine": false, 1274 "targets": [ 1275 { 1276 "expr": "max(ticdc_owner_checkpoint_ts{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (changefeed) > 0", 1277 "format": "time_series", 1278 "interval": "", 1279 "intervalFactor": 1, 1280 "legendFormat": "{{changefeed}}", 1281 "refId": "A" 1282 }, 1283 { 1284 "expr": "max(pd_cluster_tso{tidb_cluster=\"$tidb_cluster\"}) * 1000", 1285 "format": "time_series", 1286 "hide": false, 1287 "interval": "", 1288 "intervalFactor": 1, 1289 "legendFormat": "approximate current time (s)", 1290 "refId": "B" 1291 } 1292 ], 1293 "thresholds": [], 1294 "timeFrom": null, 1295 "timeRegions": [], 1296 "timeShift": null, 1297 "title": "Changefeed checkpoint", 1298 "tooltip": { 1299 "shared": true, 1300 "sort": 0, 1301 "value_type": "individual" 1302 }, 1303 "type": "graph", 1304 "xaxis": { 1305 "buckets": null, 1306 "max": null, 1307 "mode": "time", 1308 "name": null, 1309 "show": true, 1310 "values": [] 1311 }, 1312 "yaxes": [ 1313 { 1314 "format": "dateTimeAsIso", 1315 "label": null, 1316 "logBase": 1, 1317 "max": null, 1318 "min": null, 1319 "show": true 1320 }, 1321 { 1322 "format": "short", 1323 "label": null, 1324 "logBase": 1, 1325 "max": null, 1326 "min": null, 1327 "show": false 1328 } 1329 ], 1330 "yaxis": { 1331 "align": false, 1332 "alignLevel": null 1333 } 1334 }, 1335 { 1336 "aliasColors": {}, 1337 "bars": false, 1338 "dashLength": 10, 1339 "dashes": false, 1340 "datasource": "${DS_TEST-CLUSTER}", 1341 "description": "Request count of etcd operation per second", 1342 "fill": 1, 1343 "fillGradient": 0, 1344 "gridPos": { 1345 "h": 7, 1346 "w": 8, 1347 "x": 9, 1348 "y": 12 1349 }, 1350 "hiddenSeries": false, 1351 "id": 102, 1352 "legend": { 1353 "alignAsTable": true, 1354 "avg": false, 1355 "current": true, 1356 "hideEmpty": true, 1357 "hideZero": true, 1358 "max": true, 1359 "min": false, 1360 "rightSide": true, 1361 "show": true, 1362 "total": false, 1363 "values": true 1364 }, 1365 "lines": true, 1366 "linewidth": 1, 1367 "links": [], 1368 "nullPointMode": "null", 1369 "options": { 1370 "dataLinks": [] 1371 }, 1372 "paceLength": 10, 1373 "percentage": false, 1374 "pointradius": 2, 1375 "points": false, 1376 "renderer": "flot", 1377 "seriesOverrides": [], 1378 "spaceLength": 10, 1379 "stack": false, 1380 "steppedLine": false, 1381 "targets": [ 1382 { 1383 "expr": "sum(rate(ticdc_etcd_request_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (capture, type)", 1384 "format": "time_series", 1385 "interval": "", 1386 "intervalFactor": 1, 1387 "legendFormat": "{{capture}}-{{type}}", 1388 "refId": "A" 1389 } 1390 ], 1391 "thresholds": [], 1392 "timeFrom": null, 1393 "timeRegions": [], 1394 "timeShift": null, 1395 "title": "PD etcd requests/s", 1396 "tooltip": { 1397 "shared": true, 1398 "sort": 0, 1399 "value_type": "individual" 1400 }, 1401 "type": "graph", 1402 "xaxis": { 1403 "buckets": null, 1404 "mode": "time", 1405 "name": null, 1406 "show": true, 1407 "values": [] 1408 }, 1409 "yaxes": [ 1410 { 1411 "format": "short", 1412 "label": null, 1413 "logBase": 1, 1414 "max": null, 1415 "min": null, 1416 "show": true 1417 }, 1418 { 1419 "format": "short", 1420 "label": null, 1421 "logBase": 1, 1422 "max": null, 1423 "min": null, 1424 "show": true 1425 } 1426 ], 1427 "yaxis": { 1428 "align": false, 1429 "alignLevel": null 1430 } 1431 }, 1432 { 1433 "aliasColors": {}, 1434 "bars": false, 1435 "dashLength": 10, 1436 "dashes": false, 1437 "datasource": "${DS_TEST-CLUSTER}", 1438 "description": "The number of errors that interrupt changefeed per minute ", 1439 "fill": 1, 1440 "gridPos": { 1441 "h": 7, 1442 "w": 7, 1443 "x": 17, 1444 "y": 12 1445 }, 1446 "id": 82, 1447 "legend": { 1448 "alignAsTable": true, 1449 "avg": false, 1450 "current": true, 1451 "max": false, 1452 "min": false, 1453 "rightSide": true, 1454 "show": true, 1455 "total": false, 1456 "values": true 1457 }, 1458 "lines": true, 1459 "linewidth": 1, 1460 "links": [], 1461 "nullPointMode": "null", 1462 "percentage": false, 1463 "pointradius": 2, 1464 "points": false, 1465 "renderer": "flot", 1466 "seriesOverrides": [], 1467 "spaceLength": 10, 1468 "stack": false, 1469 "steppedLine": false, 1470 "targets": [ 1471 { 1472 "expr": "sum(delta(ticdc_processor_exit_with_error_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture)", 1473 "format": "time_series", 1474 "intervalFactor": 1, 1475 "legendFormat": "{{capture}}", 1476 "refId": "A" 1477 } 1478 ], 1479 "thresholds": [], 1480 "timeFrom": null, 1481 "timeRegions": [], 1482 "timeShift": null, 1483 "title": "Exit error count/m", 1484 "tooltip": { 1485 "shared": true, 1486 "sort": 0, 1487 "value_type": "individual" 1488 }, 1489 "type": "graph", 1490 "xaxis": { 1491 "buckets": null, 1492 "mode": "time", 1493 "name": null, 1494 "show": true, 1495 "values": [] 1496 }, 1497 "yaxes": [ 1498 { 1499 "format": "short", 1500 "label": null, 1501 "logBase": 1, 1502 "max": null, 1503 "min": null, 1504 "show": true 1505 }, 1506 { 1507 "format": "short", 1508 "label": null, 1509 "logBase": 1, 1510 "max": null, 1511 "min": null, 1512 "show": true 1513 } 1514 ], 1515 "yaxis": { 1516 "align": false, 1517 "alignLevel": null 1518 } 1519 }, 1520 { 1521 "aliasColors": {}, 1522 "bars": false, 1523 "dashLength": 10, 1524 "dashes": false, 1525 "datasource": "${DS_TEST-CLUSTER}", 1526 "description": "The lag between changefeed checkpoint ts and the latest ts of upstream TiDB.", 1527 "fill": 1, 1528 "fillGradient": 0, 1529 "gridPos": { 1530 "h": 7, 1531 "w": 12, 1532 "x": 0, 1533 "y": 19 1534 }, 1535 "hiddenSeries": false, 1536 "id": 3, 1537 "legend": { 1538 "alignAsTable": true, 1539 "avg": false, 1540 "current": true, 1541 "max": false, 1542 "min": false, 1543 "rightSide": true, 1544 "show": true, 1545 "total": false, 1546 "values": true 1547 }, 1548 "lines": true, 1549 "linewidth": 1, 1550 "links": [], 1551 "nullPointMode": "null", 1552 "options": { 1553 "dataLinks": [] 1554 }, 1555 "paceLength": 10, 1556 "percentage": false, 1557 "pointradius": 2, 1558 "points": false, 1559 "renderer": "flot", 1560 "seriesOverrides": [], 1561 "spaceLength": 10, 1562 "stack": false, 1563 "steppedLine": false, 1564 "targets": [ 1565 { 1566 "expr": "max(ticdc_owner_checkpoint_ts_lag{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}) by (changefeed)", 1567 "format": "time_series", 1568 "interval": "", 1569 "intervalFactor": 1, 1570 "legendFormat": "{{changefeed}}", 1571 "refId": "A" 1572 } 1573 ], 1574 "thresholds": [], 1575 "timeFrom": null, 1576 "timeRegions": [], 1577 "timeShift": null, 1578 "title": "Changefeed checkpoint lag", 1579 "tooltip": { 1580 "shared": true, 1581 "sort": 0, 1582 "value_type": "individual" 1583 }, 1584 "type": "graph", 1585 "xaxis": { 1586 "buckets": null, 1587 "mode": "time", 1588 "name": null, 1589 "show": true, 1590 "values": [] 1591 }, 1592 "yaxes": [ 1593 { 1594 "format": "s", 1595 "label": null, 1596 "logBase": 1, 1597 "max": null, 1598 "min": "0", 1599 "show": true 1600 }, 1601 { 1602 "format": "short", 1603 "label": null, 1604 "logBase": 1, 1605 "max": null, 1606 "min": null, 1607 "show": true 1608 } 1609 ], 1610 "yaxis": { 1611 "align": false, 1612 "alignLevel": null 1613 } 1614 }, 1615 { 1616 "aliasColors": {}, 1617 "bars": false, 1618 "dashLength": 10, 1619 "dashes": false, 1620 "datasource": "${DS_TEST-CLUSTER}", 1621 "description": "The lag between internal resolved ts and the latest ts of upstream TiDB.", 1622 "fill": 1, 1623 "fillGradient": 0, 1624 "gridPos": { 1625 "h": 7, 1626 "w": 12, 1627 "x": 12, 1628 "y": 19 1629 }, 1630 "hiddenSeries": false, 1631 "id": 2, 1632 "legend": { 1633 "alignAsTable": true, 1634 "avg": false, 1635 "current": true, 1636 "hideEmpty": false, 1637 "hideZero": false, 1638 "max": false, 1639 "min": false, 1640 "rightSide": true, 1641 "show": true, 1642 "total": false, 1643 "values": true 1644 }, 1645 "lines": true, 1646 "linewidth": 1, 1647 "links": [], 1648 "nullPointMode": "null", 1649 "options": { 1650 "dataLinks": [] 1651 }, 1652 "paceLength": 10, 1653 "percentage": false, 1654 "pointradius": 2, 1655 "points": false, 1656 "renderer": "flot", 1657 "seriesOverrides": [], 1658 "spaceLength": 10, 1659 "stack": false, 1660 "steppedLine": false, 1661 "targets": [ 1662 { 1663 "expr": "sum(ticdc_processor_resolved_ts_lag{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture,changefeed)", 1664 "format": "time_series", 1665 "interval": "", 1666 "intervalFactor": 1, 1667 "legendFormat": "{{capture}}--{{changefeed}}", 1668 "refId": "A" 1669 } 1670 ], 1671 "thresholds": [], 1672 "timeFrom": null, 1673 "timeRegions": [], 1674 "timeShift": null, 1675 "title": "Processor resolved ts lag", 1676 "tooltip": { 1677 "shared": true, 1678 "sort": 0, 1679 "value_type": "individual" 1680 }, 1681 "type": "graph", 1682 "xaxis": { 1683 "buckets": null, 1684 "mode": "time", 1685 "name": null, 1686 "show": true, 1687 "values": [] 1688 }, 1689 "yaxes": [ 1690 { 1691 "format": "s", 1692 "label": null, 1693 "logBase": 1, 1694 "max": null, 1695 "min": null, 1696 "show": true 1697 }, 1698 { 1699 "format": "short", 1700 "label": null, 1701 "logBase": 1, 1702 "max": null, 1703 "min": null, 1704 "show": true 1705 } 1706 ], 1707 "yaxis": { 1708 "align": false, 1709 "alignLevel": null 1710 } 1711 }, 1712 { 1713 "aliasColors": {}, 1714 "bars": false, 1715 "dashLength": 10, 1716 "dashes": false, 1717 "datasource": "${DS_TEST-CLUSTER}", 1718 "description": "The status of each changefeed.\n\n0: Normal\n\n1: Error\n\n2: Failed\n\n3: Stopped\n\n4: Finished\n\n-1: Unknown", 1719 "fill": 1, 1720 "gridPos": { 1721 "h": 7, 1722 "w": 12, 1723 "x": 0, 1724 "y": 26 1725 }, 1726 "id": 163, 1727 "legend": { 1728 "alignAsTable": true, 1729 "avg": false, 1730 "current": true, 1731 "max": true, 1732 "min": true, 1733 "show": true, 1734 "total": false, 1735 "values": true 1736 }, 1737 "lines": true, 1738 "linewidth": 1, 1739 "links": [], 1740 "nullPointMode": "null", 1741 "percentage": false, 1742 "pointradius": 1, 1743 "points": true, 1744 "renderer": "flot", 1745 "seriesOverrides": [], 1746 "spaceLength": 10, 1747 "stack": false, 1748 "steppedLine": false, 1749 "targets": [ 1750 { 1751 "expr": "ticdc_owner_status{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}", 1752 "format": "time_series", 1753 "instant": false, 1754 "intervalFactor": 1, 1755 "legendFormat": "{{changefeed}}", 1756 "refId": "A" 1757 } 1758 ], 1759 "thresholds": [], 1760 "timeFrom": null, 1761 "timeRegions": [], 1762 "timeShift": null, 1763 "title": "The status of changefeeds", 1764 "tooltip": { 1765 "shared": true, 1766 "sort": 0, 1767 "value_type": "individual" 1768 }, 1769 "type": "graph", 1770 "xaxis": { 1771 "buckets": null, 1772 "mode": "time", 1773 "name": null, 1774 "show": true, 1775 "values": [] 1776 }, 1777 "yaxes": [ 1778 { 1779 "format": "short", 1780 "label": null, 1781 "logBase": 1, 1782 "max": null, 1783 "min": null, 1784 "show": true 1785 }, 1786 { 1787 "format": "short", 1788 "label": null, 1789 "logBase": 1, 1790 "max": null, 1791 "min": null, 1792 "show": true 1793 } 1794 ], 1795 "yaxis": { 1796 "align": false, 1797 "alignLevel": null 1798 } 1799 }, 1800 { 1801 "aliasColors": {}, 1802 "bars": false, 1803 "dashLength": 10, 1804 "dashes": false, 1805 "datasource": "${DS_TEST-CLUSTER}", 1806 "description": "Percentiles of sink write duration of changefeeds", 1807 "fill": 1, 1808 "fillGradient": 0, 1809 "gridPos": { 1810 "h": 7, 1811 "w": 12, 1812 "x": 12, 1813 "y": 26 1814 }, 1815 "hiddenSeries": false, 1816 "id": 35, 1817 "legend": { 1818 "alignAsTable": true, 1819 "avg": false, 1820 "current": true, 1821 "max": false, 1822 "min": false, 1823 "rightSide": true, 1824 "show": true, 1825 "total": false, 1826 "values": true 1827 }, 1828 "lines": true, 1829 "linewidth": 1, 1830 "links": [], 1831 "nullPointMode": "null", 1832 "options": { 1833 "dataLinks": [] 1834 }, 1835 "paceLength": 10, 1836 "percentage": false, 1837 "pointradius": 2, 1838 "points": false, 1839 "renderer": "flot", 1840 "seriesOverrides": [], 1841 "spaceLength": 10, 1842 "stack": false, 1843 "steppedLine": false, 1844 "targets": [ 1845 { 1846 "expr": "histogram_quantile(0.95, sum(rate(ticdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance))", 1847 "format": "time_series", 1848 "intervalFactor": 1, 1849 "legendFormat": "{{instance}}-p95", 1850 "refId": "A" 1851 }, 1852 { 1853 "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance))", 1854 "format": "time_series", 1855 "intervalFactor": 1, 1856 "legendFormat": "{{instance}}-p99", 1857 "refId": "B" 1858 }, 1859 { 1860 "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance))", 1861 "format": "time_series", 1862 "intervalFactor": 1, 1863 "legendFormat": "{{instance}}-p999", 1864 "refId": "C" 1865 } 1866 ], 1867 "thresholds": [], 1868 "timeFrom": null, 1869 "timeRegions": [], 1870 "timeShift": null, 1871 "title": "Sink write duration percentile", 1872 "tooltip": { 1873 "shared": true, 1874 "sort": 0, 1875 "value_type": "individual" 1876 }, 1877 "type": "graph", 1878 "xaxis": { 1879 "buckets": null, 1880 "mode": "time", 1881 "name": null, 1882 "show": true, 1883 "values": [] 1884 }, 1885 "yaxes": [ 1886 { 1887 "format": "s", 1888 "label": null, 1889 "logBase": 2, 1890 "max": null, 1891 "min": null, 1892 "show": true 1893 }, 1894 { 1895 "format": "short", 1896 "label": null, 1897 "logBase": 1, 1898 "max": null, 1899 "min": null, 1900 "show": true 1901 } 1902 ], 1903 "yaxis": { 1904 "align": false, 1905 "alignLevel": null 1906 } 1907 }, 1908 { 1909 "cards": { 1910 "cardPadding": 0, 1911 "cardRound": 0 1912 }, 1913 "color": { 1914 "cardColor": "#FF9830", 1915 "colorScale": "linear", 1916 "colorScheme": "interpolateSpectral", 1917 "exponent": 0.5, 1918 "max": null, 1919 "min": 1, 1920 "mode": "spectrum" 1921 }, 1922 "dataFormat": "tsbuckets", 1923 "datasource": "${DS_TEST-CLUSTER}", 1924 "description": "Sink write duration of changefeeds", 1925 "gridPos": { 1926 "h": 7, 1927 "w": 12, 1928 "x": 0, 1929 "y": 33 1930 }, 1931 "heatmap": {}, 1932 "hideZeroBuckets": true, 1933 "highlightCards": true, 1934 "id": 94, 1935 "legend": { 1936 "alignAsTable": true, 1937 "avg": false, 1938 "current": true, 1939 "max": true, 1940 "min": false, 1941 "rightSide": true, 1942 "show": true, 1943 "sort": "current", 1944 "sortDesc": true, 1945 "total": false, 1946 "values": true 1947 }, 1948 "links": [], 1949 "repeat": null, 1950 "repeatDirection": "h", 1951 "reverseYBuckets": false, 1952 "targets": [ 1953 { 1954 "expr": "max(rate(ticdc_sink_txn_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", 1955 "format": "heatmap", 1956 "instant": false, 1957 "intervalFactor": 2, 1958 "legendFormat": "{{le}}", 1959 "refId": "A" 1960 } 1961 ], 1962 "title": "Sink write duration", 1963 "tooltip": { 1964 "show": true, 1965 "showHistogram": true 1966 }, 1967 "tooltipDecimals": 1, 1968 "type": "heatmap", 1969 "xAxis": { 1970 "show": true 1971 }, 1972 "xBucketNumber": null, 1973 "xBucketSize": null, 1974 "yAxis": { 1975 "decimals": 1, 1976 "format": "s", 1977 "logBase": 1, 1978 "max": null, 1979 "min": null, 1980 "show": true, 1981 "splitFactor": null 1982 }, 1983 "yBucketBound": "upper", 1984 "yBucketNumber": null, 1985 "yBucketSize": null 1986 }, 1987 { 1988 "aliasColors": {}, 1989 "bars": false, 1990 "dashLength": 10, 1991 "dashes": false, 1992 "datasource": "${DS_TEST-CLUSTER}", 1993 "description": "Percentiles of sink batch size", 1994 "fill": 1, 1995 "fillGradient": 0, 1996 "gridPos": { 1997 "h": 7, 1998 "w": 12, 1999 "x": 12, 2000 "y": 33 2001 }, 2002 "hiddenSeries": false, 2003 "id": 36, 2004 "legend": { 2005 "alignAsTable": true, 2006 "avg": false, 2007 "current": true, 2008 "max": false, 2009 "min": false, 2010 "rightSide": true, 2011 "show": true, 2012 "total": false, 2013 "values": true 2014 }, 2015 "lines": true, 2016 "linewidth": 1, 2017 "links": [], 2018 "nullPointMode": "null", 2019 "options": { 2020 "dataLinks": [] 2021 }, 2022 "paceLength": 10, 2023 "percentage": false, 2024 "pointradius": 2, 2025 "points": false, 2026 "renderer": "flot", 2027 "seriesOverrides": [], 2028 "spaceLength": 10, 2029 "stack": false, 2030 "steppedLine": false, 2031 "targets": [ 2032 { 2033 "expr": "histogram_quantile(0.90, sum(rate(ticdc_sink_txn_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", 2034 "format": "time_series", 2035 "intervalFactor": 1, 2036 "legendFormat": "{{capture}}-p90", 2037 "refId": "A" 2038 }, 2039 { 2040 "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_txn_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", 2041 "format": "time_series", 2042 "intervalFactor": 1, 2043 "legendFormat": "{{capture}}-p99", 2044 "refId": "B" 2045 }, 2046 { 2047 "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_batch_size_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", 2048 "format": "time_series", 2049 "hide": true, 2050 "intervalFactor": 1, 2051 "legendFormat": "{{capture}}-p999", 2052 "refId": "C" 2053 } 2054 ], 2055 "thresholds": [], 2056 "timeFrom": null, 2057 "timeRegions": [], 2058 "timeShift": null, 2059 "title": "Sink write batch size percentile", 2060 "tooltip": { 2061 "shared": true, 2062 "sort": 0, 2063 "value_type": "individual" 2064 }, 2065 "type": "graph", 2066 "xaxis": { 2067 "buckets": null, 2068 "mode": "time", 2069 "name": null, 2070 "show": true, 2071 "values": [] 2072 }, 2073 "yaxes": [ 2074 { 2075 "format": "none", 2076 "label": null, 2077 "logBase": 2, 2078 "max": null, 2079 "min": null, 2080 "show": true 2081 }, 2082 { 2083 "format": "short", 2084 "label": null, 2085 "logBase": 1, 2086 "max": null, 2087 "min": null, 2088 "show": true 2089 } 2090 ], 2091 "yaxis": { 2092 "align": false, 2093 "alignLevel": null 2094 } 2095 }, 2096 { 2097 "aliasColors": {}, 2098 "bars": false, 2099 "dashLength": 10, 2100 "dashes": false, 2101 "datasource": "${DS_TEST-CLUSTER}", 2102 "description": "The number of changed rows that are written to downstream per second", 2103 "fill": 1, 2104 "fillGradient": 0, 2105 "gridPos": { 2106 "h": 7, 2107 "w": 12, 2108 "x": 0, 2109 "y": 40 2110 }, 2111 "hiddenSeries": false, 2112 "id": 34, 2113 "legend": { 2114 "alignAsTable": true, 2115 "avg": true, 2116 "current": true, 2117 "max": true, 2118 "min": false, 2119 "rightSide": true, 2120 "show": true, 2121 "total": false, 2122 "values": true 2123 }, 2124 "lines": true, 2125 "linewidth": 1, 2126 "links": [], 2127 "nullPointMode": "null", 2128 "options": { 2129 "dataLinks": [] 2130 }, 2131 "paceLength": 10, 2132 "percentage": false, 2133 "pointradius": 2, 2134 "points": false, 2135 "renderer": "flot", 2136 "seriesOverrides": [], 2137 "spaceLength": 10, 2138 "stack": false, 2139 "steppedLine": false, 2140 "targets": [ 2141 { 2142 "expr": "sum (rate(ticdc_sink_txn_batch_size_sum{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture)", 2143 "format": "time_series", 2144 "intervalFactor": 1, 2145 "legendFormat": "{{capture}}", 2146 "refId": "A" 2147 }, 2148 { 2149 "expr": "sum (rate(ticdc_sink_txn_batch_size_sum{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (changefeed)", 2150 "format": "time_series", 2151 "hide": false, 2152 "interval": "", 2153 "intervalFactor": 1, 2154 "legendFormat": "total", 2155 "refId": "B" 2156 } 2157 ], 2158 "thresholds": [], 2159 "timeFrom": null, 2160 "timeRegions": [], 2161 "timeShift": null, 2162 "title": "Sink write rows count/s", 2163 "tooltip": { 2164 "shared": true, 2165 "sort": 0, 2166 "value_type": "individual" 2167 }, 2168 "type": "graph", 2169 "xaxis": { 2170 "buckets": null, 2171 "mode": "time", 2172 "name": null, 2173 "show": true, 2174 "values": [] 2175 }, 2176 "yaxes": [ 2177 { 2178 "format": "none", 2179 "label": null, 2180 "logBase": 1, 2181 "max": null, 2182 "min": null, 2183 "show": true 2184 }, 2185 { 2186 "format": "short", 2187 "label": null, 2188 "logBase": 1, 2189 "max": null, 2190 "min": null, 2191 "show": true 2192 } 2193 ], 2194 "yaxis": { 2195 "align": false, 2196 "alignLevel": null 2197 } 2198 }, 2199 { 2200 "aliasColors": {}, 2201 "bars": false, 2202 "dashLength": 10, 2203 "dashes": false, 2204 "datasource": "${DS_TEST-CLUSTER}", 2205 "description": "Percentiles of asynchronous flush sink duration of changefeeds", 2206 "fill": 1, 2207 "fillGradient": 0, 2208 "gridPos": { 2209 "h": 7, 2210 "w": 12, 2211 "x": 12, 2212 "y": 40 2213 }, 2214 "hiddenSeries": false, 2215 "id": 98, 2216 "legend": { 2217 "alignAsTable": true, 2218 "avg": false, 2219 "current": true, 2220 "max": false, 2221 "min": false, 2222 "rightSide": true, 2223 "show": true, 2224 "total": false, 2225 "values": true 2226 }, 2227 "lines": true, 2228 "linewidth": 1, 2229 "links": [], 2230 "nullPointMode": "null", 2231 "options": { 2232 "dataLinks": [] 2233 }, 2234 "paceLength": 10, 2235 "percentage": false, 2236 "pointradius": 2, 2237 "points": false, 2238 "renderer": "flot", 2239 "seriesOverrides": [], 2240 "spaceLength": 10, 2241 "stack": false, 2242 "steppedLine": false, 2243 "targets": [ 2244 { 2245 "expr": "histogram_quantile(0.95, sum(rate(ticdc_processor_flush_event_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance,type))", 2246 "format": "time_series", 2247 "intervalFactor": 1, 2248 "legendFormat": "{{instance}}-{{type}}-p95", 2249 "refId": "A" 2250 }, 2251 { 2252 "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_flush_event_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance,type))", 2253 "format": "time_series", 2254 "intervalFactor": 1, 2255 "legendFormat": "{{instance}}-{{type}}-p99", 2256 "refId": "B" 2257 }, 2258 { 2259 "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_flush_event_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (le,instance,type))", 2260 "format": "time_series", 2261 "intervalFactor": 1, 2262 "legendFormat": "{{instance}}-{{type}}-p999", 2263 "refId": "C" 2264 } 2265 ], 2266 "thresholds": [], 2267 "timeFrom": null, 2268 "timeRegions": [], 2269 "timeShift": null, 2270 "title": "Flush sink duration percentile", 2271 "tooltip": { 2272 "shared": true, 2273 "sort": 0, 2274 "value_type": "individual" 2275 }, 2276 "type": "graph", 2277 "xaxis": { 2278 "buckets": null, 2279 "mode": "time", 2280 "name": null, 2281 "show": true, 2282 "values": [] 2283 }, 2284 "yaxes": [ 2285 { 2286 "format": "s", 2287 "label": null, 2288 "logBase": 2, 2289 "max": null, 2290 "min": null, 2291 "show": true 2292 }, 2293 { 2294 "format": "short", 2295 "label": null, 2296 "logBase": 1, 2297 "max": null, 2298 "min": null, 2299 "show": true 2300 } 2301 ], 2302 "yaxis": { 2303 "align": false, 2304 "alignLevel": null 2305 } 2306 }, 2307 { 2308 "cards": { 2309 "cardPadding": 0, 2310 "cardRound": 0 2311 }, 2312 "color": { 2313 "cardColor": "#FF9830", 2314 "colorScale": "linear", 2315 "colorScheme": "interpolateSpectral", 2316 "exponent": 0.5, 2317 "max": null, 2318 "min": 0, 2319 "mode": "spectrum" 2320 }, 2321 "dataFormat": "tsbuckets", 2322 "datasource": "${DS_TEST-CLUSTER}", 2323 "description": "Asynchronous flush sink duration of changefeeds", 2324 "gridPos": { 2325 "h": 7, 2326 "w": 12, 2327 "x": 0, 2328 "y": 47 2329 }, 2330 "heatmap": {}, 2331 "hideZeroBuckets": true, 2332 "highlightCards": true, 2333 "id": 93, 2334 "legend": { 2335 "alignAsTable": true, 2336 "avg": false, 2337 "current": true, 2338 "max": true, 2339 "min": false, 2340 "rightSide": true, 2341 "show": true, 2342 "sort": "current", 2343 "sortDesc": true, 2344 "total": false, 2345 "values": true 2346 }, 2347 "links": [], 2348 "reverseYBuckets": false, 2349 "targets": [ 2350 { 2351 "expr": "max(rate(ticdc_sink_flush_event_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", 2352 "format": "heatmap", 2353 "instant": false, 2354 "intervalFactor": 2, 2355 "legendFormat": "{{le}}", 2356 "refId": "A" 2357 } 2358 ], 2359 "title": "Flush sink duration", 2360 "tooltip": { 2361 "show": true, 2362 "showHistogram": true 2363 }, 2364 "tooltipDecimals": 1, 2365 "type": "heatmap", 2366 "xAxis": { 2367 "show": true 2368 }, 2369 "xBucketNumber": null, 2370 "xBucketSize": null, 2371 "yAxis": { 2372 "decimals": 1, 2373 "format": "s", 2374 "logBase": 1, 2375 "max": null, 2376 "min": null, 2377 "show": true, 2378 "splitFactor": null 2379 }, 2380 "yBucketBound": "upper", 2381 "yBucketNumber": null, 2382 "yBucketSize": null 2383 }, 2384 { 2385 "aliasColors": {}, 2386 "bars": false, 2387 "dashLength": 10, 2388 "dashes": false, 2389 "datasource": "${DS_TEST-CLUSTER}", 2390 "description": "Distribution of MySQL worker loads", 2391 "fill": 1, 2392 "fillGradient": 0, 2393 "gridPos": { 2394 "h": 7, 2395 "w": 12, 2396 "x": 12, 2397 "y": 47 2398 }, 2399 "hiddenSeries": false, 2400 "id": 95, 2401 "legend": { 2402 "alignAsTable": true, 2403 "avg": true, 2404 "current": true, 2405 "max": true, 2406 "min": false, 2407 "rightSide": false, 2408 "show": true, 2409 "sort": "current", 2410 "sortDesc": true, 2411 "total": false, 2412 "values": true 2413 }, 2414 "lines": true, 2415 "linewidth": 1, 2416 "links": [], 2417 "nullPointMode": "null", 2418 "options": { 2419 "dataLinks": [] 2420 }, 2421 "percentage": false, 2422 "pointradius": 2, 2423 "points": false, 2424 "renderer": "flot", 2425 "seriesOverrides": [], 2426 "spaceLength": 10, 2427 "stack": true, 2428 "steppedLine": false, 2429 "targets": [ 2430 { 2431 "expr": "sum(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture,bucket)", 2432 "format": "time_series", 2433 "hide": true, 2434 "interval": "", 2435 "intervalFactor": 1, 2436 "legendFormat": "{{capture}}-{{bucket}}", 2437 "refId": "A" 2438 }, 2439 { 2440 "expr": "count(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) >= 0)", 2441 "format": "time_series", 2442 "hide": true, 2443 "interval": "", 2444 "intervalFactor": 1, 2445 "legendFormat": "total worker", 2446 "refId": "B" 2447 }, 2448 { 2449 "expr": "count(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) <= 2)", 2450 "format": "time_series", 2451 "interval": "", 2452 "intervalFactor": 1, 2453 "legendFormat": "0-2 row/s worker", 2454 "refId": "C" 2455 }, 2456 { 2457 "expr": "count(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) > 2 and rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) <= 10)", 2458 "format": "time_series", 2459 "hide": false, 2460 "interval": "", 2461 "intervalFactor": 1, 2462 "legendFormat": "2-10 row/s worker", 2463 "refId": "D" 2464 }, 2465 { 2466 "expr": "count(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) > 10 and rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) <= 100)", 2467 "format": "time_series", 2468 "hide": false, 2469 "interval": "", 2470 "intervalFactor": 1, 2471 "legendFormat": "10-100 row/s worker", 2472 "refId": "E" 2473 }, 2474 { 2475 "expr": "count(rate(ticdc_sink_bucket_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m]) > 100)", 2476 "format": "time_series", 2477 "hide": false, 2478 "interval": "", 2479 "intervalFactor": 1, 2480 "legendFormat": ">100 row/s worker", 2481 "refId": "F" 2482 } 2483 ], 2484 "thresholds": [], 2485 "timeFrom": null, 2486 "timeRegions": [], 2487 "timeShift": null, 2488 "title": "MySQL sink worker load", 2489 "tooltip": { 2490 "shared": true, 2491 "sort": 2, 2492 "value_type": "individual" 2493 }, 2494 "type": "graph", 2495 "xaxis": { 2496 "buckets": null, 2497 "mode": "time", 2498 "name": null, 2499 "show": true, 2500 "values": [] 2501 }, 2502 "yaxes": [ 2503 { 2504 "format": "short", 2505 "label": null, 2506 "logBase": 1, 2507 "max": null, 2508 "min": null, 2509 "show": true 2510 }, 2511 { 2512 "format": "short", 2513 "label": null, 2514 "logBase": 1, 2515 "max": null, 2516 "min": null, 2517 "show": true 2518 } 2519 ], 2520 "yaxis": { 2521 "align": false, 2522 "alignLevel": null 2523 } 2524 }, 2525 { 2526 "cards": { 2527 "cardPadding": 0, 2528 "cardRound": 0 2529 }, 2530 "color": { 2531 "cardColor": "#FF9830", 2532 "colorScale": "linear", 2533 "colorScheme": "interpolateSpectral", 2534 "exponent": 0.5, 2535 "max": null, 2536 "min": 1, 2537 "mode": "spectrum" 2538 }, 2539 "dataFormat": "tsbuckets", 2540 "datasource": "${DS_TEST-CLUSTER}", 2541 "description": "The duration of detecting and waiting conflict of MySQL sink", 2542 "gridPos": { 2543 "h": 7, 2544 "w": 12, 2545 "x": 0, 2546 "y": 54 2547 }, 2548 "heatmap": {}, 2549 "hideZeroBuckets": true, 2550 "highlightCards": true, 2551 "id": 103, 2552 "legend": { 2553 "alignAsTable": true, 2554 "avg": false, 2555 "current": true, 2556 "max": true, 2557 "min": false, 2558 "rightSide": true, 2559 "show": true, 2560 "sort": "current", 2561 "sortDesc": true, 2562 "total": false, 2563 "values": true 2564 }, 2565 "links": [], 2566 "repeatDirection": "h", 2567 "reverseYBuckets": false, 2568 "targets": [ 2569 { 2570 "expr": "max(rate(ticdc_sink_conflict_detect_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", 2571 "format": "heatmap", 2572 "instant": false, 2573 "intervalFactor": 2, 2574 "legendFormat": "{{le}}", 2575 "refId": "A" 2576 } 2577 ], 2578 "title": "MySQL sink conflict detect duration", 2579 "tooltip": { 2580 "show": true, 2581 "showHistogram": true 2582 }, 2583 "tooltipDecimals": 1, 2584 "type": "heatmap", 2585 "xAxis": { 2586 "show": true 2587 }, 2588 "xBucketNumber": null, 2589 "xBucketSize": null, 2590 "yAxis": { 2591 "decimals": 1, 2592 "format": "s", 2593 "logBase": 1, 2594 "max": null, 2595 "min": null, 2596 "show": true, 2597 "splitFactor": null 2598 }, 2599 "yBucketBound": "upper", 2600 "yBucketNumber": null, 2601 "yBucketSize": null 2602 }, 2603 { 2604 "aliasColors": {}, 2605 "bars": false, 2606 "dashLength": 10, 2607 "dashes": false, 2608 "datasource": "${DS_TEST-CLUSTER}", 2609 "description": "Percentiles of detecting and waiting conflict duration of MySQL sink", 2610 "fill": 1, 2611 "fillGradient": 0, 2612 "gridPos": { 2613 "h": 7, 2614 "w": 12, 2615 "x": 12, 2616 "y": 54 2617 }, 2618 "hiddenSeries": false, 2619 "id": 83, 2620 "legend": { 2621 "alignAsTable": true, 2622 "avg": false, 2623 "current": true, 2624 "max": true, 2625 "min": false, 2626 "rightSide": false, 2627 "show": true, 2628 "total": false, 2629 "values": true 2630 }, 2631 "lines": true, 2632 "linewidth": 1, 2633 "links": [], 2634 "nullPointMode": "null", 2635 "options": { 2636 "dataLinks": [] 2637 }, 2638 "percentage": false, 2639 "pointradius": 2, 2640 "points": false, 2641 "renderer": "flot", 2642 "seriesOverrides": [], 2643 "spaceLength": 10, 2644 "stack": false, 2645 "steppedLine": false, 2646 "targets": [ 2647 { 2648 "expr": "histogram_quantile(0.95,sum(rate(ticdc_sink_conflict_detect_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", 2649 "format": "time_series", 2650 "interval": "", 2651 "intervalFactor": 1, 2652 "legendFormat": "{{instance}}-p95", 2653 "refId": "A" 2654 }, 2655 { 2656 "expr": "histogram_quantile(0.99,sum(rate(ticdc_sink_conflict_detect_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", 2657 "format": "time_series", 2658 "interval": "", 2659 "intervalFactor": 1, 2660 "legendFormat": "{{instance}}-p99", 2661 "refId": "B" 2662 }, 2663 { 2664 "expr": "histogram_quantile(0.999,sum(rate(ticdc_sink_conflict_detect_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", 2665 "format": "time_series", 2666 "interval": "", 2667 "intervalFactor": 1, 2668 "legendFormat": "{{instance}}-p999", 2669 "refId": "C" 2670 } 2671 ], 2672 "thresholds": [], 2673 "timeFrom": null, 2674 "timeRegions": [], 2675 "timeShift": null, 2676 "title": "MySQL sink conflict detect duration percentile", 2677 "tooltip": { 2678 "shared": true, 2679 "sort": 2, 2680 "value_type": "individual" 2681 }, 2682 "type": "graph", 2683 "xaxis": { 2684 "buckets": null, 2685 "mode": "time", 2686 "name": null, 2687 "show": true, 2688 "values": [] 2689 }, 2690 "yaxes": [ 2691 { 2692 "format": "s", 2693 "label": null, 2694 "logBase": 2, 2695 "max": null, 2696 "min": null, 2697 "show": true 2698 }, 2699 { 2700 "format": "short", 2701 "label": null, 2702 "logBase": 1, 2703 "max": null, 2704 "min": null, 2705 "show": true 2706 } 2707 ], 2708 "yaxis": { 2709 "align": false, 2710 "alignLevel": null 2711 } 2712 }, 2713 { 2714 "cards": { 2715 "cardPadding": 0, 2716 "cardRound": 0 2717 }, 2718 "color": { 2719 "cardColor": "#FF9830", 2720 "colorScale": "linear", 2721 "colorScheme": "interpolateSpectral", 2722 "exponent": 0.5, 2723 "max": null, 2724 "min": 1, 2725 "mode": "spectrum" 2726 }, 2727 "dataFormat": "tsbuckets", 2728 "datasource": "${DS_TEST-CLUSTER}", 2729 "description": "The duration of executing DDL in MySQL sink", 2730 "gridPos": { 2731 "h": 7, 2732 "w": 12, 2733 "x": 0, 2734 "y": 61 2735 }, 2736 "heatmap": {}, 2737 "hideZeroBuckets": true, 2738 "highlightCards": true, 2739 "id": 165, 2740 "legend": { 2741 "alignAsTable": true, 2742 "avg": false, 2743 "current": true, 2744 "max": true, 2745 "min": false, 2746 "rightSide": true, 2747 "show": true, 2748 "sort": "current", 2749 "sortDesc": true, 2750 "total": false, 2751 "values": true 2752 }, 2753 "links": [], 2754 "repeatDirection": "h", 2755 "reverseYBuckets": false, 2756 "targets": [ 2757 { 2758 "expr": "max(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", 2759 "format": "heatmap", 2760 "instant": false, 2761 "intervalFactor": 2, 2762 "legendFormat": "{{le}}", 2763 "refId": "A" 2764 } 2765 ], 2766 "title": "MySQL sink ddl execution duration", 2767 "tooltip": { 2768 "show": true, 2769 "showHistogram": true 2770 }, 2771 "tooltipDecimals": 1, 2772 "type": "heatmap", 2773 "xAxis": { 2774 "show": true 2775 }, 2776 "xBucketNumber": null, 2777 "xBucketSize": null, 2778 "yAxis": { 2779 "decimals": 1, 2780 "format": "s", 2781 "logBase": 1, 2782 "max": null, 2783 "min": null, 2784 "show": true, 2785 "splitFactor": null 2786 }, 2787 "yBucketBound": "upper", 2788 "yBucketNumber": null, 2789 "yBucketSize": null 2790 }, 2791 { 2792 "aliasColors": {}, 2793 "bars": false, 2794 "dashLength": 10, 2795 "dashes": false, 2796 "datasource": "${DS_TEST-CLUSTER}", 2797 "description": "Percentiles of executing DDL duration of MySQL sink", 2798 "fill": 1, 2799 "fillGradient": 0, 2800 "gridPos": { 2801 "h": 7, 2802 "w": 12, 2803 "x": 12, 2804 "y": 61 2805 }, 2806 "hiddenSeries": false, 2807 "id": 166, 2808 "legend": { 2809 "alignAsTable": true, 2810 "avg": false, 2811 "current": true, 2812 "max": true, 2813 "min": false, 2814 "rightSide": false, 2815 "show": true, 2816 "total": false, 2817 "values": true 2818 }, 2819 "lines": true, 2820 "linewidth": 1, 2821 "links": [], 2822 "nullPointMode": "null", 2823 "options": { 2824 "dataLinks": [] 2825 }, 2826 "percentage": false, 2827 "pointradius": 2, 2828 "points": false, 2829 "renderer": "flot", 2830 "seriesOverrides": [], 2831 "spaceLength": 10, 2832 "stack": false, 2833 "steppedLine": false, 2834 "targets": [ 2835 { 2836 "expr": "histogram_quantile(0.95,sum(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", 2837 "format": "time_series", 2838 "interval": "", 2839 "intervalFactor": 1, 2840 "legendFormat": "{{instance}}-p95", 2841 "refId": "A" 2842 }, 2843 { 2844 "expr": "histogram_quantile(0.99,sum(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", 2845 "format": "time_series", 2846 "interval": "", 2847 "intervalFactor": 1, 2848 "legendFormat": "{{instance}}-p99", 2849 "refId": "B" 2850 }, 2851 { 2852 "expr": "histogram_quantile(0.999,sum(rate(ticdc_sink_ddl_exec_duration_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,instance))", 2853 "format": "time_series", 2854 "interval": "", 2855 "intervalFactor": 1, 2856 "legendFormat": "{{instance}}-p999", 2857 "refId": "C" 2858 } 2859 ], 2860 "thresholds": [], 2861 "timeFrom": null, 2862 "timeRegions": [], 2863 "timeShift": null, 2864 "title": "MySQL sink ddl execution duration percentile", 2865 "tooltip": { 2866 "shared": true, 2867 "sort": 2, 2868 "value_type": "individual" 2869 }, 2870 "type": "graph", 2871 "xaxis": { 2872 "buckets": null, 2873 "mode": "time", 2874 "name": null, 2875 "show": true, 2876 "values": [] 2877 }, 2878 "yaxes": [ 2879 { 2880 "format": "s", 2881 "label": null, 2882 "logBase": 2, 2883 "max": null, 2884 "min": null, 2885 "show": true 2886 }, 2887 { 2888 "format": "short", 2889 "label": null, 2890 "logBase": 1, 2891 "max": null, 2892 "min": null, 2893 "show": true 2894 } 2895 ], 2896 "yaxis": { 2897 "align": false, 2898 "alignLevel": null 2899 } 2900 }, 2901 { 2902 "aliasColors": {}, 2903 "bars": false, 2904 "dashLength": 10, 2905 "dashes": false, 2906 "datasource": "${DS_TEST-CLUSTER}", 2907 "fill": 1, 2908 "gridPos": { 2909 "h": 8, 2910 "w": 12, 2911 "x": 0, 2912 "y": 68 2913 }, 2914 "id": 149, 2915 "legend": { 2916 "avg": false, 2917 "current": false, 2918 "max": false, 2919 "min": false, 2920 "show": true, 2921 "total": false, 2922 "values": false 2923 }, 2924 "lines": true, 2925 "linewidth": 1, 2926 "links": [], 2927 "nullPointMode": "null", 2928 "percentage": false, 2929 "pointradius": 2, 2930 "points": false, 2931 "renderer": "flot", 2932 "seriesOverrides": [], 2933 "spaceLength": 10, 2934 "stack": false, 2935 "steppedLine": false, 2936 "targets": [ 2937 { 2938 "expr": "sum(rate(ticdc_processor_table_memory_consumption_sum{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s]) / rate(ticdc_processor_table_memory_consumption_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s])) by (capture)", 2939 "format": "time_series", 2940 "intervalFactor": 1, 2941 "legendFormat": "{{ capture }}", 2942 "refId": "A" 2943 } 2944 ], 2945 "thresholds": [], 2946 "timeFrom": null, 2947 "timeRegions": [], 2948 "timeShift": null, 2949 "title": "Processor Memory Consumption Per Capture", 2950 "tooltip": { 2951 "shared": true, 2952 "sort": 0, 2953 "value_type": "individual" 2954 }, 2955 "type": "graph", 2956 "xaxis": { 2957 "buckets": null, 2958 "mode": "time", 2959 "name": null, 2960 "show": true, 2961 "values": [] 2962 }, 2963 "yaxes": [ 2964 { 2965 "format": "bytes", 2966 "label": null, 2967 "logBase": 1, 2968 "max": null, 2969 "min": null, 2970 "show": true 2971 }, 2972 { 2973 "format": "short", 2974 "label": null, 2975 "logBase": 1, 2976 "max": null, 2977 "min": null, 2978 "show": true 2979 } 2980 ], 2981 "yaxis": { 2982 "align": false, 2983 "alignLevel": null 2984 } 2985 }, 2986 { 2987 "aliasColors": {}, 2988 "bars": false, 2989 "dashLength": 10, 2990 "dashes": false, 2991 "datasource": "${DS_TEST-CLUSTER}", 2992 "fill": 1, 2993 "gridPos": { 2994 "h": 8, 2995 "w": 12, 2996 "x": 12, 2997 "y": 68 2998 }, 2999 "id": 164, 3000 "legend": { 3001 "avg": false, 3002 "current": false, 3003 "max": false, 3004 "min": false, 3005 "show": true, 3006 "total": false, 3007 "values": false 3008 }, 3009 "lines": true, 3010 "linewidth": 1, 3011 "links": [], 3012 "nullPointMode": "null", 3013 "percentage": false, 3014 "pointradius": 2, 3015 "points": false, 3016 "renderer": "flot", 3017 "seriesOverrides": [], 3018 "spaceLength": 10, 3019 "stack": false, 3020 "steppedLine": false, 3021 "targets": [ 3022 { 3023 "expr": "sum(rate(ticdc_processor_table_memory_consumption_sum{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s]) / rate(ticdc_processor_table_memory_consumption_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[30s])) by (capture, changefeed)", 3024 "format": "time_series", 3025 "intervalFactor": 1, 3026 "legendFormat": "{{ capture }} - {{ changefeed }}", 3027 "refId": "A" 3028 } 3029 ], 3030 "thresholds": [], 3031 "timeFrom": null, 3032 "timeRegions": [], 3033 "timeShift": null, 3034 "title": "Processor Memory Consumption Per Changefeed", 3035 "tooltip": { 3036 "shared": true, 3037 "sort": 0, 3038 "value_type": "individual" 3039 }, 3040 "type": "graph", 3041 "xaxis": { 3042 "buckets": null, 3043 "mode": "time", 3044 "name": null, 3045 "show": true, 3046 "values": [] 3047 }, 3048 "yaxes": [ 3049 { 3050 "format": "bytes", 3051 "label": null, 3052 "logBase": 1, 3053 "max": null, 3054 "min": null, 3055 "show": true 3056 }, 3057 { 3058 "format": "short", 3059 "label": null, 3060 "logBase": 1, 3061 "max": null, 3062 "min": null, 3063 "show": true 3064 } 3065 ], 3066 "yaxis": { 3067 "align": false, 3068 "alignLevel": null 3069 } 3070 } 3071 ], 3072 "title": "Changefeed", 3073 "type": "row" 3074 }, 3075 { 3076 "collapsed": true, 3077 "gridPos": { 3078 "h": 1, 3079 "w": 24, 3080 "x": 0, 3081 "y": 2 3082 }, 3083 "id": 13, 3084 "panels": [ 3085 { 3086 "aliasColors": {}, 3087 "bars": false, 3088 "dashLength": 10, 3089 "dashes": false, 3090 "datasource": "${DS_TEST-CLUSTER}", 3091 "description": "The number of established Eventfeed RPC between TiCDC and TiKV", 3092 "fill": 1, 3093 "fillGradient": 0, 3094 "gridPos": { 3095 "h": 7, 3096 "w": 8, 3097 "x": 0, 3098 "y": 3 3099 }, 3100 "hiddenSeries": false, 3101 "id": 15, 3102 "legend": { 3103 "alignAsTable": true, 3104 "avg": false, 3105 "current": true, 3106 "hideEmpty": false, 3107 "max": false, 3108 "min": false, 3109 "rightSide": true, 3110 "show": true, 3111 "total": false, 3112 "values": true 3113 }, 3114 "lines": true, 3115 "linewidth": 1, 3116 "links": [], 3117 "nullPointMode": "null", 3118 "options": { 3119 "dataLinks": [] 3120 }, 3121 "paceLength": 10, 3122 "percentage": false, 3123 "pointradius": 2, 3124 "points": false, 3125 "renderer": "flot", 3126 "seriesOverrides": [ 3127 { 3128 "alias": "/.*-rpc/", 3129 "yaxis": 2 3130 } 3131 ], 3132 "spaceLength": 10, 3133 "stack": false, 3134 "steppedLine": false, 3135 "targets": [ 3136 { 3137 "expr": "sum(ticdc_kvclient_event_feed_count{tidb_cluster=\"$tidb_cluster\"}) by (instance)", 3138 "format": "time_series", 3139 "intervalFactor": 1, 3140 "legendFormat": "{{instance}}", 3141 "refId": "A" 3142 }, 3143 { 3144 "expr": "sum(grpc_client_started_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance) - sum(grpc_client_handled_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", 3145 "format": "time_series", 3146 "intervalFactor": 1, 3147 "legendFormat": "{{instance}}-rpc", 3148 "refId": "B" 3149 }, 3150 { 3151 "expr": "sum(grpc_client_started_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", 3152 "format": "time_series", 3153 "hide": false, 3154 "intervalFactor": 1, 3155 "legendFormat": "{{instance}}-rpc-started", 3156 "refId": "C" 3157 }, 3158 { 3159 "expr": "sum(grpc_client_handled_total{tidb_cluster=\"$tidb_cluster\", grpc_method=\"EventFeed\"}) by (instance)", 3160 "format": "time_series", 3161 "hide": false, 3162 "intervalFactor": 1, 3163 "legendFormat": "{{instance}}-rpc-handled", 3164 "refId": "D" 3165 } 3166 ], 3167 "thresholds": [], 3168 "timeFrom": null, 3169 "timeRegions": [], 3170 "timeShift": null, 3171 "title": "Eventfeed count", 3172 "tooltip": { 3173 "shared": true, 3174 "sort": 0, 3175 "value_type": "individual" 3176 }, 3177 "type": "graph", 3178 "xaxis": { 3179 "buckets": null, 3180 "mode": "time", 3181 "name": null, 3182 "show": true, 3183 "values": [] 3184 }, 3185 "yaxes": [ 3186 { 3187 "format": "short", 3188 "label": null, 3189 "logBase": 1, 3190 "max": null, 3191 "min": null, 3192 "show": true 3193 }, 3194 { 3195 "format": "short", 3196 "label": null, 3197 "logBase": 1, 3198 "max": null, 3199 "min": null, 3200 "show": true 3201 } 3202 ], 3203 "yaxis": { 3204 "align": false, 3205 "alignLevel": null 3206 } 3207 }, 3208 { 3209 "aliasColors": {}, 3210 "bars": false, 3211 "dashLength": 10, 3212 "dashes": false, 3213 "datasource": "${DS_TEST-CLUSTER}", 3214 "decimals": null, 3215 "description": "Percentiles of Eventfeed message size", 3216 "fill": 1, 3217 "fillGradient": 0, 3218 "gridPos": { 3219 "h": 7, 3220 "w": 8, 3221 "x": 8, 3222 "y": 3 3223 }, 3224 "hiddenSeries": false, 3225 "id": 17, 3226 "legend": { 3227 "alignAsTable": true, 3228 "avg": false, 3229 "current": true, 3230 "max": false, 3231 "min": false, 3232 "rightSide": true, 3233 "show": true, 3234 "total": false, 3235 "values": true 3236 }, 3237 "lines": true, 3238 "linewidth": 1, 3239 "links": [], 3240 "nullPointMode": "null", 3241 "options": { 3242 "dataLinks": [] 3243 }, 3244 "paceLength": 10, 3245 "percentage": false, 3246 "pointradius": 2, 3247 "points": false, 3248 "renderer": "flot", 3249 "seriesOverrides": [], 3250 "spaceLength": 10, 3251 "stack": false, 3252 "steppedLine": false, 3253 "targets": [ 3254 { 3255 "expr": "histogram_quantile(0.999, sum(rate(ticdc_kvclient_event_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", 3256 "format": "time_series", 3257 "intervalFactor": 1, 3258 "legendFormat": "{{instance}}-{{type}}-p999", 3259 "refId": "A" 3260 }, 3261 { 3262 "expr": "histogram_quantile(0.95, sum(rate(ticdc_kvclient_event_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance, type))", 3263 "format": "time_series", 3264 "intervalFactor": 1, 3265 "legendFormat": "{{instance}}-{{type}}-p95", 3266 "refId": "B" 3267 } 3268 ], 3269 "thresholds": [], 3270 "timeFrom": null, 3271 "timeRegions": [], 3272 "timeShift": null, 3273 "title": "Event size percentile", 3274 "tooltip": { 3275 "shared": true, 3276 "sort": 0, 3277 "value_type": "individual" 3278 }, 3279 "type": "graph", 3280 "xaxis": { 3281 "buckets": null, 3282 "mode": "time", 3283 "name": null, 3284 "show": true, 3285 "values": [] 3286 }, 3287 "yaxes": [ 3288 { 3289 "format": "bytes", 3290 "label": null, 3291 "logBase": 1, 3292 "max": null, 3293 "min": null, 3294 "show": true 3295 }, 3296 { 3297 "format": "short", 3298 "label": null, 3299 "logBase": 1, 3300 "max": null, 3301 "min": null, 3302 "show": true 3303 } 3304 ], 3305 "yaxis": { 3306 "align": false, 3307 "alignLevel": null 3308 } 3309 }, 3310 { 3311 "aliasColors": {}, 3312 "bars": true, 3313 "dashLength": 10, 3314 "dashes": false, 3315 "datasource": "${DS_TEST-CLUSTER}", 3316 "description": "The number of errors that interrupt Eventfeed RPC", 3317 "fill": 1, 3318 "fillGradient": 0, 3319 "gridPos": { 3320 "h": 7, 3321 "w": 8, 3322 "x": 16, 3323 "y": 3 3324 }, 3325 "hiddenSeries": false, 3326 "id": 28, 3327 "legend": { 3328 "alignAsTable": true, 3329 "avg": false, 3330 "current": true, 3331 "hideEmpty": true, 3332 "hideZero": true, 3333 "max": true, 3334 "min": false, 3335 "rightSide": true, 3336 "show": true, 3337 "total": false, 3338 "values": true 3339 }, 3340 "lines": true, 3341 "linewidth": 1, 3342 "links": [], 3343 "nullPointMode": "null", 3344 "options": { 3345 "dataLinks": [] 3346 }, 3347 "paceLength": 10, 3348 "percentage": false, 3349 "pointradius": 2, 3350 "points": false, 3351 "renderer": "flot", 3352 "seriesOverrides": [], 3353 "spaceLength": 10, 3354 "stack": false, 3355 "steppedLine": false, 3356 "targets": [ 3357 { 3358 "expr": "sum(increase(ticdc_kvclient_event_feed_error_count{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (type)", 3359 "format": "time_series", 3360 "hide": false, 3361 "interval": "1m", 3362 "intervalFactor": 1, 3363 "legendFormat": "{{type}}", 3364 "refId": "A" 3365 }, 3366 { 3367 "expr": "-sum(increase(pd_schedule_operators_count{tidb_cluster=\"$tidb_cluster\", event=\"create\", type=~\".*leader\"}[1m]))", 3368 "format": "time_series", 3369 "hide": false, 3370 "interval": "1m", 3371 "intervalFactor": 1, 3372 "legendFormat": "transfer-leader", 3373 "refId": "B" 3374 }, 3375 { 3376 "expr": "-sum(increase(pd_schedule_operators_count{tidb_cluster=\"$tidb_cluster\", event=\"create\", type=~\".*(peer|region)\"}[1m]))", 3377 "format": "time_series", 3378 "interval": "1m", 3379 "intervalFactor": 1, 3380 "legendFormat": "move-region", 3381 "refId": "C" 3382 } 3383 ], 3384 "thresholds": [], 3385 "timeFrom": null, 3386 "timeRegions": [], 3387 "timeShift": null, 3388 "title": "Eventfeed error/m", 3389 "tooltip": { 3390 "shared": true, 3391 "sort": 0, 3392 "value_type": "individual" 3393 }, 3394 "type": "graph", 3395 "xaxis": { 3396 "buckets": null, 3397 "mode": "time", 3398 "name": null, 3399 "show": true, 3400 "values": [] 3401 }, 3402 "yaxes": [ 3403 { 3404 "format": "short", 3405 "label": null, 3406 "logBase": 1, 3407 "max": null, 3408 "min": null, 3409 "show": true 3410 }, 3411 { 3412 "format": "short", 3413 "label": null, 3414 "logBase": 1, 3415 "max": null, 3416 "min": null, 3417 "show": true 3418 } 3419 ], 3420 "yaxis": { 3421 "align": false, 3422 "alignLevel": null 3423 } 3424 }, 3425 { 3426 "aliasColors": {}, 3427 "bars": false, 3428 "dashLength": 10, 3429 "dashes": false, 3430 "datasource": "${DS_TEST-CLUSTER}", 3431 "description": "The number of KV client received events from TiKV per seconds", 3432 "fill": 1, 3433 "fillGradient": 0, 3434 "gridPos": { 3435 "h": 7, 3436 "w": 8, 3437 "x": 0, 3438 "y": 10 3439 }, 3440 "hiddenSeries": false, 3441 "id": 29, 3442 "legend": { 3443 "alignAsTable": true, 3444 "avg": false, 3445 "current": true, 3446 "hideEmpty": true, 3447 "hideZero": true, 3448 "max": true, 3449 "min": false, 3450 "rightSide": false, 3451 "show": true, 3452 "total": false, 3453 "values": true 3454 }, 3455 "lines": true, 3456 "linewidth": 1, 3457 "links": [], 3458 "nullPointMode": "null", 3459 "options": { 3460 "dataLinks": [] 3461 }, 3462 "paceLength": 10, 3463 "percentage": false, 3464 "pointradius": 2, 3465 "points": false, 3466 "renderer": "flot", 3467 "seriesOverrides": [], 3468 "spaceLength": 10, 3469 "stack": false, 3470 "steppedLine": false, 3471 "targets": [ 3472 { 3473 "expr": "sum(rate(ticdc_kvclient_pull_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance, type)", 3474 "format": "time_series", 3475 "intervalFactor": 1, 3476 "legendFormat": "{{instance}}-{{type}}", 3477 "refId": "A" 3478 } 3479 ], 3480 "thresholds": [], 3481 "timeFrom": null, 3482 "timeRegions": [], 3483 "timeShift": null, 3484 "title": "KV client receive events/s", 3485 "tooltip": { 3486 "shared": true, 3487 "sort": 0, 3488 "value_type": "individual" 3489 }, 3490 "type": "graph", 3491 "xaxis": { 3492 "buckets": null, 3493 "mode": "time", 3494 "name": null, 3495 "show": true, 3496 "values": [] 3497 }, 3498 "yaxes": [ 3499 { 3500 "format": "short", 3501 "label": null, 3502 "logBase": 1, 3503 "max": null, 3504 "min": null, 3505 "show": true 3506 }, 3507 { 3508 "format": "short", 3509 "label": null, 3510 "logBase": 1, 3511 "max": null, 3512 "min": null, 3513 "show": true 3514 } 3515 ], 3516 "yaxis": { 3517 "align": false, 3518 "alignLevel": null 3519 } 3520 }, 3521 { 3522 "aliasColors": {}, 3523 "bars": false, 3524 "dashLength": 10, 3525 "dashes": false, 3526 "datasource": "${DS_TEST-CLUSTER}", 3527 "description": "The number of events that puller outputs to sorter \n per second", 3528 "fill": 1, 3529 "fillGradient": 0, 3530 "gridPos": { 3531 "h": 7, 3532 "w": 8, 3533 "x": 8, 3534 "y": 10 3535 }, 3536 "hiddenSeries": false, 3537 "id": 5, 3538 "legend": { 3539 "alignAsTable": true, 3540 "avg": false, 3541 "current": true, 3542 "max": true, 3543 "min": false, 3544 "rightSide": false, 3545 "show": true, 3546 "total": false, 3547 "values": true 3548 }, 3549 "lines": true, 3550 "linewidth": 1, 3551 "links": [], 3552 "nullPointMode": "null", 3553 "options": { 3554 "dataLinks": [] 3555 }, 3556 "paceLength": 10, 3557 "percentage": false, 3558 "pointradius": 2, 3559 "points": false, 3560 "renderer": "flot", 3561 "seriesOverrides": [], 3562 "spaceLength": 10, 3563 "stack": false, 3564 "steppedLine": false, 3565 "targets": [ 3566 { 3567 "expr": "sum (rate(ticdc_puller_txn_collect_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (instance, type)", 3568 "format": "time_series", 3569 "intervalFactor": 1, 3570 "legendFormat": "{{instance}} - {{type}}", 3571 "refId": "A" 3572 } 3573 ], 3574 "thresholds": [], 3575 "timeFrom": null, 3576 "timeRegions": [], 3577 "timeShift": null, 3578 "title": "Puller output events/s", 3579 "tooltip": { 3580 "shared": true, 3581 "sort": 0, 3582 "value_type": "individual" 3583 }, 3584 "type": "graph", 3585 "xaxis": { 3586 "buckets": null, 3587 "mode": "time", 3588 "name": null, 3589 "show": true, 3590 "values": [] 3591 }, 3592 "yaxes": [ 3593 { 3594 "format": "none", 3595 "label": null, 3596 "logBase": 1, 3597 "max": null, 3598 "min": null, 3599 "show": true 3600 }, 3601 { 3602 "format": "short", 3603 "label": null, 3604 "logBase": 1, 3605 "max": null, 3606 "min": null, 3607 "show": true 3608 } 3609 ], 3610 "yaxis": { 3611 "align": false, 3612 "alignLevel": null 3613 } 3614 }, 3615 { 3616 "aliasColors": {}, 3617 "bars": false, 3618 "dashLength": 10, 3619 "dashes": false, 3620 "datasource": "${DS_TEST-CLUSTER}", 3621 "description": "The number of rows that sink flushes to downstream per second", 3622 "fill": 1, 3623 "fillGradient": 0, 3624 "gridPos": { 3625 "h": 7, 3626 "w": 8, 3627 "x": 16, 3628 "y": 10 3629 }, 3630 "hiddenSeries": false, 3631 "id": 108, 3632 "legend": { 3633 "alignAsTable": true, 3634 "avg": true, 3635 "current": true, 3636 "max": true, 3637 "min": false, 3638 "rightSide": false, 3639 "show": true, 3640 "sort": "current", 3641 "sortDesc": true, 3642 "total": false, 3643 "values": true 3644 }, 3645 "lines": true, 3646 "linewidth": 1, 3647 "links": [], 3648 "nullPointMode": "null", 3649 "options": { 3650 "dataLinks": [] 3651 }, 3652 "percentage": false, 3653 "pointradius": 2, 3654 "points": false, 3655 "renderer": "flot", 3656 "seriesOverrides": [], 3657 "spaceLength": 10, 3658 "stack": false, 3659 "steppedLine": false, 3660 "targets": [ 3661 { 3662 "expr": "sum(rate(ticdc_sink_total_flushed_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture)", 3663 "format": "time_series", 3664 "intervalFactor": 1, 3665 "legendFormat": "{{capture}}", 3666 "refId": "A" 3667 } 3668 ], 3669 "thresholds": [], 3670 "timeFrom": null, 3671 "timeRegions": [], 3672 "timeShift": null, 3673 "title": "Sink flush rows/s", 3674 "tooltip": { 3675 "shared": true, 3676 "sort": 2, 3677 "value_type": "individual" 3678 }, 3679 "type": "graph", 3680 "xaxis": { 3681 "buckets": null, 3682 "mode": "time", 3683 "name": null, 3684 "show": true, 3685 "values": [] 3686 }, 3687 "yaxes": [ 3688 { 3689 "format": "short", 3690 "label": null, 3691 "logBase": 1, 3692 "max": null, 3693 "min": null, 3694 "show": true 3695 }, 3696 { 3697 "format": "short", 3698 "label": null, 3699 "logBase": 1, 3700 "max": null, 3701 "min": null, 3702 "show": true 3703 } 3704 ], 3705 "yaxis": { 3706 "align": false, 3707 "alignLevel": null 3708 } 3709 }, 3710 { 3711 "aliasColors": {}, 3712 "bars": false, 3713 "dashLength": 10, 3714 "dashes": false, 3715 "datasource": "${DS_TEST-CLUSTER}", 3716 "description": "The number of events that are buffered in Puller's memory buffer and output channel", 3717 "fill": 1, 3718 "fillGradient": 0, 3719 "gridPos": { 3720 "h": 7, 3721 "w": 6, 3722 "x": 0, 3723 "y": 17 3724 }, 3725 "hiddenSeries": false, 3726 "id": 43, 3727 "legend": { 3728 "alignAsTable": true, 3729 "avg": false, 3730 "current": true, 3731 "hideEmpty": false, 3732 "max": true, 3733 "min": false, 3734 "rightSide": false, 3735 "show": true, 3736 "total": false, 3737 "values": true 3738 }, 3739 "lines": true, 3740 "linewidth": 1, 3741 "links": [], 3742 "nullPointMode": "null", 3743 "options": { 3744 "dataLinks": [] 3745 }, 3746 "paceLength": 10, 3747 "percentage": false, 3748 "pointradius": 2, 3749 "points": false, 3750 "renderer": "flot", 3751 "seriesOverrides": [ 3752 { 3753 "alias": "/.*chan.*/", 3754 "yaxis": 2 3755 } 3756 ], 3757 "spaceLength": 10, 3758 "stack": false, 3759 "steppedLine": false, 3760 "targets": [ 3761 { 3762 "expr": "sum(rate(ticdc_puller_output_chan_size_sum{tidb_cluster=\\\"$tidb_cluster\\\", changefeed=~\\\"$changefeed\\\",capture=~\\\"$capture\\\"}[30s]) / rate(ticdc_puller_output_chan_size_count{tidb_cluster=\\\"$tidb_cluster\\\", changefeed=~\\\"$changefeed\\\",capture=~\\\"$capture\\\"}[30s])) by (capture)", 3763 "format": "time_series", 3764 "hide": true, 3765 "intervalFactor": 1, 3766 "legendFormat": "{{capture}} - output chan", 3767 "refId": "B" 3768 }, 3769 { 3770 "expr": "sum(rate(ticdc_puller_event_chan_size_sum{tidb_cluster=\\\"$tidb_cluster\\\", changefeed=~\\\"$changefeed\\\",capture=~\\\"$capture\\\"}[30s]) / rate(ticdc_puller_event_chan_size_count{tidb_cluster=\\\"$tidb_cluster\\\", changefeed=~\\\"$changefeed\\\",capture=~\\\"$capture\\\"}[30s])) by (capture)", 3771 "format": "time_series", 3772 "hide": true, 3773 "intervalFactor": 1, 3774 "legendFormat": "{{capture}} - event chan", 3775 "refId": "C" 3776 } 3777 ], 3778 "thresholds": [], 3779 "timeFrom": null, 3780 "timeRegions": [], 3781 "timeShift": null, 3782 "title": "Puller buffer size", 3783 "tooltip": { 3784 "shared": true, 3785 "sort": 0, 3786 "value_type": "individual" 3787 }, 3788 "type": "graph", 3789 "xaxis": { 3790 "buckets": null, 3791 "mode": "time", 3792 "name": null, 3793 "show": true, 3794 "values": [] 3795 }, 3796 "yaxes": [ 3797 { 3798 "format": "short", 3799 "label": null, 3800 "logBase": 1, 3801 "max": null, 3802 "min": null, 3803 "show": true 3804 }, 3805 { 3806 "format": "short", 3807 "label": null, 3808 "logBase": 1, 3809 "max": null, 3810 "min": null, 3811 "show": true 3812 } 3813 ], 3814 "yaxis": { 3815 "align": false, 3816 "alignLevel": null 3817 } 3818 }, 3819 { 3820 "aliasColors": {}, 3821 "bars": false, 3822 "dashLength": 10, 3823 "dashes": false, 3824 "datasource": "${DS_TEST-CLUSTER}", 3825 "description": "The number of events that are buffered in Sorter's unsorted events buffer and output channel", 3826 "fill": 1, 3827 "fillGradient": 0, 3828 "gridPos": { 3829 "h": 7, 3830 "w": 6, 3831 "x": 6, 3832 "y": 17 3833 }, 3834 "hiddenSeries": false, 3835 "id": 51, 3836 "legend": { 3837 "alignAsTable": true, 3838 "avg": false, 3839 "current": true, 3840 "hideEmpty": false, 3841 "max": true, 3842 "min": false, 3843 "rightSide": false, 3844 "show": true, 3845 "total": false, 3846 "values": true 3847 }, 3848 "lines": true, 3849 "linewidth": 1, 3850 "links": [], 3851 "nullPointMode": "null", 3852 "options": { 3853 "dataLinks": [] 3854 }, 3855 "paceLength": 10, 3856 "percentage": false, 3857 "pointradius": 2, 3858 "points": false, 3859 "renderer": "flot", 3860 "seriesOverrides": [ 3861 { 3862 "alias": "/.*resolvedts/", 3863 "yaxis": 2 3864 }, 3865 { 3866 "alias": "/.*chan/", 3867 "yaxis": 2 3868 } 3869 ], 3870 "spaceLength": 10, 3871 "stack": false, 3872 "steppedLine": false, 3873 "targets": [ 3874 { 3875 "expr": "sum(ticdc_puller_entry_sorter_unsorted_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", 3876 "format": "time_series", 3877 "intervalFactor": 1, 3878 "legendFormat": "{{capture}}-unsorted events", 3879 "refId": "A" 3880 }, 3881 { 3882 "expr": "-sum(ticdc_puller_entry_sorter_resolved_chan_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", 3883 "format": "time_series", 3884 "hide": true, 3885 "intervalFactor": 1, 3886 "legendFormat": "{{capture}}-resolvedts", 3887 "refId": "B" 3888 }, 3889 { 3890 "expr": "-sum(ticdc_puller_entry_sorter_output_chan_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", 3891 "format": "time_series", 3892 "hide": false, 3893 "intervalFactor": 1, 3894 "legendFormat": "{{capture}}-ouput chan", 3895 "refId": "C" 3896 } 3897 ], 3898 "thresholds": [], 3899 "timeFrom": null, 3900 "timeRegions": [], 3901 "timeShift": null, 3902 "title": "Entry sorter buffer size", 3903 "tooltip": { 3904 "shared": true, 3905 "sort": 0, 3906 "value_type": "individual" 3907 }, 3908 "type": "graph", 3909 "xaxis": { 3910 "buckets": null, 3911 "mode": "time", 3912 "name": null, 3913 "show": true, 3914 "values": [] 3915 }, 3916 "yaxes": [ 3917 { 3918 "format": "short", 3919 "label": null, 3920 "logBase": 1, 3921 "max": null, 3922 "min": null, 3923 "show": true 3924 }, 3925 { 3926 "format": "short", 3927 "label": null, 3928 "logBase": 1, 3929 "max": null, 3930 "min": null, 3931 "show": true 3932 } 3933 ], 3934 "yaxis": { 3935 "align": false, 3936 "alignLevel": null 3937 } 3938 }, 3939 { 3940 "aliasColors": {}, 3941 "bars": false, 3942 "dashLength": 10, 3943 "dashes": false, 3944 "datasource": "${DS_TEST-CLUSTER}", 3945 "description": "The number of events that are buffered in Processor's output channel and Mounter input channel", 3946 "fill": 1, 3947 "fillGradient": 0, 3948 "gridPos": { 3949 "h": 7, 3950 "w": 6, 3951 "x": 12, 3952 "y": 17 3953 }, 3954 "hiddenSeries": false, 3955 "id": 107, 3956 "legend": { 3957 "alignAsTable": true, 3958 "avg": false, 3959 "current": true, 3960 "hideEmpty": false, 3961 "max": true, 3962 "min": false, 3963 "rightSide": false, 3964 "show": true, 3965 "total": false, 3966 "values": true 3967 }, 3968 "lines": true, 3969 "linewidth": 1, 3970 "links": [], 3971 "nullPointMode": "null", 3972 "options": { 3973 "dataLinks": [] 3974 }, 3975 "paceLength": 10, 3976 "percentage": false, 3977 "pointradius": 2, 3978 "points": false, 3979 "renderer": "flot", 3980 "seriesOverrides": [ 3981 { 3982 "alias": "/.*processor.*/", 3983 "yaxis": 2 3984 } 3985 ], 3986 "spaceLength": 10, 3987 "stack": false, 3988 "steppedLine": false, 3989 "targets": [ 3990 { 3991 "expr": "sum(ticdc_mounter_input_chan_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", 3992 "format": "time_series", 3993 "intervalFactor": 1, 3994 "legendFormat": "{{capture}}-mounter input chan", 3995 "refId": "A" 3996 }, 3997 { 3998 "expr": "-sum(ticdc_sink_buffer_chan_size{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", 3999 "format": "time_series", 4000 "intervalFactor": 1, 4001 "legendFormat": "{{capture}}-sink buffer chan", 4002 "refId": "B" 4003 } 4004 ], 4005 "thresholds": [], 4006 "timeFrom": null, 4007 "timeRegions": [], 4008 "timeShift": null, 4009 "title": "Sink/Mounter buffer size", 4010 "tooltip": { 4011 "shared": true, 4012 "sort": 0, 4013 "value_type": "individual" 4014 }, 4015 "type": "graph", 4016 "xaxis": { 4017 "buckets": null, 4018 "mode": "time", 4019 "name": null, 4020 "show": true, 4021 "values": [] 4022 }, 4023 "yaxes": [ 4024 { 4025 "format": "short", 4026 "label": null, 4027 "logBase": 1, 4028 "max": null, 4029 "min": null, 4030 "show": true 4031 }, 4032 { 4033 "format": "short", 4034 "label": null, 4035 "logBase": 1, 4036 "max": null, 4037 "min": null, 4038 "show": true 4039 } 4040 ], 4041 "yaxis": { 4042 "align": false, 4043 "alignLevel": null 4044 } 4045 }, 4046 { 4047 "aliasColors": {}, 4048 "bars": false, 4049 "dashLength": 10, 4050 "dashes": false, 4051 "datasource": "${DS_TEST-CLUSTER}", 4052 "description": "The number of rows(events) that are buffered in Sink's pending flush rows buffer", 4053 "fill": 1, 4054 "fillGradient": 0, 4055 "gridPos": { 4056 "h": 7, 4057 "w": 6, 4058 "x": 18, 4059 "y": 17 4060 }, 4061 "hiddenSeries": false, 4062 "id": 96, 4063 "legend": { 4064 "alignAsTable": true, 4065 "avg": true, 4066 "current": true, 4067 "max": true, 4068 "min": false, 4069 "rightSide": false, 4070 "show": true, 4071 "sort": "current", 4072 "sortDesc": true, 4073 "total": false, 4074 "values": true 4075 }, 4076 "lines": true, 4077 "linewidth": 1, 4078 "links": [], 4079 "nullPointMode": "null", 4080 "options": { 4081 "dataLinks": [] 4082 }, 4083 "percentage": false, 4084 "pointradius": 2, 4085 "points": false, 4086 "renderer": "flot", 4087 "seriesOverrides": [], 4088 "spaceLength": 10, 4089 "stack": false, 4090 "steppedLine": false, 4091 "targets": [ 4092 { 4093 "expr": "sum(ticdc_sink_total_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture) - sum(ticdc_sink_total_flushed_rows_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}) by (capture)", 4094 "format": "time_series", 4095 "interval": "", 4096 "intervalFactor": 1, 4097 "legendFormat": "{{capture}}", 4098 "refId": "A" 4099 } 4100 ], 4101 "thresholds": [], 4102 "timeFrom": null, 4103 "timeRegions": [], 4104 "timeShift": null, 4105 "title": "Sink rows buffer size", 4106 "tooltip": { 4107 "shared": true, 4108 "sort": 2, 4109 "value_type": "individual" 4110 }, 4111 "type": "graph", 4112 "xaxis": { 4113 "buckets": null, 4114 "mode": "time", 4115 "name": null, 4116 "show": true, 4117 "values": [] 4118 }, 4119 "yaxes": [ 4120 { 4121 "format": "short", 4122 "label": null, 4123 "logBase": 1, 4124 "max": null, 4125 "min": null, 4126 "show": true 4127 }, 4128 { 4129 "format": "short", 4130 "label": null, 4131 "logBase": 1, 4132 "max": null, 4133 "min": null, 4134 "show": true 4135 } 4136 ], 4137 "yaxis": { 4138 "align": false, 4139 "alignLevel": null 4140 } 4141 }, 4142 { 4143 "cards": { 4144 "cardPadding": 0, 4145 "cardRound": 0 4146 }, 4147 "color": { 4148 "cardColor": "#FF9830", 4149 "colorScale": "linear", 4150 "colorScheme": "interpolateSpectral", 4151 "exponent": 0.5, 4152 "min": 0, 4153 "mode": "spectrum" 4154 }, 4155 "dataFormat": "tsbuckets", 4156 "datasource": "${DS_TEST-CLUSTER}", 4157 "description": "Duration of sorting unsorted events", 4158 "gridPos": { 4159 "h": 7, 4160 "w": 12, 4161 "x": 0, 4162 "y": 24 4163 }, 4164 "heatmap": {}, 4165 "hideZeroBuckets": true, 4166 "highlightCards": true, 4167 "id": 99, 4168 "legend": { 4169 "alignAsTable": true, 4170 "avg": false, 4171 "current": true, 4172 "max": true, 4173 "min": false, 4174 "rightSide": true, 4175 "show": true, 4176 "sort": "current", 4177 "sortDesc": true, 4178 "total": false, 4179 "values": true 4180 }, 4181 "links": [], 4182 "reverseYBuckets": false, 4183 "targets": [ 4184 { 4185 "expr": "sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", 4186 "format": "heatmap", 4187 "instant": false, 4188 "intervalFactor": 2, 4189 "legendFormat": "{{le}}", 4190 "refId": "A" 4191 } 4192 ], 4193 "title": "Entry sorter sort duration", 4194 "tooltip": { 4195 "show": true, 4196 "showHistogram": true 4197 }, 4198 "tooltipDecimals": 1, 4199 "type": "heatmap", 4200 "xAxis": { 4201 "show": true 4202 }, 4203 "xBucketNumber": null, 4204 "xBucketSize": null, 4205 "yAxis": { 4206 "decimals": 1, 4207 "format": "s", 4208 "logBase": 1, 4209 "max": null, 4210 "min": null, 4211 "show": true, 4212 "splitFactor": null 4213 }, 4214 "yBucketBound": "upper", 4215 "yBucketNumber": null, 4216 "yBucketSize": null 4217 }, 4218 { 4219 "aliasColors": {}, 4220 "bars": false, 4221 "dashLength": 10, 4222 "dashes": false, 4223 "datasource": "${DS_TEST-CLUSTER}", 4224 "description": "Percentiles of sorting events duration", 4225 "fill": 1, 4226 "fillGradient": 0, 4227 "gridPos": { 4228 "h": 7, 4229 "w": 12, 4230 "x": 12, 4231 "y": 24 4232 }, 4233 "hiddenSeries": false, 4234 "id": 53, 4235 "legend": { 4236 "alignAsTable": true, 4237 "avg": false, 4238 "current": true, 4239 "hideEmpty": false, 4240 "max": true, 4241 "min": false, 4242 "rightSide": true, 4243 "show": true, 4244 "total": false, 4245 "values": true 4246 }, 4247 "lines": true, 4248 "linewidth": 1, 4249 "links": [], 4250 "nullPointMode": "null", 4251 "options": { 4252 "dataLinks": [] 4253 }, 4254 "paceLength": 10, 4255 "percentage": false, 4256 "pointradius": 2, 4257 "points": false, 4258 "renderer": "flot", 4259 "seriesOverrides": [], 4260 "spaceLength": 10, 4261 "stack": false, 4262 "steppedLine": false, 4263 "targets": [ 4264 { 4265 "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", 4266 "format": "time_series", 4267 "intervalFactor": 1, 4268 "legendFormat": "{{capture}}-p999", 4269 "refId": "A" 4270 }, 4271 { 4272 "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_sort_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", 4273 "format": "time_series", 4274 "hide": false, 4275 "intervalFactor": 1, 4276 "legendFormat": "{{capture}}-p95", 4277 "refId": "B" 4278 } 4279 ], 4280 "thresholds": [], 4281 "timeFrom": null, 4282 "timeRegions": [], 4283 "timeShift": null, 4284 "title": "Entry sorter sort duration percentile", 4285 "tooltip": { 4286 "shared": true, 4287 "sort": 0, 4288 "value_type": "individual" 4289 }, 4290 "type": "graph", 4291 "xaxis": { 4292 "buckets": null, 4293 "mode": "time", 4294 "name": null, 4295 "show": true, 4296 "values": [] 4297 }, 4298 "yaxes": [ 4299 { 4300 "format": "s", 4301 "label": null, 4302 "logBase": 2, 4303 "max": null, 4304 "min": null, 4305 "show": true 4306 }, 4307 { 4308 "format": "short", 4309 "label": null, 4310 "logBase": 1, 4311 "max": null, 4312 "min": null, 4313 "show": true 4314 } 4315 ], 4316 "yaxis": { 4317 "align": false, 4318 "alignLevel": null 4319 } 4320 }, 4321 { 4322 "cards": { 4323 "cardPadding": 0, 4324 "cardRound": 0 4325 }, 4326 "color": { 4327 "cardColor": "#FF9830", 4328 "colorScale": "linear", 4329 "colorScheme": "interpolateSpectral", 4330 "exponent": 0.5, 4331 "min": 0, 4332 "mode": "spectrum" 4333 }, 4334 "dataFormat": "tsbuckets", 4335 "datasource": "${DS_TEST-CLUSTER}", 4336 "description": "Duration of merging sorted events", 4337 "gridPos": { 4338 "h": 7, 4339 "w": 12, 4340 "x": 0, 4341 "y": 31 4342 }, 4343 "heatmap": {}, 4344 "hideZeroBuckets": true, 4345 "highlightCards": true, 4346 "id": 105, 4347 "legend": { 4348 "alignAsTable": true, 4349 "avg": false, 4350 "current": true, 4351 "max": true, 4352 "min": false, 4353 "rightSide": true, 4354 "show": true, 4355 "sort": "current", 4356 "sortDesc": true, 4357 "total": false, 4358 "values": true 4359 }, 4360 "links": [], 4361 "reverseYBuckets": false, 4362 "targets": [ 4363 { 4364 "expr": "sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", 4365 "format": "heatmap", 4366 "instant": false, 4367 "intervalFactor": 2, 4368 "legendFormat": "{{le}}", 4369 "refId": "A" 4370 } 4371 ], 4372 "title": "Entry sorter merge duration", 4373 "tooltip": { 4374 "show": true, 4375 "showHistogram": true 4376 }, 4377 "tooltipDecimals": 1, 4378 "type": "heatmap", 4379 "xAxis": { 4380 "show": true 4381 }, 4382 "xBucketNumber": null, 4383 "xBucketSize": null, 4384 "yAxis": { 4385 "decimals": 1, 4386 "format": "s", 4387 "logBase": 1, 4388 "max": null, 4389 "min": null, 4390 "show": true, 4391 "splitFactor": null 4392 }, 4393 "yBucketBound": "upper", 4394 "yBucketNumber": null, 4395 "yBucketSize": null 4396 }, 4397 { 4398 "aliasColors": {}, 4399 "bars": false, 4400 "dashLength": 10, 4401 "dashes": false, 4402 "datasource": "${DS_TEST-CLUSTER}", 4403 "description": "Percentiles of merging sorted events duration", 4404 "fill": 1, 4405 "fillGradient": 0, 4406 "gridPos": { 4407 "h": 7, 4408 "w": 12, 4409 "x": 12, 4410 "y": 31 4411 }, 4412 "hiddenSeries": false, 4413 "id": 106, 4414 "legend": { 4415 "alignAsTable": true, 4416 "avg": false, 4417 "current": true, 4418 "hideEmpty": false, 4419 "max": true, 4420 "min": false, 4421 "rightSide": true, 4422 "show": true, 4423 "total": false, 4424 "values": true 4425 }, 4426 "lines": true, 4427 "linewidth": 1, 4428 "links": [], 4429 "nullPointMode": "null", 4430 "options": { 4431 "dataLinks": [] 4432 }, 4433 "paceLength": 10, 4434 "percentage": false, 4435 "pointradius": 2, 4436 "points": false, 4437 "renderer": "flot", 4438 "seriesOverrides": [], 4439 "spaceLength": 10, 4440 "stack": false, 4441 "steppedLine": false, 4442 "targets": [ 4443 { 4444 "expr": "histogram_quantile(0.999, sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", 4445 "format": "time_series", 4446 "intervalFactor": 1, 4447 "legendFormat": "{{capture}}-p999", 4448 "refId": "A" 4449 }, 4450 { 4451 "expr": "histogram_quantile(0.95, sum(rate(ticdc_puller_entry_sorter_merge_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le,capture))", 4452 "format": "time_series", 4453 "hide": false, 4454 "intervalFactor": 1, 4455 "legendFormat": "{{capture}}-p95", 4456 "refId": "B" 4457 } 4458 ], 4459 "thresholds": [], 4460 "timeFrom": null, 4461 "timeRegions": [], 4462 "timeShift": null, 4463 "title": "Entry sorter merge duration percentile", 4464 "tooltip": { 4465 "shared": true, 4466 "sort": 0, 4467 "value_type": "individual" 4468 }, 4469 "type": "graph", 4470 "xaxis": { 4471 "buckets": null, 4472 "mode": "time", 4473 "name": null, 4474 "show": true, 4475 "values": [] 4476 }, 4477 "yaxes": [ 4478 { 4479 "format": "s", 4480 "label": null, 4481 "logBase": 2, 4482 "max": null, 4483 "min": null, 4484 "show": true 4485 }, 4486 { 4487 "format": "short", 4488 "label": null, 4489 "logBase": 1, 4490 "max": null, 4491 "min": null, 4492 "show": true 4493 } 4494 ], 4495 "yaxis": { 4496 "align": false, 4497 "alignLevel": null 4498 } 4499 }, 4500 { 4501 "cards": { 4502 "cardPadding": 0, 4503 "cardRound": 0 4504 }, 4505 "color": { 4506 "cardColor": "#FF9830", 4507 "colorScale": "linear", 4508 "colorScheme": "interpolateSpectral", 4509 "exponent": 0.5, 4510 "min": 0, 4511 "mode": "spectrum" 4512 }, 4513 "dataFormat": "tsbuckets", 4514 "datasource": "${DS_TEST-CLUSTER}", 4515 "description": "Duration of unmarshal events from kv to SQL row", 4516 "gridPos": { 4517 "h": 7, 4518 "w": 12, 4519 "x": 0, 4520 "y": 38 4521 }, 4522 "heatmap": {}, 4523 "hideZeroBuckets": true, 4524 "highlightCards": true, 4525 "id": 101, 4526 "legend": { 4527 "alignAsTable": true, 4528 "avg": false, 4529 "current": true, 4530 "max": true, 4531 "min": false, 4532 "rightSide": true, 4533 "show": true, 4534 "sort": "current", 4535 "sortDesc": true, 4536 "total": false, 4537 "values": true 4538 }, 4539 "links": [], 4540 "reverseYBuckets": false, 4541 "targets": [ 4542 { 4543 "expr": "max(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}[1m])) by (le)", 4544 "format": "heatmap", 4545 "instant": false, 4546 "intervalFactor": 2, 4547 "legendFormat": "{{le}}", 4548 "refId": "A" 4549 } 4550 ], 4551 "title": "Mounter unmarshal duration", 4552 "tooltip": { 4553 "show": true, 4554 "showHistogram": true 4555 }, 4556 "tooltipDecimals": 1, 4557 "type": "heatmap", 4558 "xAxis": { 4559 "show": true 4560 }, 4561 "xBucketNumber": null, 4562 "xBucketSize": null, 4563 "yAxis": { 4564 "decimals": 1, 4565 "format": "s", 4566 "logBase": 1, 4567 "max": null, 4568 "min": null, 4569 "show": true, 4570 "splitFactor": null 4571 }, 4572 "yBucketBound": "upper", 4573 "yBucketNumber": null, 4574 "yBucketSize": null 4575 }, 4576 { 4577 "aliasColors": {}, 4578 "bars": false, 4579 "dashLength": 10, 4580 "dashes": false, 4581 "datasource": "${DS_TEST-CLUSTER}", 4582 "description": "Percentiles of unmarshal events from kv to SQL row duration", 4583 "fill": 0, 4584 "fillGradient": 0, 4585 "gridPos": { 4586 "h": 7, 4587 "w": 12, 4588 "x": 12, 4589 "y": 38 4590 }, 4591 "hiddenSeries": false, 4592 "id": 55, 4593 "legend": { 4594 "alignAsTable": true, 4595 "avg": false, 4596 "current": true, 4597 "max": true, 4598 "min": false, 4599 "rightSide": true, 4600 "show": true, 4601 "sort": "current", 4602 "sortDesc": true, 4603 "total": false, 4604 "values": true 4605 }, 4606 "lines": true, 4607 "linewidth": 1, 4608 "links": [], 4609 "nullPointMode": "null", 4610 "options": { 4611 "dataLinks": [] 4612 }, 4613 "paceLength": 10, 4614 "percentage": false, 4615 "pointradius": 2, 4616 "points": false, 4617 "renderer": "flot", 4618 "seriesOverrides": [], 4619 "spaceLength": 10, 4620 "stack": false, 4621 "steppedLine": false, 4622 "targets": [ 4623 { 4624 "expr": "histogram_quantile(0.99, sum(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le, capture))", 4625 "format": "time_series", 4626 "hide": false, 4627 "intervalFactor": 1, 4628 "legendFormat": "{{capture}}-p99", 4629 "refId": "A" 4630 }, 4631 { 4632 "expr": "histogram_quantile(0.999, sum(rate(ticdc_mounter_unmarshal_and_mount_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (le, capture))", 4633 "format": "time_series", 4634 "hide": true, 4635 "instant": false, 4636 "intervalFactor": 1, 4637 "legendFormat": "{{capture}}-p999", 4638 "refId": "B" 4639 } 4640 ], 4641 "thresholds": [], 4642 "timeFrom": null, 4643 "timeRegions": [], 4644 "timeShift": null, 4645 "title": "Mounter unmarshal duration percentile", 4646 "tooltip": { 4647 "shared": true, 4648 "sort": 2, 4649 "value_type": "individual" 4650 }, 4651 "type": "graph", 4652 "xaxis": { 4653 "buckets": null, 4654 "mode": "time", 4655 "name": null, 4656 "show": true, 4657 "values": [] 4658 }, 4659 "yaxes": [ 4660 { 4661 "format": "s", 4662 "label": null, 4663 "logBase": 1, 4664 "max": null, 4665 "min": null, 4666 "show": true 4667 }, 4668 { 4669 "format": "none", 4670 "label": null, 4671 "logBase": 1, 4672 "max": null, 4673 "min": null, 4674 "show": true 4675 } 4676 ], 4677 "yaxis": { 4678 "align": false, 4679 "alignLevel": null 4680 } 4681 }, 4682 { 4683 "aliasColors": {}, 4684 "bars": false, 4685 "dashLength": 10, 4686 "dashes": false, 4687 "datasource": "${DS_TEST-CLUSTER}", 4688 "description": "The number of KV client dispatched event per second", 4689 "fill": 1, 4690 "fillGradient": 0, 4691 "gridPos": { 4692 "h": 7, 4693 "w": 12, 4694 "x": 0, 4695 "y": 45 4696 }, 4697 "hiddenSeries": false, 4698 "id": 31, 4699 "legend": { 4700 "alignAsTable": true, 4701 "avg": false, 4702 "current": true, 4703 "hideEmpty": true, 4704 "hideZero": true, 4705 "max": true, 4706 "min": false, 4707 "rightSide": true, 4708 "show": true, 4709 "total": false, 4710 "values": true 4711 }, 4712 "lines": true, 4713 "linewidth": 1, 4714 "links": [], 4715 "nullPointMode": "null", 4716 "options": { 4717 "dataLinks": [] 4718 }, 4719 "paceLength": 10, 4720 "percentage": false, 4721 "pointradius": 2, 4722 "points": false, 4723 "renderer": "flot", 4724 "seriesOverrides": [ 4725 { 4726 "alias": "/.*batch-resolved/", 4727 "yaxis": 2 4728 } 4729 ], 4730 "spaceLength": 10, 4731 "stack": false, 4732 "steppedLine": false, 4733 "targets": [ 4734 { 4735 "expr": "sum(rate(ticdc_kvclient_send_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\"}[1m])) by (capture, changefeed, type)", 4736 "format": "time_series", 4737 "intervalFactor": 1, 4738 "legendFormat": "{{capture}}-{{changefeed}}-{{type}}", 4739 "refId": "A" 4740 }, 4741 { 4742 "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\",capture=~\"$capture\"}[1m])) by (capture, changefeed, table)", 4743 "format": "time_series", 4744 "intervalFactor": 1, 4745 "legendFormat": "{{capture}}-{{changefeed}}-batch-resolved", 4746 "refId": "B" 4747 } 4748 ], 4749 "thresholds": [], 4750 "timeFrom": null, 4751 "timeRegions": [], 4752 "timeShift": null, 4753 "title": "KV client dispatch events/s", 4754 "tooltip": { 4755 "shared": true, 4756 "sort": 0, 4757 "value_type": "individual" 4758 }, 4759 "type": "graph", 4760 "xaxis": { 4761 "buckets": null, 4762 "mode": "time", 4763 "name": null, 4764 "show": true, 4765 "values": [] 4766 }, 4767 "yaxes": [ 4768 { 4769 "format": "short", 4770 "label": null, 4771 "logBase": 1, 4772 "max": null, 4773 "min": null, 4774 "show": true 4775 }, 4776 { 4777 "format": "short", 4778 "label": null, 4779 "logBase": 1, 4780 "max": null, 4781 "min": null, 4782 "show": true 4783 } 4784 ], 4785 "yaxis": { 4786 "align": false, 4787 "alignLevel": null 4788 } 4789 }, 4790 { 4791 "cards": { 4792 "cardPadding": 0, 4793 "cardRound": 0 4794 }, 4795 "color": { 4796 "cardColor": "#FF9830", 4797 "colorScale": "linear", 4798 "colorScheme": "interpolateSpectral", 4799 "exponent": 0.5, 4800 "min": 0, 4801 "mode": "spectrum" 4802 }, 4803 "dataFormat": "tsbuckets", 4804 "datasource": "${DS_TEST-CLUSTER}", 4805 "description": "The size of batch resolved ts message from TiKV", 4806 "gridPos": { 4807 "h": 7, 4808 "w": 12, 4809 "x": 12, 4810 "y": 45 4811 }, 4812 "heatmap": {}, 4813 "hideZeroBuckets": true, 4814 "highlightCards": true, 4815 "id": 97, 4816 "legend": { 4817 "alignAsTable": true, 4818 "avg": false, 4819 "current": true, 4820 "max": true, 4821 "min": false, 4822 "rightSide": true, 4823 "show": true, 4824 "sort": "current", 4825 "sortDesc": true, 4826 "total": false, 4827 "values": true 4828 }, 4829 "links": [], 4830 "reverseYBuckets": false, 4831 "targets": [ 4832 { 4833 "expr": "sum(rate(ticdc_kvclient_batch_resolved_event_size_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)", 4834 "format": "heatmap", 4835 "instant": false, 4836 "intervalFactor": 2, 4837 "legendFormat": "{{le}}", 4838 "refId": "A" 4839 } 4840 ], 4841 "title": "KV client batch resolved size", 4842 "tooltip": { 4843 "show": true, 4844 "showHistogram": true 4845 }, 4846 "tooltipDecimals": 1, 4847 "type": "heatmap", 4848 "xAxis": { 4849 "show": true 4850 }, 4851 "xBucketNumber": null, 4852 "xBucketSize": null, 4853 "yAxis": { 4854 "decimals": 1, 4855 "format": "none", 4856 "logBase": 1, 4857 "max": null, 4858 "min": null, 4859 "show": true, 4860 "splitFactor": null 4861 }, 4862 "yBucketBound": "upper", 4863 "yBucketNumber": null, 4864 "yBucketSize": null 4865 }, 4866 { 4867 "aliasColors": {}, 4868 "bars": false, 4869 "dashLength": 10, 4870 "dashes": false, 4871 "datasource": "${DS_TEST-CLUSTER}", 4872 "description": "The number of regions that are being scanned", 4873 "fill": 1, 4874 "fillGradient": 0, 4875 "gridPos": { 4876 "h": 7, 4877 "w": 12, 4878 "x": 0, 4879 "y": 52 4880 }, 4881 "hiddenSeries": false, 4882 "id": 177, 4883 "legend": { 4884 "alignAsTable": true, 4885 "avg": false, 4886 "current": true, 4887 "hideEmpty": true, 4888 "hideZero": true, 4889 "max": true, 4890 "min": false, 4891 "rightSide": true, 4892 "show": true, 4893 "total": false, 4894 "values": true 4895 }, 4896 "lines": true, 4897 "linewidth": 1, 4898 "links": [], 4899 "nullPointMode": "null", 4900 "options": { 4901 "dataLinks": [] 4902 }, 4903 "paceLength": 10, 4904 "percentage": false, 4905 "pointradius": 2, 4906 "points": false, 4907 "renderer": "flot", 4908 "seriesOverrides": [], 4909 "spaceLength": 10, 4910 "stack": false, 4911 "steppedLine": false, 4912 "targets": [ 4913 { 4914 "expr": "sum(ticdc_kvclient_region_token{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}) by (capture, changefeed, store)", 4915 "format": "time_series", 4916 "intervalFactor": 1, 4917 "legendFormat": "{{changefeed}}-{{capture}}-{{store}}", 4918 "refId": "A" 4919 } 4920 ], 4921 "thresholds": [], 4922 "timeFrom": null, 4923 "timeRegions": [], 4924 "timeShift": null, 4925 "title": "KV client scanning regions", 4926 "tooltip": { 4927 "shared": true, 4928 "sort": 0, 4929 "value_type": "individual" 4930 }, 4931 "type": "graph", 4932 "xaxis": { 4933 "buckets": null, 4934 "mode": "time", 4935 "name": null, 4936 "show": true, 4937 "values": [] 4938 }, 4939 "yaxes": [ 4940 { 4941 "format": "short", 4942 "label": null, 4943 "logBase": 1, 4944 "max": null, 4945 "min": null, 4946 "show": true 4947 }, 4948 { 4949 "format": "short", 4950 "label": null, 4951 "logBase": 1, 4952 "max": null, 4953 "min": null, 4954 "show": true 4955 } 4956 ], 4957 "yaxis": { 4958 "align": false, 4959 "alignLevel": null 4960 } 4961 }, 4962 { 4963 "aliasColors": {}, 4964 "bars": false, 4965 "dashLength": 10, 4966 "dashes": false, 4967 "datasource": "${DS_TEST-CLUSTER}", 4968 "description": "Active stream count of each gRPC connection", 4969 "fill": 1, 4970 "fillGradient": 0, 4971 "gridPos": { 4972 "h": 7, 4973 "w": 12, 4974 "x": 12, 4975 "y": 52 4976 }, 4977 "hiddenSeries": false, 4978 "id": 188, 4979 "legend": { 4980 "alignAsTable": true, 4981 "avg": false, 4982 "current": true, 4983 "hideEmpty": true, 4984 "hideZero": true, 4985 "max": true, 4986 "min": false, 4987 "rightSide": true, 4988 "show": true, 4989 "total": false, 4990 "values": true 4991 }, 4992 "lines": true, 4993 "linewidth": 1, 4994 "links": [], 4995 "nullPointMode": "null", 4996 "options": { 4997 "dataLinks": [] 4998 }, 4999 "paceLength": 10, 5000 "percentage": false, 5001 "pointradius": 2, 5002 "points": false, 5003 "renderer": "flot", 5004 "seriesOverrides": [], 5005 "spaceLength": 10, 5006 "stack": false, 5007 "steppedLine": false, 5008 "targets": [ 5009 { 5010 "expr": "sum(ticdc_kvclient_grpc_stream_count{tidb_cluster=\"$tidb_cluster\"}) by (store)", 5011 "format": "time_series", 5012 "intervalFactor": 1, 5013 "legendFormat": "{{store}}", 5014 "refId": "A" 5015 } 5016 ], 5017 "thresholds": [], 5018 "timeFrom": null, 5019 "timeRegions": [], 5020 "timeShift": null, 5021 "title": "KV client gRPC stream count", 5022 "tooltip": { 5023 "shared": true, 5024 "sort": 0, 5025 "value_type": "individual" 5026 }, 5027 "type": "graph", 5028 "xaxis": { 5029 "buckets": null, 5030 "mode": "time", 5031 "name": null, 5032 "show": true, 5033 "values": [] 5034 }, 5035 "yaxes": [ 5036 { 5037 "format": "short", 5038 "label": null, 5039 "logBase": 1, 5040 "max": null, 5041 "min": null, 5042 "show": true 5043 }, 5044 { 5045 "format": "short", 5046 "label": null, 5047 "logBase": 1, 5048 "max": null, 5049 "min": null, 5050 "show": true 5051 } 5052 ], 5053 "yaxis": { 5054 "align": false, 5055 "alignLevel": null 5056 } 5057 } 5058 ], 5059 "title": "Events", 5060 "type": "row" 5061 }, 5062 { 5063 "collapsed": true, 5064 "gridPos": { 5065 "h": 1, 5066 "w": 24, 5067 "x": 0, 5068 "y": 3 5069 }, 5070 "id": 130, 5071 "panels": [ 5072 { 5073 "aliasColors": {}, 5074 "bars": false, 5075 "dashLength": 10, 5076 "dashes": false, 5077 "datasource": "${DS_TEST-CLUSTER}", 5078 "fill": 1, 5079 "gridPos": { 5080 "h": 8, 5081 "w": 12, 5082 "x": 0, 5083 "y": 4 5084 }, 5085 "id": 131, 5086 "legend": { 5087 "avg": false, 5088 "current": false, 5089 "max": false, 5090 "min": false, 5091 "show": true, 5092 "total": false, 5093 "values": false 5094 }, 5095 "lines": true, 5096 "linewidth": 1, 5097 "links": [], 5098 "nullPointMode": "null", 5099 "percentage": false, 5100 "pointradius": 2, 5101 "points": false, 5102 "renderer": "flot", 5103 "seriesOverrides": [], 5104 "spaceLength": 10, 5105 "stack": false, 5106 "steppedLine": false, 5107 "targets": [ 5108 { 5109 "expr": "sum(rate(ticdc_sorter_consume_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", 5110 "format": "time_series", 5111 "intervalFactor": 1, 5112 "legendFormat": "{{capture}}-{{changefeed}}", 5113 "refId": "A" 5114 } 5115 ], 5116 "thresholds": [], 5117 "timeFrom": null, 5118 "timeRegions": [], 5119 "timeShift": null, 5120 "title": "Unified Sorter intake rate", 5121 "tooltip": { 5122 "shared": true, 5123 "sort": 0, 5124 "value_type": "individual" 5125 }, 5126 "type": "graph", 5127 "xaxis": { 5128 "buckets": null, 5129 "mode": "time", 5130 "name": null, 5131 "show": true, 5132 "values": [] 5133 }, 5134 "yaxes": [ 5135 { 5136 "format": "short", 5137 "label": null, 5138 "logBase": 1, 5139 "max": null, 5140 "min": null, 5141 "show": true 5142 }, 5143 { 5144 "format": "short", 5145 "label": null, 5146 "logBase": 1, 5147 "max": null, 5148 "min": null, 5149 "show": true 5150 } 5151 ], 5152 "yaxis": { 5153 "align": false, 5154 "alignLevel": null 5155 } 5156 }, 5157 { 5158 "aliasColors": {}, 5159 "bars": false, 5160 "dashLength": 10, 5161 "dashes": false, 5162 "datasource": "${DS_TEST-CLUSTER}", 5163 "fill": 1, 5164 "gridPos": { 5165 "h": 8, 5166 "w": 12, 5167 "x": 12, 5168 "y": 4 5169 }, 5170 "id": 132, 5171 "legend": { 5172 "avg": false, 5173 "current": false, 5174 "max": false, 5175 "min": false, 5176 "show": true, 5177 "total": false, 5178 "values": false 5179 }, 5180 "lines": true, 5181 "linewidth": 1, 5182 "links": [], 5183 "nullPointMode": "null", 5184 "percentage": false, 5185 "pointradius": 2, 5186 "points": false, 5187 "renderer": "flot", 5188 "seriesOverrides": [], 5189 "spaceLength": 10, 5190 "stack": false, 5191 "steppedLine": false, 5192 "targets": [ 5193 { 5194 "expr": "sum(rate(ticdc_sorter_event_count{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (capture,changefeed)", 5195 "format": "time_series", 5196 "intervalFactor": 1, 5197 "legendFormat": "{{capture}}-{{changefeed}}", 5198 "refId": "A" 5199 } 5200 ], 5201 "thresholds": [], 5202 "timeFrom": null, 5203 "timeRegions": [], 5204 "timeShift": null, 5205 "title": "Unified Sorter event output rate", 5206 "tooltip": { 5207 "shared": true, 5208 "sort": 0, 5209 "value_type": "individual" 5210 }, 5211 "type": "graph", 5212 "xaxis": { 5213 "buckets": null, 5214 "mode": "time", 5215 "name": null, 5216 "show": true, 5217 "values": [] 5218 }, 5219 "yaxes": [ 5220 { 5221 "format": "short", 5222 "label": null, 5223 "logBase": 1, 5224 "max": null, 5225 "min": null, 5226 "show": true 5227 }, 5228 { 5229 "format": "short", 5230 "label": null, 5231 "logBase": 1, 5232 "max": null, 5233 "min": null, 5234 "show": true 5235 } 5236 ], 5237 "yaxis": { 5238 "align": false, 5239 "alignLevel": null 5240 } 5241 }, 5242 { 5243 "aliasColors": {}, 5244 "bars": false, 5245 "dashLength": 10, 5246 "dashes": false, 5247 "datasource": "${DS_TEST-CLUSTER}", 5248 "fill": 1, 5249 "gridPos": { 5250 "h": 8, 5251 "w": 12, 5252 "x": 0, 5253 "y": 12 5254 }, 5255 "id": 133, 5256 "legend": { 5257 "avg": false, 5258 "current": false, 5259 "max": false, 5260 "min": false, 5261 "show": true, 5262 "total": false, 5263 "values": false 5264 }, 5265 "lines": true, 5266 "linewidth": 1, 5267 "links": [], 5268 "nullPointMode": "null", 5269 "percentage": false, 5270 "pointradius": 2, 5271 "points": false, 5272 "renderer": "flot", 5273 "seriesOverrides": [], 5274 "spaceLength": 10, 5275 "stack": false, 5276 "steppedLine": false, 5277 "targets": [ 5278 { 5279 "expr": "sum(ticdc_sorter_on_disk_data_size_gauge{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}) by (capture)", 5280 "format": "time_series", 5281 "intervalFactor": 1, 5282 "legendFormat": "{{capture}}", 5283 "refId": "A" 5284 } 5285 ], 5286 "thresholds": [], 5287 "timeFrom": null, 5288 "timeRegions": [], 5289 "timeShift": null, 5290 "title": "Unified Sorter on disk data size", 5291 "tooltip": { 5292 "shared": true, 5293 "sort": 0, 5294 "value_type": "individual" 5295 }, 5296 "type": "graph", 5297 "xaxis": { 5298 "buckets": null, 5299 "mode": "time", 5300 "name": null, 5301 "show": true, 5302 "values": [] 5303 }, 5304 "yaxes": [ 5305 { 5306 "format": "bytes", 5307 "label": null, 5308 "logBase": 1, 5309 "max": null, 5310 "min": "0", 5311 "show": true 5312 }, 5313 { 5314 "format": "short", 5315 "label": null, 5316 "logBase": 1, 5317 "max": null, 5318 "min": null, 5319 "show": true 5320 } 5321 ], 5322 "yaxis": { 5323 "align": false, 5324 "alignLevel": null 5325 } 5326 }, 5327 { 5328 "aliasColors": {}, 5329 "bars": false, 5330 "dashLength": 10, 5331 "dashes": false, 5332 "datasource": "${DS_TEST-CLUSTER}", 5333 "fill": 1, 5334 "gridPos": { 5335 "h": 8, 5336 "w": 12, 5337 "x": 12, 5338 "y": 12 5339 }, 5340 "id": 134, 5341 "legend": { 5342 "avg": false, 5343 "current": false, 5344 "max": false, 5345 "min": false, 5346 "show": true, 5347 "total": false, 5348 "values": false 5349 }, 5350 "lines": true, 5351 "linewidth": 1, 5352 "links": [], 5353 "nullPointMode": "null", 5354 "percentage": false, 5355 "pointradius": 2, 5356 "points": false, 5357 "renderer": "flot", 5358 "seriesOverrides": [], 5359 "spaceLength": 10, 5360 "stack": false, 5361 "steppedLine": false, 5362 "targets": [ 5363 { 5364 "expr": "sum(ticdc_sorter_in_memory_data_size_gauge{tidb_cluster=\"$tidb_cluster\", capture=~\"$capture\"}) by (capture)", 5365 "format": "time_series", 5366 "intervalFactor": 1, 5367 "legendFormat": "{{capture}}", 5368 "refId": "A" 5369 } 5370 ], 5371 "thresholds": [], 5372 "timeFrom": null, 5373 "timeRegions": [], 5374 "timeShift": null, 5375 "title": "Unified Sorter in-memory data size", 5376 "tooltip": { 5377 "shared": true, 5378 "sort": 0, 5379 "value_type": "individual" 5380 }, 5381 "type": "graph", 5382 "xaxis": { 5383 "buckets": null, 5384 "mode": "time", 5385 "name": null, 5386 "show": true, 5387 "values": [] 5388 }, 5389 "yaxes": [ 5390 { 5391 "format": "bytes", 5392 "label": null, 5393 "logBase": 1, 5394 "max": null, 5395 "min": "0", 5396 "show": true 5397 }, 5398 { 5399 "format": "short", 5400 "label": null, 5401 "logBase": 1, 5402 "max": null, 5403 "min": null, 5404 "show": true 5405 } 5406 ], 5407 "yaxis": { 5408 "align": false, 5409 "alignLevel": null 5410 } 5411 }, 5412 { 5413 "cards": { 5414 "cardPadding": null, 5415 "cardRound": null 5416 }, 5417 "color": { 5418 "cardColor": "#b4ff00", 5419 "colorScale": "sqrt", 5420 "colorScheme": "interpolateOranges", 5421 "exponent": 0.5, 5422 "max": null, 5423 "min": null, 5424 "mode": "spectrum" 5425 }, 5426 "dataFormat": "tsbuckets", 5427 "datasource": "${DS_TEST-CLUSTER}", 5428 "gridPos": { 5429 "h": 8, 5430 "w": 12, 5431 "x": 0, 5432 "y": 20 5433 }, 5434 "heatmap": {}, 5435 "hideZeroBuckets": true, 5436 "highlightCards": true, 5437 "id": 135, 5438 "legend": { 5439 "show": false 5440 }, 5441 "links": [], 5442 "reverseYBuckets": false, 5443 "targets": [ 5444 { 5445 "expr": "sum(rate(ticdc_sorter_flush_count_histogram_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (le)", 5446 "format": "heatmap", 5447 "intervalFactor": 1, 5448 "legendFormat": "{{le}}", 5449 "refId": "A" 5450 } 5451 ], 5452 "timeFrom": null, 5453 "timeShift": null, 5454 "title": "Unified Sorter flush sizes", 5455 "tooltip": { 5456 "show": true, 5457 "showHistogram": false 5458 }, 5459 "type": "heatmap", 5460 "xAxis": { 5461 "show": true 5462 }, 5463 "xBucketNumber": null, 5464 "xBucketSize": null, 5465 "yAxis": { 5466 "decimals": null, 5467 "format": "none", 5468 "logBase": 1, 5469 "max": null, 5470 "min": null, 5471 "show": true, 5472 "splitFactor": null 5473 }, 5474 "yBucketBound": "auto", 5475 "yBucketNumber": null, 5476 "yBucketSize": null 5477 }, 5478 { 5479 "cards": { 5480 "cardPadding": null, 5481 "cardRound": null 5482 }, 5483 "color": { 5484 "cardColor": "#b4ff00", 5485 "colorScale": "sqrt", 5486 "colorScheme": "interpolateBlues", 5487 "exponent": 0.5, 5488 "mode": "spectrum" 5489 }, 5490 "dataFormat": "tsbuckets", 5491 "datasource": "${DS_TEST-CLUSTER}", 5492 "gridPos": { 5493 "h": 8, 5494 "w": 12, 5495 "x": 12, 5496 "y": 20 5497 }, 5498 "heatmap": {}, 5499 "hideZeroBuckets": true, 5500 "highlightCards": true, 5501 "id": 136, 5502 "legend": { 5503 "show": false 5504 }, 5505 "links": [], 5506 "reverseYBuckets": false, 5507 "targets": [ 5508 { 5509 "expr": "sum(rate(ticdc_sorter_merge_count_histogram_bucket{tidb_cluster=\"$tidb_cluster\", changefeed=~\"$changefeed\", capture=~\"$capture\"}[1m])) by (le)", 5510 "format": "heatmap", 5511 "intervalFactor": 1, 5512 "legendFormat": "{{le}}", 5513 "refId": "A" 5514 } 5515 ], 5516 "timeFrom": null, 5517 "timeShift": null, 5518 "title": "Unified Sorter merge size", 5519 "tooltip": { 5520 "show": true, 5521 "showHistogram": false 5522 }, 5523 "type": "heatmap", 5524 "xAxis": { 5525 "show": true 5526 }, 5527 "xBucketNumber": null, 5528 "xBucketSize": null, 5529 "yAxis": { 5530 "decimals": null, 5531 "format": "none", 5532 "logBase": 1, 5533 "max": null, 5534 "min": null, 5535 "show": true, 5536 "splitFactor": null 5537 }, 5538 "yBucketBound": "auto", 5539 "yBucketNumber": null, 5540 "yBucketSize": null 5541 } 5542 ], 5543 "title": "Unified Sorter", 5544 "type": "row" 5545 }, 5546 { 5547 "collapsed": true, 5548 "gridPos": { 5549 "h": 1, 5550 "w": 24, 5551 "x": 0, 5552 "y": 4 5553 }, 5554 "id": 58, 5555 "panels": [ 5556 { 5557 "aliasColors": {}, 5558 "bars": false, 5559 "dashLength": 10, 5560 "dashes": false, 5561 "datasource": "${DS_TEST-CLUSTER}", 5562 "decimals": 1, 5563 "fill": 1, 5564 "gridPos": { 5565 "h": 7, 5566 "w": 12, 5567 "x": 0, 5568 "y": 5 5569 }, 5570 "id": 60, 5571 "legend": { 5572 "alignAsTable": true, 5573 "avg": false, 5574 "current": true, 5575 "max": true, 5576 "min": false, 5577 "rightSide": true, 5578 "show": true, 5579 "sideWidth": null, 5580 "sort": "current", 5581 "sortDesc": true, 5582 "total": false, 5583 "values": true 5584 }, 5585 "lines": true, 5586 "linewidth": 1, 5587 "links": [], 5588 "nullPointMode": "null", 5589 "paceLength": 10, 5590 "percentage": false, 5591 "pointradius": 5, 5592 "points": false, 5593 "renderer": "flot", 5594 "seriesOverrides": [], 5595 "spaceLength": 10, 5596 "stack": false, 5597 "steppedLine": false, 5598 "targets": [ 5599 { 5600 "expr": "sum(rate(tikv_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"cdc_.*|cdc\"}[1m])) by (instance)", 5601 "format": "time_series", 5602 "intervalFactor": 2, 5603 "legendFormat": "{{instance}}", 5604 "refId": "A", 5605 "step": 4 5606 } 5607 ], 5608 "thresholds": [], 5609 "timeFrom": null, 5610 "timeRegions": [], 5611 "timeShift": null, 5612 "title": "CDC endpoint CPU", 5613 "tooltip": { 5614 "shared": true, 5615 "sort": 0, 5616 "value_type": "individual" 5617 }, 5618 "type": "graph", 5619 "xaxis": { 5620 "buckets": null, 5621 "mode": "time", 5622 "name": null, 5623 "show": true, 5624 "values": [] 5625 }, 5626 "yaxes": [ 5627 { 5628 "format": "percentunit", 5629 "label": null, 5630 "logBase": 1, 5631 "max": null, 5632 "min": null, 5633 "show": true 5634 }, 5635 { 5636 "format": "short", 5637 "label": null, 5638 "logBase": 1, 5639 "max": null, 5640 "min": null, 5641 "show": true 5642 } 5643 ], 5644 "yaxis": { 5645 "align": false, 5646 "alignLevel": null 5647 } 5648 }, 5649 { 5650 "aliasColors": {}, 5651 "bars": false, 5652 "dashLength": 10, 5653 "dashes": false, 5654 "datasource": "${DS_TEST-CLUSTER}", 5655 "decimals": 1, 5656 "fill": 1, 5657 "gridPos": { 5658 "h": 7, 5659 "w": 12, 5660 "x": 12, 5661 "y": 5 5662 }, 5663 "id": 62, 5664 "legend": { 5665 "alignAsTable": true, 5666 "avg": false, 5667 "current": true, 5668 "max": true, 5669 "min": false, 5670 "rightSide": true, 5671 "show": true, 5672 "sideWidth": null, 5673 "sort": "current", 5674 "sortDesc": true, 5675 "total": false, 5676 "values": true 5677 }, 5678 "lines": true, 5679 "linewidth": 1, 5680 "links": [], 5681 "nullPointMode": "null", 5682 "paceLength": 10, 5683 "percentage": false, 5684 "pointradius": 5, 5685 "points": false, 5686 "renderer": "flot", 5687 "seriesOverrides": [ 5688 { 5689 "alias": "/.*tso/", 5690 "yaxis": 2 5691 } 5692 ], 5693 "spaceLength": 10, 5694 "stack": false, 5695 "steppedLine": false, 5696 "targets": [ 5697 { 5698 "expr": "sum(rate(tikv_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"cdcwkr.*\"}[1m])) by (instance)", 5699 "format": "time_series", 5700 "intervalFactor": 2, 5701 "legendFormat": "{{instance}} - worker", 5702 "refId": "A", 5703 "step": 4 5704 }, 5705 { 5706 "expr": "sum(rate(tikv_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", name=~\"tso\"}[1m])) by (instance)", 5707 "format": "time_series", 5708 "intervalFactor": 2, 5709 "legendFormat": "{{instance}} - tso", 5710 "refId": "B", 5711 "step": 4 5712 } 5713 ], 5714 "thresholds": [], 5715 "timeFrom": null, 5716 "timeRegions": [], 5717 "timeShift": null, 5718 "title": "CDC worker CPU", 5719 "tooltip": { 5720 "shared": true, 5721 "sort": 0, 5722 "value_type": "individual" 5723 }, 5724 "type": "graph", 5725 "xaxis": { 5726 "buckets": null, 5727 "mode": "time", 5728 "name": null, 5729 "show": true, 5730 "values": [] 5731 }, 5732 "yaxes": [ 5733 { 5734 "format": "percentunit", 5735 "label": null, 5736 "logBase": 1, 5737 "max": null, 5738 "min": null, 5739 "show": true 5740 }, 5741 { 5742 "format": "percentunit", 5743 "label": null, 5744 "logBase": 1, 5745 "max": null, 5746 "min": null, 5747 "show": true 5748 } 5749 ], 5750 "yaxis": { 5751 "align": false, 5752 "alignLevel": null 5753 } 5754 }, 5755 { 5756 "aliasColors": {}, 5757 "bars": false, 5758 "dashLength": 10, 5759 "dashes": false, 5760 "datasource": "${DS_TEST-CLUSTER}", 5761 "decimals": 1, 5762 "description": "The count of different kinds of gRPC message", 5763 "fill": 1, 5764 "gridPos": { 5765 "h": 7, 5766 "w": 12, 5767 "x": 0, 5768 "y": 12 5769 }, 5770 "id": 147, 5771 "legend": { 5772 "alignAsTable": true, 5773 "avg": false, 5774 "current": true, 5775 "hideEmpty": true, 5776 "hideZero": true, 5777 "max": true, 5778 "min": false, 5779 "rightSide": true, 5780 "show": true, 5781 "sideWidth": null, 5782 "sort": "current", 5783 "sortDesc": true, 5784 "total": false, 5785 "values": true 5786 }, 5787 "lines": true, 5788 "linewidth": 1, 5789 "links": [], 5790 "nullPointMode": "null", 5791 "percentage": false, 5792 "pointradius": 5, 5793 "points": false, 5794 "renderer": "flot", 5795 "seriesOverrides": [], 5796 "spaceLength": 10, 5797 "stack": false, 5798 "steppedLine": false, 5799 "targets": [ 5800 { 5801 "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type!=\"kv_gc\"}[1m])) by (type)", 5802 "format": "time_series", 5803 "intervalFactor": 2, 5804 "legendFormat": "{{type}}", 5805 "metric": "tikv_grpc_msg_duration_seconds_bucket", 5806 "refId": "A", 5807 "step": 10 5808 } 5809 ], 5810 "thresholds": [], 5811 "timeFrom": null, 5812 "timeRegions": [], 5813 "timeShift": null, 5814 "title": "gRPC message count", 5815 "tooltip": { 5816 "shared": true, 5817 "sort": 0, 5818 "value_type": "individual" 5819 }, 5820 "type": "graph", 5821 "xaxis": { 5822 "buckets": null, 5823 "mode": "time", 5824 "name": null, 5825 "show": true, 5826 "values": [] 5827 }, 5828 "yaxes": [ 5829 { 5830 "format": "ops", 5831 "label": null, 5832 "logBase": 1, 5833 "max": null, 5834 "min": null, 5835 "show": true 5836 }, 5837 { 5838 "format": "short", 5839 "label": null, 5840 "logBase": 1, 5841 "max": null, 5842 "min": null, 5843 "show": true 5844 } 5845 ], 5846 "yaxis": { 5847 "align": false, 5848 "alignLevel": null 5849 } 5850 }, 5851 { 5852 "aliasColors": {}, 5853 "bars": false, 5854 "dashLength": 10, 5855 "dashes": false, 5856 "datasource": "${DS_TEST-CLUSTER}", 5857 "decimals": 1, 5858 "description": "The memory usage per TiKV instance", 5859 "editable": true, 5860 "error": false, 5861 "fill": 0, 5862 "grid": {}, 5863 "gridPos": { 5864 "h": 7, 5865 "w": 12, 5866 "x": 12, 5867 "y": 12 5868 }, 5869 "id": 74, 5870 "legend": { 5871 "alignAsTable": true, 5872 "avg": false, 5873 "current": true, 5874 "max": true, 5875 "min": false, 5876 "rightSide": true, 5877 "show": true, 5878 "sideWidth": null, 5879 "sort": "max", 5880 "sortDesc": true, 5881 "total": false, 5882 "values": true 5883 }, 5884 "lines": true, 5885 "linewidth": 1, 5886 "links": [], 5887 "nullPointMode": "null", 5888 "paceLength": 10, 5889 "percentage": false, 5890 "pointradius": 5, 5891 "points": false, 5892 "renderer": "flot", 5893 "seriesOverrides": [ 5894 { 5895 "alias": "/.*-cap-.*/", 5896 "yaxis": 2 5897 }, 5898 { 5899 "alias": "/.*tikv.*/", 5900 "pointradius": 1, 5901 "points": true, 5902 "yaxis": 2 5903 } 5904 ], 5905 "spaceLength": 10, 5906 "stack": false, 5907 "steppedLine": false, 5908 "targets": [ 5909 { 5910 "expr": "avg(process_resident_memory_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=~\"tikv.*\"}) by (instance)", 5911 "format": "time_series", 5912 "hide": true, 5913 "intervalFactor": 2, 5914 "legendFormat": "tikv-{{instance}}", 5915 "refId": "A", 5916 "step": 10 5917 }, 5918 { 5919 "expr": "avg(process_resident_memory_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=~\"cdc.*\"}) by (instance)", 5920 "format": "time_series", 5921 "hide": true, 5922 "intervalFactor": 2, 5923 "legendFormat": "cdc-{{instance}}", 5924 "refId": "B", 5925 "step": 10 5926 }, 5927 { 5928 "expr": "(avg(process_resident_memory_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=~\"tikv.*\"}) by (instance)) - (avg(tikv_engine_block_cache_size_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", db=\"kv\"}) by(instance))", 5929 "format": "time_series", 5930 "hide": false, 5931 "intervalFactor": 2, 5932 "legendFormat": "tikv-{{instance}}", 5933 "refId": "C", 5934 "step": 10 5935 }, 5936 { 5937 "expr": "sum(tikv_cdc_sink_memory_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", 5938 "format": "time_series", 5939 "hide": false, 5940 "intervalFactor": 2, 5941 "legendFormat": "sink-{{instance}}", 5942 "refId": "D", 5943 "step": 10 5944 }, 5945 { 5946 "expr": "sum(tikv_cdc_old_value_cache_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", 5947 "format": "time_series", 5948 "hide": false, 5949 "intervalFactor": 2, 5950 "legendFormat": "old-value-{{instance}}", 5951 "refId": "E", 5952 "step": 10 5953 }, 5954 { 5955 "expr": "sum(tikv_cdc_sink_memory_capacity{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", 5956 "format": "time_series", 5957 "hide": true, 5958 "intervalFactor": 2, 5959 "legendFormat": "sink-cap-{{instance}}", 5960 "refId": "F", 5961 "step": 10 5962 }, 5963 { 5964 "expr": "sum(tikv_cdc_old_value_cache_memory_quota{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", 5965 "format": "time_series", 5966 "hide": true, 5967 "intervalFactor": 2, 5968 "legendFormat": "old-value-cap-{{instance}}", 5969 "refId": "G", 5970 "step": 10 5971 } 5972 ], 5973 "thresholds": [], 5974 "timeFrom": null, 5975 "timeRegions": [], 5976 "timeShift": null, 5977 "title": "CDC memory", 5978 "tooltip": { 5979 "msResolution": false, 5980 "shared": true, 5981 "sort": 0, 5982 "value_type": "individual" 5983 }, 5984 "type": "graph", 5985 "xaxis": { 5986 "buckets": null, 5987 "mode": "time", 5988 "name": null, 5989 "show": true, 5990 "values": [] 5991 }, 5992 "yaxes": [ 5993 { 5994 "format": "bytes", 5995 "label": null, 5996 "logBase": 1, 5997 "max": null, 5998 "min": "0", 5999 "show": true 6000 }, 6001 { 6002 "format": "bytes", 6003 "label": null, 6004 "logBase": 1, 6005 "max": null, 6006 "min": null, 6007 "show": true 6008 } 6009 ], 6010 "yaxis": { 6011 "align": false, 6012 "alignLevel": null 6013 } 6014 }, 6015 { 6016 "aliasColors": {}, 6017 "bars": false, 6018 "dashLength": 10, 6019 "dashes": false, 6020 "datasource": "${DS_TEST-CLUSTER}", 6021 "decimals": 1, 6022 "description": "The min resolved ts of each TiKV", 6023 "editable": true, 6024 "error": false, 6025 "fill": 0, 6026 "grid": {}, 6027 "gridPos": { 6028 "h": 7, 6029 "w": 7, 6030 "x": 0, 6031 "y": 19 6032 }, 6033 "id": 152, 6034 "legend": { 6035 "alignAsTable": true, 6036 "avg": false, 6037 "current": true, 6038 "max": false, 6039 "min": false, 6040 "rightSide": false, 6041 "show": true, 6042 "sideWidth": null, 6043 "sort": null, 6044 "sortDesc": null, 6045 "total": false, 6046 "values": true 6047 }, 6048 "lines": true, 6049 "linewidth": 1, 6050 "links": [], 6051 "nullPointMode": "null", 6052 "paceLength": 10, 6053 "percentage": false, 6054 "pointradius": 1, 6055 "points": false, 6056 "renderer": "flot", 6057 "seriesOverrides": [ 6058 { 6059 "alias": "/.*-ts/", 6060 "lines": true, 6061 "linewidth": 3, 6062 "points": false, 6063 "yaxis": 2 6064 }, 6065 { 6066 "alias": "/.*-lag/", 6067 "bars": true, 6068 "fill": 1 6069 } 6070 ], 6071 "spaceLength": 10, 6072 "stack": false, 6073 "steppedLine": false, 6074 "targets": [ 6075 { 6076 "expr": "scalar(max(pd_cluster_tso{tidb_cluster=\"$tidb_cluster\"})) - avg(tikv_cdc_min_resolved_ts{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}/1000) by (instance) > 0", 6077 "format": "time_series", 6078 "hide": false, 6079 "instant": false, 6080 "intervalFactor": 2, 6081 "legendFormat": "{{instance}}-min-resolved-lag", 6082 "refId": "A", 6083 "step": 10 6084 }, 6085 { 6086 "expr": "max(pd_cluster_tso{tidb_cluster=\"$tidb_cluster\"}) * 1000", 6087 "format": "time_series", 6088 "hide": false, 6089 "intervalFactor": 2, 6090 "legendFormat": "current-ts", 6091 "refId": "B", 6092 "step": 10 6093 }, 6094 { 6095 "expr": "avg(tikv_cdc_min_resolved_ts{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", 6096 "format": "time_series", 6097 "hide": false, 6098 "intervalFactor": 2, 6099 "legendFormat": "{{instance}}-min-resolved-ts", 6100 "refId": "C", 6101 "step": 10 6102 } 6103 ], 6104 "thresholds": [], 6105 "timeFrom": null, 6106 "timeRegions": [], 6107 "timeShift": null, 6108 "title": "Min resolved ts", 6109 "tooltip": { 6110 "msResolution": false, 6111 "shared": true, 6112 "sort": 0, 6113 "value_type": "individual" 6114 }, 6115 "type": "graph", 6116 "xaxis": { 6117 "buckets": null, 6118 "mode": "time", 6119 "name": null, 6120 "show": true, 6121 "values": [] 6122 }, 6123 "yaxes": [ 6124 { 6125 "format": "s", 6126 "label": null, 6127 "logBase": 1, 6128 "max": null, 6129 "min": "0", 6130 "show": true 6131 }, 6132 { 6133 "format": "none", 6134 "label": null, 6135 "logBase": 1, 6136 "max": null, 6137 "min": null, 6138 "show": true 6139 } 6140 ], 6141 "yaxis": { 6142 "align": false, 6143 "alignLevel": null 6144 } 6145 }, 6146 { 6147 "aliasColors": {}, 6148 "bars": false, 6149 "dashLength": 10, 6150 "dashes": false, 6151 "datasource": "${DS_TEST-CLUSTER}", 6152 "decimals": 1, 6153 "description": "The ID of the min resolved region of each TiKV", 6154 "editable": true, 6155 "error": false, 6156 "fill": 0, 6157 "grid": {}, 6158 "gridPos": { 6159 "h": 7, 6160 "w": 5, 6161 "x": 7, 6162 "y": 19 6163 }, 6164 "id": 153, 6165 "legend": { 6166 "alignAsTable": true, 6167 "avg": false, 6168 "current": true, 6169 "max": true, 6170 "min": false, 6171 "rightSide": false, 6172 "show": true, 6173 "sideWidth": null, 6174 "sort": null, 6175 "sortDesc": null, 6176 "total": false, 6177 "values": true 6178 }, 6179 "lines": false, 6180 "linewidth": 1, 6181 "links": [], 6182 "nullPointMode": "null", 6183 "paceLength": 10, 6184 "percentage": false, 6185 "pointradius": 1, 6186 "points": true, 6187 "renderer": "flot", 6188 "seriesOverrides": [], 6189 "spaceLength": 10, 6190 "stack": false, 6191 "steppedLine": false, 6192 "targets": [ 6193 { 6194 "expr": "avg(tikv_cdc_min_resolved_ts_region{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", 6195 "format": "time_series", 6196 "hide": false, 6197 "intervalFactor": 2, 6198 "legendFormat": "{{instance}}-min-resolved-region", 6199 "refId": "A", 6200 "step": 10 6201 } 6202 ], 6203 "thresholds": [], 6204 "timeFrom": null, 6205 "timeRegions": [], 6206 "timeShift": null, 6207 "title": "Min resolved Region", 6208 "tooltip": { 6209 "msResolution": false, 6210 "shared": true, 6211 "sort": 0, 6212 "value_type": "individual" 6213 }, 6214 "type": "graph", 6215 "xaxis": { 6216 "buckets": null, 6217 "mode": "time", 6218 "name": null, 6219 "show": true, 6220 "values": [] 6221 }, 6222 "yaxes": [ 6223 { 6224 "format": "none", 6225 "label": null, 6226 "logBase": 1, 6227 "max": null, 6228 "min": "0", 6229 "show": true 6230 }, 6231 { 6232 "format": "none", 6233 "label": null, 6234 "logBase": 1, 6235 "max": null, 6236 "min": null, 6237 "show": false 6238 } 6239 ], 6240 "yaxis": { 6241 "align": false, 6242 "alignLevel": null 6243 } 6244 }, 6245 { 6246 "aliasColors": {}, 6247 "bars": false, 6248 "dashLength": 10, 6249 "dashes": false, 6250 "datasource": "${DS_TEST-CLUSTER}", 6251 "decimals": 1, 6252 "description": "", 6253 "fill": 1, 6254 "gridPos": { 6255 "h": 7, 6256 "w": 12, 6257 "x": 12, 6258 "y": 19 6259 }, 6260 "id": 70, 6261 "legend": { 6262 "alignAsTable": true, 6263 "avg": false, 6264 "current": true, 6265 "hideEmpty": false, 6266 "hideZero": true, 6267 "max": true, 6268 "min": false, 6269 "rightSide": true, 6270 "show": true, 6271 "sideWidth": null, 6272 "sort": null, 6273 "sortDesc": null, 6274 "total": false, 6275 "values": true 6276 }, 6277 "lines": true, 6278 "linewidth": 1, 6279 "links": [], 6280 "nullPointMode": "null", 6281 "paceLength": 10, 6282 "percentage": false, 6283 "pointradius": 5, 6284 "points": false, 6285 "renderer": "flot", 6286 "seriesOverrides": [], 6287 "spaceLength": 10, 6288 "stack": false, 6289 "steppedLine": false, 6290 "targets": [ 6291 { 6292 "expr": "histogram_quantile(0.99999, sum(rate(tikv_cdc_resolved_ts_gap_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance))", 6293 "format": "time_series", 6294 "intervalFactor": 1, 6295 "legendFormat": "{{instance}}-p9999", 6296 "refId": "A" 6297 } 6298 ], 6299 "thresholds": [], 6300 "timeFrom": null, 6301 "timeRegions": [], 6302 "timeShift": null, 6303 "title": "Resolved ts lag duration percentile", 6304 "tooltip": { 6305 "shared": true, 6306 "sort": 0, 6307 "value_type": "individual" 6308 }, 6309 "type": "graph", 6310 "xaxis": { 6311 "buckets": null, 6312 "mode": "time", 6313 "name": null, 6314 "show": true, 6315 "values": [] 6316 }, 6317 "yaxes": [ 6318 { 6319 "format": "s", 6320 "label": null, 6321 "logBase": 10, 6322 "max": null, 6323 "min": null, 6324 "show": true 6325 }, 6326 { 6327 "format": "short", 6328 "label": null, 6329 "logBase": 1, 6330 "max": null, 6331 "min": null, 6332 "show": true 6333 } 6334 ], 6335 "yaxis": { 6336 "align": false, 6337 "alignLevel": null 6338 } 6339 }, 6340 { 6341 "cards": { 6342 "cardPadding": 0, 6343 "cardRound": 0 6344 }, 6345 "color": { 6346 "cardColor": "#FF9830", 6347 "colorScale": "linear", 6348 "colorScheme": "interpolateSpectral", 6349 "exponent": 0.5, 6350 "min": 0, 6351 "mode": "spectrum" 6352 }, 6353 "dataFormat": "tsbuckets", 6354 "datasource": "${DS_TEST-CLUSTER}", 6355 "description": "The time consumed to CDC incremental scan", 6356 "gridPos": { 6357 "h": 7, 6358 "w": 12, 6359 "x": 0, 6360 "y": 26 6361 }, 6362 "heatmap": {}, 6363 "hideZeroBuckets": true, 6364 "highlightCards": true, 6365 "id": 68, 6366 "legend": { 6367 "alignAsTable": true, 6368 "avg": false, 6369 "current": true, 6370 "max": true, 6371 "min": false, 6372 "rightSide": true, 6373 "show": true, 6374 "sort": "current", 6375 "sortDesc": true, 6376 "total": false, 6377 "values": true 6378 }, 6379 "links": [], 6380 "maxPerRow": 3, 6381 "repeat": null, 6382 "repeatDirection": "h", 6383 "reverseYBuckets": false, 6384 "targets": [ 6385 { 6386 "expr": "sum(rate(tikv_cdc_scan_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)", 6387 "format": "heatmap", 6388 "instant": false, 6389 "intervalFactor": 2, 6390 "legendFormat": "{{le}}", 6391 "refId": "A" 6392 } 6393 ], 6394 "title": "Initial scan duration", 6395 "tooltip": { 6396 "show": true, 6397 "showHistogram": true 6398 }, 6399 "tooltipDecimals": 1, 6400 "type": "heatmap", 6401 "xAxis": { 6402 "show": true 6403 }, 6404 "xBucketNumber": null, 6405 "xBucketSize": null, 6406 "yAxis": { 6407 "decimals": 1, 6408 "format": "s", 6409 "logBase": 1, 6410 "max": null, 6411 "min": null, 6412 "show": true, 6413 "splitFactor": null 6414 }, 6415 "yBucketBound": "upper", 6416 "yBucketNumber": null, 6417 "yBucketSize": null 6418 }, 6419 { 6420 "aliasColors": {}, 6421 "bars": false, 6422 "dashLength": 10, 6423 "dashes": false, 6424 "datasource": "${DS_TEST-CLUSTER}", 6425 "decimals": 1, 6426 "description": "", 6427 "fill": 1, 6428 "gridPos": { 6429 "h": 7, 6430 "w": 6, 6431 "x": 12, 6432 "y": 26 6433 }, 6434 "id": 72, 6435 "legend": { 6436 "alignAsTable": true, 6437 "avg": false, 6438 "current": true, 6439 "hideEmpty": false, 6440 "hideZero": true, 6441 "max": true, 6442 "min": false, 6443 "rightSide": false, 6444 "show": true, 6445 "sideWidth": null, 6446 "sort": "current", 6447 "sortDesc": true, 6448 "total": false, 6449 "values": true 6450 }, 6451 "lines": true, 6452 "linewidth": 1, 6453 "links": [], 6454 "nullPointMode": "null", 6455 "paceLength": 10, 6456 "percentage": false, 6457 "pointradius": 5, 6458 "points": false, 6459 "renderer": "flot", 6460 "seriesOverrides": [], 6461 "spaceLength": 10, 6462 "stack": false, 6463 "steppedLine": false, 6464 "targets": [ 6465 { 6466 "expr": "histogram_quantile(0.9999, sum(rate(tikv_cdc_scan_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance))", 6467 "format": "time_series", 6468 "intervalFactor": 1, 6469 "legendFormat": "{{instance}}-p9999", 6470 "refId": "A" 6471 } 6472 ], 6473 "thresholds": [], 6474 "timeFrom": null, 6475 "timeRegions": [], 6476 "timeShift": null, 6477 "title": "Initial scan duration percentile", 6478 "tooltip": { 6479 "shared": true, 6480 "sort": 0, 6481 "value_type": "individual" 6482 }, 6483 "type": "graph", 6484 "xaxis": { 6485 "buckets": null, 6486 "mode": "time", 6487 "name": null, 6488 "show": true, 6489 "values": [] 6490 }, 6491 "yaxes": [ 6492 { 6493 "format": "s", 6494 "label": null, 6495 "logBase": 1, 6496 "max": null, 6497 "min": null, 6498 "show": true 6499 }, 6500 { 6501 "format": "short", 6502 "label": null, 6503 "logBase": 1, 6504 "max": null, 6505 "min": null, 6506 "show": false 6507 } 6508 ], 6509 "yaxis": { 6510 "align": false, 6511 "alignLevel": null 6512 } 6513 }, 6514 { 6515 "aliasColors": {}, 6516 "bars": false, 6517 "dashLength": 10, 6518 "dashes": false, 6519 "datasource": "${DS_TEST-CLUSTER}", 6520 "decimals": 1, 6521 "description": "The number of incremental scan task in different status.", 6522 "fill": 1, 6523 "gridPos": { 6524 "h": 7, 6525 "w": 6, 6526 "x": 18, 6527 "y": 26 6528 }, 6529 "id": 140, 6530 "legend": { 6531 "alignAsTable": true, 6532 "avg": false, 6533 "current": true, 6534 "hideEmpty": false, 6535 "hideZero": true, 6536 "max": true, 6537 "min": false, 6538 "rightSide": false, 6539 "show": true, 6540 "sideWidth": null, 6541 "sort": "current", 6542 "sortDesc": true, 6543 "total": false, 6544 "values": true 6545 }, 6546 "lines": true, 6547 "linewidth": 1, 6548 "links": [], 6549 "nullPointMode": "null", 6550 "paceLength": 10, 6551 "percentage": false, 6552 "pointradius": 5, 6553 "points": false, 6554 "renderer": "flot", 6555 "seriesOverrides": [ 6556 { 6557 "alias": "/.*ongoing/", 6558 "yaxis": 2 6559 } 6560 ], 6561 "spaceLength": 10, 6562 "stack": false, 6563 "steppedLine": false, 6564 "targets": [ 6565 { 6566 "expr": "sum(tikv_cdc_scan_tasks{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"ongoing\"}) by (type, instance)", 6567 "format": "time_series", 6568 "hide": false, 6569 "intervalFactor": 1, 6570 "legendFormat": "{{instance}} - {{type}}", 6571 "refId": "A" 6572 }, 6573 { 6574 "expr": "sum(tikv_cdc_scan_tasks{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=\"total\"}) by (instance) - sum(tikv_cdc_scan_tasks{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", type=~\"abort|finish\"}) by (instance)", 6575 "format": "time_series", 6576 "hide": false, 6577 "intervalFactor": 1, 6578 "legendFormat": "{{instance}} - pending", 6579 "refId": "B" 6580 } 6581 ], 6582 "thresholds": [], 6583 "timeFrom": null, 6584 "timeRegions": [], 6585 "timeShift": null, 6586 "title": "Initial scan tasks status", 6587 "tooltip": { 6588 "shared": true, 6589 "sort": 0, 6590 "value_type": "individual" 6591 }, 6592 "type": "graph", 6593 "xaxis": { 6594 "buckets": null, 6595 "mode": "time", 6596 "name": null, 6597 "show": true, 6598 "values": [] 6599 }, 6600 "yaxes": [ 6601 { 6602 "format": "none", 6603 "label": null, 6604 "logBase": 1, 6605 "max": null, 6606 "min": null, 6607 "show": true 6608 }, 6609 { 6610 "format": "none", 6611 "label": null, 6612 "logBase": 1, 6613 "max": null, 6614 "min": null, 6615 "show": true 6616 } 6617 ], 6618 "yaxis": { 6619 "align": false, 6620 "alignLevel": null 6621 } 6622 }, 6623 { 6624 "aliasColors": {}, 6625 "bars": false, 6626 "dashLength": 10, 6627 "dashes": false, 6628 "datasource": "${DS_TEST-CLUSTER}", 6629 "decimals": 1, 6630 "description": "The memory usage per TiKV instance", 6631 "editable": true, 6632 "error": false, 6633 "fill": 0, 6634 "grid": {}, 6635 "gridPos": { 6636 "h": 7, 6637 "w": 12, 6638 "x": 0, 6639 "y": 33 6640 }, 6641 "id": 78, 6642 "legend": { 6643 "alignAsTable": true, 6644 "avg": false, 6645 "current": true, 6646 "max": true, 6647 "min": false, 6648 "rightSide": true, 6649 "show": true, 6650 "sideWidth": null, 6651 "sort": null, 6652 "sortDesc": null, 6653 "total": false, 6654 "values": true 6655 }, 6656 "lines": true, 6657 "linewidth": 1, 6658 "links": [], 6659 "nullPointMode": "null", 6660 "paceLength": 10, 6661 "percentage": false, 6662 "pointradius": 5, 6663 "points": false, 6664 "renderer": "flot", 6665 "seriesOverrides": [], 6666 "spaceLength": 10, 6667 "stack": false, 6668 "steppedLine": false, 6669 "targets": [ 6670 { 6671 "expr": "avg(tikv_cdc_captured_region_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", 6672 "format": "time_series", 6673 "hide": false, 6674 "intervalFactor": 2, 6675 "legendFormat": "tikv-{{instance}}-total", 6676 "refId": "A", 6677 "step": 10 6678 }, 6679 { 6680 "expr": "sum(tikv_cdc_region_resolve_status{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance, status)", 6681 "format": "time_series", 6682 "hide": false, 6683 "intervalFactor": 2, 6684 "legendFormat": "tikv-{{instance}}-{{status}}", 6685 "refId": "B", 6686 "step": 10 6687 } 6688 ], 6689 "thresholds": [], 6690 "timeFrom": null, 6691 "timeRegions": [], 6692 "timeShift": null, 6693 "title": "Captured region count", 6694 "tooltip": { 6695 "msResolution": false, 6696 "shared": true, 6697 "sort": 0, 6698 "value_type": "individual" 6699 }, 6700 "type": "graph", 6701 "xaxis": { 6702 "buckets": null, 6703 "mode": "time", 6704 "name": null, 6705 "show": true, 6706 "values": [] 6707 }, 6708 "yaxes": [ 6709 { 6710 "format": "none", 6711 "label": null, 6712 "logBase": 1, 6713 "max": null, 6714 "min": "0", 6715 "show": true 6716 }, 6717 { 6718 "format": "short", 6719 "label": null, 6720 "logBase": 1, 6721 "max": null, 6722 "min": null, 6723 "show": true 6724 } 6725 ], 6726 "yaxis": { 6727 "align": false, 6728 "alignLevel": null 6729 } 6730 }, 6731 { 6732 "aliasColors": {}, 6733 "bars": false, 6734 "dashLength": 10, 6735 "dashes": false, 6736 "datasource": "${DS_TEST-CLUSTER}", 6737 "decimals": 1, 6738 "description": "The speed of TiKV CDC incremental scan", 6739 "editable": true, 6740 "error": false, 6741 "fill": 0, 6742 "grid": {}, 6743 "gridPos": { 6744 "h": 7, 6745 "w": 6, 6746 "x": 12, 6747 "y": 33 6748 }, 6749 "id": 76, 6750 "legend": { 6751 "alignAsTable": true, 6752 "avg": false, 6753 "current": true, 6754 "hideEmpty": true, 6755 "hideZero": true, 6756 "max": true, 6757 "min": false, 6758 "rightSide": false, 6759 "show": true, 6760 "sideWidth": null, 6761 "sort": "current", 6762 "sortDesc": true, 6763 "total": false, 6764 "values": true 6765 }, 6766 "lines": true, 6767 "linewidth": 1, 6768 "links": [], 6769 "nullPointMode": "null", 6770 "paceLength": 10, 6771 "percentage": false, 6772 "pointradius": 5, 6773 "points": false, 6774 "renderer": "flot", 6775 "seriesOverrides": [], 6776 "spaceLength": 10, 6777 "stack": false, 6778 "steppedLine": false, 6779 "targets": [ 6780 { 6781 "expr": "sum(rate(tikv_cdc_scan_bytes_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=\"tikv\"}[30s])) by (instance)", 6782 "format": "time_series", 6783 "hide": false, 6784 "intervalFactor": 2, 6785 "legendFormat": "tikv-{{instance}}", 6786 "refId": "A", 6787 "step": 10 6788 } 6789 ], 6790 "thresholds": [], 6791 "timeFrom": null, 6792 "timeRegions": [], 6793 "timeShift": null, 6794 "title": "CDC scan speed", 6795 "tooltip": { 6796 "msResolution": false, 6797 "shared": true, 6798 "sort": 0, 6799 "value_type": "individual" 6800 }, 6801 "type": "graph", 6802 "xaxis": { 6803 "buckets": null, 6804 "mode": "time", 6805 "name": null, 6806 "show": true, 6807 "values": [] 6808 }, 6809 "yaxes": [ 6810 { 6811 "format": "bytes", 6812 "label": null, 6813 "logBase": 1, 6814 "max": null, 6815 "min": "0", 6816 "show": true 6817 }, 6818 { 6819 "format": "short", 6820 "label": null, 6821 "logBase": 1, 6822 "max": null, 6823 "min": null, 6824 "show": true 6825 } 6826 ], 6827 "yaxis": { 6828 "align": false, 6829 "alignLevel": null 6830 } 6831 }, 6832 { 6833 "aliasColors": {}, 6834 "bars": true, 6835 "dashLength": 10, 6836 "dashes": false, 6837 "datasource": "${DS_TEST-CLUSTER}", 6838 "decimals": 1, 6839 "description": "The total bytes of TiKV CDC incremental scan", 6840 "editable": true, 6841 "error": false, 6842 "fill": 0, 6843 "grid": {}, 6844 "gridPos": { 6845 "h": 7, 6846 "w": 6, 6847 "x": 18, 6848 "y": 33 6849 }, 6850 "id": 139, 6851 "legend": { 6852 "alignAsTable": true, 6853 "avg": false, 6854 "current": true, 6855 "hideEmpty": true, 6856 "hideZero": true, 6857 "max": true, 6858 "min": false, 6859 "rightSide": false, 6860 "show": true, 6861 "sideWidth": null, 6862 "sort": "current", 6863 "sortDesc": true, 6864 "total": false, 6865 "values": true 6866 }, 6867 "lines": false, 6868 "linewidth": 1, 6869 "links": [], 6870 "nullPointMode": "null", 6871 "paceLength": 10, 6872 "percentage": false, 6873 "pointradius": 5, 6874 "points": false, 6875 "renderer": "flot", 6876 "seriesOverrides": [], 6877 "spaceLength": 10, 6878 "stack": false, 6879 "steppedLine": false, 6880 "targets": [ 6881 { 6882 "expr": "sum(tikv_cdc_scan_bytes_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\", job=\"tikv\"}) by (instance)", 6883 "format": "time_series", 6884 "hide": false, 6885 "intervalFactor": 2, 6886 "legendFormat": "tikv-{{instance}}", 6887 "refId": "A", 6888 "step": 10 6889 } 6890 ], 6891 "thresholds": [], 6892 "timeFrom": null, 6893 "timeRegions": [], 6894 "timeShift": null, 6895 "title": "CDC total scan bytes", 6896 "tooltip": { 6897 "msResolution": false, 6898 "shared": true, 6899 "sort": 0, 6900 "value_type": "individual" 6901 }, 6902 "type": "graph", 6903 "xaxis": { 6904 "buckets": null, 6905 "mode": "time", 6906 "name": null, 6907 "show": true, 6908 "values": [] 6909 }, 6910 "yaxes": [ 6911 { 6912 "format": "bytes", 6913 "label": null, 6914 "logBase": 1, 6915 "max": null, 6916 "min": null, 6917 "show": true 6918 }, 6919 { 6920 "format": "short", 6921 "label": null, 6922 "logBase": 1, 6923 "max": null, 6924 "min": null, 6925 "show": true 6926 } 6927 ], 6928 "yaxis": { 6929 "align": false, 6930 "alignLevel": null 6931 } 6932 }, 6933 { 6934 "aliasColors": {}, 6935 "bars": false, 6936 "dashLength": 10, 6937 "dashes": false, 6938 "datasource": "${DS_TEST-CLUSTER}", 6939 "decimals": 1, 6940 "description": "", 6941 "fill": 4, 6942 "gridPos": { 6943 "h": 7, 6944 "w": 12, 6945 "x": 0, 6946 "y": 40 6947 }, 6948 "id": 143, 6949 "legend": { 6950 "alignAsTable": true, 6951 "avg": false, 6952 "current": true, 6953 "hideEmpty": false, 6954 "hideZero": true, 6955 "max": false, 6956 "min": true, 6957 "rightSide": true, 6958 "show": true, 6959 "sideWidth": null, 6960 "sort": "current", 6961 "sortDesc": true, 6962 "total": false, 6963 "values": true 6964 }, 6965 "lines": true, 6966 "linewidth": 1, 6967 "links": [], 6968 "nullPointMode": "null", 6969 "paceLength": 10, 6970 "percentage": false, 6971 "pointradius": 1, 6972 "points": true, 6973 "renderer": "flot", 6974 "seriesOverrides": [ 6975 { 6976 "alias": "/(access|miss).*/", 6977 "fill": 0, 6978 "points": false, 6979 "yaxis": 2 6980 } 6981 ], 6982 "spaceLength": 10, 6983 "stack": false, 6984 "steppedLine": false, 6985 "targets": [ 6986 { 6987 "expr": "(sum(rate(tikv_cdc_old_value_cache_access{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance) - sum(rate(tikv_cdc_old_value_cache_miss{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)) / sum(rate(tikv_cdc_old_value_cache_access{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)", 6988 "format": "time_series", 6989 "hide": false, 6990 "intervalFactor": 1, 6991 "legendFormat": "hit-rate-{{instance}}", 6992 "refId": "A" 6993 }, 6994 { 6995 "expr": "-sum(rate(tikv_cdc_old_value_cache_access{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)", 6996 "format": "time_series", 6997 "hide": true, 6998 "intervalFactor": 1, 6999 "legendFormat": "access-{{instance}}", 7000 "refId": "B" 7001 }, 7002 { 7003 "expr": "-sum(rate(tikv_cdc_old_value_cache_miss{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)", 7004 "format": "time_series", 7005 "hide": false, 7006 "intervalFactor": 1, 7007 "legendFormat": "miss-{{instance}}", 7008 "refId": "C" 7009 }, 7010 { 7011 "expr": "-sum(rate(tikv_cdc_old_value_cache_miss_none{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance)", 7012 "format": "time_series", 7013 "hide": false, 7014 "intervalFactor": 1, 7015 "legendFormat": "miss-none-{{instance}}", 7016 "refId": "D" 7017 } 7018 ], 7019 "thresholds": [], 7020 "timeFrom": null, 7021 "timeRegions": [], 7022 "timeShift": null, 7023 "title": "Old value cache hit", 7024 "tooltip": { 7025 "shared": true, 7026 "sort": 0, 7027 "value_type": "individual" 7028 }, 7029 "type": "graph", 7030 "xaxis": { 7031 "buckets": null, 7032 "mode": "time", 7033 "name": null, 7034 "show": true, 7035 "values": [] 7036 }, 7037 "yaxes": [ 7038 { 7039 "format": "percentunit", 7040 "label": null, 7041 "logBase": 1, 7042 "max": "1", 7043 "min": "0", 7044 "show": true 7045 }, 7046 { 7047 "format": "none", 7048 "label": null, 7049 "logBase": 1, 7050 "max": null, 7051 "min": null, 7052 "show": true 7053 } 7054 ], 7055 "yaxis": { 7056 "align": false, 7057 "alignLevel": null 7058 } 7059 }, 7060 { 7061 "aliasColors": {}, 7062 "bars": false, 7063 "dashLength": 10, 7064 "dashes": false, 7065 "datasource": "${DS_TEST-CLUSTER}", 7066 "decimals": 1, 7067 "description": "The total number of cache entries in the old value cache.", 7068 "fill": 1, 7069 "gridPos": { 7070 "h": 7, 7071 "w": 12, 7072 "x": 12, 7073 "y": 40 7074 }, 7075 "id": 145, 7076 "legend": { 7077 "alignAsTable": true, 7078 "avg": false, 7079 "current": true, 7080 "hideEmpty": false, 7081 "hideZero": true, 7082 "max": true, 7083 "min": false, 7084 "rightSide": true, 7085 "show": true, 7086 "sideWidth": null, 7087 "sort": "current", 7088 "sortDesc": true, 7089 "total": false, 7090 "values": true 7091 }, 7092 "lines": true, 7093 "linewidth": 1, 7094 "links": [], 7095 "nullPointMode": "null", 7096 "paceLength": 10, 7097 "percentage": false, 7098 "pointradius": 1, 7099 "points": false, 7100 "renderer": "flot", 7101 "seriesOverrides": [ 7102 { 7103 "alias": "/.*len/", 7104 "yaxis": 2 7105 } 7106 ], 7107 "spaceLength": 10, 7108 "stack": false, 7109 "steppedLine": false, 7110 "targets": [ 7111 { 7112 "expr": "sum(tikv_cdc_old_value_cache_length{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", 7113 "format": "time_series", 7114 "hide": false, 7115 "intervalFactor": 1, 7116 "legendFormat": "{{instance}} - len", 7117 "refId": "A" 7118 }, 7119 { 7120 "expr": "sum(tikv_cdc_old_value_cache_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance) / sum(tikv_cdc_old_value_cache_length{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", 7121 "format": "time_series", 7122 "hide": false, 7123 "intervalFactor": 1, 7124 "legendFormat": "{{instance}} - avg entry bytes", 7125 "refId": "B" 7126 }, 7127 { 7128 "expr": "sum(tikv_cdc_old_value_cache_memory_quota{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}) by (instance)", 7129 "format": "time_series", 7130 "hide": false, 7131 "intervalFactor": 1, 7132 "legendFormat": "{{instance}} - quota", 7133 "refId": "C" 7134 } 7135 ], 7136 "thresholds": [], 7137 "timeFrom": null, 7138 "timeRegions": [], 7139 "timeShift": null, 7140 "title": "Old value cache size", 7141 "tooltip": { 7142 "shared": true, 7143 "sort": 0, 7144 "value_type": "individual" 7145 }, 7146 "type": "graph", 7147 "xaxis": { 7148 "buckets": null, 7149 "mode": "time", 7150 "name": null, 7151 "show": true, 7152 "values": [] 7153 }, 7154 "yaxes": [ 7155 { 7156 "format": "bytes", 7157 "label": null, 7158 "logBase": 1, 7159 "max": null, 7160 "min": "0", 7161 "show": true 7162 }, 7163 { 7164 "decimals": null, 7165 "format": "none", 7166 "label": null, 7167 "logBase": 1, 7168 "max": null, 7169 "min": "0", 7170 "show": true 7171 } 7172 ], 7173 "yaxis": { 7174 "align": false, 7175 "alignLevel": null 7176 } 7177 }, 7178 { 7179 "aliasColors": {}, 7180 "bars": false, 7181 "dashLength": 10, 7182 "dashes": false, 7183 "datasource": "${DS_TEST-CLUSTER}", 7184 "decimals": 1, 7185 "description": "", 7186 "editable": true, 7187 "error": false, 7188 "fill": 0, 7189 "grid": {}, 7190 "gridPos": { 7191 "h": 7, 7192 "w": 12, 7193 "x": 0, 7194 "y": 47 7195 }, 7196 "id": 141, 7197 "legend": { 7198 "alignAsTable": true, 7199 "avg": false, 7200 "current": true, 7201 "hideEmpty": true, 7202 "hideZero": true, 7203 "max": true, 7204 "min": false, 7205 "rightSide": true, 7206 "show": true, 7207 "sideWidth": null, 7208 "sort": "current", 7209 "sortDesc": true, 7210 "total": false, 7211 "values": true 7212 }, 7213 "lines": true, 7214 "linewidth": 1, 7215 "links": [], 7216 "nullPointMode": "null", 7217 "paceLength": 10, 7218 "percentage": false, 7219 "pointradius": 5, 7220 "points": false, 7221 "renderer": "flot", 7222 "seriesOverrides": [], 7223 "spaceLength": 10, 7224 "stack": false, 7225 "steppedLine": false, 7226 "targets": [ 7227 { 7228 "expr": "sum(rate(tikv_cdc_old_value_scan_details{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (instance, cf, tag)", 7229 "format": "time_series", 7230 "hide": false, 7231 "intervalFactor": 2, 7232 "legendFormat": "{{instance}}- {{cf}} - {{tag}}", 7233 "refId": "A", 7234 "step": 10 7235 } 7236 ], 7237 "thresholds": [], 7238 "timeFrom": null, 7239 "timeRegions": [], 7240 "timeShift": null, 7241 "title": "Old value seek operation", 7242 "tooltip": { 7243 "msResolution": false, 7244 "shared": true, 7245 "sort": 0, 7246 "value_type": "individual" 7247 }, 7248 "type": "graph", 7249 "xaxis": { 7250 "buckets": null, 7251 "mode": "time", 7252 "name": null, 7253 "show": true, 7254 "values": [] 7255 }, 7256 "yaxes": [ 7257 { 7258 "format": "ops", 7259 "label": null, 7260 "logBase": 1, 7261 "max": null, 7262 "min": "0", 7263 "show": true 7264 }, 7265 { 7266 "format": "short", 7267 "label": null, 7268 "logBase": 1, 7269 "max": null, 7270 "min": null, 7271 "show": true 7272 } 7273 ], 7274 "yaxis": { 7275 "align": false, 7276 "alignLevel": null 7277 } 7278 }, 7279 { 7280 "cards": { 7281 "cardPadding": 0, 7282 "cardRound": 0 7283 }, 7284 "color": { 7285 "cardColor": "#FF9830", 7286 "colorScale": "linear", 7287 "colorScheme": "interpolateSpectral", 7288 "exponent": 0.5, 7289 "min": 0, 7290 "mode": "spectrum" 7291 }, 7292 "dataFormat": "tsbuckets", 7293 "datasource": "${DS_TEST-CLUSTER}", 7294 "description": "The time consumed to get an old value (both from cache and from disk)", 7295 "gridPos": { 7296 "h": 7, 7297 "w": 6, 7298 "x": 12, 7299 "y": 47 7300 }, 7301 "heatmap": {}, 7302 "hideZeroBuckets": true, 7303 "highlightCards": true, 7304 "id": 146, 7305 "legend": { 7306 "alignAsTable": true, 7307 "avg": false, 7308 "current": true, 7309 "max": true, 7310 "min": false, 7311 "rightSide": true, 7312 "show": true, 7313 "sort": "current", 7314 "sortDesc": true, 7315 "total": false, 7316 "values": true 7317 }, 7318 "links": [], 7319 "maxPerRow": 3, 7320 "repeatDirection": "h", 7321 "reverseYBuckets": false, 7322 "targets": [ 7323 { 7324 "expr": "sum(rate(tikv_cdc_old_value_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le)", 7325 "format": "heatmap", 7326 "instant": false, 7327 "intervalFactor": 2, 7328 "legendFormat": "{{le}}", 7329 "refId": "A" 7330 } 7331 ], 7332 "title": "Old value seek duration", 7333 "tooltip": { 7334 "show": true, 7335 "showHistogram": true 7336 }, 7337 "tooltipDecimals": 1, 7338 "type": "heatmap", 7339 "xAxis": { 7340 "show": true 7341 }, 7342 "xBucketNumber": null, 7343 "xBucketSize": null, 7344 "yAxis": { 7345 "decimals": 1, 7346 "format": "s", 7347 "logBase": 1, 7348 "max": null, 7349 "min": null, 7350 "show": true, 7351 "splitFactor": null 7352 }, 7353 "yBucketBound": "upper", 7354 "yBucketNumber": null, 7355 "yBucketSize": null 7356 }, 7357 { 7358 "aliasColors": {}, 7359 "bars": false, 7360 "dashLength": 10, 7361 "dashes": false, 7362 "datasource": "${DS_TEST-CLUSTER}", 7363 "decimals": 1, 7364 "description": "", 7365 "fill": 1, 7366 "gridPos": { 7367 "h": 7, 7368 "w": 6, 7369 "x": 18, 7370 "y": 47 7371 }, 7372 "id": 142, 7373 "legend": { 7374 "alignAsTable": true, 7375 "avg": false, 7376 "current": true, 7377 "hideEmpty": false, 7378 "hideZero": true, 7379 "max": true, 7380 "min": false, 7381 "rightSide": false, 7382 "show": true, 7383 "sideWidth": null, 7384 "sort": "current", 7385 "sortDesc": true, 7386 "total": false, 7387 "values": true 7388 }, 7389 "lines": true, 7390 "linewidth": 1, 7391 "links": [], 7392 "nullPointMode": "null", 7393 "paceLength": 10, 7394 "percentage": false, 7395 "pointradius": 5, 7396 "points": false, 7397 "renderer": "flot", 7398 "seriesOverrides": [], 7399 "spaceLength": 10, 7400 "stack": false, 7401 "steppedLine": false, 7402 "targets": [ 7403 { 7404 "expr": "histogram_quantile(0.99, sum(rate(tikv_cdc_old_value_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance, tag))", 7405 "format": "time_series", 7406 "intervalFactor": 1, 7407 "legendFormat": "{{instance}} - 99% - {{tag}}", 7408 "refId": "A" 7409 }, 7410 { 7411 "expr": "histogram_quantile(0.95, sum(rate(tikv_cdc_old_value_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance, tag))", 7412 "format": "time_series", 7413 "intervalFactor": 1, 7414 "legendFormat": "{{instance}} - 95% - {{tag}}", 7415 "refId": "B" 7416 }, 7417 { 7418 "expr": "sum(rate(tikv_cdc_old_value_duration_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance, tag) / sum(rate(tikv_cdc_old_value_duration_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$tikv_instance\"}[1m])) by (le, instance, tag)", 7419 "format": "time_series", 7420 "intervalFactor": 1, 7421 "legendFormat": "{{instance}} - avg - {{tag}}", 7422 "refId": "C" 7423 } 7424 ], 7425 "thresholds": [], 7426 "timeFrom": null, 7427 "timeRegions": [], 7428 "timeShift": null, 7429 "title": "Old value seek duration", 7430 "tooltip": { 7431 "shared": true, 7432 "sort": 0, 7433 "value_type": "individual" 7434 }, 7435 "type": "graph", 7436 "xaxis": { 7437 "buckets": null, 7438 "mode": "time", 7439 "name": null, 7440 "show": true, 7441 "values": [] 7442 }, 7443 "yaxes": [ 7444 { 7445 "format": "s", 7446 "label": null, 7447 "logBase": 1, 7448 "max": null, 7449 "min": null, 7450 "show": true 7451 }, 7452 { 7453 "format": "short", 7454 "label": null, 7455 "logBase": 1, 7456 "max": null, 7457 "min": null, 7458 "show": false 7459 } 7460 ], 7461 "yaxis": { 7462 "align": false, 7463 "alignLevel": null 7464 } 7465 } 7466 ], 7467 "title": "TiKV", 7468 "type": "row" 7469 }, 7470 { 7471 "collapsed": true, 7472 "gridPos": { 7473 "h": 1, 7474 "w": 24, 7475 "x": 0, 7476 "y": 5 7477 }, 7478 "id": 155, 7479 "panels": [ 7480 { 7481 "aliasColors": {}, 7482 "bars": true, 7483 "cacheTimeout": null, 7484 "dashLength": 10, 7485 "dashes": false, 7486 "datasource": "${DS_TEST-CLUSTER}", 7487 "description": "TiCDC process rss memory usage. TiCDC heap memory size in use ", 7488 "editable": true, 7489 "error": false, 7490 "fill": 0, 7491 "grid": {}, 7492 "gridPos": { 7493 "h": 7, 7494 "w": 12, 7495 "x": 0, 7496 "y": 6 7497 }, 7498 "id": 157, 7499 "legend": { 7500 "alignAsTable": false, 7501 "avg": false, 7502 "current": false, 7503 "hideEmpty": true, 7504 "hideZero": true, 7505 "max": false, 7506 "min": false, 7507 "rightSide": false, 7508 "show": true, 7509 "sideWidth": null, 7510 "total": false, 7511 "values": false 7512 }, 7513 "lines": false, 7514 "linewidth": 1, 7515 "links": [], 7516 "maxPerRow": 3, 7517 "nullPointMode": "null", 7518 "percentage": false, 7519 "pluginVersion": "6.1.6", 7520 "pointradius": 5, 7521 "points": false, 7522 "renderer": "flot", 7523 "repeat": null, 7524 "repeatDirection": "h", 7525 "scopedVars": { 7526 "instance": { 7527 "selected": false, 7528 "text": "172.16.5.33:47912", 7529 "value": "172.16.5.33:47912" 7530 }, 7531 "runtime_instance": { 7532 "selected": false, 7533 "text": "172.16.5.33:47912", 7534 "value": "172.16.5.33:47912" 7535 } 7536 }, 7537 "seriesOverrides": [ 7538 { 7539 "alias": "alloc-from-os", 7540 "fill": 3, 7541 "lines": true, 7542 "stack": false 7543 }, 7544 { 7545 "alias": "gc-threshold", 7546 "bars": false, 7547 "color": "#C4162A", 7548 "lines": true, 7549 "linewidth": 2, 7550 "stack": false 7551 }, 7552 { 7553 "alias": "gc", 7554 "bars": false, 7555 "color": "#C4162A", 7556 "hideTooltip": true, 7557 "legend": false, 7558 "pointradius": 3, 7559 "points": true, 7560 "stack": false 7561 } 7562 ], 7563 "spaceLength": 10, 7564 "stack": true, 7565 "steppedLine": false, 7566 "targets": [ 7567 { 7568 "expr": "process_resident_memory_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", 7569 "format": "time_series", 7570 "hide": false, 7571 "intervalFactor": 1, 7572 "legendFormat": "alloc-from-os", 7573 "refId": "A" 7574 }, 7575 { 7576 "expr": "go_memstats_next_gc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + tidb_server_gogc{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)", 7577 "format": "time_series", 7578 "hide": false, 7579 "intervalFactor": 1, 7580 "legendFormat": "estimate-inuse", 7581 "refId": "H" 7582 }, 7583 { 7584 "expr": "go_memstats_heap_alloc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_next_gc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / (1 + tidb_server_gogc{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} / 100)", 7585 "format": "time_series", 7586 "hide": false, 7587 "intervalFactor": 1, 7588 "legendFormat": "estimate-garbage", 7589 "refId": "C" 7590 }, 7591 { 7592 "expr": "go_memstats_heap_idle_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_released_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_heap_inuse_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_alloc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", 7593 "format": "time_series", 7594 "hide": false, 7595 "intervalFactor": 1, 7596 "legendFormat": "reserved-by-go", 7597 "refId": "B" 7598 }, 7599 { 7600 "expr": "go_memstats_stack_sys_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_mspan_sys_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_mcache_sys_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_buck_hash_sys_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_gc_sys_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} + go_memstats_other_sys_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", 7601 "format": "time_series", 7602 "hide": false, 7603 "intervalFactor": 1, 7604 "legendFormat": "used-by-go", 7605 "refId": "D" 7606 }, 7607 { 7608 "expr": "go_memstats_next_gc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", 7609 "format": "time_series", 7610 "hide": false, 7611 "intervalFactor": 1, 7612 "legendFormat": "gc-threshold", 7613 "refId": "E" 7614 }, 7615 { 7616 "expr": "(clamp_max(idelta(go_memstats_last_gc_time_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[1m]), 1) * go_memstats_next_gc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}) > 0", 7617 "format": "time_series", 7618 "hide": false, 7619 "intervalFactor": 1, 7620 "legendFormat": "gc", 7621 "refId": "F" 7622 } 7623 ], 7624 "thresholds": [], 7625 "timeFrom": null, 7626 "timeRegions": [], 7627 "timeShift": null, 7628 "title": "Memory Usage", 7629 "tooltip": { 7630 "msResolution": true, 7631 "shared": true, 7632 "sort": 0, 7633 "value_type": "individual" 7634 }, 7635 "type": "graph", 7636 "xaxis": { 7637 "buckets": null, 7638 "mode": "time", 7639 "name": null, 7640 "show": true, 7641 "values": [] 7642 }, 7643 "yaxes": [ 7644 { 7645 "format": "bytes", 7646 "label": "", 7647 "logBase": 1, 7648 "max": null, 7649 "min": "0", 7650 "show": true 7651 }, 7652 { 7653 "format": "short", 7654 "label": "", 7655 "logBase": 1, 7656 "max": null, 7657 "min": null, 7658 "show": false 7659 } 7660 ], 7661 "yaxis": { 7662 "align": false, 7663 "alignLevel": null 7664 } 7665 }, 7666 { 7667 "aliasColors": {}, 7668 "bars": false, 7669 "cacheTimeout": null, 7670 "dashLength": 10, 7671 "dashes": false, 7672 "datasource": "${DS_TEST-CLUSTER}", 7673 "description": "Count of live objects.", 7674 "editable": true, 7675 "error": false, 7676 "fill": 0, 7677 "grid": {}, 7678 "gridPos": { 7679 "h": 7, 7680 "w": 12, 7681 "x": 12, 7682 "y": 6 7683 }, 7684 "id": 158, 7685 "legend": { 7686 "alignAsTable": false, 7687 "avg": false, 7688 "current": false, 7689 "hideEmpty": true, 7690 "hideZero": true, 7691 "max": false, 7692 "min": false, 7693 "rightSide": false, 7694 "show": true, 7695 "sideWidth": null, 7696 "total": false, 7697 "values": false 7698 }, 7699 "lines": true, 7700 "linewidth": 1, 7701 "links": [], 7702 "maxPerRow": 3, 7703 "nullPointMode": "null", 7704 "percentage": false, 7705 "pluginVersion": "6.1.6", 7706 "pointradius": 5, 7707 "points": false, 7708 "renderer": "flot", 7709 "repeat": null, 7710 "repeatDirection": "h", 7711 "scopedVars": { 7712 "instance": { 7713 "selected": false, 7714 "text": "172.16.5.33:47912", 7715 "value": "172.16.5.33:47912" 7716 }, 7717 "runtime_instance": { 7718 "selected": false, 7719 "text": "172.16.5.33:47912", 7720 "value": "172.16.5.33:47912" 7721 } 7722 }, 7723 "seriesOverrides": [], 7724 "spaceLength": 10, 7725 "stack": false, 7726 "steppedLine": false, 7727 "targets": [ 7728 { 7729 "expr": "go_memstats_heap_objects{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}", 7730 "format": "time_series", 7731 "hide": false, 7732 "intervalFactor": 1, 7733 "legendFormat": "objects", 7734 "refId": "A" 7735 } 7736 ], 7737 "thresholds": [], 7738 "timeFrom": null, 7739 "timeRegions": [], 7740 "timeShift": null, 7741 "title": "Estimated Live Objects", 7742 "tooltip": { 7743 "msResolution": true, 7744 "shared": true, 7745 "sort": 0, 7746 "value_type": "individual" 7747 }, 7748 "type": "graph", 7749 "xaxis": { 7750 "buckets": null, 7751 "mode": "time", 7752 "name": null, 7753 "show": true, 7754 "values": [] 7755 }, 7756 "yaxes": [ 7757 { 7758 "format": "short", 7759 "label": "", 7760 "logBase": 1, 7761 "max": null, 7762 "min": "0", 7763 "show": true 7764 }, 7765 { 7766 "format": "short", 7767 "label": "", 7768 "logBase": 1, 7769 "max": null, 7770 "min": null, 7771 "show": false 7772 } 7773 ], 7774 "yaxis": { 7775 "align": false, 7776 "alignLevel": null 7777 } 7778 }, 7779 { 7780 "aliasColors": {}, 7781 "bars": false, 7782 "cacheTimeout": null, 7783 "dashLength": 10, 7784 "dashes": false, 7785 "datasource": "${DS_TEST-CLUSTER}", 7786 "description": "TiCDC process Go garbage collection STW pause duration", 7787 "editable": true, 7788 "error": false, 7789 "fill": 1, 7790 "grid": {}, 7791 "gridPos": { 7792 "h": 7, 7793 "w": 12, 7794 "x": 0, 7795 "y": 13 7796 }, 7797 "id": 160, 7798 "legend": { 7799 "alignAsTable": false, 7800 "avg": false, 7801 "current": true, 7802 "max": false, 7803 "min": false, 7804 "rightSide": false, 7805 "show": true, 7806 "total": false, 7807 "values": true 7808 }, 7809 "lines": true, 7810 "linewidth": 1, 7811 "links": [], 7812 "nullPointMode": "null as zero", 7813 "percentage": false, 7814 "pluginVersion": "6.1.6", 7815 "pointradius": 5, 7816 "points": false, 7817 "renderer": "flot", 7818 "scopedVars": { 7819 "instance": { 7820 "selected": false, 7821 "text": "172.16.5.33:47912", 7822 "value": "172.16.5.33:47912" 7823 }, 7824 "runtime_instance": { 7825 "selected": false, 7826 "text": "172.16.5.33:47912", 7827 "value": "172.16.5.33:47912" 7828 } 7829 }, 7830 "seriesOverrides": [ 7831 {} 7832 ], 7833 "spaceLength": 10, 7834 "stack": false, 7835 "steppedLine": false, 7836 "targets": [ 7837 { 7838 "expr": "go_gc_duration_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile=\"0\"}", 7839 "format": "time_series", 7840 "hide": false, 7841 "instant": false, 7842 "intervalFactor": 1, 7843 "legendFormat": "min", 7844 "refId": "A", 7845 "step": 40 7846 }, 7847 { 7848 "expr": "go_gc_duration_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile!~\"0|1\"}", 7849 "format": "time_series", 7850 "instant": false, 7851 "intervalFactor": 1, 7852 "legendFormat": "{{quantile}}", 7853 "refId": "B" 7854 }, 7855 { 7856 "expr": "go_gc_duration_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\", quantile=\"1\"}", 7857 "format": "time_series", 7858 "instant": false, 7859 "intervalFactor": 1, 7860 "legendFormat": "max", 7861 "refId": "C" 7862 } 7863 ], 7864 "thresholds": [], 7865 "timeFrom": null, 7866 "timeRegions": [], 7867 "timeShift": null, 7868 "title": "GC STW Duration (last 256 GC cycles)", 7869 "tooltip": { 7870 "msResolution": false, 7871 "shared": true, 7872 "sort": 1, 7873 "value_type": "individual" 7874 }, 7875 "type": "graph", 7876 "xaxis": { 7877 "buckets": null, 7878 "mode": "time", 7879 "name": null, 7880 "show": true, 7881 "values": [] 7882 }, 7883 "yaxes": [ 7884 { 7885 "format": "s", 7886 "label": null, 7887 "logBase": 1, 7888 "max": null, 7889 "min": "0", 7890 "show": true 7891 }, 7892 { 7893 "format": "short", 7894 "label": null, 7895 "logBase": 1, 7896 "max": null, 7897 "min": null, 7898 "show": false 7899 } 7900 ], 7901 "yaxis": { 7902 "align": false, 7903 "alignLevel": null 7904 } 7905 }, 7906 { 7907 "aliasColors": {}, 7908 "bars": false, 7909 "dashLength": 10, 7910 "dashes": false, 7911 "datasource": "${DS_TEST-CLUSTER}", 7912 "description": "The throughput of Go's memory allocator.", 7913 "fill": 1, 7914 "gridPos": { 7915 "h": 7, 7916 "w": 12, 7917 "x": 12, 7918 "y": 13 7919 }, 7920 "id": 161, 7921 "legend": { 7922 "alignAsTable": false, 7923 "avg": false, 7924 "current": true, 7925 "max": true, 7926 "min": false, 7927 "rightSide": false, 7928 "show": true, 7929 "total": false, 7930 "values": true 7931 }, 7932 "lines": true, 7933 "linewidth": 1, 7934 "links": [], 7935 "nullPointMode": "null", 7936 "percentage": false, 7937 "pointradius": 5, 7938 "points": false, 7939 "renderer": "flot", 7940 "scopedVars": { 7941 "instance": { 7942 "selected": false, 7943 "text": "172.16.5.33:47912", 7944 "value": "172.16.5.33:47912" 7945 }, 7946 "runtime_instance": { 7947 "selected": false, 7948 "text": "172.16.5.33:47912", 7949 "value": "172.16.5.33:47912" 7950 } 7951 }, 7952 "seriesOverrides": [ 7953 { 7954 "alias": "sweep", 7955 "transform": "negative-Y" 7956 }, 7957 { 7958 "alias": "alloc-ops", 7959 "yaxis": 2 7960 }, 7961 { 7962 "alias": "swepp-ops", 7963 "transform": "negative-Y", 7964 "yaxis": 2 7965 } 7966 ], 7967 "spaceLength": 10, 7968 "stack": false, 7969 "steppedLine": false, 7970 "targets": [ 7971 { 7972 "expr": "irate(go_memstats_alloc_bytes_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[30s])", 7973 "format": "time_series", 7974 "intervalFactor": 1, 7975 "legendFormat": "alloc", 7976 "refId": "A" 7977 }, 7978 { 7979 "expr": "irate((go_memstats_alloc_bytes_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"} - go_memstats_heap_alloc_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"})[30s:])", 7980 "format": "time_series", 7981 "intervalFactor": 1, 7982 "legendFormat": "sweep", 7983 "refId": "B" 7984 }, 7985 { 7986 "expr": "irate(go_memstats_mallocs_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[30s])", 7987 "format": "time_series", 7988 "intervalFactor": 1, 7989 "legendFormat": "alloc-ops", 7990 "refId": "C" 7991 }, 7992 { 7993 "expr": "irate(go_memstats_frees_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$runtime_instance\"}[30s])", 7994 "format": "time_series", 7995 "intervalFactor": 1, 7996 "legendFormat": "swepp-ops", 7997 "refId": "D" 7998 } 7999 ], 8000 "thresholds": [], 8001 "timeFrom": null, 8002 "timeRegions": [], 8003 "timeShift": null, 8004 "title": "Allocator Throughput", 8005 "tooltip": { 8006 "shared": true, 8007 "sort": 0, 8008 "value_type": "individual" 8009 }, 8010 "type": "graph", 8011 "xaxis": { 8012 "buckets": null, 8013 "mode": "time", 8014 "name": null, 8015 "show": true, 8016 "values": [] 8017 }, 8018 "yaxes": [ 8019 { 8020 "format": "Bps", 8021 "label": null, 8022 "logBase": 1, 8023 "max": null, 8024 "min": null, 8025 "show": true 8026 }, 8027 { 8028 "format": "ops", 8029 "label": null, 8030 "logBase": 1, 8031 "max": null, 8032 "min": null, 8033 "show": true 8034 } 8035 ], 8036 "yaxis": { 8037 "align": true, 8038 "alignLevel": null 8039 } 8040 } 8041 ], 8042 "repeat": "runtime_instance", 8043 "title": "Runtime $runtime_instance", 8044 "type": "row" 8045 } 8046 ], 8047 "refresh": "1m", 8048 "schemaVersion": 18, 8049 "style": "dark", 8050 "tags": [], 8051 "templating": { 8052 "list": [ 8053 { 8054 "allValue": null, 8055 "current": {}, 8056 "datasource": "${DS_TEST-CLUSTER}", 8057 "definition": "", 8058 "hide": 2, 8059 "includeAll": false, 8060 "label": "tidb_cluster", 8061 "multi": false, 8062 "name": "tidb_cluster", 8063 "options": [], 8064 "query": "label_values(go_goroutines, tidb_cluster)", 8065 "refresh": 2, 8066 "regex": "", 8067 "skipUrlSync": false, 8068 "sort": 1, 8069 "tagValuesQuery": "", 8070 "tags": [], 8071 "tagsQuery": "", 8072 "type": "query", 8073 "useTags": false 8074 }, 8075 { 8076 "allValue": ".*", 8077 "current": {}, 8078 "datasource": "${DS_TEST-CLUSTER}", 8079 "definition": "label_values(ticdc_processor_resolved_ts{tidb_cluster=\"$tidb_cluster\"}, changefeed)", 8080 "hide": 0, 8081 "includeAll": true, 8082 "label": "Changefeed", 8083 "multi": true, 8084 "name": "changefeed", 8085 "options": [], 8086 "query": "label_values(ticdc_processor_resolved_ts{tidb_cluster=\"$tidb_cluster\"}, changefeed)", 8087 "refresh": 2, 8088 "regex": "", 8089 "skipUrlSync": false, 8090 "sort": 0, 8091 "tagValuesQuery": "", 8092 "tags": [], 8093 "tagsQuery": "", 8094 "type": "query", 8095 "useTags": false 8096 }, 8097 { 8098 "allValue": ".*", 8099 "current": {}, 8100 "datasource": "${DS_TEST-CLUSTER}", 8101 "definition": "label_values(process_start_time_seconds{tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}, instance)", 8102 "hide": 0, 8103 "includeAll": true, 8104 "label": "TiCDC", 8105 "multi": true, 8106 "name": "capture", 8107 "options": [], 8108 "query": "label_values(process_start_time_seconds{tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}, instance)", 8109 "refresh": 2, 8110 "regex": "", 8111 "skipUrlSync": false, 8112 "sort": 0, 8113 "tagValuesQuery": "", 8114 "tags": [], 8115 "tagsQuery": "", 8116 "type": "query", 8117 "useTags": false 8118 }, 8119 { 8120 "allValue": ".*", 8121 "current": {}, 8122 "datasource": "${DS_TEST-CLUSTER}", 8123 "definition": "label_values(tikv_engine_size_bytes{tidb_cluster=\"$tidb_cluster\"}, instance)", 8124 "hide": 0, 8125 "includeAll": true, 8126 "label": "TiKV", 8127 "multi": false, 8128 "name": "tikv_instance", 8129 "options": [], 8130 "query": "label_values(tikv_engine_size_bytes{tidb_cluster=\"$tidb_cluster\"}, instance)", 8131 "refresh": 2, 8132 "regex": "", 8133 "skipUrlSync": false, 8134 "sort": 1, 8135 "tagValuesQuery": "", 8136 "tags": [], 8137 "tagsQuery": "", 8138 "type": "query", 8139 "useTags": false 8140 }, 8141 { 8142 "allValue": "9999999999", 8143 "current": { 8144 "selected": true, 8145 "text": "All", 8146 "value": "$__all" 8147 }, 8148 "hide": 0, 8149 "includeAll": true, 8150 "label": "Latency spike (s) >", 8151 "multi": false, 8152 "name": "spike_threshold", 8153 "options": [ 8154 { 8155 "selected": true, 8156 "text": "All", 8157 "value": "$__all" 8158 }, 8159 { 8160 "selected": false, 8161 "text": "1", 8162 "value": "1" 8163 }, 8164 { 8165 "selected": false, 8166 "text": "3", 8167 "value": "3" 8168 }, 8169 { 8170 "selected": false, 8171 "text": "5", 8172 "value": "5" 8173 }, 8174 { 8175 "selected": false, 8176 "text": "10", 8177 "value": "10" 8178 }, 8179 { 8180 "selected": false, 8181 "text": "60", 8182 "value": "60" 8183 }, 8184 { 8185 "selected": false, 8186 "text": "300", 8187 "value": "300" 8188 } 8189 ], 8190 "query": "1, 3, 5, 10, 60, 300", 8191 "skipUrlSync": false, 8192 "type": "custom" 8193 }, 8194 { 8195 "allValue": "", 8196 "current": {}, 8197 "datasource": "${DS_TEST-CLUSTER}", 8198 "definition": "label_values(process_start_time_seconds{tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}, instance)", 8199 "hide": 0, 8200 "includeAll": true, 8201 "label": "Runtime metrics", 8202 "multi": false, 8203 "name": "runtime_instance", 8204 "options": [], 8205 "query": "label_values(process_start_time_seconds{tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}, instance)", 8206 "refresh": 2, 8207 "regex": "", 8208 "skipUrlSync": false, 8209 "sort": 0, 8210 "tagValuesQuery": "", 8211 "tags": [], 8212 "tagsQuery": "", 8213 "type": "query", 8214 "useTags": false 8215 } 8216 ] 8217 }, 8218 "time": { 8219 "from": "now-3h", 8220 "to": "now" 8221 }, 8222 "timepicker": { 8223 "refresh_intervals": [ 8224 "5s", 8225 "10s", 8226 "30s", 8227 "1m", 8228 "5m", 8229 "15m", 8230 "30m", 8231 "1h", 8232 "2h", 8233 "1d" 8234 ], 8235 "time_options": [ 8236 "5m", 8237 "15m", 8238 "1h", 8239 "6h", 8240 "12h", 8241 "24h", 8242 "2d", 8243 "7d", 8244 "30d" 8245 ] 8246 }, 8247 "timezone": "browser", 8248 "title": "Test-Cluster-TiCDC", 8249 "uid": "YiGL8hBZ1", 8250 "version": 21 8251 }