github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/metrics/grafana/TiCDC-Monitor-Summary.json (about) 1 { 2 "__inputs": [ 3 { 4 "name": "DS_TEST-CLUSTER", 5 "label": "${DS_TEST-CLUSTER}", 6 "description": "", 7 "type": "datasource", 8 "pluginId": "prometheus", 9 "pluginName": "Prometheus" 10 } 11 ], 12 "__requires": [ 13 { 14 "type": "grafana", 15 "id": "grafana", 16 "name": "Grafana", 17 "version": "7.5.11" 18 }, 19 { 20 "type": "panel", 21 "id": "graph", 22 "name": "Graph", 23 "version": "" 24 }, 25 { 26 "type": "panel", 27 "id": "heatmap", 28 "name": "Heatmap", 29 "version": "" 30 }, 31 { 32 "type": "datasource", 33 "id": "prometheus", 34 "name": "Prometheus", 35 "version": "1.0.0" 36 }, 37 { 38 "type": "panel", 39 "id": "table", 40 "name": "Table", 41 "version": "" 42 }, 43 { 44 "type": "panel", 45 "id": "timeseries", 46 "name": "Time series", 47 "version": "" 48 } 49 ], 50 "annotations": { 51 "list": [ 52 { 53 "builtIn": 1, 54 "datasource": "-- Grafana --", 55 "enable": false, 56 "expr": "", 57 "hide": true, 58 "iconColor": "#F2495C", 59 "limit": 100, 60 "name": "", 61 "showIn": 0, 62 "tagKeys": "", 63 "textFormat": "", 64 "titleFormat": "", 65 "type": "dashboard", 66 "useValueForTime": false 67 }, 68 { 69 "datasource": "${DS_TEST-CLUSTER}", 70 "enable": true, 71 "expr": "max(ticdc_owner_checkpoint_ts_lag) by (namespace,changefeed, instance) > BOOL $spike_threshold", 72 "hide": true, 73 "iconColor": "#F2495C", 74 "limit": 100, 75 "name": "Latency spike", 76 "showIn": 0, 77 "tagKeys": "changefeed", 78 "tags": [], 79 "titleFormat": "Latency spike", 80 "type": "tags", 81 "useValueForTime": false 82 }, 83 { 84 "datasource": "${DS_TEST-CLUSTER}", 85 "enable": false, 86 "expr": "delta(up{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=~\"tikv|ticdc|tidb|pd\"}[30s]) < BOOL 0", 87 "hide": false, 88 "iconColor": "#FF9830", 89 "limit": 100, 90 "name": "Server down", 91 "showIn": 0, 92 "step": "15s", 93 "tagKeys": "instance,job", 94 "tags": [], 95 "textFormat": "", 96 "titleFormat": "Down", 97 "type": "tags" 98 }, 99 { 100 "datasource": "${DS_TEST-CLUSTER}", 101 "enable": false, 102 "expr": "sum(ALERTS{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", alertstate=\"firing\", alertname=~\"ticdc.*\"}) by (alertname) > BOOL 0", 103 "hide": false, 104 "iconColor": "#B877D9", 105 "limit": 100, 106 "name": "All TiCDC alerts", 107 "showIn": 0, 108 "tagKeys": "alertname", 109 "tags": [], 110 "titleFormat": "Alert Name", 111 "type": "tags" 112 }, 113 { 114 "datasource": "${DS_TEST-CLUSTER}", 115 "enable": false, 116 "expr": "delta(tikv_cdc_region_resolve_status{status=\"resolved\"}[30s]) < BOOL -800", 117 "hide": false, 118 "iconColor": "rgba(255, 96, 96, 1)", 119 "limit": 100, 120 "name": "Resolved region drop", 121 "showIn": 0, 122 "step": "15s", 123 "tagKeys": "instance", 124 "tags": [], 125 "titleFormat": "Resolved region drop", 126 "type": "tags" 127 } 128 ] 129 }, 130 "editable": true, 131 "gnetId": null, 132 "graphTooltip": 1, 133 "id": null, 134 "iteration": 1705047462154, 135 "links": [], 136 "panels": [ 137 { 138 "collapsed": true, 139 "datasource": null, 140 "gridPos": { 141 "h": 1, 142 "w": 24, 143 "x": 0, 144 "y": 0 145 }, 146 "id": 21, 147 "panels": [ 148 { 149 "aliasColors": {}, 150 "bars": false, 151 "dashLength": 10, 152 "dashes": false, 153 "datasource": "${DS_TEST-CLUSTER}", 154 "description": "Uptime of TiCDC and TiKV", 155 "fieldConfig": { 156 "defaults": { 157 "links": [] 158 }, 159 "overrides": [] 160 }, 161 "fill": 1, 162 "fillGradient": 0, 163 "gridPos": { 164 "h": 7, 165 "w": 8, 166 "x": 0, 167 "y": 1 168 }, 169 "hiddenSeries": false, 170 "id": 19, 171 "legend": { 172 "alignAsTable": true, 173 "avg": false, 174 "current": true, 175 "max": false, 176 "min": false, 177 "rightSide": true, 178 "show": true, 179 "total": false, 180 "values": true 181 }, 182 "lines": true, 183 "linewidth": 1, 184 "links": [], 185 "nullPointMode": "null", 186 "options": { 187 "alertThreshold": true 188 }, 189 "paceLength": 10, 190 "percentage": false, 191 "pluginVersion": "7.5.11", 192 "pointradius": 2, 193 "points": false, 194 "renderer": "flot", 195 "seriesOverrides": [], 196 "spaceLength": 10, 197 "stack": false, 198 "steppedLine": false, 199 "targets": [ 200 { 201 "expr": "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"})", 202 "format": "time_series", 203 "intervalFactor": 1, 204 "legendFormat": "TiCDC-{{instance}}", 205 "refId": "A" 206 } 207 ], 208 "thresholds": [], 209 "timeFrom": null, 210 "timeRegions": [], 211 "timeShift": null, 212 "title": "Uptime", 213 "tooltip": { 214 "shared": true, 215 "sort": 0, 216 "value_type": "individual" 217 }, 218 "type": "graph", 219 "xaxis": { 220 "buckets": null, 221 "mode": "time", 222 "name": null, 223 "show": true, 224 "values": [] 225 }, 226 "yaxes": [ 227 { 228 "format": "dtdurations", 229 "label": null, 230 "logBase": 1, 231 "max": null, 232 "min": null, 233 "show": true 234 }, 235 { 236 "format": "short", 237 "label": null, 238 "logBase": 1, 239 "max": null, 240 "min": null, 241 "show": true 242 } 243 ], 244 "yaxis": { 245 "align": false, 246 "alignLevel": null 247 } 248 }, 249 { 250 "aliasColors": {}, 251 "bars": false, 252 "dashLength": 10, 253 "dashes": false, 254 "datasource": "${DS_TEST-CLUSTER}", 255 "description": "CPU usage of TiCDC", 256 "fieldConfig": { 257 "defaults": { 258 "links": [] 259 }, 260 "overrides": [] 261 }, 262 "fill": 1, 263 "fillGradient": 0, 264 "gridPos": { 265 "h": 7, 266 "w": 7, 267 "x": 8, 268 "y": 1 269 }, 270 "hiddenSeries": false, 271 "id": 24, 272 "legend": { 273 "alignAsTable": true, 274 "avg": false, 275 "current": true, 276 "hideEmpty": true, 277 "max": true, 278 "min": false, 279 "rightSide": false, 280 "show": true, 281 "total": false, 282 "values": true 283 }, 284 "lines": true, 285 "linewidth": 1, 286 "links": [], 287 "nullPointMode": "null", 288 "options": { 289 "alertThreshold": true 290 }, 291 "paceLength": 10, 292 "percentage": false, 293 "pluginVersion": "7.5.11", 294 "pointradius": 2, 295 "points": false, 296 "renderer": "flot", 297 "seriesOverrides": [ 298 { 299 "alias": "/.*MaxProcs/", 300 "fill": 0, 301 "linewidth": 2, 302 "yaxis": 2 303 } 304 ], 305 "spaceLength": 10, 306 "stack": false, 307 "steppedLine": false, 308 "targets": [ 309 { 310 "expr": "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}[1m])", 311 "format": "time_series", 312 "intervalFactor": 1, 313 "legendFormat": "{{instance}}", 314 "refId": "A" 315 }, 316 { 317 "expr": "ticdc_server_go_max_procs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}", 318 "format": "time_series", 319 "intervalFactor": 1, 320 "legendFormat": "quota-{{instance}}", 321 "refId": "B" 322 } 323 ], 324 "thresholds": [], 325 "timeFrom": null, 326 "timeRegions": [], 327 "timeShift": null, 328 "title": "CPU usage", 329 "tooltip": { 330 "shared": true, 331 "sort": 0, 332 "value_type": "individual" 333 }, 334 "type": "graph", 335 "xaxis": { 336 "buckets": null, 337 "mode": "time", 338 "name": null, 339 "show": true, 340 "values": [] 341 }, 342 "yaxes": [ 343 { 344 "format": "percentunit", 345 "label": null, 346 "logBase": 1, 347 "max": null, 348 "min": null, 349 "show": true 350 }, 351 { 352 "format": "percentunit", 353 "label": null, 354 "logBase": 1, 355 "max": null, 356 "min": null, 357 "show": true 358 } 359 ], 360 "yaxis": { 361 "align": false, 362 "alignLevel": null 363 } 364 }, 365 { 366 "aliasColors": {}, 367 "bars": false, 368 "dashLength": 10, 369 "dashes": false, 370 "datasource": "${DS_TEST-CLUSTER}", 371 "description": "Memory usage of TiCDC", 372 "fieldConfig": { 373 "defaults": { 374 "links": [] 375 }, 376 "overrides": [] 377 }, 378 "fill": 1, 379 "fillGradient": 0, 380 "gridPos": { 381 "h": 7, 382 "w": 7, 383 "x": 15, 384 "y": 1 385 }, 386 "hiddenSeries": false, 387 "id": 23, 388 "legend": { 389 "alignAsTable": true, 390 "avg": false, 391 "current": true, 392 "hideEmpty": true, 393 "max": true, 394 "min": false, 395 "rightSide": false, 396 "show": true, 397 "total": false, 398 "values": true 399 }, 400 "lines": true, 401 "linewidth": 1, 402 "links": [], 403 "nullPointMode": "null", 404 "options": { 405 "alertThreshold": true 406 }, 407 "paceLength": 10, 408 "percentage": false, 409 "pluginVersion": "7.5.11", 410 "pointradius": 2, 411 "points": false, 412 "renderer": "flot", 413 "seriesOverrides": [], 414 "spaceLength": 10, 415 "stack": false, 416 "steppedLine": false, 417 "targets": [ 418 { 419 "expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}", 420 "format": "time_series", 421 "intervalFactor": 1, 422 "legendFormat": "process-{{instance}}", 423 "refId": "A" 424 }, 425 { 426 "expr": "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}", 427 "format": "time_series", 428 "intervalFactor": 1, 429 "legendFormat": "heap-{{instance}}", 430 "refId": "B" 431 } 432 ], 433 "thresholds": [], 434 "timeFrom": null, 435 "timeRegions": [], 436 "timeShift": null, 437 "title": "Memory usage", 438 "tooltip": { 439 "shared": true, 440 "sort": 0, 441 "value_type": "individual" 442 }, 443 "type": "graph", 444 "xaxis": { 445 "buckets": null, 446 "mode": "time", 447 "name": null, 448 "show": true, 449 "values": [] 450 }, 451 "yaxes": [ 452 { 453 "format": "bytes", 454 "label": null, 455 "logBase": 1, 456 "max": null, 457 "min": null, 458 "show": true 459 }, 460 { 461 "format": "short", 462 "label": null, 463 "logBase": 1, 464 "max": null, 465 "min": null, 466 "show": true 467 } 468 ], 469 "yaxis": { 470 "align": false, 471 "alignLevel": null 472 } 473 } 474 ], 475 "title": "Server", 476 "type": "row" 477 }, 478 { 479 "collapsed": true, 480 "datasource": null, 481 "gridPos": { 482 "h": 1, 483 "w": 24, 484 "x": 0, 485 "y": 1 486 }, 487 "id": 11, 488 "panels": [ 489 { 490 "aliasColors": {}, 491 "bars": false, 492 "dashLength": 10, 493 "dashes": false, 494 "datasource": "${DS_TEST-CLUSTER}", 495 "description": "The lag between changefeed checkpoint ts and the latest ts of upstream TiDB.", 496 "fieldConfig": { 497 "defaults": { 498 "links": [] 499 }, 500 "overrides": [] 501 }, 502 "fill": 1, 503 "fillGradient": 0, 504 "gridPos": { 505 "h": 7, 506 "w": 12, 507 "x": 0, 508 "y": 2 509 }, 510 "hiddenSeries": false, 511 "id": 3, 512 "legend": { 513 "alignAsTable": true, 514 "avg": false, 515 "current": true, 516 "max": false, 517 "min": false, 518 "rightSide": false, 519 "show": true, 520 "total": false, 521 "values": true 522 }, 523 "lines": true, 524 "linewidth": 1, 525 "links": [], 526 "nullPointMode": "null", 527 "options": { 528 "alertThreshold": true 529 }, 530 "paceLength": 10, 531 "percentage": false, 532 "pluginVersion": "7.5.11", 533 "pointradius": 2, 534 "points": false, 535 "renderer": "flot", 536 "seriesOverrides": [], 537 "spaceLength": 10, 538 "stack": false, 539 "steppedLine": false, 540 "targets": [ 541 { 542 "expr": "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", 543 "format": "time_series", 544 "interval": "", 545 "intervalFactor": 1, 546 "legendFormat": "{{namespace}}-{{changefeed}}", 547 "refId": "A" 548 }, 549 { 550 "exemplar": true, 551 "expr": "max(ticdc_processor_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (instance,namespace,changefeed)", 552 "hide": true, 553 "interval": "", 554 "legendFormat": "{{instance}}-{{namespace}}-{{changefeed}}", 555 "refId": "B" 556 } 557 ], 558 "thresholds": [], 559 "timeFrom": null, 560 "timeRegions": [], 561 "timeShift": null, 562 "title": "Changefeed checkpoint lag", 563 "tooltip": { 564 "shared": true, 565 "sort": 0, 566 "value_type": "individual" 567 }, 568 "type": "graph", 569 "xaxis": { 570 "buckets": null, 571 "mode": "time", 572 "name": null, 573 "show": true, 574 "values": [] 575 }, 576 "yaxes": [ 577 { 578 "format": "s", 579 "label": null, 580 "logBase": 1, 581 "max": null, 582 "min": "0", 583 "show": true 584 }, 585 { 586 "format": "short", 587 "label": null, 588 "logBase": 1, 589 "max": null, 590 "min": null, 591 "show": true 592 } 593 ], 594 "yaxis": { 595 "align": false, 596 "alignLevel": null 597 } 598 }, 599 { 600 "aliasColors": {}, 601 "bars": false, 602 "dashLength": 10, 603 "dashes": false, 604 "datasource": "${DS_TEST-CLUSTER}", 605 "description": "The lag between changefeed resolved ts and the latest ts of upstream TiDB.", 606 "fieldConfig": { 607 "defaults": { 608 "links": [] 609 }, 610 "overrides": [] 611 }, 612 "fill": 1, 613 "fillGradient": 0, 614 "gridPos": { 615 "h": 7, 616 "w": 12, 617 "x": 12, 618 "y": 2 619 }, 620 "hiddenSeries": false, 621 "id": 513, 622 "legend": { 623 "alignAsTable": true, 624 "avg": false, 625 "current": true, 626 "max": false, 627 "min": false, 628 "rightSide": false, 629 "show": true, 630 "total": false, 631 "values": true 632 }, 633 "lines": true, 634 "linewidth": 1, 635 "links": [], 636 "nullPointMode": "null", 637 "options": { 638 "alertThreshold": true 639 }, 640 "paceLength": 10, 641 "percentage": false, 642 "pluginVersion": "7.5.11", 643 "pointradius": 2, 644 "points": false, 645 "renderer": "flot", 646 "seriesOverrides": [], 647 "spaceLength": 10, 648 "stack": false, 649 "steppedLine": false, 650 "targets": [ 651 { 652 "exemplar": true, 653 "expr": "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", 654 "format": "time_series", 655 "interval": "", 656 "intervalFactor": 1, 657 "legendFormat": "{{namespace}}-{{changefeed}}", 658 "refId": "C" 659 }, 660 { 661 "exemplar": true, 662 "expr": "max(ticdc_processor_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (instance,namespace,changefeed)", 663 "hide": true, 664 "interval": "", 665 "legendFormat": "{{instance}}-{{namespace}}-{{changefeed}}", 666 "refId": "A" 667 } 668 ], 669 "thresholds": [], 670 "timeFrom": null, 671 "timeRegions": [], 672 "timeShift": null, 673 "title": "Changefeed resolved ts lag", 674 "tooltip": { 675 "shared": true, 676 "sort": 0, 677 "value_type": "individual" 678 }, 679 "type": "graph", 680 "xaxis": { 681 "buckets": null, 682 "mode": "time", 683 "name": null, 684 "show": true, 685 "values": [] 686 }, 687 "yaxes": [ 688 { 689 "format": "s", 690 "label": null, 691 "logBase": 1, 692 "max": null, 693 "min": "0", 694 "show": true 695 }, 696 { 697 "format": "short", 698 "label": null, 699 "logBase": 1, 700 "max": null, 701 "min": null, 702 "show": true 703 } 704 ], 705 "yaxis": { 706 "align": false, 707 "alignLevel": null 708 } 709 }, 710 { 711 "datasource": "${DS_TEST-CLUSTER}", 712 "description": "The status of each changefeed.\n\n0: Normal\n\n1 and 6: Warning\n\n2: Failed\n\n3: Stopped\n\n4: Finished\n\n-1: Unknown", 713 "fieldConfig": { 714 "defaults": { 715 "color": { 716 "mode": "palette-classic" 717 }, 718 "custom": { 719 "axisLabel": "", 720 "axisPlacement": "auto", 721 "barAlignment": 0, 722 "drawStyle": "line", 723 "fillOpacity": 10, 724 "gradientMode": "none", 725 "hideFrom": { 726 "graph": false, 727 "legend": false, 728 "tooltip": false 729 }, 730 "lineInterpolation": "stepAfter", 731 "lineWidth": 1, 732 "pointSize": 4, 733 "scaleDistribution": { 734 "type": "linear" 735 }, 736 "showPoints": "always", 737 "spanNulls": true 738 }, 739 "mappings": [ 740 { 741 "from": "", 742 "id": 1, 743 "text": "Normal", 744 "to": "", 745 "type": 1, 746 "value": "0" 747 }, 748 { 749 "from": "", 750 "id": 2, 751 "text": "Warning", 752 "to": "", 753 "type": 1, 754 "value": "1" 755 }, 756 { 757 "from": "", 758 "id": 3, 759 "text": "Failed", 760 "to": "", 761 "type": 1, 762 "value": "2" 763 }, 764 { 765 "from": "", 766 "id": 4, 767 "text": "Stopped", 768 "to": "", 769 "type": 1, 770 "value": "3" 771 }, 772 { 773 "from": "", 774 "id": 5, 775 "text": "Finished", 776 "to": "", 777 "type": 1, 778 "value": "4" 779 }, 780 { 781 "from": "", 782 "id": 6, 783 "text": "Warning", 784 "to": "", 785 "type": 1, 786 "value": "6" 787 }, 788 { 789 "from": "", 790 "id": 7, 791 "text": "Unknown", 792 "to": "", 793 "type": 1, 794 "value": "-1" 795 }, 796 { 797 "from": "7", 798 "id": 8, 799 "text": "Other", 800 "to": "10000", 801 "type": 1, 802 "value": "7" 803 }, 804 { 805 "from": "7", 806 "id": 9, 807 "text": "-", 808 "to": "1000", 809 "type": 2 810 } 811 ], 812 "max": 8, 813 "min": -1, 814 "noValue": "-", 815 "thresholds": { 816 "mode": "absolute", 817 "steps": [ 818 { 819 "color": "green", 820 "value": null 821 }, 822 { 823 "color": "light-yellow", 824 "value": -1 825 }, 826 { 827 "color": "green", 828 "value": 0 829 }, 830 { 831 "color": "semi-dark-red", 832 "value": 1 833 }, 834 { 835 "color": "dark-red", 836 "value": 2 837 }, 838 { 839 "color": "dark-yellow", 840 "value": 3 841 }, 842 { 843 "color": "#6ED0E0", 844 "value": 4 845 } 846 ] 847 }, 848 "unit": "short" 849 }, 850 "overrides": [] 851 }, 852 "gridPos": { 853 "h": 7, 854 "w": 12, 855 "x": 0, 856 "y": 9 857 }, 858 "id": 163, 859 "links": [], 860 "options": { 861 "graph": {}, 862 "legend": { 863 "calcs": [ 864 "lastNotNull" 865 ], 866 "displayMode": "table", 867 "placement": "bottom" 868 }, 869 "tooltipOptions": { 870 "mode": "single" 871 } 872 }, 873 "pluginVersion": "7.5.10", 874 "targets": [ 875 { 876 "exemplar": true, 877 "expr": "max(ticdc_owner_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed)", 878 "format": "time_series", 879 "instant": false, 880 "interval": "", 881 "intervalFactor": 1, 882 "legendFormat": "{{namespace}}-{{changefeed}}", 883 "refId": "A" 884 } 885 ], 886 "timeFrom": null, 887 "timeShift": null, 888 "title": "The status of changefeeds", 889 "type": "timeseries" 890 }, 891 { 892 "aliasColors": {}, 893 "bars": false, 894 "dashLength": 10, 895 "dashes": false, 896 "datasource": "${DS_TEST-CLUSTER}", 897 "description": "The number of errors that interrupt changefeed per minute ", 898 "fieldConfig": { 899 "defaults": {}, 900 "overrides": [] 901 }, 902 "fill": 1, 903 "fillGradient": 0, 904 "gridPos": { 905 "h": 7, 906 "w": 12, 907 "x": 12, 908 "y": 9 909 }, 910 "hiddenSeries": false, 911 "id": 82, 912 "legend": { 913 "alignAsTable": true, 914 "avg": false, 915 "current": true, 916 "max": false, 917 "min": false, 918 "rightSide": false, 919 "show": true, 920 "total": false, 921 "values": true 922 }, 923 "lines": true, 924 "linewidth": 1, 925 "links": [], 926 "nullPointMode": "null", 927 "options": { 928 "alertThreshold": true 929 }, 930 "percentage": false, 931 "pluginVersion": "7.5.11", 932 "pointradius": 2, 933 "points": false, 934 "renderer": "flot", 935 "seriesOverrides": [], 936 "spaceLength": 10, 937 "stack": false, 938 "steppedLine": false, 939 "targets": [ 940 { 941 "exemplar": true, 942 "expr": "sum(rate(ticdc_processor_exit_with_error_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (instance, namespace,changefeed)", 943 "hide": false, 944 "interval": "", 945 "legendFormat": "{{instance}}-{{namespace}}-{{changefeed}}", 946 "refId": "B" 947 } 948 ], 949 "thresholds": [], 950 "timeFrom": null, 951 "timeRegions": [], 952 "timeShift": null, 953 "title": "Exit error count/m", 954 "tooltip": { 955 "shared": true, 956 "sort": 0, 957 "value_type": "individual" 958 }, 959 "type": "graph", 960 "xaxis": { 961 "buckets": null, 962 "mode": "time", 963 "name": null, 964 "show": true, 965 "values": [] 966 }, 967 "yaxes": [ 968 { 969 "format": "short", 970 "label": null, 971 "logBase": 1, 972 "max": null, 973 "min": null, 974 "show": true 975 }, 976 { 977 "format": "short", 978 "label": null, 979 "logBase": 1, 980 "max": null, 981 "min": null, 982 "show": true 983 } 984 ], 985 "yaxis": { 986 "align": false, 987 "alignLevel": null 988 } 989 } 990 ], 991 "title": "Changefeed", 992 "type": "row" 993 }, 994 { 995 "collapsed": true, 996 "datasource": null, 997 "gridPos": { 998 "h": 1, 999 "w": 24, 1000 "x": 0, 1001 "y": 2 1002 }, 1003 "id": 206, 1004 "panels": [ 1005 { 1006 "aliasColors": {}, 1007 "bars": false, 1008 "dashLength": 10, 1009 "dashes": false, 1010 "datasource": "${DS_TEST-CLUSTER}", 1011 "description": "The number of events that puller outputs to sorter \n per second", 1012 "fieldConfig": { 1013 "defaults": { 1014 "links": [] 1015 }, 1016 "overrides": [] 1017 }, 1018 "fill": 1, 1019 "fillGradient": 0, 1020 "gridPos": { 1021 "h": 4, 1022 "w": 12, 1023 "x": 0, 1024 "y": 3 1025 }, 1026 "hiddenSeries": false, 1027 "id": 218, 1028 "legend": { 1029 "alignAsTable": true, 1030 "avg": false, 1031 "current": true, 1032 "max": true, 1033 "min": false, 1034 "rightSide": false, 1035 "show": true, 1036 "total": false, 1037 "values": true 1038 }, 1039 "lines": true, 1040 "linewidth": 1, 1041 "links": [], 1042 "nullPointMode": "null", 1043 "options": { 1044 "alertThreshold": true 1045 }, 1046 "paceLength": 10, 1047 "percentage": false, 1048 "pluginVersion": "7.5.11", 1049 "pointradius": 2, 1050 "points": false, 1051 "renderer": "flot", 1052 "seriesOverrides": [], 1053 "spaceLength": 10, 1054 "stack": false, 1055 "steppedLine": false, 1056 "targets": [ 1057 { 1058 "expr": "sum (rate(ticdc_puller_txn_collect_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, type)", 1059 "format": "time_series", 1060 "intervalFactor": 1, 1061 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{type}}", 1062 "refId": "A" 1063 } 1064 ], 1065 "thresholds": [], 1066 "timeFrom": null, 1067 "timeRegions": [], 1068 "timeShift": null, 1069 "title": "Puller output events/s", 1070 "tooltip": { 1071 "shared": true, 1072 "sort": 0, 1073 "value_type": "individual" 1074 }, 1075 "type": "graph", 1076 "xaxis": { 1077 "buckets": null, 1078 "mode": "time", 1079 "name": null, 1080 "show": true, 1081 "values": [] 1082 }, 1083 "yaxes": [ 1084 { 1085 "$$hashKey": "object:234", 1086 "format": "short", 1087 "label": null, 1088 "logBase": 1, 1089 "max": null, 1090 "min": null, 1091 "show": true 1092 }, 1093 { 1094 "$$hashKey": "object:235", 1095 "format": "short", 1096 "label": null, 1097 "logBase": 1, 1098 "max": null, 1099 "min": null, 1100 "show": false 1101 } 1102 ], 1103 "yaxis": { 1104 "align": false, 1105 "alignLevel": null 1106 } 1107 }, 1108 { 1109 "aliasColors": {}, 1110 "bars": false, 1111 "dashLength": 10, 1112 "dashes": false, 1113 "datasource": "${DS_TEST-CLUSTER}", 1114 "description": "The total number of events that puller outputs", 1115 "fieldConfig": { 1116 "defaults": { 1117 "links": [] 1118 }, 1119 "overrides": [] 1120 }, 1121 "fill": 1, 1122 "fillGradient": 0, 1123 "gridPos": { 1124 "h": 4, 1125 "w": 12, 1126 "x": 12, 1127 "y": 3 1128 }, 1129 "hiddenSeries": false, 1130 "id": 229, 1131 "legend": { 1132 "alignAsTable": true, 1133 "avg": false, 1134 "current": true, 1135 "max": true, 1136 "min": false, 1137 "rightSide": false, 1138 "show": true, 1139 "total": false, 1140 "values": true 1141 }, 1142 "lines": true, 1143 "linewidth": 1, 1144 "links": [], 1145 "nullPointMode": "null", 1146 "options": { 1147 "alertThreshold": true 1148 }, 1149 "paceLength": 10, 1150 "percentage": false, 1151 "pluginVersion": "7.5.11", 1152 "pointradius": 2, 1153 "points": false, 1154 "renderer": "flot", 1155 "seriesOverrides": [], 1156 "spaceLength": 10, 1157 "stack": false, 1158 "steppedLine": false, 1159 "targets": [ 1160 { 1161 "expr": "sum(ticdc_puller_txn_collect_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, type)", 1162 "format": "time_series", 1163 "intervalFactor": 1, 1164 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{type}}", 1165 "refId": "A" 1166 } 1167 ], 1168 "thresholds": [], 1169 "timeFrom": null, 1170 "timeRegions": [], 1171 "timeShift": null, 1172 "title": "Puller output events", 1173 "tooltip": { 1174 "shared": true, 1175 "sort": 0, 1176 "value_type": "individual" 1177 }, 1178 "type": "graph", 1179 "xaxis": { 1180 "buckets": null, 1181 "mode": "time", 1182 "name": null, 1183 "show": true, 1184 "values": [] 1185 }, 1186 "yaxes": [ 1187 { 1188 "format": "none", 1189 "label": null, 1190 "logBase": 1, 1191 "max": null, 1192 "min": null, 1193 "show": true 1194 }, 1195 { 1196 "format": "short", 1197 "label": null, 1198 "logBase": 1, 1199 "max": null, 1200 "min": null, 1201 "show": true 1202 } 1203 ], 1204 "yaxis": { 1205 "align": false, 1206 "alignLevel": null 1207 } 1208 }, 1209 { 1210 "aliasColors": {}, 1211 "bars": false, 1212 "dashLength": 10, 1213 "dashes": false, 1214 "datasource": "${DS_TEST-CLUSTER}", 1215 "description": "The number of events that sorter outputs to puller \n per second", 1216 "fieldConfig": { 1217 "defaults": { 1218 "links": [] 1219 }, 1220 "overrides": [] 1221 }, 1222 "fill": 1, 1223 "fillGradient": 0, 1224 "gridPos": { 1225 "h": 4, 1226 "w": 12, 1227 "x": 0, 1228 "y": 7 1229 }, 1230 "hiddenSeries": false, 1231 "id": 228, 1232 "legend": { 1233 "alignAsTable": true, 1234 "avg": false, 1235 "current": true, 1236 "max": true, 1237 "min": false, 1238 "rightSide": false, 1239 "show": true, 1240 "total": false, 1241 "values": true 1242 }, 1243 "lines": true, 1244 "linewidth": 1, 1245 "links": [], 1246 "nullPointMode": "null", 1247 "options": { 1248 "alertThreshold": true 1249 }, 1250 "paceLength": 10, 1251 "percentage": false, 1252 "pluginVersion": "7.5.11", 1253 "pointradius": 2, 1254 "points": false, 1255 "renderer": "flot", 1256 "seriesOverrides": [], 1257 "spaceLength": 10, 1258 "stack": false, 1259 "steppedLine": false, 1260 "targets": [ 1261 { 1262 "expr": "sum(rate(ticdc_sorter_output_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, type)", 1263 "format": "time_series", 1264 "intervalFactor": 1, 1265 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{type}}", 1266 "refId": "A" 1267 } 1268 ], 1269 "thresholds": [], 1270 "timeFrom": null, 1271 "timeRegions": [], 1272 "timeShift": null, 1273 "title": "Sorter output events/s", 1274 "tooltip": { 1275 "shared": true, 1276 "sort": 0, 1277 "value_type": "individual" 1278 }, 1279 "type": "graph", 1280 "xaxis": { 1281 "buckets": null, 1282 "mode": "time", 1283 "name": null, 1284 "show": true, 1285 "values": [] 1286 }, 1287 "yaxes": [ 1288 { 1289 "$$hashKey": "object:279", 1290 "format": "short", 1291 "label": null, 1292 "logBase": 1, 1293 "max": null, 1294 "min": null, 1295 "show": true 1296 }, 1297 { 1298 "$$hashKey": "object:280", 1299 "format": "short", 1300 "label": null, 1301 "logBase": 1, 1302 "max": null, 1303 "min": null, 1304 "show": false 1305 } 1306 ], 1307 "yaxis": { 1308 "align": false, 1309 "alignLevel": null 1310 } 1311 }, 1312 { 1313 "aliasColors": {}, 1314 "bars": false, 1315 "dashLength": 10, 1316 "dashes": false, 1317 "datasource": "${DS_TEST-CLUSTER}", 1318 "description": "The total number of events that sorter outputs", 1319 "fieldConfig": { 1320 "defaults": { 1321 "links": [] 1322 }, 1323 "overrides": [] 1324 }, 1325 "fill": 1, 1326 "fillGradient": 0, 1327 "gridPos": { 1328 "h": 4, 1329 "w": 12, 1330 "x": 12, 1331 "y": 7 1332 }, 1333 "hiddenSeries": false, 1334 "id": 220, 1335 "legend": { 1336 "alignAsTable": true, 1337 "avg": false, 1338 "current": true, 1339 "max": true, 1340 "min": false, 1341 "rightSide": false, 1342 "show": true, 1343 "total": false, 1344 "values": true 1345 }, 1346 "lines": true, 1347 "linewidth": 1, 1348 "links": [], 1349 "nullPointMode": "null", 1350 "options": { 1351 "alertThreshold": true 1352 }, 1353 "paceLength": 10, 1354 "percentage": false, 1355 "pluginVersion": "7.5.11", 1356 "pointradius": 2, 1357 "points": false, 1358 "renderer": "flot", 1359 "seriesOverrides": [], 1360 "spaceLength": 10, 1361 "stack": false, 1362 "steppedLine": false, 1363 "targets": [ 1364 { 1365 "expr": "sum(ticdc_sorter_output_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (instance,namespace, changefeed, type)", 1366 "format": "time_series", 1367 "intervalFactor": 1, 1368 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{type}}", 1369 "refId": "A" 1370 } 1371 ], 1372 "thresholds": [], 1373 "timeFrom": null, 1374 "timeRegions": [], 1375 "timeShift": null, 1376 "title": "Sorter output events", 1377 "tooltip": { 1378 "shared": true, 1379 "sort": 0, 1380 "value_type": "individual" 1381 }, 1382 "type": "graph", 1383 "xaxis": { 1384 "buckets": null, 1385 "mode": "time", 1386 "name": null, 1387 "show": true, 1388 "values": [] 1389 }, 1390 "yaxes": [ 1391 { 1392 "format": "none", 1393 "label": null, 1394 "logBase": 1, 1395 "max": null, 1396 "min": null, 1397 "show": true 1398 }, 1399 { 1400 "format": "short", 1401 "label": null, 1402 "logBase": 1, 1403 "max": null, 1404 "min": null, 1405 "show": true 1406 } 1407 ], 1408 "yaxis": { 1409 "align": false, 1410 "alignLevel": null 1411 } 1412 }, 1413 { 1414 "aliasColors": {}, 1415 "bars": false, 1416 "dashLength": 10, 1417 "dashes": false, 1418 "datasource": "${DS_TEST-CLUSTER}", 1419 "description": "The number of events that mounter outputs to sink per second", 1420 "fieldConfig": { 1421 "defaults": { 1422 "links": [] 1423 }, 1424 "overrides": [] 1425 }, 1426 "fill": 1, 1427 "fillGradient": 0, 1428 "gridPos": { 1429 "h": 4, 1430 "w": 12, 1431 "x": 0, 1432 "y": 11 1433 }, 1434 "hiddenSeries": false, 1435 "id": 219, 1436 "legend": { 1437 "alignAsTable": true, 1438 "avg": false, 1439 "current": true, 1440 "max": true, 1441 "min": false, 1442 "rightSide": false, 1443 "show": true, 1444 "total": false, 1445 "values": true 1446 }, 1447 "lines": true, 1448 "linewidth": 1, 1449 "links": [], 1450 "nullPointMode": "null", 1451 "options": { 1452 "alertThreshold": true 1453 }, 1454 "paceLength": 10, 1455 "percentage": false, 1456 "pluginVersion": "7.5.11", 1457 "pointradius": 2, 1458 "points": false, 1459 "renderer": "flot", 1460 "seriesOverrides": [], 1461 "spaceLength": 10, 1462 "stack": false, 1463 "steppedLine": false, 1464 "targets": [ 1465 { 1466 "expr": "sum(rate(ticdc_mounter_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (instance,namespace,changefeed)", 1467 "format": "time_series", 1468 "intervalFactor": 1, 1469 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 1470 "refId": "A" 1471 } 1472 ], 1473 "thresholds": [], 1474 "timeFrom": null, 1475 "timeRegions": [], 1476 "timeShift": null, 1477 "title": "Mounter output events/s", 1478 "tooltip": { 1479 "shared": true, 1480 "sort": 0, 1481 "value_type": "individual" 1482 }, 1483 "type": "graph", 1484 "xaxis": { 1485 "buckets": null, 1486 "mode": "time", 1487 "name": null, 1488 "show": true, 1489 "values": [] 1490 }, 1491 "yaxes": [ 1492 { 1493 "$$hashKey": "object:344", 1494 "format": "short", 1495 "label": null, 1496 "logBase": 1, 1497 "max": null, 1498 "min": null, 1499 "show": true 1500 }, 1501 { 1502 "$$hashKey": "object:345", 1503 "format": "short", 1504 "label": null, 1505 "logBase": 1, 1506 "max": null, 1507 "min": null, 1508 "show": false 1509 } 1510 ], 1511 "yaxis": { 1512 "align": false, 1513 "alignLevel": null 1514 } 1515 }, 1516 { 1517 "aliasColors": {}, 1518 "bars": false, 1519 "dashLength": 10, 1520 "dashes": false, 1521 "datasource": "${DS_TEST-CLUSTER}", 1522 "description": "The total number of events that mounter outputs", 1523 "fieldConfig": { 1524 "defaults": { 1525 "links": [] 1526 }, 1527 "overrides": [] 1528 }, 1529 "fill": 1, 1530 "fillGradient": 0, 1531 "gridPos": { 1532 "h": 4, 1533 "w": 12, 1534 "x": 12, 1535 "y": 11 1536 }, 1537 "hiddenSeries": false, 1538 "id": 224, 1539 "legend": { 1540 "alignAsTable": true, 1541 "avg": false, 1542 "current": true, 1543 "max": true, 1544 "min": false, 1545 "rightSide": false, 1546 "show": true, 1547 "total": false, 1548 "values": true 1549 }, 1550 "lines": true, 1551 "linewidth": 1, 1552 "links": [], 1553 "nullPointMode": "null", 1554 "options": { 1555 "alertThreshold": true 1556 }, 1557 "paceLength": 10, 1558 "percentage": false, 1559 "pluginVersion": "7.5.11", 1560 "pointradius": 2, 1561 "points": false, 1562 "renderer": "flot", 1563 "seriesOverrides": [], 1564 "spaceLength": 10, 1565 "stack": false, 1566 "steppedLine": false, 1567 "targets": [ 1568 { 1569 "expr": "sum(ticdc_mounter_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance)", 1570 "format": "time_series", 1571 "intervalFactor": 1, 1572 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 1573 "refId": "A" 1574 } 1575 ], 1576 "thresholds": [], 1577 "timeFrom": null, 1578 "timeRegions": [], 1579 "timeShift": null, 1580 "title": "Mounter output events", 1581 "tooltip": { 1582 "shared": true, 1583 "sort": 0, 1584 "value_type": "individual" 1585 }, 1586 "type": "graph", 1587 "xaxis": { 1588 "buckets": null, 1589 "mode": "time", 1590 "name": null, 1591 "show": true, 1592 "values": [] 1593 }, 1594 "yaxes": [ 1595 { 1596 "format": "none", 1597 "label": null, 1598 "logBase": 1, 1599 "max": null, 1600 "min": null, 1601 "show": true 1602 }, 1603 { 1604 "format": "short", 1605 "label": null, 1606 "logBase": 1, 1607 "max": null, 1608 "min": null, 1609 "show": true 1610 } 1611 ], 1612 "yaxis": { 1613 "align": false, 1614 "alignLevel": null 1615 } 1616 }, 1617 { 1618 "aliasColors": {}, 1619 "bars": false, 1620 "dashLength": 10, 1621 "dashes": false, 1622 "datasource": "${DS_TEST-CLUSTER}", 1623 "description": "The number of rows that sink flushes to downstream per second.", 1624 "fieldConfig": { 1625 "defaults": { 1626 "links": [] 1627 }, 1628 "overrides": [] 1629 }, 1630 "fill": 1, 1631 "fillGradient": 0, 1632 "gridPos": { 1633 "h": 4, 1634 "w": 12, 1635 "x": 0, 1636 "y": 15 1637 }, 1638 "hiddenSeries": false, 1639 "id": 664, 1640 "legend": { 1641 "alignAsTable": true, 1642 "avg": false, 1643 "current": true, 1644 "max": true, 1645 "min": false, 1646 "rightSide": false, 1647 "show": true, 1648 "sort": "current", 1649 "sortDesc": true, 1650 "total": false, 1651 "values": true 1652 }, 1653 "lines": true, 1654 "linewidth": 1, 1655 "links": [], 1656 "nullPointMode": "null", 1657 "options": { 1658 "alertThreshold": true 1659 }, 1660 "percentage": false, 1661 "pluginVersion": "7.5.11", 1662 "pointradius": 2, 1663 "points": false, 1664 "renderer": "flot", 1665 "seriesOverrides": [], 1666 "spaceLength": 10, 1667 "stack": false, 1668 "steppedLine": false, 1669 "targets": [ 1670 { 1671 "exemplar": true, 1672 "expr": "sum(rate(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)", 1673 "format": "time_series", 1674 "interval": "", 1675 "intervalFactor": 1, 1676 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 1677 "refId": "A" 1678 } 1679 ], 1680 "thresholds": [], 1681 "timeFrom": null, 1682 "timeRegions": [], 1683 "timeShift": null, 1684 "title": "Sink flush rows/s", 1685 "tooltip": { 1686 "shared": true, 1687 "sort": 2, 1688 "value_type": "individual" 1689 }, 1690 "type": "graph", 1691 "xaxis": { 1692 "buckets": null, 1693 "mode": "time", 1694 "name": null, 1695 "show": true, 1696 "values": [] 1697 }, 1698 "yaxes": [ 1699 { 1700 "$$hashKey": "object:389", 1701 "format": "short", 1702 "label": null, 1703 "logBase": 1, 1704 "max": null, 1705 "min": null, 1706 "show": true 1707 }, 1708 { 1709 "$$hashKey": "object:390", 1710 "format": "none", 1711 "label": null, 1712 "logBase": 1, 1713 "max": null, 1714 "min": null, 1715 "show": false 1716 } 1717 ], 1718 "yaxis": { 1719 "align": false, 1720 "alignLevel": null 1721 } 1722 }, 1723 { 1724 "aliasColors": {}, 1725 "bars": false, 1726 "dashLength": 10, 1727 "dashes": false, 1728 "datasource": "${DS_TEST-CLUSTER}", 1729 "description": "The number of rows(events) that are flushed by sink.", 1730 "fieldConfig": { 1731 "defaults": { 1732 "links": [] 1733 }, 1734 "overrides": [] 1735 }, 1736 "fill": 1, 1737 "fillGradient": 0, 1738 "gridPos": { 1739 "h": 4, 1740 "w": 12, 1741 "x": 12, 1742 "y": 15 1743 }, 1744 "hiddenSeries": false, 1745 "id": 665, 1746 "legend": { 1747 "alignAsTable": true, 1748 "avg": false, 1749 "current": true, 1750 "max": true, 1751 "min": false, 1752 "rightSide": false, 1753 "show": true, 1754 "sort": "current", 1755 "sortDesc": true, 1756 "total": false, 1757 "values": true 1758 }, 1759 "lines": true, 1760 "linewidth": 1, 1761 "links": [], 1762 "nullPointMode": "null", 1763 "options": { 1764 "alertThreshold": true 1765 }, 1766 "percentage": false, 1767 "pluginVersion": "7.5.11", 1768 "pointradius": 2, 1769 "points": false, 1770 "renderer": "flot", 1771 "seriesOverrides": [], 1772 "spaceLength": 10, 1773 "stack": false, 1774 "steppedLine": false, 1775 "targets": [ 1776 { 1777 "exemplar": true, 1778 "expr": "sum(ticdc_sink_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace,changefeed, instance)", 1779 "format": "time_series", 1780 "interval": "", 1781 "intervalFactor": 1, 1782 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 1783 "refId": "A" 1784 } 1785 ], 1786 "thresholds": [], 1787 "timeFrom": null, 1788 "timeRegions": [], 1789 "timeShift": null, 1790 "title": "Sink flush rows", 1791 "tooltip": { 1792 "shared": true, 1793 "sort": 2, 1794 "value_type": "individual" 1795 }, 1796 "type": "graph", 1797 "xaxis": { 1798 "buckets": null, 1799 "mode": "time", 1800 "name": null, 1801 "show": true, 1802 "values": [] 1803 }, 1804 "yaxes": [ 1805 { 1806 "format": "none", 1807 "label": null, 1808 "logBase": 1, 1809 "max": null, 1810 "min": null, 1811 "show": true 1812 }, 1813 { 1814 "format": "short", 1815 "label": null, 1816 "logBase": 1, 1817 "max": null, 1818 "min": null, 1819 "show": true 1820 } 1821 ], 1822 "yaxis": { 1823 "align": false, 1824 "alignLevel": null 1825 } 1826 } 1827 ], 1828 "title": "Dataflow", 1829 "type": "row" 1830 }, 1831 { 1832 "collapsed": true, 1833 "datasource": null, 1834 "gridPos": { 1835 "h": 1, 1836 "w": 24, 1837 "x": 0, 1838 "y": 3 1839 }, 1840 "id": 616, 1841 "panels": [ 1842 { 1843 "aliasColors": {}, 1844 "bars": false, 1845 "dashLength": 10, 1846 "dashes": false, 1847 "datasource": "${DS_TEST-CLUSTER}", 1848 "description": "Sink worker busy ratio", 1849 "fieldConfig": { 1850 "defaults": {}, 1851 "overrides": [] 1852 }, 1853 "fill": 1, 1854 "fillGradient": 0, 1855 "gridPos": { 1856 "h": 8, 1857 "w": 12, 1858 "x": 0, 1859 "y": 20 1860 }, 1861 "hiddenSeries": false, 1862 "id": 635, 1863 "legend": { 1864 "alignAsTable": true, 1865 "avg": false, 1866 "current": true, 1867 "max": false, 1868 "min": false, 1869 "show": true, 1870 "total": false, 1871 "values": true 1872 }, 1873 "lines": true, 1874 "linewidth": 1, 1875 "nullPointMode": "null", 1876 "options": { 1877 "alertThreshold": true 1878 }, 1879 "percentage": false, 1880 "pluginVersion": "7.5.11", 1881 "pointradius": 2, 1882 "points": false, 1883 "renderer": "flot", 1884 "seriesOverrides": [], 1885 "spaceLength": 10, 1886 "stack": false, 1887 "steppedLine": false, 1888 "targets": [ 1889 { 1890 "exemplar": true, 1891 "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_sink_dml_batch_commit_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,namespace,changefeed,instance))", 1892 "interval": "", 1893 "legendFormat": "99.9-{{namespace}}-{{changefeed}}-{{instance}}", 1894 "queryType": "randomWalk", 1895 "refId": "A" 1896 }, 1897 { 1898 "exemplar": true, 1899 "expr": "sum(rate(ticdc_sink_txn_sink_dml_batch_commit_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_txn_sink_dml_batch_commit_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) by (namespace,changefeed,instance)", 1900 "hide": false, 1901 "interval": "", 1902 "legendFormat": "avg-{{namespace}}-{{changefeed}}-{{instance}}", 1903 "refId": "B" 1904 } 1905 ], 1906 "thresholds": [], 1907 "timeFrom": null, 1908 "timeRegions": [], 1909 "timeShift": null, 1910 "title": "Backend Flush Duration", 1911 "tooltip": { 1912 "shared": true, 1913 "sort": 0, 1914 "value_type": "individual" 1915 }, 1916 "type": "graph", 1917 "xaxis": { 1918 "buckets": null, 1919 "mode": "time", 1920 "name": null, 1921 "show": true, 1922 "values": [] 1923 }, 1924 "yaxes": [ 1925 { 1926 "format": "s", 1927 "label": null, 1928 "logBase": 1, 1929 "max": null, 1930 "min": null, 1931 "show": true 1932 }, 1933 { 1934 "format": "short", 1935 "label": null, 1936 "logBase": 1, 1937 "max": null, 1938 "min": null, 1939 "show": false 1940 } 1941 ], 1942 "yaxis": { 1943 "align": false, 1944 "alignLevel": null 1945 } 1946 }, 1947 { 1948 "aliasColors": {}, 1949 "bars": false, 1950 "dashLength": 10, 1951 "dashes": false, 1952 "datasource": "${DS_TEST-CLUSTER}", 1953 "description": "Full flush (backend flush + callback + conflict detector notify) duration", 1954 "fieldConfig": { 1955 "defaults": {}, 1956 "overrides": [] 1957 }, 1958 "fill": 1, 1959 "fillGradient": 0, 1960 "gridPos": { 1961 "h": 8, 1962 "w": 12, 1963 "x": 12, 1964 "y": 20 1965 }, 1966 "hiddenSeries": false, 1967 "id": 620, 1968 "legend": { 1969 "alignAsTable": true, 1970 "avg": false, 1971 "current": true, 1972 "max": false, 1973 "min": false, 1974 "rightSide": false, 1975 "show": true, 1976 "total": false, 1977 "values": true 1978 }, 1979 "lines": true, 1980 "linewidth": 1, 1981 "nullPointMode": "null", 1982 "options": { 1983 "alertThreshold": true 1984 }, 1985 "percentage": false, 1986 "pluginVersion": "7.5.11", 1987 "pointradius": 2, 1988 "points": false, 1989 "renderer": "flot", 1990 "seriesOverrides": [], 1991 "spaceLength": 10, 1992 "stack": false, 1993 "steppedLine": false, 1994 "targets": [ 1995 { 1996 "exemplar": true, 1997 "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_txn_worker_flush_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,namespace,changefeed,instance))", 1998 "interval": "", 1999 "legendFormat": "99.9-{{namespace}}-{{changefeed}}-{{instance}}", 2000 "queryType": "randomWalk", 2001 "refId": "A" 2002 }, 2003 { 2004 "exemplar": true, 2005 "expr": "sum(rate(ticdc_sink_txn_worker_flush_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_txn_worker_flush_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) by (namespace,changefeed,instance)", 2006 "hide": false, 2007 "interval": "", 2008 "legendFormat": "avg-{{namespace}}-{{changefeed}}-{{instance}}", 2009 "refId": "B" 2010 } 2011 ], 2012 "thresholds": [], 2013 "timeFrom": null, 2014 "timeRegions": [], 2015 "timeShift": null, 2016 "title": "Full Flush Duration", 2017 "tooltip": { 2018 "shared": true, 2019 "sort": 0, 2020 "value_type": "individual" 2021 }, 2022 "type": "graph", 2023 "xaxis": { 2024 "buckets": null, 2025 "mode": "time", 2026 "name": null, 2027 "show": true, 2028 "values": [] 2029 }, 2030 "yaxes": [ 2031 { 2032 "format": "s", 2033 "label": null, 2034 "logBase": 2, 2035 "max": null, 2036 "min": null, 2037 "show": true 2038 }, 2039 { 2040 "format": "short", 2041 "label": null, 2042 "logBase": 1, 2043 "max": null, 2044 "min": null, 2045 "show": false 2046 } 2047 ], 2048 "yaxis": { 2049 "align": false, 2050 "alignLevel": null 2051 } 2052 } 2053 ], 2054 "title": "Transaction Sink", 2055 "type": "row" 2056 }, 2057 { 2058 "collapsed": true, 2059 "datasource": null, 2060 "gridPos": { 2061 "h": 1, 2062 "w": 24, 2063 "x": 0, 2064 "y": 4 2065 }, 2066 "id": 623, 2067 "panels": [ 2068 { 2069 "aliasColors": {}, 2070 "bars": false, 2071 "dashLength": 10, 2072 "dashes": false, 2073 "datasource": "${DS_TEST-CLUSTER}", 2074 "description": "MQ worker send messages to Kafka, this metric record the time cost on send every message.", 2075 "fieldConfig": { 2076 "defaults": {}, 2077 "overrides": [] 2078 }, 2079 "fill": 1, 2080 "fillGradient": 0, 2081 "gridPos": { 2082 "h": 7, 2083 "w": 12, 2084 "x": 0, 2085 "y": 5 2086 }, 2087 "hiddenSeries": false, 2088 "id": 653, 2089 "legend": { 2090 "alignAsTable": true, 2091 "avg": false, 2092 "current": true, 2093 "max": true, 2094 "min": false, 2095 "rightSide": false, 2096 "show": true, 2097 "total": false, 2098 "values": true 2099 }, 2100 "lines": true, 2101 "linewidth": 1, 2102 "nullPointMode": "null", 2103 "options": { 2104 "alertThreshold": true 2105 }, 2106 "percentage": false, 2107 "pluginVersion": "7.5.11", 2108 "pointradius": 2, 2109 "points": false, 2110 "renderer": "flot", 2111 "seriesOverrides": [], 2112 "spaceLength": 10, 2113 "stack": false, 2114 "steppedLine": false, 2115 "targets": [ 2116 { 2117 "exemplar": true, 2118 "expr": "histogram_quantile(0.999, sum(rate(ticdc_sink_mq_worker_send_message_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,namespace,changefeed,instance))", 2119 "interval": "", 2120 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-P999", 2121 "queryType": "randomWalk", 2122 "refId": "A" 2123 }, 2124 { 2125 "exemplar": true, 2126 "expr": "sum(rate(ticdc_sink_mq_worker_send_message_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) by (namespace,changefeed,instance) / \nsum(rate(ticdc_sink_mq_worker_send_message_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) by (namespace,changefeed,instance)", 2127 "hide": false, 2128 "interval": "", 2129 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-avg", 2130 "refId": "B" 2131 } 2132 ], 2133 "thresholds": [], 2134 "timeFrom": null, 2135 "timeRegions": [], 2136 "timeShift": null, 2137 "title": "Worker Send Message Duration Percentile", 2138 "tooltip": { 2139 "shared": true, 2140 "sort": 0, 2141 "value_type": "individual" 2142 }, 2143 "type": "graph", 2144 "xaxis": { 2145 "buckets": null, 2146 "mode": "time", 2147 "name": null, 2148 "show": true, 2149 "values": [] 2150 }, 2151 "yaxes": [ 2152 { 2153 "format": "s", 2154 "label": null, 2155 "logBase": 2, 2156 "max": null, 2157 "min": null, 2158 "show": true 2159 }, 2160 { 2161 "format": "short", 2162 "label": null, 2163 "logBase": 1, 2164 "max": null, 2165 "min": null, 2166 "show": false 2167 } 2168 ], 2169 "yaxis": { 2170 "align": false, 2171 "alignLevel": null 2172 } 2173 }, 2174 { 2175 "aliasColors": {}, 2176 "bars": false, 2177 "dashLength": 10, 2178 "dashes": false, 2179 "datasource": "${DS_TEST-CLUSTER}", 2180 "description": "Bytes/second written off all brokers.\nvalue = one-minute moving average rate of Bytes per second", 2181 "fieldConfig": { 2182 "defaults": { 2183 "links": [] 2184 }, 2185 "overrides": [] 2186 }, 2187 "fill": 1, 2188 "fillGradient": 0, 2189 "gridPos": { 2190 "h": 7, 2191 "w": 12, 2192 "x": 12, 2193 "y": 5 2194 }, 2195 "hiddenSeries": false, 2196 "id": 628, 2197 "legend": { 2198 "alignAsTable": true, 2199 "avg": true, 2200 "current": true, 2201 "max": true, 2202 "min": true, 2203 "rightSide": false, 2204 "show": true, 2205 "total": false, 2206 "values": true 2207 }, 2208 "lines": true, 2209 "linewidth": 1, 2210 "links": [], 2211 "nullPointMode": "null", 2212 "options": { 2213 "alertThreshold": true 2214 }, 2215 "paceLength": 10, 2216 "percentage": false, 2217 "pluginVersion": "7.5.11", 2218 "pointradius": 2, 2219 "points": false, 2220 "renderer": "flot", 2221 "seriesOverrides": [], 2222 "spaceLength": 10, 2223 "stack": false, 2224 "steppedLine": false, 2225 "targets": [ 2226 { 2227 "exemplar": true, 2228 "expr": "sum(ticdc_sink_kafka_producer_outgoing_byte_rate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)", 2229 "format": "time_series", 2230 "interval": "", 2231 "intervalFactor": 1, 2232 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}", 2233 "refId": "A" 2234 } 2235 ], 2236 "thresholds": [], 2237 "timeFrom": null, 2238 "timeRegions": [], 2239 "timeShift": null, 2240 "title": "Kafka Outgoing Bytes", 2241 "tooltip": { 2242 "shared": true, 2243 "sort": 0, 2244 "value_type": "individual" 2245 }, 2246 "type": "graph", 2247 "xaxis": { 2248 "buckets": null, 2249 "mode": "time", 2250 "name": null, 2251 "show": true, 2252 "values": [] 2253 }, 2254 "yaxes": [ 2255 { 2256 "format": "bytes", 2257 "label": null, 2258 "logBase": 1, 2259 "max": null, 2260 "min": "0", 2261 "show": true 2262 }, 2263 { 2264 "format": "short", 2265 "label": null, 2266 "logBase": 1, 2267 "max": null, 2268 "min": null, 2269 "show": false 2270 } 2271 ], 2272 "yaxis": { 2273 "align": false, 2274 "alignLevel": null 2275 } 2276 } 2277 ], 2278 "title": "MQ Sink", 2279 "type": "row" 2280 }, 2281 { 2282 "collapsed": true, 2283 "datasource": null, 2284 "gridPos": { 2285 "h": 1, 2286 "w": 24, 2287 "x": 0, 2288 "y": 5 2289 }, 2290 "id": 713, 2291 "panels": [ 2292 { 2293 "aliasColors": {}, 2294 "bars": false, 2295 "dashLength": 10, 2296 "dashes": false, 2297 "datasource": "${DS_TEST-CLUSTER}", 2298 "fieldConfig": { 2299 "defaults": {}, 2300 "overrides": [] 2301 }, 2302 "fill": 1, 2303 "fillGradient": 0, 2304 "gridPos": { 2305 "h": 8, 2306 "w": 12, 2307 "x": 0, 2308 "y": 6 2309 }, 2310 "hiddenSeries": false, 2311 "id": 714, 2312 "legend": { 2313 "avg": false, 2314 "current": false, 2315 "max": false, 2316 "min": false, 2317 "show": true, 2318 "total": false, 2319 "values": false 2320 }, 2321 "lines": true, 2322 "linewidth": 1, 2323 "nullPointMode": "null", 2324 "options": { 2325 "alertThreshold": true 2326 }, 2327 "percentage": false, 2328 "pluginVersion": "7.5.11", 2329 "pointradius": 2, 2330 "points": false, 2331 "renderer": "flot", 2332 "seriesOverrides": [], 2333 "spaceLength": 10, 2334 "stack": false, 2335 "steppedLine": false, 2336 "targets": [ 2337 { 2338 "exemplar": true, 2339 "expr": "sum(rate(ticdc_sink_cloud_storage_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (namespace, changefeed, instance)\n", 2340 "interval": "", 2341 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 2342 "queryType": "randomWalk", 2343 "refId": "A" 2344 } 2345 ], 2346 "thresholds": [], 2347 "timeFrom": null, 2348 "timeRegions": [], 2349 "timeShift": null, 2350 "title": "Write Bytes/s", 2351 "tooltip": { 2352 "shared": true, 2353 "sort": 0, 2354 "value_type": "individual" 2355 }, 2356 "type": "graph", 2357 "xaxis": { 2358 "buckets": null, 2359 "mode": "time", 2360 "name": null, 2361 "show": true, 2362 "values": [] 2363 }, 2364 "yaxes": [ 2365 { 2366 "format": "short", 2367 "label": null, 2368 "logBase": 1, 2369 "max": null, 2370 "min": null, 2371 "show": true 2372 }, 2373 { 2374 "format": "short", 2375 "label": null, 2376 "logBase": 1, 2377 "max": null, 2378 "min": null, 2379 "show": true 2380 } 2381 ], 2382 "yaxis": { 2383 "align": false, 2384 "alignLevel": null 2385 } 2386 }, 2387 { 2388 "aliasColors": {}, 2389 "bars": false, 2390 "dashLength": 10, 2391 "dashes": false, 2392 "datasource": "${DS_TEST-CLUSTER}", 2393 "fieldConfig": { 2394 "defaults": {}, 2395 "overrides": [] 2396 }, 2397 "fill": 1, 2398 "fillGradient": 0, 2399 "gridPos": { 2400 "h": 8, 2401 "w": 12, 2402 "x": 12, 2403 "y": 6 2404 }, 2405 "hiddenSeries": false, 2406 "id": 715, 2407 "legend": { 2408 "avg": false, 2409 "current": false, 2410 "max": false, 2411 "min": false, 2412 "show": true, 2413 "total": false, 2414 "values": false 2415 }, 2416 "lines": true, 2417 "linewidth": 1, 2418 "nullPointMode": "null", 2419 "options": { 2420 "alertThreshold": true 2421 }, 2422 "percentage": false, 2423 "pluginVersion": "7.5.11", 2424 "pointradius": 2, 2425 "points": false, 2426 "renderer": "flot", 2427 "seriesOverrides": [], 2428 "spaceLength": 10, 2429 "stack": false, 2430 "steppedLine": false, 2431 "targets": [ 2432 { 2433 "exemplar": true, 2434 "expr": "sum(ticdc_sink_cloud_storage_file_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}) by (namespace, changefeed, instance)", 2435 "interval": "", 2436 "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", 2437 "queryType": "randomWalk", 2438 "refId": "A" 2439 } 2440 ], 2441 "thresholds": [], 2442 "timeFrom": null, 2443 "timeRegions": [], 2444 "timeShift": null, 2445 "title": "File Count", 2446 "tooltip": { 2447 "shared": true, 2448 "sort": 0, 2449 "value_type": "individual" 2450 }, 2451 "type": "graph", 2452 "xaxis": { 2453 "buckets": null, 2454 "mode": "time", 2455 "name": null, 2456 "show": true, 2457 "values": [] 2458 }, 2459 "yaxes": [ 2460 { 2461 "format": "short", 2462 "label": null, 2463 "logBase": 1, 2464 "max": null, 2465 "min": null, 2466 "show": true 2467 }, 2468 { 2469 "format": "short", 2470 "label": null, 2471 "logBase": 1, 2472 "max": null, 2473 "min": null, 2474 "show": true 2475 } 2476 ], 2477 "yaxis": { 2478 "align": false, 2479 "alignLevel": null 2480 } 2481 } 2482 ], 2483 "title": "Cloud Storage Sink", 2484 "type": "row" 2485 }, 2486 { 2487 "collapsed": true, 2488 "datasource": null, 2489 "gridPos": { 2490 "h": 1, 2491 "w": 24, 2492 "x": 0, 2493 "y": 6 2494 }, 2495 "id": 187, 2496 "panels": [ 2497 { 2498 "aliasColors": {}, 2499 "bars": false, 2500 "dashLength": 10, 2501 "dashes": false, 2502 "datasource": "${DS_TEST-CLUSTER}", 2503 "description": "The total count of rows that are processed by redo writer", 2504 "fieldConfig": { 2505 "defaults": {}, 2506 "overrides": [] 2507 }, 2508 "fill": 1, 2509 "fillGradient": 0, 2510 "gridPos": { 2511 "h": 8, 2512 "w": 12, 2513 "x": 0, 2514 "y": 7 2515 }, 2516 "hiddenSeries": false, 2517 "id": 170, 2518 "legend": { 2519 "alignAsTable": true, 2520 "avg": true, 2521 "current": true, 2522 "max": true, 2523 "min": false, 2524 "rightSide": true, 2525 "show": true, 2526 "total": false, 2527 "values": true 2528 }, 2529 "lines": true, 2530 "linewidth": 1, 2531 "nullPointMode": "null", 2532 "options": { 2533 "alertThreshold": true 2534 }, 2535 "percentage": false, 2536 "pluginVersion": "7.5.11", 2537 "pointradius": 2, 2538 "points": false, 2539 "renderer": "flot", 2540 "seriesOverrides": [], 2541 "spaceLength": 10, 2542 "stack": false, 2543 "steppedLine": false, 2544 "targets": [ 2545 { 2546 "exemplar": true, 2547 "expr": "sum(rate(ticdc_redo_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (instance)", 2548 "format": "time_series", 2549 "interval": "", 2550 "intervalFactor": 1, 2551 "legendFormat": "{{instance}}", 2552 "queryType": "randomWalk", 2553 "refId": "A" 2554 }, 2555 { 2556 "exemplar": true, 2557 "expr": "sum(rate(ticdc_redo_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\"}[1m])) by (namespace,changefeed)", 2558 "format": "time_series", 2559 "hide": false, 2560 "interval": "", 2561 "intervalFactor": 1, 2562 "legendFormat": "total", 2563 "refId": "B" 2564 } 2565 ], 2566 "thresholds": [], 2567 "timeFrom": null, 2568 "timeRegions": [], 2569 "timeShift": null, 2570 "title": "Redo Write rows/s ", 2571 "tooltip": { 2572 "shared": true, 2573 "sort": 0, 2574 "value_type": "individual" 2575 }, 2576 "type": "graph", 2577 "xaxis": { 2578 "buckets": null, 2579 "mode": "time", 2580 "name": null, 2581 "show": true, 2582 "values": [] 2583 }, 2584 "yaxes": [ 2585 { 2586 "format": "short", 2587 "label": null, 2588 "logBase": 1, 2589 "max": null, 2590 "min": null, 2591 "show": true 2592 }, 2593 { 2594 "format": "short", 2595 "label": null, 2596 "logBase": 1, 2597 "max": null, 2598 "min": null, 2599 "show": false 2600 } 2601 ], 2602 "yaxis": { 2603 "align": false, 2604 "alignLevel": null 2605 } 2606 }, 2607 { 2608 "aliasColors": {}, 2609 "bars": false, 2610 "dashLength": 10, 2611 "dashes": false, 2612 "datasource": "${DS_TEST-CLUSTER}", 2613 "description": "Total number of bytes redo log written", 2614 "fieldConfig": { 2615 "defaults": {}, 2616 "overrides": [] 2617 }, 2618 "fill": 1, 2619 "fillGradient": 0, 2620 "gridPos": { 2621 "h": 8, 2622 "w": 12, 2623 "x": 12, 2624 "y": 7 2625 }, 2626 "hiddenSeries": false, 2627 "id": 173, 2628 "legend": { 2629 "avg": false, 2630 "current": false, 2631 "max": false, 2632 "min": false, 2633 "show": true, 2634 "total": false, 2635 "values": false 2636 }, 2637 "lines": true, 2638 "linewidth": 1, 2639 "nullPointMode": "null", 2640 "options": { 2641 "alertThreshold": true 2642 }, 2643 "percentage": false, 2644 "pluginVersion": "7.5.11", 2645 "pointradius": 2, 2646 "points": false, 2647 "renderer": "flot", 2648 "seriesOverrides": [], 2649 "spaceLength": 10, 2650 "stack": false, 2651 "steppedLine": false, 2652 "targets": [ 2653 { 2654 "exemplar": true, 2655 "expr": "sum(rate(ticdc_redo_write_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (instance)", 2656 "interval": "", 2657 "legendFormat": "{{instance}}", 2658 "queryType": "randomWalk", 2659 "refId": "A" 2660 } 2661 ], 2662 "thresholds": [], 2663 "timeFrom": null, 2664 "timeRegions": [], 2665 "timeShift": null, 2666 "title": "Redo Write bytes/s ", 2667 "tooltip": { 2668 "shared": true, 2669 "sort": 0, 2670 "value_type": "individual" 2671 }, 2672 "type": "graph", 2673 "xaxis": { 2674 "buckets": null, 2675 "mode": "time", 2676 "name": null, 2677 "show": true, 2678 "values": [] 2679 }, 2680 "yaxes": [ 2681 { 2682 "format": "bytes", 2683 "label": null, 2684 "logBase": 1, 2685 "max": null, 2686 "min": null, 2687 "show": true 2688 }, 2689 { 2690 "format": "short", 2691 "label": null, 2692 "logBase": 1, 2693 "max": null, 2694 "min": null, 2695 "show": true 2696 } 2697 ], 2698 "yaxis": { 2699 "align": false, 2700 "alignLevel": null 2701 } 2702 }, 2703 { 2704 "cards": { 2705 "cardPadding": 0, 2706 "cardRound": 0 2707 }, 2708 "color": { 2709 "cardColor": "#b4ff00", 2710 "colorScale": "sqrt", 2711 "colorScheme": "interpolateSpectral", 2712 "exponent": 0.5, 2713 "min": 0, 2714 "mode": "spectrum" 2715 }, 2716 "dataFormat": "tsbuckets", 2717 "datasource": "${DS_TEST-CLUSTER}", 2718 "description": "The latency distributions of flushLog called by redoManager", 2719 "fieldConfig": { 2720 "defaults": {}, 2721 "overrides": [] 2722 }, 2723 "gridPos": { 2724 "h": 8, 2725 "w": 12, 2726 "x": 0, 2727 "y": 15 2728 }, 2729 "heatmap": {}, 2730 "hideZeroBuckets": true, 2731 "highlightCards": true, 2732 "id": 178, 2733 "legend": { 2734 "show": true 2735 }, 2736 "pluginVersion": "6.1.6", 2737 "reverseYBuckets": false, 2738 "targets": [ 2739 { 2740 "exemplar": true, 2741 "expr": "max(rate(ticdc_redo_flush_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)", 2742 "format": "heatmap", 2743 "interval": "", 2744 "intervalFactor": 2, 2745 "legendFormat": "{{le}}", 2746 "queryType": "randomWalk", 2747 "refId": "A" 2748 } 2749 ], 2750 "timeFrom": null, 2751 "timeShift": null, 2752 "title": "Redo flush log duration", 2753 "tooltip": { 2754 "show": true, 2755 "showHistogram": true 2756 }, 2757 "tooltipDecimals": 1, 2758 "type": "heatmap", 2759 "xAxis": { 2760 "show": true 2761 }, 2762 "xBucketNumber": null, 2763 "xBucketSize": null, 2764 "yAxis": { 2765 "decimals": 1, 2766 "format": "s", 2767 "logBase": 1, 2768 "max": null, 2769 "min": null, 2770 "show": true, 2771 "splitFactor": null 2772 }, 2773 "yBucketBound": "upper", 2774 "yBucketNumber": null, 2775 "yBucketSize": null 2776 }, 2777 { 2778 "cards": { 2779 "cardPadding": 0, 2780 "cardRound": 0 2781 }, 2782 "color": { 2783 "cardColor": "#b4ff00", 2784 "colorScale": "sqrt", 2785 "colorScheme": "interpolateSpectral", 2786 "exponent": 0.5, 2787 "min": 0, 2788 "mode": "spectrum" 2789 }, 2790 "dataFormat": "tsbuckets", 2791 "datasource": "${DS_TEST-CLUSTER}", 2792 "description": "The latency distributions of flushall called by redo writer", 2793 "fieldConfig": { 2794 "defaults": {}, 2795 "overrides": [] 2796 }, 2797 "gridPos": { 2798 "h": 8, 2799 "w": 12, 2800 "x": 12, 2801 "y": 15 2802 }, 2803 "heatmap": {}, 2804 "hideZeroBuckets": true, 2805 "highlightCards": true, 2806 "id": 180, 2807 "legend": { 2808 "show": true 2809 }, 2810 "pluginVersion": "6.1.6", 2811 "reverseYBuckets": false, 2812 "targets": [ 2813 { 2814 "exemplar": true, 2815 "expr": "max(rate(ticdc_redo_flushall_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\", changefeed=~\"$changefeed\",instance=~\"$ticdc_instance\"}[1m])) by (le)", 2816 "format": "heatmap", 2817 "interval": "", 2818 "intervalFactor": 2, 2819 "legendFormat": "{{le}}", 2820 "queryType": "randomWalk", 2821 "refId": "A" 2822 } 2823 ], 2824 "timeFrom": null, 2825 "timeShift": null, 2826 "title": "Redo flushall duration", 2827 "tooltip": { 2828 "show": true, 2829 "showHistogram": true 2830 }, 2831 "tooltipDecimals": 1, 2832 "type": "heatmap", 2833 "xAxis": { 2834 "show": true 2835 }, 2836 "xBucketNumber": null, 2837 "xBucketSize": null, 2838 "yAxis": { 2839 "decimals": 1, 2840 "format": "s", 2841 "logBase": 1, 2842 "max": null, 2843 "min": null, 2844 "show": true, 2845 "splitFactor": null 2846 }, 2847 "yBucketBound": "upper", 2848 "yBucketNumber": null, 2849 "yBucketSize": null 2850 } 2851 ], 2852 "title": "Redo", 2853 "type": "row" 2854 } 2855 ], 2856 "refresh": "1m", 2857 "schemaVersion": 27, 2858 "style": "dark", 2859 "tags": [], 2860 "templating": { 2861 "list": [ 2862 { 2863 "allValue": null, 2864 "current": {}, 2865 "datasource": "${DS_TEST-CLUSTER}", 2866 "definition": "", 2867 "description": null, 2868 "error": null, 2869 "hide": 0, 2870 "includeAll": false, 2871 "label": "K8s-cluster", 2872 "multi": false, 2873 "name": "k8s_cluster", 2874 "options": [], 2875 "query": { 2876 "query": "label_values(go_goroutines, k8s_cluster)", 2877 "refId": "debug-k8s_cluster-Variable-Query" 2878 }, 2879 "refresh": 2, 2880 "regex": "", 2881 "skipUrlSync": false, 2882 "sort": 1, 2883 "tagValuesQuery": "", 2884 "tags": [], 2885 "tagsQuery": "", 2886 "type": "query", 2887 "useTags": false 2888 }, 2889 { 2890 "allValue": null, 2891 "current": {}, 2892 "datasource": "${DS_TEST-CLUSTER}", 2893 "definition": "", 2894 "description": null, 2895 "error": null, 2896 "hide": 0, 2897 "includeAll": false, 2898 "label": "tidb_cluster", 2899 "multi": false, 2900 "name": "tidb_cluster", 2901 "options": [], 2902 "query": { 2903 "query": "label_values(go_goroutines{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)", 2904 "refId": "debug-tidb_cluster-Variable-Query" 2905 }, 2906 "refresh": 2, 2907 "regex": "", 2908 "skipUrlSync": false, 2909 "sort": 1, 2910 "tagValuesQuery": "", 2911 "tags": [], 2912 "tagsQuery": "", 2913 "type": "query", 2914 "useTags": false 2915 }, 2916 { 2917 "allValue": ".*", 2918 "current": {}, 2919 "datasource": "${DS_TEST-CLUSTER}", 2920 "definition": "label_values(ticdc_processor_processor_tick_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, namespace)", 2921 "description": null, 2922 "error": null, 2923 "hide": 0, 2924 "includeAll": true, 2925 "label": "Namespace", 2926 "multi": true, 2927 "name": "namespace", 2928 "options": [], 2929 "query": { 2930 "query": "label_values(ticdc_processor_processor_tick_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, namespace)", 2931 "refId": "StandardVariableQuery" 2932 }, 2933 "refresh": 2, 2934 "regex": "", 2935 "skipUrlSync": false, 2936 "sort": 0, 2937 "tagValuesQuery": "", 2938 "tags": [], 2939 "tagsQuery": "", 2940 "type": "query", 2941 "useTags": false 2942 }, 2943 { 2944 "allValue": ".*", 2945 "current": {}, 2946 "datasource": "${DS_TEST-CLUSTER}", 2947 "definition": "label_values(ticdc_processor_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, changefeed)", 2948 "description": null, 2949 "error": null, 2950 "hide": 0, 2951 "includeAll": true, 2952 "label": "Changefeed", 2953 "multi": true, 2954 "name": "changefeed", 2955 "options": [], 2956 "query": { 2957 "query": "label_values(ticdc_processor_resolved_ts{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, changefeed)", 2958 "refId": "debug-changefeed-Variable-Query" 2959 }, 2960 "refresh": 2, 2961 "regex": "", 2962 "skipUrlSync": false, 2963 "sort": 0, 2964 "tagValuesQuery": "", 2965 "tags": [], 2966 "tagsQuery": "", 2967 "type": "query", 2968 "useTags": false 2969 }, 2970 { 2971 "allValue": ".*", 2972 "current": {}, 2973 "datasource": "${DS_TEST-CLUSTER}", 2974 "definition": "label_values(process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}, instance)", 2975 "description": null, 2976 "error": null, 2977 "hide": 0, 2978 "includeAll": true, 2979 "label": "TiCDC", 2980 "multi": true, 2981 "name": "ticdc_instance", 2982 "options": [], 2983 "query": { 2984 "query": "label_values(process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}, instance)", 2985 "refId": "debug-ticdc_instance-Variable-Query" 2986 }, 2987 "refresh": 2, 2988 "regex": "", 2989 "skipUrlSync": false, 2990 "sort": 0, 2991 "tagValuesQuery": "", 2992 "tags": [], 2993 "tagsQuery": "", 2994 "type": "query", 2995 "useTags": false 2996 }, 2997 { 2998 "allValue": ".*", 2999 "current": {}, 3000 "datasource": "${DS_TEST-CLUSTER}", 3001 "definition": "label_values(tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, instance)", 3002 "description": null, 3003 "error": null, 3004 "hide": 0, 3005 "includeAll": true, 3006 "label": "TiKV", 3007 "multi": false, 3008 "name": "tikv_instance", 3009 "options": [], 3010 "query": { 3011 "query": "label_values(tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, instance)", 3012 "refId": "debug-tikv_instance-Variable-Query" 3013 }, 3014 "refresh": 2, 3015 "regex": "", 3016 "skipUrlSync": false, 3017 "sort": 1, 3018 "tagValuesQuery": "", 3019 "tags": [], 3020 "tagsQuery": "", 3021 "type": "query", 3022 "useTags": false 3023 }, 3024 { 3025 "allValue": "9999999999", 3026 "current": { 3027 "selected": true, 3028 "text": "All", 3029 "value": "$__all" 3030 }, 3031 "description": null, 3032 "error": null, 3033 "hide": 0, 3034 "includeAll": true, 3035 "label": "Latency spike (s) >", 3036 "multi": false, 3037 "name": "spike_threshold", 3038 "options": [ 3039 { 3040 "selected": true, 3041 "text": "All", 3042 "value": "$__all" 3043 }, 3044 { 3045 "selected": false, 3046 "text": "1", 3047 "value": "1" 3048 }, 3049 { 3050 "selected": false, 3051 "text": "3", 3052 "value": "3" 3053 }, 3054 { 3055 "selected": false, 3056 "text": "5", 3057 "value": "5" 3058 }, 3059 { 3060 "selected": false, 3061 "text": "10", 3062 "value": "10" 3063 }, 3064 { 3065 "selected": false, 3066 "text": "60", 3067 "value": "60" 3068 }, 3069 { 3070 "selected": false, 3071 "text": "300", 3072 "value": "300" 3073 } 3074 ], 3075 "query": "1, 3, 5, 10, 60, 300", 3076 "skipUrlSync": false, 3077 "type": "custom" 3078 }, 3079 { 3080 "allValue": "", 3081 "current": {}, 3082 "datasource": "${DS_TEST-CLUSTER}", 3083 "definition": "label_values(process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}, instance)", 3084 "description": null, 3085 "error": null, 3086 "hide": 0, 3087 "includeAll": true, 3088 "label": "Runtime metrics", 3089 "multi": false, 3090 "name": "runtime_instance", 3091 "options": [], 3092 "query": { 3093 "query": "label_values(process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}, instance)", 3094 "refId": "debug-runtime_instance-Variable-Query" 3095 }, 3096 "refresh": 2, 3097 "regex": "", 3098 "skipUrlSync": false, 3099 "sort": 0, 3100 "tagValuesQuery": "", 3101 "tags": [], 3102 "tagsQuery": "", 3103 "type": "query", 3104 "useTags": false 3105 }, 3106 { 3107 "allValue": null, 3108 "current": {}, 3109 "datasource": "${DS_TEST-CLUSTER}", 3110 "definition": "label_values(ticdc_actor_number_of_workers{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}, name)", 3111 "description": null, 3112 "error": null, 3113 "hide": 0, 3114 "includeAll": true, 3115 "label": "Actor", 3116 "multi": true, 3117 "name": "actor_name", 3118 "options": [], 3119 "query": { 3120 "query": "label_values(ticdc_actor_number_of_workers{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}, name)", 3121 "refId": "debug-actor_name-Variable-Query" 3122 }, 3123 "refresh": 2, 3124 "regex": "", 3125 "skipUrlSync": false, 3126 "sort": 0, 3127 "tagValuesQuery": "", 3128 "tags": [], 3129 "tagsQuery": "", 3130 "type": "query", 3131 "useTags": false 3132 } 3133 ] 3134 }, 3135 "time": { 3136 "from": "now-3h", 3137 "to": "now" 3138 }, 3139 "timepicker": { 3140 "refresh_intervals": [ 3141 "5s", 3142 "10s", 3143 "30s", 3144 "1m", 3145 "5m", 3146 "15m", 3147 "30m", 3148 "1h", 3149 "2h", 3150 "1d" 3151 ], 3152 "time_options": [ 3153 "5m", 3154 "15m", 3155 "1h", 3156 "6h", 3157 "12h", 3158 "24h", 3159 "2d", 3160 "7d", 3161 "30d" 3162 ] 3163 }, 3164 "timezone": "browser", 3165 "title": "Test-Cluster-TiCDC-Summary", 3166 "uid": "McUpY954z", 3167 "version": 2 3168 }