github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/metrics/grafana/DM-Monitor-Professional.json (about) 1 { 2 "__inputs": [ 3 { 4 "name": "DS_TEST-CLUSTER", 5 "label": "prometheus-local", 6 "description": "", 7 "type": "datasource", 8 "pluginId": "prometheus", 9 "pluginName": "Prometheus" 10 } 11 ], 12 "__requires": [ 13 { 14 "type": "grafana", 15 "id": "grafana", 16 "name": "Grafana", 17 "version": "7.5.11" 18 }, 19 { 20 "type": "panel", 21 "id": "graph", 22 "name": "Graph", 23 "version": "" 24 }, 25 { 26 "type": "datasource", 27 "id": "prometheus", 28 "name": "Prometheus", 29 "version": "1.0.0" 30 }, 31 { 32 "type": "panel", 33 "id": "timeseries", 34 "name": "Time series", 35 "version": "" 36 } 37 ], 38 "annotations": { 39 "list": [ 40 { 41 "builtIn": 1, 42 "datasource": "${DS_TEST-CLUSTER}", 43 "enable": true, 44 "hide": false, 45 "iconColor": "rgba(0, 211, 255, 1)", 46 "name": "Annotations & Alerts", 47 "type": "dashboard" 48 } 49 ] 50 }, 51 "editable": true, 52 "gnetId": null, 53 "graphTooltip": 0, 54 "id": null, 55 "iteration": 1653965508897, 56 "links": [], 57 "panels": [ 58 { 59 "cacheTimeout": null, 60 "datasource": "${DS_TEST-CLUSTER}", 61 "description": "", 62 "fieldConfig": { 63 "defaults": { 64 "color": { 65 "mode": "palette-classic" 66 }, 67 "custom": { 68 "axisLabel": "", 69 "axisPlacement": "auto", 70 "barAlignment": 0, 71 "drawStyle": "line", 72 "fillOpacity": 8, 73 "gradientMode": "none", 74 "hideFrom": { 75 "graph": false, 76 "legend": false, 77 "tooltip": false 78 }, 79 "lineInterpolation": "stepBefore", 80 "lineStyle": { 81 "fill": "solid" 82 }, 83 "lineWidth": 1, 84 "pointSize": 5, 85 "scaleDistribution": { 86 "type": "linear" 87 }, 88 "showPoints": "auto", 89 "spanNulls": false 90 }, 91 "decimals": 0, 92 "mappings": [ 93 { 94 "from": "", 95 "id": 1, 96 "text": "Invaild", 97 "to": "", 98 "type": 1, 99 "value": "0" 100 }, 101 { 102 "from": "", 103 "id": 2, 104 "text": "New", 105 "to": "", 106 "type": 1, 107 "value": "1" 108 }, 109 { 110 "from": "", 111 "id": 3, 112 "text": "Running", 113 "to": "", 114 "type": 1, 115 "value": "2" 116 }, 117 { 118 "from": "", 119 "id": 4, 120 "text": "Paused", 121 "to": "", 122 "type": 1, 123 "value": "3" 124 }, 125 { 126 "from": "", 127 "id": 5, 128 "text": "Stopped", 129 "to": "", 130 "type": 1, 131 "value": "4" 132 }, 133 { 134 "from": "", 135 "id": 6, 136 "text": "Finished", 137 "to": "", 138 "type": 1, 139 "value": "5" 140 }, 141 { 142 "from": "", 143 "id": 7, 144 "text": "Error", 145 "to": "", 146 "type": 1, 147 "value": "15" 148 } 149 ], 150 "min": 0, 151 "thresholds": { 152 "mode": "absolute", 153 "steps": [ 154 { 155 "color": "green", 156 "value": null 157 }, 158 { 159 "color": "#EAB839", 160 "value": 0 161 }, 162 { 163 "color": "light-green", 164 "value": 1 165 }, 166 { 167 "color": "super-light-green", 168 "value": 2 169 }, 170 { 171 "color": "light-red", 172 "value": 3 173 }, 174 { 175 "color": "semi-dark-red", 176 "value": 4 177 }, 178 { 179 "color": "dark-red", 180 "value": 5 181 } 182 ] 183 }, 184 "unit": "short" 185 }, 186 "overrides": [] 187 }, 188 "gridPos": { 189 "h": 7, 190 "w": 24, 191 "x": 0, 192 "y": 0 193 }, 194 "id": 104, 195 "links": [], 196 "options": { 197 "graph": {}, 198 "legend": { 199 "calcs": [], 200 "displayMode": "list", 201 "placement": "bottom" 202 }, 203 "tooltipOptions": { 204 "mode": "single" 205 } 206 }, 207 "pluginVersion": "7.5.7", 208 "targets": [ 209 { 210 "exemplar": true, 211 "expr": "dm_worker_task_state{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}", 212 "format": "time_series", 213 "interval": "$interval", 214 "intervalFactor": 1, 215 "legendFormat": "{{task}}-{{source_id}}-{{worker}}", 216 "refId": "A" 217 } 218 ], 219 "timeFrom": null, 220 "timeShift": null, 221 "title": "Task State", 222 "type": "timeseries" 223 }, 224 { 225 "collapsed": true, 226 "datasource": "${DS_TEST-CLUSTER}", 227 "gridPos": { 228 "h": 1, 229 "w": 24, 230 "x": 0, 231 "y": 7 232 }, 233 "id": 56, 234 "panels": [ 235 { 236 "aliasColors": {}, 237 "bars": false, 238 "cacheTimeout": null, 239 "dashLength": 10, 240 "dashes": false, 241 "datasource": "${DS_TEST-CLUSTER}", 242 "decimals": 1, 243 "description": "The data import process percentage of dump unit. The value is an estimated value and may exceed less than 100%.", 244 "fieldConfig": { 245 "defaults": {}, 246 "overrides": [] 247 }, 248 "fill": 1, 249 "fillGradient": 0, 250 "gridPos": { 251 "h": 7, 252 "w": 8, 253 "x": 0, 254 "y": 8 255 }, 256 "hiddenSeries": false, 257 "id": 88, 258 "legend": { 259 "alignAsTable": true, 260 "avg": false, 261 "current": true, 262 "max": false, 263 "min": false, 264 "rightSide": true, 265 "show": true, 266 "total": false, 267 "values": true 268 }, 269 "lines": true, 270 "linewidth": 1, 271 "links": [], 272 "nullPointMode": "null", 273 "options": { 274 "alertThreshold": true 275 }, 276 "percentage": false, 277 "pluginVersion": "7.5.7", 278 "pointradius": 2, 279 "points": false, 280 "renderer": "flot", 281 "seriesOverrides": [], 282 "spaceLength": 10, 283 "stack": false, 284 "steppedLine": false, 285 "targets": [ 286 { 287 "exemplar": true, 288 "expr": "dumpling_dump_finished_rows{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"} / dumpling_dump_estimate_total_rows{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}", 289 "format": "time_series", 290 "hide": false, 291 "interval": "$interval", 292 "intervalFactor": 2, 293 "legendFormat": "{{task}} - {{source_id}}", 294 "refId": "A" 295 } 296 ], 297 "thresholds": [], 298 "timeFrom": null, 299 "timeRegions": [], 300 "timeShift": null, 301 "title": "dump progress", 302 "tooltip": { 303 "shared": true, 304 "sort": 0, 305 "value_type": "individual" 306 }, 307 "type": "graph", 308 "xaxis": { 309 "buckets": null, 310 "mode": "time", 311 "name": null, 312 "show": true, 313 "values": [] 314 }, 315 "yaxes": [ 316 { 317 "decimals": 1, 318 "format": "percentunit", 319 "label": null, 320 "logBase": 1, 321 "max": null, 322 "min": "0", 323 "show": true 324 }, 325 { 326 "format": "short", 327 "label": null, 328 "logBase": 1, 329 "max": null, 330 "min": null, 331 "show": true 332 } 333 ], 334 "yaxis": { 335 "align": false, 336 "alignLevel": null 337 } 338 }, 339 { 340 "aliasColors": {}, 341 "bars": false, 342 "cacheTimeout": null, 343 "dashLength": 10, 344 "dashes": false, 345 "datasource": "${DS_TEST-CLUSTER}", 346 "description": "The data import process percentage of load unit. The value range is 0% ~ 100%", 347 "fieldConfig": { 348 "defaults": {}, 349 "overrides": [] 350 }, 351 "fill": 1, 352 "fillGradient": 0, 353 "gridPos": { 354 "h": 7, 355 "w": 8, 356 "x": 8, 357 "y": 8 358 }, 359 "hiddenSeries": false, 360 "id": 87, 361 "legend": { 362 "alignAsTable": true, 363 "avg": false, 364 "current": true, 365 "max": false, 366 "min": false, 367 "rightSide": true, 368 "show": true, 369 "total": false, 370 "values": true 371 }, 372 "lines": true, 373 "linewidth": 1, 374 "links": [], 375 "nullPointMode": "null", 376 "options": { 377 "alertThreshold": true 378 }, 379 "percentage": false, 380 "pluginVersion": "7.5.7", 381 "pointradius": 2, 382 "points": false, 383 "renderer": "flot", 384 "seriesOverrides": [], 385 "spaceLength": 10, 386 "stack": false, 387 "steppedLine": false, 388 "targets": [ 389 { 390 "exemplar": true, 391 "expr": "lightning_chunks{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\",state=\"finished\"} / ignoring(state) lightning_chunks{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\",state=\"estimated\"}", 392 "format": "time_series", 393 "interval": "$interval", 394 "intervalFactor": 2, 395 "legendFormat": "{{task}} - {{source_id}}", 396 "refId": "A" 397 } 398 ], 399 "thresholds": [], 400 "timeFrom": null, 401 "timeRegions": [], 402 "timeShift": null, 403 "title": "load progress", 404 "tooltip": { 405 "shared": true, 406 "sort": 0, 407 "value_type": "individual" 408 }, 409 "type": "graph", 410 "xaxis": { 411 "buckets": null, 412 "mode": "time", 413 "name": null, 414 "show": true, 415 "values": [] 416 }, 417 "yaxes": [ 418 { 419 "decimals": 1, 420 "format": "percentunit", 421 "label": null, 422 "logBase": 1, 423 "max": null, 424 "min": "0", 425 "show": true 426 }, 427 { 428 "format": "short", 429 "label": null, 430 "logBase": 1, 431 "max": null, 432 "min": null, 433 "show": true 434 } 435 ], 436 "yaxis": { 437 "align": false, 438 "alignLevel": null 439 } 440 }, 441 { 442 "aliasColors": {}, 443 "bars": false, 444 "cacheTimeout": null, 445 "dashLength": 10, 446 "dashes": false, 447 "datasource": "${DS_TEST-CLUSTER}", 448 "description": "The checksum process percentage of load unit. The value range is 0% ~ 100%", 449 "fieldConfig": { 450 "defaults": {}, 451 "overrides": [] 452 }, 453 "fill": 1, 454 "fillGradient": 0, 455 "gridPos": { 456 "h": 7, 457 "w": 8, 458 "x": 16, 459 "y": 8 460 }, 461 "hiddenSeries": false, 462 "id": 95, 463 "interval": null, 464 "legend": { 465 "alignAsTable": true, 466 "avg": false, 467 "current": true, 468 "max": true, 469 "min": true, 470 "rightSide": false, 471 "show": true, 472 "total": false, 473 "values": true 474 }, 475 "lines": true, 476 "linewidth": 1, 477 "links": [], 478 "maxDataPoints": 100, 479 "nullPointMode": "null", 480 "options": { 481 "alertThreshold": true 482 }, 483 "percentage": false, 484 "pluginVersion": "7.5.7", 485 "pointradius": 2, 486 "points": false, 487 "renderer": "flot", 488 "seriesOverrides": [], 489 "spaceLength": 10, 490 "stack": false, 491 "steppedLine": false, 492 "targets": [ 493 { 494 "exemplar": true, 495 "expr": "lightning_tables{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\",state=\"completed\"} / ignoring(state) lightning_tables{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\",state=\"pending\"}", 496 "format": "time_series", 497 "interval": "", 498 "intervalFactor": 2, 499 "legendFormat": "{{task}} - {{source_id}}", 500 "refId": "A" 501 } 502 ], 503 "thresholds": [], 504 "timeFrom": null, 505 "timeRegions": [], 506 "timeShift": null, 507 "title": "checksum progress", 508 "tooltip": { 509 "shared": true, 510 "sort": 0, 511 "value_type": "individual" 512 }, 513 "type": "graph", 514 "xaxis": { 515 "buckets": null, 516 "mode": "time", 517 "name": null, 518 "show": true, 519 "values": [] 520 }, 521 "yaxes": [ 522 { 523 "decimals": 1, 524 "format": "percentunit", 525 "label": null, 526 "logBase": 1, 527 "max": "1", 528 "min": "0", 529 "show": true 530 }, 531 { 532 "format": "short", 533 "label": null, 534 "logBase": 1, 535 "max": null, 536 "min": null, 537 "show": true 538 } 539 ], 540 "yaxis": { 541 "align": false, 542 "alignLevel": null 543 } 544 }, 545 { 546 "aliasColors": {}, 547 "bars": false, 548 "cacheTimeout": null, 549 "dashLength": 10, 550 "dashes": false, 551 "datasource": "${DS_TEST-CLUSTER}", 552 "description": "Dumper encounters an error within the DM-worker and exits", 553 "fieldConfig": { 554 "defaults": {}, 555 "overrides": [] 556 }, 557 "fill": 1, 558 "fillGradient": 0, 559 "gridPos": { 560 "h": 7, 561 "w": 8, 562 "x": 0, 563 "y": 15 564 }, 565 "hiddenSeries": false, 566 "id": 32, 567 "legend": { 568 "alignAsTable": true, 569 "avg": false, 570 "current": true, 571 "max": true, 572 "min": true, 573 "show": true, 574 "total": false, 575 "values": true 576 }, 577 "lines": true, 578 "linewidth": 1, 579 "links": [], 580 "nullPointMode": "null", 581 "options": { 582 "alertThreshold": true 583 }, 584 "percentage": false, 585 "pluginVersion": "7.5.7", 586 "pointradius": 2, 587 "points": false, 588 "renderer": "flot", 589 "seriesOverrides": [], 590 "spaceLength": 10, 591 "stack": false, 592 "steppedLine": false, 593 "targets": [ 594 { 595 "expr": "changes(dm_dumpling_exit_with_error_count{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}[30m])", 596 "format": "time_series", 597 "intervalFactor": 2, 598 "legendFormat": "{{task}} - {{source_id}}", 599 "refId": "A" 600 } 601 ], 602 "thresholds": [], 603 "timeFrom": null, 604 "timeRegions": [], 605 "timeShift": null, 606 "title": "dump process exits with error", 607 "tooltip": { 608 "shared": true, 609 "sort": 0, 610 "value_type": "individual" 611 }, 612 "type": "graph", 613 "xaxis": { 614 "buckets": null, 615 "mode": "time", 616 "name": null, 617 "show": true, 618 "values": [] 619 }, 620 "yaxes": [ 621 { 622 "decimals": 0, 623 "format": "short", 624 "label": null, 625 "logBase": 1, 626 "max": null, 627 "min": "0", 628 "show": true 629 }, 630 { 631 "format": "short", 632 "label": null, 633 "logBase": 1, 634 "max": null, 635 "min": null, 636 "show": true 637 } 638 ], 639 "yaxis": { 640 "align": false, 641 "alignLevel": null 642 } 643 }, 644 { 645 "aliasColors": {}, 646 "bars": false, 647 "cacheTimeout": null, 648 "dashLength": 10, 649 "dashes": false, 650 "datasource": "${DS_TEST-CLUSTER}", 651 "description": "load unit encounters an error within the DM-worker and exits", 652 "fieldConfig": { 653 "defaults": {}, 654 "overrides": [] 655 }, 656 "fill": 1, 657 "fillGradient": 0, 658 "gridPos": { 659 "h": 7, 660 "w": 8, 661 "x": 8, 662 "y": 15 663 }, 664 "hiddenSeries": false, 665 "id": 33, 666 "legend": { 667 "alignAsTable": true, 668 "avg": false, 669 "current": true, 670 "max": true, 671 "min": true, 672 "show": true, 673 "total": false, 674 "values": true 675 }, 676 "lines": true, 677 "linewidth": 1, 678 "links": [], 679 "nullPointMode": "null", 680 "options": { 681 "alertThreshold": true 682 }, 683 "percentage": false, 684 "pluginVersion": "7.5.7", 685 "pointradius": 2, 686 "points": false, 687 "renderer": "flot", 688 "seriesOverrides": [], 689 "spaceLength": 10, 690 "stack": false, 691 "steppedLine": false, 692 "targets": [ 693 { 694 "expr": "changes(dm_loader_exit_with_error_count{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}[30m])", 695 "format": "time_series", 696 "intervalFactor": 2, 697 "legendFormat": "{{task}} - {{source_id}}", 698 "refId": "A" 699 } 700 ], 701 "thresholds": [], 702 "timeFrom": null, 703 "timeRegions": [], 704 "timeShift": null, 705 "title": "load process exits with error", 706 "tooltip": { 707 "shared": true, 708 "sort": 0, 709 "value_type": "individual" 710 }, 711 "type": "graph", 712 "xaxis": { 713 "buckets": null, 714 "mode": "time", 715 "name": null, 716 "show": true, 717 "values": [] 718 }, 719 "yaxes": [ 720 { 721 "decimals": 0, 722 "format": "short", 723 "label": "", 724 "logBase": 1, 725 "max": null, 726 "min": "0", 727 "show": true 728 }, 729 { 730 "format": "short", 731 "label": null, 732 "logBase": 1, 733 "max": null, 734 "min": null, 735 "show": true 736 } 737 ], 738 "yaxis": { 739 "align": false, 740 "alignLevel": null 741 } 742 }, 743 { 744 "aliasColors": {}, 745 "bars": false, 746 "dashLength": 10, 747 "dashes": false, 748 "datasource": "${DS_TEST-CLUSTER}", 749 "decimals": 1, 750 "description": "idle load unit workers", 751 "fieldConfig": { 752 "defaults": {}, 753 "overrides": [] 754 }, 755 "fill": 1, 756 "fillGradient": 0, 757 "gridPos": { 758 "h": 7, 759 "w": 12, 760 "x": 0, 761 "y": 22 762 }, 763 "hiddenSeries": false, 764 "id": 18, 765 "legend": { 766 "alignAsTable": true, 767 "avg": false, 768 "current": true, 769 "max": true, 770 "min": false, 771 "rightSide": true, 772 "show": true, 773 "sort": "current", 774 "sortDesc": true, 775 "total": false, 776 "values": true 777 }, 778 "lines": true, 779 "linewidth": 1, 780 "links": [], 781 "nullPointMode": "null", 782 "options": { 783 "alertThreshold": true 784 }, 785 "percentage": false, 786 "pluginVersion": "7.5.7", 787 "pointradius": 5, 788 "points": false, 789 "renderer": "flot", 790 "seriesOverrides": [], 791 "spaceLength": 10, 792 "stack": false, 793 "steppedLine": false, 794 "targets": [ 795 { 796 "expr": "lightning_idle_workers{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}", 797 "format": "time_series", 798 "intervalFactor": 2, 799 "legendFormat": "{{name}} - {{task}} - {{source_id}}", 800 "refId": "A" 801 } 802 ], 803 "thresholds": [], 804 "timeFrom": null, 805 "timeRegions": [], 806 "timeShift": null, 807 "title": "Idle workers", 808 "tooltip": { 809 "shared": true, 810 "sort": 0, 811 "value_type": "individual" 812 }, 813 "type": "graph", 814 "xaxis": { 815 "buckets": null, 816 "mode": "time", 817 "name": null, 818 "show": true, 819 "values": [] 820 }, 821 "yaxes": [ 822 { 823 "decimals": 0, 824 "format": "none", 825 "label": null, 826 "logBase": 1, 827 "max": null, 828 "min": "0", 829 "show": true 830 }, 831 { 832 "format": "short", 833 "label": null, 834 "logBase": 1, 835 "max": null, 836 "min": null, 837 "show": true 838 } 839 ] 840 }, 841 { 842 "aliasColors": {}, 843 "bars": false, 844 "dashLength": 10, 845 "dashes": false, 846 "datasource": "${DS_TEST-CLUSTER}", 847 "description": "The duration that load unit process a chunk (in seconds)", 848 "fieldConfig": { 849 "defaults": {}, 850 "overrides": [] 851 }, 852 "fill": 1, 853 "fillGradient": 0, 854 "gridPos": { 855 "h": 7, 856 "w": 12, 857 "x": 12, 858 "y": 22 859 }, 860 "hiddenSeries": false, 861 "id": 96, 862 "legend": { 863 "alignAsTable": true, 864 "avg": false, 865 "current": false, 866 "max": false, 867 "min": false, 868 "rightSide": true, 869 "show": true, 870 "total": false, 871 "values": false 872 }, 873 "lines": true, 874 "linewidth": 1, 875 "links": [], 876 "nullPointMode": "null", 877 "options": { 878 "alertThreshold": true 879 }, 880 "percentage": false, 881 "pluginVersion": "7.5.7", 882 "pointradius": 5, 883 "points": false, 884 "renderer": "flot", 885 "seriesOverrides": [], 886 "spaceLength": 10, 887 "stack": false, 888 "steppedLine": false, 889 "targets": [ 890 { 891 "exemplar": true, 892 "expr": "1/rate(lightning_chunks{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",state=\"finished\"}[1m])", 893 "format": "time_series", 894 "interval": "", 895 "intervalFactor": 2, 896 "legendFormat": "{{task}} - {{source_id}}", 897 "refId": "A" 898 } 899 ], 900 "thresholds": [], 901 "timeFrom": null, 902 "timeRegions": [], 903 "timeShift": null, 904 "title": "Chunk process duration", 905 "tooltip": { 906 "shared": true, 907 "sort": 0, 908 "value_type": "individual" 909 }, 910 "type": "graph", 911 "xaxis": { 912 "buckets": null, 913 "mode": "time", 914 "name": null, 915 "show": true, 916 "values": [] 917 }, 918 "yaxes": [ 919 { 920 "decimals": 2, 921 "format": "s", 922 "label": null, 923 "logBase": 1, 924 "max": null, 925 "min": 0, 926 "show": true 927 }, 928 { 929 "format": "short", 930 "label": null, 931 "logBase": 1, 932 "max": null, 933 "min": null, 934 "show": false 935 } 936 ], 937 "yaxis": { 938 "align": false, 939 "alignLevel": null 940 } 941 }, 942 { 943 "aliasColors": {}, 944 "bars": false, 945 "cacheTimeout": null, 946 "dashLength": 10, 947 "dashes": false, 948 "datasource": "${DS_TEST-CLUSTER}", 949 "description": "The number of external resources used by load unit", 950 "fieldConfig": { 951 "defaults": {}, 952 "overrides": [] 953 }, 954 "fill": 1, 955 "fillGradient": 0, 956 "gridPos": { 957 "h": 7, 958 "w": 12, 959 "x": 0, 960 "y": 29 961 }, 962 "hiddenSeries": false, 963 "id": 89, 964 "legend": { 965 "avg": false, 966 "current": false, 967 "max": false, 968 "min": false, 969 "show": true, 970 "total": false, 971 "values": false 972 }, 973 "lines": true, 974 "linewidth": 1, 975 "links": [], 976 "nullPointMode": "null", 977 "options": { 978 "alertThreshold": true 979 }, 980 "percentage": false, 981 "pluginVersion": "7.5.7", 982 "pointradius": 2, 983 "points": false, 984 "renderer": "flot", 985 "seriesOverrides": [], 986 "spaceLength": 10, 987 "stack": false, 988 "steppedLine": false, 989 "targets": [ 990 { 991 "expr": "lightning_kv_encoder{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\", type=\"open\"} - ignoring(type) lightning_kv_encoder{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\", type=\"closed\"}", 992 "format": "time_series", 993 "intervalFactor": 2, 994 "legendFormat": "KV Encoder", 995 "refId": "A" 996 }, 997 { 998 "expr": "lightning_importer_engine{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\", type=\"open\"} - ignoring(type) lightning_importer_engine{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\", type=\"closed\"}", 999 "format": "time_series", 1000 "intervalFactor": 2, 1001 "legendFormat": "Importer Engines (via Lightning)", 1002 "refId": "B" 1003 } 1004 ], 1005 "thresholds": [], 1006 "timeFrom": null, 1007 "timeRegions": [], 1008 "timeShift": null, 1009 "title": "External resources", 1010 "tooltip": { 1011 "shared": true, 1012 "sort": 0, 1013 "value_type": "individual" 1014 }, 1015 "type": "graph", 1016 "xaxis": { 1017 "buckets": null, 1018 "mode": "time", 1019 "name": null, 1020 "show": true, 1021 "values": [] 1022 }, 1023 "yaxes": [ 1024 { 1025 "decimals": 0, 1026 "format": "none", 1027 "label": null, 1028 "logBase": 1, 1029 "max": null, 1030 "min": "0", 1031 "show": true 1032 }, 1033 { 1034 "format": "short", 1035 "label": null, 1036 "logBase": 1, 1037 "max": null, 1038 "min": null, 1039 "show": true 1040 } 1041 ] 1042 }, 1043 { 1044 "aliasColors": {}, 1045 "bars": false, 1046 "cacheTimeout": null, 1047 "dashLength": 10, 1048 "dashes": false, 1049 "datasource": "${DS_TEST-CLUSTER}", 1050 "description": "The statistic of bytes in different status of data files processed by load unit", 1051 "fieldConfig": { 1052 "defaults": {}, 1053 "overrides": [] 1054 }, 1055 "fill": 1, 1056 "fillGradient": 0, 1057 "gridPos": { 1058 "h": 7, 1059 "w": 8, 1060 "x": 16, 1061 "y": 15 1062 }, 1063 "hiddenSeries": false, 1064 "id": 17, 1065 "legend": { 1066 "avg": false, 1067 "current": false, 1068 "max": false, 1069 "min": false, 1070 "show": true, 1071 "total": false, 1072 "values": false 1073 }, 1074 "lines": true, 1075 "linewidth": 1, 1076 "links": [], 1077 "nullPointMode": "null", 1078 "options": { 1079 "alertThreshold": true 1080 }, 1081 "percentage": false, 1082 "pluginVersion": "7.5.7", 1083 "pointradius": 2, 1084 "points": false, 1085 "renderer": "flot", 1086 "seriesOverrides": [], 1087 "spaceLength": 10, 1088 "stack": false, 1089 "steppedLine": false, 1090 "targets": [ 1091 { 1092 "expr": "lightning_row_read_bytes_sum{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}", 1093 "format": "time_series", 1094 "intervalFactor": 2, 1095 "legendFormat": "parser read size - {{task}} - {{source_id}}", 1096 "refId": "A" 1097 }, 1098 { 1099 "expr": "lightning_block_deliver_bytes_sum{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}", 1100 "format": "time_series", 1101 "intervalFactor": 2, 1102 "legendFormat": "{{kind}} deliver size - {{task}} - {{source_id}}", 1103 "refId": "B" 1104 } 1105 ], 1106 "thresholds": [], 1107 "timeFrom": null, 1108 "timeRegions": [], 1109 "timeShift": null, 1110 "title": "Total bytes for load unit", 1111 "tooltip": { 1112 "shared": true, 1113 "sort": 0, 1114 "value_type": "individual" 1115 }, 1116 "type": "graph", 1117 "xaxis": { 1118 "buckets": null, 1119 "mode": "time", 1120 "name": null, 1121 "show": true, 1122 "values": [] 1123 }, 1124 "yaxes": [ 1125 { 1126 "format": "decbytes", 1127 "label": null, 1128 "logBase": 1, 1129 "max": null, 1130 "min": null, 1131 "show": true 1132 }, 1133 { 1134 "format": "short", 1135 "label": null, 1136 "logBase": 1, 1137 "max": null, 1138 "min": null, 1139 "show": true 1140 } 1141 ] 1142 }, 1143 { 1144 "aliasColors": {}, 1145 "bars": false, 1146 "cacheTimeout": null, 1147 "dashLength": 10, 1148 "dashes": false, 1149 "datasource": "${DS_TEST-CLUSTER}", 1150 "description": "The speed of load unit processing source files internally", 1151 "fieldConfig": { 1152 "defaults": {}, 1153 "overrides": [] 1154 }, 1155 "fill": 1, 1156 "fillGradient": 0, 1157 "gridPos": { 1158 "h": 7, 1159 "w": 12, 1160 "x": 12, 1161 "y": 29 1162 }, 1163 "hiddenSeries": false, 1164 "id": 16, 1165 "legend": { 1166 "avg": false, 1167 "current": false, 1168 "max": false, 1169 "min": false, 1170 "show": true, 1171 "total": false, 1172 "values": false 1173 }, 1174 "lines": true, 1175 "linewidth": 1, 1176 "links": [], 1177 "nullPointMode": "null", 1178 "options": { 1179 "alertThreshold": true 1180 }, 1181 "percentage": false, 1182 "pluginVersion": "7.5.7", 1183 "pointradius": 2, 1184 "points": false, 1185 "renderer": "flot", 1186 "seriesOverrides": [], 1187 "spaceLength": 10, 1188 "stack": false, 1189 "steppedLine": false, 1190 "targets": [ 1191 { 1192 "expr": "rate(lightning_block_deliver_bytes_sum{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}[30s])", 1193 "format": "time_series", 1194 "intervalFactor": 2, 1195 "legendFormat": "{{kind}} deliver rate {{task}} - {{source_id}}", 1196 "refId": "B" 1197 }, 1198 { 1199 "exemplar": true, 1200 "expr": "sum(rate(lightning_block_deliver_bytes_sum{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}[30s]))", 1201 "format": "time_series", 1202 "intervalFactor": 2, 1203 "legendFormat": "total deliver rate {{task}} - {{source_id}}", 1204 "refId": "A" 1205 } 1206 ], 1207 "title": "SQL process rate", 1208 "type": "graph", 1209 "xaxis": { 1210 "buckets": null, 1211 "mode": "time", 1212 "name": null, 1213 "show": true, 1214 "values": [] 1215 }, 1216 "yaxes": [ 1217 { 1218 "format": "Bps", 1219 "label": null, 1220 "logBase": 1, 1221 "max": null, 1222 "min": null, 1223 "show": true 1224 }, 1225 { 1226 "format": "short", 1227 "label": null, 1228 "logBase": 1, 1229 "max": null, 1230 "min": null, 1231 "show": true 1232 } 1233 ] 1234 } 1235 ], 1236 "repeat": null, 1237 "title": "dump and load ", 1238 "type": "row" 1239 }, 1240 { 1241 "collapsed": true, 1242 "datasource": "${DS_TEST-CLUSTER}", 1243 "gridPos": { 1244 "h": 1, 1245 "w": 24, 1246 "x": 0, 1247 "y": 8 1248 }, 1249 "id": 57, 1250 "panels": [ 1251 { 1252 "aliasColors": { 1253 "admin queue-flush": "blue" 1254 }, 1255 "bars": false, 1256 "dashLength": 10, 1257 "dashes": false, 1258 "datasource": "${DS_TEST-CLUSTER}", 1259 "decimals": 1, 1260 "description": "number of sql executed per second(one minute average value, dm_syncer_finished_jobs_total )", 1261 "fieldConfig": { 1262 "defaults": {}, 1263 "overrides": [] 1264 }, 1265 "fill": 1, 1266 "fillGradient": 0, 1267 "gridPos": { 1268 "h": 12, 1269 "w": 15, 1270 "x": 0, 1271 "y": 10 1272 }, 1273 "hiddenSeries": false, 1274 "id": 102, 1275 "legend": { 1276 "alignAsTable": true, 1277 "avg": false, 1278 "current": true, 1279 "max": true, 1280 "min": true, 1281 "rightSide": true, 1282 "show": true, 1283 "sort": "current", 1284 "sortDesc": true, 1285 "total": false, 1286 "values": true 1287 }, 1288 "lines": true, 1289 "linewidth": 1, 1290 "links": [], 1291 "nullPointMode": "null", 1292 "options": { 1293 "alertThreshold": true 1294 }, 1295 "percentage": false, 1296 "pluginVersion": "7.5.11", 1297 "pointradius": 5, 1298 "points": false, 1299 "renderer": "flot", 1300 "seriesOverrides": [], 1301 "spaceLength": 10, 1302 "stack": false, 1303 "steppedLine": false, 1304 "targets": [ 1305 { 1306 "exemplar": true, 1307 "expr": "sum(rate(dm_syncer_finished_jobs_total{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",target_schema=~\"$target_schema\",type=~\"$type\"}[$interval]))by (task,source_id,target_schema,type)", 1308 "format": "time_series", 1309 "interval": "$interval", 1310 "intervalFactor": 2, 1311 "legendFormat": "{{task}} - {{target_schema}} - {{type}} ", 1312 "refId": "A" 1313 } 1314 ], 1315 "thresholds": [], 1316 "timeFrom": null, 1317 "timeRegions": [], 1318 "timeShift": null, 1319 "title": "Replication QPS", 1320 "tooltip": { 1321 "shared": true, 1322 "sort": 0, 1323 "value_type": "individual" 1324 }, 1325 "type": "graph", 1326 "xaxis": { 1327 "buckets": null, 1328 "mode": "time", 1329 "name": null, 1330 "show": true, 1331 "values": [] 1332 }, 1333 "yaxes": [ 1334 { 1335 "$$hashKey": "object:805", 1336 "decimals": 0, 1337 "format": "short", 1338 "label": "", 1339 "logBase": 1, 1340 "max": null, 1341 "min": "0", 1342 "show": true 1343 }, 1344 { 1345 "$$hashKey": "object:806", 1346 "format": "short", 1347 "label": null, 1348 "logBase": 1, 1349 "max": null, 1350 "min": null, 1351 "show": false 1352 } 1353 ], 1354 "yaxis": { 1355 "align": false, 1356 "alignLevel": null 1357 } 1358 }, 1359 { 1360 "aliasColors": {}, 1361 "bars": false, 1362 "cacheTimeout": null, 1363 "dashLength": 10, 1364 "dashes": false, 1365 "datasource": "${DS_TEST-CLUSTER}", 1366 "decimals": 1, 1367 "description": "The predicted remaining time it takes Syncer to be completely synchronized with the master (in seconds)", 1368 "fieldConfig": { 1369 "defaults": {}, 1370 "overrides": [] 1371 }, 1372 "fill": 1, 1373 "fillGradient": 0, 1374 "gridPos": { 1375 "h": 6, 1376 "w": 9, 1377 "x": 15, 1378 "y": 10 1379 }, 1380 "hiddenSeries": false, 1381 "id": 37, 1382 "legend": { 1383 "alignAsTable": true, 1384 "avg": false, 1385 "current": true, 1386 "max": true, 1387 "min": true, 1388 "rightSide": false, 1389 "show": true, 1390 "total": false, 1391 "values": true 1392 }, 1393 "lines": true, 1394 "linewidth": 1, 1395 "links": [], 1396 "nullPointMode": "null", 1397 "options": { 1398 "alertThreshold": true 1399 }, 1400 "percentage": false, 1401 "pluginVersion": "7.5.11", 1402 "pointradius": 2, 1403 "points": false, 1404 "renderer": "flot", 1405 "seriesOverrides": [], 1406 "spaceLength": 10, 1407 "stack": false, 1408 "steppedLine": false, 1409 "targets": [ 1410 { 1411 "expr": "dm_syncer_remaining_time{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}", 1412 "format": "time_series", 1413 "interval": "$interval", 1414 "intervalFactor": 2, 1415 "legendFormat": "{{task}} - {{source_id}}", 1416 "refId": "A" 1417 } 1418 ], 1419 "thresholds": [], 1420 "timeFrom": null, 1421 "timeRegions": [], 1422 "timeShift": null, 1423 "title": "Remaining time to sync", 1424 "tooltip": { 1425 "shared": true, 1426 "sort": 0, 1427 "value_type": "individual" 1428 }, 1429 "type": "graph", 1430 "xaxis": { 1431 "buckets": null, 1432 "mode": "time", 1433 "name": null, 1434 "show": true, 1435 "values": [] 1436 }, 1437 "yaxes": [ 1438 { 1439 "decimals": 1, 1440 "format": "s", 1441 "label": null, 1442 "logBase": 1, 1443 "max": null, 1444 "min": "0", 1445 "show": true 1446 }, 1447 { 1448 "decimals": null, 1449 "format": "short", 1450 "label": null, 1451 "logBase": 1, 1452 "max": null, 1453 "min": null, 1454 "show": true 1455 } 1456 ], 1457 "yaxis": { 1458 "align": false, 1459 "alignLevel": null 1460 } 1461 }, 1462 { 1463 "aliasColors": {}, 1464 "bars": false, 1465 "cacheTimeout": null, 1466 "dashLength": 10, 1467 "dashes": false, 1468 "datasource": "${DS_TEST-CLUSTER}", 1469 "decimals": 1, 1470 "description": "The latency time it takes to replicate the binlog from master to Syncer (in seconds)", 1471 "fieldConfig": { 1472 "defaults": {}, 1473 "overrides": [] 1474 }, 1475 "fill": 1, 1476 "fillGradient": 0, 1477 "gridPos": { 1478 "h": 6, 1479 "w": 9, 1480 "x": 15, 1481 "y": 16 1482 }, 1483 "hiddenSeries": false, 1484 "id": 38, 1485 "interval": null, 1486 "legend": { 1487 "alignAsTable": true, 1488 "avg": false, 1489 "current": true, 1490 "max": true, 1491 "min": true, 1492 "rightSide": false, 1493 "show": true, 1494 "total": false, 1495 "values": true 1496 }, 1497 "lines": true, 1498 "linewidth": 1, 1499 "links": [], 1500 "maxDataPoints": 100, 1501 "nullPointMode": "null", 1502 "options": { 1503 "alertThreshold": true 1504 }, 1505 "percentage": false, 1506 "pluginVersion": "7.5.11", 1507 "pointradius": 2, 1508 "points": false, 1509 "renderer": "flot", 1510 "seriesOverrides": [], 1511 "spaceLength": 10, 1512 "stack": false, 1513 "steppedLine": false, 1514 "targets": [ 1515 { 1516 "exemplar": true, 1517 "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_replication_lag_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,source_id,task))", 1518 "format": "time_series", 1519 "hide": false, 1520 "interval": "$interval", 1521 "intervalFactor": 2, 1522 "legendFormat": "{{task}} - {{source_id}} - 90", 1523 "refId": "A" 1524 }, 1525 { 1526 "exemplar": true, 1527 "expr": "dm_syncer_replication_lag_gauge{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}", 1528 "format": "time_series", 1529 "hide": false, 1530 "interval": "$interval", 1531 "intervalFactor": 2, 1532 "legendFormat": "{{task}} - {{source_id}} - gauge", 1533 "refId": "D" 1534 } 1535 ], 1536 "thresholds": [], 1537 "timeFrom": null, 1538 "timeRegions": [], 1539 "timeShift": null, 1540 "title": "Replication lag", 1541 "tooltip": { 1542 "shared": true, 1543 "sort": 0, 1544 "value_type": "individual" 1545 }, 1546 "type": "graph", 1547 "xaxis": { 1548 "buckets": null, 1549 "mode": "time", 1550 "name": null, 1551 "show": true, 1552 "values": [] 1553 }, 1554 "yaxes": [ 1555 { 1556 "$$hashKey": "object:669", 1557 "decimals": 1, 1558 "format": "s", 1559 "label": null, 1560 "logBase": 1, 1561 "max": null, 1562 "min": "0", 1563 "show": true 1564 }, 1565 { 1566 "$$hashKey": "object:670", 1567 "decimals": null, 1568 "format": "short", 1569 "label": null, 1570 "logBase": 1, 1571 "max": null, 1572 "min": null, 1573 "show": true 1574 } 1575 ], 1576 "yaxis": { 1577 "align": false, 1578 "alignLevel": null 1579 } 1580 }, 1581 { 1582 "aliasColors": { 1583 "admin queue-flush": "blue" 1584 }, 1585 "bars": false, 1586 "dashLength": 10, 1587 "dashes": false, 1588 "datasource": "${DS_TEST-CLUSTER}", 1589 "decimals": 1, 1590 "description": "number of sql executed per second(one minute average value, dm_syncer_finished_jobs_total )\n\nThe maximum TPS estimated according to the recent SQL time-consuming, may vary greatly with different SQL", 1591 "fieldConfig": { 1592 "defaults": {}, 1593 "overrides": [] 1594 }, 1595 "fill": 1, 1596 "fillGradient": 0, 1597 "gridPos": { 1598 "h": 8, 1599 "w": 15, 1600 "x": 0, 1601 "y": 22 1602 }, 1603 "hiddenSeries": false, 1604 "id": 110, 1605 "legend": { 1606 "alignAsTable": true, 1607 "avg": false, 1608 "current": true, 1609 "max": true, 1610 "min": true, 1611 "rightSide": true, 1612 "show": true, 1613 "sort": "current", 1614 "sortDesc": true, 1615 "total": false, 1616 "values": true 1617 }, 1618 "lines": true, 1619 "linewidth": 1, 1620 "links": [], 1621 "nullPointMode": "null", 1622 "options": { 1623 "alertThreshold": true 1624 }, 1625 "percentage": false, 1626 "pluginVersion": "7.5.11", 1627 "pointradius": 5, 1628 "points": false, 1629 "renderer": "flot", 1630 "seriesOverrides": [], 1631 "spaceLength": 10, 1632 "stack": false, 1633 "steppedLine": false, 1634 "targets": [ 1635 { 1636 "exemplar": true, 1637 "expr": "sum(rate(dm_syncer_finished_jobs_total{job_id=~\"$job_id\",task=~\"$task\"}[$interval])) by (task)", 1638 "format": "time_series", 1639 "interval": "$interval", 1640 "intervalFactor": 2, 1641 "legendFormat": "{{task}} ", 1642 "refId": "A" 1643 }, 1644 { 1645 "exemplar": true, 1646 "expr": "sum(dm_syncer_ideal_qps{job_id=~\"$job_id\",task=~\"$task\"}) by (task)", 1647 "hide": false, 1648 "interval": "$interval", 1649 "intervalFactor": 2, 1650 "legendFormat": "{{task}} - ideal", 1651 "refId": "B" 1652 } 1653 ], 1654 "thresholds": [], 1655 "timeFrom": null, 1656 "timeRegions": [], 1657 "timeShift": null, 1658 "title": "Replication QPS (Sum by task)", 1659 "tooltip": { 1660 "shared": true, 1661 "sort": 0, 1662 "value_type": "individual" 1663 }, 1664 "type": "graph", 1665 "xaxis": { 1666 "buckets": null, 1667 "mode": "time", 1668 "name": null, 1669 "show": true, 1670 "values": [] 1671 }, 1672 "yaxes": [ 1673 { 1674 "$$hashKey": "object:805", 1675 "decimals": 0, 1676 "format": "short", 1677 "label": "", 1678 "logBase": 1, 1679 "max": null, 1680 "min": "0", 1681 "show": true 1682 }, 1683 { 1684 "$$hashKey": "object:806", 1685 "format": "short", 1686 "label": null, 1687 "logBase": 1, 1688 "max": null, 1689 "min": null, 1690 "show": false 1691 } 1692 ], 1693 "yaxis": { 1694 "align": false, 1695 "alignLevel": null 1696 } 1697 }, 1698 { 1699 "aliasColors": {}, 1700 "bars": false, 1701 "cacheTimeout": null, 1702 "dashLength": 10, 1703 "dashes": false, 1704 "datasource": "${DS_TEST-CLUSTER}", 1705 "description": "The binlog replication unit process encounters an error within the DM-worker and exits", 1706 "fieldConfig": { 1707 "defaults": {}, 1708 "overrides": [] 1709 }, 1710 "fill": 1, 1711 "fillGradient": 0, 1712 "gridPos": { 1713 "h": 8, 1714 "w": 9, 1715 "x": 15, 1716 "y": 22 1717 }, 1718 "hiddenSeries": false, 1719 "id": 39, 1720 "legend": { 1721 "alignAsTable": true, 1722 "avg": false, 1723 "current": true, 1724 "max": true, 1725 "min": true, 1726 "show": true, 1727 "total": false, 1728 "values": true 1729 }, 1730 "lines": true, 1731 "linewidth": 1, 1732 "links": [], 1733 "nullPointMode": "null", 1734 "options": { 1735 "alertThreshold": true 1736 }, 1737 "percentage": false, 1738 "pluginVersion": "7.5.11", 1739 "pointradius": 2, 1740 "points": false, 1741 "renderer": "flot", 1742 "seriesOverrides": [], 1743 "spaceLength": 10, 1744 "stack": false, 1745 "steppedLine": false, 1746 "targets": [ 1747 { 1748 "expr": "changes(dm_syncer_exit_with_error_count{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[30m])", 1749 "format": "time_series", 1750 "intervalFactor": 2, 1751 "legendFormat": "{{task}} - {{source_id}}", 1752 "refId": "A" 1753 } 1754 ], 1755 "thresholds": [], 1756 "timeFrom": null, 1757 "timeRegions": [], 1758 "timeShift": null, 1759 "title": "process exits with error", 1760 "tooltip": { 1761 "shared": true, 1762 "sort": 0, 1763 "value_type": "individual" 1764 }, 1765 "type": "graph", 1766 "xaxis": { 1767 "buckets": null, 1768 "mode": "time", 1769 "name": null, 1770 "show": true, 1771 "values": [] 1772 }, 1773 "yaxes": [ 1774 { 1775 "decimals": 0, 1776 "format": "short", 1777 "label": null, 1778 "logBase": 1, 1779 "max": null, 1780 "min": "0", 1781 "show": true 1782 }, 1783 { 1784 "format": "short", 1785 "label": null, 1786 "logBase": 1, 1787 "max": null, 1788 "min": null, 1789 "show": true 1790 } 1791 ], 1792 "yaxis": { 1793 "align": false, 1794 "alignLevel": null 1795 } 1796 }, 1797 { 1798 "aliasColors": { 1799 "admin queue-flush": "blue" 1800 }, 1801 "bars": false, 1802 "dashLength": 10, 1803 "dashes": false, 1804 "datasource": "${DS_TEST-CLUSTER}", 1805 "description": "The number of add/finished jobs per unit of time", 1806 "fieldConfig": { 1807 "defaults": {}, 1808 "overrides": [] 1809 }, 1810 "fill": 1, 1811 "fillGradient": 0, 1812 "gridPos": { 1813 "h": 8, 1814 "w": 15, 1815 "x": 0, 1816 "y": 30 1817 }, 1818 "hiddenSeries": false, 1819 "id": 2, 1820 "legend": { 1821 "alignAsTable": true, 1822 "avg": false, 1823 "current": true, 1824 "max": true, 1825 "min": false, 1826 "rightSide": true, 1827 "show": true, 1828 "sort": "current", 1829 "sortDesc": true, 1830 "total": false, 1831 "values": true 1832 }, 1833 "lines": true, 1834 "linewidth": 1, 1835 "links": [], 1836 "nullPointMode": "null", 1837 "options": { 1838 "alertThreshold": true 1839 }, 1840 "percentage": false, 1841 "pluginVersion": "7.5.11", 1842 "pointradius": 5, 1843 "points": false, 1844 "renderer": "flot", 1845 "seriesOverrides": [], 1846 "spaceLength": 10, 1847 "stack": false, 1848 "steppedLine": false, 1849 "targets": [ 1850 { 1851 "exemplar": true, 1852 "expr": "sum(rate(dm_syncer_finished_jobs_total{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",target_schema=~\"$target_schema\",type=~\"$type\"}[$interval])) by (task,source_id,target_schema,type)", 1853 "format": "time_series", 1854 "interval": "$interval", 1855 "intervalFactor": 2, 1856 "legendFormat": "finished-{{task}} - {{target_schema}} - {{type}} ", 1857 "refId": "A" 1858 }, 1859 { 1860 "exemplar": true, 1861 "expr": "sum(-rate(dm_syncer_added_jobs_total{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",target_schema=~\"$target_schema\",type=~\"$type\"}[$interval])) by (task,source_id,target_schema,type)", 1862 "format": "time_series", 1863 "hide": false, 1864 "interval": "$interval", 1865 "intervalFactor": 2, 1866 "legendFormat": "add-{{task}} - {{target_schema}} - {{type}}", 1867 "refId": "B" 1868 } 1869 ], 1870 "thresholds": [], 1871 "timeFrom": null, 1872 "timeRegions": [], 1873 "timeShift": null, 1874 "title": "DM SQL jobs ", 1875 "tooltip": { 1876 "shared": true, 1877 "sort": 0, 1878 "value_type": "individual" 1879 }, 1880 "type": "graph", 1881 "xaxis": { 1882 "buckets": null, 1883 "mode": "time", 1884 "name": null, 1885 "show": true, 1886 "values": [] 1887 }, 1888 "yaxes": [ 1889 { 1890 "$$hashKey": "object:731", 1891 "format": "short", 1892 "label": null, 1893 "logBase": 1, 1894 "max": null, 1895 "min": null, 1896 "show": true 1897 }, 1898 { 1899 "$$hashKey": "object:732", 1900 "format": "short", 1901 "label": null, 1902 "logBase": 1, 1903 "max": null, 1904 "min": null, 1905 "show": false 1906 } 1907 ], 1908 "yaxis": { 1909 "align": false, 1910 "alignLevel": null 1911 } 1912 }, 1913 { 1914 "aliasColors": {}, 1915 "bars": false, 1916 "dashLength": 10, 1917 "dashes": false, 1918 "datasource": "${DS_TEST-CLUSTER}", 1919 "decimals": 1, 1920 "description": "The time it takes binlog replication unit to detect conflicts between DMLs (in seconds)", 1921 "fieldConfig": { 1922 "defaults": {}, 1923 "overrides": [] 1924 }, 1925 "fill": 1, 1926 "fillGradient": 0, 1927 "gridPos": { 1928 "h": 8, 1929 "w": 9, 1930 "x": 15, 1931 "y": 30 1932 }, 1933 "hiddenSeries": false, 1934 "id": 64, 1935 "legend": { 1936 "alignAsTable": true, 1937 "avg": false, 1938 "current": true, 1939 "max": true, 1940 "min": false, 1941 "rightSide": false, 1942 "show": true, 1943 "total": false, 1944 "values": true 1945 }, 1946 "lines": true, 1947 "linewidth": 1, 1948 "links": [], 1949 "nullPointMode": "null", 1950 "options": { 1951 "alertThreshold": true 1952 }, 1953 "percentage": false, 1954 "pluginVersion": "7.5.11", 1955 "pointradius": 5, 1956 "points": false, 1957 "renderer": "flot", 1958 "seriesOverrides": [], 1959 "spaceLength": 10, 1960 "stack": false, 1961 "steppedLine": false, 1962 "targets": [ 1963 { 1964 "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_conflict_detect_duration_bucket{job_id=~\"$job_id\",task=~\"$task\"}[$interval])) by (le,task))", 1965 "format": "time_series", 1966 "interval": "$interval", 1967 "intervalFactor": 2, 1968 "legendFormat": "{{task}} - 90", 1969 "refId": "A" 1970 }, 1971 { 1972 "expr": "histogram_quantile(0.95, sum(rate(dm_syncer_conflict_detect_duration_bucket{job_id=~\"$job_id\",task=~\"$task\"}[$interval])) by (le,task))", 1973 "format": "time_series", 1974 "interval": "$interval", 1975 "intervalFactor": 2, 1976 "legendFormat": "{{task}} - 95", 1977 "refId": "B" 1978 }, 1979 { 1980 "expr": "histogram_quantile(0.99, sum(rate(dm_syncer_conflict_detect_duration_bucket{job_id=~\"$job_id\",task=~\"$task\"}[$interval])) by (le,task))", 1981 "format": "time_series", 1982 "interval": "$interval", 1983 "intervalFactor": 2, 1984 "legendFormat": "{{task}} - 99", 1985 "refId": "C" 1986 } 1987 ], 1988 "thresholds": [], 1989 "timeFrom": null, 1990 "timeRegions": [], 1991 "timeShift": null, 1992 "title": "DML conflict detect duration", 1993 "tooltip": { 1994 "shared": true, 1995 "sort": 0, 1996 "value_type": "individual" 1997 }, 1998 "type": "graph", 1999 "xaxis": { 2000 "buckets": null, 2001 "mode": "time", 2002 "name": null, 2003 "show": true, 2004 "values": [] 2005 }, 2006 "yaxes": [ 2007 { 2008 "decimals": 2, 2009 "format": "s", 2010 "label": null, 2011 "logBase": 1, 2012 "max": null, 2013 "min": 0, 2014 "show": true 2015 }, 2016 { 2017 "format": "short", 2018 "label": null, 2019 "logBase": 1, 2020 "max": null, 2021 "min": null, 2022 "show": false 2023 } 2024 ], 2025 "yaxis": { 2026 "align": false, 2027 "alignLevel": null 2028 } 2029 }, 2030 { 2031 "aliasColors": {}, 2032 "bars": false, 2033 "dashLength": 10, 2034 "dashes": false, 2035 "datasource": "${DS_TEST-CLUSTER}", 2036 "decimals": 1, 2037 "description": "The number of binlog events received per unit of time (this number does not include the events that need to be skipped)", 2038 "fieldConfig": { 2039 "defaults": {}, 2040 "overrides": [] 2041 }, 2042 "fill": 1, 2043 "fillGradient": 0, 2044 "gridPos": { 2045 "h": 8, 2046 "w": 15, 2047 "x": 0, 2048 "y": 38 2049 }, 2050 "hiddenSeries": false, 2051 "id": 4, 2052 "legend": { 2053 "alignAsTable": true, 2054 "avg": false, 2055 "current": true, 2056 "max": true, 2057 "min": false, 2058 "rightSide": true, 2059 "show": true, 2060 "total": false, 2061 "values": true 2062 }, 2063 "lines": true, 2064 "linewidth": 1, 2065 "links": [], 2066 "nullPointMode": "null", 2067 "options": { 2068 "alertThreshold": true 2069 }, 2070 "percentage": false, 2071 "pluginVersion": "7.5.11", 2072 "pointradius": 5, 2073 "points": false, 2074 "renderer": "flot", 2075 "seriesOverrides": [], 2076 "spaceLength": 10, 2077 "stack": false, 2078 "steppedLine": false, 2079 "targets": [ 2080 { 2081 "exemplar": true, 2082 "expr": "sum by (instance,task,stage) (rate(dm_syncer_binlog_transform_cost_count{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\", source_id=~\"$source\",stage=~\"$stage\"}[$interval]))", 2083 "format": "time_series", 2084 "hide": false, 2085 "interval": "$interval", 2086 "intervalFactor": 2, 2087 "legendFormat": "{{task}} - {{instance}} - {{stage}}", 2088 "refId": "A" 2089 } 2090 ], 2091 "thresholds": [], 2092 "timeFrom": null, 2093 "timeRegions": [], 2094 "timeShift": null, 2095 "title": "binlog event QPS", 2096 "tooltip": { 2097 "shared": true, 2098 "sort": 0, 2099 "value_type": "individual" 2100 }, 2101 "type": "graph", 2102 "xaxis": { 2103 "buckets": null, 2104 "mode": "time", 2105 "name": null, 2106 "show": true, 2107 "values": [] 2108 }, 2109 "yaxes": [ 2110 { 2111 "$$hashKey": "object:325", 2112 "format": "short", 2113 "label": null, 2114 "logBase": 1, 2115 "max": null, 2116 "min": null, 2117 "show": true 2118 }, 2119 { 2120 "$$hashKey": "object:326", 2121 "format": "short", 2122 "label": null, 2123 "logBase": 1, 2124 "max": null, 2125 "min": null, 2126 "show": false 2127 } 2128 ], 2129 "yaxis": { 2130 "align": false, 2131 "alignLevel": null 2132 } 2133 }, 2134 { 2135 "aliasColors": {}, 2136 "bars": false, 2137 "dashLength": 10, 2138 "dashes": false, 2139 "datasource": "${DS_TEST-CLUSTER}", 2140 "decimals": 1, 2141 "description": "The time it takes binlog replication unit to add a job to the queue (in seconds)", 2142 "fieldConfig": { 2143 "defaults": {}, 2144 "overrides": [] 2145 }, 2146 "fill": 1, 2147 "fillGradient": 0, 2148 "gridPos": { 2149 "h": 8, 2150 "w": 9, 2151 "x": 15, 2152 "y": 38 2153 }, 2154 "hiddenSeries": false, 2155 "id": 63, 2156 "legend": { 2157 "alignAsTable": true, 2158 "avg": false, 2159 "current": true, 2160 "max": true, 2161 "min": true, 2162 "rightSide": false, 2163 "show": true, 2164 "total": false, 2165 "values": true 2166 }, 2167 "lines": true, 2168 "linewidth": 1, 2169 "links": [], 2170 "nullPointMode": "null", 2171 "options": { 2172 "alertThreshold": true 2173 }, 2174 "percentage": false, 2175 "pluginVersion": "7.5.11", 2176 "pointradius": 5, 2177 "points": false, 2178 "renderer": "flot", 2179 "seriesOverrides": [], 2180 "spaceLength": 10, 2181 "stack": false, 2182 "steppedLine": false, 2183 "targets": [ 2184 { 2185 "expr": "histogram_quantile(0.90, sum (rate(dm_syncer_add_job_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",queueNo=~\"$queueNo\",type=~\"$type\"}[$interval])) by (task,le))", 2186 "format": "time_series", 2187 "interval": "$interval", 2188 "intervalFactor": 2, 2189 "legendFormat": "{{task}} - 90", 2190 "refId": "A" 2191 }, 2192 { 2193 "expr": "histogram_quantile(0.95, sum(rate(dm_syncer_add_job_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",queueNo=~\"$queueNo\",type=~\"$type\"}[$interval])) by (task,le))", 2194 "format": "time_series", 2195 "interval": "$interval", 2196 "intervalFactor": 2, 2197 "legendFormat": "{{task}} - 95", 2198 "refId": "B" 2199 }, 2200 { 2201 "expr": "histogram_quantile(0.99, sum(rate(dm_syncer_add_job_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\",queueNo=~\"$queueNo\",type=~\"$type\"}[$interval])) by (task,le))", 2202 "format": "time_series", 2203 "interval": "$interval", 2204 "intervalFactor": 2, 2205 "legendFormat": "{{task}} - 99", 2206 "refId": "C" 2207 } 2208 ], 2209 "thresholds": [], 2210 "timeFrom": null, 2211 "timeRegions": [], 2212 "timeShift": null, 2213 "title": "add job duration", 2214 "tooltip": { 2215 "shared": true, 2216 "sort": 0, 2217 "value_type": "individual" 2218 }, 2219 "type": "graph", 2220 "xaxis": { 2221 "buckets": null, 2222 "mode": "time", 2223 "name": null, 2224 "show": true, 2225 "values": [] 2226 }, 2227 "yaxes": [ 2228 { 2229 "decimals": 1, 2230 "format": "s", 2231 "label": null, 2232 "logBase": 1, 2233 "max": null, 2234 "min": 0, 2235 "show": true 2236 }, 2237 { 2238 "format": "short", 2239 "label": null, 2240 "logBase": 1, 2241 "max": null, 2242 "min": null, 2243 "show": false 2244 } 2245 ], 2246 "yaxis": { 2247 "align": false, 2248 "alignLevel": null 2249 } 2250 }, 2251 { 2252 "aliasColors": {}, 2253 "bars": false, 2254 "dashLength": 10, 2255 "dashes": false, 2256 "datasource": "${DS_TEST-CLUSTER}", 2257 "decimals": 1, 2258 "description": "The time it takes binlog replication unit to execute the transaction to the downstream (in seconds)", 2259 "fieldConfig": { 2260 "defaults": {}, 2261 "overrides": [] 2262 }, 2263 "fill": 1, 2264 "fillGradient": 0, 2265 "gridPos": { 2266 "h": 8, 2267 "w": 15, 2268 "x": 0, 2269 "y": 46 2270 }, 2271 "hiddenSeries": false, 2272 "id": 60, 2273 "legend": { 2274 "alignAsTable": true, 2275 "avg": false, 2276 "current": true, 2277 "max": true, 2278 "min": false, 2279 "rightSide": true, 2280 "show": true, 2281 "sort": "current", 2282 "sortDesc": true, 2283 "total": false, 2284 "values": true 2285 }, 2286 "lines": true, 2287 "linewidth": 1, 2288 "links": [], 2289 "nullPointMode": "null", 2290 "options": { 2291 "alertThreshold": true 2292 }, 2293 "percentage": false, 2294 "pluginVersion": "7.5.11", 2295 "pointradius": 5, 2296 "points": false, 2297 "renderer": "flot", 2298 "seriesOverrides": [], 2299 "spaceLength": 10, 2300 "stack": false, 2301 "steppedLine": false, 2302 "targets": [ 2303 { 2304 "exemplar": true, 2305 "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_txn_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", 2306 "format": "time_series", 2307 "hide": false, 2308 "interval": "$interval", 2309 "intervalFactor": 2, 2310 "legendFormat": "{{task}} - {{instance}} - txn-90", 2311 "refId": "A" 2312 }, 2313 { 2314 "exemplar": true, 2315 "expr": "histogram_quantile(0.95, sum(rate(dm_syncer_txn_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", 2316 "format": "time_series", 2317 "hide": false, 2318 "interval": "$interval", 2319 "intervalFactor": 2, 2320 "legendFormat": "{{task}} - {{instance}} - txn-95", 2321 "refId": "B" 2322 }, 2323 { 2324 "exemplar": true, 2325 "expr": "histogram_quantile(0.99, sum(rate(dm_syncer_txn_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", 2326 "format": "time_series", 2327 "hide": false, 2328 "interval": "$interval", 2329 "intervalFactor": 2, 2330 "legendFormat": "{{task}} - {{instance}} - txn-99", 2331 "refId": "C" 2332 }, 2333 { 2334 "exemplar": true, 2335 "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_query_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", 2336 "format": "time_series", 2337 "hide": false, 2338 "interval": "$interval", 2339 "intervalFactor": 2, 2340 "legendFormat": "{{task}} - {{instance}} - ddl-90", 2341 "refId": "D" 2342 }, 2343 { 2344 "exemplar": true, 2345 "expr": "histogram_quantile(0.95, sum(rate(dm_syncer_query_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", 2346 "format": "time_series", 2347 "hide": false, 2348 "interval": "$interval", 2349 "intervalFactor": 2, 2350 "legendFormat": "{{task}} - {{instance}} - ddl-95", 2351 "refId": "E" 2352 }, 2353 { 2354 "exemplar": true, 2355 "expr": "histogram_quantile(0.99, sum(rate(dm_syncer_query_duration_time_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", 2356 "format": "time_series", 2357 "hide": false, 2358 "interval": "$interval", 2359 "intervalFactor": 2, 2360 "legendFormat": "{{task}} - {{instance}} - ddl-99", 2361 "refId": "F" 2362 } 2363 ], 2364 "thresholds": [], 2365 "timeFrom": null, 2366 "timeRegions": [], 2367 "timeShift": null, 2368 "title": "transaction/ddl query execution latency", 2369 "tooltip": { 2370 "shared": true, 2371 "sort": 0, 2372 "value_type": "individual" 2373 }, 2374 "type": "graph", 2375 "xaxis": { 2376 "buckets": null, 2377 "mode": "time", 2378 "name": null, 2379 "show": true, 2380 "values": [] 2381 }, 2382 "yaxes": [ 2383 { 2384 "$$hashKey": "object:401", 2385 "decimals": 1, 2386 "format": "s", 2387 "label": null, 2388 "logBase": 1, 2389 "max": null, 2390 "min": 0, 2391 "show": true 2392 }, 2393 { 2394 "$$hashKey": "object:402", 2395 "format": "short", 2396 "label": null, 2397 "logBase": 1, 2398 "max": null, 2399 "min": null, 2400 "show": false 2401 } 2402 ], 2403 "yaxis": { 2404 "align": false, 2405 "alignLevel": null 2406 } 2407 }, 2408 { 2409 "aliasColors": {}, 2410 "bars": false, 2411 "dashLength": 10, 2412 "dashes": false, 2413 "datasource": "${DS_TEST-CLUSTER}", 2414 "description": "The time it takes binlog replication unit to execute every statement to the downstream (in seconds)", 2415 "fieldConfig": { 2416 "defaults": {}, 2417 "overrides": [] 2418 }, 2419 "fill": 1, 2420 "fillGradient": 0, 2421 "gridPos": { 2422 "h": 7, 2423 "w": 9, 2424 "x": 15, 2425 "y": 46 2426 }, 2427 "hiddenSeries": false, 2428 "id": 1, 2429 "legend": { 2430 "alignAsTable": true, 2431 "avg": false, 2432 "current": true, 2433 "max": true, 2434 "min": false, 2435 "rightSide": false, 2436 "show": true, 2437 "total": false, 2438 "values": true 2439 }, 2440 "lines": true, 2441 "linewidth": 1, 2442 "links": [], 2443 "nullPointMode": "null", 2444 "options": { 2445 "alertThreshold": true 2446 }, 2447 "percentage": false, 2448 "pluginVersion": "7.5.11", 2449 "pointradius": 5, 2450 "points": false, 2451 "renderer": "flot", 2452 "seriesOverrides": [], 2453 "spaceLength": 10, 2454 "stack": false, 2455 "steppedLine": false, 2456 "targets": [ 2457 { 2458 "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_stmt_duration_time_bucket{job_id=~\"$job_id\",task=~\"$task\", type=\"begin\", instance=~\"$instance\"}[$interval])) by (le,task,instance))", 2459 "format": "time_series", 2460 "interval": "$interval", 2461 "intervalFactor": 2, 2462 "legendFormat": "{{task}} - {{instance}} - begin", 2463 "refId": "A" 2464 }, 2465 { 2466 "exemplar": true, 2467 "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_stmt_duration_time_bucket{job_id=~\"$job_id\",task=~\"$task\", type=\"stmt\", instance=~\"$instance\"}[$interval])) by (le,task,instance))", 2468 "format": "time_series", 2469 "interval": "$interval", 2470 "intervalFactor": 2, 2471 "legendFormat": "{{task}} - {{instance}} - stmt", 2472 "refId": "B" 2473 }, 2474 { 2475 "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_stmt_duration_time_bucket{job_id=~\"$job_id\",task=~\"$task\", type=\"commit\", instance=~\"$instance\"}[$interval])) by (le,task,instance))", 2476 "format": "time_series", 2477 "interval": "$interval", 2478 "intervalFactor": 2, 2479 "legendFormat": "{{task}} - {{instance}} - commit", 2480 "refId": "C" 2481 }, 2482 { 2483 "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_stmt_duration_time_bucket{job_id=~\"$job_id\",task=~\"$task\", type=\"rollback\", instance=~\"$instance\"}[1m])) by (le,task,instance))", 2484 "format": "time_series", 2485 "interval": "$interval", 2486 "intervalFactor": 2, 2487 "legendFormat": "{{task}} - {{instance}} - rollback", 2488 "refId": "D" 2489 } 2490 ], 2491 "thresholds": [], 2492 "timeFrom": null, 2493 "timeRegions": [], 2494 "timeShift": null, 2495 "title": "statement execution latency - 90", 2496 "tooltip": { 2497 "shared": true, 2498 "sort": 0, 2499 "value_type": "individual" 2500 }, 2501 "type": "graph", 2502 "xaxis": { 2503 "buckets": null, 2504 "mode": "time", 2505 "name": null, 2506 "show": true, 2507 "values": [] 2508 }, 2509 "yaxes": [ 2510 { 2511 "decimals": 1, 2512 "format": "s", 2513 "label": null, 2514 "logBase": 1, 2515 "max": null, 2516 "min": 0, 2517 "show": true 2518 }, 2519 { 2520 "format": "short", 2521 "label": null, 2522 "logBase": 1, 2523 "max": null, 2524 "min": null, 2525 "show": false 2526 } 2527 ], 2528 "yaxis": { 2529 "align": false, 2530 "alignLevel": null 2531 } 2532 }, 2533 { 2534 "aliasColors": {}, 2535 "bars": false, 2536 "dashLength": 10, 2537 "dashes": false, 2538 "datasource": "${DS_TEST-CLUSTER}", 2539 "decimals": 1, 2540 "description": "The number of binlog events received per unit of time that needs to be skipped", 2541 "fieldConfig": { 2542 "defaults": {}, 2543 "overrides": [] 2544 }, 2545 "fill": 1, 2546 "fillGradient": 0, 2547 "gridPos": { 2548 "h": 8, 2549 "w": 9, 2550 "x": 15, 2551 "y": 53 2552 }, 2553 "hiddenSeries": false, 2554 "id": 35, 2555 "legend": { 2556 "alignAsTable": true, 2557 "avg": false, 2558 "current": true, 2559 "max": true, 2560 "min": false, 2561 "rightSide": false, 2562 "show": true, 2563 "total": false, 2564 "values": true 2565 }, 2566 "lines": true, 2567 "linewidth": 1, 2568 "links": [], 2569 "nullPointMode": "null", 2570 "options": { 2571 "alertThreshold": true 2572 }, 2573 "percentage": false, 2574 "pluginVersion": "7.5.11", 2575 "pointradius": 5, 2576 "points": false, 2577 "renderer": "flot", 2578 "seriesOverrides": [], 2579 "spaceLength": 10, 2580 "stack": false, 2581 "steppedLine": false, 2582 "targets": [ 2583 { 2584 "expr": "rate(dm_syncer_skip_binlog_duration_count{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\", type=\"rows\"}[$interval])", 2585 "format": "time_series", 2586 "interval": "$interval", 2587 "intervalFactor": 2, 2588 "legendFormat": "{{task}} - {{source_id}} - rows", 2589 "refId": "A" 2590 }, 2591 { 2592 "expr": "rate(dm_syncer_skip_binlog_duration_count{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\", type=\"query\"}[$interval])", 2593 "format": "time_series", 2594 "interval": "$interval", 2595 "intervalFactor": 2, 2596 "legendFormat": "{{task}} - {{source_id}} - query", 2597 "refId": "B" 2598 } 2599 ], 2600 "thresholds": [], 2601 "timeFrom": null, 2602 "timeRegions": [], 2603 "timeShift": null, 2604 "title": "skipped binlog event QPS", 2605 "tooltip": { 2606 "shared": true, 2607 "sort": 0, 2608 "value_type": "individual" 2609 }, 2610 "type": "graph", 2611 "xaxis": { 2612 "buckets": null, 2613 "mode": "time", 2614 "name": null, 2615 "show": true, 2616 "values": [] 2617 }, 2618 "yaxes": [ 2619 { 2620 "decimals": 1, 2621 "format": "short", 2622 "label": null, 2623 "logBase": 1, 2624 "max": null, 2625 "min": "0", 2626 "show": true 2627 }, 2628 { 2629 "format": "short", 2630 "label": null, 2631 "logBase": 1, 2632 "max": null, 2633 "min": null, 2634 "show": false 2635 } 2636 ], 2637 "yaxis": { 2638 "align": false, 2639 "alignLevel": null 2640 } 2641 }, 2642 { 2643 "aliasColors": {}, 2644 "bars": false, 2645 "dashLength": 10, 2646 "dashes": false, 2647 "datasource": "${DS_TEST-CLUSTER}", 2648 "decimals": 0, 2649 "description": "The remain length of DML job queues", 2650 "fieldConfig": { 2651 "defaults": {}, 2652 "overrides": [] 2653 }, 2654 "fill": 1, 2655 "fillGradient": 0, 2656 "gridPos": { 2657 "h": 7, 2658 "w": 15, 2659 "x": 0, 2660 "y": 54 2661 }, 2662 "hiddenSeries": false, 2663 "id": 62, 2664 "legend": { 2665 "alignAsTable": true, 2666 "avg": false, 2667 "current": true, 2668 "max": true, 2669 "min": true, 2670 "rightSide": true, 2671 "show": true, 2672 "sort": "current", 2673 "sortDesc": true, 2674 "total": false, 2675 "values": true 2676 }, 2677 "lines": true, 2678 "linewidth": 1, 2679 "links": [], 2680 "nullPointMode": "null", 2681 "options": { 2682 "alertThreshold": true 2683 }, 2684 "percentage": false, 2685 "pluginVersion": "7.5.11", 2686 "pointradius": 5, 2687 "points": false, 2688 "renderer": "flot", 2689 "seriesOverrides": [], 2690 "spaceLength": 10, 2691 "stack": false, 2692 "steppedLine": false, 2693 "targets": [ 2694 { 2695 "exemplar": true, 2696 "expr": "sum(dm_syncer_queue_size{job_id=~\"$job_id\",task=~\"$task\"}) by (task)", 2697 "format": "time_series", 2698 "interval": "$interval", 2699 "intervalFactor": 2, 2700 "legendFormat": "{{task}}", 2701 "refId": "A" 2702 } 2703 ], 2704 "thresholds": [], 2705 "timeFrom": null, 2706 "timeRegions": [], 2707 "timeShift": null, 2708 "title": "DML queue remain length", 2709 "tooltip": { 2710 "shared": true, 2711 "sort": 0, 2712 "value_type": "individual" 2713 }, 2714 "type": "graph", 2715 "xaxis": { 2716 "buckets": null, 2717 "mode": "time", 2718 "name": null, 2719 "show": true, 2720 "values": [] 2721 }, 2722 "yaxes": [ 2723 { 2724 "$$hashKey": "object:246", 2725 "format": "short", 2726 "label": null, 2727 "logBase": 1, 2728 "max": null, 2729 "min": null, 2730 "show": true 2731 }, 2732 { 2733 "$$hashKey": "object:247", 2734 "format": "short", 2735 "label": null, 2736 "logBase": 1, 2737 "max": null, 2738 "min": null, 2739 "show": false 2740 } 2741 ], 2742 "yaxis": { 2743 "align": false, 2744 "alignLevel": null 2745 } 2746 }, 2747 { 2748 "aliasColors": {}, 2749 "bars": false, 2750 "dashLength": 10, 2751 "dashes": false, 2752 "datasource": "${DS_TEST-CLUSTER}", 2753 "decimals": 1, 2754 "description": "The size of a single binlog event that the binlog replication unit reads from relay log or upstream master", 2755 "fieldConfig": { 2756 "defaults": {}, 2757 "overrides": [] 2758 }, 2759 "fill": 1, 2760 "fillGradient": 0, 2761 "gridPos": { 2762 "h": 8, 2763 "w": 8, 2764 "x": 0, 2765 "y": 61 2766 }, 2767 "hiddenSeries": false, 2768 "id": 61, 2769 "legend": { 2770 "alignAsTable": true, 2771 "avg": false, 2772 "current": true, 2773 "max": true, 2774 "min": false, 2775 "rightSide": false, 2776 "show": true, 2777 "sort": null, 2778 "sortDesc": null, 2779 "total": false, 2780 "values": true 2781 }, 2782 "lines": true, 2783 "linewidth": 1, 2784 "links": [], 2785 "nullPointMode": "null", 2786 "options": { 2787 "alertThreshold": true 2788 }, 2789 "percentage": false, 2790 "pluginVersion": "7.5.11", 2791 "pointradius": 5, 2792 "points": false, 2793 "renderer": "flot", 2794 "seriesOverrides": [], 2795 "spaceLength": 10, 2796 "stack": false, 2797 "steppedLine": false, 2798 "targets": [ 2799 { 2800 "exemplar": true, 2801 "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_binlog_event_size_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,worker,task))", 2802 "format": "time_series", 2803 "interval": "$interval", 2804 "intervalFactor": 2, 2805 "legendFormat": "{{task}} - {{worker}} - 90", 2806 "refId": "A" 2807 }, 2808 { 2809 "exemplar": true, 2810 "expr": "histogram_quantile(0.95, sum(rate(dm_syncer_binlog_event_size_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[1m])) by (le,worker,task))", 2811 "format": "time_series", 2812 "interval": "$interval", 2813 "intervalFactor": 2, 2814 "legendFormat": "{{task}} - {{worker}} - 95", 2815 "refId": "B" 2816 }, 2817 { 2818 "exemplar": true, 2819 "expr": "histogram_quantile(0.99, sum(rate(dm_syncer_binlog_event_size_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[1m])) by (le,worker,task))", 2820 "format": "time_series", 2821 "interval": "$interval", 2822 "intervalFactor": 2, 2823 "legendFormat": "{{task}} - {{worker}} - 99", 2824 "refId": "C" 2825 } 2826 ], 2827 "thresholds": [], 2828 "timeFrom": null, 2829 "timeRegions": [], 2830 "timeShift": null, 2831 "title": "binlog event size", 2832 "tooltip": { 2833 "shared": true, 2834 "sort": 0, 2835 "value_type": "individual" 2836 }, 2837 "type": "graph", 2838 "xaxis": { 2839 "buckets": null, 2840 "mode": "time", 2841 "name": null, 2842 "show": true, 2843 "values": [] 2844 }, 2845 "yaxes": [ 2846 { 2847 "$$hashKey": "object:1109", 2848 "decimals": 1, 2849 "format": "decbytes", 2850 "label": null, 2851 "logBase": 1, 2852 "max": null, 2853 "min": null, 2854 "show": true 2855 }, 2856 { 2857 "$$hashKey": "object:1110", 2858 "format": "short", 2859 "label": null, 2860 "logBase": 1, 2861 "max": null, 2862 "min": null, 2863 "show": false 2864 } 2865 ], 2866 "yaxis": { 2867 "align": false, 2868 "alignLevel": null 2869 } 2870 }, 2871 { 2872 "aliasColors": {}, 2873 "bars": false, 2874 "dashLength": 10, 2875 "dashes": false, 2876 "datasource": "${DS_TEST-CLUSTER}", 2877 "description": "The number of binlog files in binlog replication unit that are behind the master", 2878 "fieldConfig": { 2879 "defaults": {}, 2880 "overrides": [] 2881 }, 2882 "fill": 1, 2883 "fillGradient": 0, 2884 "gridPos": { 2885 "h": 8, 2886 "w": 7, 2887 "x": 8, 2888 "y": 61 2889 }, 2890 "hiddenSeries": false, 2891 "id": 43, 2892 "legend": { 2893 "alignAsTable": true, 2894 "avg": false, 2895 "current": true, 2896 "max": true, 2897 "min": false, 2898 "show": true, 2899 "total": false, 2900 "values": true 2901 }, 2902 "lines": true, 2903 "linewidth": 1, 2904 "links": [], 2905 "nullPointMode": "null", 2906 "options": { 2907 "alertThreshold": true 2908 }, 2909 "percentage": false, 2910 "pluginVersion": "7.5.11", 2911 "pointradius": 5, 2912 "points": false, 2913 "renderer": "flot", 2914 "seriesOverrides": [], 2915 "spaceLength": 10, 2916 "stack": false, 2917 "steppedLine": false, 2918 "targets": [ 2919 { 2920 "expr": "dm_syncer_binlog_file{job_id=~\"$job_id\",source_id=~\"$source\", task=~\"$task\", node=\"master\"} - ON(task,source_id) dm_syncer_binlog_file{job_id=~\"$job_id\",source_id=~\"$source\", task=~\"$task\", node=\"syncer\"}", 2921 "format": "time_series", 2922 "hide": false, 2923 "interval": "$interval", 2924 "intervalFactor": 2, 2925 "legendFormat": "{{task}} - {{source_id}}", 2926 "refId": "B" 2927 } 2928 ], 2929 "thresholds": [], 2930 "timeFrom": null, 2931 "timeRegions": [], 2932 "timeShift": null, 2933 "title": "binlog file gap between master and syncer", 2934 "tooltip": { 2935 "shared": true, 2936 "sort": 0, 2937 "value_type": "individual" 2938 }, 2939 "type": "graph", 2940 "xaxis": { 2941 "buckets": null, 2942 "mode": "time", 2943 "name": null, 2944 "show": true, 2945 "values": [] 2946 }, 2947 "yaxes": [ 2948 { 2949 "decimals": 0, 2950 "format": "short", 2951 "label": null, 2952 "logBase": 1, 2953 "max": null, 2954 "min": null, 2955 "show": true 2956 }, 2957 { 2958 "format": "short", 2959 "label": null, 2960 "logBase": 1, 2961 "max": null, 2962 "min": null, 2963 "show": false 2964 } 2965 ], 2966 "yaxis": { 2967 "align": false, 2968 "alignLevel": null 2969 } 2970 }, 2971 { 2972 "aliasColors": {}, 2973 "bars": false, 2974 "dashLength": 10, 2975 "dashes": false, 2976 "datasource": "${DS_TEST-CLUSTER}", 2977 "decimals": 1, 2978 "description": "The number of rows of each binlog event that enters the DM", 2979 "fieldConfig": { 2980 "defaults": {}, 2981 "overrides": [] 2982 }, 2983 "fill": 1, 2984 "fillGradient": 0, 2985 "gridPos": { 2986 "h": 8, 2987 "w": 9, 2988 "x": 15, 2989 "y": 61 2990 }, 2991 "hiddenSeries": false, 2992 "id": 106, 2993 "legend": { 2994 "alignAsTable": true, 2995 "avg": false, 2996 "current": true, 2997 "max": true, 2998 "min": false, 2999 "show": true, 3000 "total": false, 3001 "values": true 3002 }, 3003 "lines": true, 3004 "linewidth": 1, 3005 "links": [], 3006 "nullPointMode": "null", 3007 "options": { 3008 "alertThreshold": true 3009 }, 3010 "percentage": false, 3011 "pluginVersion": "7.5.11", 3012 "pointradius": 5, 3013 "points": false, 3014 "renderer": "flot", 3015 "seriesOverrides": [], 3016 "spaceLength": 10, 3017 "stack": false, 3018 "steppedLine": false, 3019 "targets": [ 3020 { 3021 "exemplar": true, 3022 "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_binlog_event_row_bucket{job_id=~\"$job_id\",task=~\"$task\",source_id=~\"$source\"}[$interval])) by (le,task,source))", 3023 "hide": false, 3024 "interval": "", 3025 "legendFormat": "{{task}}-{{source}}", 3026 "refId": "A" 3027 } 3028 ], 3029 "thresholds": [], 3030 "timeFrom": null, 3031 "timeRegions": [], 3032 "timeShift": null, 3033 "title": "binlog event row number", 3034 "tooltip": { 3035 "shared": true, 3036 "sort": 0, 3037 "value_type": "individual" 3038 }, 3039 "type": "graph", 3040 "xaxis": { 3041 "buckets": null, 3042 "mode": "time", 3043 "name": null, 3044 "show": true, 3045 "values": [] 3046 }, 3047 "yaxes": [ 3048 { 3049 "decimals": 0, 3050 "format": "short", 3051 "label": null, 3052 "logBase": 1, 3053 "max": null, 3054 "min": null, 3055 "show": true 3056 }, 3057 { 3058 "format": "short", 3059 "label": null, 3060 "logBase": 1, 3061 "max": null, 3062 "min": null, 3063 "show": false 3064 } 3065 ], 3066 "yaxis": { 3067 "align": false, 3068 "alignLevel": null 3069 } 3070 }, 3071 { 3072 "aliasColors": {}, 3073 "bars": false, 3074 "dashLength": 10, 3075 "dashes": false, 3076 "datasource": "${DS_TEST-CLUSTER}", 3077 "decimals": 1, 3078 "description": "The duration that the binlog replication unit reads binlog event from the relay log or upstream MySQL (in seconds)", 3079 "fieldConfig": { 3080 "defaults": {}, 3081 "overrides": [] 3082 }, 3083 "fill": 1, 3084 "fillGradient": 0, 3085 "gridPos": { 3086 "h": 8, 3087 "w": 8, 3088 "x": 0, 3089 "y": 69 3090 }, 3091 "hiddenSeries": false, 3092 "id": 36, 3093 "legend": { 3094 "alignAsTable": true, 3095 "avg": false, 3096 "current": true, 3097 "max": true, 3098 "min": false, 3099 "rightSide": false, 3100 "show": true, 3101 "total": false, 3102 "values": true 3103 }, 3104 "lines": true, 3105 "linewidth": 1, 3106 "links": [], 3107 "nullPointMode": "null", 3108 "options": { 3109 "alertThreshold": true 3110 }, 3111 "percentage": false, 3112 "pluginVersion": "7.5.11", 3113 "pointradius": 5, 3114 "points": false, 3115 "renderer": "flot", 3116 "seriesOverrides": [], 3117 "spaceLength": 10, 3118 "stack": false, 3119 "steppedLine": false, 3120 "targets": [ 3121 { 3122 "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_read_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", 3123 "format": "time_series", 3124 "interval": "$interval", 3125 "intervalFactor": 2, 3126 "legendFormat": "{{task}} - {{source_id}} - 90", 3127 "refId": "A" 3128 }, 3129 { 3130 "expr": "histogram_quantile(0.95, sum(rate(dm_syncer_read_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", 3131 "format": "time_series", 3132 "interval": "$interval", 3133 "intervalFactor": 2, 3134 "legendFormat": "{{task}} - {{source_id}} - 95", 3135 "refId": "B" 3136 }, 3137 { 3138 "expr": "histogram_quantile(0.99, sum(rate(dm_syncer_read_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", 3139 "format": "time_series", 3140 "interval": "$interval", 3141 "intervalFactor": 2, 3142 "legendFormat": "{{task}} - {{source_id}} - 99", 3143 "refId": "C" 3144 } 3145 ], 3146 "thresholds": [], 3147 "timeFrom": null, 3148 "timeRegions": [], 3149 "timeShift": null, 3150 "title": "read binlog event duration", 3151 "tooltip": { 3152 "shared": true, 3153 "sort": 0, 3154 "value_type": "individual" 3155 }, 3156 "type": "graph", 3157 "xaxis": { 3158 "buckets": null, 3159 "mode": "time", 3160 "name": null, 3161 "show": true, 3162 "values": [] 3163 }, 3164 "yaxes": [ 3165 { 3166 "decimals": 1, 3167 "format": "s", 3168 "label": null, 3169 "logBase": 1, 3170 "max": null, 3171 "min": 0, 3172 "show": true 3173 }, 3174 { 3175 "format": "short", 3176 "label": null, 3177 "logBase": 1, 3178 "max": null, 3179 "min": null, 3180 "show": false 3181 } 3182 ], 3183 "yaxis": { 3184 "align": false, 3185 "alignLevel": null 3186 } 3187 }, 3188 { 3189 "aliasColors": {}, 3190 "bars": false, 3191 "dashLength": 10, 3192 "dashes": false, 3193 "datasource": "${DS_TEST-CLUSTER}", 3194 "description": "The time it takes binlog replication unit to parse and transform the binlog into SQL statements (in seconds)", 3195 "fieldConfig": { 3196 "defaults": {}, 3197 "overrides": [] 3198 }, 3199 "fill": 1, 3200 "fillGradient": 0, 3201 "gridPos": { 3202 "h": 8, 3203 "w": 7, 3204 "x": 8, 3205 "y": 69 3206 }, 3207 "hiddenSeries": false, 3208 "id": 58, 3209 "legend": { 3210 "alignAsTable": true, 3211 "avg": false, 3212 "current": true, 3213 "max": true, 3214 "min": false, 3215 "rightSide": false, 3216 "show": true, 3217 "total": false, 3218 "values": true 3219 }, 3220 "lines": true, 3221 "linewidth": 1, 3222 "links": [], 3223 "nullPointMode": "null", 3224 "options": { 3225 "alertThreshold": true 3226 }, 3227 "percentage": false, 3228 "pluginVersion": "7.5.11", 3229 "pointradius": 5, 3230 "points": false, 3231 "renderer": "flot", 3232 "seriesOverrides": [], 3233 "spaceLength": 10, 3234 "stack": false, 3235 "steppedLine": false, 3236 "targets": [ 3237 { 3238 "exemplar": true, 3239 "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[1m])) by (le,worker,task,stage))", 3240 "format": "time_series", 3241 "hide": false, 3242 "interval": "", 3243 "intervalFactor": 2, 3244 "legendFormat": "90-[[worker]]-[[task]]-{{stage}}", 3245 "refId": "A" 3246 }, 3247 { 3248 "exemplar": true, 3249 "expr": "histogram_quantile(0.95, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[1m])) by (le,worker,task,stage))", 3250 "format": "time_series", 3251 "hide": true, 3252 "interval": "", 3253 "intervalFactor": 2, 3254 "legendFormat": "95-[[worker]]-[[task]]-{{stage}}", 3255 "refId": "B" 3256 }, 3257 { 3258 "exemplar": true, 3259 "expr": "histogram_quantile(0.99, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",worker=~\"$worker\",task=~\"$task\", source_id=~\"$source\"}[1m])) by (le,worker,task,stage))", 3260 "format": "time_series", 3261 "hide": true, 3262 "interval": "", 3263 "intervalFactor": 2, 3264 "legendFormat": "99-[[worker]]-[[task]]-{{stage}}", 3265 "refId": "C" 3266 } 3267 ], 3268 "thresholds": [], 3269 "timeFrom": null, 3270 "timeRegions": [], 3271 "timeShift": null, 3272 "title": "binlog event cost duration", 3273 "tooltip": { 3274 "shared": true, 3275 "sort": 0, 3276 "value_type": "individual" 3277 }, 3278 "type": "graph", 3279 "xaxis": { 3280 "buckets": null, 3281 "mode": "time", 3282 "name": null, 3283 "show": true, 3284 "values": [] 3285 }, 3286 "yaxes": [ 3287 { 3288 "decimals": 2, 3289 "format": "s", 3290 "label": null, 3291 "logBase": 1, 3292 "max": null, 3293 "min": 0, 3294 "show": true 3295 }, 3296 { 3297 "format": "short", 3298 "label": null, 3299 "logBase": 1, 3300 "max": null, 3301 "min": null, 3302 "show": false 3303 } 3304 ], 3305 "yaxis": { 3306 "align": false, 3307 "alignLevel": null 3308 } 3309 }, 3310 { 3311 "aliasColors": {}, 3312 "bars": false, 3313 "dashLength": 10, 3314 "dashes": false, 3315 "datasource": "${DS_TEST-CLUSTER}", 3316 "description": "The time it takes binlog replication unit to dispatch a binlog event (in seconds)", 3317 "fieldConfig": { 3318 "defaults": {}, 3319 "overrides": [] 3320 }, 3321 "fill": 1, 3322 "fillGradient": 0, 3323 "gridPos": { 3324 "h": 8, 3325 "w": 9, 3326 "x": 15, 3327 "y": 69 3328 }, 3329 "hiddenSeries": false, 3330 "id": 59, 3331 "legend": { 3332 "alignAsTable": true, 3333 "avg": false, 3334 "current": true, 3335 "max": true, 3336 "min": false, 3337 "rightSide": false, 3338 "show": true, 3339 "total": false, 3340 "values": true 3341 }, 3342 "lines": true, 3343 "linewidth": 1, 3344 "links": [], 3345 "nullPointMode": "null", 3346 "options": { 3347 "alertThreshold": true 3348 }, 3349 "percentage": false, 3350 "pluginVersion": "7.5.11", 3351 "pointradius": 5, 3352 "points": false, 3353 "renderer": "flot", 3354 "seriesOverrides": [], 3355 "spaceLength": 10, 3356 "stack": false, 3357 "steppedLine": false, 3358 "targets": [ 3359 { 3360 "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_dispatch_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", 3361 "format": "time_series", 3362 "interval": "$interval", 3363 "intervalFactor": 2, 3364 "legendFormat": "{{task}} - {{source_id}} - 90", 3365 "refId": "A" 3366 }, 3367 { 3368 "expr": "histogram_quantile(0.95, sum(rate(dm_syncer_dispatch_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", 3369 "format": "time_series", 3370 "interval": "$interval", 3371 "intervalFactor": 2, 3372 "legendFormat": "{{task}} - {{source_id}} - 95", 3373 "refId": "B" 3374 }, 3375 { 3376 "expr": "histogram_quantile(0.99, sum(rate(dm_syncer_dispatch_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", 3377 "format": "time_series", 3378 "interval": "$interval", 3379 "intervalFactor": 2, 3380 "legendFormat": "{{task}} - {{source_id}} - 99", 3381 "refId": "C" 3382 } 3383 ], 3384 "thresholds": [], 3385 "timeFrom": null, 3386 "timeRegions": [], 3387 "timeShift": null, 3388 "title": "dispatch binlog event duration", 3389 "tooltip": { 3390 "shared": true, 3391 "sort": 0, 3392 "value_type": "individual" 3393 }, 3394 "type": "graph", 3395 "xaxis": { 3396 "buckets": null, 3397 "mode": "time", 3398 "name": null, 3399 "show": true, 3400 "values": [] 3401 }, 3402 "yaxes": [ 3403 { 3404 "decimals": 1, 3405 "format": "s", 3406 "label": null, 3407 "logBase": 1, 3408 "max": null, 3409 "min": 0, 3410 "show": true 3411 }, 3412 { 3413 "format": "short", 3414 "label": null, 3415 "logBase": 1, 3416 "max": null, 3417 "min": null, 3418 "show": false 3419 } 3420 ], 3421 "yaxis": { 3422 "align": false, 3423 "alignLevel": null 3424 } 3425 }, 3426 { 3427 "aliasColors": {}, 3428 "bars": false, 3429 "dashLength": 10, 3430 "dashes": false, 3431 "datasource": "${DS_TEST-CLUSTER}", 3432 "decimals": 1, 3433 "description": "The number of unsynced tables in the subtask", 3434 "fieldConfig": { 3435 "defaults": {}, 3436 "overrides": [] 3437 }, 3438 "fill": 1, 3439 "fillGradient": 0, 3440 "gridPos": { 3441 "h": 8, 3442 "w": 8, 3443 "x": 0, 3444 "y": 77 3445 }, 3446 "hiddenSeries": false, 3447 "id": 45, 3448 "legend": { 3449 "alignAsTable": true, 3450 "avg": false, 3451 "current": true, 3452 "max": true, 3453 "min": false, 3454 "show": true, 3455 "total": false, 3456 "values": true 3457 }, 3458 "lines": true, 3459 "linewidth": 1, 3460 "links": [], 3461 "nullPointMode": "null", 3462 "options": { 3463 "alertThreshold": true 3464 }, 3465 "percentage": false, 3466 "pluginVersion": "7.5.11", 3467 "pointradius": 5, 3468 "points": false, 3469 "renderer": "flot", 3470 "seriesOverrides": [], 3471 "spaceLength": 10, 3472 "stack": false, 3473 "steppedLine": false, 3474 "targets": [ 3475 { 3476 "expr": "dm_syncer_unsynced_table_number{job_id=~\"$job_id\",source_id=~\"$source\", task=~\"$task\"}", 3477 "format": "time_series", 3478 "instant": false, 3479 "interval": "$interval", 3480 "intervalFactor": 2, 3481 "legendFormat": "{{task}} - {{source_id}}", 3482 "refId": "A" 3483 } 3484 ], 3485 "thresholds": [], 3486 "timeFrom": null, 3487 "timeRegions": [], 3488 "timeShift": null, 3489 "title": "unsynced tables", 3490 "tooltip": { 3491 "shared": true, 3492 "sort": 0, 3493 "value_type": "individual" 3494 }, 3495 "type": "graph", 3496 "xaxis": { 3497 "buckets": null, 3498 "mode": "time", 3499 "name": null, 3500 "show": true, 3501 "values": [] 3502 }, 3503 "yaxes": [ 3504 { 3505 "decimals": 0, 3506 "format": "short", 3507 "label": null, 3508 "logBase": 1, 3509 "max": null, 3510 "min": null, 3511 "show": true 3512 }, 3513 { 3514 "format": "short", 3515 "label": null, 3516 "logBase": 1, 3517 "max": null, 3518 "min": null, 3519 "show": false 3520 } 3521 ], 3522 "yaxis": { 3523 "align": false, 3524 "alignLevel": null 3525 } 3526 }, 3527 { 3528 "aliasColors": {}, 3529 "bars": false, 3530 "dashLength": 10, 3531 "dashes": false, 3532 "datasource": "${DS_TEST-CLUSTER}", 3533 "decimals": 1, 3534 "description": "The number of binlog files in binlog replication unit that are behind relay", 3535 "fieldConfig": { 3536 "defaults": {}, 3537 "overrides": [] 3538 }, 3539 "fill": 1, 3540 "fillGradient": 0, 3541 "gridPos": { 3542 "h": 8, 3543 "w": 7, 3544 "x": 8, 3545 "y": 77 3546 }, 3547 "hiddenSeries": false, 3548 "id": 44, 3549 "legend": { 3550 "alignAsTable": true, 3551 "avg": false, 3552 "current": true, 3553 "max": true, 3554 "min": false, 3555 "show": true, 3556 "total": false, 3557 "values": true 3558 }, 3559 "lines": true, 3560 "linewidth": 1, 3561 "links": [], 3562 "nullPointMode": "null", 3563 "options": { 3564 "alertThreshold": true 3565 }, 3566 "percentage": false, 3567 "pluginVersion": "7.5.11", 3568 "pointradius": 5, 3569 "points": false, 3570 "renderer": "flot", 3571 "seriesOverrides": [], 3572 "spaceLength": 10, 3573 "stack": false, 3574 "steppedLine": false, 3575 "targets": [ 3576 { 3577 "expr": "dm_relay_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", node=\"relay\"} - ON(instance) group_right dm_syncer_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", task=~\"$task\", node=\"syncer\"}", 3578 "format": "time_series", 3579 "hide": false, 3580 "interval": "$interval", 3581 "intervalFactor": 2, 3582 "legendFormat": "{{instance}} - {{task}}", 3583 "refId": "B" 3584 } 3585 ], 3586 "thresholds": [], 3587 "timeFrom": null, 3588 "timeRegions": [], 3589 "timeShift": null, 3590 "title": "binlog file gap between relay and syncer", 3591 "tooltip": { 3592 "shared": true, 3593 "sort": 0, 3594 "value_type": "individual" 3595 }, 3596 "type": "graph", 3597 "xaxis": { 3598 "buckets": null, 3599 "mode": "time", 3600 "name": null, 3601 "show": true, 3602 "values": [] 3603 }, 3604 "yaxes": [ 3605 { 3606 "decimals": 0, 3607 "format": "short", 3608 "label": null, 3609 "logBase": 1, 3610 "max": null, 3611 "min": null, 3612 "show": true 3613 }, 3614 { 3615 "format": "short", 3616 "label": null, 3617 "logBase": 1, 3618 "max": null, 3619 "min": null, 3620 "show": false 3621 } 3622 ], 3623 "yaxis": { 3624 "align": false, 3625 "alignLevel": null 3626 } 3627 }, 3628 { 3629 "aliasColors": {}, 3630 "bars": false, 3631 "dashLength": 10, 3632 "dashes": false, 3633 "datasource": "${DS_TEST-CLUSTER}", 3634 "decimals": 1, 3635 "description": "The time interval of DM written to disk", 3636 "fieldConfig": { 3637 "defaults": {}, 3638 "overrides": [] 3639 }, 3640 "fill": 1, 3641 "fillGradient": 0, 3642 "gridPos": { 3643 "h": 8, 3644 "w": 9, 3645 "x": 15, 3646 "y": 77 3647 }, 3648 "hiddenSeries": false, 3649 "id": 107, 3650 "legend": { 3651 "alignAsTable": true, 3652 "avg": false, 3653 "current": true, 3654 "max": true, 3655 "min": false, 3656 "rightSide": false, 3657 "show": true, 3658 "total": false, 3659 "values": true 3660 }, 3661 "lines": true, 3662 "linewidth": 1, 3663 "links": [], 3664 "nullPointMode": "null", 3665 "options": { 3666 "alertThreshold": true 3667 }, 3668 "percentage": false, 3669 "pluginVersion": "7.5.11", 3670 "pointradius": 5, 3671 "points": false, 3672 "renderer": "flot", 3673 "seriesOverrides": [], 3674 "spaceLength": 10, 3675 "stack": false, 3676 "steppedLine": false, 3677 "targets": [ 3678 { 3679 "exemplar": true, 3680 "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_flush_checkpoints_time_interval_bucket{job_id=~\"$job_id\",task=~\"$task\"}[$interval])) by (le,task,worker))", 3681 "hide": false, 3682 "interval": "", 3683 "legendFormat": "{{task}}-{{worker}}", 3684 "refId": "B" 3685 } 3686 ], 3687 "thresholds": [], 3688 "timeFrom": null, 3689 "timeRegions": [], 3690 "timeShift": null, 3691 "title": "Flushed Checkpoint Time Interval", 3692 "tooltip": { 3693 "shared": true, 3694 "sort": 0, 3695 "value_type": "individual" 3696 }, 3697 "type": "graph", 3698 "xaxis": { 3699 "buckets": null, 3700 "mode": "time", 3701 "name": null, 3702 "show": true, 3703 "values": [] 3704 }, 3705 "yaxes": [ 3706 { 3707 "$$hashKey": "object:644", 3708 "decimals": 1, 3709 "format": "s", 3710 "label": null, 3711 "logBase": 1, 3712 "max": null, 3713 "min": 0, 3714 "show": true 3715 }, 3716 { 3717 "$$hashKey": "object:645", 3718 "format": "short", 3719 "label": null, 3720 "logBase": 1, 3721 "max": null, 3722 "min": null, 3723 "show": false 3724 } 3725 ], 3726 "yaxis": { 3727 "align": false, 3728 "alignLevel": null 3729 } 3730 }, 3731 { 3732 "aliasColors": {}, 3733 "bars": false, 3734 "dashLength": 10, 3735 "dashes": false, 3736 "datasource": "${DS_TEST-CLUSTER}", 3737 "description": "Is waiting shard DDL lock to be resolved, >0 means waiting", 3738 "fieldConfig": { 3739 "defaults": {}, 3740 "overrides": [] 3741 }, 3742 "fill": 1, 3743 "fillGradient": 0, 3744 "gridPos": { 3745 "h": 8, 3746 "w": 8, 3747 "x": 0, 3748 "y": 85 3749 }, 3750 "hiddenSeries": false, 3751 "id": 46, 3752 "legend": { 3753 "alignAsTable": true, 3754 "avg": false, 3755 "current": true, 3756 "max": true, 3757 "min": false, 3758 "show": true, 3759 "total": false, 3760 "values": true 3761 }, 3762 "lines": true, 3763 "linewidth": 1, 3764 "links": [], 3765 "nullPointMode": "null", 3766 "options": { 3767 "alertThreshold": true 3768 }, 3769 "percentage": false, 3770 "pluginVersion": "7.5.11", 3771 "pointradius": 5, 3772 "points": false, 3773 "renderer": "flot", 3774 "seriesOverrides": [], 3775 "spaceLength": 10, 3776 "stack": false, 3777 "steppedLine": false, 3778 "targets": [ 3779 { 3780 "expr": "dm_syncer_shard_lock_resolving{job_id=~\"$job_id\",source_id=~\"$source\", task=~\"$task\"}", 3781 "format": "time_series", 3782 "instant": false, 3783 "interval": "$interval", 3784 "intervalFactor": 2, 3785 "legendFormat": "{{task}} - {{source}}", 3786 "refId": "A" 3787 } 3788 ], 3789 "thresholds": [], 3790 "timeFrom": null, 3791 "timeRegions": [], 3792 "timeShift": null, 3793 "title": "shard lock resolving", 3794 "tooltip": { 3795 "shared": true, 3796 "sort": 0, 3797 "value_type": "individual" 3798 }, 3799 "type": "graph", 3800 "xaxis": { 3801 "buckets": null, 3802 "mode": "time", 3803 "name": null, 3804 "show": true, 3805 "values": [] 3806 }, 3807 "yaxes": [ 3808 { 3809 "decimals": 0, 3810 "format": "short", 3811 "label": "", 3812 "logBase": 1, 3813 "max": "1", 3814 "min": "0", 3815 "show": true 3816 }, 3817 { 3818 "format": "short", 3819 "label": null, 3820 "logBase": 1, 3821 "max": null, 3822 "min": null, 3823 "show": false 3824 } 3825 ], 3826 "yaxis": { 3827 "align": false, 3828 "alignLevel": null 3829 } 3830 }, 3831 { 3832 "aliasColors": {}, 3833 "bars": false, 3834 "dashLength": 10, 3835 "dashes": false, 3836 "datasource": "${DS_TEST-CLUSTER}", 3837 "decimals": 1, 3838 "description": "The time it takes binlog replication unit to skip a binlog event (in seconds)", 3839 "fieldConfig": { 3840 "defaults": {}, 3841 "overrides": [] 3842 }, 3843 "fill": 1, 3844 "fillGradient": 0, 3845 "gridPos": { 3846 "h": 8, 3847 "w": 7, 3848 "x": 8, 3849 "y": 85 3850 }, 3851 "hiddenSeries": false, 3852 "id": 65, 3853 "legend": { 3854 "alignAsTable": true, 3855 "avg": false, 3856 "current": true, 3857 "max": true, 3858 "min": false, 3859 "rightSide": false, 3860 "show": true, 3861 "total": false, 3862 "values": true 3863 }, 3864 "lines": true, 3865 "linewidth": 1, 3866 "links": [], 3867 "nullPointMode": "null", 3868 "options": { 3869 "alertThreshold": true 3870 }, 3871 "percentage": false, 3872 "pluginVersion": "7.5.11", 3873 "pointradius": 5, 3874 "points": false, 3875 "renderer": "flot", 3876 "seriesOverrides": [], 3877 "spaceLength": 10, 3878 "stack": false, 3879 "steppedLine": false, 3880 "targets": [ 3881 { 3882 "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_skip_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", 3883 "format": "time_series", 3884 "interval": "$interval", 3885 "intervalFactor": 2, 3886 "legendFormat": "{{task}} - {{source_id}} - 90", 3887 "refId": "A" 3888 }, 3889 { 3890 "expr": "histogram_quantile(0.95, sum(rate(dm_syncer_skip_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", 3891 "format": "time_series", 3892 "interval": "$interval", 3893 "intervalFactor": 2, 3894 "legendFormat": "{{task}} - {{source_id}} - 95", 3895 "refId": "B" 3896 }, 3897 { 3898 "expr": "histogram_quantile(0.99, sum(rate(dm_syncer_skip_binlog_duration_bucket{job_id=~\"$job_id\",task=~\"$task\", source_id=~\"$source\"}[$interval])) by (le,task,source_id))", 3899 "format": "time_series", 3900 "interval": "$interval", 3901 "intervalFactor": 2, 3902 "legendFormat": "{{task}} - {{source_id}} - 99", 3903 "refId": "C" 3904 } 3905 ], 3906 "thresholds": [], 3907 "timeFrom": null, 3908 "timeRegions": [], 3909 "timeShift": null, 3910 "title": "skip event duration", 3911 "tooltip": { 3912 "shared": true, 3913 "sort": 0, 3914 "value_type": "individual" 3915 }, 3916 "type": "graph", 3917 "xaxis": { 3918 "buckets": null, 3919 "mode": "time", 3920 "name": null, 3921 "show": true, 3922 "values": [] 3923 }, 3924 "yaxes": [ 3925 { 3926 "decimals": 1, 3927 "format": "s", 3928 "label": null, 3929 "logBase": 1, 3930 "max": null, 3931 "min": 0, 3932 "show": true 3933 }, 3934 { 3935 "format": "short", 3936 "label": null, 3937 "logBase": 1, 3938 "max": null, 3939 "min": null, 3940 "show": false 3941 } 3942 ], 3943 "yaxis": { 3944 "align": false, 3945 "alignLevel": null 3946 } 3947 }, 3948 { 3949 "aliasColors": {}, 3950 "bars": false, 3951 "dashLength": 10, 3952 "dashes": false, 3953 "datasource": "${DS_TEST-CLUSTER}", 3954 "decimals": 1, 3955 "description": "Display the number of sql contained in the dm copy incremental data to the downstream transaction from each latitude", 3956 "fieldConfig": { 3957 "defaults": {}, 3958 "overrides": [] 3959 }, 3960 "fill": 1, 3961 "fillGradient": 0, 3962 "gridPos": { 3963 "h": 8, 3964 "w": 9, 3965 "x": 15, 3966 "y": 85 3967 }, 3968 "hiddenSeries": false, 3969 "id": 105, 3970 "legend": { 3971 "alignAsTable": true, 3972 "avg": false, 3973 "current": true, 3974 "max": true, 3975 "min": false, 3976 "rightSide": false, 3977 "show": true, 3978 "total": false, 3979 "values": true 3980 }, 3981 "lines": true, 3982 "linewidth": 1, 3983 "links": [], 3984 "nullPointMode": "null", 3985 "options": { 3986 "alertThreshold": true 3987 }, 3988 "percentage": false, 3989 "pluginVersion": "7.5.11", 3990 "pointradius": 5, 3991 "points": false, 3992 "renderer": "flot", 3993 "seriesOverrides": [], 3994 "spaceLength": 10, 3995 "stack": false, 3996 "steppedLine": false, 3997 "targets": [ 3998 { 3999 "exemplar": true, 4000 "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_replication_transaction_batch_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task,queueNo,type))", 4001 "hide": false, 4002 "interval": "", 4003 "legendFormat": "{{task}} - {{queueNo}} - {{type}}", 4004 "refId": "B" 4005 } 4006 ], 4007 "thresholds": [], 4008 "timeFrom": null, 4009 "timeRegions": [], 4010 "timeShift": null, 4011 "title": "replication transaction batch", 4012 "tooltip": { 4013 "shared": true, 4014 "sort": 0, 4015 "value_type": "individual" 4016 }, 4017 "type": "graph", 4018 "xaxis": { 4019 "buckets": null, 4020 "mode": "time", 4021 "name": null, 4022 "show": true, 4023 "values": [] 4024 }, 4025 "yaxes": [ 4026 { 4027 "$$hashKey": "object:581", 4028 "decimals": 1, 4029 "format": "short", 4030 "label": null, 4031 "logBase": 1, 4032 "max": null, 4033 "min": 0, 4034 "show": true 4035 }, 4036 { 4037 "$$hashKey": "object:582", 4038 "format": "short", 4039 "label": null, 4040 "logBase": 1, 4041 "max": null, 4042 "min": null, 4043 "show": false 4044 } 4045 ], 4046 "yaxis": { 4047 "align": false, 4048 "alignLevel": null 4049 } 4050 }, 4051 { 4052 "aliasColors": {}, 4053 "bars": false, 4054 "dashLength": 10, 4055 "dashes": false, 4056 "datasource": "${DS_TEST-CLUSTER}", 4057 "decimals": 1, 4058 "description": "", 4059 "fieldConfig": { 4060 "defaults": {}, 4061 "overrides": [] 4062 }, 4063 "fill": 1, 4064 "fillGradient": 0, 4065 "gridPos": { 4066 "h": 8, 4067 "w": 15, 4068 "x": 0, 4069 "y": 93 4070 }, 4071 "hiddenSeries": false, 4072 "id": 109, 4073 "legend": { 4074 "alignAsTable": true, 4075 "avg": false, 4076 "current": true, 4077 "max": true, 4078 "min": true, 4079 "rightSide": true, 4080 "show": true, 4081 "total": false, 4082 "values": true 4083 }, 4084 "lines": true, 4085 "linewidth": 1, 4086 "links": [], 4087 "nullPointMode": "null", 4088 "options": { 4089 "alertThreshold": true 4090 }, 4091 "percentage": false, 4092 "pluginVersion": "7.5.11", 4093 "pointradius": 5, 4094 "points": false, 4095 "renderer": "flot", 4096 "seriesOverrides": [], 4097 "spaceLength": 10, 4098 "stack": false, 4099 "steppedLine": false, 4100 "targets": [ 4101 { 4102 "exemplar": true, 4103 "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", 4104 "format": "time_series", 4105 "interval": "$interval", 4106 "intervalFactor": 2, 4107 "legendFormat": "{{task}} - {{source_id}} - {{instance}} - 90", 4108 "refId": "A" 4109 }, 4110 { 4111 "exemplar": true, 4112 "expr": "histogram_quantile(0.95, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", 4113 "format": "time_series", 4114 "hide": false, 4115 "interval": "$interval", 4116 "intervalFactor": 2, 4117 "legendFormat": "{{task}} - {{source_id}} - {{instance}} - 95", 4118 "refId": "B" 4119 }, 4120 { 4121 "exemplar": true, 4122 "expr": "histogram_quantile(0.99, sum(rate(dm_syncer_binlog_transform_cost_bucket{job_id=~\"$job_id\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", 4123 "format": "time_series", 4124 "hide": false, 4125 "interval": "$interval", 4126 "intervalFactor": 2, 4127 "legendFormat": "{{task}} - {{source_id}} - {{instance}} - 99", 4128 "refId": "C" 4129 } 4130 ], 4131 "thresholds": [], 4132 "timeFrom": null, 4133 "timeRegions": [], 4134 "timeShift": null, 4135 "title": " transform binlog event duration", 4136 "tooltip": { 4137 "shared": true, 4138 "sort": 0, 4139 "value_type": "individual" 4140 }, 4141 "type": "graph", 4142 "xaxis": { 4143 "buckets": null, 4144 "mode": "time", 4145 "name": null, 4146 "show": true, 4147 "values": [] 4148 }, 4149 "yaxes": [ 4150 { 4151 "$$hashKey": "object:659", 4152 "decimals": 1, 4153 "format": "short", 4154 "label": null, 4155 "logBase": 1, 4156 "max": null, 4157 "min": 0, 4158 "show": true 4159 }, 4160 { 4161 "$$hashKey": "object:660", 4162 "format": "short", 4163 "label": null, 4164 "logBase": 1, 4165 "max": null, 4166 "min": null, 4167 "show": false 4168 } 4169 ], 4170 "yaxis": { 4171 "align": false, 4172 "alignLevel": null 4173 } 4174 } 4175 ], 4176 "repeat": null, 4177 "title": "binlog replication", 4178 "type": "row" 4179 }, 4180 { 4181 "collapsed": true, 4182 "datasource": "${DS_TEST-CLUSTER}", 4183 "gridPos": { 4184 "h": 1, 4185 "w": 24, 4186 "x": 0, 4187 "y": 9 4188 }, 4189 "id": 55, 4190 "panels": [ 4191 { 4192 "aliasColors": {}, 4193 "bars": false, 4194 "cacheTimeout": null, 4195 "dashLength": 10, 4196 "dashes": false, 4197 "datasource": "${DS_TEST-CLUSTER}", 4198 "decimals": 1, 4199 "description": "The storage capacity of the disk occupied by the relay log", 4200 "fieldConfig": { 4201 "defaults": {}, 4202 "overrides": [] 4203 }, 4204 "fill": 1, 4205 "fillGradient": 0, 4206 "gridPos": { 4207 "h": 7, 4208 "w": 6, 4209 "x": 0, 4210 "y": 11 4211 }, 4212 "hiddenSeries": false, 4213 "id": 20, 4214 "legend": { 4215 "alignAsTable": true, 4216 "avg": false, 4217 "current": true, 4218 "max": true, 4219 "min": false, 4220 "show": true, 4221 "total": false, 4222 "values": true 4223 }, 4224 "lines": true, 4225 "linewidth": 1, 4226 "links": [], 4227 "nullPointMode": "null", 4228 "options": { 4229 "alertThreshold": true 4230 }, 4231 "percentage": false, 4232 "pluginVersion": "7.5.11", 4233 "pointradius": 2, 4234 "points": false, 4235 "renderer": "flot", 4236 "seriesOverrides": [], 4237 "spaceLength": 10, 4238 "stack": false, 4239 "steppedLine": false, 4240 "targets": [ 4241 { 4242 "expr": "dm_relay_space{job_id=~\"$job_id\",instance=~\"$instance\", type=\"capacity\"}", 4243 "format": "time_series", 4244 "interval": "$interval", 4245 "intervalFactor": 2, 4246 "legendFormat": "{{instance}}", 4247 "refId": "A" 4248 } 4249 ], 4250 "thresholds": [], 4251 "timeFrom": null, 4252 "timeRegions": [], 4253 "timeShift": null, 4254 "title": "storage capacity", 4255 "tooltip": { 4256 "shared": true, 4257 "sort": 0, 4258 "value_type": "individual" 4259 }, 4260 "type": "graph", 4261 "xaxis": { 4262 "buckets": null, 4263 "mode": "time", 4264 "name": null, 4265 "show": true, 4266 "values": [] 4267 }, 4268 "yaxes": [ 4269 { 4270 "decimals": 1, 4271 "format": "bytes", 4272 "label": null, 4273 "logBase": 1, 4274 "max": null, 4275 "min": "0", 4276 "show": true 4277 }, 4278 { 4279 "format": "short", 4280 "label": null, 4281 "logBase": 1, 4282 "max": null, 4283 "min": null, 4284 "show": true 4285 } 4286 ], 4287 "yaxis": { 4288 "align": false, 4289 "alignLevel": null 4290 } 4291 }, 4292 { 4293 "aliasColors": {}, 4294 "bars": false, 4295 "cacheTimeout": null, 4296 "dashLength": 10, 4297 "dashes": false, 4298 "datasource": "${DS_TEST-CLUSTER}", 4299 "decimals": 1, 4300 "description": "The remaining storage capacity of the disk occupied by the relay log", 4301 "fieldConfig": { 4302 "defaults": {}, 4303 "overrides": [] 4304 }, 4305 "fill": 1, 4306 "fillGradient": 0, 4307 "gridPos": { 4308 "h": 7, 4309 "w": 6, 4310 "x": 6, 4311 "y": 11 4312 }, 4313 "hiddenSeries": false, 4314 "id": 21, 4315 "legend": { 4316 "alignAsTable": true, 4317 "avg": false, 4318 "current": true, 4319 "max": true, 4320 "min": false, 4321 "show": true, 4322 "total": false, 4323 "values": true 4324 }, 4325 "lines": true, 4326 "linewidth": 1, 4327 "links": [], 4328 "nullPointMode": "null", 4329 "options": { 4330 "alertThreshold": true 4331 }, 4332 "percentage": false, 4333 "pluginVersion": "7.5.11", 4334 "pointradius": 2, 4335 "points": false, 4336 "renderer": "flot", 4337 "seriesOverrides": [], 4338 "spaceLength": 10, 4339 "stack": false, 4340 "steppedLine": false, 4341 "targets": [ 4342 { 4343 "expr": "dm_relay_space{job_id=~\"$job_id\",instance=~\"$instance\", type=\"available\"}", 4344 "format": "time_series", 4345 "interval": "$interval", 4346 "intervalFactor": 2, 4347 "legendFormat": "{{instance}}", 4348 "refId": "A" 4349 } 4350 ], 4351 "thresholds": [], 4352 "timeFrom": null, 4353 "timeRegions": [], 4354 "timeShift": null, 4355 "title": "storage remain", 4356 "tooltip": { 4357 "shared": true, 4358 "sort": 0, 4359 "value_type": "individual" 4360 }, 4361 "type": "graph", 4362 "xaxis": { 4363 "buckets": null, 4364 "mode": "time", 4365 "name": null, 4366 "show": true, 4367 "values": [] 4368 }, 4369 "yaxes": [ 4370 { 4371 "decimals": 1, 4372 "format": "bytes", 4373 "label": null, 4374 "logBase": 1, 4375 "max": null, 4376 "min": "0", 4377 "show": true 4378 }, 4379 { 4380 "format": "short", 4381 "label": null, 4382 "logBase": 1, 4383 "max": null, 4384 "min": null, 4385 "show": true 4386 } 4387 ], 4388 "yaxis": { 4389 "align": false, 4390 "alignLevel": null 4391 } 4392 }, 4393 { 4394 "aliasColors": {}, 4395 "bars": false, 4396 "cacheTimeout": null, 4397 "dashLength": 10, 4398 "dashes": false, 4399 "datasource": "${DS_TEST-CLUSTER}", 4400 "decimals": 0, 4401 "description": "The relay log encounters an error within the DM-worker and exits", 4402 "fieldConfig": { 4403 "defaults": {}, 4404 "overrides": [] 4405 }, 4406 "fill": 1, 4407 "fillGradient": 0, 4408 "gridPos": { 4409 "h": 7, 4410 "w": 6, 4411 "x": 12, 4412 "y": 11 4413 }, 4414 "hiddenSeries": false, 4415 "id": 30, 4416 "legend": { 4417 "alignAsTable": true, 4418 "avg": false, 4419 "current": true, 4420 "max": true, 4421 "min": false, 4422 "show": true, 4423 "total": false, 4424 "values": true 4425 }, 4426 "lines": true, 4427 "linewidth": 1, 4428 "links": [], 4429 "nullPointMode": "null as zero", 4430 "options": { 4431 "alertThreshold": true 4432 }, 4433 "percentage": false, 4434 "pluginVersion": "7.5.11", 4435 "pointradius": 2, 4436 "points": false, 4437 "renderer": "flot", 4438 "seriesOverrides": [], 4439 "spaceLength": 10, 4440 "stack": false, 4441 "steppedLine": false, 4442 "targets": [ 4443 { 4444 "expr": "dm_relay_exit_with_error_count{job_id=~\"$job_id\",instance=~\"$instance\"}", 4445 "format": "time_series", 4446 "interval": "$interval", 4447 "intervalFactor": 2, 4448 "legendFormat": " {{instance}}", 4449 "refId": "A" 4450 } 4451 ], 4452 "thresholds": [], 4453 "timeFrom": null, 4454 "timeRegions": [], 4455 "timeShift": null, 4456 "title": "process exits with error", 4457 "tooltip": { 4458 "shared": true, 4459 "sort": 0, 4460 "value_type": "individual" 4461 }, 4462 "type": "graph", 4463 "xaxis": { 4464 "buckets": null, 4465 "mode": "time", 4466 "name": null, 4467 "show": true, 4468 "values": [] 4469 }, 4470 "yaxes": [ 4471 { 4472 "decimals": 0, 4473 "format": "short", 4474 "label": null, 4475 "logBase": 1, 4476 "max": null, 4477 "min": "0", 4478 "show": true 4479 }, 4480 { 4481 "decimals": null, 4482 "format": "short", 4483 "label": null, 4484 "logBase": 1, 4485 "max": null, 4486 "min": null, 4487 "show": true 4488 } 4489 ], 4490 "yaxis": { 4491 "align": false, 4492 "alignLevel": null 4493 } 4494 }, 4495 { 4496 "aliasColors": {}, 4497 "bars": false, 4498 "cacheTimeout": null, 4499 "dashLength": 10, 4500 "dashes": false, 4501 "datasource": "${DS_TEST-CLUSTER}", 4502 "decimals": null, 4503 "description": "The number of corrupted relay log files", 4504 "fieldConfig": { 4505 "defaults": {}, 4506 "overrides": [] 4507 }, 4508 "fill": 1, 4509 "fillGradient": 0, 4510 "gridPos": { 4511 "h": 7, 4512 "w": 6, 4513 "x": 18, 4514 "y": 11 4515 }, 4516 "hiddenSeries": false, 4517 "id": 31, 4518 "legend": { 4519 "alignAsTable": true, 4520 "avg": false, 4521 "current": true, 4522 "max": true, 4523 "min": false, 4524 "show": true, 4525 "total": false, 4526 "values": true 4527 }, 4528 "lines": true, 4529 "linewidth": 1, 4530 "links": [], 4531 "nullPointMode": "null", 4532 "options": { 4533 "alertThreshold": true 4534 }, 4535 "percentage": false, 4536 "pluginVersion": "7.5.11", 4537 "pointradius": 2, 4538 "points": false, 4539 "renderer": "flot", 4540 "seriesOverrides": [], 4541 "spaceLength": 10, 4542 "stack": false, 4543 "steppedLine": false, 4544 "targets": [ 4545 { 4546 "expr": "dm_relay_data_corruption{job_id=~\"$job_id\",instance=~\"$instance\"}", 4547 "format": "time_series", 4548 "interval": "$interval", 4549 "intervalFactor": 2, 4550 "legendFormat": "{{instance}}", 4551 "refId": "A" 4552 } 4553 ], 4554 "thresholds": [], 4555 "timeFrom": null, 4556 "timeRegions": [], 4557 "timeShift": null, 4558 "title": "relay log data corruption", 4559 "tooltip": { 4560 "shared": true, 4561 "sort": 0, 4562 "value_type": "individual" 4563 }, 4564 "type": "graph", 4565 "xaxis": { 4566 "buckets": null, 4567 "mode": "time", 4568 "name": null, 4569 "show": true, 4570 "values": [] 4571 }, 4572 "yaxes": [ 4573 { 4574 "format": "short", 4575 "label": "", 4576 "logBase": 1, 4577 "max": null, 4578 "min": "0", 4579 "show": true 4580 }, 4581 { 4582 "format": "short", 4583 "label": null, 4584 "logBase": 1, 4585 "max": null, 4586 "min": null, 4587 "show": true 4588 } 4589 ], 4590 "yaxis": { 4591 "align": false, 4592 "alignLevel": null 4593 } 4594 }, 4595 { 4596 "aliasColors": {}, 4597 "bars": false, 4598 "cacheTimeout": null, 4599 "dashLength": 10, 4600 "dashes": false, 4601 "datasource": "${DS_TEST-CLUSTER}", 4602 "description": "The number of errors encountered when the relay log reads the binlog from the upstream MySQL", 4603 "fieldConfig": { 4604 "defaults": {}, 4605 "overrides": [] 4606 }, 4607 "fill": 1, 4608 "fillGradient": 0, 4609 "gridPos": { 4610 "h": 7, 4611 "w": 6, 4612 "x": 0, 4613 "y": 18 4614 }, 4615 "hiddenSeries": false, 4616 "id": 28, 4617 "legend": { 4618 "alignAsTable": true, 4619 "avg": false, 4620 "current": true, 4621 "max": true, 4622 "min": false, 4623 "show": true, 4624 "total": false, 4625 "values": true 4626 }, 4627 "lines": true, 4628 "linewidth": 1, 4629 "links": [], 4630 "nullPointMode": "null", 4631 "options": { 4632 "alertThreshold": true 4633 }, 4634 "percentage": false, 4635 "pluginVersion": "7.5.11", 4636 "pointradius": 2, 4637 "points": false, 4638 "renderer": "flot", 4639 "seriesOverrides": [], 4640 "spaceLength": 10, 4641 "stack": false, 4642 "steppedLine": false, 4643 "targets": [ 4644 { 4645 "expr": "dm_relay_read_error_count{job_id=~\"$job_id\",instance=~\"$instance\"}", 4646 "format": "time_series", 4647 "interval": "$interval", 4648 "intervalFactor": 2, 4649 "legendFormat": "{{instance}}", 4650 "refId": "A" 4651 } 4652 ], 4653 "thresholds": [], 4654 "timeFrom": null, 4655 "timeRegions": [], 4656 "timeShift": null, 4657 "title": "fail to read binlog from master", 4658 "tooltip": { 4659 "shared": true, 4660 "sort": 0, 4661 "value_type": "individual" 4662 }, 4663 "type": "graph", 4664 "xaxis": { 4665 "buckets": null, 4666 "mode": "time", 4667 "name": null, 4668 "show": true, 4669 "values": [] 4670 }, 4671 "yaxes": [ 4672 { 4673 "decimals": null, 4674 "format": "short", 4675 "label": "", 4676 "logBase": 1, 4677 "max": null, 4678 "min": "0", 4679 "show": true 4680 }, 4681 { 4682 "format": "short", 4683 "label": null, 4684 "logBase": 1, 4685 "max": null, 4686 "min": null, 4687 "show": true 4688 } 4689 ], 4690 "yaxis": { 4691 "align": false, 4692 "alignLevel": null 4693 } 4694 }, 4695 { 4696 "aliasColors": {}, 4697 "bars": false, 4698 "cacheTimeout": null, 4699 "dashLength": 10, 4700 "dashes": false, 4701 "datasource": "${DS_TEST-CLUSTER}", 4702 "description": "The number of errors encountered when the relay log writes the binlog to disks", 4703 "fieldConfig": { 4704 "defaults": {}, 4705 "overrides": [] 4706 }, 4707 "fill": 1, 4708 "fillGradient": 0, 4709 "gridPos": { 4710 "h": 7, 4711 "w": 6, 4712 "x": 6, 4713 "y": 18 4714 }, 4715 "hiddenSeries": false, 4716 "id": 29, 4717 "legend": { 4718 "alignAsTable": true, 4719 "avg": false, 4720 "current": true, 4721 "max": true, 4722 "min": false, 4723 "show": true, 4724 "total": false, 4725 "values": true 4726 }, 4727 "lines": true, 4728 "linewidth": 1, 4729 "links": [], 4730 "nullPointMode": "null", 4731 "options": { 4732 "alertThreshold": true 4733 }, 4734 "percentage": false, 4735 "pluginVersion": "7.5.11", 4736 "pointradius": 2, 4737 "points": false, 4738 "renderer": "flot", 4739 "seriesOverrides": [], 4740 "spaceLength": 10, 4741 "stack": false, 4742 "steppedLine": false, 4743 "targets": [ 4744 { 4745 "expr": "dm_relay_write_error_count{job_id=~\"$job_id\",instance=~\"$instance\"}", 4746 "format": "time_series", 4747 "interval": "$interval", 4748 "intervalFactor": 2, 4749 "legendFormat": "{{instance}}", 4750 "refId": "A" 4751 } 4752 ], 4753 "thresholds": [], 4754 "timeFrom": null, 4755 "timeRegions": [], 4756 "timeShift": null, 4757 "title": "fail to write relay log", 4758 "tooltip": { 4759 "shared": true, 4760 "sort": 0, 4761 "value_type": "individual" 4762 }, 4763 "type": "graph", 4764 "xaxis": { 4765 "buckets": null, 4766 "mode": "time", 4767 "name": null, 4768 "show": true, 4769 "values": [] 4770 }, 4771 "yaxes": [ 4772 { 4773 "format": "short", 4774 "label": null, 4775 "logBase": 1, 4776 "max": null, 4777 "min": null, 4778 "show": true 4779 }, 4780 { 4781 "format": "short", 4782 "label": null, 4783 "logBase": 1, 4784 "max": null, 4785 "min": null, 4786 "show": true 4787 } 4788 ], 4789 "yaxis": { 4790 "align": false, 4791 "alignLevel": null 4792 } 4793 }, 4794 { 4795 "aliasColors": {}, 4796 "bars": false, 4797 "dashLength": 10, 4798 "dashes": false, 4799 "datasource": "${DS_TEST-CLUSTER}", 4800 "description": "The largest index number of relay log files. E.g. \"value = 1\" indicates \"mysql-bin.000001\"", 4801 "fieldConfig": { 4802 "defaults": {}, 4803 "overrides": [] 4804 }, 4805 "fill": 1, 4806 "fillGradient": 0, 4807 "gridPos": { 4808 "h": 7, 4809 "w": 6, 4810 "x": 12, 4811 "y": 18 4812 }, 4813 "hiddenSeries": false, 4814 "id": 7, 4815 "legend": { 4816 "alignAsTable": true, 4817 "avg": false, 4818 "current": false, 4819 "max": false, 4820 "min": false, 4821 "rightSide": true, 4822 "show": true, 4823 "total": false, 4824 "values": false 4825 }, 4826 "lines": true, 4827 "linewidth": 1, 4828 "links": [], 4829 "nullPointMode": "null", 4830 "options": { 4831 "alertThreshold": true 4832 }, 4833 "percentage": false, 4834 "pluginVersion": "7.5.11", 4835 "pointradius": 5, 4836 "points": false, 4837 "renderer": "flot", 4838 "seriesOverrides": [], 4839 "spaceLength": 10, 4840 "stack": false, 4841 "steppedLine": false, 4842 "targets": [ 4843 { 4844 "expr": "dm_relay_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", node=\"master\"}", 4845 "format": "time_series", 4846 "interval": "$interval", 4847 "intervalFactor": 2, 4848 "legendFormat": "master", 4849 "refId": "A" 4850 }, 4851 { 4852 "expr": "dm_relay_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", node=\"relay\"}", 4853 "format": "time_series", 4854 "interval": "$interval", 4855 "intervalFactor": 2, 4856 "legendFormat": "relay", 4857 "refId": "B" 4858 } 4859 ], 4860 "thresholds": [], 4861 "timeFrom": null, 4862 "timeRegions": [], 4863 "timeShift": null, 4864 "title": "binlog file index", 4865 "tooltip": { 4866 "shared": true, 4867 "sort": 0, 4868 "value_type": "individual" 4869 }, 4870 "type": "graph", 4871 "xaxis": { 4872 "buckets": null, 4873 "mode": "time", 4874 "name": null, 4875 "show": true, 4876 "values": [] 4877 }, 4878 "yaxes": [ 4879 { 4880 "decimals": 0, 4881 "format": "short", 4882 "label": null, 4883 "logBase": 1, 4884 "max": null, 4885 "min": null, 4886 "show": true 4887 }, 4888 { 4889 "format": "short", 4890 "label": null, 4891 "logBase": 1, 4892 "max": null, 4893 "min": null, 4894 "show": false 4895 } 4896 ], 4897 "yaxis": { 4898 "align": false, 4899 "alignLevel": null 4900 } 4901 }, 4902 { 4903 "aliasColors": {}, 4904 "bars": false, 4905 "dashLength": 10, 4906 "dashes": false, 4907 "datasource": "${DS_TEST-CLUSTER}", 4908 "description": "The number of binlog files in the relay log that are behind the upstream master", 4909 "fieldConfig": { 4910 "defaults": {}, 4911 "overrides": [] 4912 }, 4913 "fill": 1, 4914 "fillGradient": 0, 4915 "gridPos": { 4916 "h": 7, 4917 "w": 6, 4918 "x": 18, 4919 "y": 18 4920 }, 4921 "hiddenSeries": false, 4922 "id": 42, 4923 "legend": { 4924 "avg": false, 4925 "current": false, 4926 "max": false, 4927 "min": false, 4928 "show": false, 4929 "total": false, 4930 "values": false 4931 }, 4932 "lines": true, 4933 "linewidth": 1, 4934 "links": [], 4935 "nullPointMode": "null", 4936 "options": { 4937 "alertThreshold": true 4938 }, 4939 "percentage": false, 4940 "pluginVersion": "7.5.11", 4941 "pointradius": 5, 4942 "points": false, 4943 "renderer": "flot", 4944 "seriesOverrides": [], 4945 "spaceLength": 10, 4946 "stack": false, 4947 "steppedLine": false, 4948 "targets": [ 4949 { 4950 "expr": "dm_relay_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", node=\"master\"} - ON(instance) dm_relay_binlog_file{job_id=~\"$job_id\",instance=~\"$instance\", node=\"relay\"}", 4951 "format": "time_series", 4952 "hide": false, 4953 "interval": "$interval", 4954 "intervalFactor": 2, 4955 "legendFormat": "{{instance}}", 4956 "refId": "B" 4957 } 4958 ], 4959 "thresholds": [], 4960 "timeFrom": null, 4961 "timeRegions": [], 4962 "timeShift": null, 4963 "title": "binlog file gap between master and relay", 4964 "tooltip": { 4965 "shared": true, 4966 "sort": 0, 4967 "value_type": "individual" 4968 }, 4969 "type": "graph", 4970 "xaxis": { 4971 "buckets": null, 4972 "mode": "time", 4973 "name": null, 4974 "show": true, 4975 "values": [] 4976 }, 4977 "yaxes": [ 4978 { 4979 "decimals": 0, 4980 "format": "short", 4981 "label": null, 4982 "logBase": 1, 4983 "max": null, 4984 "min": null, 4985 "show": true 4986 }, 4987 { 4988 "format": "short", 4989 "label": null, 4990 "logBase": 1, 4991 "max": null, 4992 "min": null, 4993 "show": false 4994 } 4995 ], 4996 "yaxis": { 4997 "align": false, 4998 "alignLevel": null 4999 } 5000 }, 5001 { 5002 "aliasColors": {}, 5003 "bars": false, 5004 "dashLength": 10, 5005 "dashes": false, 5006 "datasource": "${DS_TEST-CLUSTER}", 5007 "description": "The write offset of the latest relay log file", 5008 "fieldConfig": { 5009 "defaults": {}, 5010 "overrides": [] 5011 }, 5012 "fill": 1, 5013 "fillGradient": 0, 5014 "gridPos": { 5015 "h": 7, 5016 "w": 6, 5017 "x": 0, 5018 "y": 25 5019 }, 5020 "hiddenSeries": false, 5021 "id": 6, 5022 "legend": { 5023 "alignAsTable": true, 5024 "avg": false, 5025 "current": true, 5026 "max": true, 5027 "min": false, 5028 "rightSide": false, 5029 "show": true, 5030 "total": false, 5031 "values": true 5032 }, 5033 "lines": true, 5034 "linewidth": 1, 5035 "links": [], 5036 "nullPointMode": "null", 5037 "options": { 5038 "alertThreshold": true 5039 }, 5040 "percentage": false, 5041 "pluginVersion": "7.5.11", 5042 "pointradius": 5, 5043 "points": false, 5044 "renderer": "flot", 5045 "seriesOverrides": [], 5046 "spaceLength": 10, 5047 "stack": false, 5048 "steppedLine": false, 5049 "targets": [ 5050 { 5051 "expr": "dm_relay_binlog_pos{job_id=~\"$job_id\",instance=~\"$instance\", node=\"master\"}", 5052 "format": "time_series", 5053 "interval": "$interval", 5054 "intervalFactor": 2, 5055 "legendFormat": "master - {{instance}}", 5056 "refId": "A" 5057 }, 5058 { 5059 "expr": "dm_relay_binlog_pos{job_id=~\"$job_id\",instance=~\"$instance\", node=\"relay\"}", 5060 "format": "time_series", 5061 "interval": "$interval", 5062 "intervalFactor": 2, 5063 "legendFormat": "relay - {{instance}}", 5064 "refId": "B" 5065 } 5066 ], 5067 "thresholds": [], 5068 "timeFrom": null, 5069 "timeRegions": [], 5070 "timeShift": null, 5071 "title": "binlog pos", 5072 "tooltip": { 5073 "shared": true, 5074 "sort": 0, 5075 "value_type": "individual" 5076 }, 5077 "type": "graph", 5078 "xaxis": { 5079 "buckets": null, 5080 "mode": "time", 5081 "name": null, 5082 "show": true, 5083 "values": [] 5084 }, 5085 "yaxes": [ 5086 { 5087 "decimals": 0, 5088 "format": "short", 5089 "label": null, 5090 "logBase": 1, 5091 "max": null, 5092 "min": null, 5093 "show": true 5094 }, 5095 { 5096 "format": "short", 5097 "label": null, 5098 "logBase": 1, 5099 "max": null, 5100 "min": null, 5101 "show": false 5102 } 5103 ], 5104 "yaxis": { 5105 "align": false, 5106 "alignLevel": null 5107 } 5108 }, 5109 { 5110 "aliasColors": {}, 5111 "bars": false, 5112 "dashLength": 10, 5113 "dashes": false, 5114 "datasource": "${DS_TEST-CLUSTER}", 5115 "description": "The duration that the relay log reads binlog event from the upstream MySQL (in seconds)", 5116 "fieldConfig": { 5117 "defaults": {}, 5118 "overrides": [] 5119 }, 5120 "fill": 1, 5121 "fillGradient": 0, 5122 "gridPos": { 5123 "h": 7, 5124 "w": 6, 5125 "x": 6, 5126 "y": 25 5127 }, 5128 "hiddenSeries": false, 5129 "id": 9, 5130 "legend": { 5131 "alignAsTable": true, 5132 "avg": false, 5133 "current": false, 5134 "max": false, 5135 "min": false, 5136 "rightSide": true, 5137 "show": true, 5138 "total": false, 5139 "values": false 5140 }, 5141 "lines": true, 5142 "linewidth": 1, 5143 "links": [], 5144 "nullPointMode": "null", 5145 "options": { 5146 "alertThreshold": true 5147 }, 5148 "percentage": false, 5149 "pluginVersion": "7.5.11", 5150 "pointradius": 5, 5151 "points": false, 5152 "renderer": "flot", 5153 "seriesOverrides": [], 5154 "spaceLength": 10, 5155 "stack": false, 5156 "steppedLine": false, 5157 "targets": [ 5158 { 5159 "expr": "histogram_quantile(0.90, sum(rate(dm_relay_read_binlog_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[1m])) by (le))", 5160 "format": "time_series", 5161 "interval": "", 5162 "intervalFactor": 2, 5163 "legendFormat": "90", 5164 "refId": "A" 5165 }, 5166 { 5167 "expr": "histogram_quantile(0.95, sum(rate(dm_relay_read_binlog_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[1m])) by (le))", 5168 "format": "time_series", 5169 "intervalFactor": 2, 5170 "legendFormat": "95", 5171 "refId": "B" 5172 }, 5173 { 5174 "expr": "histogram_quantile(0.99, sum(rate(dm_relay_read_binlog_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[1m])) by (le))", 5175 "format": "time_series", 5176 "intervalFactor": 2, 5177 "legendFormat": "99", 5178 "refId": "C" 5179 } 5180 ], 5181 "thresholds": [], 5182 "timeFrom": null, 5183 "timeRegions": [], 5184 "timeShift": null, 5185 "title": "read binlog event duration", 5186 "tooltip": { 5187 "shared": true, 5188 "sort": 0, 5189 "value_type": "individual" 5190 }, 5191 "type": "graph", 5192 "xaxis": { 5193 "buckets": null, 5194 "mode": "time", 5195 "name": null, 5196 "show": true, 5197 "values": [] 5198 }, 5199 "yaxes": [ 5200 { 5201 "decimals": 2, 5202 "format": "s", 5203 "label": null, 5204 "logBase": 1, 5205 "max": null, 5206 "min": 0, 5207 "show": true 5208 }, 5209 { 5210 "format": "short", 5211 "label": null, 5212 "logBase": 1, 5213 "max": null, 5214 "min": null, 5215 "show": false 5216 } 5217 ], 5218 "yaxis": { 5219 "align": false, 5220 "alignLevel": null 5221 } 5222 }, 5223 { 5224 "aliasColors": {}, 5225 "bars": false, 5226 "dashLength": 10, 5227 "dashes": false, 5228 "datasource": "${DS_TEST-CLUSTER}", 5229 "description": "The duration that the relay log writes binlog event into the disks each time (in seconds)", 5230 "fieldConfig": { 5231 "defaults": {}, 5232 "overrides": [] 5233 }, 5234 "fill": 1, 5235 "fillGradient": 0, 5236 "gridPos": { 5237 "h": 7, 5238 "w": 6, 5239 "x": 12, 5240 "y": 25 5241 }, 5242 "hiddenSeries": false, 5243 "id": 12, 5244 "legend": { 5245 "alignAsTable": true, 5246 "avg": false, 5247 "current": false, 5248 "max": false, 5249 "min": false, 5250 "rightSide": true, 5251 "show": true, 5252 "total": false, 5253 "values": false 5254 }, 5255 "lines": true, 5256 "linewidth": 1, 5257 "links": [], 5258 "nullPointMode": "null", 5259 "options": { 5260 "alertThreshold": true 5261 }, 5262 "percentage": false, 5263 "pluginVersion": "7.5.11", 5264 "pointradius": 5, 5265 "points": false, 5266 "renderer": "flot", 5267 "seriesOverrides": [], 5268 "spaceLength": 10, 5269 "stack": false, 5270 "steppedLine": false, 5271 "targets": [ 5272 { 5273 "expr": "histogram_quantile(0.90, sum(rate(dm_relay_write_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[1m])) by (le))", 5274 "format": "time_series", 5275 "intervalFactor": 2, 5276 "legendFormat": "90", 5277 "refId": "A" 5278 }, 5279 { 5280 "expr": "histogram_quantile(0.95, sum(rate(dm_relay_write_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[1m])) by (le))", 5281 "format": "time_series", 5282 "intervalFactor": 2, 5283 "legendFormat": "95", 5284 "refId": "B" 5285 }, 5286 { 5287 "expr": "histogram_quantile(0.99, sum(rate(dm_relay_write_duration_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[1m])) by (le))", 5288 "format": "time_series", 5289 "intervalFactor": 2, 5290 "legendFormat": "99", 5291 "refId": "C" 5292 } 5293 ], 5294 "thresholds": [], 5295 "timeFrom": null, 5296 "timeRegions": [], 5297 "timeShift": null, 5298 "title": "write relay log duration", 5299 "tooltip": { 5300 "shared": true, 5301 "sort": 0, 5302 "value_type": "individual" 5303 }, 5304 "type": "graph", 5305 "xaxis": { 5306 "buckets": null, 5307 "mode": "time", 5308 "name": null, 5309 "show": true, 5310 "values": [] 5311 }, 5312 "yaxes": [ 5313 { 5314 "decimals": 2, 5315 "format": "s", 5316 "label": null, 5317 "logBase": 1, 5318 "max": null, 5319 "min": 0, 5320 "show": true 5321 }, 5322 { 5323 "format": "short", 5324 "label": null, 5325 "logBase": 1, 5326 "max": null, 5327 "min": null, 5328 "show": false 5329 } 5330 ], 5331 "yaxis": { 5332 "align": false, 5333 "alignLevel": null 5334 } 5335 }, 5336 { 5337 "aliasColors": {}, 5338 "bars": false, 5339 "dashLength": 10, 5340 "dashes": false, 5341 "datasource": "${DS_TEST-CLUSTER}", 5342 "description": "The size of a single binlog event that the relay log writes into the disks", 5343 "fieldConfig": { 5344 "defaults": {}, 5345 "overrides": [] 5346 }, 5347 "fill": 1, 5348 "fillGradient": 0, 5349 "gridPos": { 5350 "h": 7, 5351 "w": 6, 5352 "x": 18, 5353 "y": 25 5354 }, 5355 "hiddenSeries": false, 5356 "id": 25, 5357 "legend": { 5358 "alignAsTable": true, 5359 "avg": false, 5360 "current": false, 5361 "max": false, 5362 "min": false, 5363 "rightSide": true, 5364 "show": true, 5365 "total": false, 5366 "values": false 5367 }, 5368 "lines": true, 5369 "linewidth": 1, 5370 "links": [], 5371 "nullPointMode": "null", 5372 "options": { 5373 "alertThreshold": true 5374 }, 5375 "percentage": false, 5376 "pluginVersion": "7.5.11", 5377 "pointradius": 5, 5378 "points": false, 5379 "renderer": "flot", 5380 "seriesOverrides": [], 5381 "spaceLength": 10, 5382 "stack": false, 5383 "steppedLine": false, 5384 "targets": [ 5385 { 5386 "expr": "histogram_quantile(0.90, sum(rate(dm_relay_write_size_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[1m])) by (le))", 5387 "format": "time_series", 5388 "intervalFactor": 2, 5389 "legendFormat": "90", 5390 "refId": "A" 5391 }, 5392 { 5393 "expr": "histogram_quantile(0.95, sum(rate(dm_relay_write_size_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[1m])) by (le))", 5394 "format": "time_series", 5395 "intervalFactor": 2, 5396 "legendFormat": "95", 5397 "refId": "B" 5398 }, 5399 { 5400 "expr": "histogram_quantile(0.99, sum(rate(dm_relay_write_size_bucket{job_id=~\"$job_id\",instance=~\"$instance\"}[1m])) by (le))", 5401 "format": "time_series", 5402 "intervalFactor": 2, 5403 "legendFormat": "99", 5404 "refId": "C" 5405 } 5406 ], 5407 "thresholds": [], 5408 "timeFrom": null, 5409 "timeRegions": [], 5410 "timeShift": null, 5411 "title": "binlog event size", 5412 "tooltip": { 5413 "shared": true, 5414 "sort": 0, 5415 "value_type": "individual" 5416 }, 5417 "type": "graph", 5418 "xaxis": { 5419 "buckets": null, 5420 "mode": "time", 5421 "name": null, 5422 "show": true, 5423 "values": [] 5424 }, 5425 "yaxes": [ 5426 { 5427 "decimals": 2, 5428 "format": "decbytes", 5429 "label": null, 5430 "logBase": 1, 5431 "max": null, 5432 "min": null, 5433 "show": true 5434 }, 5435 { 5436 "format": "short", 5437 "label": null, 5438 "logBase": 1, 5439 "max": null, 5440 "min": null, 5441 "show": false 5442 } 5443 ], 5444 "yaxis": { 5445 "align": false, 5446 "alignLevel": null 5447 } 5448 } 5449 ], 5450 "repeat": null, 5451 "title": "relay log", 5452 "type": "row" 5453 }, 5454 { 5455 "collapsed": true, 5456 "datasource": "${DS_TEST-CLUSTER}", 5457 "gridPos": { 5458 "h": 1, 5459 "w": 24, 5460 "x": 0, 5461 "y": 10 5462 }, 5463 "id": 80, 5464 "panels": [ 5465 { 5466 "aliasColors": {}, 5467 "bars": false, 5468 "dashLength": 10, 5469 "dashes": false, 5470 "datasource": "${DS_TEST-CLUSTER}", 5471 "description": "number of dm-masters try to start leader components per minute", 5472 "fieldConfig": { 5473 "defaults": {}, 5474 "overrides": [] 5475 }, 5476 "fill": 1, 5477 "fillGradient": 0, 5478 "gridPos": { 5479 "h": 7, 5480 "w": 6, 5481 "x": 0, 5482 "y": 12 5483 }, 5484 "hiddenSeries": false, 5485 "id": 86, 5486 "legend": { 5487 "avg": false, 5488 "current": false, 5489 "max": false, 5490 "min": false, 5491 "show": false, 5492 "total": false, 5493 "values": false 5494 }, 5495 "lines": true, 5496 "linewidth": 1, 5497 "links": [], 5498 "nullPointMode": "null", 5499 "options": { 5500 "alertThreshold": true 5501 }, 5502 "percentage": false, 5503 "pluginVersion": "7.5.11", 5504 "pointradius": 5, 5505 "points": false, 5506 "renderer": "flot", 5507 "seriesOverrides": [], 5508 "spaceLength": 10, 5509 "stack": false, 5510 "steppedLine": false, 5511 "targets": [ 5512 { 5513 "expr": "dm_master_start_leader_counter", 5514 "format": "time_series", 5515 "instant": false, 5516 "intervalFactor": 2, 5517 "legendFormat": "{{instance}}", 5518 "refId": "A" 5519 } 5520 ], 5521 "thresholds": [], 5522 "timeFrom": null, 5523 "timeRegions": [], 5524 "timeShift": null, 5525 "title": "number of dm-masters start leader components per minute", 5526 "tooltip": { 5527 "shared": true, 5528 "sort": 0, 5529 "value_type": "individual" 5530 }, 5531 "type": "graph", 5532 "xaxis": { 5533 "buckets": null, 5534 "mode": "time", 5535 "name": null, 5536 "show": true, 5537 "values": [] 5538 }, 5539 "yaxes": [ 5540 { 5541 "decimals": 0, 5542 "format": "short", 5543 "label": null, 5544 "logBase": 1, 5545 "max": null, 5546 "min": null, 5547 "show": true 5548 }, 5549 { 5550 "format": "short", 5551 "label": null, 5552 "logBase": 1, 5553 "max": null, 5554 "min": null, 5555 "show": false 5556 } 5557 ], 5558 "yaxis": { 5559 "align": false, 5560 "alignLevel": null 5561 } 5562 }, 5563 { 5564 "aliasColors": {}, 5565 "bars": false, 5566 "dashLength": 10, 5567 "dashes": false, 5568 "datasource": "${DS_TEST-CLUSTER}", 5569 "description": "number of worker in different states, -1 - unrecognized, 0 - offline, 1 - free, 2 - bound", 5570 "fieldConfig": { 5571 "defaults": {}, 5572 "overrides": [] 5573 }, 5574 "fill": 1, 5575 "fillGradient": 0, 5576 "gridPos": { 5577 "h": 7, 5578 "w": 6, 5579 "x": 6, 5580 "y": 12 5581 }, 5582 "hiddenSeries": false, 5583 "id": 83, 5584 "legend": { 5585 "avg": false, 5586 "current": false, 5587 "max": false, 5588 "min": false, 5589 "show": false, 5590 "total": false, 5591 "values": false 5592 }, 5593 "lines": true, 5594 "linewidth": 1, 5595 "links": [], 5596 "nullPointMode": "null", 5597 "options": { 5598 "alertThreshold": true 5599 }, 5600 "percentage": false, 5601 "pluginVersion": "7.5.11", 5602 "pointradius": 5, 5603 "points": false, 5604 "renderer": "flot", 5605 "seriesOverrides": [], 5606 "spaceLength": 10, 5607 "stack": false, 5608 "steppedLine": false, 5609 "targets": [ 5610 { 5611 "expr": "count_values(\"state\", dm_master_worker_state)", 5612 "format": "time_series", 5613 "instant": false, 5614 "intervalFactor": 2, 5615 "refId": "A" 5616 } 5617 ], 5618 "thresholds": [], 5619 "timeFrom": null, 5620 "timeRegions": [], 5621 "timeShift": null, 5622 "title": "number of workers in different state", 5623 "tooltip": { 5624 "shared": true, 5625 "sort": 0, 5626 "value_type": "individual" 5627 }, 5628 "type": "graph", 5629 "xaxis": { 5630 "buckets": null, 5631 "mode": "time", 5632 "name": null, 5633 "show": true, 5634 "values": [] 5635 }, 5636 "yaxes": [ 5637 { 5638 "decimals": 0, 5639 "format": "short", 5640 "label": null, 5641 "logBase": 1, 5642 "max": null, 5643 "min": null, 5644 "show": true 5645 }, 5646 { 5647 "format": "short", 5648 "label": null, 5649 "logBase": 1, 5650 "max": null, 5651 "min": null, 5652 "show": false 5653 } 5654 ], 5655 "yaxis": { 5656 "align": false, 5657 "alignLevel": null 5658 } 5659 }, 5660 { 5661 "aliasColors": {}, 5662 "bars": false, 5663 "dashLength": 10, 5664 "dashes": false, 5665 "datasource": "${DS_TEST-CLUSTER}", 5666 "description": "state of worker, -1 - unrecognized, 0 - offline, 1 - free, 2 - bound", 5667 "fieldConfig": { 5668 "defaults": {}, 5669 "overrides": [] 5670 }, 5671 "fill": 1, 5672 "fillGradient": 0, 5673 "gridPos": { 5674 "h": 7, 5675 "w": 6, 5676 "x": 12, 5677 "y": 12 5678 }, 5679 "hiddenSeries": false, 5680 "id": 81, 5681 "legend": { 5682 "avg": false, 5683 "current": false, 5684 "max": false, 5685 "min": false, 5686 "show": false, 5687 "total": false, 5688 "values": false 5689 }, 5690 "lines": true, 5691 "linewidth": 1, 5692 "links": [], 5693 "nullPointMode": "null", 5694 "options": { 5695 "alertThreshold": true 5696 }, 5697 "percentage": false, 5698 "pluginVersion": "7.5.11", 5699 "pointradius": 5, 5700 "points": false, 5701 "renderer": "flot", 5702 "seriesOverrides": [], 5703 "spaceLength": 10, 5704 "stack": false, 5705 "steppedLine": false, 5706 "targets": [ 5707 { 5708 "expr": "sum by (worker) (dm_master_worker_state)", 5709 "format": "time_series", 5710 "instant": false, 5711 "intervalFactor": 2, 5712 "refId": "A" 5713 } 5714 ], 5715 "thresholds": [], 5716 "timeFrom": null, 5717 "timeRegions": [], 5718 "timeShift": null, 5719 "title": "workers' state", 5720 "tooltip": { 5721 "shared": true, 5722 "sort": 0, 5723 "value_type": "individual" 5724 }, 5725 "type": "graph", 5726 "xaxis": { 5727 "buckets": null, 5728 "mode": "time", 5729 "name": null, 5730 "show": true, 5731 "values": [] 5732 }, 5733 "yaxes": [ 5734 { 5735 "decimals": 0, 5736 "format": "short", 5737 "label": null, 5738 "logBase": 1, 5739 "max": null, 5740 "min": null, 5741 "show": true 5742 }, 5743 { 5744 "format": "short", 5745 "label": null, 5746 "logBase": 1, 5747 "max": null, 5748 "min": null, 5749 "show": false 5750 } 5751 ], 5752 "yaxis": { 5753 "align": false, 5754 "alignLevel": null 5755 } 5756 }, 5757 { 5758 "aliasColors": {}, 5759 "bars": false, 5760 "dashLength": 10, 5761 "dashes": false, 5762 "datasource": "${DS_TEST-CLUSTER}", 5763 "description": "number of worker event error", 5764 "fieldConfig": { 5765 "defaults": {}, 5766 "overrides": [] 5767 }, 5768 "fill": 1, 5769 "fillGradient": 0, 5770 "gridPos": { 5771 "h": 7, 5772 "w": 6, 5773 "x": 18, 5774 "y": 12 5775 }, 5776 "hiddenSeries": false, 5777 "id": 85, 5778 "legend": { 5779 "avg": false, 5780 "current": false, 5781 "max": false, 5782 "min": false, 5783 "show": false, 5784 "total": false, 5785 "values": false 5786 }, 5787 "lines": true, 5788 "linewidth": 1, 5789 "links": [], 5790 "nullPointMode": "null", 5791 "options": { 5792 "alertThreshold": true 5793 }, 5794 "percentage": false, 5795 "pluginVersion": "7.5.11", 5796 "pointradius": 5, 5797 "points": false, 5798 "renderer": "flot", 5799 "seriesOverrides": [], 5800 "spaceLength": 10, 5801 "stack": false, 5802 "steppedLine": false, 5803 "targets": [ 5804 { 5805 "expr": "sum by (type) (dm_master_worker_event_error)", 5806 "format": "time_series", 5807 "instant": false, 5808 "intervalFactor": 2, 5809 "refId": "A" 5810 } 5811 ], 5812 "thresholds": [], 5813 "timeFrom": null, 5814 "timeRegions": [], 5815 "timeShift": null, 5816 "title": "number of worker event error", 5817 "tooltip": { 5818 "shared": true, 5819 "sort": 0, 5820 "value_type": "individual" 5821 }, 5822 "type": "graph", 5823 "xaxis": { 5824 "buckets": null, 5825 "mode": "time", 5826 "name": null, 5827 "show": true, 5828 "values": [] 5829 }, 5830 "yaxes": [ 5831 { 5832 "decimals": 0, 5833 "format": "short", 5834 "label": null, 5835 "logBase": 1, 5836 "max": null, 5837 "min": null, 5838 "show": true 5839 }, 5840 { 5841 "format": "short", 5842 "label": null, 5843 "logBase": 1, 5844 "max": null, 5845 "min": null, 5846 "show": false 5847 } 5848 ], 5849 "yaxis": { 5850 "align": false, 5851 "alignLevel": null 5852 } 5853 }, 5854 { 5855 "aliasColors": {}, 5856 "bars": false, 5857 "dashLength": 10, 5858 "dashes": false, 5859 "datasource": "${DS_TEST-CLUSTER}", 5860 "description": "number of shard DDL lock/operation error in one minute", 5861 "fieldConfig": { 5862 "defaults": {}, 5863 "overrides": [] 5864 }, 5865 "fill": 1, 5866 "fillGradient": 0, 5867 "gridPos": { 5868 "h": 7, 5869 "w": 12, 5870 "x": 0, 5871 "y": 19 5872 }, 5873 "hiddenSeries": false, 5874 "id": 82, 5875 "legend": { 5876 "avg": false, 5877 "current": false, 5878 "max": false, 5879 "min": false, 5880 "show": false, 5881 "total": false, 5882 "values": false 5883 }, 5884 "lines": true, 5885 "linewidth": 1, 5886 "links": [], 5887 "nullPointMode": "null", 5888 "options": { 5889 "alertThreshold": true 5890 }, 5891 "percentage": false, 5892 "pluginVersion": "7.5.11", 5893 "pointradius": 5, 5894 "points": false, 5895 "renderer": "flot", 5896 "seriesOverrides": [], 5897 "spaceLength": 10, 5898 "stack": false, 5899 "steppedLine": false, 5900 "targets": [ 5901 { 5902 "expr": "sum by (task, type) (increase(dm_master_shard_ddl_error[1m]))", 5903 "format": "time_series", 5904 "instant": false, 5905 "intervalFactor": 2, 5906 "refId": "A" 5907 } 5908 ], 5909 "thresholds": [], 5910 "timeFrom": null, 5911 "timeRegions": [], 5912 "timeShift": null, 5913 "title": "shard ddl error per minute", 5914 "tooltip": { 5915 "shared": true, 5916 "sort": 0, 5917 "value_type": "individual" 5918 }, 5919 "type": "graph", 5920 "xaxis": { 5921 "buckets": null, 5922 "mode": "time", 5923 "name": null, 5924 "show": true, 5925 "values": [] 5926 }, 5927 "yaxes": [ 5928 { 5929 "decimals": 0, 5930 "format": "short", 5931 "label": null, 5932 "logBase": 1, 5933 "max": null, 5934 "min": null, 5935 "show": true 5936 }, 5937 { 5938 "format": "short", 5939 "label": null, 5940 "logBase": 1, 5941 "max": null, 5942 "min": null, 5943 "show": false 5944 } 5945 ], 5946 "yaxis": { 5947 "align": false, 5948 "alignLevel": null 5949 } 5950 }, 5951 { 5952 "aliasColors": {}, 5953 "bars": false, 5954 "dashLength": 10, 5955 "dashes": false, 5956 "datasource": "${DS_TEST-CLUSTER}", 5957 "description": "number of pending shard DDL in different states, Un-synced (waiting all upstream), Synced (all upstream finished, waiting all downstream)", 5958 "fieldConfig": { 5959 "defaults": {}, 5960 "overrides": [] 5961 }, 5962 "fill": 1, 5963 "fillGradient": 0, 5964 "gridPos": { 5965 "h": 7, 5966 "w": 12, 5967 "x": 12, 5968 "y": 19 5969 }, 5970 "hiddenSeries": false, 5971 "id": 84, 5972 "legend": { 5973 "avg": false, 5974 "current": false, 5975 "max": false, 5976 "min": false, 5977 "show": false, 5978 "total": false, 5979 "values": false 5980 }, 5981 "lines": true, 5982 "linewidth": 1, 5983 "links": [], 5984 "nullPointMode": "null", 5985 "options": { 5986 "alertThreshold": true 5987 }, 5988 "percentage": false, 5989 "pluginVersion": "7.5.11", 5990 "pointradius": 5, 5991 "points": false, 5992 "renderer": "flot", 5993 "seriesOverrides": [], 5994 "spaceLength": 10, 5995 "stack": false, 5996 "steppedLine": false, 5997 "targets": [ 5998 { 5999 "expr": "sum by (task, type) (dm_master_ddl_state_number)", 6000 "format": "time_series", 6001 "instant": false, 6002 "intervalFactor": 2, 6003 "refId": "A" 6004 } 6005 ], 6006 "thresholds": [], 6007 "timeFrom": null, 6008 "timeRegions": [], 6009 "timeShift": null, 6010 "title": "number of pending shard ddl", 6011 "tooltip": { 6012 "shared": true, 6013 "sort": 0, 6014 "value_type": "individual" 6015 }, 6016 "type": "graph", 6017 "xaxis": { 6018 "buckets": null, 6019 "mode": "time", 6020 "name": null, 6021 "show": true, 6022 "values": [] 6023 }, 6024 "yaxes": [ 6025 { 6026 "decimals": 0, 6027 "format": "short", 6028 "label": null, 6029 "logBase": 1, 6030 "max": null, 6031 "min": null, 6032 "show": true 6033 }, 6034 { 6035 "format": "short", 6036 "label": null, 6037 "logBase": 1, 6038 "max": null, 6039 "min": null, 6040 "show": false 6041 } 6042 ], 6043 "yaxis": { 6044 "align": false, 6045 "alignLevel": null 6046 } 6047 } 6048 ], 6049 "repeat": null, 6050 "title": "HA", 6051 "type": "row" 6052 }, 6053 { 6054 "collapsed": true, 6055 "datasource": "${DS_TEST-CLUSTER}", 6056 "gridPos": { 6057 "h": 1, 6058 "w": 24, 6059 "x": 0, 6060 "y": 11 6061 }, 6062 "id": 67, 6063 "panels": [ 6064 { 6065 "aliasColors": {}, 6066 "bars": false, 6067 "dashLength": 10, 6068 "dashes": false, 6069 "datasource": "${DS_TEST-CLUSTER}", 6070 "description": "The number of error happens before operate", 6071 "fieldConfig": { 6072 "defaults": {}, 6073 "overrides": [] 6074 }, 6075 "fill": 1, 6076 "fillGradient": 0, 6077 "gridPos": { 6078 "h": 7, 6079 "w": 6, 6080 "x": 0, 6081 "y": 28 6082 }, 6083 "hiddenSeries": false, 6084 "id": 71, 6085 "legend": { 6086 "avg": false, 6087 "current": false, 6088 "max": false, 6089 "min": false, 6090 "show": false, 6091 "total": false, 6092 "values": false 6093 }, 6094 "lines": true, 6095 "linewidth": 1, 6096 "links": [], 6097 "nullPointMode": "null", 6098 "options": { 6099 "alertThreshold": true 6100 }, 6101 "percentage": false, 6102 "pluginVersion": "7.5.11", 6103 "pointradius": 5, 6104 "points": false, 6105 "renderer": "flot", 6106 "seriesOverrides": [], 6107 "spaceLength": 10, 6108 "stack": false, 6109 "steppedLine": false, 6110 "targets": [ 6111 { 6112 "expr": "dm_worker_operate_error{type=\"BeforeAnyOp\"}", 6113 "format": "time_series", 6114 "intervalFactor": 2, 6115 "refId": "A" 6116 } 6117 ], 6118 "thresholds": [], 6119 "timeFrom": null, 6120 "timeRegions": [], 6121 "timeShift": null, 6122 "title": "before any operate error", 6123 "tooltip": { 6124 "shared": true, 6125 "sort": 0, 6126 "value_type": "individual" 6127 }, 6128 "type": "graph", 6129 "xaxis": { 6130 "buckets": null, 6131 "mode": "time", 6132 "name": null, 6133 "show": true, 6134 "values": [] 6135 }, 6136 "yaxes": [ 6137 { 6138 "decimals": 0, 6139 "format": "short", 6140 "label": null, 6141 "logBase": 1, 6142 "show": true 6143 }, 6144 { 6145 "format": "short", 6146 "label": null, 6147 "logBase": 1, 6148 "max": null, 6149 "min": null, 6150 "show": false 6151 } 6152 ], 6153 "yaxis": { 6154 "align": false, 6155 "alignLevel": null 6156 } 6157 }, 6158 { 6159 "aliasColors": {}, 6160 "bars": false, 6161 "dashLength": 10, 6162 "dashes": false, 6163 "datasource": "${DS_TEST-CLUSTER}", 6164 "description": "The number of error happens in operate source bound", 6165 "fieldConfig": { 6166 "defaults": {}, 6167 "overrides": [] 6168 }, 6169 "fill": 1, 6170 "fillGradient": 0, 6171 "gridPos": { 6172 "h": 7, 6173 "w": 6, 6174 "x": 6, 6175 "y": 28 6176 }, 6177 "hiddenSeries": false, 6178 "id": 72, 6179 "legend": { 6180 "avg": false, 6181 "current": false, 6182 "max": false, 6183 "min": false, 6184 "show": false, 6185 "total": false, 6186 "values": false 6187 }, 6188 "lines": true, 6189 "linewidth": 1, 6190 "links": [], 6191 "nullPointMode": "null", 6192 "options": { 6193 "alertThreshold": true 6194 }, 6195 "percentage": false, 6196 "pluginVersion": "7.5.11", 6197 "pointradius": 5, 6198 "points": false, 6199 "renderer": "flot", 6200 "seriesOverrides": [], 6201 "spaceLength": 10, 6202 "stack": false, 6203 "steppedLine": false, 6204 "targets": [ 6205 { 6206 "expr": "dm_worker_operate_error{type=\"SourceBound\"}", 6207 "format": "time_series", 6208 "intervalFactor": 2, 6209 "refId": "A" 6210 } 6211 ], 6212 "thresholds": [], 6213 "timeFrom": null, 6214 "timeRegions": [], 6215 "timeShift": null, 6216 "title": "source bound error", 6217 "tooltip": { 6218 "shared": true, 6219 "sort": 0, 6220 "value_type": "individual" 6221 }, 6222 "type": "graph", 6223 "xaxis": { 6224 "buckets": null, 6225 "mode": "time", 6226 "name": null, 6227 "show": true, 6228 "values": [] 6229 }, 6230 "yaxes": [ 6231 { 6232 "decimals": 0, 6233 "format": "short", 6234 "label": null, 6235 "logBase": 1, 6236 "show": true 6237 }, 6238 { 6239 "format": "short", 6240 "label": null, 6241 "logBase": 1, 6242 "max": null, 6243 "min": null, 6244 "show": false 6245 } 6246 ], 6247 "yaxis": { 6248 "align": false, 6249 "alignLevel": null 6250 } 6251 }, 6252 { 6253 "aliasColors": {}, 6254 "bars": false, 6255 "dashLength": 10, 6256 "dashes": false, 6257 "datasource": "${DS_TEST-CLUSTER}", 6258 "description": "The number of error happens in operate start", 6259 "fieldConfig": { 6260 "defaults": {}, 6261 "overrides": [] 6262 }, 6263 "fill": 1, 6264 "fillGradient": 0, 6265 "gridPos": { 6266 "h": 7, 6267 "w": 6, 6268 "x": 12, 6269 "y": 28 6270 }, 6271 "hiddenSeries": false, 6272 "id": 76, 6273 "legend": { 6274 "avg": false, 6275 "current": false, 6276 "max": false, 6277 "min": false, 6278 "show": false, 6279 "total": false, 6280 "values": false 6281 }, 6282 "lines": true, 6283 "linewidth": 1, 6284 "links": [], 6285 "nullPointMode": "null", 6286 "options": { 6287 "alertThreshold": true 6288 }, 6289 "percentage": false, 6290 "pluginVersion": "7.5.11", 6291 "pointradius": 5, 6292 "points": false, 6293 "renderer": "flot", 6294 "seriesOverrides": [], 6295 "spaceLength": 10, 6296 "stack": false, 6297 "steppedLine": false, 6298 "targets": [ 6299 { 6300 "expr": "dm_worker_operate_error{type=\"Start\"}", 6301 "format": "time_series", 6302 "intervalFactor": 2, 6303 "refId": "A" 6304 } 6305 ], 6306 "thresholds": [], 6307 "timeFrom": null, 6308 "timeRegions": [], 6309 "timeShift": null, 6310 "title": "start error", 6311 "tooltip": { 6312 "shared": true, 6313 "sort": 0, 6314 "value_type": "individual" 6315 }, 6316 "type": "graph", 6317 "xaxis": { 6318 "buckets": null, 6319 "mode": "time", 6320 "name": null, 6321 "show": true, 6322 "values": [] 6323 }, 6324 "yaxes": [ 6325 { 6326 "decimals": 0, 6327 "format": "short", 6328 "label": null, 6329 "logBase": 1, 6330 "show": true 6331 }, 6332 { 6333 "format": "short", 6334 "label": null, 6335 "logBase": 1, 6336 "max": null, 6337 "min": null, 6338 "show": false 6339 } 6340 ], 6341 "yaxis": { 6342 "align": false, 6343 "alignLevel": null 6344 } 6345 }, 6346 { 6347 "aliasColors": {}, 6348 "bars": false, 6349 "dashLength": 10, 6350 "dashes": false, 6351 "datasource": "${DS_TEST-CLUSTER}", 6352 "description": "The number of error happens in operate pause", 6353 "fieldConfig": { 6354 "defaults": {}, 6355 "overrides": [] 6356 }, 6357 "fill": 1, 6358 "fillGradient": 0, 6359 "gridPos": { 6360 "h": 7, 6361 "w": 6, 6362 "x": 18, 6363 "y": 28 6364 }, 6365 "hiddenSeries": false, 6366 "id": 74, 6367 "legend": { 6368 "avg": false, 6369 "current": false, 6370 "max": false, 6371 "min": false, 6372 "show": false, 6373 "total": false, 6374 "values": false 6375 }, 6376 "lines": true, 6377 "linewidth": 1, 6378 "links": [], 6379 "nullPointMode": "null", 6380 "options": { 6381 "alertThreshold": true 6382 }, 6383 "percentage": false, 6384 "pluginVersion": "7.5.11", 6385 "pointradius": 5, 6386 "points": false, 6387 "renderer": "flot", 6388 "seriesOverrides": [], 6389 "spaceLength": 10, 6390 "stack": false, 6391 "steppedLine": false, 6392 "targets": [ 6393 { 6394 "expr": "dm_worker_operate_error{type=\"Pause\"}", 6395 "format": "time_series", 6396 "intervalFactor": 2, 6397 "refId": "A" 6398 } 6399 ], 6400 "thresholds": [], 6401 "timeFrom": null, 6402 "timeRegions": [], 6403 "timeShift": null, 6404 "title": "pause error", 6405 "tooltip": { 6406 "shared": true, 6407 "sort": 0, 6408 "value_type": "individual" 6409 }, 6410 "type": "graph", 6411 "xaxis": { 6412 "buckets": null, 6413 "mode": "time", 6414 "name": null, 6415 "show": true, 6416 "values": [] 6417 }, 6418 "yaxes": [ 6419 { 6420 "decimals": 0, 6421 "format": "short", 6422 "label": null, 6423 "logBase": 1, 6424 "show": true 6425 }, 6426 { 6427 "format": "short", 6428 "label": null, 6429 "logBase": 1, 6430 "max": null, 6431 "min": null, 6432 "show": false 6433 } 6434 ], 6435 "yaxis": { 6436 "align": false, 6437 "alignLevel": null 6438 } 6439 }, 6440 { 6441 "aliasColors": {}, 6442 "bars": false, 6443 "dashLength": 10, 6444 "dashes": false, 6445 "datasource": "${DS_TEST-CLUSTER}", 6446 "description": "The number of error happens in operate resume", 6447 "fieldConfig": { 6448 "defaults": {}, 6449 "overrides": [] 6450 }, 6451 "fill": 1, 6452 "fillGradient": 0, 6453 "gridPos": { 6454 "h": 7, 6455 "w": 6, 6456 "x": 0, 6457 "y": 35 6458 }, 6459 "hiddenSeries": false, 6460 "id": 78, 6461 "legend": { 6462 "avg": false, 6463 "current": false, 6464 "max": false, 6465 "min": false, 6466 "show": false, 6467 "total": false, 6468 "values": false 6469 }, 6470 "lines": true, 6471 "linewidth": 1, 6472 "links": [], 6473 "nullPointMode": "null", 6474 "options": { 6475 "alertThreshold": true 6476 }, 6477 "percentage": false, 6478 "pluginVersion": "7.5.11", 6479 "pointradius": 5, 6480 "points": false, 6481 "renderer": "flot", 6482 "seriesOverrides": [], 6483 "spaceLength": 10, 6484 "stack": false, 6485 "steppedLine": false, 6486 "targets": [ 6487 { 6488 "expr": "dm_worker_operate_error{type=\"Resume\"}", 6489 "format": "time_series", 6490 "intervalFactor": 2, 6491 "refId": "A" 6492 } 6493 ], 6494 "thresholds": [], 6495 "timeFrom": null, 6496 "timeRegions": [], 6497 "timeShift": null, 6498 "title": "resume error", 6499 "tooltip": { 6500 "shared": true, 6501 "sort": 0, 6502 "value_type": "individual" 6503 }, 6504 "type": "graph", 6505 "xaxis": { 6506 "buckets": null, 6507 "mode": "time", 6508 "name": null, 6509 "show": true, 6510 "values": [] 6511 }, 6512 "yaxes": [ 6513 { 6514 "decimals": 0, 6515 "format": "short", 6516 "label": null, 6517 "logBase": 1, 6518 "show": true 6519 }, 6520 { 6521 "format": "short", 6522 "label": null, 6523 "logBase": 1, 6524 "max": null, 6525 "min": null, 6526 "show": false 6527 } 6528 ], 6529 "yaxis": { 6530 "align": false, 6531 "alignLevel": null 6532 } 6533 }, 6534 { 6535 "aliasColors": {}, 6536 "bars": false, 6537 "dashLength": 10, 6538 "dashes": false, 6539 "datasource": "${DS_TEST-CLUSTER}", 6540 "description": "The number of error happens in operate auto-resume", 6541 "fieldConfig": { 6542 "defaults": {}, 6543 "overrides": [] 6544 }, 6545 "fill": 1, 6546 "fillGradient": 0, 6547 "gridPos": { 6548 "h": 7, 6549 "w": 6, 6550 "x": 6, 6551 "y": 35 6552 }, 6553 "hiddenSeries": false, 6554 "id": 69, 6555 "legend": { 6556 "avg": false, 6557 "current": false, 6558 "max": false, 6559 "min": false, 6560 "show": false, 6561 "total": false, 6562 "values": false 6563 }, 6564 "lines": true, 6565 "linewidth": 1, 6566 "links": [], 6567 "nullPointMode": "null", 6568 "options": { 6569 "alertThreshold": true 6570 }, 6571 "percentage": false, 6572 "pluginVersion": "7.5.11", 6573 "pointradius": 5, 6574 "points": false, 6575 "renderer": "flot", 6576 "seriesOverrides": [], 6577 "spaceLength": 10, 6578 "stack": false, 6579 "steppedLine": false, 6580 "targets": [ 6581 { 6582 "expr": "dm_worker_operate_error{type=\"AutoResume\"}", 6583 "format": "time_series", 6584 "intervalFactor": 2, 6585 "refId": "A" 6586 } 6587 ], 6588 "thresholds": [], 6589 "timeFrom": null, 6590 "timeRegions": [], 6591 "timeShift": null, 6592 "title": "auto-resume error", 6593 "tooltip": { 6594 "shared": true, 6595 "sort": 0, 6596 "value_type": "individual" 6597 }, 6598 "type": "graph", 6599 "xaxis": { 6600 "buckets": null, 6601 "mode": "time", 6602 "name": null, 6603 "show": true, 6604 "values": [] 6605 }, 6606 "yaxes": [ 6607 { 6608 "decimals": 0, 6609 "format": "short", 6610 "label": null, 6611 "logBase": 1, 6612 "show": true 6613 }, 6614 { 6615 "format": "short", 6616 "label": null, 6617 "logBase": 1, 6618 "max": null, 6619 "min": null, 6620 "show": false 6621 } 6622 ], 6623 "yaxis": { 6624 "align": false, 6625 "alignLevel": null 6626 } 6627 }, 6628 { 6629 "aliasColors": {}, 6630 "bars": false, 6631 "dashLength": 10, 6632 "dashes": false, 6633 "datasource": "${DS_TEST-CLUSTER}", 6634 "description": "The number of error happens in operate update", 6635 "fieldConfig": { 6636 "defaults": {}, 6637 "overrides": [] 6638 }, 6639 "fill": 1, 6640 "fillGradient": 0, 6641 "gridPos": { 6642 "h": 7, 6643 "w": 6, 6644 "x": 12, 6645 "y": 35 6646 }, 6647 "hiddenSeries": false, 6648 "id": 77, 6649 "legend": { 6650 "avg": false, 6651 "current": false, 6652 "max": false, 6653 "min": false, 6654 "show": false, 6655 "total": false, 6656 "values": false 6657 }, 6658 "lines": true, 6659 "linewidth": 1, 6660 "links": [], 6661 "nullPointMode": "null", 6662 "options": { 6663 "alertThreshold": true 6664 }, 6665 "percentage": false, 6666 "pluginVersion": "7.5.11", 6667 "pointradius": 5, 6668 "points": false, 6669 "renderer": "flot", 6670 "seriesOverrides": [], 6671 "spaceLength": 10, 6672 "stack": false, 6673 "steppedLine": false, 6674 "targets": [ 6675 { 6676 "expr": "dm_worker_operate_error{type=\"Update\"}", 6677 "format": "time_series", 6678 "intervalFactor": 2, 6679 "refId": "A" 6680 } 6681 ], 6682 "thresholds": [], 6683 "timeFrom": null, 6684 "timeRegions": [], 6685 "timeShift": null, 6686 "title": "update error", 6687 "tooltip": { 6688 "shared": true, 6689 "sort": 0, 6690 "value_type": "individual" 6691 }, 6692 "type": "graph", 6693 "xaxis": { 6694 "buckets": null, 6695 "mode": "time", 6696 "name": null, 6697 "show": true, 6698 "values": [] 6699 }, 6700 "yaxes": [ 6701 { 6702 "decimals": 0, 6703 "format": "short", 6704 "label": null, 6705 "logBase": 1, 6706 "show": true 6707 }, 6708 { 6709 "format": "short", 6710 "label": null, 6711 "logBase": 1, 6712 "max": null, 6713 "min": null, 6714 "show": false 6715 } 6716 ], 6717 "yaxis": { 6718 "align": false, 6719 "alignLevel": null 6720 } 6721 }, 6722 { 6723 "aliasColors": {}, 6724 "bars": false, 6725 "dashLength": 10, 6726 "dashes": false, 6727 "datasource": "${DS_TEST-CLUSTER}", 6728 "description": "The number of error happens in operate stop", 6729 "fieldConfig": { 6730 "defaults": {}, 6731 "overrides": [] 6732 }, 6733 "fill": 1, 6734 "fillGradient": 0, 6735 "gridPos": { 6736 "h": 7, 6737 "w": 6, 6738 "x": 18, 6739 "y": 35 6740 }, 6741 "hiddenSeries": false, 6742 "id": 75, 6743 "legend": { 6744 "avg": false, 6745 "current": false, 6746 "max": false, 6747 "min": false, 6748 "show": false, 6749 "total": false, 6750 "values": false 6751 }, 6752 "lines": true, 6753 "linewidth": 1, 6754 "links": [], 6755 "nullPointMode": "null", 6756 "options": { 6757 "alertThreshold": true 6758 }, 6759 "percentage": false, 6760 "pluginVersion": "7.5.11", 6761 "pointradius": 5, 6762 "points": false, 6763 "renderer": "flot", 6764 "seriesOverrides": [], 6765 "spaceLength": 10, 6766 "stack": false, 6767 "steppedLine": false, 6768 "targets": [ 6769 { 6770 "expr": "dm_worker_operate_error{type=\"Stop\"}", 6771 "format": "time_series", 6772 "intervalFactor": 2, 6773 "refId": "A" 6774 } 6775 ], 6776 "thresholds": [], 6777 "timeFrom": null, 6778 "timeRegions": [], 6779 "timeShift": null, 6780 "title": "stop error", 6781 "tooltip": { 6782 "shared": true, 6783 "sort": 0, 6784 "value_type": "individual" 6785 }, 6786 "type": "graph", 6787 "xaxis": { 6788 "buckets": null, 6789 "mode": "time", 6790 "name": null, 6791 "show": true, 6792 "values": [] 6793 }, 6794 "yaxes": [ 6795 { 6796 "decimals": 0, 6797 "format": "short", 6798 "label": null, 6799 "logBase": 1, 6800 "show": true 6801 }, 6802 { 6803 "format": "short", 6804 "label": null, 6805 "logBase": 1, 6806 "max": null, 6807 "min": null, 6808 "show": false 6809 } 6810 ], 6811 "yaxis": { 6812 "align": false, 6813 "alignLevel": null 6814 } 6815 } 6816 ], 6817 "title": "operate error", 6818 "type": "row" 6819 }, 6820 { 6821 "collapsed": true, 6822 "datasource": "${DS_TEST-CLUSTER}", 6823 "gridPos": { 6824 "h": 1, 6825 "w": 24, 6826 "x": 0, 6827 "y": 12 6828 }, 6829 "id": 122, 6830 "panels": [ 6831 { 6832 "aliasColors": {}, 6833 "bars": false, 6834 "dashLength": 10, 6835 "dashes": false, 6836 "datasource": "${DS_TEST-CLUSTER}", 6837 "fieldConfig": { 6838 "defaults": {}, 6839 "overrides": [] 6840 }, 6841 "fill": 1, 6842 "fillGradient": 0, 6843 "gridPos": { 6844 "h": 8, 6845 "w": 24, 6846 "x": 0, 6847 "y": 13 6848 }, 6849 "hiddenSeries": false, 6850 "id": 120, 6851 "legend": { 6852 "avg": false, 6853 "current": false, 6854 "max": false, 6855 "min": false, 6856 "show": true, 6857 "total": false, 6858 "values": false 6859 }, 6860 "lines": true, 6861 "linewidth": 1, 6862 "nullPointMode": "null", 6863 "options": { 6864 "alertThreshold": true 6865 }, 6866 "percentage": false, 6867 "pluginVersion": "7.5.11", 6868 "pointradius": 2, 6869 "points": false, 6870 "renderer": "flot", 6871 "seriesOverrides": [], 6872 "spaceLength": 10, 6873 "stack": false, 6874 "steppedLine": false, 6875 "targets": [ 6876 { 6877 "exemplar": true, 6878 "expr": "dm_validator_validator_error_count", 6879 "interval": "", 6880 "legendFormat": "", 6881 "queryType": "randomWalk", 6882 "refId": "A" 6883 } 6884 ], 6885 "thresholds": [], 6886 "timeFrom": null, 6887 "timeRegions": [], 6888 "timeShift": null, 6889 "title": "Validator Error Count", 6890 "tooltip": { 6891 "shared": true, 6892 "sort": 0, 6893 "value_type": "individual" 6894 }, 6895 "type": "graph", 6896 "xaxis": { 6897 "buckets": null, 6898 "mode": "time", 6899 "name": null, 6900 "show": true, 6901 "values": [] 6902 }, 6903 "yaxes": [ 6904 { 6905 "format": "short", 6906 "label": null, 6907 "logBase": 1, 6908 "max": null, 6909 "min": null, 6910 "show": true 6911 }, 6912 { 6913 "format": "short", 6914 "label": null, 6915 "logBase": 1, 6916 "max": null, 6917 "min": null, 6918 "show": true 6919 } 6920 ], 6921 "yaxis": { 6922 "align": false, 6923 "alignLevel": null 6924 } 6925 }, 6926 { 6927 "aliasColors": {}, 6928 "bars": false, 6929 "dashLength": 10, 6930 "dashes": false, 6931 "datasource": "${DS_TEST-CLUSTER}", 6932 "fieldConfig": { 6933 "defaults": {}, 6934 "overrides": [] 6935 }, 6936 "fill": 1, 6937 "fillGradient": 0, 6938 "gridPos": { 6939 "h": 8, 6940 "w": 12, 6941 "x": 0, 6942 "y": 21 6943 }, 6944 "hiddenSeries": false, 6945 "id": 116, 6946 "legend": { 6947 "avg": false, 6948 "current": false, 6949 "max": false, 6950 "min": false, 6951 "show": true, 6952 "total": false, 6953 "values": false 6954 }, 6955 "lines": true, 6956 "linewidth": 1, 6957 "nullPointMode": "null", 6958 "options": { 6959 "alertThreshold": true 6960 }, 6961 "percentage": false, 6962 "pluginVersion": "7.5.11", 6963 "pointradius": 2, 6964 "points": false, 6965 "renderer": "flot", 6966 "seriesOverrides": [], 6967 "spaceLength": 10, 6968 "stack": false, 6969 "steppedLine": false, 6970 "targets": [ 6971 { 6972 "exemplar": true, 6973 "expr": "dm_validator_validator_logpos_latency", 6974 "interval": "", 6975 "legendFormat": "", 6976 "queryType": "randomWalk", 6977 "refId": "A" 6978 } 6979 ], 6980 "thresholds": [], 6981 "timeFrom": null, 6982 "timeRegions": [], 6983 "timeShift": null, 6984 "title": "Validator Binlog Latency (in byte)", 6985 "tooltip": { 6986 "shared": true, 6987 "sort": 0, 6988 "value_type": "individual" 6989 }, 6990 "type": "graph", 6991 "xaxis": { 6992 "buckets": null, 6993 "mode": "time", 6994 "name": null, 6995 "show": true, 6996 "values": [] 6997 }, 6998 "yaxes": [ 6999 { 7000 "format": "short", 7001 "label": null, 7002 "logBase": 1, 7003 "max": null, 7004 "min": null, 7005 "show": true 7006 }, 7007 { 7008 "format": "short", 7009 "label": null, 7010 "logBase": 1, 7011 "max": null, 7012 "min": null, 7013 "show": true 7014 } 7015 ], 7016 "yaxis": { 7017 "align": false, 7018 "alignLevel": null 7019 } 7020 }, 7021 { 7022 "aliasColors": {}, 7023 "bars": false, 7024 "dashLength": 10, 7025 "dashes": false, 7026 "datasource": "${DS_TEST-CLUSTER}", 7027 "fieldConfig": { 7028 "defaults": {}, 7029 "overrides": [] 7030 }, 7031 "fill": 1, 7032 "fillGradient": 0, 7033 "gridPos": { 7034 "h": 8, 7035 "w": 12, 7036 "x": 12, 7037 "y": 21 7038 }, 7039 "hiddenSeries": false, 7040 "id": 114, 7041 "legend": { 7042 "avg": false, 7043 "current": false, 7044 "max": false, 7045 "min": false, 7046 "show": true, 7047 "total": false, 7048 "values": false 7049 }, 7050 "lines": true, 7051 "linewidth": 1, 7052 "nullPointMode": "null", 7053 "options": { 7054 "alertThreshold": true 7055 }, 7056 "percentage": false, 7057 "pluginVersion": "7.5.11", 7058 "pointradius": 2, 7059 "points": false, 7060 "renderer": "flot", 7061 "seriesOverrides": [], 7062 "spaceLength": 10, 7063 "stack": false, 7064 "steppedLine": false, 7065 "targets": [ 7066 { 7067 "exemplar": true, 7068 "expr": "dm_validator_validator_binlog_file", 7069 "interval": "", 7070 "legendFormat": "", 7071 "queryType": "randomWalk", 7072 "refId": "A" 7073 } 7074 ], 7075 "thresholds": [], 7076 "timeFrom": null, 7077 "timeRegions": [], 7078 "timeShift": null, 7079 "title": "Validator Binlog Position (in file)", 7080 "tooltip": { 7081 "shared": true, 7082 "sort": 0, 7083 "value_type": "individual" 7084 }, 7085 "type": "graph", 7086 "xaxis": { 7087 "buckets": null, 7088 "mode": "time", 7089 "name": null, 7090 "show": true, 7091 "values": [] 7092 }, 7093 "yaxes": [ 7094 { 7095 "format": "short", 7096 "label": null, 7097 "logBase": 1, 7098 "max": null, 7099 "min": null, 7100 "show": true 7101 }, 7102 { 7103 "format": "short", 7104 "label": null, 7105 "logBase": 1, 7106 "max": null, 7107 "min": null, 7108 "show": true 7109 } 7110 ], 7111 "yaxis": { 7112 "align": false, 7113 "alignLevel": null 7114 } 7115 }, 7116 { 7117 "aliasColors": {}, 7118 "bars": false, 7119 "dashLength": 10, 7120 "dashes": false, 7121 "datasource": "${DS_TEST-CLUSTER}", 7122 "fieldConfig": { 7123 "defaults": {}, 7124 "overrides": [] 7125 }, 7126 "fill": 1, 7127 "fillGradient": 0, 7128 "gridPos": { 7129 "h": 8, 7130 "w": 12, 7131 "x": 0, 7132 "y": 29 7133 }, 7134 "hiddenSeries": false, 7135 "id": 118, 7136 "legend": { 7137 "avg": false, 7138 "current": false, 7139 "max": false, 7140 "min": false, 7141 "show": true, 7142 "total": false, 7143 "values": false 7144 }, 7145 "lines": true, 7146 "linewidth": 1, 7147 "nullPointMode": "null", 7148 "options": { 7149 "alertThreshold": true 7150 }, 7151 "percentage": false, 7152 "pluginVersion": "7.5.11", 7153 "pointradius": 2, 7154 "points": false, 7155 "renderer": "flot", 7156 "seriesOverrides": [], 7157 "spaceLength": 10, 7158 "stack": false, 7159 "steppedLine": false, 7160 "targets": [ 7161 { 7162 "exemplar": true, 7163 "expr": "dm_validator_validator_logfile_latency", 7164 "interval": "", 7165 "legendFormat": "", 7166 "queryType": "randomWalk", 7167 "refId": "A" 7168 } 7169 ], 7170 "thresholds": [], 7171 "timeFrom": null, 7172 "timeRegions": [], 7173 "timeShift": null, 7174 "title": "Validator Binlog Latency (in file)", 7175 "tooltip": { 7176 "shared": true, 7177 "sort": 0, 7178 "value_type": "individual" 7179 }, 7180 "type": "graph", 7181 "xaxis": { 7182 "buckets": null, 7183 "mode": "time", 7184 "name": null, 7185 "show": true, 7186 "values": [] 7187 }, 7188 "yaxes": [ 7189 { 7190 "format": "short", 7191 "label": null, 7192 "logBase": 1, 7193 "max": null, 7194 "min": null, 7195 "show": true 7196 }, 7197 { 7198 "format": "short", 7199 "label": null, 7200 "logBase": 1, 7201 "max": null, 7202 "min": null, 7203 "show": true 7204 } 7205 ], 7206 "yaxis": { 7207 "align": false, 7208 "alignLevel": null 7209 } 7210 }, 7211 { 7212 "aliasColors": {}, 7213 "bars": false, 7214 "dashLength": 10, 7215 "dashes": false, 7216 "datasource": "${DS_TEST-CLUSTER}", 7217 "fieldConfig": { 7218 "defaults": {}, 7219 "overrides": [] 7220 }, 7221 "fill": 1, 7222 "fillGradient": 0, 7223 "gridPos": { 7224 "h": 8, 7225 "w": 12, 7226 "x": 12, 7227 "y": 29 7228 }, 7229 "hiddenSeries": false, 7230 "id": 112, 7231 "legend": { 7232 "avg": false, 7233 "current": false, 7234 "max": false, 7235 "min": false, 7236 "show": false, 7237 "total": false, 7238 "values": false 7239 }, 7240 "lines": true, 7241 "linewidth": 1, 7242 "nullPointMode": "null", 7243 "options": { 7244 "alertThreshold": true 7245 }, 7246 "percentage": false, 7247 "pluginVersion": "7.5.11", 7248 "pointradius": 2, 7249 "points": false, 7250 "renderer": "flot", 7251 "seriesOverrides": [], 7252 "spaceLength": 10, 7253 "stack": false, 7254 "steppedLine": false, 7255 "targets": [ 7256 { 7257 "exemplar": true, 7258 "expr": "dm_validator_validator_binlog_pos", 7259 "interval": "", 7260 "legendFormat": "", 7261 "queryType": "randomWalk", 7262 "refId": "A" 7263 } 7264 ], 7265 "thresholds": [], 7266 "timeFrom": null, 7267 "timeRegions": [], 7268 "timeShift": null, 7269 "title": "Validator Binlog Position (in byte)", 7270 "tooltip": { 7271 "shared": true, 7272 "sort": 0, 7273 "value_type": "individual" 7274 }, 7275 "type": "graph", 7276 "xaxis": { 7277 "buckets": null, 7278 "mode": "time", 7279 "name": null, 7280 "show": true, 7281 "values": [] 7282 }, 7283 "yaxes": [ 7284 { 7285 "format": "short", 7286 "label": null, 7287 "logBase": 1, 7288 "max": null, 7289 "min": null, 7290 "show": true 7291 }, 7292 { 7293 "format": "short", 7294 "label": null, 7295 "logBase": 1, 7296 "max": null, 7297 "min": null, 7298 "show": true 7299 } 7300 ], 7301 "yaxis": { 7302 "align": false, 7303 "alignLevel": null 7304 } 7305 } 7306 ], 7307 "title": "Validator Statistics", 7308 "type": "row" 7309 } 7310 ], 7311 "refresh": false, 7312 "schemaVersion": 27, 7313 "style": "dark", 7314 "tags": [], 7315 "templating": { 7316 "list": [ 7317 { 7318 "allValue": null, 7319 "current": {}, 7320 "datasource": "${DS_TEST-CLUSTER}", 7321 "definition": "label_values(dm_worker_task_state, job_id)", 7322 "description": null, 7323 "error": null, 7324 "hide": 0, 7325 "includeAll": true, 7326 "label": "JobID", 7327 "multi": true, 7328 "name": "job_id", 7329 "options": [], 7330 "query": { 7331 "query": "label_values(dm_worker_task_state, job_id)", 7332 "refId": "StandardVariableQuery" 7333 }, 7334 "refresh": 2, 7335 "regex": "", 7336 "skipUrlSync": false, 7337 "sort": 1, 7338 "tagValuesQuery": "", 7339 "tags": [], 7340 "tagsQuery": "", 7341 "type": "query", 7342 "useTags": false 7343 }, 7344 { 7345 "allValue": null, 7346 "current": {}, 7347 "datasource": "${DS_TEST-CLUSTER}", 7348 "definition": "label_values(dm_worker_task_state, task)", 7349 "description": null, 7350 "error": null, 7351 "hide": 0, 7352 "includeAll": true, 7353 "label": "Task", 7354 "multi": true, 7355 "name": "task", 7356 "options": [], 7357 "query": { 7358 "query": "label_values(dm_worker_task_state, task)", 7359 "refId": "prometheus-local-task-Variable-Query" 7360 }, 7361 "refresh": 2, 7362 "regex": "", 7363 "skipUrlSync": false, 7364 "sort": 1, 7365 "tagValuesQuery": "", 7366 "tags": [], 7367 "tagsQuery": "", 7368 "type": "query", 7369 "useTags": false 7370 }, 7371 { 7372 "allValue": null, 7373 "current": {}, 7374 "datasource": "${DS_TEST-CLUSTER}", 7375 "definition": "label_values(dm_worker_task_state, source_id)", 7376 "description": null, 7377 "error": null, 7378 "hide": 0, 7379 "includeAll": true, 7380 "label": "Source", 7381 "multi": true, 7382 "name": "source", 7383 "options": [], 7384 "query": { 7385 "query": "label_values(dm_worker_task_state, source_id)", 7386 "refId": "prometheus-local-source-Variable-Query" 7387 }, 7388 "refresh": 2, 7389 "regex": "", 7390 "skipUrlSync": false, 7391 "sort": 1, 7392 "tagValuesQuery": "", 7393 "tags": [], 7394 "tagsQuery": "", 7395 "type": "query", 7396 "useTags": false 7397 }, 7398 { 7399 "allValue": null, 7400 "current": {}, 7401 "datasource": "${DS_TEST-CLUSTER}", 7402 "definition": "label_values(dm_worker_task_state{job_id=~\"$job_id\",task=~\"$task\"}, instance)", 7403 "description": null, 7404 "error": null, 7405 "hide": 0, 7406 "includeAll": true, 7407 "label": "Instance", 7408 "multi": true, 7409 "name": "instance", 7410 "options": [], 7411 "query": { 7412 "query": "label_values(dm_worker_task_state{job_id=~\"$job_id\",task=~\"$task\"}, instance)", 7413 "refId": "prometheus-local-instance-Variable-Query" 7414 }, 7415 "refresh": 1, 7416 "regex": "", 7417 "skipUrlSync": false, 7418 "sort": 0, 7419 "tagValuesQuery": "", 7420 "tags": [], 7421 "tagsQuery": "", 7422 "type": "query", 7423 "useTags": false 7424 }, 7425 { 7426 "allValue": null, 7427 "current": {}, 7428 "datasource": "${DS_TEST-CLUSTER}", 7429 "definition": "label_values(dm_syncer_finished_jobs_total, queueNo)", 7430 "description": null, 7431 "error": null, 7432 "hide": 2, 7433 "includeAll": true, 7434 "label": "Queue", 7435 "multi": true, 7436 "name": "queueNo", 7437 "options": [], 7438 "query": { 7439 "query": "label_values(dm_syncer_finished_jobs_total, queueNo)", 7440 "refId": "prometheus-local-queueNo-Variable-Query" 7441 }, 7442 "refresh": 2, 7443 "regex": "", 7444 "skipUrlSync": false, 7445 "sort": 0, 7446 "tagValuesQuery": "", 7447 "tags": [], 7448 "tagsQuery": "", 7449 "type": "query", 7450 "useTags": false 7451 }, 7452 { 7453 "allValue": null, 7454 "current": {}, 7455 "datasource": "${DS_TEST-CLUSTER}", 7456 "definition": "label_values(dm_syncer_finished_jobs_total,type)", 7457 "description": null, 7458 "error": null, 7459 "hide": 0, 7460 "includeAll": true, 7461 "label": "Type", 7462 "multi": true, 7463 "name": "type", 7464 "options": [], 7465 "query": { 7466 "query": "label_values(dm_syncer_finished_jobs_total,type)", 7467 "refId": "prometheus-local-type-Variable-Query" 7468 }, 7469 "refresh": 2, 7470 "regex": "", 7471 "skipUrlSync": false, 7472 "sort": 0, 7473 "tagValuesQuery": "", 7474 "tags": [], 7475 "tagsQuery": "", 7476 "type": "query", 7477 "useTags": false 7478 }, 7479 { 7480 "allValue": null, 7481 "current": {}, 7482 "datasource": "${DS_TEST-CLUSTER}", 7483 "definition": "label_values(dm_syncer_binlog_transform_cost_count,stage)", 7484 "description": null, 7485 "error": null, 7486 "hide": 2, 7487 "includeAll": true, 7488 "label": "Stage", 7489 "multi": true, 7490 "name": "stage", 7491 "options": [], 7492 "query": { 7493 "query": "label_values(dm_syncer_binlog_transform_cost_count,stage)", 7494 "refId": "prometheus-local-stage-Variable-Query" 7495 }, 7496 "refresh": 2, 7497 "regex": "", 7498 "skipUrlSync": false, 7499 "sort": 0, 7500 "tagValuesQuery": "", 7501 "tags": [], 7502 "tagsQuery": "", 7503 "type": "query", 7504 "useTags": false 7505 }, 7506 { 7507 "allValue": null, 7508 "current": {}, 7509 "datasource": "${DS_TEST-CLUSTER}", 7510 "definition": "label_values(dm_syncer_finished_jobs_total, target_schema)", 7511 "description": null, 7512 "error": null, 7513 "hide": 0, 7514 "includeAll": true, 7515 "label": "Target Schema", 7516 "multi": true, 7517 "name": "target_schema", 7518 "options": [], 7519 "query": { 7520 "query": "label_values(dm_syncer_finished_jobs_total, target_schema)", 7521 "refId": "prometheus-local-target_schema-Variable-Query" 7522 }, 7523 "refresh": 2, 7524 "regex": "", 7525 "skipUrlSync": false, 7526 "sort": 0, 7527 "tagValuesQuery": "", 7528 "tags": [], 7529 "tagsQuery": "", 7530 "type": "query", 7531 "useTags": false 7532 }, 7533 { 7534 "allValue": null, 7535 "current": {}, 7536 "datasource": "${DS_TEST-CLUSTER}", 7537 "definition": "label_values(dm_syncer_finished_jobs_total, target_table)", 7538 "description": null, 7539 "error": null, 7540 "hide": 2, 7541 "includeAll": true, 7542 "label": "Target_table", 7543 "multi": true, 7544 "name": "target_table", 7545 "options": [], 7546 "query": { 7547 "query": "label_values(dm_syncer_finished_jobs_total, target_table)", 7548 "refId": "prometheus-local-target_table-Variable-Query" 7549 }, 7550 "refresh": 2, 7551 "regex": "", 7552 "skipUrlSync": false, 7553 "sort": 0, 7554 "tagValuesQuery": "", 7555 "tags": [], 7556 "tagsQuery": "", 7557 "type": "query", 7558 "useTags": false 7559 }, 7560 { 7561 "auto": true, 7562 "auto_count": 30, 7563 "auto_min": "10s", 7564 "current": { 7565 "selected": false, 7566 "text": "auto", 7567 "value": "$__auto_interval_interval" 7568 }, 7569 "description": null, 7570 "error": null, 7571 "hide": 0, 7572 "label": "Interval", 7573 "name": "interval", 7574 "options": [ 7575 { 7576 "selected": true, 7577 "text": "auto", 7578 "value": "$__auto_interval_interval" 7579 }, 7580 { 7581 "selected": false, 7582 "text": "30s", 7583 "value": "30s" 7584 }, 7585 { 7586 "selected": false, 7587 "text": "1m", 7588 "value": "1m" 7589 }, 7590 { 7591 "selected": false, 7592 "text": "10m", 7593 "value": "10m" 7594 }, 7595 { 7596 "selected": false, 7597 "text": "30m", 7598 "value": "30m" 7599 }, 7600 { 7601 "selected": false, 7602 "text": "1h", 7603 "value": "1h" 7604 }, 7605 { 7606 "selected": false, 7607 "text": "6h", 7608 "value": "6h" 7609 }, 7610 { 7611 "selected": false, 7612 "text": "12h", 7613 "value": "12h" 7614 }, 7615 { 7616 "selected": false, 7617 "text": "1d", 7618 "value": "1d" 7619 }, 7620 { 7621 "selected": false, 7622 "text": "7d", 7623 "value": "7d" 7624 }, 7625 { 7626 "selected": false, 7627 "text": "14d", 7628 "value": "14d" 7629 }, 7630 { 7631 "selected": false, 7632 "text": "30d", 7633 "value": "30d" 7634 } 7635 ], 7636 "query": "30s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", 7637 "refresh": 2, 7638 "skipUrlSync": false, 7639 "type": "interval" 7640 }, 7641 { 7642 "allValue": null, 7643 "current": {}, 7644 "datasource": "${DS_TEST-CLUSTER}", 7645 "definition": "label_values(dm_syncer_finished_jobs_total,worker)", 7646 "description": null, 7647 "error": null, 7648 "hide": 2, 7649 "includeAll": true, 7650 "label": "Worker", 7651 "multi": true, 7652 "name": "worker", 7653 "options": [], 7654 "query": { 7655 "query": "label_values(dm_syncer_finished_jobs_total,worker)", 7656 "refId": "prometheus-local-worker-Variable-Query" 7657 }, 7658 "refresh": 2, 7659 "regex": "", 7660 "skipUrlSync": false, 7661 "sort": 0, 7662 "tagValuesQuery": "", 7663 "tags": [], 7664 "tagsQuery": "", 7665 "type": "query", 7666 "useTags": false 7667 } 7668 ] 7669 }, 7670 "time": { 7671 "from": "now-1h", 7672 "to": "now" 7673 }, 7674 "timepicker": { 7675 "refresh_intervals": [ 7676 "5s", 7677 "10s", 7678 "30s", 7679 "1m", 7680 "5m", 7681 "15m", 7682 "30m", 7683 "1h", 7684 "2h", 7685 "1d" 7686 ], 7687 "time_options": [ 7688 "5m", 7689 "15m", 7690 "1h", 7691 "6h", 7692 "12h", 7693 "24h", 7694 "2d", 7695 "7d", 7696 "30d" 7697 ] 7698 }, 7699 "timezone": "", 7700 "title": "DM-Monitor-Professional", 7701 "uid": "Ja68YqRnz", 7702 "version": 13 7703 }