github.com/pingcap/br@v5.3.0-alpha.0.20220125034240-ec59c7b6ce30+incompatible/metrics/grafana/br.json (about) 1 { 2 "__inputs": [ 3 { 4 "name": "DS_TEST-CLUSTER", 5 "label": "test-cluster", 6 "description": "", 7 "type": "datasource", 8 "pluginId": "prometheus", 9 "pluginName": "Prometheus" 10 } 11 ], 12 "__requires": [ 13 { 14 "type": "grafana", 15 "id": "grafana", 16 "name": "Grafana", 17 "version": "6.1.6" 18 }, 19 { 20 "type": "panel", 21 "id": "graph", 22 "name": "Graph", 23 "version": "" 24 }, 25 { 26 "type": "panel", 27 "id": "heatmap", 28 "name": "Heatmap", 29 "version": "" 30 }, 31 { 32 "type": "datasource", 33 "id": "prometheus", 34 "name": "Prometheus", 35 "version": "1.0.0" 36 }, 37 { 38 "type": "panel", 39 "id": "table", 40 "name": "Table", 41 "version": "" 42 } 43 ], 44 "annotations": { 45 "list": [ 46 { 47 "builtIn": 1, 48 "datasource": "${DS_TEST-CLUSTER}", 49 "enable": true, 50 "hide": true, 51 "iconColor": "rgba(0, 211, 255, 1)", 52 "name": "Annotations & Alerts", 53 "type": "dashboard" 54 } 55 ] 56 }, 57 "editable": true, 58 "gnetId": null, 59 "graphTooltip": 1, 60 "id": null, 61 "iteration": 1609129954697, 62 "links": [], 63 "panels": [ 64 { 65 "collapsed": true, 66 "gridPos": { 67 "h": 1, 68 "w": 24, 69 "x": 0, 70 "y": 0 71 }, 72 "id": 15, 73 "panels": [ 74 { 75 "aliasColors": {}, 76 "bars": false, 77 "dashLength": 10, 78 "dashes": false, 79 "datasource": "${DS_TEST-CLUSTER}", 80 "decimals": 1, 81 "editable": true, 82 "error": false, 83 "fill": 0, 84 "grid": {}, 85 "gridPos": { 86 "h": 6, 87 "w": 12, 88 "x": 0, 89 "y": 1 90 }, 91 "id": 4, 92 "legend": { 93 "alignAsTable": true, 94 "avg": false, 95 "current": true, 96 "hideEmpty": true, 97 "max": true, 98 "min": false, 99 "rightSide": true, 100 "show": true, 101 "sideWidth": null, 102 "sort": "current", 103 "sortDesc": true, 104 "total": false, 105 "values": true 106 }, 107 "lines": true, 108 "linewidth": 1, 109 "links": [], 110 "nullPointMode": "null", 111 "percentage": false, 112 "pointradius": 5, 113 "points": false, 114 "renderer": "flot", 115 "seriesOverrides": [ 116 { 117 "alias": "/backup-count.*/", 118 "yaxis": 2 119 } 120 ], 121 "spaceLength": 10, 122 "stack": false, 123 "steppedLine": false, 124 "targets": [ 125 { 126 "expr": "sum(rate(tikv_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(backup-worker|bkwkr).*\"}[1m])) by (instance)", 127 "format": "time_series", 128 "hide": false, 129 "intervalFactor": 2, 130 "legendFormat": "backup-{{instance}}", 131 "metric": "tikv_thread_cpu_seconds_total", 132 "refId": "A", 133 "step": 4 134 }, 135 { 136 "expr": "sum(rate(tikv_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"backup_endpoint\"}[1m])) by (instance)", 137 "format": "time_series", 138 "hide": true, 139 "intervalFactor": 2, 140 "legendFormat": "backup-endpoint", 141 "metric": "tikv_thread_cpu_seconds_total", 142 "refId": "B", 143 "step": 4 144 }, 145 { 146 "expr": "sum(rate(tikv_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"(backup-worker|bkwkr).*\"}[1m])) by (instance, tid) > 0", 147 "format": "time_series", 148 "hide": true, 149 "intervalFactor": 2, 150 "legendFormat": "backup-{{instance}}-{{tid}}", 151 "metric": "tikv_thread_cpu_seconds_total", 152 "refId": "C", 153 "step": 4 154 }, 155 { 156 "expr": "sum(tikv_backup_thread_pool_size{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by(instance)", 157 "format": "time_series", 158 "hide": true, 159 "intervalFactor": 2, 160 "legendFormat": "backup-count-{{instance}}", 161 "metric": "tikv_thread_cpu_seconds_total", 162 "refId": "D", 163 "step": 4 164 } 165 ], 166 "thresholds": [], 167 "timeFrom": null, 168 "timeRegions": [], 169 "timeShift": null, 170 "title": "Backup CPU Utilization", 171 "tooltip": { 172 "msResolution": false, 173 "shared": true, 174 "sort": 0, 175 "value_type": "individual" 176 }, 177 "type": "graph", 178 "xaxis": { 179 "buckets": null, 180 "mode": "time", 181 "name": null, 182 "show": true, 183 "values": [] 184 }, 185 "yaxes": [ 186 { 187 "format": "percentunit", 188 "label": null, 189 "logBase": 1, 190 "max": null, 191 "min": null, 192 "show": true 193 }, 194 { 195 "format": "short", 196 "label": null, 197 "logBase": 1, 198 "max": null, 199 "min": null, 200 "show": true 201 } 202 ], 203 "yaxis": { 204 "align": false, 205 "alignLevel": null 206 } 207 }, 208 { 209 "columns": [ 210 { 211 "text": "Current", 212 "value": "current" 213 } 214 ], 215 "datasource": "${DS_TEST-CLUSTER}", 216 "fontSize": "100%", 217 "gridPos": { 218 "h": 6, 219 "w": 4, 220 "x": 12, 221 "y": 1 222 }, 223 "id": 35, 224 "links": [], 225 "pageSize": null, 226 "scroll": true, 227 "showHeader": true, 228 "sort": { 229 "col": 0, 230 "desc": true 231 }, 232 "styles": [ 233 { 234 "alias": "Time", 235 "dateFormat": "YYYY-MM-DD HH:mm:ss", 236 "pattern": "Time", 237 "type": "date" 238 }, 239 { 240 "alias": "", 241 "colorMode": null, 242 "colors": [ 243 "rgba(245, 54, 54, 0.9)", 244 "rgba(237, 129, 40, 0.89)", 245 "rgba(50, 172, 45, 0.97)" 246 ], 247 "decimals": 2, 248 "pattern": "/.*/", 249 "thresholds": [], 250 "type": "number", 251 "unit": "short" 252 } 253 ], 254 "targets": [ 255 { 256 "expr": "sum(tikv_backup_thread_pool_size{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}) by(instance)", 257 "format": "time_series", 258 "interval": "", 259 "intervalFactor": 1, 260 "legendFormat": "{{instance}}", 261 "refId": "A" 262 } 263 ], 264 "timeFrom": null, 265 "timeShift": null, 266 "title": "Backup Thread Count", 267 "transform": "timeseries_aggregations", 268 "type": "table" 269 }, 270 { 271 "aliasColors": {}, 272 "bars": false, 273 "dashLength": 10, 274 "dashes": false, 275 "datasource": "${DS_TEST-CLUSTER}", 276 "decimals": 2, 277 "description": "", 278 "fill": 1, 279 "gridPos": { 280 "h": 6, 281 "w": 8, 282 "x": 16, 283 "y": 1 284 }, 285 "id": 10, 286 "legend": { 287 "alignAsTable": true, 288 "avg": false, 289 "current": true, 290 "max": true, 291 "min": false, 292 "rightSide": true, 293 "show": true, 294 "total": false, 295 "values": true 296 }, 297 "lines": true, 298 "linewidth": 1, 299 "links": [], 300 "nullPointMode": "null as zero", 301 "percentage": false, 302 "pointradius": 2, 303 "points": false, 304 "renderer": "flot", 305 "seriesOverrides": [], 306 "spaceLength": 10, 307 "stack": false, 308 "steppedLine": false, 309 "targets": [ 310 { 311 "expr": "delta(tikv_backup_error_counter{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])", 312 "format": "time_series", 313 "hide": false, 314 "intervalFactor": 1, 315 "legendFormat": "{{error}} {{instance}}", 316 "refId": "A" 317 } 318 ], 319 "thresholds": [], 320 "timeFrom": null, 321 "timeRegions": [], 322 "timeShift": null, 323 "title": "Backup Errors", 324 "tooltip": { 325 "shared": true, 326 "sort": 0, 327 "value_type": "individual" 328 }, 329 "type": "graph", 330 "xaxis": { 331 "buckets": null, 332 "mode": "time", 333 "name": null, 334 "show": true, 335 "values": [] 336 }, 337 "yaxes": [ 338 { 339 "format": "none", 340 "label": null, 341 "logBase": 1, 342 "max": null, 343 "min": "0", 344 "show": true 345 }, 346 { 347 "format": "none", 348 "label": null, 349 "logBase": 1, 350 "max": null, 351 "min": "0", 352 "show": true 353 } 354 ], 355 "yaxis": { 356 "align": false, 357 "alignLevel": null 358 } 359 }, 360 { 361 "cards": { 362 "cardPadding": 0, 363 "cardRound": 0 364 }, 365 "color": { 366 "cardColor": "#FF9830", 367 "colorScale": "linear", 368 "colorScheme": "interpolateSpectral", 369 "exponent": 0.5, 370 "max": null, 371 "min": 0, 372 "mode": "spectrum" 373 }, 374 "dataFormat": "tsbuckets", 375 "datasource": "${DS_TEST-CLUSTER}", 376 "description": "", 377 "gridPos": { 378 "h": 6, 379 "w": 8, 380 "x": 0, 381 "y": 7 382 }, 383 "heatmap": {}, 384 "hideZeroBuckets": true, 385 "highlightCards": true, 386 "id": 37, 387 "legend": { 388 "alignAsTable": true, 389 "avg": false, 390 "current": true, 391 "max": true, 392 "min": false, 393 "rightSide": true, 394 "show": true, 395 "sort": "current", 396 "sortDesc": true, 397 "total": false, 398 "values": true 399 }, 400 "links": [], 401 "reverseYBuckets": false, 402 "targets": [ 403 { 404 "expr": "max(rate(tikv_backup_range_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", cf=\"write\"}[1m])) by (le)", 405 "format": "heatmap", 406 "instant": false, 407 "intervalFactor": 2, 408 "legendFormat": "{{le}}", 409 "refId": "A" 410 } 411 ], 412 "title": "Backup Write CF SST Size", 413 "tooltip": { 414 "show": true, 415 "showHistogram": true 416 }, 417 "tooltipDecimals": 1, 418 "type": "heatmap", 419 "xAxis": { 420 "show": true 421 }, 422 "xBucketNumber": null, 423 "xBucketSize": null, 424 "yAxis": { 425 "decimals": 1, 426 "format": "bytes", 427 "logBase": 1, 428 "max": null, 429 "min": null, 430 "show": true, 431 "splitFactor": null 432 }, 433 "yBucketBound": "upper", 434 "yBucketNumber": null, 435 "yBucketSize": null 436 }, 437 { 438 "cards": { 439 "cardPadding": 0, 440 "cardRound": 0 441 }, 442 "color": { 443 "cardColor": "#FF9830", 444 "colorScale": "linear", 445 "colorScheme": "interpolateSpectral", 446 "exponent": 0.5, 447 "max": null, 448 "min": 0, 449 "mode": "spectrum" 450 }, 451 "dataFormat": "tsbuckets", 452 "datasource": "${DS_TEST-CLUSTER}", 453 "description": "", 454 "gridPos": { 455 "h": 6, 456 "w": 8, 457 "x": 8, 458 "y": 7 459 }, 460 "heatmap": {}, 461 "hideZeroBuckets": true, 462 "highlightCards": true, 463 "id": 38, 464 "legend": { 465 "alignAsTable": true, 466 "avg": false, 467 "current": true, 468 "max": true, 469 "min": false, 470 "rightSide": true, 471 "show": true, 472 "sort": "current", 473 "sortDesc": true, 474 "total": false, 475 "values": true 476 }, 477 "links": [], 478 "reverseYBuckets": false, 479 "targets": [ 480 { 481 "expr": "max(rate(tikv_backup_range_size_bytes_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", cf=\"default\"}[1m])) by (le)", 482 "format": "heatmap", 483 "instant": false, 484 "intervalFactor": 2, 485 "legendFormat": "{{le}}", 486 "refId": "A" 487 } 488 ], 489 "title": "Backup Default CF SST Size", 490 "tooltip": { 491 "show": true, 492 "showHistogram": true 493 }, 494 "tooltipDecimals": 1, 495 "type": "heatmap", 496 "xAxis": { 497 "show": true 498 }, 499 "xBucketNumber": null, 500 "xBucketSize": null, 501 "yAxis": { 502 "decimals": 1, 503 "format": "bytes", 504 "logBase": 1, 505 "max": null, 506 "min": null, 507 "show": true, 508 "splitFactor": null 509 }, 510 "yBucketBound": "upper", 511 "yBucketNumber": null, 512 "yBucketSize": null 513 }, 514 { 515 "aliasColors": {}, 516 "bars": false, 517 "dashLength": 10, 518 "dashes": false, 519 "datasource": "${DS_TEST-CLUSTER}", 520 "editable": true, 521 "error": false, 522 "fill": 1, 523 "grid": {}, 524 "gridPos": { 525 "h": 6, 526 "w": 8, 527 "x": 16, 528 "y": 7 529 }, 530 "id": 2, 531 "legend": { 532 "alignAsTable": true, 533 "avg": false, 534 "current": true, 535 "max": true, 536 "min": false, 537 "rightSide": true, 538 "show": true, 539 "sort": "max", 540 "sortDesc": true, 541 "total": false, 542 "values": true 543 }, 544 "lines": true, 545 "linewidth": 1, 546 "links": [], 547 "nullPointMode": "null", 548 "percentage": false, 549 "pointradius": 5, 550 "points": false, 551 "renderer": "flot", 552 "seriesOverrides": [ 553 { 554 "alias": "total", 555 "yaxis": 2 556 } 557 ], 558 "spaceLength": 10, 559 "stack": false, 560 "steppedLine": false, 561 "targets": [ 562 { 563 "expr": "sum(rate(tikv_backup_range_size_bytes_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", 564 "format": "time_series", 565 "hide": false, 566 "intervalFactor": 2, 567 "legendFormat": "total", 568 "metric": "", 569 "refId": "A", 570 "step": 4 571 }, 572 { 573 "expr": "rate(tikv_backup_range_size_bytes_sum{tidb_cluster=\"$tidb_cluster\"}[1m])", 574 "format": "time_series", 575 "hide": false, 576 "intervalFactor": 2, 577 "legendFormat": "{{instance}} {{cf}}", 578 "metric": "", 579 "refId": "B", 580 "step": 4 581 } 582 ], 583 "thresholds": [], 584 "timeFrom": null, 585 "timeRegions": [], 586 "timeShift": null, 587 "title": "Backup SST Generation Throughput", 588 "tooltip": { 589 "msResolution": false, 590 "shared": true, 591 "sort": 0, 592 "value_type": "cumulative" 593 }, 594 "type": "graph", 595 "xaxis": { 596 "buckets": null, 597 "mode": "time", 598 "name": null, 599 "show": true, 600 "values": [] 601 }, 602 "yaxes": [ 603 { 604 "format": "Bps", 605 "label": null, 606 "logBase": 1, 607 "max": null, 608 "min": null, 609 "show": true 610 }, 611 { 612 "format": "Bps", 613 "label": null, 614 "logBase": 1, 615 "max": null, 616 "min": null, 617 "show": true 618 } 619 ], 620 "yaxis": { 621 "align": false, 622 "alignLevel": null 623 } 624 }, 625 { 626 "cards": { 627 "cardPadding": 0, 628 "cardRound": 0 629 }, 630 "color": { 631 "cardColor": "#FF9830", 632 "colorScale": "linear", 633 "colorScheme": "interpolateSpectral", 634 "exponent": 0.5, 635 "max": null, 636 "min": 0, 637 "mode": "spectrum" 638 }, 639 "dataFormat": "tsbuckets", 640 "datasource": "${DS_TEST-CLUSTER}", 641 "description": "", 642 "gridPos": { 643 "h": 6, 644 "w": 4, 645 "x": 0, 646 "y": 13 647 }, 648 "heatmap": {}, 649 "hideZeroBuckets": true, 650 "highlightCards": true, 651 "id": 44, 652 "legend": { 653 "alignAsTable": true, 654 "avg": false, 655 "current": true, 656 "max": true, 657 "min": false, 658 "rightSide": true, 659 "show": true, 660 "sort": "current", 661 "sortDesc": true, 662 "total": false, 663 "values": true 664 }, 665 "links": [], 666 "reverseYBuckets": false, 667 "targets": [ 668 { 669 "expr": "max(rate(tikv_backup_range_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"snapshot\"}[1m])) by (le)", 670 "format": "heatmap", 671 "instant": false, 672 "intervalFactor": 2, 673 "legendFormat": "{{le}}", 674 "refId": "A" 675 } 676 ], 677 "title": "Backup Snapshot Duration", 678 "tooltip": { 679 "show": true, 680 "showHistogram": true 681 }, 682 "tooltipDecimals": 1, 683 "type": "heatmap", 684 "xAxis": { 685 "show": true 686 }, 687 "xBucketNumber": null, 688 "xBucketSize": null, 689 "yAxis": { 690 "decimals": 1, 691 "format": "s", 692 "logBase": 1, 693 "max": null, 694 "min": null, 695 "show": true, 696 "splitFactor": null 697 }, 698 "yBucketBound": "upper", 699 "yBucketNumber": null, 700 "yBucketSize": null 701 }, 702 { 703 "cards": { 704 "cardPadding": 0, 705 "cardRound": 0 706 }, 707 "color": { 708 "cardColor": "#FF9830", 709 "colorScale": "linear", 710 "colorScheme": "interpolateSpectral", 711 "exponent": 0.5, 712 "max": null, 713 "min": 0, 714 "mode": "spectrum" 715 }, 716 "dataFormat": "tsbuckets", 717 "datasource": "${DS_TEST-CLUSTER}", 718 "description": "", 719 "gridPos": { 720 "h": 6, 721 "w": 6, 722 "x": 4, 723 "y": 13 724 }, 725 "heatmap": {}, 726 "hideZeroBuckets": true, 727 "highlightCards": true, 728 "id": 40, 729 "legend": { 730 "alignAsTable": true, 731 "avg": false, 732 "current": true, 733 "max": true, 734 "min": false, 735 "rightSide": true, 736 "show": true, 737 "sort": "current", 738 "sortDesc": true, 739 "total": false, 740 "values": true 741 }, 742 "links": [], 743 "reverseYBuckets": false, 744 "targets": [ 745 { 746 "expr": "max(rate(tikv_backup_range_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"scan\"}[1m])) by (le)", 747 "format": "heatmap", 748 "instant": false, 749 "intervalFactor": 2, 750 "legendFormat": "{{le}}", 751 "refId": "A" 752 } 753 ], 754 "title": "Backup Scan SST Duration", 755 "tooltip": { 756 "show": true, 757 "showHistogram": true 758 }, 759 "tooltipDecimals": 1, 760 "type": "heatmap", 761 "xAxis": { 762 "show": true 763 }, 764 "xBucketNumber": null, 765 "xBucketSize": null, 766 "yAxis": { 767 "decimals": 1, 768 "format": "s", 769 "logBase": 1, 770 "max": null, 771 "min": null, 772 "show": true, 773 "splitFactor": null 774 }, 775 "yBucketBound": "upper", 776 "yBucketNumber": null, 777 "yBucketSize": null 778 }, 779 { 780 "cards": { 781 "cardPadding": 0, 782 "cardRound": 0 783 }, 784 "color": { 785 "cardColor": "#FF9830", 786 "colorScale": "linear", 787 "colorScheme": "interpolateSpectral", 788 "exponent": 0.5, 789 "max": null, 790 "min": 0, 791 "mode": "spectrum" 792 }, 793 "dataFormat": "tsbuckets", 794 "datasource": "${DS_TEST-CLUSTER}", 795 "description": "", 796 "gridPos": { 797 "h": 6, 798 "w": 6, 799 "x": 10, 800 "y": 13 801 }, 802 "heatmap": {}, 803 "hideZeroBuckets": true, 804 "highlightCards": true, 805 "id": 39, 806 "legend": { 807 "alignAsTable": true, 808 "avg": false, 809 "current": true, 810 "max": true, 811 "min": false, 812 "rightSide": true, 813 "show": true, 814 "sort": "current", 815 "sortDesc": true, 816 "total": false, 817 "values": true 818 }, 819 "links": [], 820 "reverseYBuckets": false, 821 "targets": [ 822 { 823 "expr": "max(rate(tikv_backup_range_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"save.*\"}[1m])) by (le)", 824 "format": "heatmap", 825 "instant": false, 826 "intervalFactor": 2, 827 "legendFormat": "{{le}}", 828 "refId": "A" 829 } 830 ], 831 "title": "Backup Save SST Duration", 832 "tooltip": { 833 "show": true, 834 "showHistogram": true 835 }, 836 "tooltipDecimals": 1, 837 "type": "heatmap", 838 "xAxis": { 839 "show": true 840 }, 841 "xBucketNumber": null, 842 "xBucketSize": null, 843 "yAxis": { 844 "decimals": 1, 845 "format": "s", 846 "logBase": 1, 847 "max": null, 848 "min": null, 849 "show": true, 850 "splitFactor": null 851 }, 852 "yBucketBound": "upper", 853 "yBucketNumber": null, 854 "yBucketSize": null 855 }, 856 { 857 "aliasColors": {}, 858 "bars": false, 859 "dashLength": 10, 860 "dashes": false, 861 "datasource": "${DS_TEST-CLUSTER}", 862 "editable": true, 863 "error": false, 864 "fill": 1, 865 "grid": {}, 866 "gridPos": { 867 "h": 6, 868 "w": 8, 869 "x": 16, 870 "y": 13 871 }, 872 "id": 6, 873 "legend": { 874 "alignAsTable": true, 875 "avg": false, 876 "current": true, 877 "max": true, 878 "min": false, 879 "rightSide": true, 880 "show": true, 881 "sort": "max", 882 "sortDesc": true, 883 "total": false, 884 "values": true 885 }, 886 "lines": true, 887 "linewidth": 1, 888 "links": [], 889 "nullPointMode": "null", 890 "percentage": false, 891 "pointradius": 5, 892 "points": false, 893 "renderer": "flot", 894 "seriesOverrides": [], 895 "spaceLength": 10, 896 "stack": false, 897 "steppedLine": false, 898 "targets": [ 899 { 900 "expr": "histogram_quantile(0.99, sum(rate(tikv_backup_range_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", 901 "format": "time_series", 902 "intervalFactor": 2, 903 "legendFormat": "{{type}} - 99%", 904 "metric": "", 905 "refId": "A", 906 "step": 4 907 }, 908 { 909 "expr": "histogram_quantile(0.95, sum(rate(tikv_backup_range_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, type))", 910 "format": "time_series", 911 "intervalFactor": 2, 912 "legendFormat": "{{type}} - 95%", 913 "refId": "B", 914 "step": 4 915 }, 916 { 917 "expr": "sum(rate(tikv_backup_range_duration_seconds_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type) / sum(rate(tikv_backup_range_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (type)", 918 "format": "time_series", 919 "intervalFactor": 2, 920 "legendFormat": "{{type}} - avg", 921 "refId": "C", 922 "step": 4 923 } 924 ], 925 "thresholds": [], 926 "timeFrom": null, 927 "timeRegions": [], 928 "timeShift": null, 929 "title": "Backup SST Duration", 930 "tooltip": { 931 "msResolution": false, 932 "shared": true, 933 "sort": 0, 934 "value_type": "cumulative" 935 }, 936 "type": "graph", 937 "xaxis": { 938 "buckets": null, 939 "mode": "time", 940 "name": null, 941 "show": true, 942 "values": [] 943 }, 944 "yaxes": [ 945 { 946 "format": "s", 947 "label": null, 948 "logBase": 1, 949 "max": null, 950 "min": null, 951 "show": true 952 }, 953 { 954 "format": "short", 955 "label": null, 956 "logBase": 1, 957 "max": null, 958 "min": null, 959 "show": true 960 } 961 ], 962 "yaxis": { 963 "align": false, 964 "alignLevel": null 965 } 966 }, 967 { 968 "cards": { 969 "cardPadding": 0, 970 "cardRound": 0 971 }, 972 "color": { 973 "cardColor": "#FF9830", 974 "colorScale": "linear", 975 "colorScheme": "interpolateSpectral", 976 "exponent": 0.5, 977 "max": null, 978 "min": 0, 979 "mode": "spectrum" 980 }, 981 "dataFormat": "tsbuckets", 982 "datasource": "${DS_TEST-CLUSTER}", 983 "description": "", 984 "gridPos": { 985 "h": 7, 986 "w": 12, 987 "x": 0, 988 "y": 19 989 }, 990 "heatmap": {}, 991 "hideZeroBuckets": true, 992 "highlightCards": true, 993 "id": 41, 994 "legend": { 995 "alignAsTable": true, 996 "avg": false, 997 "current": true, 998 "max": true, 999 "min": false, 1000 "rightSide": true, 1001 "show": true, 1002 "sort": "current", 1003 "sortDesc": true, 1004 "total": false, 1005 "values": true 1006 }, 1007 "links": [], 1008 "reverseYBuckets": false, 1009 "targets": [ 1010 { 1011 "expr": "max(rate(tikv_external_storage_create_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le)", 1012 "format": "heatmap", 1013 "instant": false, 1014 "intervalFactor": 2, 1015 "legendFormat": "{{le}}", 1016 "refId": "A" 1017 } 1018 ], 1019 "title": "External Storage Create Duration", 1020 "tooltip": { 1021 "show": true, 1022 "showHistogram": true 1023 }, 1024 "tooltipDecimals": 1, 1025 "type": "heatmap", 1026 "xAxis": { 1027 "show": true 1028 }, 1029 "xBucketNumber": null, 1030 "xBucketSize": null, 1031 "yAxis": { 1032 "decimals": 1, 1033 "format": "s", 1034 "logBase": 1, 1035 "max": null, 1036 "min": null, 1037 "show": true, 1038 "splitFactor": null 1039 }, 1040 "yBucketBound": "upper", 1041 "yBucketNumber": null, 1042 "yBucketSize": null 1043 }, 1044 { 1045 "aliasColors": {}, 1046 "bars": false, 1047 "dashLength": 10, 1048 "dashes": false, 1049 "datasource": "${DS_TEST-CLUSTER}", 1050 "decimals": 1, 1051 "description": "", 1052 "editable": true, 1053 "error": false, 1054 "fill": 1, 1055 "grid": {}, 1056 "gridPos": { 1057 "h": 7, 1058 "w": 12, 1059 "x": 12, 1060 "y": 19 1061 }, 1062 "id": 43, 1063 "legend": { 1064 "alignAsTable": true, 1065 "avg": false, 1066 "current": true, 1067 "hideEmpty": false, 1068 "max": true, 1069 "min": false, 1070 "rightSide": true, 1071 "show": true, 1072 "sideWidth": null, 1073 "sort": "current", 1074 "sortDesc": true, 1075 "total": false, 1076 "values": true 1077 }, 1078 "lines": true, 1079 "linewidth": 1, 1080 "links": [], 1081 "nullPointMode": "null as zero", 1082 "percentage": false, 1083 "pointradius": 5, 1084 "points": false, 1085 "renderer": "flot", 1086 "seriesOverrides": [], 1087 "spaceLength": 10, 1088 "stack": false, 1089 "steppedLine": false, 1090 "targets": [ 1091 { 1092 "expr": "histogram_quantile(1, sum(rate(tikv_external_storage_create_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le,type))", 1093 "format": "time_series", 1094 "intervalFactor": 2, 1095 "legendFormat": "{{type}}-100%", 1096 "refId": "E" 1097 }, 1098 { 1099 "expr": "histogram_quantile(0.99, sum(rate(tikv_external_storage_create_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le,type))", 1100 "format": "time_series", 1101 "intervalFactor": 2, 1102 "legendFormat": "{{type}}-99%", 1103 "refId": "A", 1104 "step": 4 1105 } 1106 ], 1107 "thresholds": [], 1108 "timeFrom": null, 1109 "timeRegions": [], 1110 "timeShift": null, 1111 "title": "External Storage Create Duration", 1112 "tooltip": { 1113 "msResolution": false, 1114 "shared": true, 1115 "sort": 1, 1116 "value_type": "cumulative" 1117 }, 1118 "type": "graph", 1119 "xaxis": { 1120 "buckets": null, 1121 "mode": "time", 1122 "name": null, 1123 "show": true, 1124 "values": [] 1125 }, 1126 "yaxes": [ 1127 { 1128 "decimals": null, 1129 "format": "s", 1130 "label": null, 1131 "logBase": 1, 1132 "max": null, 1133 "min": null, 1134 "show": true 1135 }, 1136 { 1137 "format": "short", 1138 "label": null, 1139 "logBase": 1, 1140 "max": null, 1141 "min": null, 1142 "show": false 1143 } 1144 ], 1145 "yaxis": { 1146 "align": false, 1147 "alignLevel": null 1148 } 1149 }, 1150 { 1151 "aliasColors": {}, 1152 "bars": false, 1153 "dashLength": 10, 1154 "dashes": false, 1155 "datasource": "${DS_TEST-CLUSTER}", 1156 "decimals": 1, 1157 "description": "", 1158 "editable": true, 1159 "error": false, 1160 "fill": 1, 1161 "grid": {}, 1162 "gridPos": { 1163 "h": 6, 1164 "w": 12, 1165 "x": 0, 1166 "y": 26 1167 }, 1168 "id": 12, 1169 "legend": { 1170 "alignAsTable": true, 1171 "avg": false, 1172 "current": true, 1173 "hideEmpty": false, 1174 "max": true, 1175 "min": false, 1176 "rightSide": true, 1177 "show": true, 1178 "sideWidth": null, 1179 "sort": "current", 1180 "sortDesc": true, 1181 "total": false, 1182 "values": true 1183 }, 1184 "lines": true, 1185 "linewidth": 1, 1186 "links": [], 1187 "nullPointMode": "null as zero", 1188 "percentage": false, 1189 "pointradius": 5, 1190 "points": false, 1191 "renderer": "flot", 1192 "seriesOverrides": [], 1193 "spaceLength": 10, 1194 "stack": false, 1195 "steppedLine": false, 1196 "targets": [ 1197 { 1198 "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[1m])) by (le,req))", 1199 "format": "time_series", 1200 "intervalFactor": 2, 1201 "legendFormat": "{{req}}-100%", 1202 "refId": "E" 1203 }, 1204 { 1205 "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[1m])) by (le,req))", 1206 "format": "time_series", 1207 "intervalFactor": 2, 1208 "legendFormat": "{{req}}-99%", 1209 "refId": "A", 1210 "step": 4 1211 } 1212 ], 1213 "thresholds": [], 1214 "timeFrom": null, 1215 "timeRegions": [], 1216 "timeShift": null, 1217 "title": "Checksum Request Duration", 1218 "tooltip": { 1219 "msResolution": false, 1220 "shared": true, 1221 "sort": 1, 1222 "value_type": "cumulative" 1223 }, 1224 "type": "graph", 1225 "xaxis": { 1226 "buckets": null, 1227 "mode": "time", 1228 "name": null, 1229 "show": true, 1230 "values": [] 1231 }, 1232 "yaxes": [ 1233 { 1234 "decimals": null, 1235 "format": "s", 1236 "label": null, 1237 "logBase": 1, 1238 "max": null, 1239 "min": null, 1240 "show": true 1241 }, 1242 { 1243 "format": "short", 1244 "label": null, 1245 "logBase": 1, 1246 "max": null, 1247 "min": null, 1248 "show": false 1249 } 1250 ], 1251 "yaxis": { 1252 "align": false, 1253 "alignLevel": null 1254 } 1255 }, 1256 { 1257 "aliasColors": {}, 1258 "bars": false, 1259 "dashLength": 10, 1260 "dashes": false, 1261 "datasource": "${DS_TEST-CLUSTER}", 1262 "decimals": 1, 1263 "editable": true, 1264 "error": false, 1265 "fill": 0, 1266 "grid": {}, 1267 "gridPos": { 1268 "h": 6, 1269 "w": 12, 1270 "x": 12, 1271 "y": 26 1272 }, 1273 "id": 13, 1274 "legend": { 1275 "alignAsTable": true, 1276 "avg": false, 1277 "current": true, 1278 "hideEmpty": true, 1279 "max": true, 1280 "min": false, 1281 "rightSide": true, 1282 "show": true, 1283 "sideWidth": null, 1284 "sort": "current", 1285 "sortDesc": true, 1286 "total": false, 1287 "values": true 1288 }, 1289 "lines": false, 1290 "linewidth": 1, 1291 "links": [], 1292 "nullPointMode": "null", 1293 "percentage": false, 1294 "pointradius": 2, 1295 "points": true, 1296 "renderer": "flot", 1297 "seriesOverrides": [], 1298 "spaceLength": 10, 1299 "stack": false, 1300 "steppedLine": false, 1301 "targets": [ 1302 { 1303 "expr": "rate(node_disk_io_time_seconds_total{tidb_cluster=\"$tidb_cluster\"}[1m])", 1304 "format": "time_series", 1305 "intervalFactor": 2, 1306 "legendFormat": "{{instance}} - {{device}}", 1307 "metric": "tikv_thread_cpu_seconds_total", 1308 "refId": "A", 1309 "step": 4 1310 } 1311 ], 1312 "thresholds": [], 1313 "timeFrom": null, 1314 "timeRegions": [], 1315 "timeShift": null, 1316 "title": "IO Utilization", 1317 "tooltip": { 1318 "msResolution": false, 1319 "shared": true, 1320 "sort": 0, 1321 "value_type": "individual" 1322 }, 1323 "type": "graph", 1324 "xaxis": { 1325 "buckets": null, 1326 "mode": "time", 1327 "name": null, 1328 "show": true, 1329 "values": [] 1330 }, 1331 "yaxes": [ 1332 { 1333 "format": "percentunit", 1334 "label": null, 1335 "logBase": 1, 1336 "max": null, 1337 "min": null, 1338 "show": true 1339 }, 1340 { 1341 "format": "short", 1342 "label": null, 1343 "logBase": 1, 1344 "max": null, 1345 "min": null, 1346 "show": true 1347 } 1348 ], 1349 "yaxis": { 1350 "align": false, 1351 "alignLevel": null 1352 } 1353 } 1354 ], 1355 "title": "Backup", 1356 "type": "row" 1357 }, 1358 { 1359 "collapsed": true, 1360 "gridPos": { 1361 "h": 1, 1362 "w": 24, 1363 "x": 0, 1364 "y": 1 1365 }, 1366 "id": 17, 1367 "panels": [ 1368 { 1369 "aliasColors": {}, 1370 "bars": false, 1371 "dashLength": 10, 1372 "dashes": false, 1373 "datasource": "${DS_TEST-CLUSTER}", 1374 "decimals": 1, 1375 "editable": true, 1376 "error": false, 1377 "fill": 0, 1378 "grid": {}, 1379 "gridPos": { 1380 "h": 6, 1381 "w": 12, 1382 "x": 0, 1383 "y": 2 1384 }, 1385 "id": 55, 1386 "legend": { 1387 "alignAsTable": true, 1388 "avg": false, 1389 "current": true, 1390 "hideEmpty": true, 1391 "max": true, 1392 "min": false, 1393 "rightSide": true, 1394 "show": true, 1395 "sideWidth": null, 1396 "sort": "current", 1397 "sortDesc": true, 1398 "total": false, 1399 "values": true 1400 }, 1401 "lines": true, 1402 "linewidth": 1, 1403 "links": [], 1404 "nullPointMode": "null", 1405 "percentage": false, 1406 "pointradius": 5, 1407 "points": false, 1408 "renderer": "flot", 1409 "seriesOverrides": [ 1410 { 1411 "alias": "/import-count.*/", 1412 "yaxis": 2 1413 } 1414 ], 1415 "spaceLength": 10, 1416 "stack": false, 1417 "steppedLine": false, 1418 "targets": [ 1419 { 1420 "expr": "sum(rate(tikv_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance)", 1421 "format": "time_series", 1422 "hide": false, 1423 "intervalFactor": 2, 1424 "legendFormat": "import-{{instance}}", 1425 "metric": "tikv_thread_cpu_seconds_total", 1426 "refId": "A", 1427 "step": 4 1428 }, 1429 { 1430 "expr": "sum(rate(tikv_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance, tid) > 0", 1431 "format": "time_series", 1432 "hide": true, 1433 "intervalFactor": 2, 1434 "legendFormat": "backup-{{instance}}-{{tid}}", 1435 "metric": "tikv_thread_cpu_seconds_total", 1436 "refId": "C", 1437 "step": 4 1438 }, 1439 { 1440 "expr": "count(rate(tikv_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance)", 1441 "format": "time_series", 1442 "hide": true, 1443 "intervalFactor": 2, 1444 "legendFormat": "import-count-{{instance}}", 1445 "metric": "tikv_thread_cpu_seconds_total", 1446 "refId": "D", 1447 "step": 4 1448 } 1449 ], 1450 "thresholds": [], 1451 "timeFrom": null, 1452 "timeRegions": [], 1453 "timeShift": null, 1454 "title": "Import CPU Utilization", 1455 "tooltip": { 1456 "msResolution": false, 1457 "shared": true, 1458 "sort": 0, 1459 "value_type": "individual" 1460 }, 1461 "type": "graph", 1462 "xaxis": { 1463 "buckets": null, 1464 "mode": "time", 1465 "name": null, 1466 "show": true, 1467 "values": [] 1468 }, 1469 "yaxes": [ 1470 { 1471 "format": "percentunit", 1472 "label": null, 1473 "logBase": 1, 1474 "max": null, 1475 "min": null, 1476 "show": true 1477 }, 1478 { 1479 "format": "short", 1480 "label": null, 1481 "logBase": 1, 1482 "max": null, 1483 "min": null, 1484 "show": true 1485 } 1486 ], 1487 "yaxis": { 1488 "align": false, 1489 "alignLevel": null 1490 } 1491 }, 1492 { 1493 "columns": [ 1494 { 1495 "text": "Current", 1496 "value": "current" 1497 } 1498 ], 1499 "datasource": "${DS_TEST-CLUSTER}", 1500 "fontSize": "100%", 1501 "gridPos": { 1502 "h": 6, 1503 "w": 4, 1504 "x": 12, 1505 "y": 2 1506 }, 1507 "id": 56, 1508 "links": [], 1509 "pageSize": null, 1510 "scroll": true, 1511 "showHeader": true, 1512 "sort": { 1513 "col": 0, 1514 "desc": true 1515 }, 1516 "styles": [ 1517 { 1518 "alias": "Time", 1519 "dateFormat": "YYYY-MM-DD HH:mm:ss", 1520 "pattern": "Time", 1521 "type": "date" 1522 }, 1523 { 1524 "alias": "", 1525 "colorMode": null, 1526 "colors": [ 1527 "rgba(245, 54, 54, 0.9)", 1528 "rgba(237, 129, 40, 0.89)", 1529 "rgba(50, 172, 45, 0.97)" 1530 ], 1531 "decimals": 2, 1532 "pattern": "/.*/", 1533 "thresholds": [], 1534 "type": "number", 1535 "unit": "short" 1536 } 1537 ], 1538 "targets": [ 1539 { 1540 "expr": "count(rate(tikv_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sst_.*\"}[1m])) by (instance)", 1541 "format": "time_series", 1542 "interval": "", 1543 "intervalFactor": 1, 1544 "legendFormat": "{{instance}}", 1545 "refId": "A" 1546 } 1547 ], 1548 "timeFrom": null, 1549 "timeShift": null, 1550 "title": "Import Thread Count", 1551 "transform": "timeseries_aggregations", 1552 "type": "table" 1553 }, 1554 { 1555 "aliasColors": {}, 1556 "bars": false, 1557 "dashLength": 10, 1558 "dashes": false, 1559 "datasource": "${DS_TEST-CLUSTER}", 1560 "decimals": 2, 1561 "description": "", 1562 "fill": 1, 1563 "gridPos": { 1564 "h": 6, 1565 "w": 8, 1566 "x": 16, 1567 "y": 2 1568 }, 1569 "id": 27, 1570 "legend": { 1571 "alignAsTable": true, 1572 "avg": false, 1573 "current": true, 1574 "max": true, 1575 "min": false, 1576 "rightSide": true, 1577 "show": true, 1578 "sideWidth": null, 1579 "total": false, 1580 "values": true 1581 }, 1582 "lines": true, 1583 "linewidth": 1, 1584 "links": [], 1585 "nullPointMode": "null as zero", 1586 "percentage": false, 1587 "pointradius": 2, 1588 "points": false, 1589 "renderer": "flot", 1590 "seriesOverrides": [], 1591 "spaceLength": 10, 1592 "stack": false, 1593 "steppedLine": false, 1594 "targets": [ 1595 { 1596 "expr": "delta(tikv_import_error_counter{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])", 1597 "format": "time_series", 1598 "hide": false, 1599 "intervalFactor": 1, 1600 "legendFormat": "{{type}} {{error}} {{instance}}", 1601 "refId": "D" 1602 } 1603 ], 1604 "thresholds": [], 1605 "timeFrom": null, 1606 "timeRegions": [], 1607 "timeShift": null, 1608 "title": "Import Errors", 1609 "tooltip": { 1610 "shared": true, 1611 "sort": 0, 1612 "value_type": "individual" 1613 }, 1614 "type": "graph", 1615 "xaxis": { 1616 "buckets": null, 1617 "mode": "time", 1618 "name": null, 1619 "show": true, 1620 "values": [] 1621 }, 1622 "yaxes": [ 1623 { 1624 "format": "none", 1625 "label": null, 1626 "logBase": 1, 1627 "max": null, 1628 "min": "0", 1629 "show": true 1630 }, 1631 { 1632 "format": "none", 1633 "label": null, 1634 "logBase": 1, 1635 "max": null, 1636 "min": "0", 1637 "show": true 1638 } 1639 ], 1640 "yaxis": { 1641 "align": false, 1642 "alignLevel": null 1643 } 1644 }, 1645 { 1646 "aliasColors": {}, 1647 "bars": false, 1648 "dashLength": 10, 1649 "dashes": false, 1650 "datasource": "${DS_TEST-CLUSTER}", 1651 "fill": 1, 1652 "gridPos": { 1653 "h": 6, 1654 "w": 12, 1655 "x": 0, 1656 "y": 8 1657 }, 1658 "id": 49, 1659 "legend": { 1660 "alignAsTable": true, 1661 "avg": false, 1662 "current": true, 1663 "max": true, 1664 "min": false, 1665 "rightSide": true, 1666 "show": true, 1667 "total": false, 1668 "values": true 1669 }, 1670 "lines": true, 1671 "linewidth": 1, 1672 "links": [], 1673 "nullPointMode": "null", 1674 "percentage": false, 1675 "pointradius": 2, 1676 "points": false, 1677 "renderer": "flot", 1678 "seriesOverrides": [], 1679 "spaceLength": 10, 1680 "stack": false, 1681 "steppedLine": false, 1682 "targets": [ 1683 { 1684 "expr": "histogram_quantile(0.99, sum(rate(tikv_import_rpc_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le, request))", 1685 "format": "time_series", 1686 "hide": false, 1687 "intervalFactor": 1, 1688 "legendFormat": "{{request}}-99%", 1689 "refId": "A" 1690 } 1691 ], 1692 "thresholds": [], 1693 "timeFrom": null, 1694 "timeRegions": [], 1695 "timeShift": null, 1696 "title": "Import RPC Duration", 1697 "tooltip": { 1698 "shared": true, 1699 "sort": 0, 1700 "value_type": "individual" 1701 }, 1702 "type": "graph", 1703 "xaxis": { 1704 "buckets": null, 1705 "mode": "time", 1706 "name": null, 1707 "show": true, 1708 "values": [] 1709 }, 1710 "yaxes": [ 1711 { 1712 "format": "s", 1713 "label": null, 1714 "logBase": 1, 1715 "max": null, 1716 "min": null, 1717 "show": true 1718 }, 1719 { 1720 "format": "short", 1721 "label": null, 1722 "logBase": 1, 1723 "max": null, 1724 "min": null, 1725 "show": true 1726 } 1727 ], 1728 "yaxis": { 1729 "align": false, 1730 "alignLevel": null 1731 } 1732 }, 1733 { 1734 "aliasColors": {}, 1735 "bars": false, 1736 "dashLength": 10, 1737 "dashes": false, 1738 "datasource": "${DS_TEST-CLUSTER}", 1739 "decimals": 1, 1740 "editable": true, 1741 "error": false, 1742 "fill": 0, 1743 "grid": {}, 1744 "gridPos": { 1745 "h": 6, 1746 "w": 12, 1747 "x": 12, 1748 "y": 8 1749 }, 1750 "id": 19, 1751 "legend": { 1752 "alignAsTable": true, 1753 "avg": false, 1754 "current": true, 1755 "hideEmpty": true, 1756 "max": true, 1757 "min": false, 1758 "rightSide": true, 1759 "show": true, 1760 "sideWidth": null, 1761 "sort": "current", 1762 "sortDesc": true, 1763 "total": false, 1764 "values": true 1765 }, 1766 "lines": true, 1767 "linewidth": 1, 1768 "links": [], 1769 "nullPointMode": "null", 1770 "percentage": false, 1771 "pointradius": 5, 1772 "points": false, 1773 "renderer": "flot", 1774 "seriesOverrides": [], 1775 "spaceLength": 10, 1776 "stack": false, 1777 "steppedLine": false, 1778 "targets": [ 1779 { 1780 "expr": "rate(node_disk_io_time_seconds_total{tidb_cluster=\"$tidb_cluster\"}[1m])", 1781 "format": "time_series", 1782 "intervalFactor": 2, 1783 "legendFormat": "{{instance}} - {{device}}", 1784 "metric": "tikv_thread_cpu_seconds_total", 1785 "refId": "A", 1786 "step": 4 1787 } 1788 ], 1789 "thresholds": [], 1790 "timeFrom": null, 1791 "timeRegions": [], 1792 "timeShift": null, 1793 "title": "IO Utilization", 1794 "tooltip": { 1795 "msResolution": false, 1796 "shared": true, 1797 "sort": 0, 1798 "value_type": "individual" 1799 }, 1800 "type": "graph", 1801 "xaxis": { 1802 "buckets": null, 1803 "mode": "time", 1804 "name": null, 1805 "show": true, 1806 "values": [] 1807 }, 1808 "yaxes": [ 1809 { 1810 "format": "percentunit", 1811 "label": null, 1812 "logBase": 1, 1813 "max": null, 1814 "min": null, 1815 "show": true 1816 }, 1817 { 1818 "format": "short", 1819 "label": null, 1820 "logBase": 1, 1821 "max": null, 1822 "min": null, 1823 "show": true 1824 } 1825 ], 1826 "yaxis": { 1827 "align": false, 1828 "alignLevel": null 1829 } 1830 }, 1831 { 1832 "cards": { 1833 "cardPadding": 0, 1834 "cardRound": 0 1835 }, 1836 "color": { 1837 "cardColor": "#FF9830", 1838 "colorScale": "linear", 1839 "colorScheme": "interpolateSpectral", 1840 "exponent": 0.5, 1841 "max": null, 1842 "min": 0, 1843 "mode": "spectrum" 1844 }, 1845 "dataFormat": "tsbuckets", 1846 "datasource": "${DS_TEST-CLUSTER}", 1847 "description": "", 1848 "gridPos": { 1849 "h": 7, 1850 "w": 6, 1851 "x": 0, 1852 "y": 14 1853 }, 1854 "heatmap": {}, 1855 "hideZeroBuckets": true, 1856 "highlightCards": true, 1857 "id": 46, 1858 "legend": { 1859 "alignAsTable": true, 1860 "avg": false, 1861 "current": true, 1862 "max": true, 1863 "min": false, 1864 "rightSide": true, 1865 "show": true, 1866 "sort": "current", 1867 "sortDesc": true, 1868 "total": false, 1869 "values": true 1870 }, 1871 "links": [], 1872 "reverseYBuckets": false, 1873 "targets": [ 1874 { 1875 "expr": "max(rate(tikv_import_rpc_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", request=~\"download|write\"}[1m])) by (le)", 1876 "format": "heatmap", 1877 "instant": false, 1878 "intervalFactor": 2, 1879 "legendFormat": "{{le}}", 1880 "refId": "A" 1881 } 1882 ], 1883 "title": "Import Write/Download RPC Duration", 1884 "tooltip": { 1885 "show": true, 1886 "showHistogram": true 1887 }, 1888 "tooltipDecimals": 1, 1889 "type": "heatmap", 1890 "xAxis": { 1891 "show": true 1892 }, 1893 "xBucketNumber": null, 1894 "xBucketSize": null, 1895 "yAxis": { 1896 "decimals": 1, 1897 "format": "s", 1898 "logBase": 1, 1899 "max": null, 1900 "min": null, 1901 "show": true, 1902 "splitFactor": null 1903 }, 1904 "yBucketBound": "upper", 1905 "yBucketNumber": null, 1906 "yBucketSize": null 1907 }, 1908 { 1909 "cards": { 1910 "cardPadding": 0, 1911 "cardRound": 0 1912 }, 1913 "color": { 1914 "cardColor": "#FF9830", 1915 "colorScale": "linear", 1916 "colorScheme": "interpolateSpectral", 1917 "exponent": 0.5, 1918 "max": null, 1919 "min": 0, 1920 "mode": "spectrum" 1921 }, 1922 "dataFormat": "tsbuckets", 1923 "datasource": "${DS_TEST-CLUSTER}", 1924 "description": "", 1925 "gridPos": { 1926 "h": 7, 1927 "w": 6, 1928 "x": 6, 1929 "y": 14 1930 }, 1931 "heatmap": {}, 1932 "hideZeroBuckets": true, 1933 "highlightCards": true, 1934 "id": 59, 1935 "legend": { 1936 "alignAsTable": true, 1937 "avg": false, 1938 "current": true, 1939 "max": true, 1940 "min": false, 1941 "rightSide": true, 1942 "show": true, 1943 "sort": "current", 1944 "sortDesc": true, 1945 "total": false, 1946 "values": true 1947 }, 1948 "links": [], 1949 "reverseYBuckets": false, 1950 "targets": [ 1951 { 1952 "expr": "max(rate(tikv_import_download_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"queue\"}[1m])) by (le)", 1953 "format": "heatmap", 1954 "instant": false, 1955 "intervalFactor": 2, 1956 "legendFormat": "{{le}}", 1957 "refId": "A" 1958 } 1959 ], 1960 "title": "Import Wait Duration", 1961 "tooltip": { 1962 "show": true, 1963 "showHistogram": true 1964 }, 1965 "tooltipDecimals": 1, 1966 "type": "heatmap", 1967 "xAxis": { 1968 "show": true 1969 }, 1970 "xBucketNumber": null, 1971 "xBucketSize": null, 1972 "yAxis": { 1973 "decimals": 1, 1974 "format": "s", 1975 "logBase": 1, 1976 "max": null, 1977 "min": null, 1978 "show": true, 1979 "splitFactor": null 1980 }, 1981 "yBucketBound": "upper", 1982 "yBucketNumber": null, 1983 "yBucketSize": null 1984 }, 1985 { 1986 "cards": { 1987 "cardPadding": 0, 1988 "cardRound": 0 1989 }, 1990 "color": { 1991 "cardColor": "#FF9830", 1992 "colorScale": "linear", 1993 "colorScheme": "interpolateSpectral", 1994 "exponent": 0.5, 1995 "max": null, 1996 "min": 0, 1997 "mode": "spectrum" 1998 }, 1999 "dataFormat": "tsbuckets", 2000 "datasource": "${DS_TEST-CLUSTER}", 2001 "description": "", 2002 "gridPos": { 2003 "h": 7, 2004 "w": 6, 2005 "x": 12, 2006 "y": 14 2007 }, 2008 "heatmap": {}, 2009 "hideZeroBuckets": true, 2010 "highlightCards": true, 2011 "id": 52, 2012 "legend": { 2013 "alignAsTable": true, 2014 "avg": false, 2015 "current": true, 2016 "max": true, 2017 "min": false, 2018 "rightSide": true, 2019 "show": true, 2020 "sort": "current", 2021 "sortDesc": true, 2022 "total": false, 2023 "values": true 2024 }, 2025 "links": [], 2026 "reverseYBuckets": false, 2027 "targets": [ 2028 { 2029 "expr": "max(rate(tikv_import_download_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"read\"}[1m])) by (le)", 2030 "format": "heatmap", 2031 "instant": false, 2032 "intervalFactor": 2, 2033 "legendFormat": "{{le}}", 2034 "refId": "A" 2035 } 2036 ], 2037 "title": "Import Read SST Duration", 2038 "tooltip": { 2039 "show": true, 2040 "showHistogram": true 2041 }, 2042 "tooltipDecimals": 1, 2043 "type": "heatmap", 2044 "xAxis": { 2045 "show": true 2046 }, 2047 "xBucketNumber": null, 2048 "xBucketSize": null, 2049 "yAxis": { 2050 "decimals": 1, 2051 "format": "s", 2052 "logBase": 1, 2053 "max": null, 2054 "min": null, 2055 "show": true, 2056 "splitFactor": null 2057 }, 2058 "yBucketBound": "upper", 2059 "yBucketNumber": null, 2060 "yBucketSize": null 2061 }, 2062 { 2063 "cards": { 2064 "cardPadding": 0, 2065 "cardRound": 0 2066 }, 2067 "color": { 2068 "cardColor": "#FF9830", 2069 "colorScale": "linear", 2070 "colorScheme": "interpolateSpectral", 2071 "exponent": 0.5, 2072 "max": null, 2073 "min": 0, 2074 "mode": "spectrum" 2075 }, 2076 "dataFormat": "tsbuckets", 2077 "datasource": "${DS_TEST-CLUSTER}", 2078 "description": "", 2079 "gridPos": { 2080 "h": 7, 2081 "w": 6, 2082 "x": 18, 2083 "y": 14 2084 }, 2085 "heatmap": {}, 2086 "hideZeroBuckets": true, 2087 "highlightCards": true, 2088 "id": 50, 2089 "legend": { 2090 "alignAsTable": true, 2091 "avg": false, 2092 "current": true, 2093 "max": true, 2094 "min": false, 2095 "rightSide": true, 2096 "show": true, 2097 "sort": "current", 2098 "sortDesc": true, 2099 "total": false, 2100 "values": true 2101 }, 2102 "links": [], 2103 "reverseYBuckets": false, 2104 "targets": [ 2105 { 2106 "expr": "max(rate(tikv_import_download_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"rewrite\"}[1m])) by (le)", 2107 "format": "heatmap", 2108 "instant": false, 2109 "intervalFactor": 2, 2110 "legendFormat": "{{le}}", 2111 "refId": "A" 2112 } 2113 ], 2114 "title": "Import Rewrite SST Duration", 2115 "tooltip": { 2116 "show": true, 2117 "showHistogram": true 2118 }, 2119 "tooltipDecimals": 1, 2120 "type": "heatmap", 2121 "xAxis": { 2122 "show": true 2123 }, 2124 "xBucketNumber": null, 2125 "xBucketSize": null, 2126 "yAxis": { 2127 "decimals": 1, 2128 "format": "s", 2129 "logBase": 1, 2130 "max": null, 2131 "min": null, 2132 "show": true, 2133 "splitFactor": null 2134 }, 2135 "yBucketBound": "upper", 2136 "yBucketNumber": null, 2137 "yBucketSize": null 2138 }, 2139 { 2140 "cards": { 2141 "cardPadding": 0, 2142 "cardRound": 0 2143 }, 2144 "color": { 2145 "cardColor": "#FF9830", 2146 "colorScale": "linear", 2147 "colorScheme": "interpolateSpectral", 2148 "exponent": 0.5, 2149 "max": null, 2150 "min": 0, 2151 "mode": "spectrum" 2152 }, 2153 "dataFormat": "tsbuckets", 2154 "datasource": "${DS_TEST-CLUSTER}", 2155 "description": "", 2156 "gridPos": { 2157 "h": 7, 2158 "w": 6, 2159 "x": 0, 2160 "y": 21 2161 }, 2162 "heatmap": {}, 2163 "hideZeroBuckets": true, 2164 "highlightCards": true, 2165 "id": 47, 2166 "legend": { 2167 "alignAsTable": true, 2168 "avg": false, 2169 "current": true, 2170 "max": true, 2171 "min": false, 2172 "rightSide": true, 2173 "show": true, 2174 "sort": "current", 2175 "sortDesc": true, 2176 "total": false, 2177 "values": true 2178 }, 2179 "links": [], 2180 "reverseYBuckets": false, 2181 "targets": [ 2182 { 2183 "expr": "max(rate(tikv_import_rpc_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", request=~\"ingest\"}[1m])) by (le)", 2184 "format": "heatmap", 2185 "instant": false, 2186 "intervalFactor": 2, 2187 "legendFormat": "{{le}}", 2188 "refId": "A" 2189 } 2190 ], 2191 "title": "Import Ingest RPC Duration", 2192 "tooltip": { 2193 "show": true, 2194 "showHistogram": true 2195 }, 2196 "tooltipDecimals": 1, 2197 "type": "heatmap", 2198 "xAxis": { 2199 "show": true 2200 }, 2201 "xBucketNumber": null, 2202 "xBucketSize": null, 2203 "yAxis": { 2204 "decimals": 1, 2205 "format": "s", 2206 "logBase": 1, 2207 "max": null, 2208 "min": null, 2209 "show": true, 2210 "splitFactor": null 2211 }, 2212 "yBucketBound": "upper", 2213 "yBucketNumber": null, 2214 "yBucketSize": null 2215 }, 2216 { 2217 "cards": { 2218 "cardPadding": 0, 2219 "cardRound": 0 2220 }, 2221 "color": { 2222 "cardColor": "#FF9830", 2223 "colorScale": "linear", 2224 "colorScheme": "interpolateSpectral", 2225 "exponent": 0.5, 2226 "max": null, 2227 "min": 0, 2228 "mode": "spectrum" 2229 }, 2230 "dataFormat": "tsbuckets", 2231 "datasource": "${DS_TEST-CLUSTER}", 2232 "description": "", 2233 "gridPos": { 2234 "h": 7, 2235 "w": 6, 2236 "x": 6, 2237 "y": 21 2238 }, 2239 "heatmap": {}, 2240 "hideZeroBuckets": true, 2241 "highlightCards": true, 2242 "id": 53, 2243 "legend": { 2244 "alignAsTable": true, 2245 "avg": false, 2246 "current": true, 2247 "max": true, 2248 "min": false, 2249 "rightSide": true, 2250 "show": true, 2251 "sort": "current", 2252 "sortDesc": true, 2253 "total": false, 2254 "values": true 2255 }, 2256 "links": [], 2257 "reverseYBuckets": false, 2258 "targets": [ 2259 { 2260 "expr": "max(rate(tikv_import_ingest_duration_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"ingest\"}[1m])) by (le)", 2261 "format": "heatmap", 2262 "instant": false, 2263 "intervalFactor": 2, 2264 "legendFormat": "{{le}}", 2265 "refId": "A" 2266 } 2267 ], 2268 "title": "Import Ingest SST Duration", 2269 "tooltip": { 2270 "show": true, 2271 "showHistogram": true 2272 }, 2273 "tooltipDecimals": 1, 2274 "type": "heatmap", 2275 "xAxis": { 2276 "show": true 2277 }, 2278 "xBucketNumber": null, 2279 "xBucketSize": null, 2280 "yAxis": { 2281 "decimals": 1, 2282 "format": "s", 2283 "logBase": 1, 2284 "max": null, 2285 "min": null, 2286 "show": true, 2287 "splitFactor": null 2288 }, 2289 "yBucketBound": "upper", 2290 "yBucketNumber": null, 2291 "yBucketSize": null 2292 }, 2293 { 2294 "cards": { 2295 "cardPadding": 0, 2296 "cardRound": 0 2297 }, 2298 "color": { 2299 "cardColor": "#FF9830", 2300 "colorScale": "linear", 2301 "colorScheme": "interpolateSpectral", 2302 "exponent": 0.5, 2303 "max": null, 2304 "min": 0, 2305 "mode": "spectrum" 2306 }, 2307 "dataFormat": "tsbuckets", 2308 "datasource": "${DS_TEST-CLUSTER}", 2309 "description": "", 2310 "gridPos": { 2311 "h": 7, 2312 "w": 6, 2313 "x": 12, 2314 "y": 21 2315 }, 2316 "heatmap": {}, 2317 "hideZeroBuckets": true, 2318 "highlightCards": true, 2319 "id": 54, 2320 "legend": { 2321 "alignAsTable": true, 2322 "avg": false, 2323 "current": true, 2324 "max": true, 2325 "min": false, 2326 "rightSide": true, 2327 "show": true, 2328 "sort": "current", 2329 "sortDesc": true, 2330 "total": false, 2331 "values": true 2332 }, 2333 "links": [], 2334 "reverseYBuckets": false, 2335 "targets": [ 2336 { 2337 "expr": "max(rate(tikv_import_ingest_byte{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le)", 2338 "format": "heatmap", 2339 "instant": false, 2340 "intervalFactor": 2, 2341 "legendFormat": "{{le}}", 2342 "refId": "A" 2343 } 2344 ], 2345 "title": "Import Ingest SST Bytes", 2346 "tooltip": { 2347 "show": true, 2348 "showHistogram": true 2349 }, 2350 "tooltipDecimals": 1, 2351 "type": "heatmap", 2352 "xAxis": { 2353 "show": true 2354 }, 2355 "xBucketNumber": null, 2356 "xBucketSize": null, 2357 "yAxis": { 2358 "decimals": 1, 2359 "format": "s", 2360 "logBase": 1, 2361 "max": null, 2362 "min": null, 2363 "show": true, 2364 "splitFactor": null 2365 }, 2366 "yBucketBound": "upper", 2367 "yBucketNumber": null, 2368 "yBucketSize": null 2369 }, 2370 { 2371 "aliasColors": {}, 2372 "bars": false, 2373 "dashLength": 10, 2374 "dashes": false, 2375 "datasource": "${DS_TEST-CLUSTER}", 2376 "fill": 1, 2377 "gridPos": { 2378 "h": 7, 2379 "w": 6, 2380 "x": 18, 2381 "y": 21 2382 }, 2383 "id": 31, 2384 "legend": { 2385 "alignAsTable": true, 2386 "avg": false, 2387 "current": true, 2388 "max": true, 2389 "min": false, 2390 "rightSide": false, 2391 "show": true, 2392 "total": false, 2393 "values": true 2394 }, 2395 "lines": true, 2396 "linewidth": 1, 2397 "links": [], 2398 "nullPointMode": "null", 2399 "percentage": false, 2400 "pointradius": 2, 2401 "points": false, 2402 "renderer": "flot", 2403 "seriesOverrides": [], 2404 "spaceLength": 10, 2405 "stack": false, 2406 "steppedLine": false, 2407 "targets": [ 2408 { 2409 "expr": "sum(rate(tikv_import_download_bytes_sum{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m]))", 2410 "format": "time_series", 2411 "intervalFactor": 1, 2412 "legendFormat": "total", 2413 "refId": "A" 2414 }, 2415 { 2416 "expr": "rate(tikv_import_download_bytes_sum{tidb_cluster=\"$tidb_cluster\"}[1m])", 2417 "format": "time_series", 2418 "intervalFactor": 1, 2419 "legendFormat": "{{instance}}", 2420 "refId": "B" 2421 } 2422 ], 2423 "thresholds": [], 2424 "timeFrom": null, 2425 "timeRegions": [], 2426 "timeShift": null, 2427 "title": "Import Download SST Throughput", 2428 "tooltip": { 2429 "shared": true, 2430 "sort": 0, 2431 "value_type": "individual" 2432 }, 2433 "type": "graph", 2434 "xaxis": { 2435 "buckets": null, 2436 "mode": "time", 2437 "name": null, 2438 "show": true, 2439 "values": [] 2440 }, 2441 "yaxes": [ 2442 { 2443 "format": "Bps", 2444 "label": null, 2445 "logBase": 1, 2446 "max": null, 2447 "min": null, 2448 "show": true 2449 }, 2450 { 2451 "format": "short", 2452 "label": null, 2453 "logBase": 1, 2454 "max": null, 2455 "min": null, 2456 "show": true 2457 } 2458 ], 2459 "yaxis": { 2460 "align": false, 2461 "alignLevel": null 2462 } 2463 }, 2464 { 2465 "aliasColors": {}, 2466 "bars": false, 2467 "dashLength": 10, 2468 "dashes": false, 2469 "datasource": "${DS_TEST-CLUSTER}", 2470 "decimals": 1, 2471 "description": "The number of leaders on each TiKV instance", 2472 "editable": true, 2473 "error": false, 2474 "fill": 0, 2475 "grid": {}, 2476 "gridPos": { 2477 "h": 6, 2478 "w": 12, 2479 "x": 0, 2480 "y": 28 2481 }, 2482 "id": 25, 2483 "legend": { 2484 "alignAsTable": true, 2485 "avg": false, 2486 "current": true, 2487 "max": true, 2488 "min": false, 2489 "rightSide": true, 2490 "show": true, 2491 "sideWidth": null, 2492 "sort": "current", 2493 "sortDesc": true, 2494 "total": false, 2495 "values": true 2496 }, 2497 "lines": true, 2498 "linewidth": 1, 2499 "links": [], 2500 "nullPointMode": "null", 2501 "percentage": false, 2502 "pointradius": 5, 2503 "points": false, 2504 "renderer": "flot", 2505 "seriesOverrides": [ 2506 { 2507 "alias": "total", 2508 "lines": false 2509 } 2510 ], 2511 "spaceLength": 10, 2512 "stack": false, 2513 "steppedLine": false, 2514 "targets": [ 2515 { 2516 "expr": "sum(tikv_raftstore_region_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}) by (instance)", 2517 "format": "time_series", 2518 "hide": false, 2519 "intervalFactor": 2, 2520 "legendFormat": "{{instance}}", 2521 "refId": "A", 2522 "step": 10 2523 }, 2524 { 2525 "expr": "delta(tikv_raftstore_region_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"leader\"}[30s]) < -10", 2526 "format": "time_series", 2527 "hide": true, 2528 "intervalFactor": 2, 2529 "legendFormat": "", 2530 "refId": "B" 2531 } 2532 ], 2533 "thresholds": [], 2534 "timeFrom": null, 2535 "timeRegions": [], 2536 "timeShift": null, 2537 "title": "Leader", 2538 "tooltip": { 2539 "msResolution": false, 2540 "shared": true, 2541 "sort": 0, 2542 "value_type": "cumulative" 2543 }, 2544 "type": "graph", 2545 "xaxis": { 2546 "buckets": null, 2547 "mode": "time", 2548 "name": null, 2549 "show": true, 2550 "values": [] 2551 }, 2552 "yaxes": [ 2553 { 2554 "format": "short", 2555 "label": null, 2556 "logBase": 1, 2557 "max": null, 2558 "min": "0", 2559 "show": true 2560 }, 2561 { 2562 "format": "short", 2563 "label": null, 2564 "logBase": 1, 2565 "max": null, 2566 "min": null, 2567 "show": true 2568 } 2569 ], 2570 "yaxis": { 2571 "align": false, 2572 "alignLevel": null 2573 } 2574 }, 2575 { 2576 "aliasColors": {}, 2577 "bars": false, 2578 "dashLength": 10, 2579 "dashes": false, 2580 "datasource": "${DS_TEST-CLUSTER}", 2581 "decimals": 1, 2582 "description": "The number of Regions on each TiKV instance", 2583 "editable": true, 2584 "error": false, 2585 "fill": 0, 2586 "grid": {}, 2587 "gridPos": { 2588 "h": 6, 2589 "w": 12, 2590 "x": 12, 2591 "y": 28 2592 }, 2593 "id": 29, 2594 "legend": { 2595 "alignAsTable": true, 2596 "avg": false, 2597 "current": true, 2598 "max": true, 2599 "min": false, 2600 "rightSide": true, 2601 "show": true, 2602 "sideWidth": null, 2603 "sort": "current", 2604 "sortDesc": true, 2605 "total": false, 2606 "values": true 2607 }, 2608 "lines": true, 2609 "linewidth": 1, 2610 "links": [], 2611 "nullPointMode": "null", 2612 "percentage": false, 2613 "pointradius": 5, 2614 "points": false, 2615 "renderer": "flot", 2616 "seriesOverrides": [], 2617 "spaceLength": 10, 2618 "stack": false, 2619 "steppedLine": false, 2620 "targets": [ 2621 { 2622 "expr": "sum(tikv_raftstore_region_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"region\"}) by (instance)", 2623 "format": "time_series", 2624 "intervalFactor": 2, 2625 "legendFormat": "{{instance}}", 2626 "refId": "A", 2627 "step": 10 2628 } 2629 ], 2630 "thresholds": [], 2631 "timeFrom": null, 2632 "timeRegions": [], 2633 "timeShift": null, 2634 "title": "Region", 2635 "tooltip": { 2636 "msResolution": false, 2637 "shared": true, 2638 "sort": 0, 2639 "value_type": "cumulative" 2640 }, 2641 "type": "graph", 2642 "xaxis": { 2643 "buckets": null, 2644 "mode": "time", 2645 "name": null, 2646 "show": true, 2647 "values": [] 2648 }, 2649 "yaxes": [ 2650 { 2651 "format": "short", 2652 "label": "", 2653 "logBase": 1, 2654 "max": null, 2655 "min": null, 2656 "show": true 2657 }, 2658 { 2659 "format": "short", 2660 "label": null, 2661 "logBase": 1, 2662 "max": null, 2663 "min": null, 2664 "show": false 2665 } 2666 ], 2667 "yaxis": { 2668 "align": false, 2669 "alignLevel": null 2670 } 2671 }, 2672 { 2673 "aliasColors": {}, 2674 "bars": false, 2675 "dashLength": 10, 2676 "dashes": false, 2677 "datasource": "${DS_TEST-CLUSTER}", 2678 "decimals": 1, 2679 "description": "", 2680 "editable": true, 2681 "error": false, 2682 "fill": 1, 2683 "grid": {}, 2684 "gridPos": { 2685 "h": 6, 2686 "w": 12, 2687 "x": 0, 2688 "y": 34 2689 }, 2690 "id": 57, 2691 "legend": { 2692 "alignAsTable": true, 2693 "avg": false, 2694 "current": true, 2695 "hideEmpty": false, 2696 "max": true, 2697 "min": false, 2698 "rightSide": true, 2699 "show": true, 2700 "sideWidth": null, 2701 "sort": "current", 2702 "sortDesc": true, 2703 "total": false, 2704 "values": true 2705 }, 2706 "lines": true, 2707 "linewidth": 1, 2708 "links": [], 2709 "nullPointMode": "null as zero", 2710 "percentage": false, 2711 "pointradius": 5, 2712 "points": false, 2713 "renderer": "flot", 2714 "seriesOverrides": [], 2715 "spaceLength": 10, 2716 "stack": false, 2717 "steppedLine": false, 2718 "targets": [ 2719 { 2720 "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[1m])) by (le,req))", 2721 "format": "time_series", 2722 "intervalFactor": 2, 2723 "legendFormat": "{{req}}-100%", 2724 "refId": "E" 2725 }, 2726 { 2727 "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"checksum.*|analyze.*\"}[1m])) by (le,req))", 2728 "format": "time_series", 2729 "intervalFactor": 2, 2730 "legendFormat": "{{req}}-99%", 2731 "refId": "A", 2732 "step": 4 2733 } 2734 ], 2735 "thresholds": [], 2736 "timeFrom": null, 2737 "timeRegions": [], 2738 "timeShift": null, 2739 "title": "Checksum Request Duration", 2740 "tooltip": { 2741 "msResolution": false, 2742 "shared": true, 2743 "sort": 1, 2744 "value_type": "cumulative" 2745 }, 2746 "type": "graph", 2747 "xaxis": { 2748 "buckets": null, 2749 "mode": "time", 2750 "name": null, 2751 "show": true, 2752 "values": [] 2753 }, 2754 "yaxes": [ 2755 { 2756 "decimals": null, 2757 "format": "s", 2758 "label": null, 2759 "logBase": 1, 2760 "max": null, 2761 "min": null, 2762 "show": true 2763 }, 2764 { 2765 "format": "short", 2766 "label": null, 2767 "logBase": 1, 2768 "max": null, 2769 "min": null, 2770 "show": false 2771 } 2772 ], 2773 "yaxis": { 2774 "align": false, 2775 "alignLevel": null 2776 } 2777 }, 2778 { 2779 "aliasColors": {}, 2780 "bars": false, 2781 "dashLength": 10, 2782 "dashes": false, 2783 "datasource": "${DS_TEST-CLUSTER}", 2784 "decimals": 1, 2785 "editable": true, 2786 "error": false, 2787 "fill": 0, 2788 "grid": {}, 2789 "gridPos": { 2790 "h": 6, 2791 "w": 12, 2792 "x": 12, 2793 "y": 34 2794 }, 2795 "id": 58, 2796 "legend": { 2797 "alignAsTable": true, 2798 "avg": false, 2799 "current": true, 2800 "hideEmpty": true, 2801 "max": true, 2802 "min": false, 2803 "rightSide": true, 2804 "show": true, 2805 "sideWidth": null, 2806 "sort": "current", 2807 "sortDesc": true, 2808 "total": false, 2809 "values": true 2810 }, 2811 "lines": false, 2812 "linewidth": 1, 2813 "links": [], 2814 "nullPointMode": "null", 2815 "percentage": false, 2816 "pointradius": 2, 2817 "points": true, 2818 "renderer": "flot", 2819 "seriesOverrides": [], 2820 "spaceLength": 10, 2821 "stack": false, 2822 "steppedLine": false, 2823 "targets": [ 2824 { 2825 "expr": "rate(node_disk_io_time_seconds_total{tidb_cluster=\"$tidb_cluster\"}[1m])", 2826 "format": "time_series", 2827 "intervalFactor": 2, 2828 "legendFormat": "{{instance}} - {{device}}", 2829 "metric": "tikv_thread_cpu_seconds_total", 2830 "refId": "A", 2831 "step": 4 2832 } 2833 ], 2834 "thresholds": [], 2835 "timeFrom": null, 2836 "timeRegions": [], 2837 "timeShift": null, 2838 "title": "IO Utilization", 2839 "tooltip": { 2840 "msResolution": false, 2841 "shared": true, 2842 "sort": 0, 2843 "value_type": "individual" 2844 }, 2845 "type": "graph", 2846 "xaxis": { 2847 "buckets": null, 2848 "mode": "time", 2849 "name": null, 2850 "show": true, 2851 "values": [] 2852 }, 2853 "yaxes": [ 2854 { 2855 "format": "percentunit", 2856 "label": null, 2857 "logBase": 1, 2858 "max": null, 2859 "min": null, 2860 "show": true 2861 }, 2862 { 2863 "format": "short", 2864 "label": null, 2865 "logBase": 1, 2866 "max": null, 2867 "min": null, 2868 "show": true 2869 } 2870 ], 2871 "yaxis": { 2872 "align": false, 2873 "alignLevel": null 2874 } 2875 } 2876 ], 2877 "title": "Import", 2878 "type": "row" 2879 } 2880 ], 2881 "refresh": false, 2882 "schemaVersion": 18, 2883 "style": "dark", 2884 "tags": [], 2885 "templating": { 2886 "list": [ 2887 { 2888 "allValue": null, 2889 "current": { 2890 }, 2891 "datasource": "${DS_TEST-CLUSTER}", 2892 "hide": 2, 2893 "includeAll": false, 2894 "label": "tidb_cluster", 2895 "multi": false, 2896 "name": "tidb_cluster", 2897 "options": [ 2898 2899 ], 2900 "query": "label_values(tikv_engine_size_bytes, tidb_cluster)", 2901 "refresh": 2, 2902 "regex": "", 2903 "sort": 1, 2904 "tagValuesQuery": "", 2905 "tags": [ 2906 2907 ], 2908 "tagsQuery": "", 2909 "type": "query", 2910 "useTags": false 2911 }, 2912 { 2913 "allValue": ".*", 2914 "current": {}, 2915 "datasource": "${DS_TEST-CLUSTER}", 2916 "definition": "label_values(tikv_engine_size_bytes, instance)", 2917 "hide": 0, 2918 "includeAll": true, 2919 "label": "TiKV", 2920 "multi": false, 2921 "name": "instance", 2922 "options": [], 2923 "query": "label_values(tikv_engine_size_bytes, instance)", 2924 "refresh": 1, 2925 "regex": "", 2926 "skipUrlSync": false, 2927 "sort": 1, 2928 "tagValuesQuery": "", 2929 "tags": [], 2930 "tagsQuery": "", 2931 "type": "query", 2932 "useTags": false 2933 } 2934 ] 2935 }, 2936 "time": { 2937 "from": "now-30m", 2938 "to": "now" 2939 }, 2940 "timepicker": { 2941 "refresh_intervals": [ 2942 "5s", 2943 "10s", 2944 "30s", 2945 "1m", 2946 "5m", 2947 "15m", 2948 "30m", 2949 "1h", 2950 "2h", 2951 "1d" 2952 ], 2953 "time_options": [ 2954 "5m", 2955 "15m", 2956 "1h", 2957 "6h", 2958 "12h", 2959 "24h", 2960 "2d", 2961 "7d", 2962 "30d" 2963 ] 2964 }, 2965 "timezone": "", 2966 "title": "Test-Cluster-Backup & Import", 2967 "uid": "AzvioWLWz", 2968 "version": 27 2969 }