k8s.io/perf-tests/clusterloader2@v0.0.0-20240304094227-64bdb12da87e/pkg/prometheus/manifests/dashboards/master-dashboard.json (about) 1 { 2 "__inputs": [], 3 "annotations": { 4 "list": [] 5 }, 6 "description": "", 7 "editable": true, 8 "gnetId": null, 9 "graphTooltip": 0, 10 "hideControls": false, 11 "id": null, 12 "links": [], 13 "panels": [], 14 "refresh": "", 15 "rows": [ 16 { 17 "collapse": false, 18 "editable": true, 19 "height": "300px", 20 "panels": [ 21 { 22 "aliasColors": {}, 23 "bars": false, 24 "cacheTimeout": null, 25 "datasource": "$source", 26 "description": null, 27 "editable": true, 28 "error": false, 29 "fieldConfig": { 30 "defaults": { 31 "thresholds": { 32 "mode": "absolute", 33 "steps": [] 34 } 35 } 36 }, 37 "fill": 1, 38 "grid": { 39 "threshold1": null, 40 "threshold1Color": "rgba(216, 200, 27, 0.27)", 41 "threshold2": null, 42 "threshold2Color": "rgba(234, 112, 112, 0.22)" 43 }, 44 "gridPos": null, 45 "height": null, 46 "hideTimeOverride": false, 47 "id": 1, 48 "interval": null, 49 "isNew": true, 50 "legend": { 51 "alignAsTable": false, 52 "avg": false, 53 "current": false, 54 "hideEmpty": false, 55 "hideZero": false, 56 "max": false, 57 "min": false, 58 "rightSide": false, 59 "show": true, 60 "sideWidth": null, 61 "sort": null, 62 "sortDesc": false, 63 "total": false, 64 "values": false 65 }, 66 "lines": true, 67 "linewidth": 2, 68 "links": [], 69 "maxDataPoints": 100, 70 "maxPerRow": null, 71 "minSpan": null, 72 "nullPointMode": null, 73 "options": { 74 "alertThreshold": true, 75 "dataLinks": [] 76 }, 77 "percentage": false, 78 "pointradius": 5, 79 "points": false, 80 "renderer": "flot", 81 "repeat": null, 82 "repeatDirection": null, 83 "seriesOverrides": [], 84 "span": 12, 85 "stack": false, 86 "steppedLine": false, 87 "targets": [ 88 { 89 "datasource": "$source", 90 "expr": "1", 91 "format": "time_series", 92 "hide": false, 93 "instant": false, 94 "interval": "", 95 "intervalFactor": 2, 96 "legendFormat": "threshold", 97 "metric": "", 98 "query": "1", 99 "refId": "", 100 "step": 10, 101 "target": "" 102 }, 103 { 104 "datasource": "$source", 105 "expr": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"GET\", scope=~\"resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 106 "format": "time_series", 107 "hide": false, 108 "instant": false, 109 "interval": "", 110 "intervalFactor": 2, 111 "legendFormat": "", 112 "metric": "", 113 "query": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"GET\", scope=~\"resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 114 "refId": "", 115 "step": 10, 116 "target": "" 117 } 118 ], 119 "thresholds": [], 120 "timeFrom": null, 121 "timeShift": null, 122 "title": "GET resource latency (percentaile=99, scope=resource, threshold=1s)", 123 "tooltip": { 124 "msResolution": true, 125 "shared": true, 126 "sort": 2, 127 "value_type": "cumulative" 128 }, 129 "transformations": [], 130 "transparent": false, 131 "type": "graph", 132 "xaxis": { 133 "mode": "time", 134 "name": null, 135 "show": true, 136 "values": [] 137 }, 138 "yaxes": [ 139 { 140 "decimals": null, 141 "format": "s", 142 "label": null, 143 "logBase": 1, 144 "max": null, 145 "min": null, 146 "show": true 147 }, 148 { 149 "decimals": null, 150 "format": "short", 151 "label": null, 152 "logBase": 1, 153 "max": null, 154 "min": null, 155 "show": true 156 } 157 ], 158 "yaxis": { 159 "align": false, 160 "alignLevel": 0 161 } 162 }, 163 { 164 "aliasColors": {}, 165 "bars": false, 166 "cacheTimeout": null, 167 "datasource": "$source", 168 "description": null, 169 "editable": true, 170 "error": false, 171 "fieldConfig": { 172 "defaults": { 173 "thresholds": { 174 "mode": "absolute", 175 "steps": [] 176 } 177 } 178 }, 179 "fill": 1, 180 "grid": { 181 "threshold1": null, 182 "threshold1Color": "rgba(216, 200, 27, 0.27)", 183 "threshold2": null, 184 "threshold2Color": "rgba(234, 112, 112, 0.22)" 185 }, 186 "gridPos": null, 187 "height": null, 188 "hideTimeOverride": false, 189 "id": 2, 190 "interval": null, 191 "isNew": true, 192 "legend": { 193 "alignAsTable": false, 194 "avg": false, 195 "current": false, 196 "hideEmpty": false, 197 "hideZero": false, 198 "max": false, 199 "min": false, 200 "rightSide": false, 201 "show": true, 202 "sideWidth": null, 203 "sort": null, 204 "sortDesc": false, 205 "total": false, 206 "values": false 207 }, 208 "lines": true, 209 "linewidth": 2, 210 "links": [], 211 "maxDataPoints": 100, 212 "maxPerRow": null, 213 "minSpan": null, 214 "nullPointMode": null, 215 "options": { 216 "alertThreshold": true, 217 "dataLinks": [] 218 }, 219 "percentage": false, 220 "pointradius": 5, 221 "points": false, 222 "renderer": "flot", 223 "repeat": null, 224 "repeatDirection": null, 225 "seriesOverrides": [], 226 "span": 12, 227 "stack": false, 228 "steppedLine": false, 229 "targets": [ 230 { 231 "datasource": "$source", 232 "expr": "5", 233 "format": "time_series", 234 "hide": false, 235 "instant": false, 236 "interval": "", 237 "intervalFactor": 2, 238 "legendFormat": "threshold", 239 "metric": "", 240 "query": "5", 241 "refId": "", 242 "step": 10, 243 "target": "" 244 }, 245 { 246 "datasource": "$source", 247 "expr": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"namespace\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 248 "format": "time_series", 249 "hide": false, 250 "instant": false, 251 "interval": "", 252 "intervalFactor": 2, 253 "legendFormat": "", 254 "metric": "", 255 "query": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"namespace\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 256 "refId": "", 257 "step": 10, 258 "target": "" 259 } 260 ], 261 "thresholds": [], 262 "timeFrom": null, 263 "timeShift": null, 264 "title": "LIST namespace latency (percentaile=99, scope=namespace, threshold=5s)", 265 "tooltip": { 266 "msResolution": true, 267 "shared": true, 268 "sort": 2, 269 "value_type": "cumulative" 270 }, 271 "transformations": [], 272 "transparent": false, 273 "type": "graph", 274 "xaxis": { 275 "mode": "time", 276 "name": null, 277 "show": true, 278 "values": [] 279 }, 280 "yaxes": [ 281 { 282 "decimals": null, 283 "format": "s", 284 "label": null, 285 "logBase": 1, 286 "max": null, 287 "min": null, 288 "show": true 289 }, 290 { 291 "decimals": null, 292 "format": "short", 293 "label": null, 294 "logBase": 1, 295 "max": null, 296 "min": null, 297 "show": true 298 } 299 ], 300 "yaxis": { 301 "align": false, 302 "alignLevel": 0 303 } 304 }, 305 { 306 "aliasColors": {}, 307 "bars": false, 308 "cacheTimeout": null, 309 "datasource": "$source", 310 "description": null, 311 "editable": true, 312 "error": false, 313 "fieldConfig": { 314 "defaults": { 315 "thresholds": { 316 "mode": "absolute", 317 "steps": [] 318 } 319 } 320 }, 321 "fill": 1, 322 "grid": { 323 "threshold1": null, 324 "threshold1Color": "rgba(216, 200, 27, 0.27)", 325 "threshold2": null, 326 "threshold2Color": "rgba(234, 112, 112, 0.22)" 327 }, 328 "gridPos": null, 329 "height": null, 330 "hideTimeOverride": false, 331 "id": 3, 332 "interval": null, 333 "isNew": true, 334 "legend": { 335 "alignAsTable": false, 336 "avg": false, 337 "current": false, 338 "hideEmpty": false, 339 "hideZero": false, 340 "max": false, 341 "min": false, 342 "rightSide": false, 343 "show": true, 344 "sideWidth": null, 345 "sort": null, 346 "sortDesc": false, 347 "total": false, 348 "values": false 349 }, 350 "lines": true, 351 "linewidth": 2, 352 "links": [], 353 "maxDataPoints": 100, 354 "maxPerRow": null, 355 "minSpan": null, 356 "nullPointMode": null, 357 "options": { 358 "alertThreshold": true, 359 "dataLinks": [] 360 }, 361 "percentage": false, 362 "pointradius": 5, 363 "points": false, 364 "renderer": "flot", 365 "repeat": null, 366 "repeatDirection": null, 367 "seriesOverrides": [], 368 "span": 12, 369 "stack": false, 370 "steppedLine": false, 371 "targets": [ 372 { 373 "datasource": "$source", 374 "expr": "30", 375 "format": "time_series", 376 "hide": false, 377 "instant": false, 378 "interval": "", 379 "intervalFactor": 2, 380 "legendFormat": "threshold", 381 "metric": "", 382 "query": "30", 383 "refId": "", 384 "step": 10, 385 "target": "" 386 }, 387 { 388 "datasource": "$source", 389 "expr": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"cluster\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 390 "format": "time_series", 391 "hide": false, 392 "instant": false, 393 "interval": "", 394 "intervalFactor": 2, 395 "legendFormat": "", 396 "metric": "", 397 "query": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"cluster\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 398 "refId": "", 399 "step": 10, 400 "target": "" 401 } 402 ], 403 "thresholds": [], 404 "timeFrom": null, 405 "timeShift": null, 406 "title": "LIST cluster latency (percentaile=99, scope=cluster, threshold=30s)", 407 "tooltip": { 408 "msResolution": true, 409 "shared": true, 410 "sort": 2, 411 "value_type": "cumulative" 412 }, 413 "transformations": [], 414 "transparent": false, 415 "type": "graph", 416 "xaxis": { 417 "mode": "time", 418 "name": null, 419 "show": true, 420 "values": [] 421 }, 422 "yaxes": [ 423 { 424 "decimals": null, 425 "format": "s", 426 "label": null, 427 "logBase": 1, 428 "max": null, 429 "min": null, 430 "show": true 431 }, 432 { 433 "decimals": null, 434 "format": "short", 435 "label": null, 436 "logBase": 1, 437 "max": null, 438 "min": null, 439 "show": true 440 } 441 ], 442 "yaxis": { 443 "align": false, 444 "alignLevel": 0 445 } 446 }, 447 { 448 "aliasColors": {}, 449 "bars": false, 450 "cacheTimeout": null, 451 "datasource": "$source", 452 "description": null, 453 "editable": true, 454 "error": false, 455 "fieldConfig": { 456 "defaults": { 457 "thresholds": { 458 "mode": "absolute", 459 "steps": [] 460 } 461 } 462 }, 463 "fill": 1, 464 "grid": { 465 "threshold1": null, 466 "threshold1Color": "rgba(216, 200, 27, 0.27)", 467 "threshold2": null, 468 "threshold2Color": "rgba(234, 112, 112, 0.22)" 469 }, 470 "gridPos": null, 471 "height": null, 472 "hideTimeOverride": false, 473 "id": 4, 474 "interval": null, 475 "isNew": true, 476 "legend": { 477 "alignAsTable": false, 478 "avg": false, 479 "current": false, 480 "hideEmpty": false, 481 "hideZero": false, 482 "max": false, 483 "min": false, 484 "rightSide": false, 485 "show": true, 486 "sideWidth": null, 487 "sort": null, 488 "sortDesc": false, 489 "total": false, 490 "values": false 491 }, 492 "lines": true, 493 "linewidth": 2, 494 "links": [], 495 "maxDataPoints": 100, 496 "maxPerRow": null, 497 "minSpan": null, 498 "nullPointMode": null, 499 "options": { 500 "alertThreshold": true, 501 "dataLinks": [] 502 }, 503 "percentage": false, 504 "pointradius": 5, 505 "points": false, 506 "renderer": "flot", 507 "repeat": null, 508 "repeatDirection": null, 509 "seriesOverrides": [], 510 "span": 12, 511 "stack": false, 512 "steppedLine": false, 513 "targets": [ 514 { 515 "datasource": "$source", 516 "expr": "1", 517 "format": "time_series", 518 "hide": false, 519 "instant": false, 520 "interval": "", 521 "intervalFactor": 2, 522 "legendFormat": "threshold", 523 "metric": "", 524 "query": "1", 525 "refId": "", 526 "step": 10, 527 "target": "" 528 }, 529 { 530 "datasource": "$source", 531 "expr": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"CREATE|DELETE|PATCH|POST|PUT\", scope=~\"namespace|cluster|resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 532 "format": "time_series", 533 "hide": false, 534 "instant": false, 535 "interval": "", 536 "intervalFactor": 2, 537 "legendFormat": "", 538 "metric": "", 539 "query": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"CREATE|DELETE|PATCH|POST|PUT\", scope=~\"namespace|cluster|resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 540 "refId": "", 541 "step": 10, 542 "target": "" 543 } 544 ], 545 "thresholds": [], 546 "timeFrom": null, 547 "timeShift": null, 548 "title": "Mutating API call latency (threshold=1s)", 549 "tooltip": { 550 "msResolution": true, 551 "shared": true, 552 "sort": 2, 553 "value_type": "cumulative" 554 }, 555 "transformations": [], 556 "transparent": false, 557 "type": "graph", 558 "xaxis": { 559 "mode": "time", 560 "name": null, 561 "show": true, 562 "values": [] 563 }, 564 "yaxes": [ 565 { 566 "decimals": null, 567 "format": "s", 568 "label": null, 569 "logBase": 1, 570 "max": null, 571 "min": null, 572 "show": true 573 }, 574 { 575 "decimals": null, 576 "format": "short", 577 "label": null, 578 "logBase": 1, 579 "max": null, 580 "min": null, 581 "show": true 582 } 583 ], 584 "yaxis": { 585 "align": false, 586 "alignLevel": 0 587 } 588 } 589 ], 590 "repeat": null, 591 "showTitle": true, 592 "title": "API call latency" 593 }, 594 { 595 "collapse": true, 596 "editable": true, 597 "height": "300px", 598 "panels": [ 599 { 600 "aliasColors": {}, 601 "bars": false, 602 "cacheTimeout": null, 603 "datasource": "$source", 604 "description": null, 605 "editable": true, 606 "error": false, 607 "fieldConfig": { 608 "defaults": { 609 "thresholds": { 610 "mode": "absolute", 611 "steps": [] 612 } 613 } 614 }, 615 "fill": 1, 616 "grid": { 617 "threshold1": null, 618 "threshold1Color": "rgba(216, 200, 27, 0.27)", 619 "threshold2": null, 620 "threshold2Color": "rgba(234, 112, 112, 0.22)" 621 }, 622 "gridPos": null, 623 "height": null, 624 "hideTimeOverride": false, 625 "id": 5, 626 "interval": null, 627 "isNew": true, 628 "legend": { 629 "alignAsTable": false, 630 "avg": false, 631 "current": false, 632 "hideEmpty": false, 633 "hideZero": false, 634 "max": false, 635 "min": false, 636 "rightSide": false, 637 "show": true, 638 "sideWidth": null, 639 "sort": null, 640 "sortDesc": false, 641 "total": false, 642 "values": false 643 }, 644 "lines": true, 645 "linewidth": 2, 646 "links": [], 647 "maxDataPoints": 100, 648 "maxPerRow": null, 649 "minSpan": null, 650 "nullPointMode": null, 651 "options": { 652 "alertThreshold": true, 653 "dataLinks": [] 654 }, 655 "percentage": false, 656 "pointradius": 5, 657 "points": false, 658 "renderer": "flot", 659 "repeat": null, 660 "repeatDirection": null, 661 "seriesOverrides": [], 662 "span": 12, 663 "stack": false, 664 "steppedLine": false, 665 "targets": [ 666 { 667 "datasource": "$source", 668 "expr": "1", 669 "format": "time_series", 670 "hide": false, 671 "instant": false, 672 "interval": "", 673 "intervalFactor": 2, 674 "legendFormat": "threshold", 675 "metric": "", 676 "query": "1", 677 "refId": "", 678 "step": 10, 679 "target": "" 680 }, 681 { 682 "datasource": "$source", 683 "expr": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"GET\", scope=~\"resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 684 "format": "time_series", 685 "hide": false, 686 "instant": false, 687 "interval": "", 688 "intervalFactor": 2, 689 "legendFormat": "", 690 "metric": "", 691 "query": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"GET\", scope=~\"resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 692 "refId": "", 693 "step": 10, 694 "target": "" 695 } 696 ], 697 "thresholds": [], 698 "timeFrom": null, 699 "timeShift": null, 700 "title": "GET resource latency (percentaile=99, scope=resource, threshold=1s)", 701 "tooltip": { 702 "msResolution": true, 703 "shared": true, 704 "sort": 2, 705 "value_type": "cumulative" 706 }, 707 "transformations": [], 708 "transparent": false, 709 "type": "graph", 710 "xaxis": { 711 "mode": "time", 712 "name": null, 713 "show": true, 714 "values": [] 715 }, 716 "yaxes": [ 717 { 718 "decimals": null, 719 "format": "s", 720 "label": null, 721 "logBase": 1, 722 "max": null, 723 "min": null, 724 "show": true 725 }, 726 { 727 "decimals": null, 728 "format": "short", 729 "label": null, 730 "logBase": 1, 731 "max": null, 732 "min": null, 733 "show": true 734 } 735 ], 736 "yaxis": { 737 "align": false, 738 "alignLevel": 0 739 } 740 }, 741 { 742 "aliasColors": {}, 743 "bars": false, 744 "cacheTimeout": null, 745 "datasource": "$source", 746 "description": null, 747 "editable": true, 748 "error": false, 749 "fieldConfig": { 750 "defaults": { 751 "thresholds": { 752 "mode": "absolute", 753 "steps": [] 754 } 755 } 756 }, 757 "fill": 1, 758 "grid": { 759 "threshold1": null, 760 "threshold1Color": "rgba(216, 200, 27, 0.27)", 761 "threshold2": null, 762 "threshold2Color": "rgba(234, 112, 112, 0.22)" 763 }, 764 "gridPos": null, 765 "height": null, 766 "hideTimeOverride": false, 767 "id": 6, 768 "interval": null, 769 "isNew": true, 770 "legend": { 771 "alignAsTable": false, 772 "avg": false, 773 "current": false, 774 "hideEmpty": false, 775 "hideZero": false, 776 "max": false, 777 "min": false, 778 "rightSide": false, 779 "show": true, 780 "sideWidth": null, 781 "sort": null, 782 "sortDesc": false, 783 "total": false, 784 "values": false 785 }, 786 "lines": true, 787 "linewidth": 2, 788 "links": [], 789 "maxDataPoints": 100, 790 "maxPerRow": null, 791 "minSpan": null, 792 "nullPointMode": null, 793 "options": { 794 "alertThreshold": true, 795 "dataLinks": [] 796 }, 797 "percentage": false, 798 "pointradius": 5, 799 "points": false, 800 "renderer": "flot", 801 "repeat": null, 802 "repeatDirection": null, 803 "seriesOverrides": [], 804 "span": 12, 805 "stack": false, 806 "steppedLine": false, 807 "targets": [ 808 { 809 "datasource": "$source", 810 "expr": "5", 811 "format": "time_series", 812 "hide": false, 813 "instant": false, 814 "interval": "", 815 "intervalFactor": 2, 816 "legendFormat": "threshold", 817 "metric": "", 818 "query": "5", 819 "refId": "", 820 "step": 10, 821 "target": "" 822 }, 823 { 824 "datasource": "$source", 825 "expr": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"namespace\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 826 "format": "time_series", 827 "hide": false, 828 "instant": false, 829 "interval": "", 830 "intervalFactor": 2, 831 "legendFormat": "", 832 "metric": "", 833 "query": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"namespace\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 834 "refId": "", 835 "step": 10, 836 "target": "" 837 } 838 ], 839 "thresholds": [], 840 "timeFrom": null, 841 "timeShift": null, 842 "title": "LIST namespace latency (percentaile=99, scope=namespace, threshold=5s)", 843 "tooltip": { 844 "msResolution": true, 845 "shared": true, 846 "sort": 2, 847 "value_type": "cumulative" 848 }, 849 "transformations": [], 850 "transparent": false, 851 "type": "graph", 852 "xaxis": { 853 "mode": "time", 854 "name": null, 855 "show": true, 856 "values": [] 857 }, 858 "yaxes": [ 859 { 860 "decimals": null, 861 "format": "s", 862 "label": null, 863 "logBase": 1, 864 "max": null, 865 "min": null, 866 "show": true 867 }, 868 { 869 "decimals": null, 870 "format": "short", 871 "label": null, 872 "logBase": 1, 873 "max": null, 874 "min": null, 875 "show": true 876 } 877 ], 878 "yaxis": { 879 "align": false, 880 "alignLevel": 0 881 } 882 }, 883 { 884 "aliasColors": {}, 885 "bars": false, 886 "cacheTimeout": null, 887 "datasource": "$source", 888 "description": null, 889 "editable": true, 890 "error": false, 891 "fieldConfig": { 892 "defaults": { 893 "thresholds": { 894 "mode": "absolute", 895 "steps": [] 896 } 897 } 898 }, 899 "fill": 1, 900 "grid": { 901 "threshold1": null, 902 "threshold1Color": "rgba(216, 200, 27, 0.27)", 903 "threshold2": null, 904 "threshold2Color": "rgba(234, 112, 112, 0.22)" 905 }, 906 "gridPos": null, 907 "height": null, 908 "hideTimeOverride": false, 909 "id": 7, 910 "interval": null, 911 "isNew": true, 912 "legend": { 913 "alignAsTable": false, 914 "avg": false, 915 "current": false, 916 "hideEmpty": false, 917 "hideZero": false, 918 "max": false, 919 "min": false, 920 "rightSide": false, 921 "show": true, 922 "sideWidth": null, 923 "sort": null, 924 "sortDesc": false, 925 "total": false, 926 "values": false 927 }, 928 "lines": true, 929 "linewidth": 2, 930 "links": [], 931 "maxDataPoints": 100, 932 "maxPerRow": null, 933 "minSpan": null, 934 "nullPointMode": null, 935 "options": { 936 "alertThreshold": true, 937 "dataLinks": [] 938 }, 939 "percentage": false, 940 "pointradius": 5, 941 "points": false, 942 "renderer": "flot", 943 "repeat": null, 944 "repeatDirection": null, 945 "seriesOverrides": [], 946 "span": 12, 947 "stack": false, 948 "steppedLine": false, 949 "targets": [ 950 { 951 "datasource": "$source", 952 "expr": "30", 953 "format": "time_series", 954 "hide": false, 955 "instant": false, 956 "interval": "", 957 "intervalFactor": 2, 958 "legendFormat": "threshold", 959 "metric": "", 960 "query": "30", 961 "refId": "", 962 "step": 10, 963 "target": "" 964 }, 965 { 966 "datasource": "$source", 967 "expr": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"cluster\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 968 "format": "time_series", 969 "hide": false, 970 "instant": false, 971 "interval": "", 972 "intervalFactor": 2, 973 "legendFormat": "", 974 "metric": "", 975 "query": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"cluster\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 976 "refId": "", 977 "step": 10, 978 "target": "" 979 } 980 ], 981 "thresholds": [], 982 "timeFrom": null, 983 "timeShift": null, 984 "title": "LIST cluster latency (percentaile=99, scope=cluster, threshold=30s)", 985 "tooltip": { 986 "msResolution": true, 987 "shared": true, 988 "sort": 2, 989 "value_type": "cumulative" 990 }, 991 "transformations": [], 992 "transparent": false, 993 "type": "graph", 994 "xaxis": { 995 "mode": "time", 996 "name": null, 997 "show": true, 998 "values": [] 999 }, 1000 "yaxes": [ 1001 { 1002 "decimals": null, 1003 "format": "s", 1004 "label": null, 1005 "logBase": 1, 1006 "max": null, 1007 "min": null, 1008 "show": true 1009 }, 1010 { 1011 "decimals": null, 1012 "format": "short", 1013 "label": null, 1014 "logBase": 1, 1015 "max": null, 1016 "min": null, 1017 "show": true 1018 } 1019 ], 1020 "yaxis": { 1021 "align": false, 1022 "alignLevel": 0 1023 } 1024 }, 1025 { 1026 "aliasColors": {}, 1027 "bars": false, 1028 "cacheTimeout": null, 1029 "datasource": "$source", 1030 "description": null, 1031 "editable": true, 1032 "error": false, 1033 "fieldConfig": { 1034 "defaults": { 1035 "thresholds": { 1036 "mode": "absolute", 1037 "steps": [] 1038 } 1039 } 1040 }, 1041 "fill": 1, 1042 "grid": { 1043 "threshold1": null, 1044 "threshold1Color": "rgba(216, 200, 27, 0.27)", 1045 "threshold2": null, 1046 "threshold2Color": "rgba(234, 112, 112, 0.22)" 1047 }, 1048 "gridPos": null, 1049 "height": null, 1050 "hideTimeOverride": false, 1051 "id": 8, 1052 "interval": null, 1053 "isNew": true, 1054 "legend": { 1055 "alignAsTable": false, 1056 "avg": false, 1057 "current": false, 1058 "hideEmpty": false, 1059 "hideZero": false, 1060 "max": false, 1061 "min": false, 1062 "rightSide": false, 1063 "show": true, 1064 "sideWidth": null, 1065 "sort": null, 1066 "sortDesc": false, 1067 "total": false, 1068 "values": false 1069 }, 1070 "lines": true, 1071 "linewidth": 2, 1072 "links": [], 1073 "maxDataPoints": 100, 1074 "maxPerRow": null, 1075 "minSpan": null, 1076 "nullPointMode": null, 1077 "options": { 1078 "alertThreshold": true, 1079 "dataLinks": [] 1080 }, 1081 "percentage": false, 1082 "pointradius": 5, 1083 "points": false, 1084 "renderer": "flot", 1085 "repeat": null, 1086 "repeatDirection": null, 1087 "seriesOverrides": [], 1088 "span": 12, 1089 "stack": false, 1090 "steppedLine": false, 1091 "targets": [ 1092 { 1093 "datasource": "$source", 1094 "expr": "1", 1095 "format": "time_series", 1096 "hide": false, 1097 "instant": false, 1098 "interval": "", 1099 "intervalFactor": 2, 1100 "legendFormat": "threshold", 1101 "metric": "", 1102 "query": "1", 1103 "refId": "", 1104 "step": 10, 1105 "target": "" 1106 }, 1107 { 1108 "datasource": "$source", 1109 "expr": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"CREATE|DELETE|PATCH|POST|PUT\", scope=~\"namespace|cluster|resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 1110 "format": "time_series", 1111 "hide": false, 1112 "instant": false, 1113 "interval": "", 1114 "intervalFactor": 2, 1115 "legendFormat": "", 1116 "metric": "", 1117 "query": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"CREATE|DELETE|PATCH|POST|PUT\", scope=~\"namespace|cluster|resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 1118 "refId": "", 1119 "step": 10, 1120 "target": "" 1121 } 1122 ], 1123 "thresholds": [], 1124 "timeFrom": null, 1125 "timeShift": null, 1126 "title": "Mutating API call latency (threshold=1s)", 1127 "tooltip": { 1128 "msResolution": true, 1129 "shared": true, 1130 "sort": 2, 1131 "value_type": "cumulative" 1132 }, 1133 "transformations": [], 1134 "transparent": false, 1135 "type": "graph", 1136 "xaxis": { 1137 "mode": "time", 1138 "name": null, 1139 "show": true, 1140 "values": [] 1141 }, 1142 "yaxes": [ 1143 { 1144 "decimals": null, 1145 "format": "s", 1146 "label": null, 1147 "logBase": 1, 1148 "max": null, 1149 "min": null, 1150 "show": true 1151 }, 1152 { 1153 "decimals": null, 1154 "format": "short", 1155 "label": null, 1156 "logBase": 1, 1157 "max": null, 1158 "min": null, 1159 "show": true 1160 } 1161 ], 1162 "yaxis": { 1163 "align": false, 1164 "alignLevel": 0 1165 } 1166 } 1167 ], 1168 "repeat": null, 1169 "showTitle": true, 1170 "title": "API call latency aggregated with quantile" 1171 }, 1172 { 1173 "collapse": true, 1174 "editable": true, 1175 "height": "300px", 1176 "panels": [ 1177 { 1178 "aliasColors": {}, 1179 "bars": false, 1180 "cacheTimeout": null, 1181 "datasource": "$source", 1182 "description": null, 1183 "editable": true, 1184 "error": false, 1185 "fieldConfig": { 1186 "defaults": { 1187 "thresholds": { 1188 "mode": "absolute", 1189 "steps": [] 1190 } 1191 } 1192 }, 1193 "fill": 1, 1194 "grid": { 1195 "threshold1": null, 1196 "threshold1Color": "rgba(216, 200, 27, 0.27)", 1197 "threshold2": null, 1198 "threshold2Color": "rgba(234, 112, 112, 0.22)" 1199 }, 1200 "gridPos": null, 1201 "height": null, 1202 "hideTimeOverride": false, 1203 "id": 9, 1204 "interval": null, 1205 "isNew": true, 1206 "legend": { 1207 "alignAsTable": false, 1208 "avg": false, 1209 "current": false, 1210 "hideEmpty": false, 1211 "hideZero": false, 1212 "max": false, 1213 "min": false, 1214 "rightSide": false, 1215 "show": true, 1216 "sideWidth": null, 1217 "sort": null, 1218 "sortDesc": false, 1219 "total": false, 1220 "values": false 1221 }, 1222 "lines": true, 1223 "linewidth": 2, 1224 "links": [], 1225 "maxDataPoints": 100, 1226 "maxPerRow": null, 1227 "minSpan": null, 1228 "nullPointMode": null, 1229 "options": { 1230 "alertThreshold": true, 1231 "dataLinks": [] 1232 }, 1233 "percentage": false, 1234 "pointradius": 5, 1235 "points": false, 1236 "renderer": "flot", 1237 "repeat": null, 1238 "repeatDirection": null, 1239 "seriesOverrides": [], 1240 "span": 12, 1241 "stack": false, 1242 "steppedLine": false, 1243 "targets": [ 1244 { 1245 "datasource": "$source", 1246 "expr": "histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_wait_duration_seconds_bucket{instance=~\".*(${instance:pipe})\"}[1m])) by (le, instance, priority_level))", 1247 "format": "time_series", 1248 "hide": false, 1249 "instant": false, 1250 "interval": "5s", 1251 "intervalFactor": 1, 1252 "legendFormat": "{{instance}} {{priority_level}}", 1253 "metric": "", 1254 "query": "histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_wait_duration_seconds_bucket{instance=~\".*(${instance:pipe})\"}[1m])) by (le, instance, priority_level))", 1255 "refId": "", 1256 "step": 10, 1257 "target": "" 1258 } 1259 ], 1260 "thresholds": [], 1261 "timeFrom": null, 1262 "timeShift": null, 1263 "title": "Requests waiting time", 1264 "tooltip": { 1265 "msResolution": true, 1266 "shared": true, 1267 "sort": 2, 1268 "value_type": "cumulative" 1269 }, 1270 "transformations": [], 1271 "transparent": false, 1272 "type": "graph", 1273 "xaxis": { 1274 "mode": "time", 1275 "name": null, 1276 "show": true, 1277 "values": [] 1278 }, 1279 "yaxes": [ 1280 { 1281 "decimals": null, 1282 "format": "s", 1283 "label": null, 1284 "logBase": 1, 1285 "max": null, 1286 "min": null, 1287 "show": true 1288 }, 1289 { 1290 "decimals": null, 1291 "format": "short", 1292 "label": null, 1293 "logBase": 1, 1294 "max": null, 1295 "min": null, 1296 "show": true 1297 } 1298 ], 1299 "yaxis": { 1300 "align": false, 1301 "alignLevel": 0 1302 } 1303 }, 1304 { 1305 "aliasColors": {}, 1306 "bars": false, 1307 "cacheTimeout": null, 1308 "datasource": "$source", 1309 "description": null, 1310 "editable": true, 1311 "error": false, 1312 "fieldConfig": { 1313 "defaults": { 1314 "thresholds": { 1315 "mode": "absolute", 1316 "steps": [] 1317 } 1318 } 1319 }, 1320 "fill": 1, 1321 "grid": { 1322 "threshold1": null, 1323 "threshold1Color": "rgba(216, 200, 27, 0.27)", 1324 "threshold2": null, 1325 "threshold2Color": "rgba(234, 112, 112, 0.22)" 1326 }, 1327 "gridPos": null, 1328 "height": null, 1329 "hideTimeOverride": false, 1330 "id": 10, 1331 "interval": null, 1332 "isNew": true, 1333 "legend": { 1334 "alignAsTable": false, 1335 "avg": false, 1336 "current": false, 1337 "hideEmpty": false, 1338 "hideZero": false, 1339 "max": false, 1340 "min": false, 1341 "rightSide": false, 1342 "show": true, 1343 "sideWidth": null, 1344 "sort": null, 1345 "sortDesc": false, 1346 "total": false, 1347 "values": false 1348 }, 1349 "lines": true, 1350 "linewidth": 2, 1351 "links": [], 1352 "maxDataPoints": 100, 1353 "maxPerRow": null, 1354 "minSpan": null, 1355 "nullPointMode": null, 1356 "options": { 1357 "alertThreshold": true, 1358 "dataLinks": [] 1359 }, 1360 "percentage": false, 1361 "pointradius": 5, 1362 "points": false, 1363 "renderer": "flot", 1364 "repeat": null, 1365 "repeatDirection": null, 1366 "seriesOverrides": [], 1367 "span": 12, 1368 "stack": false, 1369 "steppedLine": false, 1370 "targets": [ 1371 { 1372 "datasource": "$source", 1373 "expr": "histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_execution_seconds_bucket{instance=~\".*(${instance:pipe})\"}[1m])) by (le, instance, priority_level))", 1374 "format": "time_series", 1375 "hide": false, 1376 "instant": false, 1377 "interval": "5s", 1378 "intervalFactor": 1, 1379 "legendFormat": "{{instance}} {{priority_level}}", 1380 "metric": "", 1381 "query": "histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_execution_seconds_bucket{instance=~\".*(${instance:pipe})\"}[1m])) by (le, instance, priority_level))", 1382 "refId": "", 1383 "step": 10, 1384 "target": "" 1385 } 1386 ], 1387 "thresholds": [], 1388 "timeFrom": null, 1389 "timeShift": null, 1390 "title": "Execution time", 1391 "tooltip": { 1392 "msResolution": true, 1393 "shared": true, 1394 "sort": 2, 1395 "value_type": "cumulative" 1396 }, 1397 "transformations": [], 1398 "transparent": false, 1399 "type": "graph", 1400 "xaxis": { 1401 "mode": "time", 1402 "name": null, 1403 "show": true, 1404 "values": [] 1405 }, 1406 "yaxes": [ 1407 { 1408 "decimals": null, 1409 "format": "s", 1410 "label": null, 1411 "logBase": 1, 1412 "max": null, 1413 "min": null, 1414 "show": true 1415 }, 1416 { 1417 "decimals": null, 1418 "format": "short", 1419 "label": null, 1420 "logBase": 1, 1421 "max": null, 1422 "min": null, 1423 "show": true 1424 } 1425 ], 1426 "yaxis": { 1427 "align": false, 1428 "alignLevel": 0 1429 } 1430 }, 1431 { 1432 "aliasColors": {}, 1433 "bars": false, 1434 "cacheTimeout": null, 1435 "datasource": "$source", 1436 "description": null, 1437 "editable": true, 1438 "error": false, 1439 "fieldConfig": { 1440 "defaults": { 1441 "thresholds": { 1442 "mode": "absolute", 1443 "steps": [] 1444 } 1445 } 1446 }, 1447 "fill": 1, 1448 "grid": { 1449 "threshold1": null, 1450 "threshold1Color": "rgba(216, 200, 27, 0.27)", 1451 "threshold2": null, 1452 "threshold2Color": "rgba(234, 112, 112, 0.22)" 1453 }, 1454 "gridPos": null, 1455 "height": null, 1456 "hideTimeOverride": false, 1457 "id": 11, 1458 "interval": null, 1459 "isNew": true, 1460 "legend": { 1461 "alignAsTable": false, 1462 "avg": false, 1463 "current": false, 1464 "hideEmpty": false, 1465 "hideZero": false, 1466 "max": false, 1467 "min": false, 1468 "rightSide": false, 1469 "show": true, 1470 "sideWidth": null, 1471 "sort": null, 1472 "sortDesc": false, 1473 "total": false, 1474 "values": false 1475 }, 1476 "lines": true, 1477 "linewidth": 2, 1478 "links": [], 1479 "maxDataPoints": 100, 1480 "maxPerRow": null, 1481 "minSpan": null, 1482 "nullPointMode": null, 1483 "options": { 1484 "alertThreshold": true, 1485 "dataLinks": [] 1486 }, 1487 "percentage": false, 1488 "pointradius": 5, 1489 "points": false, 1490 "renderer": "flot", 1491 "repeat": null, 1492 "repeatDirection": null, 1493 "seriesOverrides": [], 1494 "span": 12, 1495 "stack": false, 1496 "steppedLine": false, 1497 "targets": [ 1498 { 1499 "datasource": "$source", 1500 "expr": "sum(irate(apiserver_flowcontrol_request_execution_seconds_sum{instance=~\".*(${instance:pipe})\"}[1m])) by (instance, priority_level)", 1501 "format": "time_series", 1502 "hide": false, 1503 "instant": false, 1504 "interval": "5s", 1505 "intervalFactor": 1, 1506 "legendFormat": "{{instance}} {{priority_level}}", 1507 "metric": "", 1508 "query": "sum(irate(apiserver_flowcontrol_request_execution_seconds_sum{instance=~\".*(${instance:pipe})\"}[1m])) by (instance, priority_level)", 1509 "refId": "", 1510 "step": 10, 1511 "target": "" 1512 } 1513 ], 1514 "thresholds": [], 1515 "timeFrom": null, 1516 "timeShift": null, 1517 "title": "Total execution time per second", 1518 "tooltip": { 1519 "msResolution": true, 1520 "shared": true, 1521 "sort": 2, 1522 "value_type": "cumulative" 1523 }, 1524 "transformations": [], 1525 "transparent": false, 1526 "type": "graph", 1527 "xaxis": { 1528 "mode": "time", 1529 "name": null, 1530 "show": true, 1531 "values": [] 1532 }, 1533 "yaxes": [ 1534 { 1535 "decimals": null, 1536 "format": "s", 1537 "label": null, 1538 "logBase": 1, 1539 "max": null, 1540 "min": null, 1541 "show": true 1542 }, 1543 { 1544 "decimals": null, 1545 "format": "short", 1546 "label": null, 1547 "logBase": 1, 1548 "max": null, 1549 "min": null, 1550 "show": true 1551 } 1552 ], 1553 "yaxis": { 1554 "align": false, 1555 "alignLevel": 0 1556 } 1557 }, 1558 { 1559 "aliasColors": {}, 1560 "bars": false, 1561 "cacheTimeout": null, 1562 "datasource": "$source", 1563 "description": null, 1564 "editable": true, 1565 "error": false, 1566 "fieldConfig": { 1567 "defaults": { 1568 "thresholds": { 1569 "mode": "absolute", 1570 "steps": [] 1571 } 1572 } 1573 }, 1574 "fill": 1, 1575 "grid": { 1576 "threshold1": null, 1577 "threshold1Color": "rgba(216, 200, 27, 0.27)", 1578 "threshold2": null, 1579 "threshold2Color": "rgba(234, 112, 112, 0.22)" 1580 }, 1581 "gridPos": null, 1582 "height": null, 1583 "hideTimeOverride": false, 1584 "id": 12, 1585 "interval": null, 1586 "isNew": true, 1587 "legend": { 1588 "alignAsTable": false, 1589 "avg": false, 1590 "current": false, 1591 "hideEmpty": false, 1592 "hideZero": false, 1593 "max": false, 1594 "min": false, 1595 "rightSide": false, 1596 "show": true, 1597 "sideWidth": null, 1598 "sort": null, 1599 "sortDesc": false, 1600 "total": false, 1601 "values": false 1602 }, 1603 "lines": true, 1604 "linewidth": 2, 1605 "links": [], 1606 "maxDataPoints": 100, 1607 "maxPerRow": null, 1608 "minSpan": null, 1609 "nullPointMode": null, 1610 "options": { 1611 "alertThreshold": true, 1612 "dataLinks": [] 1613 }, 1614 "percentage": false, 1615 "pointradius": 5, 1616 "points": false, 1617 "renderer": "flot", 1618 "repeat": null, 1619 "repeatDirection": null, 1620 "seriesOverrides": [], 1621 "span": 12, 1622 "stack": false, 1623 "steppedLine": false, 1624 "targets": [ 1625 { 1626 "datasource": "$source", 1627 "expr": "sum(irate(apiserver_flowcontrol_dispatched_requests_total{instance=~\".*(${instance:pipe})\"}[1m])) by (instance, priority_level)", 1628 "format": "time_series", 1629 "hide": false, 1630 "instant": false, 1631 "interval": "5s", 1632 "intervalFactor": 1, 1633 "legendFormat": "{{instance}} {{priority_level}}", 1634 "metric": "", 1635 "query": "sum(irate(apiserver_flowcontrol_dispatched_requests_total{instance=~\".*(${instance:pipe})\"}[1m])) by (instance, priority_level)", 1636 "refId": "", 1637 "step": 10, 1638 "target": "" 1639 } 1640 ], 1641 "thresholds": [], 1642 "timeFrom": null, 1643 "timeShift": null, 1644 "title": "Requests rate by priority level", 1645 "tooltip": { 1646 "msResolution": true, 1647 "shared": true, 1648 "sort": 2, 1649 "value_type": "cumulative" 1650 }, 1651 "transformations": [], 1652 "transparent": false, 1653 "type": "graph", 1654 "xaxis": { 1655 "mode": "time", 1656 "name": null, 1657 "show": true, 1658 "values": [] 1659 }, 1660 "yaxes": [ 1661 { 1662 "decimals": null, 1663 "format": "ops", 1664 "label": null, 1665 "logBase": 1, 1666 "max": null, 1667 "min": null, 1668 "show": true 1669 }, 1670 { 1671 "decimals": null, 1672 "format": "short", 1673 "label": null, 1674 "logBase": 1, 1675 "max": null, 1676 "min": null, 1677 "show": true 1678 } 1679 ], 1680 "yaxis": { 1681 "align": false, 1682 "alignLevel": 0 1683 } 1684 }, 1685 { 1686 "aliasColors": {}, 1687 "bars": false, 1688 "cacheTimeout": null, 1689 "datasource": "$source", 1690 "description": null, 1691 "editable": true, 1692 "error": false, 1693 "fieldConfig": { 1694 "defaults": { 1695 "thresholds": { 1696 "mode": "absolute", 1697 "steps": [] 1698 } 1699 } 1700 }, 1701 "fill": 1, 1702 "grid": { 1703 "threshold1": null, 1704 "threshold1Color": "rgba(216, 200, 27, 0.27)", 1705 "threshold2": null, 1706 "threshold2Color": "rgba(234, 112, 112, 0.22)" 1707 }, 1708 "gridPos": null, 1709 "height": null, 1710 "hideTimeOverride": false, 1711 "id": 13, 1712 "interval": null, 1713 "isNew": true, 1714 "legend": { 1715 "alignAsTable": false, 1716 "avg": false, 1717 "current": false, 1718 "hideEmpty": false, 1719 "hideZero": false, 1720 "max": false, 1721 "min": false, 1722 "rightSide": false, 1723 "show": true, 1724 "sideWidth": null, 1725 "sort": null, 1726 "sortDesc": false, 1727 "total": false, 1728 "values": false 1729 }, 1730 "lines": true, 1731 "linewidth": 2, 1732 "links": [], 1733 "maxDataPoints": 100, 1734 "maxPerRow": null, 1735 "minSpan": null, 1736 "nullPointMode": null, 1737 "options": { 1738 "alertThreshold": true, 1739 "dataLinks": [] 1740 }, 1741 "percentage": false, 1742 "pointradius": 5, 1743 "points": false, 1744 "renderer": "flot", 1745 "repeat": null, 1746 "repeatDirection": null, 1747 "seriesOverrides": [], 1748 "span": 12, 1749 "stack": false, 1750 "steppedLine": false, 1751 "targets": [ 1752 { 1753 "datasource": "$source", 1754 "expr": "sum(apiserver_flowcontrol_request_concurrency_in_use{instance=~\".*(${instance:pipe})\"}) by (instance, priority_level)", 1755 "format": "time_series", 1756 "hide": false, 1757 "instant": false, 1758 "interval": "5s", 1759 "intervalFactor": 1, 1760 "legendFormat": "{{instance}} {{priority_level}}", 1761 "metric": "", 1762 "query": "sum(apiserver_flowcontrol_request_concurrency_in_use{instance=~\".*(${instance:pipe})\"}) by (instance, priority_level)", 1763 "refId": "", 1764 "step": 10, 1765 "target": "" 1766 } 1767 ], 1768 "thresholds": [], 1769 "timeFrom": null, 1770 "timeShift": null, 1771 "title": "Concurrency in use", 1772 "tooltip": { 1773 "msResolution": true, 1774 "shared": true, 1775 "sort": 2, 1776 "value_type": "cumulative" 1777 }, 1778 "transformations": [], 1779 "transparent": false, 1780 "type": "graph", 1781 "xaxis": { 1782 "mode": "time", 1783 "name": null, 1784 "show": true, 1785 "values": [] 1786 }, 1787 "yaxes": [ 1788 { 1789 "decimals": null, 1790 "format": "ops", 1791 "label": null, 1792 "logBase": 1, 1793 "max": null, 1794 "min": null, 1795 "show": true 1796 }, 1797 { 1798 "decimals": null, 1799 "format": "short", 1800 "label": null, 1801 "logBase": 1, 1802 "max": null, 1803 "min": null, 1804 "show": true 1805 } 1806 ], 1807 "yaxis": { 1808 "align": false, 1809 "alignLevel": 0 1810 } 1811 }, 1812 { 1813 "aliasColors": {}, 1814 "bars": false, 1815 "cacheTimeout": null, 1816 "datasource": "$source", 1817 "description": null, 1818 "editable": true, 1819 "error": false, 1820 "fieldConfig": { 1821 "defaults": { 1822 "thresholds": { 1823 "mode": "absolute", 1824 "steps": [] 1825 } 1826 } 1827 }, 1828 "fill": 1, 1829 "grid": { 1830 "threshold1": null, 1831 "threshold1Color": "rgba(216, 200, 27, 0.27)", 1832 "threshold2": null, 1833 "threshold2Color": "rgba(234, 112, 112, 0.22)" 1834 }, 1835 "gridPos": null, 1836 "height": null, 1837 "hideTimeOverride": false, 1838 "id": 14, 1839 "interval": null, 1840 "isNew": true, 1841 "legend": { 1842 "alignAsTable": false, 1843 "avg": false, 1844 "current": false, 1845 "hideEmpty": false, 1846 "hideZero": false, 1847 "max": false, 1848 "min": false, 1849 "rightSide": false, 1850 "show": true, 1851 "sideWidth": null, 1852 "sort": null, 1853 "sortDesc": false, 1854 "total": false, 1855 "values": false 1856 }, 1857 "lines": true, 1858 "linewidth": 2, 1859 "links": [], 1860 "maxDataPoints": 100, 1861 "maxPerRow": null, 1862 "minSpan": null, 1863 "nullPointMode": null, 1864 "options": { 1865 "alertThreshold": true, 1866 "dataLinks": [] 1867 }, 1868 "percentage": false, 1869 "pointradius": 5, 1870 "points": false, 1871 "renderer": "flot", 1872 "repeat": null, 1873 "repeatDirection": null, 1874 "seriesOverrides": [], 1875 "span": 12, 1876 "stack": false, 1877 "steppedLine": false, 1878 "targets": [ 1879 { 1880 "datasource": "$source", 1881 "expr": "sum(apiserver_flowcontrol_current_executing_requests{instance=~\".*(${instance:pipe})\"}) by (instance, priority_level)", 1882 "format": "time_series", 1883 "hide": false, 1884 "instant": false, 1885 "interval": "5s", 1886 "intervalFactor": 1, 1887 "legendFormat": "{{instance}} {{priority_level}}", 1888 "metric": "", 1889 "query": "sum(apiserver_flowcontrol_current_executing_requests{instance=~\".*(${instance:pipe})\"}) by (instance, priority_level)", 1890 "refId": "", 1891 "step": 10, 1892 "target": "" 1893 } 1894 ], 1895 "thresholds": [], 1896 "timeFrom": null, 1897 "timeShift": null, 1898 "title": "Current executing requests", 1899 "tooltip": { 1900 "msResolution": true, 1901 "shared": true, 1902 "sort": 2, 1903 "value_type": "cumulative" 1904 }, 1905 "transformations": [], 1906 "transparent": false, 1907 "type": "graph", 1908 "xaxis": { 1909 "mode": "time", 1910 "name": null, 1911 "show": true, 1912 "values": [] 1913 }, 1914 "yaxes": [ 1915 { 1916 "decimals": null, 1917 "format": "ops", 1918 "label": null, 1919 "logBase": 1, 1920 "max": null, 1921 "min": null, 1922 "show": true 1923 }, 1924 { 1925 "decimals": null, 1926 "format": "short", 1927 "label": null, 1928 "logBase": 1, 1929 "max": null, 1930 "min": null, 1931 "show": true 1932 } 1933 ], 1934 "yaxis": { 1935 "align": false, 1936 "alignLevel": 0 1937 } 1938 }, 1939 { 1940 "aliasColors": {}, 1941 "bars": false, 1942 "cacheTimeout": null, 1943 "datasource": "$source", 1944 "description": null, 1945 "editable": true, 1946 "error": false, 1947 "fieldConfig": { 1948 "defaults": { 1949 "thresholds": { 1950 "mode": "absolute", 1951 "steps": [] 1952 } 1953 } 1954 }, 1955 "fill": 1, 1956 "grid": { 1957 "threshold1": null, 1958 "threshold1Color": "rgba(216, 200, 27, 0.27)", 1959 "threshold2": null, 1960 "threshold2Color": "rgba(234, 112, 112, 0.22)" 1961 }, 1962 "gridPos": null, 1963 "height": null, 1964 "hideTimeOverride": false, 1965 "id": 15, 1966 "interval": null, 1967 "isNew": true, 1968 "legend": { 1969 "alignAsTable": false, 1970 "avg": false, 1971 "current": false, 1972 "hideEmpty": false, 1973 "hideZero": false, 1974 "max": false, 1975 "min": false, 1976 "rightSide": false, 1977 "show": true, 1978 "sideWidth": null, 1979 "sort": null, 1980 "sortDesc": false, 1981 "total": false, 1982 "values": false 1983 }, 1984 "lines": true, 1985 "linewidth": 2, 1986 "links": [], 1987 "maxDataPoints": 100, 1988 "maxPerRow": null, 1989 "minSpan": null, 1990 "nullPointMode": null, 1991 "options": { 1992 "alertThreshold": true, 1993 "dataLinks": [] 1994 }, 1995 "percentage": false, 1996 "pointradius": 5, 1997 "points": false, 1998 "renderer": "flot", 1999 "repeat": null, 2000 "repeatDirection": null, 2001 "seriesOverrides": [], 2002 "span": 12, 2003 "stack": false, 2004 "steppedLine": false, 2005 "targets": [ 2006 { 2007 "datasource": "$source", 2008 "expr": "sum(apiserver_flowcontrol_current_inqueue_requests{instance=~\".*(${instance:pipe})\"}) by (instance, priority_level)", 2009 "format": "time_series", 2010 "hide": false, 2011 "instant": false, 2012 "interval": "5s", 2013 "intervalFactor": 1, 2014 "legendFormat": "{{instance}} {{priority_level}}", 2015 "metric": "", 2016 "query": "sum(apiserver_flowcontrol_current_inqueue_requests{instance=~\".*(${instance:pipe})\"}) by (instance, priority_level)", 2017 "refId": "", 2018 "step": 10, 2019 "target": "" 2020 } 2021 ], 2022 "thresholds": [], 2023 "timeFrom": null, 2024 "timeShift": null, 2025 "title": "Inqueue requests", 2026 "tooltip": { 2027 "msResolution": true, 2028 "shared": true, 2029 "sort": 2, 2030 "value_type": "cumulative" 2031 }, 2032 "transformations": [], 2033 "transparent": false, 2034 "type": "graph", 2035 "xaxis": { 2036 "mode": "time", 2037 "name": null, 2038 "show": true, 2039 "values": [] 2040 }, 2041 "yaxes": [ 2042 { 2043 "decimals": null, 2044 "format": "ops", 2045 "label": null, 2046 "logBase": 1, 2047 "max": null, 2048 "min": null, 2049 "show": true 2050 }, 2051 { 2052 "decimals": null, 2053 "format": "short", 2054 "label": null, 2055 "logBase": 1, 2056 "max": null, 2057 "min": null, 2058 "show": true 2059 } 2060 ], 2061 "yaxis": { 2062 "align": false, 2063 "alignLevel": 0 2064 } 2065 }, 2066 { 2067 "aliasColors": {}, 2068 "bars": false, 2069 "cacheTimeout": null, 2070 "datasource": "$source", 2071 "description": null, 2072 "editable": true, 2073 "error": false, 2074 "fieldConfig": { 2075 "defaults": { 2076 "thresholds": { 2077 "mode": "absolute", 2078 "steps": [] 2079 } 2080 } 2081 }, 2082 "fill": 1, 2083 "grid": { 2084 "threshold1": null, 2085 "threshold1Color": "rgba(216, 200, 27, 0.27)", 2086 "threshold2": null, 2087 "threshold2Color": "rgba(234, 112, 112, 0.22)" 2088 }, 2089 "gridPos": null, 2090 "height": null, 2091 "hideTimeOverride": false, 2092 "id": 16, 2093 "interval": null, 2094 "isNew": true, 2095 "legend": { 2096 "alignAsTable": false, 2097 "avg": false, 2098 "current": false, 2099 "hideEmpty": false, 2100 "hideZero": false, 2101 "max": false, 2102 "min": false, 2103 "rightSide": false, 2104 "show": true, 2105 "sideWidth": null, 2106 "sort": null, 2107 "sortDesc": false, 2108 "total": false, 2109 "values": false 2110 }, 2111 "lines": true, 2112 "linewidth": 2, 2113 "links": [], 2114 "maxDataPoints": 100, 2115 "maxPerRow": null, 2116 "minSpan": null, 2117 "nullPointMode": null, 2118 "options": { 2119 "alertThreshold": true, 2120 "dataLinks": [] 2121 }, 2122 "percentage": false, 2123 "pointradius": 5, 2124 "points": false, 2125 "renderer": "flot", 2126 "repeat": null, 2127 "repeatDirection": null, 2128 "seriesOverrides": [], 2129 "span": 12, 2130 "stack": false, 2131 "steppedLine": false, 2132 "targets": [ 2133 { 2134 "datasource": "$source", 2135 "expr": "avg(apiserver_flowcontrol_nominal_limit_seats{instance=~\".*(${instance:pipe})\"}) by (priority_level)", 2136 "format": "time_series", 2137 "hide": false, 2138 "instant": false, 2139 "interval": "5s", 2140 "intervalFactor": 1, 2141 "legendFormat": "{{priority_level}}", 2142 "metric": "", 2143 "query": "avg(apiserver_flowcontrol_nominal_limit_seats{instance=~\".*(${instance:pipe})\"}) by (priority_level)", 2144 "refId": "", 2145 "step": 10, 2146 "target": "" 2147 } 2148 ], 2149 "thresholds": [], 2150 "timeFrom": null, 2151 "timeShift": null, 2152 "title": "Nominal number of execution seats", 2153 "tooltip": { 2154 "msResolution": true, 2155 "shared": true, 2156 "sort": 2, 2157 "value_type": "cumulative" 2158 }, 2159 "transformations": [], 2160 "transparent": false, 2161 "type": "graph", 2162 "xaxis": { 2163 "mode": "time", 2164 "name": null, 2165 "show": true, 2166 "values": [] 2167 }, 2168 "yaxes": [ 2169 { 2170 "decimals": null, 2171 "format": "short", 2172 "label": null, 2173 "logBase": 1, 2174 "max": null, 2175 "min": null, 2176 "show": true 2177 }, 2178 { 2179 "decimals": null, 2180 "format": "short", 2181 "label": null, 2182 "logBase": 1, 2183 "max": null, 2184 "min": null, 2185 "show": true 2186 } 2187 ], 2188 "yaxis": { 2189 "align": false, 2190 "alignLevel": 0 2191 } 2192 }, 2193 { 2194 "aliasColors": {}, 2195 "bars": false, 2196 "cacheTimeout": null, 2197 "datasource": "$source", 2198 "description": null, 2199 "editable": true, 2200 "error": false, 2201 "fieldConfig": { 2202 "defaults": { 2203 "thresholds": { 2204 "mode": "absolute", 2205 "steps": [] 2206 } 2207 } 2208 }, 2209 "fill": 1, 2210 "grid": { 2211 "threshold1": null, 2212 "threshold1Color": "rgba(216, 200, 27, 0.27)", 2213 "threshold2": null, 2214 "threshold2Color": "rgba(234, 112, 112, 0.22)" 2215 }, 2216 "gridPos": null, 2217 "height": null, 2218 "hideTimeOverride": false, 2219 "id": 17, 2220 "interval": null, 2221 "isNew": true, 2222 "legend": { 2223 "alignAsTable": false, 2224 "avg": false, 2225 "current": false, 2226 "hideEmpty": false, 2227 "hideZero": false, 2228 "max": false, 2229 "min": false, 2230 "rightSide": false, 2231 "show": true, 2232 "sideWidth": null, 2233 "sort": null, 2234 "sortDesc": false, 2235 "total": false, 2236 "values": false 2237 }, 2238 "lines": true, 2239 "linewidth": 2, 2240 "links": [], 2241 "maxDataPoints": 100, 2242 "maxPerRow": null, 2243 "minSpan": null, 2244 "nullPointMode": null, 2245 "options": { 2246 "alertThreshold": true, 2247 "dataLinks": [] 2248 }, 2249 "percentage": false, 2250 "pointradius": 5, 2251 "points": false, 2252 "renderer": "flot", 2253 "repeat": null, 2254 "repeatDirection": null, 2255 "seriesOverrides": [], 2256 "span": 12, 2257 "stack": false, 2258 "steppedLine": false, 2259 "targets": [ 2260 { 2261 "datasource": "$source", 2262 "expr": "avg(apiserver_flowcontrol_lower_limit_seats{instance=~\".*(${instance:pipe})\"}) by (priority_level)", 2263 "format": "time_series", 2264 "hide": false, 2265 "instant": false, 2266 "interval": "5s", 2267 "intervalFactor": 1, 2268 "legendFormat": "{{priority_level}}", 2269 "metric": "", 2270 "query": "avg(apiserver_flowcontrol_lower_limit_seats{instance=~\".*(${instance:pipe})\"}) by (priority_level)", 2271 "refId": "", 2272 "step": 10, 2273 "target": "" 2274 } 2275 ], 2276 "thresholds": [], 2277 "timeFrom": null, 2278 "timeShift": null, 2279 "title": "Lower bound on number of execution seats", 2280 "tooltip": { 2281 "msResolution": true, 2282 "shared": true, 2283 "sort": 2, 2284 "value_type": "cumulative" 2285 }, 2286 "transformations": [], 2287 "transparent": false, 2288 "type": "graph", 2289 "xaxis": { 2290 "mode": "time", 2291 "name": null, 2292 "show": true, 2293 "values": [] 2294 }, 2295 "yaxes": [ 2296 { 2297 "decimals": null, 2298 "format": "short", 2299 "label": null, 2300 "logBase": 1, 2301 "max": null, 2302 "min": null, 2303 "show": true 2304 }, 2305 { 2306 "decimals": null, 2307 "format": "short", 2308 "label": null, 2309 "logBase": 1, 2310 "max": null, 2311 "min": null, 2312 "show": true 2313 } 2314 ], 2315 "yaxis": { 2316 "align": false, 2317 "alignLevel": 0 2318 } 2319 }, 2320 { 2321 "aliasColors": {}, 2322 "bars": false, 2323 "cacheTimeout": null, 2324 "datasource": "$source", 2325 "description": null, 2326 "editable": true, 2327 "error": false, 2328 "fieldConfig": { 2329 "defaults": { 2330 "thresholds": { 2331 "mode": "absolute", 2332 "steps": [] 2333 } 2334 } 2335 }, 2336 "fill": 1, 2337 "grid": { 2338 "threshold1": null, 2339 "threshold1Color": "rgba(216, 200, 27, 0.27)", 2340 "threshold2": null, 2341 "threshold2Color": "rgba(234, 112, 112, 0.22)" 2342 }, 2343 "gridPos": null, 2344 "height": null, 2345 "hideTimeOverride": false, 2346 "id": 18, 2347 "interval": null, 2348 "isNew": true, 2349 "legend": { 2350 "alignAsTable": false, 2351 "avg": false, 2352 "current": false, 2353 "hideEmpty": false, 2354 "hideZero": false, 2355 "max": false, 2356 "min": false, 2357 "rightSide": false, 2358 "show": true, 2359 "sideWidth": null, 2360 "sort": null, 2361 "sortDesc": false, 2362 "total": false, 2363 "values": false 2364 }, 2365 "lines": true, 2366 "linewidth": 2, 2367 "links": [], 2368 "maxDataPoints": 100, 2369 "maxPerRow": null, 2370 "minSpan": null, 2371 "nullPointMode": null, 2372 "options": { 2373 "alertThreshold": true, 2374 "dataLinks": [] 2375 }, 2376 "percentage": false, 2377 "pointradius": 5, 2378 "points": false, 2379 "renderer": "flot", 2380 "repeat": null, 2381 "repeatDirection": null, 2382 "seriesOverrides": [], 2383 "span": 12, 2384 "stack": false, 2385 "steppedLine": false, 2386 "targets": [ 2387 { 2388 "datasource": "$source", 2389 "expr": "avg(apiserver_flowcontrol_upper_limit_seats{instance=~\".*(${instance:pipe})\"}) by (priority_level)", 2390 "format": "time_series", 2391 "hide": false, 2392 "instant": false, 2393 "interval": "5s", 2394 "intervalFactor": 1, 2395 "legendFormat": "{{priority_level}}", 2396 "metric": "", 2397 "query": "avg(apiserver_flowcontrol_upper_limit_seats{instance=~\".*(${instance:pipe})\"}) by (priority_level)", 2398 "refId": "", 2399 "step": 10, 2400 "target": "" 2401 } 2402 ], 2403 "thresholds": [], 2404 "timeFrom": null, 2405 "timeShift": null, 2406 "title": "Upper bound on number of execution seats", 2407 "tooltip": { 2408 "msResolution": true, 2409 "shared": true, 2410 "sort": 2, 2411 "value_type": "cumulative" 2412 }, 2413 "transformations": [], 2414 "transparent": false, 2415 "type": "graph", 2416 "xaxis": { 2417 "mode": "time", 2418 "name": null, 2419 "show": true, 2420 "values": [] 2421 }, 2422 "yaxes": [ 2423 { 2424 "decimals": null, 2425 "format": "short", 2426 "label": null, 2427 "logBase": 1, 2428 "max": null, 2429 "min": null, 2430 "show": true 2431 }, 2432 { 2433 "decimals": null, 2434 "format": "short", 2435 "label": null, 2436 "logBase": 1, 2437 "max": null, 2438 "min": null, 2439 "show": true 2440 } 2441 ], 2442 "yaxis": { 2443 "align": false, 2444 "alignLevel": 0 2445 } 2446 }, 2447 { 2448 "aliasColors": {}, 2449 "bars": false, 2450 "cacheTimeout": null, 2451 "datasource": "$source", 2452 "description": null, 2453 "editable": true, 2454 "error": false, 2455 "fieldConfig": { 2456 "defaults": { 2457 "thresholds": { 2458 "mode": "absolute", 2459 "steps": [] 2460 } 2461 } 2462 }, 2463 "fill": 1, 2464 "grid": { 2465 "threshold1": null, 2466 "threshold1Color": "rgba(216, 200, 27, 0.27)", 2467 "threshold2": null, 2468 "threshold2Color": "rgba(234, 112, 112, 0.22)" 2469 }, 2470 "gridPos": null, 2471 "height": null, 2472 "hideTimeOverride": false, 2473 "id": 19, 2474 "interval": null, 2475 "isNew": true, 2476 "legend": { 2477 "alignAsTable": false, 2478 "avg": false, 2479 "current": false, 2480 "hideEmpty": false, 2481 "hideZero": false, 2482 "max": false, 2483 "min": false, 2484 "rightSide": false, 2485 "show": true, 2486 "sideWidth": null, 2487 "sort": null, 2488 "sortDesc": false, 2489 "total": false, 2490 "values": false 2491 }, 2492 "lines": true, 2493 "linewidth": 2, 2494 "links": [], 2495 "maxDataPoints": 100, 2496 "maxPerRow": null, 2497 "minSpan": null, 2498 "nullPointMode": null, 2499 "options": { 2500 "alertThreshold": true, 2501 "dataLinks": [] 2502 }, 2503 "percentage": false, 2504 "pointradius": 5, 2505 "points": false, 2506 "renderer": "flot", 2507 "repeat": null, 2508 "repeatDirection": null, 2509 "seriesOverrides": [], 2510 "span": 12, 2511 "stack": false, 2512 "steppedLine": false, 2513 "targets": [ 2514 { 2515 "datasource": "$source", 2516 "expr": "histogram_quantile(0.5, rate(apiserver_flowcontrol_demand_seats_bucket{instance=~\".*(${instance:pipe})\"}[10s]))", 2517 "format": "time_series", 2518 "hide": false, 2519 "instant": false, 2520 "interval": "5s", 2521 "intervalFactor": 1, 2522 "legendFormat": "{{instance}} {{priority_level}}", 2523 "metric": "", 2524 "query": "histogram_quantile(0.5, rate(apiserver_flowcontrol_demand_seats_bucket{instance=~\".*(${instance:pipe})\"}[10s]))", 2525 "refId": "", 2526 "step": 10, 2527 "target": "" 2528 } 2529 ], 2530 "thresholds": [], 2531 "timeFrom": null, 2532 "timeShift": null, 2533 "title": "Number of seats Priority Level could use divided by nominal seats (50th percentile)", 2534 "tooltip": { 2535 "msResolution": true, 2536 "shared": true, 2537 "sort": 2, 2538 "value_type": "cumulative" 2539 }, 2540 "transformations": [], 2541 "transparent": false, 2542 "type": "graph", 2543 "xaxis": { 2544 "mode": "time", 2545 "name": null, 2546 "show": true, 2547 "values": [] 2548 }, 2549 "yaxes": [ 2550 { 2551 "decimals": null, 2552 "format": "short", 2553 "label": null, 2554 "logBase": 1, 2555 "max": null, 2556 "min": null, 2557 "show": true 2558 }, 2559 { 2560 "decimals": null, 2561 "format": "short", 2562 "label": null, 2563 "logBase": 1, 2564 "max": null, 2565 "min": null, 2566 "show": true 2567 } 2568 ], 2569 "yaxis": { 2570 "align": false, 2571 "alignLevel": 0 2572 } 2573 }, 2574 { 2575 "aliasColors": {}, 2576 "bars": false, 2577 "cacheTimeout": null, 2578 "datasource": "$source", 2579 "description": null, 2580 "editable": true, 2581 "error": false, 2582 "fieldConfig": { 2583 "defaults": { 2584 "thresholds": { 2585 "mode": "absolute", 2586 "steps": [] 2587 } 2588 } 2589 }, 2590 "fill": 1, 2591 "grid": { 2592 "threshold1": null, 2593 "threshold1Color": "rgba(216, 200, 27, 0.27)", 2594 "threshold2": null, 2595 "threshold2Color": "rgba(234, 112, 112, 0.22)" 2596 }, 2597 "gridPos": null, 2598 "height": null, 2599 "hideTimeOverride": false, 2600 "id": 20, 2601 "interval": null, 2602 "isNew": true, 2603 "legend": { 2604 "alignAsTable": false, 2605 "avg": false, 2606 "current": false, 2607 "hideEmpty": false, 2608 "hideZero": false, 2609 "max": false, 2610 "min": false, 2611 "rightSide": false, 2612 "show": true, 2613 "sideWidth": null, 2614 "sort": null, 2615 "sortDesc": false, 2616 "total": false, 2617 "values": false 2618 }, 2619 "lines": true, 2620 "linewidth": 2, 2621 "links": [], 2622 "maxDataPoints": 100, 2623 "maxPerRow": null, 2624 "minSpan": null, 2625 "nullPointMode": null, 2626 "options": { 2627 "alertThreshold": true, 2628 "dataLinks": [] 2629 }, 2630 "percentage": false, 2631 "pointradius": 5, 2632 "points": false, 2633 "renderer": "flot", 2634 "repeat": null, 2635 "repeatDirection": null, 2636 "seriesOverrides": [], 2637 "span": 12, 2638 "stack": false, 2639 "steppedLine": false, 2640 "targets": [ 2641 { 2642 "datasource": "$source", 2643 "expr": "apiserver_flowcontrol_demand_seats_high_watermark{instance=~\".*(${instance:pipe})\"}", 2644 "format": "time_series", 2645 "hide": false, 2646 "instant": false, 2647 "interval": "5s", 2648 "intervalFactor": 1, 2649 "legendFormat": "{{instance}} {{priority_level}}", 2650 "metric": "", 2651 "query": "apiserver_flowcontrol_demand_seats_high_watermark{instance=~\".*(${instance:pipe})\"}", 2652 "refId": "", 2653 "step": 10, 2654 "target": "" 2655 } 2656 ], 2657 "thresholds": [], 2658 "timeFrom": null, 2659 "timeShift": null, 2660 "title": "High watermark of demand seats over last adjustment period", 2661 "tooltip": { 2662 "msResolution": true, 2663 "shared": true, 2664 "sort": 2, 2665 "value_type": "cumulative" 2666 }, 2667 "transformations": [], 2668 "transparent": false, 2669 "type": "graph", 2670 "xaxis": { 2671 "mode": "time", 2672 "name": null, 2673 "show": true, 2674 "values": [] 2675 }, 2676 "yaxes": [ 2677 { 2678 "decimals": null, 2679 "format": "short", 2680 "label": null, 2681 "logBase": 1, 2682 "max": null, 2683 "min": null, 2684 "show": true 2685 }, 2686 { 2687 "decimals": null, 2688 "format": "short", 2689 "label": null, 2690 "logBase": 1, 2691 "max": null, 2692 "min": null, 2693 "show": true 2694 } 2695 ], 2696 "yaxis": { 2697 "align": false, 2698 "alignLevel": 0 2699 } 2700 }, 2701 { 2702 "aliasColors": {}, 2703 "bars": false, 2704 "cacheTimeout": null, 2705 "datasource": "$source", 2706 "description": null, 2707 "editable": true, 2708 "error": false, 2709 "fieldConfig": { 2710 "defaults": { 2711 "thresholds": { 2712 "mode": "absolute", 2713 "steps": [] 2714 } 2715 } 2716 }, 2717 "fill": 1, 2718 "grid": { 2719 "threshold1": null, 2720 "threshold1Color": "rgba(216, 200, 27, 0.27)", 2721 "threshold2": null, 2722 "threshold2Color": "rgba(234, 112, 112, 0.22)" 2723 }, 2724 "gridPos": null, 2725 "height": null, 2726 "hideTimeOverride": false, 2727 "id": 21, 2728 "interval": null, 2729 "isNew": true, 2730 "legend": { 2731 "alignAsTable": false, 2732 "avg": false, 2733 "current": false, 2734 "hideEmpty": false, 2735 "hideZero": false, 2736 "max": false, 2737 "min": false, 2738 "rightSide": false, 2739 "show": true, 2740 "sideWidth": null, 2741 "sort": null, 2742 "sortDesc": false, 2743 "total": false, 2744 "values": false 2745 }, 2746 "lines": true, 2747 "linewidth": 2, 2748 "links": [], 2749 "maxDataPoints": 100, 2750 "maxPerRow": null, 2751 "minSpan": null, 2752 "nullPointMode": null, 2753 "options": { 2754 "alertThreshold": true, 2755 "dataLinks": [] 2756 }, 2757 "percentage": false, 2758 "pointradius": 5, 2759 "points": false, 2760 "renderer": "flot", 2761 "repeat": null, 2762 "repeatDirection": null, 2763 "seriesOverrides": [], 2764 "span": 12, 2765 "stack": false, 2766 "steppedLine": false, 2767 "targets": [ 2768 { 2769 "datasource": "$source", 2770 "expr": "apiserver_flowcontrol_demand_seats_smoothed{instance=~\".*(${instance:pipe})\"}", 2771 "format": "time_series", 2772 "hide": false, 2773 "instant": false, 2774 "interval": "5s", 2775 "intervalFactor": 1, 2776 "legendFormat": "{{instance}} {{priority_level}}", 2777 "metric": "", 2778 "query": "apiserver_flowcontrol_demand_seats_smoothed{instance=~\".*(${instance:pipe})\"}", 2779 "refId": "", 2780 "step": 10, 2781 "target": "" 2782 } 2783 ], 2784 "thresholds": [], 2785 "timeFrom": null, 2786 "timeShift": null, 2787 "title": "Smoothed seat demands", 2788 "tooltip": { 2789 "msResolution": true, 2790 "shared": true, 2791 "sort": 2, 2792 "value_type": "cumulative" 2793 }, 2794 "transformations": [], 2795 "transparent": false, 2796 "type": "graph", 2797 "xaxis": { 2798 "mode": "time", 2799 "name": null, 2800 "show": true, 2801 "values": [] 2802 }, 2803 "yaxes": [ 2804 { 2805 "decimals": null, 2806 "format": "short", 2807 "label": null, 2808 "logBase": 1, 2809 "max": null, 2810 "min": null, 2811 "show": true 2812 }, 2813 { 2814 "decimals": null, 2815 "format": "short", 2816 "label": null, 2817 "logBase": 1, 2818 "max": null, 2819 "min": null, 2820 "show": true 2821 } 2822 ], 2823 "yaxis": { 2824 "align": false, 2825 "alignLevel": 0 2826 } 2827 }, 2828 { 2829 "aliasColors": {}, 2830 "bars": false, 2831 "cacheTimeout": null, 2832 "datasource": "$source", 2833 "description": null, 2834 "editable": true, 2835 "error": false, 2836 "fieldConfig": { 2837 "defaults": { 2838 "thresholds": { 2839 "mode": "absolute", 2840 "steps": [] 2841 } 2842 } 2843 }, 2844 "fill": 1, 2845 "grid": { 2846 "threshold1": null, 2847 "threshold1Color": "rgba(216, 200, 27, 0.27)", 2848 "threshold2": null, 2849 "threshold2Color": "rgba(234, 112, 112, 0.22)" 2850 }, 2851 "gridPos": null, 2852 "height": null, 2853 "hideTimeOverride": false, 2854 "id": 22, 2855 "interval": null, 2856 "isNew": true, 2857 "legend": { 2858 "alignAsTable": false, 2859 "avg": false, 2860 "current": false, 2861 "hideEmpty": false, 2862 "hideZero": false, 2863 "max": false, 2864 "min": false, 2865 "rightSide": false, 2866 "show": true, 2867 "sideWidth": null, 2868 "sort": null, 2869 "sortDesc": false, 2870 "total": false, 2871 "values": false 2872 }, 2873 "lines": true, 2874 "linewidth": 2, 2875 "links": [], 2876 "maxDataPoints": 100, 2877 "maxPerRow": null, 2878 "minSpan": null, 2879 "nullPointMode": null, 2880 "options": { 2881 "alertThreshold": true, 2882 "dataLinks": [] 2883 }, 2884 "percentage": false, 2885 "pointradius": 5, 2886 "points": false, 2887 "renderer": "flot", 2888 "repeat": null, 2889 "repeatDirection": null, 2890 "seriesOverrides": [], 2891 "span": 12, 2892 "stack": false, 2893 "steppedLine": false, 2894 "targets": [ 2895 { 2896 "datasource": "$source", 2897 "expr": "apiserver_flowcontrol_current_limit_seats{instance=~\".*(${instance:pipe})\"}", 2898 "format": "time_series", 2899 "hide": false, 2900 "instant": false, 2901 "interval": "5s", 2902 "intervalFactor": 1, 2903 "legendFormat": "{{instance}} {{priority_level}}", 2904 "metric": "", 2905 "query": "apiserver_flowcontrol_current_limit_seats{instance=~\".*(${instance:pipe})\"}", 2906 "refId": "", 2907 "step": 10, 2908 "target": "" 2909 } 2910 ], 2911 "thresholds": [], 2912 "timeFrom": null, 2913 "timeShift": null, 2914 "title": "Current seat limit for each Priority Level", 2915 "tooltip": { 2916 "msResolution": true, 2917 "shared": true, 2918 "sort": 2, 2919 "value_type": "cumulative" 2920 }, 2921 "transformations": [], 2922 "transparent": false, 2923 "type": "graph", 2924 "xaxis": { 2925 "mode": "time", 2926 "name": null, 2927 "show": true, 2928 "values": [] 2929 }, 2930 "yaxes": [ 2931 { 2932 "decimals": null, 2933 "format": "short", 2934 "label": null, 2935 "logBase": 1, 2936 "max": null, 2937 "min": null, 2938 "show": true 2939 }, 2940 { 2941 "decimals": null, 2942 "format": "short", 2943 "label": null, 2944 "logBase": 1, 2945 "max": null, 2946 "min": null, 2947 "show": true 2948 } 2949 ], 2950 "yaxis": { 2951 "align": false, 2952 "alignLevel": 0 2953 } 2954 } 2955 ], 2956 "repeat": null, 2957 "showTitle": true, 2958 "title": "P&F metrics" 2959 }, 2960 { 2961 "collapse": true, 2962 "editable": true, 2963 "height": "300px", 2964 "panels": [ 2965 { 2966 "aliasColors": {}, 2967 "bars": false, 2968 "cacheTimeout": null, 2969 "datasource": "$source", 2970 "description": null, 2971 "editable": true, 2972 "error": false, 2973 "fieldConfig": { 2974 "defaults": { 2975 "thresholds": { 2976 "mode": "absolute", 2977 "steps": [] 2978 } 2979 } 2980 }, 2981 "fill": 1, 2982 "grid": { 2983 "threshold1": null, 2984 "threshold1Color": "rgba(216, 200, 27, 0.27)", 2985 "threshold2": null, 2986 "threshold2Color": "rgba(234, 112, 112, 0.22)" 2987 }, 2988 "gridPos": null, 2989 "height": null, 2990 "hideTimeOverride": false, 2991 "id": 23, 2992 "interval": null, 2993 "isNew": true, 2994 "legend": { 2995 "alignAsTable": false, 2996 "avg": false, 2997 "current": false, 2998 "hideEmpty": false, 2999 "hideZero": false, 3000 "max": false, 3001 "min": false, 3002 "rightSide": false, 3003 "show": true, 3004 "sideWidth": null, 3005 "sort": null, 3006 "sortDesc": false, 3007 "total": false, 3008 "values": false 3009 }, 3010 "lines": true, 3011 "linewidth": 2, 3012 "links": [], 3013 "maxDataPoints": 100, 3014 "maxPerRow": null, 3015 "minSpan": null, 3016 "nullPointMode": null, 3017 "options": { 3018 "alertThreshold": true, 3019 "dataLinks": [] 3020 }, 3021 "percentage": false, 3022 "pointradius": 5, 3023 "points": false, 3024 "renderer": "flot", 3025 "repeat": null, 3026 "repeatDirection": null, 3027 "seriesOverrides": [], 3028 "span": 12, 3029 "stack": false, 3030 "steppedLine": false, 3031 "targets": [ 3032 { 3033 "datasource": "$source", 3034 "expr": "sum(node_collector_unhealthy_nodes_in_zone) by (zone)", 3035 "format": "time_series", 3036 "hide": false, 3037 "instant": false, 3038 "interval": "5s", 3039 "intervalFactor": 1, 3040 "legendFormat": "{{zone}}", 3041 "metric": "", 3042 "query": "sum(node_collector_unhealthy_nodes_in_zone) by (zone)", 3043 "refId": "", 3044 "step": 10, 3045 "target": "" 3046 } 3047 ], 3048 "thresholds": [], 3049 "timeFrom": null, 3050 "timeShift": null, 3051 "title": "Unhealthy nodes", 3052 "tooltip": { 3053 "msResolution": true, 3054 "shared": true, 3055 "sort": 2, 3056 "value_type": "cumulative" 3057 }, 3058 "transformations": [], 3059 "transparent": false, 3060 "type": "graph", 3061 "xaxis": { 3062 "mode": "time", 3063 "name": null, 3064 "show": true, 3065 "values": [] 3066 }, 3067 "yaxes": [ 3068 { 3069 "decimals": null, 3070 "format": "short", 3071 "label": null, 3072 "logBase": 1, 3073 "max": null, 3074 "min": null, 3075 "show": true 3076 }, 3077 { 3078 "decimals": null, 3079 "format": "short", 3080 "label": null, 3081 "logBase": 1, 3082 "max": null, 3083 "min": null, 3084 "show": true 3085 } 3086 ], 3087 "yaxis": { 3088 "align": false, 3089 "alignLevel": 0 3090 } 3091 }, 3092 { 3093 "aliasColors": {}, 3094 "bars": false, 3095 "cacheTimeout": null, 3096 "datasource": "$source", 3097 "description": null, 3098 "editable": true, 3099 "error": false, 3100 "fieldConfig": { 3101 "defaults": { 3102 "thresholds": { 3103 "mode": "absolute", 3104 "steps": [] 3105 } 3106 } 3107 }, 3108 "fill": 1, 3109 "grid": { 3110 "threshold1": null, 3111 "threshold1Color": "rgba(216, 200, 27, 0.27)", 3112 "threshold2": null, 3113 "threshold2Color": "rgba(234, 112, 112, 0.22)" 3114 }, 3115 "gridPos": null, 3116 "height": null, 3117 "hideTimeOverride": false, 3118 "id": 24, 3119 "interval": null, 3120 "isNew": true, 3121 "legend": { 3122 "alignAsTable": false, 3123 "avg": false, 3124 "current": false, 3125 "hideEmpty": false, 3126 "hideZero": false, 3127 "max": false, 3128 "min": false, 3129 "rightSide": false, 3130 "show": true, 3131 "sideWidth": null, 3132 "sort": null, 3133 "sortDesc": false, 3134 "total": false, 3135 "values": false 3136 }, 3137 "lines": true, 3138 "linewidth": 2, 3139 "links": [], 3140 "maxDataPoints": 100, 3141 "maxPerRow": null, 3142 "minSpan": null, 3143 "nullPointMode": null, 3144 "options": { 3145 "alertThreshold": true, 3146 "dataLinks": [] 3147 }, 3148 "percentage": false, 3149 "pointradius": 5, 3150 "points": false, 3151 "renderer": "flot", 3152 "repeat": null, 3153 "repeatDirection": null, 3154 "seriesOverrides": [], 3155 "span": 12, 3156 "stack": false, 3157 "steppedLine": false, 3158 "targets": [ 3159 { 3160 "datasource": "$source", 3161 "expr": "sum(irate(apiserver_request_total{verb=\"POST\", resource=\"pods\", subresource=\"\"}[1m]))", 3162 "format": "time_series", 3163 "hide": false, 3164 "instant": false, 3165 "interval": "5s", 3166 "intervalFactor": 1, 3167 "legendFormat": "", 3168 "metric": "", 3169 "query": "sum(irate(apiserver_request_total{verb=\"POST\", resource=\"pods\", subresource=\"\"}[1m]))", 3170 "refId": "", 3171 "step": 10, 3172 "target": "" 3173 } 3174 ], 3175 "thresholds": [], 3176 "timeFrom": null, 3177 "timeShift": null, 3178 "title": "Pod creations", 3179 "tooltip": { 3180 "msResolution": true, 3181 "shared": true, 3182 "sort": 2, 3183 "value_type": "cumulative" 3184 }, 3185 "transformations": [], 3186 "transparent": false, 3187 "type": "graph", 3188 "xaxis": { 3189 "mode": "time", 3190 "name": null, 3191 "show": true, 3192 "values": [] 3193 }, 3194 "yaxes": [ 3195 { 3196 "decimals": null, 3197 "format": "ops", 3198 "label": null, 3199 "logBase": 1, 3200 "max": null, 3201 "min": null, 3202 "show": true 3203 }, 3204 { 3205 "decimals": null, 3206 "format": "short", 3207 "label": null, 3208 "logBase": 1, 3209 "max": null, 3210 "min": null, 3211 "show": true 3212 } 3213 ], 3214 "yaxis": { 3215 "align": false, 3216 "alignLevel": 0 3217 } 3218 }, 3219 { 3220 "aliasColors": {}, 3221 "bars": false, 3222 "cacheTimeout": null, 3223 "datasource": "$source", 3224 "description": null, 3225 "editable": true, 3226 "error": false, 3227 "fieldConfig": { 3228 "defaults": { 3229 "thresholds": { 3230 "mode": "absolute", 3231 "steps": [] 3232 } 3233 } 3234 }, 3235 "fill": 1, 3236 "grid": { 3237 "threshold1": null, 3238 "threshold1Color": "rgba(216, 200, 27, 0.27)", 3239 "threshold2": null, 3240 "threshold2Color": "rgba(234, 112, 112, 0.22)" 3241 }, 3242 "gridPos": null, 3243 "height": null, 3244 "hideTimeOverride": false, 3245 "id": 25, 3246 "interval": null, 3247 "isNew": true, 3248 "legend": { 3249 "alignAsTable": false, 3250 "avg": false, 3251 "current": false, 3252 "hideEmpty": false, 3253 "hideZero": false, 3254 "max": false, 3255 "min": false, 3256 "rightSide": false, 3257 "show": true, 3258 "sideWidth": null, 3259 "sort": null, 3260 "sortDesc": false, 3261 "total": false, 3262 "values": false 3263 }, 3264 "lines": true, 3265 "linewidth": 2, 3266 "links": [], 3267 "maxDataPoints": 100, 3268 "maxPerRow": null, 3269 "minSpan": null, 3270 "nullPointMode": null, 3271 "options": { 3272 "alertThreshold": true, 3273 "dataLinks": [] 3274 }, 3275 "percentage": false, 3276 "pointradius": 5, 3277 "points": false, 3278 "renderer": "flot", 3279 "repeat": null, 3280 "repeatDirection": null, 3281 "seriesOverrides": [], 3282 "span": 12, 3283 "stack": false, 3284 "steppedLine": false, 3285 "targets": [ 3286 { 3287 "datasource": "$source", 3288 "expr": "sum(irate(apiserver_request_total{verb=\"POST\", resource=\"pods\", subresource=\"binding\"}[1m]))", 3289 "format": "time_series", 3290 "hide": false, 3291 "instant": false, 3292 "interval": "5s", 3293 "intervalFactor": 1, 3294 "legendFormat": "", 3295 "metric": "", 3296 "query": "sum(irate(apiserver_request_total{verb=\"POST\", resource=\"pods\", subresource=\"binding\"}[1m]))", 3297 "refId": "", 3298 "step": 10, 3299 "target": "" 3300 } 3301 ], 3302 "thresholds": [], 3303 "timeFrom": null, 3304 "timeShift": null, 3305 "title": "Pod bindings", 3306 "tooltip": { 3307 "msResolution": true, 3308 "shared": true, 3309 "sort": 2, 3310 "value_type": "cumulative" 3311 }, 3312 "transformations": [], 3313 "transparent": false, 3314 "type": "graph", 3315 "xaxis": { 3316 "mode": "time", 3317 "name": null, 3318 "show": true, 3319 "values": [] 3320 }, 3321 "yaxes": [ 3322 { 3323 "decimals": null, 3324 "format": "ops", 3325 "label": null, 3326 "logBase": 1, 3327 "max": null, 3328 "min": null, 3329 "show": true 3330 }, 3331 { 3332 "decimals": null, 3333 "format": "short", 3334 "label": null, 3335 "logBase": 1, 3336 "max": null, 3337 "min": null, 3338 "show": true 3339 } 3340 ], 3341 "yaxis": { 3342 "align": false, 3343 "alignLevel": 0 3344 } 3345 }, 3346 { 3347 "aliasColors": {}, 3348 "bars": false, 3349 "cacheTimeout": null, 3350 "datasource": "$source", 3351 "description": null, 3352 "editable": true, 3353 "error": false, 3354 "fieldConfig": { 3355 "defaults": { 3356 "thresholds": { 3357 "mode": "absolute", 3358 "steps": [] 3359 } 3360 } 3361 }, 3362 "fill": 1, 3363 "grid": { 3364 "threshold1": null, 3365 "threshold1Color": "rgba(216, 200, 27, 0.27)", 3366 "threshold2": null, 3367 "threshold2Color": "rgba(234, 112, 112, 0.22)" 3368 }, 3369 "gridPos": null, 3370 "height": null, 3371 "hideTimeOverride": false, 3372 "id": 26, 3373 "interval": null, 3374 "isNew": true, 3375 "legend": { 3376 "alignAsTable": false, 3377 "avg": false, 3378 "current": false, 3379 "hideEmpty": false, 3380 "hideZero": false, 3381 "max": false, 3382 "min": false, 3383 "rightSide": false, 3384 "show": true, 3385 "sideWidth": null, 3386 "sort": null, 3387 "sortDesc": false, 3388 "total": false, 3389 "values": false 3390 }, 3391 "lines": true, 3392 "linewidth": 2, 3393 "links": [], 3394 "maxDataPoints": 100, 3395 "maxPerRow": null, 3396 "minSpan": null, 3397 "nullPointMode": null, 3398 "options": { 3399 "alertThreshold": true, 3400 "dataLinks": [] 3401 }, 3402 "percentage": false, 3403 "pointradius": 5, 3404 "points": false, 3405 "renderer": "flot", 3406 "repeat": null, 3407 "repeatDirection": null, 3408 "seriesOverrides": [], 3409 "span": 12, 3410 "stack": false, 3411 "steppedLine": false, 3412 "targets": [ 3413 { 3414 "datasource": "$source", 3415 "expr": "sum(rate(process_start_time_seconds[1m]) > bool 0) by (job, endpoint)", 3416 "format": "time_series", 3417 "hide": false, 3418 "instant": false, 3419 "interval": "5s", 3420 "intervalFactor": 1, 3421 "legendFormat": "", 3422 "metric": "", 3423 "query": "sum(rate(process_start_time_seconds[1m]) > bool 0) by (job, endpoint)", 3424 "refId": "", 3425 "step": 10, 3426 "target": "" 3427 } 3428 ], 3429 "thresholds": [], 3430 "timeFrom": null, 3431 "timeShift": null, 3432 "title": "Component restarts", 3433 "tooltip": { 3434 "msResolution": true, 3435 "shared": true, 3436 "sort": 2, 3437 "value_type": "cumulative" 3438 }, 3439 "transformations": [], 3440 "transparent": false, 3441 "type": "graph", 3442 "xaxis": { 3443 "mode": "time", 3444 "name": null, 3445 "show": true, 3446 "values": [] 3447 }, 3448 "yaxes": [ 3449 { 3450 "decimals": null, 3451 "format": "short", 3452 "label": null, 3453 "logBase": 1, 3454 "max": null, 3455 "min": null, 3456 "show": true 3457 }, 3458 { 3459 "decimals": null, 3460 "format": "short", 3461 "label": null, 3462 "logBase": 1, 3463 "max": null, 3464 "min": null, 3465 "show": true 3466 } 3467 ], 3468 "yaxis": { 3469 "align": false, 3470 "alignLevel": 0 3471 } 3472 }, 3473 { 3474 "aliasColors": {}, 3475 "bars": false, 3476 "cacheTimeout": null, 3477 "datasource": "$source", 3478 "description": null, 3479 "editable": true, 3480 "error": false, 3481 "fieldConfig": { 3482 "defaults": { 3483 "thresholds": { 3484 "mode": "absolute", 3485 "steps": [] 3486 } 3487 } 3488 }, 3489 "fill": 1, 3490 "grid": { 3491 "threshold1": null, 3492 "threshold1Color": "rgba(216, 200, 27, 0.27)", 3493 "threshold2": null, 3494 "threshold2Color": "rgba(234, 112, 112, 0.22)" 3495 }, 3496 "gridPos": null, 3497 "height": null, 3498 "hideTimeOverride": false, 3499 "id": 27, 3500 "interval": null, 3501 "isNew": true, 3502 "legend": { 3503 "alignAsTable": false, 3504 "avg": false, 3505 "current": false, 3506 "hideEmpty": false, 3507 "hideZero": false, 3508 "max": false, 3509 "min": false, 3510 "rightSide": false, 3511 "show": true, 3512 "sideWidth": null, 3513 "sort": null, 3514 "sortDesc": false, 3515 "total": false, 3516 "values": false 3517 }, 3518 "lines": true, 3519 "linewidth": 2, 3520 "links": [], 3521 "maxDataPoints": 100, 3522 "maxPerRow": null, 3523 "minSpan": null, 3524 "nullPointMode": null, 3525 "options": { 3526 "alertThreshold": true, 3527 "dataLinks": [] 3528 }, 3529 "percentage": false, 3530 "pointradius": 5, 3531 "points": false, 3532 "renderer": "flot", 3533 "repeat": null, 3534 "repeatDirection": null, 3535 "seriesOverrides": [], 3536 "span": 12, 3537 "stack": false, 3538 "steppedLine": false, 3539 "targets": [ 3540 { 3541 "datasource": "$source", 3542 "expr": "sum(min_over_time(container_start_time_seconds{container!=\"\",container!=\"POD\"}[2m])) by (container)", 3543 "format": "time_series", 3544 "hide": false, 3545 "instant": false, 3546 "interval": "5s", 3547 "intervalFactor": 1, 3548 "legendFormat": "", 3549 "metric": "", 3550 "query": "sum(min_over_time(container_start_time_seconds{container!=\"\",container!=\"POD\"}[2m])) by (container)", 3551 "refId": "", 3552 "step": 10, 3553 "target": "" 3554 } 3555 ], 3556 "thresholds": [], 3557 "timeFrom": null, 3558 "timeShift": null, 3559 "title": "Component restarts 2", 3560 "tooltip": { 3561 "msResolution": true, 3562 "shared": true, 3563 "sort": 2, 3564 "value_type": "cumulative" 3565 }, 3566 "transformations": [], 3567 "transparent": false, 3568 "type": "graph", 3569 "xaxis": { 3570 "mode": "time", 3571 "name": null, 3572 "show": true, 3573 "values": [] 3574 }, 3575 "yaxes": [ 3576 { 3577 "decimals": null, 3578 "format": "short", 3579 "label": null, 3580 "logBase": 1, 3581 "max": null, 3582 "min": null, 3583 "show": true 3584 }, 3585 { 3586 "decimals": null, 3587 "format": "short", 3588 "label": null, 3589 "logBase": 1, 3590 "max": null, 3591 "min": null, 3592 "show": true 3593 } 3594 ], 3595 "yaxis": { 3596 "align": false, 3597 "alignLevel": 0 3598 } 3599 }, 3600 { 3601 "aliasColors": {}, 3602 "bars": false, 3603 "cacheTimeout": null, 3604 "datasource": "$source", 3605 "description": null, 3606 "editable": true, 3607 "error": false, 3608 "fieldConfig": { 3609 "defaults": { 3610 "thresholds": { 3611 "mode": "absolute", 3612 "steps": [] 3613 } 3614 } 3615 }, 3616 "fill": 1, 3617 "grid": { 3618 "threshold1": null, 3619 "threshold1Color": "rgba(216, 200, 27, 0.27)", 3620 "threshold2": null, 3621 "threshold2Color": "rgba(234, 112, 112, 0.22)" 3622 }, 3623 "gridPos": null, 3624 "height": null, 3625 "hideTimeOverride": false, 3626 "id": 28, 3627 "interval": null, 3628 "isNew": true, 3629 "legend": { 3630 "alignAsTable": false, 3631 "avg": false, 3632 "current": false, 3633 "hideEmpty": false, 3634 "hideZero": false, 3635 "max": false, 3636 "min": false, 3637 "rightSide": false, 3638 "show": true, 3639 "sideWidth": null, 3640 "sort": null, 3641 "sortDesc": false, 3642 "total": false, 3643 "values": false 3644 }, 3645 "lines": true, 3646 "linewidth": 2, 3647 "links": [], 3648 "maxDataPoints": 100, 3649 "maxPerRow": null, 3650 "minSpan": null, 3651 "nullPointMode": null, 3652 "options": { 3653 "alertThreshold": true, 3654 "dataLinks": [] 3655 }, 3656 "percentage": false, 3657 "pointradius": 5, 3658 "points": false, 3659 "renderer": "flot", 3660 "repeat": null, 3661 "repeatDirection": null, 3662 "seriesOverrides": [], 3663 "span": 12, 3664 "stack": false, 3665 "steppedLine": false, 3666 "targets": [ 3667 { 3668 "datasource": "$source", 3669 "expr": "sum(leader_election_master_status) by (name, instance)", 3670 "format": "time_series", 3671 "hide": false, 3672 "instant": false, 3673 "interval": "5s", 3674 "intervalFactor": 1, 3675 "legendFormat": "", 3676 "metric": "", 3677 "query": "sum(leader_election_master_status) by (name, instance)", 3678 "refId": "", 3679 "step": 10, 3680 "target": "" 3681 } 3682 ], 3683 "thresholds": [], 3684 "timeFrom": null, 3685 "timeShift": null, 3686 "title": "Active component", 3687 "tooltip": { 3688 "msResolution": true, 3689 "shared": true, 3690 "sort": 2, 3691 "value_type": "cumulative" 3692 }, 3693 "transformations": [], 3694 "transparent": false, 3695 "type": "graph", 3696 "xaxis": { 3697 "mode": "time", 3698 "name": null, 3699 "show": true, 3700 "values": [] 3701 }, 3702 "yaxes": [ 3703 { 3704 "decimals": null, 3705 "format": "short", 3706 "label": null, 3707 "logBase": 1, 3708 "max": null, 3709 "min": null, 3710 "show": true 3711 }, 3712 { 3713 "decimals": null, 3714 "format": "short", 3715 "label": null, 3716 "logBase": 1, 3717 "max": null, 3718 "min": null, 3719 "show": true 3720 } 3721 ], 3722 "yaxis": { 3723 "align": false, 3724 "alignLevel": 0 3725 } 3726 } 3727 ], 3728 "repeat": null, 3729 "showTitle": true, 3730 "title": "Overall cluster health" 3731 }, 3732 { 3733 "collapse": true, 3734 "editable": true, 3735 "height": "300px", 3736 "panels": [ 3737 { 3738 "aliasColors": {}, 3739 "bars": false, 3740 "cacheTimeout": null, 3741 "datasource": "$source", 3742 "description": null, 3743 "editable": true, 3744 "error": false, 3745 "fieldConfig": { 3746 "defaults": { 3747 "thresholds": { 3748 "mode": "absolute", 3749 "steps": [] 3750 } 3751 } 3752 }, 3753 "fill": 1, 3754 "grid": { 3755 "threshold1": null, 3756 "threshold1Color": "rgba(216, 200, 27, 0.27)", 3757 "threshold2": null, 3758 "threshold2Color": "rgba(234, 112, 112, 0.22)" 3759 }, 3760 "gridPos": null, 3761 "height": null, 3762 "hideTimeOverride": false, 3763 "id": 29, 3764 "interval": null, 3765 "isNew": true, 3766 "legend": { 3767 "alignAsTable": false, 3768 "avg": false, 3769 "current": false, 3770 "hideEmpty": false, 3771 "hideZero": false, 3772 "max": false, 3773 "min": false, 3774 "rightSide": false, 3775 "show": true, 3776 "sideWidth": null, 3777 "sort": null, 3778 "sortDesc": false, 3779 "total": false, 3780 "values": false 3781 }, 3782 "lines": true, 3783 "linewidth": 2, 3784 "links": [], 3785 "maxDataPoints": 100, 3786 "maxPerRow": null, 3787 "minSpan": null, 3788 "nullPointMode": null, 3789 "options": { 3790 "alertThreshold": true, 3791 "dataLinks": [] 3792 }, 3793 "percentage": false, 3794 "pointradius": 5, 3795 "points": false, 3796 "renderer": "flot", 3797 "repeat": null, 3798 "repeatDirection": null, 3799 "seriesOverrides": [], 3800 "span": 12, 3801 "stack": false, 3802 "steppedLine": false, 3803 "targets": [ 3804 { 3805 "datasource": "$source", 3806 "expr": "etcd_server_is_leader", 3807 "format": "time_series", 3808 "hide": false, 3809 "instant": false, 3810 "interval": "5s", 3811 "intervalFactor": 1, 3812 "legendFormat": "{{instance}}", 3813 "metric": "", 3814 "query": "etcd_server_is_leader", 3815 "refId": "", 3816 "step": 10, 3817 "target": "" 3818 } 3819 ], 3820 "thresholds": [], 3821 "timeFrom": null, 3822 "timeShift": null, 3823 "title": "etcd leader", 3824 "tooltip": { 3825 "msResolution": true, 3826 "shared": true, 3827 "sort": 2, 3828 "value_type": "cumulative" 3829 }, 3830 "transformations": [], 3831 "transparent": false, 3832 "type": "graph", 3833 "xaxis": { 3834 "mode": "time", 3835 "name": null, 3836 "show": true, 3837 "values": [] 3838 }, 3839 "yaxes": [ 3840 { 3841 "decimals": null, 3842 "format": "short", 3843 "label": null, 3844 "logBase": 1, 3845 "max": null, 3846 "min": null, 3847 "show": true 3848 }, 3849 { 3850 "decimals": null, 3851 "format": "short", 3852 "label": null, 3853 "logBase": 1, 3854 "max": null, 3855 "min": null, 3856 "show": true 3857 } 3858 ], 3859 "yaxis": { 3860 "align": false, 3861 "alignLevel": 0 3862 } 3863 }, 3864 { 3865 "aliasColors": {}, 3866 "bars": false, 3867 "cacheTimeout": null, 3868 "datasource": "$source", 3869 "description": null, 3870 "editable": true, 3871 "error": false, 3872 "fieldConfig": { 3873 "defaults": { 3874 "thresholds": { 3875 "mode": "absolute", 3876 "steps": [] 3877 } 3878 } 3879 }, 3880 "fill": 1, 3881 "grid": { 3882 "threshold1": null, 3883 "threshold1Color": "rgba(216, 200, 27, 0.27)", 3884 "threshold2": null, 3885 "threshold2Color": "rgba(234, 112, 112, 0.22)" 3886 }, 3887 "gridPos": null, 3888 "height": null, 3889 "hideTimeOverride": false, 3890 "id": 30, 3891 "interval": null, 3892 "isNew": true, 3893 "legend": { 3894 "alignAsTable": false, 3895 "avg": false, 3896 "current": false, 3897 "hideEmpty": false, 3898 "hideZero": false, 3899 "max": false, 3900 "min": false, 3901 "rightSide": false, 3902 "show": true, 3903 "sideWidth": null, 3904 "sort": null, 3905 "sortDesc": false, 3906 "total": false, 3907 "values": false 3908 }, 3909 "lines": true, 3910 "linewidth": 2, 3911 "links": [], 3912 "maxDataPoints": 100, 3913 "maxPerRow": null, 3914 "minSpan": null, 3915 "nullPointMode": null, 3916 "options": { 3917 "alertThreshold": true, 3918 "dataLinks": [] 3919 }, 3920 "percentage": false, 3921 "pointradius": 5, 3922 "points": false, 3923 "renderer": "flot", 3924 "repeat": null, 3925 "repeatDirection": null, 3926 "seriesOverrides": [], 3927 "span": 12, 3928 "stack": false, 3929 "steppedLine": false, 3930 "targets": [ 3931 { 3932 "datasource": "$source", 3933 "expr": "rate(etcd_network_client_grpc_sent_bytes_total[1m])", 3934 "format": "time_series", 3935 "hide": false, 3936 "instant": false, 3937 "interval": "5s", 3938 "intervalFactor": 1, 3939 "legendFormat": "{{instance}}", 3940 "metric": "", 3941 "query": "rate(etcd_network_client_grpc_sent_bytes_total[1m])", 3942 "refId": "", 3943 "step": 10, 3944 "target": "" 3945 } 3946 ], 3947 "thresholds": [], 3948 "timeFrom": null, 3949 "timeShift": null, 3950 "title": "etcd bytes sent", 3951 "tooltip": { 3952 "msResolution": true, 3953 "shared": true, 3954 "sort": 2, 3955 "value_type": "cumulative" 3956 }, 3957 "transformations": [], 3958 "transparent": false, 3959 "type": "graph", 3960 "xaxis": { 3961 "mode": "time", 3962 "name": null, 3963 "show": true, 3964 "values": [] 3965 }, 3966 "yaxes": [ 3967 { 3968 "decimals": null, 3969 "format": "Bps", 3970 "label": null, 3971 "logBase": 1, 3972 "max": null, 3973 "min": null, 3974 "show": true 3975 }, 3976 { 3977 "decimals": null, 3978 "format": "short", 3979 "label": null, 3980 "logBase": 1, 3981 "max": null, 3982 "min": null, 3983 "show": true 3984 } 3985 ], 3986 "yaxis": { 3987 "align": false, 3988 "alignLevel": 0 3989 } 3990 }, 3991 { 3992 "aliasColors": {}, 3993 "bars": false, 3994 "cacheTimeout": null, 3995 "datasource": "$source", 3996 "description": null, 3997 "editable": true, 3998 "error": false, 3999 "fieldConfig": { 4000 "defaults": { 4001 "thresholds": { 4002 "mode": "absolute", 4003 "steps": [] 4004 } 4005 } 4006 }, 4007 "fill": 1, 4008 "grid": { 4009 "threshold1": null, 4010 "threshold1Color": "rgba(216, 200, 27, 0.27)", 4011 "threshold2": null, 4012 "threshold2Color": "rgba(234, 112, 112, 0.22)" 4013 }, 4014 "gridPos": null, 4015 "height": null, 4016 "hideTimeOverride": false, 4017 "id": 31, 4018 "interval": null, 4019 "isNew": true, 4020 "legend": { 4021 "alignAsTable": false, 4022 "avg": false, 4023 "current": false, 4024 "hideEmpty": false, 4025 "hideZero": false, 4026 "max": false, 4027 "min": false, 4028 "rightSide": false, 4029 "show": true, 4030 "sideWidth": null, 4031 "sort": null, 4032 "sortDesc": false, 4033 "total": false, 4034 "values": false 4035 }, 4036 "lines": true, 4037 "linewidth": 2, 4038 "links": [], 4039 "maxDataPoints": 100, 4040 "maxPerRow": null, 4041 "minSpan": null, 4042 "nullPointMode": null, 4043 "options": { 4044 "alertThreshold": true, 4045 "dataLinks": [] 4046 }, 4047 "percentage": false, 4048 "pointradius": 5, 4049 "points": false, 4050 "renderer": "flot", 4051 "repeat": null, 4052 "repeatDirection": null, 4053 "seriesOverrides": [], 4054 "span": 12, 4055 "stack": false, 4056 "steppedLine": false, 4057 "targets": [ 4058 { 4059 "datasource": "$source", 4060 "expr": "sum(rate(etcd_request_duration_seconds_count{operation=~\"${etcd_operation:regex}\", type=~\".*(${etcd_type:pipe})\"}[1m])) by (operation, type)", 4061 "format": "time_series", 4062 "hide": false, 4063 "instant": false, 4064 "interval": "5s", 4065 "intervalFactor": 1, 4066 "legendFormat": "{{operation}} {{type}}", 4067 "metric": "", 4068 "query": "sum(rate(etcd_request_duration_seconds_count{operation=~\"${etcd_operation:regex}\", type=~\".*(${etcd_type:pipe})\"}[1m])) by (operation, type)", 4069 "refId": "", 4070 "step": 10, 4071 "target": "" 4072 } 4073 ], 4074 "thresholds": [], 4075 "timeFrom": null, 4076 "timeShift": null, 4077 "title": "etcd operations rate", 4078 "tooltip": { 4079 "msResolution": true, 4080 "shared": true, 4081 "sort": 2, 4082 "value_type": "cumulative" 4083 }, 4084 "transformations": [], 4085 "transparent": false, 4086 "type": "graph", 4087 "xaxis": { 4088 "mode": "time", 4089 "name": null, 4090 "show": true, 4091 "values": [] 4092 }, 4093 "yaxes": [ 4094 { 4095 "decimals": null, 4096 "format": "ops", 4097 "label": null, 4098 "logBase": 1, 4099 "max": null, 4100 "min": null, 4101 "show": true 4102 }, 4103 { 4104 "decimals": null, 4105 "format": "short", 4106 "label": null, 4107 "logBase": 1, 4108 "max": null, 4109 "min": null, 4110 "show": true 4111 } 4112 ], 4113 "yaxis": { 4114 "align": false, 4115 "alignLevel": 0 4116 } 4117 }, 4118 { 4119 "aliasColors": {}, 4120 "bars": false, 4121 "cacheTimeout": null, 4122 "datasource": "$source", 4123 "description": null, 4124 "editable": true, 4125 "error": false, 4126 "fieldConfig": { 4127 "defaults": { 4128 "thresholds": { 4129 "mode": "absolute", 4130 "steps": [] 4131 } 4132 } 4133 }, 4134 "fill": 1, 4135 "grid": { 4136 "threshold1": null, 4137 "threshold1Color": "rgba(216, 200, 27, 0.27)", 4138 "threshold2": null, 4139 "threshold2Color": "rgba(234, 112, 112, 0.22)" 4140 }, 4141 "gridPos": null, 4142 "height": null, 4143 "hideTimeOverride": false, 4144 "id": 32, 4145 "interval": null, 4146 "isNew": true, 4147 "legend": { 4148 "alignAsTable": false, 4149 "avg": false, 4150 "current": false, 4151 "hideEmpty": false, 4152 "hideZero": false, 4153 "max": false, 4154 "min": false, 4155 "rightSide": false, 4156 "show": true, 4157 "sideWidth": null, 4158 "sort": null, 4159 "sortDesc": false, 4160 "total": false, 4161 "values": false 4162 }, 4163 "lines": true, 4164 "linewidth": 2, 4165 "links": [], 4166 "maxDataPoints": 100, 4167 "maxPerRow": null, 4168 "minSpan": null, 4169 "nullPointMode": null, 4170 "options": { 4171 "alertThreshold": true, 4172 "dataLinks": [] 4173 }, 4174 "percentage": false, 4175 "pointradius": 5, 4176 "points": false, 4177 "renderer": "flot", 4178 "repeat": null, 4179 "repeatDirection": null, 4180 "seriesOverrides": [], 4181 "span": 12, 4182 "stack": false, 4183 "steppedLine": false, 4184 "targets": [ 4185 { 4186 "datasource": "$source", 4187 "expr": "histogram_quantile(0.99, sum(rate(etcd_request_duration_seconds_bucket{operation=~\"${etcd_operation:regex}\", type=~\".*(${etcd_type:pipe})\"}[1m])) by (le, operation, type, instance))", 4188 "format": "time_series", 4189 "hide": false, 4190 "instant": false, 4191 "interval": "5s", 4192 "intervalFactor": 1, 4193 "legendFormat": "{{operation}} {{type}} on {{instance}}", 4194 "metric": "", 4195 "query": "histogram_quantile(0.99, sum(rate(etcd_request_duration_seconds_bucket{operation=~\"${etcd_operation:regex}\", type=~\".*(${etcd_type:pipe})\"}[1m])) by (le, operation, type, instance))", 4196 "refId": "", 4197 "step": 10, 4198 "target": "" 4199 } 4200 ], 4201 "thresholds": [], 4202 "timeFrom": null, 4203 "timeShift": null, 4204 "title": "etcd get latency by type (99th percentile)", 4205 "tooltip": { 4206 "msResolution": true, 4207 "shared": true, 4208 "sort": 2, 4209 "value_type": "cumulative" 4210 }, 4211 "transformations": [], 4212 "transparent": false, 4213 "type": "graph", 4214 "xaxis": { 4215 "mode": "time", 4216 "name": null, 4217 "show": true, 4218 "values": [] 4219 }, 4220 "yaxes": [ 4221 { 4222 "decimals": null, 4223 "format": "s", 4224 "label": null, 4225 "logBase": 1, 4226 "max": null, 4227 "min": null, 4228 "show": true 4229 }, 4230 { 4231 "decimals": null, 4232 "format": "short", 4233 "label": null, 4234 "logBase": 1, 4235 "max": null, 4236 "min": null, 4237 "show": true 4238 } 4239 ], 4240 "yaxis": { 4241 "align": false, 4242 "alignLevel": 0 4243 } 4244 }, 4245 { 4246 "aliasColors": {}, 4247 "bars": false, 4248 "cacheTimeout": null, 4249 "datasource": "$source", 4250 "description": null, 4251 "editable": true, 4252 "error": false, 4253 "fieldConfig": { 4254 "defaults": { 4255 "thresholds": { 4256 "mode": "absolute", 4257 "steps": [] 4258 } 4259 } 4260 }, 4261 "fill": 1, 4262 "grid": { 4263 "threshold1": null, 4264 "threshold1Color": "rgba(216, 200, 27, 0.27)", 4265 "threshold2": null, 4266 "threshold2Color": "rgba(234, 112, 112, 0.22)" 4267 }, 4268 "gridPos": null, 4269 "height": null, 4270 "hideTimeOverride": false, 4271 "id": 33, 4272 "interval": null, 4273 "isNew": true, 4274 "legend": { 4275 "alignAsTable": false, 4276 "avg": false, 4277 "current": false, 4278 "hideEmpty": false, 4279 "hideZero": false, 4280 "max": false, 4281 "min": false, 4282 "rightSide": false, 4283 "show": true, 4284 "sideWidth": null, 4285 "sort": null, 4286 "sortDesc": false, 4287 "total": false, 4288 "values": false 4289 }, 4290 "lines": true, 4291 "linewidth": 2, 4292 "links": [], 4293 "maxDataPoints": 100, 4294 "maxPerRow": null, 4295 "minSpan": null, 4296 "nullPointMode": null, 4297 "options": { 4298 "alertThreshold": true, 4299 "dataLinks": [] 4300 }, 4301 "percentage": false, 4302 "pointradius": 5, 4303 "points": false, 4304 "renderer": "flot", 4305 "repeat": null, 4306 "repeatDirection": null, 4307 "seriesOverrides": [], 4308 "span": 12, 4309 "stack": false, 4310 "steppedLine": false, 4311 "targets": [ 4312 { 4313 "datasource": "$source", 4314 "expr": "histogram_quantile(0.50, sum(rate(etcd_request_duration_seconds_bucket{operation=~\"${etcd_operation:regex}\", type=~\".*(${etcd_type:pipe})\"}[1m])) by (le, operation, type, instance))", 4315 "format": "time_series", 4316 "hide": false, 4317 "instant": false, 4318 "interval": "5s", 4319 "intervalFactor": 1, 4320 "legendFormat": "", 4321 "metric": "", 4322 "query": "histogram_quantile(0.50, sum(rate(etcd_request_duration_seconds_bucket{operation=~\"${etcd_operation:regex}\", type=~\".*(${etcd_type:pipe})\"}[1m])) by (le, operation, type, instance))", 4323 "refId": "", 4324 "step": 10, 4325 "target": "" 4326 } 4327 ], 4328 "thresholds": [], 4329 "timeFrom": null, 4330 "timeShift": null, 4331 "title": "etcd get latency by type (50th percentile)", 4332 "tooltip": { 4333 "msResolution": true, 4334 "shared": true, 4335 "sort": 2, 4336 "value_type": "cumulative" 4337 }, 4338 "transformations": [], 4339 "transparent": false, 4340 "type": "graph", 4341 "xaxis": { 4342 "mode": "time", 4343 "name": null, 4344 "show": true, 4345 "values": [] 4346 }, 4347 "yaxes": [ 4348 { 4349 "decimals": null, 4350 "format": "s", 4351 "label": null, 4352 "logBase": 1, 4353 "max": null, 4354 "min": null, 4355 "show": true 4356 }, 4357 { 4358 "decimals": null, 4359 "format": "short", 4360 "label": null, 4361 "logBase": 1, 4362 "max": null, 4363 "min": null, 4364 "show": true 4365 } 4366 ], 4367 "yaxis": { 4368 "align": false, 4369 "alignLevel": 0 4370 } 4371 }, 4372 { 4373 "aliasColors": {}, 4374 "bars": false, 4375 "cacheTimeout": null, 4376 "datasource": "$source", 4377 "description": null, 4378 "editable": true, 4379 "error": false, 4380 "fieldConfig": { 4381 "defaults": { 4382 "thresholds": { 4383 "mode": "absolute", 4384 "steps": [] 4385 } 4386 } 4387 }, 4388 "fill": 1, 4389 "grid": { 4390 "threshold1": null, 4391 "threshold1Color": "rgba(216, 200, 27, 0.27)", 4392 "threshold2": null, 4393 "threshold2Color": "rgba(234, 112, 112, 0.22)" 4394 }, 4395 "gridPos": null, 4396 "height": null, 4397 "hideTimeOverride": false, 4398 "id": 34, 4399 "interval": null, 4400 "isNew": true, 4401 "legend": { 4402 "alignAsTable": false, 4403 "avg": false, 4404 "current": false, 4405 "hideEmpty": false, 4406 "hideZero": false, 4407 "max": false, 4408 "min": false, 4409 "rightSide": false, 4410 "show": true, 4411 "sideWidth": null, 4412 "sort": null, 4413 "sortDesc": false, 4414 "total": false, 4415 "values": false 4416 }, 4417 "lines": true, 4418 "linewidth": 2, 4419 "links": [], 4420 "maxDataPoints": 100, 4421 "maxPerRow": null, 4422 "minSpan": null, 4423 "nullPointMode": null, 4424 "options": { 4425 "alertThreshold": true, 4426 "dataLinks": [] 4427 }, 4428 "percentage": false, 4429 "pointradius": 5, 4430 "points": false, 4431 "renderer": "flot", 4432 "repeat": null, 4433 "repeatDirection": null, 4434 "seriesOverrides": [], 4435 "span": 12, 4436 "stack": false, 4437 "steppedLine": false, 4438 "targets": [ 4439 { 4440 "datasource": "$source", 4441 "expr": "sum(etcd_server_id) by (instance, server_id)", 4442 "format": "time_series", 4443 "hide": false, 4444 "instant": false, 4445 "interval": "5s", 4446 "intervalFactor": 1, 4447 "legendFormat": "", 4448 "metric": "", 4449 "query": "sum(etcd_server_id) by (instance, server_id)", 4450 "refId": "", 4451 "step": 10, 4452 "target": "" 4453 } 4454 ], 4455 "thresholds": [], 4456 "timeFrom": null, 4457 "timeShift": null, 4458 "title": "etcd instance id", 4459 "tooltip": { 4460 "msResolution": true, 4461 "shared": true, 4462 "sort": 2, 4463 "value_type": "cumulative" 4464 }, 4465 "transformations": [], 4466 "transparent": false, 4467 "type": "graph", 4468 "xaxis": { 4469 "mode": "time", 4470 "name": null, 4471 "show": true, 4472 "values": [] 4473 }, 4474 "yaxes": [ 4475 { 4476 "decimals": null, 4477 "format": "short", 4478 "label": null, 4479 "logBase": 1, 4480 "max": null, 4481 "min": null, 4482 "show": true 4483 }, 4484 { 4485 "decimals": null, 4486 "format": "short", 4487 "label": null, 4488 "logBase": 1, 4489 "max": null, 4490 "min": null, 4491 "show": true 4492 } 4493 ], 4494 "yaxis": { 4495 "align": false, 4496 "alignLevel": 0 4497 } 4498 }, 4499 { 4500 "aliasColors": {}, 4501 "bars": false, 4502 "cacheTimeout": null, 4503 "datasource": "$source", 4504 "description": null, 4505 "editable": true, 4506 "error": false, 4507 "fieldConfig": { 4508 "defaults": { 4509 "thresholds": { 4510 "mode": "absolute", 4511 "steps": [] 4512 } 4513 } 4514 }, 4515 "fill": 1, 4516 "grid": { 4517 "threshold1": null, 4518 "threshold1Color": "rgba(216, 200, 27, 0.27)", 4519 "threshold2": null, 4520 "threshold2Color": "rgba(234, 112, 112, 0.22)" 4521 }, 4522 "gridPos": null, 4523 "height": null, 4524 "hideTimeOverride": false, 4525 "id": 35, 4526 "interval": null, 4527 "isNew": true, 4528 "legend": { 4529 "alignAsTable": false, 4530 "avg": false, 4531 "current": false, 4532 "hideEmpty": false, 4533 "hideZero": false, 4534 "max": false, 4535 "min": false, 4536 "rightSide": false, 4537 "show": true, 4538 "sideWidth": null, 4539 "sort": null, 4540 "sortDesc": false, 4541 "total": false, 4542 "values": false 4543 }, 4544 "lines": true, 4545 "linewidth": 2, 4546 "links": [], 4547 "maxDataPoints": 100, 4548 "maxPerRow": null, 4549 "minSpan": null, 4550 "nullPointMode": null, 4551 "options": { 4552 "alertThreshold": true, 4553 "dataLinks": [] 4554 }, 4555 "percentage": false, 4556 "pointradius": 5, 4557 "points": false, 4558 "renderer": "flot", 4559 "repeat": null, 4560 "repeatDirection": null, 4561 "seriesOverrides": [], 4562 "span": 12, 4563 "stack": false, 4564 "steppedLine": false, 4565 "targets": [ 4566 { 4567 "datasource": "$source", 4568 "expr": "histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket[1m])) by (le, instance, To))", 4569 "format": "time_series", 4570 "hide": false, 4571 "instant": false, 4572 "interval": "5s", 4573 "intervalFactor": 1, 4574 "legendFormat": "", 4575 "metric": "", 4576 "query": "histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket[1m])) by (le, instance, To))", 4577 "refId": "", 4578 "step": 10, 4579 "target": "" 4580 } 4581 ], 4582 "thresholds": [], 4583 "timeFrom": null, 4584 "timeShift": null, 4585 "title": "etcd network latency (99th percentile)", 4586 "tooltip": { 4587 "msResolution": true, 4588 "shared": true, 4589 "sort": 2, 4590 "value_type": "cumulative" 4591 }, 4592 "transformations": [], 4593 "transparent": false, 4594 "type": "graph", 4595 "xaxis": { 4596 "mode": "time", 4597 "name": null, 4598 "show": true, 4599 "values": [] 4600 }, 4601 "yaxes": [ 4602 { 4603 "decimals": null, 4604 "format": "s", 4605 "label": null, 4606 "logBase": 1, 4607 "max": null, 4608 "min": null, 4609 "show": true 4610 }, 4611 { 4612 "decimals": null, 4613 "format": "short", 4614 "label": null, 4615 "logBase": 1, 4616 "max": null, 4617 "min": null, 4618 "show": true 4619 } 4620 ], 4621 "yaxis": { 4622 "align": false, 4623 "alignLevel": 0 4624 } 4625 }, 4626 { 4627 "aliasColors": {}, 4628 "bars": false, 4629 "cacheTimeout": null, 4630 "datasource": "$source", 4631 "description": null, 4632 "editable": true, 4633 "error": false, 4634 "fieldConfig": { 4635 "defaults": { 4636 "thresholds": { 4637 "mode": "absolute", 4638 "steps": [] 4639 } 4640 } 4641 }, 4642 "fill": 1, 4643 "grid": { 4644 "threshold1": null, 4645 "threshold1Color": "rgba(216, 200, 27, 0.27)", 4646 "threshold2": null, 4647 "threshold2Color": "rgba(234, 112, 112, 0.22)" 4648 }, 4649 "gridPos": null, 4650 "height": null, 4651 "hideTimeOverride": false, 4652 "id": 36, 4653 "interval": null, 4654 "isNew": true, 4655 "legend": { 4656 "alignAsTable": false, 4657 "avg": false, 4658 "current": false, 4659 "hideEmpty": false, 4660 "hideZero": false, 4661 "max": false, 4662 "min": false, 4663 "rightSide": false, 4664 "show": true, 4665 "sideWidth": null, 4666 "sort": null, 4667 "sortDesc": false, 4668 "total": false, 4669 "values": false 4670 }, 4671 "lines": true, 4672 "linewidth": 2, 4673 "links": [], 4674 "maxDataPoints": 100, 4675 "maxPerRow": null, 4676 "minSpan": null, 4677 "nullPointMode": null, 4678 "options": { 4679 "alertThreshold": true, 4680 "dataLinks": [] 4681 }, 4682 "percentage": false, 4683 "pointradius": 5, 4684 "points": false, 4685 "renderer": "flot", 4686 "repeat": null, 4687 "repeatDirection": null, 4688 "seriesOverrides": [], 4689 "span": 12, 4690 "stack": false, 4691 "steppedLine": false, 4692 "targets": [ 4693 { 4694 "datasource": "$source", 4695 "expr": "delta(etcd_debugging_mvcc_db_compaction_keys_total[1m])", 4696 "format": "time_series", 4697 "hide": false, 4698 "instant": false, 4699 "interval": "5s", 4700 "intervalFactor": 1, 4701 "legendFormat": "", 4702 "metric": "", 4703 "query": "delta(etcd_debugging_mvcc_db_compaction_keys_total[1m])", 4704 "refId": "", 4705 "step": 10, 4706 "target": "" 4707 } 4708 ], 4709 "thresholds": [], 4710 "timeFrom": null, 4711 "timeShift": null, 4712 "title": "etcd compaction keys", 4713 "tooltip": { 4714 "msResolution": true, 4715 "shared": true, 4716 "sort": 2, 4717 "value_type": "cumulative" 4718 }, 4719 "transformations": [], 4720 "transparent": false, 4721 "type": "graph", 4722 "xaxis": { 4723 "mode": "time", 4724 "name": null, 4725 "show": true, 4726 "values": [] 4727 }, 4728 "yaxes": [ 4729 { 4730 "decimals": null, 4731 "format": "short", 4732 "label": null, 4733 "logBase": 1, 4734 "max": null, 4735 "min": null, 4736 "show": true 4737 }, 4738 { 4739 "decimals": null, 4740 "format": "short", 4741 "label": null, 4742 "logBase": 1, 4743 "max": null, 4744 "min": null, 4745 "show": true 4746 } 4747 ], 4748 "yaxis": { 4749 "align": false, 4750 "alignLevel": 0 4751 } 4752 }, 4753 { 4754 "aliasColors": {}, 4755 "bars": false, 4756 "cacheTimeout": null, 4757 "datasource": "$source", 4758 "description": null, 4759 "editable": true, 4760 "error": false, 4761 "fieldConfig": { 4762 "defaults": { 4763 "thresholds": { 4764 "mode": "absolute", 4765 "steps": [] 4766 } 4767 } 4768 }, 4769 "fill": 1, 4770 "grid": { 4771 "threshold1": null, 4772 "threshold1Color": "rgba(216, 200, 27, 0.27)", 4773 "threshold2": null, 4774 "threshold2Color": "rgba(234, 112, 112, 0.22)" 4775 }, 4776 "gridPos": null, 4777 "height": null, 4778 "hideTimeOverride": false, 4779 "id": 37, 4780 "interval": null, 4781 "isNew": true, 4782 "legend": { 4783 "alignAsTable": false, 4784 "avg": false, 4785 "current": false, 4786 "hideEmpty": false, 4787 "hideZero": false, 4788 "max": false, 4789 "min": false, 4790 "rightSide": false, 4791 "show": true, 4792 "sideWidth": null, 4793 "sort": null, 4794 "sortDesc": false, 4795 "total": false, 4796 "values": false 4797 }, 4798 "lines": true, 4799 "linewidth": 2, 4800 "links": [], 4801 "maxDataPoints": 100, 4802 "maxPerRow": null, 4803 "minSpan": null, 4804 "nullPointMode": null, 4805 "options": { 4806 "alertThreshold": true, 4807 "dataLinks": [] 4808 }, 4809 "percentage": false, 4810 "pointradius": 5, 4811 "points": false, 4812 "renderer": "flot", 4813 "repeat": null, 4814 "repeatDirection": null, 4815 "seriesOverrides": [], 4816 "span": 12, 4817 "stack": false, 4818 "steppedLine": false, 4819 "targets": [ 4820 { 4821 "datasource": "$source", 4822 "expr": "delta(etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds_sum[1m])", 4823 "format": "time_series", 4824 "hide": false, 4825 "instant": false, 4826 "interval": "5s", 4827 "intervalFactor": 1, 4828 "legendFormat": "", 4829 "metric": "", 4830 "query": "delta(etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds_sum[1m])", 4831 "refId": "", 4832 "step": 10, 4833 "target": "" 4834 } 4835 ], 4836 "thresholds": [], 4837 "timeFrom": null, 4838 "timeShift": null, 4839 "title": "etcd compaction pause sum duration", 4840 "tooltip": { 4841 "msResolution": true, 4842 "shared": true, 4843 "sort": 2, 4844 "value_type": "cumulative" 4845 }, 4846 "transformations": [], 4847 "transparent": false, 4848 "type": "graph", 4849 "xaxis": { 4850 "mode": "time", 4851 "name": null, 4852 "show": true, 4853 "values": [] 4854 }, 4855 "yaxes": [ 4856 { 4857 "decimals": null, 4858 "format": "ms", 4859 "label": null, 4860 "logBase": 1, 4861 "max": null, 4862 "min": null, 4863 "show": true 4864 }, 4865 { 4866 "decimals": null, 4867 "format": "short", 4868 "label": null, 4869 "logBase": 1, 4870 "max": null, 4871 "min": null, 4872 "show": true 4873 } 4874 ], 4875 "yaxis": { 4876 "align": false, 4877 "alignLevel": 0 4878 } 4879 }, 4880 { 4881 "aliasColors": {}, 4882 "bars": false, 4883 "cacheTimeout": null, 4884 "datasource": "$source", 4885 "description": null, 4886 "editable": true, 4887 "error": false, 4888 "fieldConfig": { 4889 "defaults": { 4890 "thresholds": { 4891 "mode": "absolute", 4892 "steps": [] 4893 } 4894 } 4895 }, 4896 "fill": 1, 4897 "grid": { 4898 "threshold1": null, 4899 "threshold1Color": "rgba(216, 200, 27, 0.27)", 4900 "threshold2": null, 4901 "threshold2Color": "rgba(234, 112, 112, 0.22)" 4902 }, 4903 "gridPos": null, 4904 "height": null, 4905 "hideTimeOverride": false, 4906 "id": 38, 4907 "interval": null, 4908 "isNew": true, 4909 "legend": { 4910 "alignAsTable": false, 4911 "avg": false, 4912 "current": false, 4913 "hideEmpty": false, 4914 "hideZero": false, 4915 "max": false, 4916 "min": false, 4917 "rightSide": false, 4918 "show": true, 4919 "sideWidth": null, 4920 "sort": null, 4921 "sortDesc": false, 4922 "total": false, 4923 "values": false 4924 }, 4925 "lines": true, 4926 "linewidth": 2, 4927 "links": [], 4928 "maxDataPoints": 100, 4929 "maxPerRow": null, 4930 "minSpan": null, 4931 "nullPointMode": null, 4932 "options": { 4933 "alertThreshold": true, 4934 "dataLinks": [] 4935 }, 4936 "percentage": false, 4937 "pointradius": 5, 4938 "points": false, 4939 "renderer": "flot", 4940 "repeat": null, 4941 "repeatDirection": null, 4942 "seriesOverrides": [], 4943 "span": 12, 4944 "stack": false, 4945 "steppedLine": false, 4946 "targets": [ 4947 { 4948 "datasource": "$source", 4949 "expr": "delta(etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds_count[1m])", 4950 "format": "time_series", 4951 "hide": false, 4952 "instant": false, 4953 "interval": "5s", 4954 "intervalFactor": 1, 4955 "legendFormat": "", 4956 "metric": "", 4957 "query": "delta(etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds_count[1m])", 4958 "refId": "", 4959 "step": 10, 4960 "target": "" 4961 } 4962 ], 4963 "thresholds": [], 4964 "timeFrom": null, 4965 "timeShift": null, 4966 "title": "etcd compaction pause num chunks", 4967 "tooltip": { 4968 "msResolution": true, 4969 "shared": true, 4970 "sort": 2, 4971 "value_type": "cumulative" 4972 }, 4973 "transformations": [], 4974 "transparent": false, 4975 "type": "graph", 4976 "xaxis": { 4977 "mode": "time", 4978 "name": null, 4979 "show": true, 4980 "values": [] 4981 }, 4982 "yaxes": [ 4983 { 4984 "decimals": null, 4985 "format": "short", 4986 "label": null, 4987 "logBase": 1, 4988 "max": null, 4989 "min": null, 4990 "show": true 4991 }, 4992 { 4993 "decimals": null, 4994 "format": "short", 4995 "label": null, 4996 "logBase": 1, 4997 "max": null, 4998 "min": null, 4999 "show": true 5000 } 5001 ], 5002 "yaxis": { 5003 "align": false, 5004 "alignLevel": 0 5005 } 5006 }, 5007 { 5008 "aliasColors": {}, 5009 "bars": false, 5010 "cacheTimeout": null, 5011 "datasource": "$source", 5012 "description": null, 5013 "editable": true, 5014 "error": false, 5015 "fieldConfig": { 5016 "defaults": { 5017 "thresholds": { 5018 "mode": "absolute", 5019 "steps": [] 5020 } 5021 } 5022 }, 5023 "fill": 1, 5024 "grid": { 5025 "threshold1": null, 5026 "threshold1Color": "rgba(216, 200, 27, 0.27)", 5027 "threshold2": null, 5028 "threshold2Color": "rgba(234, 112, 112, 0.22)" 5029 }, 5030 "gridPos": null, 5031 "height": null, 5032 "hideTimeOverride": false, 5033 "id": 39, 5034 "interval": null, 5035 "isNew": true, 5036 "legend": { 5037 "alignAsTable": false, 5038 "avg": false, 5039 "current": false, 5040 "hideEmpty": false, 5041 "hideZero": false, 5042 "max": false, 5043 "min": false, 5044 "rightSide": false, 5045 "show": true, 5046 "sideWidth": null, 5047 "sort": null, 5048 "sortDesc": false, 5049 "total": false, 5050 "values": false 5051 }, 5052 "lines": true, 5053 "linewidth": 2, 5054 "links": [], 5055 "maxDataPoints": 100, 5056 "maxPerRow": null, 5057 "minSpan": null, 5058 "nullPointMode": null, 5059 "options": { 5060 "alertThreshold": true, 5061 "dataLinks": [] 5062 }, 5063 "percentage": false, 5064 "pointradius": 5, 5065 "points": false, 5066 "renderer": "flot", 5067 "repeat": null, 5068 "repeatDirection": null, 5069 "seriesOverrides": [], 5070 "span": 12, 5071 "stack": false, 5072 "steppedLine": false, 5073 "targets": [ 5074 { 5075 "datasource": "$source", 5076 "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[1m])) by (le, instance))", 5077 "format": "time_series", 5078 "hide": false, 5079 "instant": false, 5080 "interval": "5s", 5081 "intervalFactor": 1, 5082 "legendFormat": "", 5083 "metric": "", 5084 "query": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[1m])) by (le, instance))", 5085 "refId": "", 5086 "step": 10, 5087 "target": "" 5088 } 5089 ], 5090 "thresholds": [], 5091 "timeFrom": null, 5092 "timeShift": null, 5093 "title": "etcd_disk_backend_commit_duration_seconds", 5094 "tooltip": { 5095 "msResolution": true, 5096 "shared": true, 5097 "sort": 2, 5098 "value_type": "cumulative" 5099 }, 5100 "transformations": [], 5101 "transparent": false, 5102 "type": "graph", 5103 "xaxis": { 5104 "mode": "time", 5105 "name": null, 5106 "show": true, 5107 "values": [] 5108 }, 5109 "yaxes": [ 5110 { 5111 "decimals": null, 5112 "format": "s", 5113 "label": null, 5114 "logBase": 1, 5115 "max": null, 5116 "min": null, 5117 "show": true 5118 }, 5119 { 5120 "decimals": null, 5121 "format": "short", 5122 "label": null, 5123 "logBase": 1, 5124 "max": null, 5125 "min": null, 5126 "show": true 5127 } 5128 ], 5129 "yaxis": { 5130 "align": false, 5131 "alignLevel": 0 5132 } 5133 }, 5134 { 5135 "aliasColors": {}, 5136 "bars": false, 5137 "cacheTimeout": null, 5138 "datasource": "$source", 5139 "description": null, 5140 "editable": true, 5141 "error": false, 5142 "fieldConfig": { 5143 "defaults": { 5144 "thresholds": { 5145 "mode": "absolute", 5146 "steps": [] 5147 } 5148 } 5149 }, 5150 "fill": 1, 5151 "grid": { 5152 "threshold1": null, 5153 "threshold1Color": "rgba(216, 200, 27, 0.27)", 5154 "threshold2": null, 5155 "threshold2Color": "rgba(234, 112, 112, 0.22)" 5156 }, 5157 "gridPos": null, 5158 "height": null, 5159 "hideTimeOverride": false, 5160 "id": 40, 5161 "interval": null, 5162 "isNew": true, 5163 "legend": { 5164 "alignAsTable": false, 5165 "avg": false, 5166 "current": false, 5167 "hideEmpty": false, 5168 "hideZero": false, 5169 "max": false, 5170 "min": false, 5171 "rightSide": false, 5172 "show": true, 5173 "sideWidth": null, 5174 "sort": null, 5175 "sortDesc": false, 5176 "total": false, 5177 "values": false 5178 }, 5179 "lines": true, 5180 "linewidth": 2, 5181 "links": [], 5182 "maxDataPoints": 100, 5183 "maxPerRow": null, 5184 "minSpan": null, 5185 "nullPointMode": null, 5186 "options": { 5187 "alertThreshold": true, 5188 "dataLinks": [] 5189 }, 5190 "percentage": false, 5191 "pointradius": 5, 5192 "points": false, 5193 "renderer": "flot", 5194 "repeat": null, 5195 "repeatDirection": null, 5196 "seriesOverrides": [], 5197 "span": 12, 5198 "stack": false, 5199 "steppedLine": false, 5200 "targets": [ 5201 { 5202 "datasource": "$source", 5203 "expr": "histogram_quantile(1.0, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[1m])) by (le, endpoint))", 5204 "format": "time_series", 5205 "hide": false, 5206 "instant": false, 5207 "interval": "5s", 5208 "intervalFactor": 1, 5209 "legendFormat": "", 5210 "metric": "", 5211 "query": "histogram_quantile(1.0, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[1m])) by (le, endpoint))", 5212 "refId": "", 5213 "step": 10, 5214 "target": "" 5215 } 5216 ], 5217 "thresholds": [], 5218 "timeFrom": null, 5219 "timeShift": null, 5220 "title": "etcd wal fsync duration", 5221 "tooltip": { 5222 "msResolution": true, 5223 "shared": true, 5224 "sort": 2, 5225 "value_type": "cumulative" 5226 }, 5227 "transformations": [], 5228 "transparent": false, 5229 "type": "graph", 5230 "xaxis": { 5231 "mode": "time", 5232 "name": null, 5233 "show": true, 5234 "values": [] 5235 }, 5236 "yaxes": [ 5237 { 5238 "decimals": null, 5239 "format": "s", 5240 "label": null, 5241 "logBase": 1, 5242 "max": null, 5243 "min": null, 5244 "show": true 5245 }, 5246 { 5247 "decimals": null, 5248 "format": "short", 5249 "label": null, 5250 "logBase": 1, 5251 "max": null, 5252 "min": null, 5253 "show": true 5254 } 5255 ], 5256 "yaxis": { 5257 "align": false, 5258 "alignLevel": 0 5259 } 5260 }, 5261 { 5262 "aliasColors": {}, 5263 "bars": false, 5264 "cacheTimeout": null, 5265 "datasource": "$source", 5266 "description": null, 5267 "editable": true, 5268 "error": false, 5269 "fieldConfig": { 5270 "defaults": { 5271 "thresholds": { 5272 "mode": "absolute", 5273 "steps": [] 5274 } 5275 } 5276 }, 5277 "fill": 1, 5278 "grid": { 5279 "threshold1": null, 5280 "threshold1Color": "rgba(216, 200, 27, 0.27)", 5281 "threshold2": null, 5282 "threshold2Color": "rgba(234, 112, 112, 0.22)" 5283 }, 5284 "gridPos": null, 5285 "height": null, 5286 "hideTimeOverride": false, 5287 "id": 41, 5288 "interval": null, 5289 "isNew": true, 5290 "legend": { 5291 "alignAsTable": false, 5292 "avg": false, 5293 "current": false, 5294 "hideEmpty": false, 5295 "hideZero": false, 5296 "max": false, 5297 "min": false, 5298 "rightSide": false, 5299 "show": true, 5300 "sideWidth": null, 5301 "sort": null, 5302 "sortDesc": false, 5303 "total": false, 5304 "values": false 5305 }, 5306 "lines": false, 5307 "linewidth": 2, 5308 "links": [], 5309 "maxDataPoints": 100, 5310 "maxPerRow": null, 5311 "minSpan": null, 5312 "nullPointMode": null, 5313 "options": { 5314 "alertThreshold": true, 5315 "dataLinks": [] 5316 }, 5317 "percentage": false, 5318 "pointradius": 5, 5319 "points": true, 5320 "renderer": "flot", 5321 "repeat": null, 5322 "repeatDirection": null, 5323 "seriesOverrides": [], 5324 "span": 12, 5325 "stack": false, 5326 "steppedLine": false, 5327 "targets": [ 5328 { 5329 "datasource": "$source", 5330 "expr": "histogram_quantile(1.0, sum(rate(etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds_bucket[1m])) by (le, instance))", 5331 "format": "time_series", 5332 "hide": false, 5333 "instant": false, 5334 "interval": "", 5335 "intervalFactor": 2, 5336 "legendFormat": "", 5337 "metric": "", 5338 "query": "histogram_quantile(1.0, sum(rate(etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds_bucket[1m])) by (le, instance))", 5339 "refId": "", 5340 "step": 10, 5341 "target": "" 5342 } 5343 ], 5344 "thresholds": [], 5345 "timeFrom": null, 5346 "timeShift": null, 5347 "title": "etcd compaction max pause", 5348 "tooltip": { 5349 "msResolution": true, 5350 "shared": true, 5351 "sort": 2, 5352 "value_type": "cumulative" 5353 }, 5354 "transformations": [], 5355 "transparent": false, 5356 "type": "graph", 5357 "xaxis": { 5358 "mode": "time", 5359 "name": null, 5360 "show": true, 5361 "values": [] 5362 }, 5363 "yaxes": [ 5364 { 5365 "decimals": null, 5366 "format": "ms", 5367 "label": null, 5368 "logBase": 1, 5369 "max": null, 5370 "min": null, 5371 "show": true 5372 }, 5373 { 5374 "decimals": null, 5375 "format": "short", 5376 "label": null, 5377 "logBase": 1, 5378 "max": null, 5379 "min": null, 5380 "show": true 5381 } 5382 ], 5383 "yaxis": { 5384 "align": false, 5385 "alignLevel": 0 5386 } 5387 }, 5388 { 5389 "aliasColors": {}, 5390 "bars": false, 5391 "cacheTimeout": null, 5392 "datasource": "$source", 5393 "description": null, 5394 "editable": true, 5395 "error": false, 5396 "fieldConfig": { 5397 "defaults": { 5398 "thresholds": { 5399 "mode": "absolute", 5400 "steps": [] 5401 } 5402 } 5403 }, 5404 "fill": 1, 5405 "grid": { 5406 "threshold1": null, 5407 "threshold1Color": "rgba(216, 200, 27, 0.27)", 5408 "threshold2": null, 5409 "threshold2Color": "rgba(234, 112, 112, 0.22)" 5410 }, 5411 "gridPos": null, 5412 "height": null, 5413 "hideTimeOverride": false, 5414 "id": 42, 5415 "interval": null, 5416 "isNew": true, 5417 "legend": { 5418 "alignAsTable": false, 5419 "avg": false, 5420 "current": false, 5421 "hideEmpty": false, 5422 "hideZero": false, 5423 "max": false, 5424 "min": false, 5425 "rightSide": false, 5426 "show": true, 5427 "sideWidth": null, 5428 "sort": null, 5429 "sortDesc": false, 5430 "total": false, 5431 "values": false 5432 }, 5433 "lines": true, 5434 "linewidth": 2, 5435 "links": [], 5436 "maxDataPoints": 100, 5437 "maxPerRow": null, 5438 "minSpan": null, 5439 "nullPointMode": null, 5440 "options": { 5441 "alertThreshold": true, 5442 "dataLinks": [] 5443 }, 5444 "percentage": false, 5445 "pointradius": 5, 5446 "points": false, 5447 "renderer": "flot", 5448 "repeat": null, 5449 "repeatDirection": null, 5450 "seriesOverrides": [], 5451 "span": 12, 5452 "stack": false, 5453 "steppedLine": false, 5454 "targets": [ 5455 { 5456 "datasource": "$source", 5457 "expr": "sum(etcd_object_counts) by (resource, instance)", 5458 "format": "time_series", 5459 "hide": false, 5460 "instant": false, 5461 "interval": "5s", 5462 "intervalFactor": 1, 5463 "legendFormat": "{{instance}}: {{resource}}", 5464 "metric": "", 5465 "query": "sum(etcd_object_counts) by (resource, instance)", 5466 "refId": "", 5467 "step": 10, 5468 "target": "" 5469 } 5470 ], 5471 "thresholds": [], 5472 "timeFrom": null, 5473 "timeShift": null, 5474 "title": "etcd objects", 5475 "tooltip": { 5476 "msResolution": true, 5477 "shared": true, 5478 "sort": 2, 5479 "value_type": "cumulative" 5480 }, 5481 "transformations": [], 5482 "transparent": false, 5483 "type": "graph", 5484 "xaxis": { 5485 "mode": "time", 5486 "name": null, 5487 "show": true, 5488 "values": [] 5489 }, 5490 "yaxes": [ 5491 { 5492 "decimals": null, 5493 "format": "short", 5494 "label": null, 5495 "logBase": 1, 5496 "max": null, 5497 "min": null, 5498 "show": true 5499 }, 5500 { 5501 "decimals": null, 5502 "format": "short", 5503 "label": null, 5504 "logBase": 1, 5505 "max": null, 5506 "min": null, 5507 "show": true 5508 } 5509 ], 5510 "yaxis": { 5511 "align": false, 5512 "alignLevel": 0 5513 } 5514 }, 5515 { 5516 "aliasColors": {}, 5517 "bars": false, 5518 "cacheTimeout": null, 5519 "datasource": "$source", 5520 "description": null, 5521 "editable": true, 5522 "error": false, 5523 "fieldConfig": { 5524 "defaults": { 5525 "thresholds": { 5526 "mode": "absolute", 5527 "steps": [] 5528 } 5529 } 5530 }, 5531 "fill": 1, 5532 "grid": { 5533 "threshold1": null, 5534 "threshold1Color": "rgba(216, 200, 27, 0.27)", 5535 "threshold2": null, 5536 "threshold2Color": "rgba(234, 112, 112, 0.22)" 5537 }, 5538 "gridPos": null, 5539 "height": null, 5540 "hideTimeOverride": false, 5541 "id": 43, 5542 "interval": null, 5543 "isNew": true, 5544 "legend": { 5545 "alignAsTable": false, 5546 "avg": false, 5547 "current": false, 5548 "hideEmpty": false, 5549 "hideZero": false, 5550 "max": false, 5551 "min": false, 5552 "rightSide": false, 5553 "show": true, 5554 "sideWidth": null, 5555 "sort": null, 5556 "sortDesc": false, 5557 "total": false, 5558 "values": false 5559 }, 5560 "lines": true, 5561 "linewidth": 2, 5562 "links": [], 5563 "maxDataPoints": 100, 5564 "maxPerRow": null, 5565 "minSpan": null, 5566 "nullPointMode": null, 5567 "options": { 5568 "alertThreshold": true, 5569 "dataLinks": [] 5570 }, 5571 "percentage": false, 5572 "pointradius": 5, 5573 "points": false, 5574 "renderer": "flot", 5575 "repeat": null, 5576 "repeatDirection": null, 5577 "seriesOverrides": [], 5578 "span": 12, 5579 "stack": false, 5580 "steppedLine": false, 5581 "targets": [ 5582 { 5583 "datasource": "$source", 5584 "expr": "etcd_mvcc_db_total_size_in_bytes", 5585 "format": "time_series", 5586 "hide": false, 5587 "instant": false, 5588 "interval": "5s", 5589 "intervalFactor": 1, 5590 "legendFormat": "", 5591 "metric": "", 5592 "query": "etcd_mvcc_db_total_size_in_bytes", 5593 "refId": "", 5594 "step": 10, 5595 "target": "" 5596 }, 5597 { 5598 "datasource": "$source", 5599 "expr": "etcd_mvcc_db_total_size_in_use_in_bytes", 5600 "format": "time_series", 5601 "hide": false, 5602 "instant": false, 5603 "interval": "5s", 5604 "intervalFactor": 1, 5605 "legendFormat": "", 5606 "metric": "", 5607 "query": "etcd_mvcc_db_total_size_in_use_in_bytes", 5608 "refId": "", 5609 "step": 10, 5610 "target": "" 5611 }, 5612 { 5613 "datasource": "$source", 5614 "expr": "etcd_server_quota_backend_bytes", 5615 "format": "time_series", 5616 "hide": false, 5617 "instant": false, 5618 "interval": "5s", 5619 "intervalFactor": 1, 5620 "legendFormat": "", 5621 "metric": "", 5622 "query": "etcd_server_quota_backend_bytes", 5623 "refId": "", 5624 "step": 10, 5625 "target": "" 5626 } 5627 ], 5628 "thresholds": [], 5629 "timeFrom": null, 5630 "timeShift": null, 5631 "title": "etcd db size", 5632 "tooltip": { 5633 "msResolution": true, 5634 "shared": true, 5635 "sort": 2, 5636 "value_type": "cumulative" 5637 }, 5638 "transformations": [], 5639 "transparent": false, 5640 "type": "graph", 5641 "xaxis": { 5642 "mode": "time", 5643 "name": null, 5644 "show": true, 5645 "values": [] 5646 }, 5647 "yaxes": [ 5648 { 5649 "decimals": null, 5650 "format": "bytes", 5651 "label": null, 5652 "logBase": 1, 5653 "max": null, 5654 "min": null, 5655 "show": true 5656 }, 5657 { 5658 "decimals": null, 5659 "format": "short", 5660 "label": null, 5661 "logBase": 1, 5662 "max": null, 5663 "min": null, 5664 "show": true 5665 } 5666 ], 5667 "yaxis": { 5668 "align": false, 5669 "alignLevel": 0 5670 } 5671 } 5672 ], 5673 "repeat": null, 5674 "showTitle": true, 5675 "title": "etcd" 5676 }, 5677 { 5678 "collapse": true, 5679 "editable": true, 5680 "height": "300px", 5681 "panels": [ 5682 { 5683 "aliasColors": {}, 5684 "bars": false, 5685 "cacheTimeout": null, 5686 "datasource": "$source", 5687 "description": null, 5688 "editable": true, 5689 "error": false, 5690 "fieldConfig": { 5691 "defaults": { 5692 "thresholds": { 5693 "mode": "absolute", 5694 "steps": [] 5695 } 5696 } 5697 }, 5698 "fill": 1, 5699 "grid": { 5700 "threshold1": null, 5701 "threshold1Color": "rgba(216, 200, 27, 0.27)", 5702 "threshold2": null, 5703 "threshold2Color": "rgba(234, 112, 112, 0.22)" 5704 }, 5705 "gridPos": null, 5706 "height": null, 5707 "hideTimeOverride": false, 5708 "id": 44, 5709 "interval": null, 5710 "isNew": true, 5711 "legend": { 5712 "alignAsTable": false, 5713 "avg": false, 5714 "current": false, 5715 "hideEmpty": false, 5716 "hideZero": false, 5717 "max": false, 5718 "min": false, 5719 "rightSide": false, 5720 "show": true, 5721 "sideWidth": null, 5722 "sort": null, 5723 "sortDesc": false, 5724 "total": false, 5725 "values": false 5726 }, 5727 "lines": true, 5728 "linewidth": 2, 5729 "links": [], 5730 "maxDataPoints": 100, 5731 "maxPerRow": null, 5732 "minSpan": null, 5733 "nullPointMode": null, 5734 "options": { 5735 "alertThreshold": true, 5736 "dataLinks": [] 5737 }, 5738 "percentage": false, 5739 "pointradius": 5, 5740 "points": false, 5741 "renderer": "flot", 5742 "repeat": null, 5743 "repeatDirection": null, 5744 "seriesOverrides": [], 5745 "span": 12, 5746 "stack": false, 5747 "steppedLine": false, 5748 "targets": [ 5749 { 5750 "datasource": "$source", 5751 "expr": "go_goroutines{job=\"master\", endpoint=\"apiserver\"}", 5752 "format": "time_series", 5753 "hide": false, 5754 "instant": false, 5755 "interval": "5s", 5756 "intervalFactor": 1, 5757 "legendFormat": "{{instance}}", 5758 "metric": "", 5759 "query": "go_goroutines{job=\"master\", endpoint=\"apiserver\"}", 5760 "refId": "", 5761 "step": 10, 5762 "target": "" 5763 } 5764 ], 5765 "thresholds": [], 5766 "timeFrom": null, 5767 "timeShift": null, 5768 "title": "goroutines", 5769 "tooltip": { 5770 "msResolution": true, 5771 "shared": true, 5772 "sort": 2, 5773 "value_type": "cumulative" 5774 }, 5775 "transformations": [], 5776 "transparent": false, 5777 "type": "graph", 5778 "xaxis": { 5779 "mode": "time", 5780 "name": null, 5781 "show": true, 5782 "values": [] 5783 }, 5784 "yaxes": [ 5785 { 5786 "decimals": null, 5787 "format": "short", 5788 "label": null, 5789 "logBase": 1, 5790 "max": null, 5791 "min": null, 5792 "show": true 5793 }, 5794 { 5795 "decimals": null, 5796 "format": "short", 5797 "label": null, 5798 "logBase": 1, 5799 "max": null, 5800 "min": null, 5801 "show": true 5802 } 5803 ], 5804 "yaxis": { 5805 "align": false, 5806 "alignLevel": 0 5807 } 5808 }, 5809 { 5810 "aliasColors": {}, 5811 "bars": false, 5812 "cacheTimeout": null, 5813 "datasource": "$source", 5814 "description": null, 5815 "editable": true, 5816 "error": false, 5817 "fieldConfig": { 5818 "defaults": { 5819 "thresholds": { 5820 "mode": "absolute", 5821 "steps": [] 5822 } 5823 } 5824 }, 5825 "fill": 1, 5826 "grid": { 5827 "threshold1": null, 5828 "threshold1Color": "rgba(216, 200, 27, 0.27)", 5829 "threshold2": null, 5830 "threshold2Color": "rgba(234, 112, 112, 0.22)" 5831 }, 5832 "gridPos": null, 5833 "height": null, 5834 "hideTimeOverride": false, 5835 "id": 45, 5836 "interval": null, 5837 "isNew": true, 5838 "legend": { 5839 "alignAsTable": false, 5840 "avg": false, 5841 "current": false, 5842 "hideEmpty": false, 5843 "hideZero": false, 5844 "max": false, 5845 "min": false, 5846 "rightSide": false, 5847 "show": true, 5848 "sideWidth": null, 5849 "sort": null, 5850 "sortDesc": false, 5851 "total": false, 5852 "values": false 5853 }, 5854 "lines": true, 5855 "linewidth": 2, 5856 "links": [], 5857 "maxDataPoints": 100, 5858 "maxPerRow": null, 5859 "minSpan": null, 5860 "nullPointMode": null, 5861 "options": { 5862 "alertThreshold": true, 5863 "dataLinks": [] 5864 }, 5865 "percentage": false, 5866 "pointradius": 5, 5867 "points": false, 5868 "renderer": "flot", 5869 "repeat": null, 5870 "repeatDirection": null, 5871 "seriesOverrides": [], 5872 "span": 12, 5873 "stack": false, 5874 "steppedLine": false, 5875 "targets": [ 5876 { 5877 "datasource": "$source", 5878 "expr": "rate(go_gc_duration_seconds_count{job=\"master\", endpoint=\"apiserver\"}[1m])", 5879 "format": "time_series", 5880 "hide": false, 5881 "instant": false, 5882 "interval": "5s", 5883 "intervalFactor": 1, 5884 "legendFormat": "{{instance}}", 5885 "metric": "", 5886 "query": "rate(go_gc_duration_seconds_count{job=\"master\", endpoint=\"apiserver\"}[1m])", 5887 "refId": "", 5888 "step": 10, 5889 "target": "" 5890 } 5891 ], 5892 "thresholds": [], 5893 "timeFrom": null, 5894 "timeShift": null, 5895 "title": "gc rate", 5896 "tooltip": { 5897 "msResolution": true, 5898 "shared": true, 5899 "sort": 2, 5900 "value_type": "cumulative" 5901 }, 5902 "transformations": [], 5903 "transparent": false, 5904 "type": "graph", 5905 "xaxis": { 5906 "mode": "time", 5907 "name": null, 5908 "show": true, 5909 "values": [] 5910 }, 5911 "yaxes": [ 5912 { 5913 "decimals": null, 5914 "format": "short", 5915 "label": null, 5916 "logBase": 1, 5917 "max": null, 5918 "min": null, 5919 "show": true 5920 }, 5921 { 5922 "decimals": null, 5923 "format": "short", 5924 "label": null, 5925 "logBase": 1, 5926 "max": null, 5927 "min": null, 5928 "show": true 5929 } 5930 ], 5931 "yaxis": { 5932 "align": false, 5933 "alignLevel": 0 5934 } 5935 }, 5936 { 5937 "aliasColors": {}, 5938 "bars": false, 5939 "cacheTimeout": null, 5940 "datasource": "$source", 5941 "description": null, 5942 "editable": true, 5943 "error": false, 5944 "fieldConfig": { 5945 "defaults": { 5946 "thresholds": { 5947 "mode": "absolute", 5948 "steps": [] 5949 } 5950 } 5951 }, 5952 "fill": 1, 5953 "grid": { 5954 "threshold1": null, 5955 "threshold1Color": "rgba(216, 200, 27, 0.27)", 5956 "threshold2": null, 5957 "threshold2Color": "rgba(234, 112, 112, 0.22)" 5958 }, 5959 "gridPos": null, 5960 "height": null, 5961 "hideTimeOverride": false, 5962 "id": 46, 5963 "interval": null, 5964 "isNew": true, 5965 "legend": { 5966 "alignAsTable": false, 5967 "avg": false, 5968 "current": false, 5969 "hideEmpty": false, 5970 "hideZero": false, 5971 "max": false, 5972 "min": false, 5973 "rightSide": false, 5974 "show": true, 5975 "sideWidth": null, 5976 "sort": null, 5977 "sortDesc": false, 5978 "total": false, 5979 "values": false 5980 }, 5981 "lines": true, 5982 "linewidth": 2, 5983 "links": [], 5984 "maxDataPoints": 100, 5985 "maxPerRow": null, 5986 "minSpan": null, 5987 "nullPointMode": null, 5988 "options": { 5989 "alertThreshold": true, 5990 "dataLinks": [] 5991 }, 5992 "percentage": false, 5993 "pointradius": 5, 5994 "points": false, 5995 "renderer": "flot", 5996 "repeat": null, 5997 "repeatDirection": null, 5998 "seriesOverrides": [], 5999 "span": 12, 6000 "stack": false, 6001 "steppedLine": false, 6002 "targets": [ 6003 { 6004 "datasource": "$source", 6005 "expr": "rate(go_memstats_alloc_bytes_total{job=\"master\", endpoint=\"apiserver\"}[1m])", 6006 "format": "time_series", 6007 "hide": false, 6008 "instant": false, 6009 "interval": "5s", 6010 "intervalFactor": 1, 6011 "legendFormat": "{{instance}}", 6012 "metric": "", 6013 "query": "rate(go_memstats_alloc_bytes_total{job=\"master\", endpoint=\"apiserver\"}[1m])", 6014 "refId": "", 6015 "step": 10, 6016 "target": "" 6017 } 6018 ], 6019 "thresholds": [], 6020 "timeFrom": null, 6021 "timeShift": null, 6022 "title": "alloc rate", 6023 "tooltip": { 6024 "msResolution": true, 6025 "shared": true, 6026 "sort": 2, 6027 "value_type": "cumulative" 6028 }, 6029 "transformations": [], 6030 "transparent": false, 6031 "type": "graph", 6032 "xaxis": { 6033 "mode": "time", 6034 "name": null, 6035 "show": true, 6036 "values": [] 6037 }, 6038 "yaxes": [ 6039 { 6040 "decimals": null, 6041 "format": "Bps", 6042 "label": null, 6043 "logBase": 1, 6044 "max": null, 6045 "min": null, 6046 "show": true 6047 }, 6048 { 6049 "decimals": null, 6050 "format": "short", 6051 "label": null, 6052 "logBase": 1, 6053 "max": null, 6054 "min": null, 6055 "show": true 6056 } 6057 ], 6058 "yaxis": { 6059 "align": false, 6060 "alignLevel": 0 6061 } 6062 }, 6063 { 6064 "aliasColors": {}, 6065 "bars": false, 6066 "cacheTimeout": null, 6067 "datasource": "$source", 6068 "description": null, 6069 "editable": true, 6070 "error": false, 6071 "fieldConfig": { 6072 "defaults": { 6073 "thresholds": { 6074 "mode": "absolute", 6075 "steps": [] 6076 } 6077 } 6078 }, 6079 "fill": 1, 6080 "grid": { 6081 "threshold1": null, 6082 "threshold1Color": "rgba(216, 200, 27, 0.27)", 6083 "threshold2": null, 6084 "threshold2Color": "rgba(234, 112, 112, 0.22)" 6085 }, 6086 "gridPos": null, 6087 "height": null, 6088 "hideTimeOverride": false, 6089 "id": 47, 6090 "interval": null, 6091 "isNew": true, 6092 "legend": { 6093 "alignAsTable": false, 6094 "avg": false, 6095 "current": false, 6096 "hideEmpty": false, 6097 "hideZero": false, 6098 "max": false, 6099 "min": false, 6100 "rightSide": false, 6101 "show": true, 6102 "sideWidth": null, 6103 "sort": null, 6104 "sortDesc": false, 6105 "total": false, 6106 "values": false 6107 }, 6108 "lines": true, 6109 "linewidth": 2, 6110 "links": [], 6111 "maxDataPoints": 100, 6112 "maxPerRow": null, 6113 "minSpan": null, 6114 "nullPointMode": null, 6115 "options": { 6116 "alertThreshold": true, 6117 "dataLinks": [] 6118 }, 6119 "percentage": false, 6120 "pointradius": 5, 6121 "points": false, 6122 "renderer": "flot", 6123 "repeat": null, 6124 "repeatDirection": null, 6125 "seriesOverrides": [], 6126 "span": 12, 6127 "stack": false, 6128 "steppedLine": false, 6129 "targets": [ 6130 { 6131 "datasource": "$source", 6132 "expr": "sum(apiserver_registered_watchers{kind=~\"(?i:(${resource:regex}))s*\"}) by (instance, group, version, kind)", 6133 "format": "time_series", 6134 "hide": false, 6135 "instant": false, 6136 "interval": "5s", 6137 "intervalFactor": 1, 6138 "legendFormat": "{{instance}}: {{version}}.{{group}}.{{kind}}", 6139 "metric": "", 6140 "query": "sum(apiserver_registered_watchers{kind=~\"(?i:(${resource:regex}))s*\"}) by (instance, group, version, kind)", 6141 "refId": "", 6142 "step": 10, 6143 "target": "" 6144 } 6145 ], 6146 "thresholds": [], 6147 "timeFrom": null, 6148 "timeShift": null, 6149 "title": "Number of active watches", 6150 "tooltip": { 6151 "msResolution": true, 6152 "shared": true, 6153 "sort": 2, 6154 "value_type": "cumulative" 6155 }, 6156 "transformations": [], 6157 "transparent": false, 6158 "type": "graph", 6159 "xaxis": { 6160 "mode": "time", 6161 "name": null, 6162 "show": true, 6163 "values": [] 6164 }, 6165 "yaxes": [ 6166 { 6167 "decimals": null, 6168 "format": "short", 6169 "label": null, 6170 "logBase": 1, 6171 "max": null, 6172 "min": null, 6173 "show": true 6174 }, 6175 { 6176 "decimals": null, 6177 "format": "short", 6178 "label": null, 6179 "logBase": 1, 6180 "max": null, 6181 "min": null, 6182 "show": true 6183 } 6184 ], 6185 "yaxis": { 6186 "align": false, 6187 "alignLevel": 0 6188 } 6189 }, 6190 { 6191 "aliasColors": {}, 6192 "bars": false, 6193 "cacheTimeout": null, 6194 "datasource": "$source", 6195 "description": null, 6196 "editable": true, 6197 "error": false, 6198 "fieldConfig": { 6199 "defaults": { 6200 "thresholds": { 6201 "mode": "absolute", 6202 "steps": [] 6203 } 6204 } 6205 }, 6206 "fill": 1, 6207 "grid": { 6208 "threshold1": null, 6209 "threshold1Color": "rgba(216, 200, 27, 0.27)", 6210 "threshold2": null, 6211 "threshold2Color": "rgba(234, 112, 112, 0.22)" 6212 }, 6213 "gridPos": null, 6214 "height": null, 6215 "hideTimeOverride": false, 6216 "id": 48, 6217 "interval": null, 6218 "isNew": true, 6219 "legend": { 6220 "alignAsTable": false, 6221 "avg": false, 6222 "current": false, 6223 "hideEmpty": false, 6224 "hideZero": false, 6225 "max": false, 6226 "min": false, 6227 "rightSide": false, 6228 "show": true, 6229 "sideWidth": null, 6230 "sort": null, 6231 "sortDesc": false, 6232 "total": false, 6233 "values": false 6234 }, 6235 "lines": true, 6236 "linewidth": 2, 6237 "links": [], 6238 "maxDataPoints": 100, 6239 "maxPerRow": null, 6240 "minSpan": null, 6241 "nullPointMode": null, 6242 "options": { 6243 "alertThreshold": true, 6244 "dataLinks": [] 6245 }, 6246 "percentage": false, 6247 "pointradius": 5, 6248 "points": false, 6249 "renderer": "flot", 6250 "repeat": null, 6251 "repeatDirection": null, 6252 "seriesOverrides": [], 6253 "span": 12, 6254 "stack": false, 6255 "steppedLine": false, 6256 "targets": [ 6257 { 6258 "datasource": "$source", 6259 "expr": "sum(irate(apiserver_watch_events_total{kind=~\"(?i:(${resource:regex}))s*\"}[1m])) by (instance, group, version, kind)", 6260 "format": "time_series", 6261 "hide": false, 6262 "instant": false, 6263 "interval": "5s", 6264 "intervalFactor": 1, 6265 "legendFormat": "{{instance}}: {{version}}.{{group}}.{{kind}}", 6266 "metric": "", 6267 "query": "sum(irate(apiserver_watch_events_total{kind=~\"(?i:(${resource:regex}))s*\"}[1m])) by (instance, group, version, kind)", 6268 "refId": "", 6269 "step": 10, 6270 "target": "" 6271 } 6272 ], 6273 "thresholds": [], 6274 "timeFrom": null, 6275 "timeShift": null, 6276 "title": "Watch events rate", 6277 "tooltip": { 6278 "msResolution": true, 6279 "shared": true, 6280 "sort": 2, 6281 "value_type": "cumulative" 6282 }, 6283 "transformations": [], 6284 "transparent": false, 6285 "type": "graph", 6286 "xaxis": { 6287 "mode": "time", 6288 "name": null, 6289 "show": true, 6290 "values": [] 6291 }, 6292 "yaxes": [ 6293 { 6294 "decimals": null, 6295 "format": "short", 6296 "label": null, 6297 "logBase": 1, 6298 "max": null, 6299 "min": null, 6300 "show": true 6301 }, 6302 { 6303 "decimals": null, 6304 "format": "short", 6305 "label": null, 6306 "logBase": 1, 6307 "max": null, 6308 "min": null, 6309 "show": true 6310 } 6311 ], 6312 "yaxis": { 6313 "align": false, 6314 "alignLevel": 0 6315 } 6316 }, 6317 { 6318 "aliasColors": {}, 6319 "bars": false, 6320 "cacheTimeout": null, 6321 "datasource": "$source", 6322 "description": null, 6323 "editable": true, 6324 "error": false, 6325 "fieldConfig": { 6326 "defaults": { 6327 "thresholds": { 6328 "mode": "absolute", 6329 "steps": [] 6330 } 6331 } 6332 }, 6333 "fill": 1, 6334 "grid": { 6335 "threshold1": null, 6336 "threshold1Color": "rgba(216, 200, 27, 0.27)", 6337 "threshold2": null, 6338 "threshold2Color": "rgba(234, 112, 112, 0.22)" 6339 }, 6340 "gridPos": null, 6341 "height": null, 6342 "hideTimeOverride": false, 6343 "id": 49, 6344 "interval": null, 6345 "isNew": true, 6346 "legend": { 6347 "alignAsTable": false, 6348 "avg": false, 6349 "current": false, 6350 "hideEmpty": false, 6351 "hideZero": false, 6352 "max": false, 6353 "min": false, 6354 "rightSide": false, 6355 "show": true, 6356 "sideWidth": null, 6357 "sort": null, 6358 "sortDesc": false, 6359 "total": false, 6360 "values": false 6361 }, 6362 "lines": true, 6363 "linewidth": 2, 6364 "links": [], 6365 "maxDataPoints": 100, 6366 "maxPerRow": null, 6367 "minSpan": null, 6368 "nullPointMode": null, 6369 "options": { 6370 "alertThreshold": true, 6371 "dataLinks": [] 6372 }, 6373 "percentage": false, 6374 "pointradius": 5, 6375 "points": false, 6376 "renderer": "flot", 6377 "repeat": null, 6378 "repeatDirection": null, 6379 "seriesOverrides": [], 6380 "span": 12, 6381 "stack": false, 6382 "steppedLine": false, 6383 "targets": [ 6384 { 6385 "datasource": "$source", 6386 "expr": "sum(irate(apiserver_watch_events_sizes_sum{kind=~\"(?i:(${resource:regex}))s*\"}[1m])) by (instance, group, version, kind)", 6387 "format": "time_series", 6388 "hide": false, 6389 "instant": false, 6390 "interval": "5s", 6391 "intervalFactor": 1, 6392 "legendFormat": "{{instance}}: {{version}}.{{group}}.{{kind}}", 6393 "metric": "", 6394 "query": "sum(irate(apiserver_watch_events_sizes_sum{kind=~\"(?i:(${resource:regex}))s*\"}[1m])) by (instance, group, version, kind)", 6395 "refId": "", 6396 "step": 10, 6397 "target": "" 6398 } 6399 ], 6400 "thresholds": [], 6401 "timeFrom": null, 6402 "timeShift": null, 6403 "title": "Watch events traffic", 6404 "tooltip": { 6405 "msResolution": true, 6406 "shared": true, 6407 "sort": 2, 6408 "value_type": "cumulative" 6409 }, 6410 "transformations": [], 6411 "transparent": false, 6412 "type": "graph", 6413 "xaxis": { 6414 "mode": "time", 6415 "name": null, 6416 "show": true, 6417 "values": [] 6418 }, 6419 "yaxes": [ 6420 { 6421 "decimals": null, 6422 "format": "Bps", 6423 "label": null, 6424 "logBase": 1, 6425 "max": null, 6426 "min": null, 6427 "show": true 6428 }, 6429 { 6430 "decimals": null, 6431 "format": "short", 6432 "label": null, 6433 "logBase": 1, 6434 "max": null, 6435 "min": null, 6436 "show": true 6437 } 6438 ], 6439 "yaxis": { 6440 "align": false, 6441 "alignLevel": 0 6442 } 6443 }, 6444 { 6445 "aliasColors": {}, 6446 "bars": false, 6447 "cacheTimeout": null, 6448 "datasource": "$source", 6449 "description": null, 6450 "editable": true, 6451 "error": false, 6452 "fieldConfig": { 6453 "defaults": { 6454 "thresholds": { 6455 "mode": "absolute", 6456 "steps": [] 6457 } 6458 } 6459 }, 6460 "fill": 1, 6461 "grid": { 6462 "threshold1": null, 6463 "threshold1Color": "rgba(216, 200, 27, 0.27)", 6464 "threshold2": null, 6465 "threshold2Color": "rgba(234, 112, 112, 0.22)" 6466 }, 6467 "gridPos": null, 6468 "height": null, 6469 "hideTimeOverride": false, 6470 "id": 50, 6471 "interval": null, 6472 "isNew": true, 6473 "legend": { 6474 "alignAsTable": false, 6475 "avg": false, 6476 "current": false, 6477 "hideEmpty": false, 6478 "hideZero": false, 6479 "max": false, 6480 "min": false, 6481 "rightSide": false, 6482 "show": true, 6483 "sideWidth": null, 6484 "sort": null, 6485 "sortDesc": false, 6486 "total": false, 6487 "values": false 6488 }, 6489 "lines": true, 6490 "linewidth": 2, 6491 "links": [], 6492 "maxDataPoints": 100, 6493 "maxPerRow": null, 6494 "minSpan": null, 6495 "nullPointMode": null, 6496 "options": { 6497 "alertThreshold": true, 6498 "dataLinks": [] 6499 }, 6500 "percentage": false, 6501 "pointradius": 5, 6502 "points": false, 6503 "renderer": "flot", 6504 "repeat": null, 6505 "repeatDirection": null, 6506 "seriesOverrides": [], 6507 "span": 12, 6508 "stack": false, 6509 "steppedLine": false, 6510 "targets": [ 6511 { 6512 "datasource": "$source", 6513 "expr": "sum(rate(apiserver_watch_events_sizes_sum{kind=~\"(?i:(${resource:regex}))s*\"}[1m])/rate(apiserver_watch_events_sizes_count{kind=~\"(?i:(${resource:regex}))s*\"}[1m])) by (instance, group, version, kind)", 6514 "format": "time_series", 6515 "hide": false, 6516 "instant": false, 6517 "interval": "5s", 6518 "intervalFactor": 1, 6519 "legendFormat": "{{instance}}: {{version}}.{{group}}.{{kind}}", 6520 "metric": "", 6521 "query": "sum(rate(apiserver_watch_events_sizes_sum{kind=~\"(?i:(${resource:regex}))s*\"}[1m])/rate(apiserver_watch_events_sizes_count{kind=~\"(?i:(${resource:regex}))s*\"}[1m])) by (instance, group, version, kind)", 6522 "refId": "", 6523 "step": 10, 6524 "target": "" 6525 } 6526 ], 6527 "thresholds": [], 6528 "timeFrom": null, 6529 "timeShift": null, 6530 "title": "Watch event avg size", 6531 "tooltip": { 6532 "msResolution": true, 6533 "shared": true, 6534 "sort": 2, 6535 "value_type": "cumulative" 6536 }, 6537 "transformations": [], 6538 "transparent": false, 6539 "type": "graph", 6540 "xaxis": { 6541 "mode": "time", 6542 "name": null, 6543 "show": true, 6544 "values": [] 6545 }, 6546 "yaxes": [ 6547 { 6548 "decimals": null, 6549 "format": "short", 6550 "label": null, 6551 "logBase": 1, 6552 "max": null, 6553 "min": null, 6554 "show": true 6555 }, 6556 { 6557 "decimals": null, 6558 "format": "short", 6559 "label": null, 6560 "logBase": 1, 6561 "max": null, 6562 "min": null, 6563 "show": true 6564 } 6565 ], 6566 "yaxis": { 6567 "align": false, 6568 "alignLevel": 0 6569 } 6570 }, 6571 { 6572 "aliasColors": {}, 6573 "bars": false, 6574 "cacheTimeout": null, 6575 "datasource": "$source", 6576 "description": null, 6577 "editable": true, 6578 "error": false, 6579 "fieldConfig": { 6580 "defaults": { 6581 "thresholds": { 6582 "mode": "absolute", 6583 "steps": [] 6584 } 6585 } 6586 }, 6587 "fill": 1, 6588 "grid": { 6589 "threshold1": null, 6590 "threshold1Color": "rgba(216, 200, 27, 0.27)", 6591 "threshold2": null, 6592 "threshold2Color": "rgba(234, 112, 112, 0.22)" 6593 }, 6594 "gridPos": null, 6595 "height": null, 6596 "hideTimeOverride": false, 6597 "id": 51, 6598 "interval": null, 6599 "isNew": true, 6600 "legend": { 6601 "alignAsTable": false, 6602 "avg": false, 6603 "current": false, 6604 "hideEmpty": false, 6605 "hideZero": false, 6606 "max": false, 6607 "min": false, 6608 "rightSide": false, 6609 "show": true, 6610 "sideWidth": null, 6611 "sort": null, 6612 "sortDesc": false, 6613 "total": false, 6614 "values": false 6615 }, 6616 "lines": true, 6617 "linewidth": 2, 6618 "links": [], 6619 "maxDataPoints": 100, 6620 "maxPerRow": null, 6621 "minSpan": null, 6622 "nullPointMode": null, 6623 "options": { 6624 "alertThreshold": true, 6625 "dataLinks": [] 6626 }, 6627 "percentage": false, 6628 "pointradius": 5, 6629 "points": false, 6630 "renderer": "flot", 6631 "repeat": null, 6632 "repeatDirection": null, 6633 "seriesOverrides": [], 6634 "span": 12, 6635 "stack": false, 6636 "steppedLine": false, 6637 "targets": [ 6638 { 6639 "datasource": "$source", 6640 "expr": "sum(rate(apiserver_terminated_watchers_total{}[1m])) by (resource, instance)", 6641 "format": "time_series", 6642 "hide": false, 6643 "instant": false, 6644 "interval": "5s", 6645 "intervalFactor": 1, 6646 "legendFormat": "{{instance}}: {{resource}}", 6647 "metric": "", 6648 "query": "sum(rate(apiserver_terminated_watchers_total{}[1m])) by (resource, instance)", 6649 "refId": "", 6650 "step": 10, 6651 "target": "" 6652 } 6653 ], 6654 "thresholds": [], 6655 "timeFrom": null, 6656 "timeShift": null, 6657 "title": "Watch terminated total", 6658 "tooltip": { 6659 "msResolution": true, 6660 "shared": true, 6661 "sort": 2, 6662 "value_type": "cumulative" 6663 }, 6664 "transformations": [], 6665 "transparent": false, 6666 "type": "graph", 6667 "xaxis": { 6668 "mode": "time", 6669 "name": null, 6670 "show": true, 6671 "values": [] 6672 }, 6673 "yaxes": [ 6674 { 6675 "decimals": null, 6676 "format": "short", 6677 "label": null, 6678 "logBase": 1, 6679 "max": null, 6680 "min": null, 6681 "show": true 6682 }, 6683 { 6684 "decimals": null, 6685 "format": "short", 6686 "label": null, 6687 "logBase": 1, 6688 "max": null, 6689 "min": null, 6690 "show": true 6691 } 6692 ], 6693 "yaxis": { 6694 "align": false, 6695 "alignLevel": 0 6696 } 6697 }, 6698 { 6699 "aliasColors": {}, 6700 "bars": false, 6701 "cacheTimeout": null, 6702 "datasource": "$source", 6703 "description": null, 6704 "editable": true, 6705 "error": false, 6706 "fieldConfig": { 6707 "defaults": { 6708 "thresholds": { 6709 "mode": "absolute", 6710 "steps": [] 6711 } 6712 } 6713 }, 6714 "fill": 1, 6715 "grid": { 6716 "threshold1": null, 6717 "threshold1Color": "rgba(216, 200, 27, 0.27)", 6718 "threshold2": null, 6719 "threshold2Color": "rgba(234, 112, 112, 0.22)" 6720 }, 6721 "gridPos": null, 6722 "height": null, 6723 "hideTimeOverride": false, 6724 "id": 52, 6725 "interval": null, 6726 "isNew": true, 6727 "legend": { 6728 "alignAsTable": false, 6729 "avg": false, 6730 "current": false, 6731 "hideEmpty": false, 6732 "hideZero": false, 6733 "max": false, 6734 "min": false, 6735 "rightSide": false, 6736 "show": true, 6737 "sideWidth": null, 6738 "sort": null, 6739 "sortDesc": false, 6740 "total": false, 6741 "values": false 6742 }, 6743 "lines": true, 6744 "linewidth": 2, 6745 "links": [], 6746 "maxDataPoints": 100, 6747 "maxPerRow": null, 6748 "minSpan": null, 6749 "nullPointMode": null, 6750 "options": { 6751 "alertThreshold": true, 6752 "dataLinks": [] 6753 }, 6754 "percentage": false, 6755 "pointradius": 5, 6756 "points": false, 6757 "renderer": "flot", 6758 "repeat": null, 6759 "repeatDirection": null, 6760 "seriesOverrides": [], 6761 "span": 12, 6762 "stack": false, 6763 "steppedLine": false, 6764 "targets": [ 6765 { 6766 "datasource": "$source", 6767 "expr": "sum(apiserver_current_inflight_requests) by (requestKind, instance)", 6768 "format": "time_series", 6769 "hide": false, 6770 "instant": false, 6771 "interval": "5s", 6772 "intervalFactor": 1, 6773 "legendFormat": "{{instance}}: {{requestKind}}", 6774 "metric": "", 6775 "query": "sum(apiserver_current_inflight_requests) by (requestKind, instance)", 6776 "refId": "", 6777 "step": 10, 6778 "target": "" 6779 } 6780 ], 6781 "thresholds": [], 6782 "timeFrom": null, 6783 "timeShift": null, 6784 "title": "Inflight requests", 6785 "tooltip": { 6786 "msResolution": true, 6787 "shared": true, 6788 "sort": 2, 6789 "value_type": "cumulative" 6790 }, 6791 "transformations": [], 6792 "transparent": false, 6793 "type": "graph", 6794 "xaxis": { 6795 "mode": "time", 6796 "name": null, 6797 "show": true, 6798 "values": [] 6799 }, 6800 "yaxes": [ 6801 { 6802 "decimals": null, 6803 "format": "short", 6804 "label": null, 6805 "logBase": 1, 6806 "max": null, 6807 "min": null, 6808 "show": true 6809 }, 6810 { 6811 "decimals": null, 6812 "format": "short", 6813 "label": null, 6814 "logBase": 1, 6815 "max": null, 6816 "min": null, 6817 "show": true 6818 } 6819 ], 6820 "yaxis": { 6821 "align": false, 6822 "alignLevel": 0 6823 } 6824 }, 6825 { 6826 "aliasColors": {}, 6827 "bars": false, 6828 "cacheTimeout": null, 6829 "datasource": "$source", 6830 "description": null, 6831 "editable": true, 6832 "error": false, 6833 "fieldConfig": { 6834 "defaults": { 6835 "thresholds": { 6836 "mode": "absolute", 6837 "steps": [] 6838 } 6839 } 6840 }, 6841 "fill": 1, 6842 "grid": { 6843 "threshold1": null, 6844 "threshold1Color": "rgba(216, 200, 27, 0.27)", 6845 "threshold2": null, 6846 "threshold2Color": "rgba(234, 112, 112, 0.22)" 6847 }, 6848 "gridPos": null, 6849 "height": null, 6850 "hideTimeOverride": false, 6851 "id": 53, 6852 "interval": null, 6853 "isNew": true, 6854 "legend": { 6855 "alignAsTable": false, 6856 "avg": false, 6857 "current": false, 6858 "hideEmpty": false, 6859 "hideZero": false, 6860 "max": false, 6861 "min": false, 6862 "rightSide": false, 6863 "show": true, 6864 "sideWidth": null, 6865 "sort": null, 6866 "sortDesc": false, 6867 "total": false, 6868 "values": false 6869 }, 6870 "lines": true, 6871 "linewidth": 2, 6872 "links": [], 6873 "maxDataPoints": 100, 6874 "maxPerRow": null, 6875 "minSpan": null, 6876 "nullPointMode": null, 6877 "options": { 6878 "alertThreshold": true, 6879 "dataLinks": [] 6880 }, 6881 "percentage": false, 6882 "pointradius": 5, 6883 "points": false, 6884 "renderer": "flot", 6885 "repeat": null, 6886 "repeatDirection": null, 6887 "seriesOverrides": [], 6888 "span": 12, 6889 "stack": false, 6890 "steppedLine": false, 6891 "targets": [ 6892 { 6893 "datasource": "$source", 6894 "expr": "sum(rate(apiserver_request_total{verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}[1m])) by (verb, resource, subresource, instance)", 6895 "format": "time_series", 6896 "hide": false, 6897 "instant": false, 6898 "interval": "5s", 6899 "intervalFactor": 1, 6900 "legendFormat": "", 6901 "metric": "", 6902 "query": "sum(rate(apiserver_request_total{verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}[1m])) by (verb, resource, subresource, instance)", 6903 "refId": "", 6904 "step": 10, 6905 "target": "" 6906 } 6907 ], 6908 "thresholds": [], 6909 "timeFrom": null, 6910 "timeShift": null, 6911 "title": "Request rate", 6912 "tooltip": { 6913 "msResolution": true, 6914 "shared": true, 6915 "sort": 2, 6916 "value_type": "cumulative" 6917 }, 6918 "transformations": [], 6919 "transparent": false, 6920 "type": "graph", 6921 "xaxis": { 6922 "mode": "time", 6923 "name": null, 6924 "show": true, 6925 "values": [] 6926 }, 6927 "yaxes": [ 6928 { 6929 "decimals": null, 6930 "format": "short", 6931 "label": null, 6932 "logBase": 1, 6933 "max": null, 6934 "min": null, 6935 "show": true 6936 }, 6937 { 6938 "decimals": null, 6939 "format": "short", 6940 "label": null, 6941 "logBase": 1, 6942 "max": null, 6943 "min": null, 6944 "show": true 6945 } 6946 ], 6947 "yaxis": { 6948 "align": false, 6949 "alignLevel": 0 6950 } 6951 }, 6952 { 6953 "aliasColors": {}, 6954 "bars": false, 6955 "cacheTimeout": null, 6956 "datasource": "$source", 6957 "description": null, 6958 "editable": true, 6959 "error": false, 6960 "fieldConfig": { 6961 "defaults": { 6962 "thresholds": { 6963 "mode": "absolute", 6964 "steps": [] 6965 } 6966 } 6967 }, 6968 "fill": 1, 6969 "grid": { 6970 "threshold1": null, 6971 "threshold1Color": "rgba(216, 200, 27, 0.27)", 6972 "threshold2": null, 6973 "threshold2Color": "rgba(234, 112, 112, 0.22)" 6974 }, 6975 "gridPos": null, 6976 "height": null, 6977 "hideTimeOverride": false, 6978 "id": 54, 6979 "interval": null, 6980 "isNew": true, 6981 "legend": { 6982 "alignAsTable": false, 6983 "avg": false, 6984 "current": false, 6985 "hideEmpty": false, 6986 "hideZero": false, 6987 "max": false, 6988 "min": false, 6989 "rightSide": false, 6990 "show": true, 6991 "sideWidth": null, 6992 "sort": null, 6993 "sortDesc": false, 6994 "total": false, 6995 "values": false 6996 }, 6997 "lines": true, 6998 "linewidth": 2, 6999 "links": [], 7000 "maxDataPoints": 100, 7001 "maxPerRow": null, 7002 "minSpan": null, 7003 "nullPointMode": null, 7004 "options": { 7005 "alertThreshold": true, 7006 "dataLinks": [] 7007 }, 7008 "percentage": false, 7009 "pointradius": 5, 7010 "points": false, 7011 "renderer": "flot", 7012 "repeat": null, 7013 "repeatDirection": null, 7014 "seriesOverrides": [], 7015 "span": 12, 7016 "stack": false, 7017 "steppedLine": false, 7018 "targets": [ 7019 { 7020 "datasource": "$source", 7021 "expr": "sum(rate(apiserver_request_total[1m])) by (code, instance)", 7022 "format": "time_series", 7023 "hide": false, 7024 "instant": false, 7025 "interval": "5s", 7026 "intervalFactor": 1, 7027 "legendFormat": "{{instance}}: {{code}}", 7028 "metric": "", 7029 "query": "sum(rate(apiserver_request_total[1m])) by (code, instance)", 7030 "refId": "", 7031 "step": 10, 7032 "target": "" 7033 } 7034 ], 7035 "thresholds": [], 7036 "timeFrom": null, 7037 "timeShift": null, 7038 "title": "Request rate by code", 7039 "tooltip": { 7040 "msResolution": true, 7041 "shared": true, 7042 "sort": 2, 7043 "value_type": "cumulative" 7044 }, 7045 "transformations": [], 7046 "transparent": false, 7047 "type": "graph", 7048 "xaxis": { 7049 "mode": "time", 7050 "name": null, 7051 "show": true, 7052 "values": [] 7053 }, 7054 "yaxes": [ 7055 { 7056 "decimals": null, 7057 "format": "short", 7058 "label": null, 7059 "logBase": 1, 7060 "max": null, 7061 "min": null, 7062 "show": true 7063 }, 7064 { 7065 "decimals": null, 7066 "format": "short", 7067 "label": null, 7068 "logBase": 1, 7069 "max": null, 7070 "min": null, 7071 "show": true 7072 } 7073 ], 7074 "yaxis": { 7075 "align": false, 7076 "alignLevel": 0 7077 } 7078 }, 7079 { 7080 "aliasColors": {}, 7081 "bars": false, 7082 "cacheTimeout": null, 7083 "datasource": "$source", 7084 "description": null, 7085 "editable": true, 7086 "error": false, 7087 "fieldConfig": { 7088 "defaults": { 7089 "thresholds": { 7090 "mode": "absolute", 7091 "steps": [] 7092 } 7093 } 7094 }, 7095 "fill": 1, 7096 "grid": { 7097 "threshold1": null, 7098 "threshold1Color": "rgba(216, 200, 27, 0.27)", 7099 "threshold2": null, 7100 "threshold2Color": "rgba(234, 112, 112, 0.22)" 7101 }, 7102 "gridPos": null, 7103 "height": null, 7104 "hideTimeOverride": false, 7105 "id": 55, 7106 "interval": null, 7107 "isNew": true, 7108 "legend": { 7109 "alignAsTable": false, 7110 "avg": false, 7111 "current": false, 7112 "hideEmpty": false, 7113 "hideZero": false, 7114 "max": false, 7115 "min": false, 7116 "rightSide": false, 7117 "show": true, 7118 "sideWidth": null, 7119 "sort": null, 7120 "sortDesc": false, 7121 "total": false, 7122 "values": false 7123 }, 7124 "lines": true, 7125 "linewidth": 2, 7126 "links": [], 7127 "maxDataPoints": 100, 7128 "maxPerRow": null, 7129 "minSpan": null, 7130 "nullPointMode": null, 7131 "options": { 7132 "alertThreshold": true, 7133 "dataLinks": [] 7134 }, 7135 "percentage": false, 7136 "pointradius": 5, 7137 "points": false, 7138 "renderer": "flot", 7139 "repeat": null, 7140 "repeatDirection": null, 7141 "seriesOverrides": [], 7142 "span": 12, 7143 "stack": false, 7144 "steppedLine": false, 7145 "targets": [ 7146 { 7147 "datasource": "$source", 7148 "expr": "apiserver:apiserver_request_latency:histogram_quantile{quantile=\"0.50\", verb!=\"WATCH\", verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}", 7149 "format": "time_series", 7150 "hide": false, 7151 "instant": false, 7152 "interval": "5s", 7153 "intervalFactor": 1, 7154 "legendFormat": "", 7155 "metric": "", 7156 "query": "apiserver:apiserver_request_latency:histogram_quantile{quantile=\"0.50\", verb!=\"WATCH\", verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}", 7157 "refId": "", 7158 "step": 10, 7159 "target": "" 7160 } 7161 ], 7162 "thresholds": [], 7163 "timeFrom": null, 7164 "timeShift": null, 7165 "title": "Request latency (50th percentile) (excl. WATCH)", 7166 "tooltip": { 7167 "msResolution": true, 7168 "shared": true, 7169 "sort": 2, 7170 "value_type": "cumulative" 7171 }, 7172 "transformations": [], 7173 "transparent": false, 7174 "type": "graph", 7175 "xaxis": { 7176 "mode": "time", 7177 "name": null, 7178 "show": true, 7179 "values": [] 7180 }, 7181 "yaxes": [ 7182 { 7183 "decimals": null, 7184 "format": "s", 7185 "label": null, 7186 "logBase": 1, 7187 "max": null, 7188 "min": null, 7189 "show": true 7190 }, 7191 { 7192 "decimals": null, 7193 "format": "short", 7194 "label": null, 7195 "logBase": 1, 7196 "max": null, 7197 "min": null, 7198 "show": true 7199 } 7200 ], 7201 "yaxis": { 7202 "align": false, 7203 "alignLevel": 0 7204 } 7205 }, 7206 { 7207 "aliasColors": {}, 7208 "bars": false, 7209 "cacheTimeout": null, 7210 "datasource": "$source", 7211 "description": null, 7212 "editable": true, 7213 "error": false, 7214 "fieldConfig": { 7215 "defaults": { 7216 "thresholds": { 7217 "mode": "absolute", 7218 "steps": [] 7219 } 7220 } 7221 }, 7222 "fill": 1, 7223 "grid": { 7224 "threshold1": null, 7225 "threshold1Color": "rgba(216, 200, 27, 0.27)", 7226 "threshold2": null, 7227 "threshold2Color": "rgba(234, 112, 112, 0.22)" 7228 }, 7229 "gridPos": null, 7230 "height": null, 7231 "hideTimeOverride": false, 7232 "id": 56, 7233 "interval": null, 7234 "isNew": true, 7235 "legend": { 7236 "alignAsTable": false, 7237 "avg": false, 7238 "current": false, 7239 "hideEmpty": false, 7240 "hideZero": false, 7241 "max": false, 7242 "min": false, 7243 "rightSide": false, 7244 "show": true, 7245 "sideWidth": null, 7246 "sort": null, 7247 "sortDesc": false, 7248 "total": false, 7249 "values": false 7250 }, 7251 "lines": true, 7252 "linewidth": 2, 7253 "links": [], 7254 "maxDataPoints": 100, 7255 "maxPerRow": null, 7256 "minSpan": null, 7257 "nullPointMode": null, 7258 "options": { 7259 "alertThreshold": true, 7260 "dataLinks": [] 7261 }, 7262 "percentage": false, 7263 "pointradius": 5, 7264 "points": false, 7265 "renderer": "flot", 7266 "repeat": null, 7267 "repeatDirection": null, 7268 "seriesOverrides": [], 7269 "span": 12, 7270 "stack": false, 7271 "steppedLine": false, 7272 "targets": [ 7273 { 7274 "datasource": "$source", 7275 "expr": "apiserver:apiserver_request_latency:histogram_quantile{quantile=\"0.99\", verb!=\"WATCH\", verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}", 7276 "format": "time_series", 7277 "hide": false, 7278 "instant": false, 7279 "interval": "5s", 7280 "intervalFactor": 1, 7281 "legendFormat": "", 7282 "metric": "", 7283 "query": "apiserver:apiserver_request_latency:histogram_quantile{quantile=\"0.99\", verb!=\"WATCH\", verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}", 7284 "refId": "", 7285 "step": 10, 7286 "target": "" 7287 } 7288 ], 7289 "thresholds": [], 7290 "timeFrom": null, 7291 "timeShift": null, 7292 "title": "Request latency (99th percentile) (excl. WATCH)", 7293 "tooltip": { 7294 "msResolution": true, 7295 "shared": true, 7296 "sort": 2, 7297 "value_type": "cumulative" 7298 }, 7299 "transformations": [], 7300 "transparent": false, 7301 "type": "graph", 7302 "xaxis": { 7303 "mode": "time", 7304 "name": null, 7305 "show": true, 7306 "values": [] 7307 }, 7308 "yaxes": [ 7309 { 7310 "decimals": null, 7311 "format": "s", 7312 "label": null, 7313 "logBase": 1, 7314 "max": null, 7315 "min": null, 7316 "show": true 7317 }, 7318 { 7319 "decimals": null, 7320 "format": "short", 7321 "label": null, 7322 "logBase": 1, 7323 "max": null, 7324 "min": null, 7325 "show": true 7326 } 7327 ], 7328 "yaxis": { 7329 "align": false, 7330 "alignLevel": 0 7331 } 7332 }, 7333 { 7334 "aliasColors": {}, 7335 "bars": false, 7336 "cacheTimeout": null, 7337 "datasource": "$source", 7338 "description": null, 7339 "editable": true, 7340 "error": false, 7341 "fieldConfig": { 7342 "defaults": { 7343 "thresholds": { 7344 "mode": "absolute", 7345 "steps": [] 7346 } 7347 } 7348 }, 7349 "fill": 1, 7350 "grid": { 7351 "threshold1": null, 7352 "threshold1Color": "rgba(216, 200, 27, 0.27)", 7353 "threshold2": null, 7354 "threshold2Color": "rgba(234, 112, 112, 0.22)" 7355 }, 7356 "gridPos": null, 7357 "height": null, 7358 "hideTimeOverride": false, 7359 "id": 57, 7360 "interval": null, 7361 "isNew": true, 7362 "legend": { 7363 "alignAsTable": false, 7364 "avg": false, 7365 "current": false, 7366 "hideEmpty": false, 7367 "hideZero": false, 7368 "max": false, 7369 "min": false, 7370 "rightSide": false, 7371 "show": true, 7372 "sideWidth": null, 7373 "sort": null, 7374 "sortDesc": false, 7375 "total": false, 7376 "values": false 7377 }, 7378 "lines": true, 7379 "linewidth": 2, 7380 "links": [], 7381 "maxDataPoints": 100, 7382 "maxPerRow": null, 7383 "minSpan": null, 7384 "nullPointMode": null, 7385 "options": { 7386 "alertThreshold": true, 7387 "dataLinks": [] 7388 }, 7389 "percentage": false, 7390 "pointradius": 5, 7391 "points": false, 7392 "renderer": "flot", 7393 "repeat": null, 7394 "repeatDirection": null, 7395 "seriesOverrides": [], 7396 "span": 12, 7397 "stack": false, 7398 "steppedLine": false, 7399 "targets": [ 7400 { 7401 "datasource": "$source", 7402 "expr": "sum(rate(apiserver_response_sizes_sum{verb!=\"WATCH\", verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}[1m])) by (verb, version, resource, subresource, scope, instance)", 7403 "format": "time_series", 7404 "hide": false, 7405 "instant": false, 7406 "interval": "5s", 7407 "intervalFactor": 1, 7408 "legendFormat": "", 7409 "metric": "", 7410 "query": "sum(rate(apiserver_response_sizes_sum{verb!=\"WATCH\", verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}[1m])) by (verb, version, resource, subresource, scope, instance)", 7411 "refId": "", 7412 "step": 10, 7413 "target": "" 7414 } 7415 ], 7416 "thresholds": [], 7417 "timeFrom": null, 7418 "timeShift": null, 7419 "title": "Traffic (excl. WATCH)", 7420 "tooltip": { 7421 "msResolution": true, 7422 "shared": true, 7423 "sort": 2, 7424 "value_type": "cumulative" 7425 }, 7426 "transformations": [], 7427 "transparent": false, 7428 "type": "graph", 7429 "xaxis": { 7430 "mode": "time", 7431 "name": null, 7432 "show": true, 7433 "values": [] 7434 }, 7435 "yaxes": [ 7436 { 7437 "decimals": null, 7438 "format": "Bps", 7439 "label": null, 7440 "logBase": 1, 7441 "max": null, 7442 "min": null, 7443 "show": true 7444 }, 7445 { 7446 "decimals": null, 7447 "format": "short", 7448 "label": null, 7449 "logBase": 1, 7450 "max": null, 7451 "min": null, 7452 "show": true 7453 } 7454 ], 7455 "yaxis": { 7456 "align": false, 7457 "alignLevel": 0 7458 } 7459 }, 7460 { 7461 "aliasColors": {}, 7462 "bars": false, 7463 "cacheTimeout": null, 7464 "datasource": "$source", 7465 "description": null, 7466 "editable": true, 7467 "error": false, 7468 "fieldConfig": { 7469 "defaults": { 7470 "thresholds": { 7471 "mode": "absolute", 7472 "steps": [] 7473 } 7474 } 7475 }, 7476 "fill": 1, 7477 "grid": { 7478 "threshold1": null, 7479 "threshold1Color": "rgba(216, 200, 27, 0.27)", 7480 "threshold2": null, 7481 "threshold2Color": "rgba(234, 112, 112, 0.22)" 7482 }, 7483 "gridPos": null, 7484 "height": null, 7485 "hideTimeOverride": false, 7486 "id": 58, 7487 "interval": null, 7488 "isNew": true, 7489 "legend": { 7490 "alignAsTable": false, 7491 "avg": false, 7492 "current": false, 7493 "hideEmpty": false, 7494 "hideZero": false, 7495 "max": false, 7496 "min": false, 7497 "rightSide": false, 7498 "show": true, 7499 "sideWidth": null, 7500 "sort": null, 7501 "sortDesc": false, 7502 "total": false, 7503 "values": false 7504 }, 7505 "lines": true, 7506 "linewidth": 2, 7507 "links": [], 7508 "maxDataPoints": 100, 7509 "maxPerRow": null, 7510 "minSpan": null, 7511 "nullPointMode": null, 7512 "options": { 7513 "alertThreshold": true, 7514 "dataLinks": [] 7515 }, 7516 "percentage": false, 7517 "pointradius": 5, 7518 "points": false, 7519 "renderer": "flot", 7520 "repeat": null, 7521 "repeatDirection": null, 7522 "seriesOverrides": [], 7523 "span": 12, 7524 "stack": false, 7525 "steppedLine": false, 7526 "targets": [ 7527 { 7528 "datasource": "$source", 7529 "expr": "histogram_quantile(0.99, sum(rate(apiserver_admission_webhook_admission_duration_seconds_bucket[1m])) by (le, type, name))", 7530 "format": "time_series", 7531 "hide": false, 7532 "instant": false, 7533 "interval": "5s", 7534 "intervalFactor": 1, 7535 "legendFormat": "{{type}}: {{name}}", 7536 "metric": "", 7537 "query": "histogram_quantile(0.99, sum(rate(apiserver_admission_webhook_admission_duration_seconds_bucket[1m])) by (le, type, name))", 7538 "refId": "", 7539 "step": 10, 7540 "target": "" 7541 } 7542 ], 7543 "thresholds": [], 7544 "timeFrom": null, 7545 "timeShift": null, 7546 "title": "Webhook admission duration (99th percentile)", 7547 "tooltip": { 7548 "msResolution": true, 7549 "shared": true, 7550 "sort": 2, 7551 "value_type": "cumulative" 7552 }, 7553 "transformations": [], 7554 "transparent": false, 7555 "type": "graph", 7556 "xaxis": { 7557 "mode": "time", 7558 "name": null, 7559 "show": true, 7560 "values": [] 7561 }, 7562 "yaxes": [ 7563 { 7564 "decimals": null, 7565 "format": "s", 7566 "label": null, 7567 "logBase": 1, 7568 "max": null, 7569 "min": null, 7570 "show": true 7571 }, 7572 { 7573 "decimals": null, 7574 "format": "short", 7575 "label": null, 7576 "logBase": 1, 7577 "max": null, 7578 "min": null, 7579 "show": true 7580 } 7581 ], 7582 "yaxis": { 7583 "align": false, 7584 "alignLevel": 0 7585 } 7586 }, 7587 { 7588 "aliasColors": {}, 7589 "bars": false, 7590 "cacheTimeout": null, 7591 "datasource": "$source", 7592 "description": null, 7593 "editable": true, 7594 "error": false, 7595 "fieldConfig": { 7596 "defaults": { 7597 "thresholds": { 7598 "mode": "absolute", 7599 "steps": [] 7600 } 7601 } 7602 }, 7603 "fill": 1, 7604 "grid": { 7605 "threshold1": null, 7606 "threshold1Color": "rgba(216, 200, 27, 0.27)", 7607 "threshold2": null, 7608 "threshold2Color": "rgba(234, 112, 112, 0.22)" 7609 }, 7610 "gridPos": null, 7611 "height": null, 7612 "hideTimeOverride": false, 7613 "id": 59, 7614 "interval": null, 7615 "isNew": true, 7616 "legend": { 7617 "alignAsTable": false, 7618 "avg": false, 7619 "current": false, 7620 "hideEmpty": false, 7621 "hideZero": false, 7622 "max": false, 7623 "min": false, 7624 "rightSide": false, 7625 "show": true, 7626 "sideWidth": null, 7627 "sort": null, 7628 "sortDesc": false, 7629 "total": false, 7630 "values": false 7631 }, 7632 "lines": true, 7633 "linewidth": 2, 7634 "links": [], 7635 "maxDataPoints": 100, 7636 "maxPerRow": null, 7637 "minSpan": null, 7638 "nullPointMode": null, 7639 "options": { 7640 "alertThreshold": true, 7641 "dataLinks": [] 7642 }, 7643 "percentage": false, 7644 "pointradius": 5, 7645 "points": false, 7646 "renderer": "flot", 7647 "repeat": null, 7648 "repeatDirection": null, 7649 "seriesOverrides": [], 7650 "span": 12, 7651 "stack": false, 7652 "steppedLine": false, 7653 "targets": [ 7654 { 7655 "datasource": "$source", 7656 "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_filter_duration_seconds_bucket[1m])) by (le, filter))", 7657 "format": "time_series", 7658 "hide": false, 7659 "instant": false, 7660 "interval": "5s", 7661 "intervalFactor": 1, 7662 "legendFormat": "{{filter}}", 7663 "metric": "", 7664 "query": "histogram_quantile(0.99, sum(rate(apiserver_request_filter_duration_seconds_bucket[1m])) by (le, filter))", 7665 "refId": "", 7666 "step": 10, 7667 "target": "" 7668 } 7669 ], 7670 "thresholds": [], 7671 "timeFrom": null, 7672 "timeShift": null, 7673 "title": "Request filter latency for each filter type (99th percentile)", 7674 "tooltip": { 7675 "msResolution": true, 7676 "shared": true, 7677 "sort": 2, 7678 "value_type": "cumulative" 7679 }, 7680 "transformations": [], 7681 "transparent": false, 7682 "type": "graph", 7683 "xaxis": { 7684 "mode": "time", 7685 "name": null, 7686 "show": true, 7687 "values": [] 7688 }, 7689 "yaxes": [ 7690 { 7691 "decimals": null, 7692 "format": "s", 7693 "label": null, 7694 "logBase": 1, 7695 "max": null, 7696 "min": null, 7697 "show": true 7698 }, 7699 { 7700 "decimals": null, 7701 "format": "short", 7702 "label": null, 7703 "logBase": 1, 7704 "max": null, 7705 "min": null, 7706 "show": true 7707 } 7708 ], 7709 "yaxis": { 7710 "align": false, 7711 "alignLevel": 0 7712 } 7713 }, 7714 { 7715 "aliasColors": {}, 7716 "bars": false, 7717 "cacheTimeout": null, 7718 "datasource": "$source", 7719 "description": null, 7720 "editable": true, 7721 "error": false, 7722 "fieldConfig": { 7723 "defaults": { 7724 "thresholds": { 7725 "mode": "absolute", 7726 "steps": [] 7727 } 7728 } 7729 }, 7730 "fill": 1, 7731 "grid": { 7732 "threshold1": null, 7733 "threshold1Color": "rgba(216, 200, 27, 0.27)", 7734 "threshold2": null, 7735 "threshold2Color": "rgba(234, 112, 112, 0.22)" 7736 }, 7737 "gridPos": null, 7738 "height": null, 7739 "hideTimeOverride": false, 7740 "id": 60, 7741 "interval": null, 7742 "isNew": true, 7743 "legend": { 7744 "alignAsTable": false, 7745 "avg": false, 7746 "current": false, 7747 "hideEmpty": false, 7748 "hideZero": false, 7749 "max": false, 7750 "min": false, 7751 "rightSide": false, 7752 "show": true, 7753 "sideWidth": null, 7754 "sort": null, 7755 "sortDesc": false, 7756 "total": false, 7757 "values": false 7758 }, 7759 "lines": true, 7760 "linewidth": 2, 7761 "links": [], 7762 "maxDataPoints": 100, 7763 "maxPerRow": null, 7764 "minSpan": null, 7765 "nullPointMode": null, 7766 "options": { 7767 "alertThreshold": true, 7768 "dataLinks": [] 7769 }, 7770 "percentage": false, 7771 "pointradius": 5, 7772 "points": false, 7773 "renderer": "flot", 7774 "repeat": null, 7775 "repeatDirection": null, 7776 "seriesOverrides": [], 7777 "span": 12, 7778 "stack": false, 7779 "steppedLine": false, 7780 "targets": [ 7781 { 7782 "datasource": "$source", 7783 "expr": "sum(rate(rest_client_requests_total{endpoint=\"apiserver\", code!=\"200\", host!=\"[::1]:443\"}[1m])) by (code, instance, method)", 7784 "format": "time_series", 7785 "hide": false, 7786 "instant": false, 7787 "interval": "5s", 7788 "intervalFactor": 1, 7789 "legendFormat": "{{instance}}: {{code}} {{method}}", 7790 "metric": "", 7791 "query": "sum(rate(rest_client_requests_total{endpoint=\"apiserver\", code!=\"200\", host!=\"[::1]:443\"}[1m])) by (code, instance, method)", 7792 "refId": "", 7793 "step": 10, 7794 "target": "" 7795 } 7796 ], 7797 "thresholds": [], 7798 "timeFrom": null, 7799 "timeShift": null, 7800 "title": "Failed external requests", 7801 "tooltip": { 7802 "msResolution": true, 7803 "shared": true, 7804 "sort": 2, 7805 "value_type": "cumulative" 7806 }, 7807 "transformations": [], 7808 "transparent": false, 7809 "type": "graph", 7810 "xaxis": { 7811 "mode": "time", 7812 "name": null, 7813 "show": true, 7814 "values": [] 7815 }, 7816 "yaxes": [ 7817 { 7818 "decimals": null, 7819 "format": "short", 7820 "label": null, 7821 "logBase": 1, 7822 "max": null, 7823 "min": null, 7824 "show": true 7825 }, 7826 { 7827 "decimals": null, 7828 "format": "short", 7829 "label": null, 7830 "logBase": 1, 7831 "max": null, 7832 "min": null, 7833 "show": true 7834 } 7835 ], 7836 "yaxis": { 7837 "align": false, 7838 "alignLevel": 0 7839 } 7840 }, 7841 { 7842 "aliasColors": {}, 7843 "bars": false, 7844 "cacheTimeout": null, 7845 "datasource": "$source", 7846 "description": null, 7847 "editable": true, 7848 "error": false, 7849 "fieldConfig": { 7850 "defaults": { 7851 "thresholds": { 7852 "mode": "absolute", 7853 "steps": [] 7854 } 7855 } 7856 }, 7857 "fill": 1, 7858 "grid": { 7859 "threshold1": null, 7860 "threshold1Color": "rgba(216, 200, 27, 0.27)", 7861 "threshold2": null, 7862 "threshold2Color": "rgba(234, 112, 112, 0.22)" 7863 }, 7864 "gridPos": null, 7865 "height": null, 7866 "hideTimeOverride": false, 7867 "id": 61, 7868 "interval": null, 7869 "isNew": true, 7870 "legend": { 7871 "alignAsTable": false, 7872 "avg": false, 7873 "current": false, 7874 "hideEmpty": false, 7875 "hideZero": false, 7876 "max": false, 7877 "min": false, 7878 "rightSide": false, 7879 "show": true, 7880 "sideWidth": null, 7881 "sort": null, 7882 "sortDesc": false, 7883 "total": false, 7884 "values": false 7885 }, 7886 "lines": true, 7887 "linewidth": 2, 7888 "links": [], 7889 "maxDataPoints": 100, 7890 "maxPerRow": null, 7891 "minSpan": null, 7892 "nullPointMode": null, 7893 "options": { 7894 "alertThreshold": true, 7895 "dataLinks": [] 7896 }, 7897 "percentage": false, 7898 "pointradius": 5, 7899 "points": false, 7900 "renderer": "flot", 7901 "repeat": null, 7902 "repeatDirection": null, 7903 "seriesOverrides": [], 7904 "span": 12, 7905 "stack": false, 7906 "steppedLine": false, 7907 "targets": [ 7908 { 7909 "datasource": "$source", 7910 "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{endpoint=\"apiserver\", host!=\"[::1]:443\"}[1m])) by (verb, host, instance, le))", 7911 "format": "time_series", 7912 "hide": false, 7913 "instant": false, 7914 "interval": "5s", 7915 "intervalFactor": 1, 7916 "legendFormat": "{{instance}}: {{verb}} {{host}}", 7917 "metric": "", 7918 "query": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{endpoint=\"apiserver\", host!=\"[::1]:443\"}[1m])) by (verb, host, instance, le))", 7919 "refId": "", 7920 "step": 10, 7921 "target": "" 7922 } 7923 ], 7924 "thresholds": [], 7925 "timeFrom": null, 7926 "timeShift": null, 7927 "title": "Extrernal requests latency (99th percentile)", 7928 "tooltip": { 7929 "msResolution": true, 7930 "shared": true, 7931 "sort": 2, 7932 "value_type": "cumulative" 7933 }, 7934 "transformations": [], 7935 "transparent": false, 7936 "type": "graph", 7937 "xaxis": { 7938 "mode": "time", 7939 "name": null, 7940 "show": true, 7941 "values": [] 7942 }, 7943 "yaxes": [ 7944 { 7945 "decimals": null, 7946 "format": "s", 7947 "label": null, 7948 "logBase": 1, 7949 "max": null, 7950 "min": null, 7951 "show": true 7952 }, 7953 { 7954 "decimals": null, 7955 "format": "short", 7956 "label": null, 7957 "logBase": 1, 7958 "max": null, 7959 "min": null, 7960 "show": true 7961 } 7962 ], 7963 "yaxis": { 7964 "align": false, 7965 "alignLevel": 0 7966 } 7967 } 7968 ], 7969 "repeat": null, 7970 "showTitle": true, 7971 "title": "kube-apiserver" 7972 }, 7973 { 7974 "collapse": true, 7975 "editable": true, 7976 "height": "300px", 7977 "panels": [ 7978 { 7979 "aliasColors": {}, 7980 "bars": false, 7981 "cacheTimeout": null, 7982 "datasource": "$source", 7983 "description": null, 7984 "editable": true, 7985 "error": false, 7986 "fieldConfig": { 7987 "defaults": { 7988 "thresholds": { 7989 "mode": "absolute", 7990 "steps": [] 7991 } 7992 } 7993 }, 7994 "fill": 1, 7995 "grid": { 7996 "threshold1": null, 7997 "threshold1Color": "rgba(216, 200, 27, 0.27)", 7998 "threshold2": null, 7999 "threshold2Color": "rgba(234, 112, 112, 0.22)" 8000 }, 8001 "gridPos": null, 8002 "height": null, 8003 "hideTimeOverride": false, 8004 "id": 62, 8005 "interval": null, 8006 "isNew": true, 8007 "legend": { 8008 "alignAsTable": false, 8009 "avg": false, 8010 "current": false, 8011 "hideEmpty": false, 8012 "hideZero": false, 8013 "max": false, 8014 "min": false, 8015 "rightSide": false, 8016 "show": true, 8017 "sideWidth": null, 8018 "sort": null, 8019 "sortDesc": false, 8020 "total": false, 8021 "values": false 8022 }, 8023 "lines": true, 8024 "linewidth": 2, 8025 "links": [], 8026 "maxDataPoints": 100, 8027 "maxPerRow": null, 8028 "minSpan": null, 8029 "nullPointMode": null, 8030 "options": { 8031 "alertThreshold": true, 8032 "dataLinks": [] 8033 }, 8034 "percentage": false, 8035 "pointradius": 5, 8036 "points": false, 8037 "renderer": "flot", 8038 "repeat": null, 8039 "repeatDirection": null, 8040 "seriesOverrides": [], 8041 "span": 12, 8042 "stack": false, 8043 "steppedLine": false, 8044 "targets": [ 8045 { 8046 "datasource": "$source", 8047 "expr": "workqueue_depth{endpoint=\"kube-controller-manager\"}", 8048 "format": "time_series", 8049 "hide": false, 8050 "instant": false, 8051 "interval": "5s", 8052 "intervalFactor": 1, 8053 "legendFormat": "{{name}}", 8054 "metric": "", 8055 "query": "workqueue_depth{endpoint=\"kube-controller-manager\"}", 8056 "refId": "", 8057 "step": 10, 8058 "target": "" 8059 } 8060 ], 8061 "thresholds": [], 8062 "timeFrom": null, 8063 "timeShift": null, 8064 "title": "Workqueue depths", 8065 "tooltip": { 8066 "msResolution": true, 8067 "shared": true, 8068 "sort": 2, 8069 "value_type": "cumulative" 8070 }, 8071 "transformations": [], 8072 "transparent": false, 8073 "type": "graph", 8074 "xaxis": { 8075 "mode": "time", 8076 "name": null, 8077 "show": true, 8078 "values": [] 8079 }, 8080 "yaxes": [ 8081 { 8082 "decimals": null, 8083 "format": "short", 8084 "label": null, 8085 "logBase": 1, 8086 "max": null, 8087 "min": null, 8088 "show": true 8089 }, 8090 { 8091 "decimals": null, 8092 "format": "short", 8093 "label": null, 8094 "logBase": 1, 8095 "max": null, 8096 "min": null, 8097 "show": true 8098 } 8099 ], 8100 "yaxis": { 8101 "align": false, 8102 "alignLevel": 0 8103 } 8104 } 8105 ], 8106 "repeat": null, 8107 "showTitle": true, 8108 "title": "kube-controller-manager" 8109 }, 8110 { 8111 "collapse": true, 8112 "editable": true, 8113 "height": "300px", 8114 "panels": [ 8115 { 8116 "aliasColors": {}, 8117 "bars": false, 8118 "cacheTimeout": null, 8119 "datasource": "$source", 8120 "description": null, 8121 "editable": true, 8122 "error": false, 8123 "fieldConfig": { 8124 "defaults": { 8125 "thresholds": { 8126 "mode": "absolute", 8127 "steps": [] 8128 } 8129 } 8130 }, 8131 "fill": 1, 8132 "grid": { 8133 "threshold1": null, 8134 "threshold1Color": "rgba(216, 200, 27, 0.27)", 8135 "threshold2": null, 8136 "threshold2Color": "rgba(234, 112, 112, 0.22)" 8137 }, 8138 "gridPos": null, 8139 "height": null, 8140 "hideTimeOverride": false, 8141 "id": 63, 8142 "interval": null, 8143 "isNew": true, 8144 "legend": { 8145 "alignAsTable": false, 8146 "avg": false, 8147 "current": false, 8148 "hideEmpty": false, 8149 "hideZero": false, 8150 "max": false, 8151 "min": false, 8152 "rightSide": false, 8153 "show": true, 8154 "sideWidth": null, 8155 "sort": null, 8156 "sortDesc": false, 8157 "total": false, 8158 "values": false 8159 }, 8160 "lines": true, 8161 "linewidth": 2, 8162 "links": [], 8163 "maxDataPoints": 100, 8164 "maxPerRow": null, 8165 "minSpan": null, 8166 "nullPointMode": null, 8167 "options": { 8168 "alertThreshold": true, 8169 "dataLinks": [] 8170 }, 8171 "percentage": false, 8172 "pointradius": 5, 8173 "points": false, 8174 "renderer": "flot", 8175 "repeat": null, 8176 "repeatDirection": null, 8177 "seriesOverrides": [], 8178 "span": 12, 8179 "stack": false, 8180 "steppedLine": false, 8181 "targets": [ 8182 { 8183 "datasource": "$source", 8184 "expr": "sum(rate(container_fs_reads_bytes_total[1m])) by (container, instance)", 8185 "format": "time_series", 8186 "hide": false, 8187 "instant": false, 8188 "interval": "5s", 8189 "intervalFactor": 1, 8190 "legendFormat": "{{instance}}: {{container}}", 8191 "metric": "", 8192 "query": "sum(rate(container_fs_reads_bytes_total[1m])) by (container, instance)", 8193 "refId": "", 8194 "step": 10, 8195 "target": "" 8196 } 8197 ], 8198 "thresholds": [], 8199 "timeFrom": null, 8200 "timeShift": null, 8201 "title": "fs bytes reads by container", 8202 "tooltip": { 8203 "msResolution": true, 8204 "shared": true, 8205 "sort": 2, 8206 "value_type": "cumulative" 8207 }, 8208 "transformations": [], 8209 "transparent": false, 8210 "type": "graph", 8211 "xaxis": { 8212 "mode": "time", 8213 "name": null, 8214 "show": true, 8215 "values": [] 8216 }, 8217 "yaxes": [ 8218 { 8219 "decimals": null, 8220 "format": "bytes", 8221 "label": null, 8222 "logBase": 1, 8223 "max": null, 8224 "min": null, 8225 "show": true 8226 }, 8227 { 8228 "decimals": null, 8229 "format": "short", 8230 "label": null, 8231 "logBase": 1, 8232 "max": null, 8233 "min": null, 8234 "show": true 8235 } 8236 ], 8237 "yaxis": { 8238 "align": false, 8239 "alignLevel": 0 8240 } 8241 }, 8242 { 8243 "aliasColors": {}, 8244 "bars": false, 8245 "cacheTimeout": null, 8246 "datasource": "$source", 8247 "description": null, 8248 "editable": true, 8249 "error": false, 8250 "fieldConfig": { 8251 "defaults": { 8252 "thresholds": { 8253 "mode": "absolute", 8254 "steps": [] 8255 } 8256 } 8257 }, 8258 "fill": 1, 8259 "grid": { 8260 "threshold1": null, 8261 "threshold1Color": "rgba(216, 200, 27, 0.27)", 8262 "threshold2": null, 8263 "threshold2Color": "rgba(234, 112, 112, 0.22)" 8264 }, 8265 "gridPos": null, 8266 "height": null, 8267 "hideTimeOverride": false, 8268 "id": 64, 8269 "interval": null, 8270 "isNew": true, 8271 "legend": { 8272 "alignAsTable": false, 8273 "avg": false, 8274 "current": false, 8275 "hideEmpty": false, 8276 "hideZero": false, 8277 "max": false, 8278 "min": false, 8279 "rightSide": false, 8280 "show": true, 8281 "sideWidth": null, 8282 "sort": null, 8283 "sortDesc": false, 8284 "total": false, 8285 "values": false 8286 }, 8287 "lines": true, 8288 "linewidth": 2, 8289 "links": [], 8290 "maxDataPoints": 100, 8291 "maxPerRow": null, 8292 "minSpan": null, 8293 "nullPointMode": null, 8294 "options": { 8295 "alertThreshold": true, 8296 "dataLinks": [] 8297 }, 8298 "percentage": false, 8299 "pointradius": 5, 8300 "points": false, 8301 "renderer": "flot", 8302 "repeat": null, 8303 "repeatDirection": null, 8304 "seriesOverrides": [], 8305 "span": 12, 8306 "stack": false, 8307 "steppedLine": false, 8308 "targets": [ 8309 { 8310 "datasource": "$source", 8311 "expr": "sum(rate(container_fs_reads_total[1m])) by (container, instance)", 8312 "format": "time_series", 8313 "hide": false, 8314 "instant": false, 8315 "interval": "5s", 8316 "intervalFactor": 1, 8317 "legendFormat": "{{instance}}: {{container}}", 8318 "metric": "", 8319 "query": "sum(rate(container_fs_reads_total[1m])) by (container, instance)", 8320 "refId": "", 8321 "step": 10, 8322 "target": "" 8323 } 8324 ], 8325 "thresholds": [], 8326 "timeFrom": null, 8327 "timeShift": null, 8328 "title": "fs reads by container", 8329 "tooltip": { 8330 "msResolution": true, 8331 "shared": true, 8332 "sort": 2, 8333 "value_type": "cumulative" 8334 }, 8335 "transformations": [], 8336 "transparent": false, 8337 "type": "graph", 8338 "xaxis": { 8339 "mode": "time", 8340 "name": null, 8341 "show": true, 8342 "values": [] 8343 }, 8344 "yaxes": [ 8345 { 8346 "decimals": null, 8347 "format": "short", 8348 "label": null, 8349 "logBase": 1, 8350 "max": null, 8351 "min": null, 8352 "show": true 8353 }, 8354 { 8355 "decimals": null, 8356 "format": "short", 8357 "label": null, 8358 "logBase": 1, 8359 "max": null, 8360 "min": null, 8361 "show": true 8362 } 8363 ], 8364 "yaxis": { 8365 "align": false, 8366 "alignLevel": 0 8367 } 8368 }, 8369 { 8370 "aliasColors": {}, 8371 "bars": false, 8372 "cacheTimeout": null, 8373 "datasource": "$source", 8374 "description": null, 8375 "editable": true, 8376 "error": false, 8377 "fieldConfig": { 8378 "defaults": { 8379 "thresholds": { 8380 "mode": "absolute", 8381 "steps": [] 8382 } 8383 } 8384 }, 8385 "fill": 1, 8386 "grid": { 8387 "threshold1": null, 8388 "threshold1Color": "rgba(216, 200, 27, 0.27)", 8389 "threshold2": null, 8390 "threshold2Color": "rgba(234, 112, 112, 0.22)" 8391 }, 8392 "gridPos": null, 8393 "height": null, 8394 "hideTimeOverride": false, 8395 "id": 65, 8396 "interval": null, 8397 "isNew": true, 8398 "legend": { 8399 "alignAsTable": false, 8400 "avg": false, 8401 "current": false, 8402 "hideEmpty": false, 8403 "hideZero": false, 8404 "max": false, 8405 "min": false, 8406 "rightSide": false, 8407 "show": true, 8408 "sideWidth": null, 8409 "sort": null, 8410 "sortDesc": false, 8411 "total": false, 8412 "values": false 8413 }, 8414 "lines": true, 8415 "linewidth": 2, 8416 "links": [], 8417 "maxDataPoints": 100, 8418 "maxPerRow": null, 8419 "minSpan": null, 8420 "nullPointMode": null, 8421 "options": { 8422 "alertThreshold": true, 8423 "dataLinks": [] 8424 }, 8425 "percentage": false, 8426 "pointradius": 5, 8427 "points": false, 8428 "renderer": "flot", 8429 "repeat": null, 8430 "repeatDirection": null, 8431 "seriesOverrides": [], 8432 "span": 12, 8433 "stack": false, 8434 "steppedLine": false, 8435 "targets": [ 8436 { 8437 "datasource": "$source", 8438 "expr": "sum(rate(container_fs_writes_bytes_total[1m])) by (container, instance)", 8439 "format": "time_series", 8440 "hide": false, 8441 "instant": false, 8442 "interval": "5s", 8443 "intervalFactor": 1, 8444 "legendFormat": "{{instance}}: {{container}}", 8445 "metric": "", 8446 "query": "sum(rate(container_fs_writes_bytes_total[1m])) by (container, instance)", 8447 "refId": "", 8448 "step": 10, 8449 "target": "" 8450 } 8451 ], 8452 "thresholds": [], 8453 "timeFrom": null, 8454 "timeShift": null, 8455 "title": "fs bytes writes by container", 8456 "tooltip": { 8457 "msResolution": true, 8458 "shared": true, 8459 "sort": 2, 8460 "value_type": "cumulative" 8461 }, 8462 "transformations": [], 8463 "transparent": false, 8464 "type": "graph", 8465 "xaxis": { 8466 "mode": "time", 8467 "name": null, 8468 "show": true, 8469 "values": [] 8470 }, 8471 "yaxes": [ 8472 { 8473 "decimals": null, 8474 "format": "bytes", 8475 "label": null, 8476 "logBase": 1, 8477 "max": null, 8478 "min": null, 8479 "show": true 8480 }, 8481 { 8482 "decimals": null, 8483 "format": "short", 8484 "label": null, 8485 "logBase": 1, 8486 "max": null, 8487 "min": null, 8488 "show": true 8489 } 8490 ], 8491 "yaxis": { 8492 "align": false, 8493 "alignLevel": 0 8494 } 8495 }, 8496 { 8497 "aliasColors": {}, 8498 "bars": false, 8499 "cacheTimeout": null, 8500 "datasource": "$source", 8501 "description": null, 8502 "editable": true, 8503 "error": false, 8504 "fieldConfig": { 8505 "defaults": { 8506 "thresholds": { 8507 "mode": "absolute", 8508 "steps": [] 8509 } 8510 } 8511 }, 8512 "fill": 1, 8513 "grid": { 8514 "threshold1": null, 8515 "threshold1Color": "rgba(216, 200, 27, 0.27)", 8516 "threshold2": null, 8517 "threshold2Color": "rgba(234, 112, 112, 0.22)" 8518 }, 8519 "gridPos": null, 8520 "height": null, 8521 "hideTimeOverride": false, 8522 "id": 66, 8523 "interval": null, 8524 "isNew": true, 8525 "legend": { 8526 "alignAsTable": false, 8527 "avg": false, 8528 "current": false, 8529 "hideEmpty": false, 8530 "hideZero": false, 8531 "max": false, 8532 "min": false, 8533 "rightSide": false, 8534 "show": true, 8535 "sideWidth": null, 8536 "sort": null, 8537 "sortDesc": false, 8538 "total": false, 8539 "values": false 8540 }, 8541 "lines": true, 8542 "linewidth": 2, 8543 "links": [], 8544 "maxDataPoints": 100, 8545 "maxPerRow": null, 8546 "minSpan": null, 8547 "nullPointMode": null, 8548 "options": { 8549 "alertThreshold": true, 8550 "dataLinks": [] 8551 }, 8552 "percentage": false, 8553 "pointradius": 5, 8554 "points": false, 8555 "renderer": "flot", 8556 "repeat": null, 8557 "repeatDirection": null, 8558 "seriesOverrides": [], 8559 "span": 12, 8560 "stack": false, 8561 "steppedLine": false, 8562 "targets": [ 8563 { 8564 "datasource": "$source", 8565 "expr": "sum(rate(container_fs_writes_total[1m])) by (container, instance)", 8566 "format": "time_series", 8567 "hide": false, 8568 "instant": false, 8569 "interval": "5s", 8570 "intervalFactor": 1, 8571 "legendFormat": "{{instance}}: {{container}}", 8572 "metric": "", 8573 "query": "sum(rate(container_fs_writes_total[1m])) by (container, instance)", 8574 "refId": "", 8575 "step": 10, 8576 "target": "" 8577 } 8578 ], 8579 "thresholds": [], 8580 "timeFrom": null, 8581 "timeShift": null, 8582 "title": "fs writes by container", 8583 "tooltip": { 8584 "msResolution": true, 8585 "shared": true, 8586 "sort": 2, 8587 "value_type": "cumulative" 8588 }, 8589 "transformations": [], 8590 "transparent": false, 8591 "type": "graph", 8592 "xaxis": { 8593 "mode": "time", 8594 "name": null, 8595 "show": true, 8596 "values": [] 8597 }, 8598 "yaxes": [ 8599 { 8600 "decimals": null, 8601 "format": "short", 8602 "label": null, 8603 "logBase": 1, 8604 "max": null, 8605 "min": null, 8606 "show": true 8607 }, 8608 { 8609 "decimals": null, 8610 "format": "short", 8611 "label": null, 8612 "logBase": 1, 8613 "max": null, 8614 "min": null, 8615 "show": true 8616 } 8617 ], 8618 "yaxis": { 8619 "align": false, 8620 "alignLevel": 0 8621 } 8622 }, 8623 { 8624 "aliasColors": {}, 8625 "bars": false, 8626 "cacheTimeout": null, 8627 "datasource": "$source", 8628 "description": null, 8629 "editable": true, 8630 "error": false, 8631 "fieldConfig": { 8632 "defaults": { 8633 "thresholds": { 8634 "mode": "absolute", 8635 "steps": [] 8636 } 8637 } 8638 }, 8639 "fill": 1, 8640 "grid": { 8641 "threshold1": null, 8642 "threshold1Color": "rgba(216, 200, 27, 0.27)", 8643 "threshold2": null, 8644 "threshold2Color": "rgba(234, 112, 112, 0.22)" 8645 }, 8646 "gridPos": null, 8647 "height": null, 8648 "hideTimeOverride": false, 8649 "id": 67, 8650 "interval": null, 8651 "isNew": true, 8652 "legend": { 8653 "alignAsTable": false, 8654 "avg": false, 8655 "current": false, 8656 "hideEmpty": false, 8657 "hideZero": false, 8658 "max": false, 8659 "min": false, 8660 "rightSide": false, 8661 "show": true, 8662 "sideWidth": null, 8663 "sort": null, 8664 "sortDesc": false, 8665 "total": false, 8666 "values": false 8667 }, 8668 "lines": true, 8669 "linewidth": 2, 8670 "links": [], 8671 "maxDataPoints": 100, 8672 "maxPerRow": null, 8673 "minSpan": null, 8674 "nullPointMode": null, 8675 "options": { 8676 "alertThreshold": true, 8677 "dataLinks": [] 8678 }, 8679 "percentage": false, 8680 "pointradius": 5, 8681 "points": false, 8682 "renderer": "flot", 8683 "repeat": null, 8684 "repeatDirection": null, 8685 "seriesOverrides": [], 8686 "span": 12, 8687 "stack": false, 8688 "steppedLine": false, 8689 "targets": [ 8690 { 8691 "datasource": "$source", 8692 "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"\"}[1m])) by (container, instance)", 8693 "format": "time_series", 8694 "hide": false, 8695 "instant": false, 8696 "interval": "5s", 8697 "intervalFactor": 1, 8698 "legendFormat": "{{instance}}: {{container}}", 8699 "metric": "", 8700 "query": "sum(rate(container_cpu_usage_seconds_total{container!=\"\"}[1m])) by (container, instance)", 8701 "refId": "", 8702 "step": 10, 8703 "target": "" 8704 }, 8705 { 8706 "datasource": "$source", 8707 "expr": "machine_cpu_cores", 8708 "format": "time_series", 8709 "hide": false, 8710 "instant": false, 8711 "interval": "5s", 8712 "intervalFactor": 1, 8713 "legendFormat": "limit", 8714 "metric": "", 8715 "query": "machine_cpu_cores", 8716 "refId": "", 8717 "step": 10, 8718 "target": "" 8719 } 8720 ], 8721 "thresholds": [], 8722 "timeFrom": null, 8723 "timeShift": null, 8724 "title": "CPU usage by container", 8725 "tooltip": { 8726 "msResolution": true, 8727 "shared": true, 8728 "sort": 2, 8729 "value_type": "cumulative" 8730 }, 8731 "transformations": [], 8732 "transparent": false, 8733 "type": "graph", 8734 "xaxis": { 8735 "mode": "time", 8736 "name": null, 8737 "show": true, 8738 "values": [] 8739 }, 8740 "yaxes": [ 8741 { 8742 "decimals": null, 8743 "format": "short", 8744 "label": null, 8745 "logBase": 1, 8746 "max": null, 8747 "min": null, 8748 "show": true 8749 }, 8750 { 8751 "decimals": null, 8752 "format": "short", 8753 "label": null, 8754 "logBase": 1, 8755 "max": null, 8756 "min": null, 8757 "show": true 8758 } 8759 ], 8760 "yaxis": { 8761 "align": false, 8762 "alignLevel": 0 8763 } 8764 }, 8765 { 8766 "aliasColors": {}, 8767 "bars": false, 8768 "cacheTimeout": null, 8769 "datasource": "$source", 8770 "description": null, 8771 "editable": true, 8772 "error": false, 8773 "fieldConfig": { 8774 "defaults": { 8775 "thresholds": { 8776 "mode": "absolute", 8777 "steps": [] 8778 } 8779 } 8780 }, 8781 "fill": 1, 8782 "grid": { 8783 "threshold1": null, 8784 "threshold1Color": "rgba(216, 200, 27, 0.27)", 8785 "threshold2": null, 8786 "threshold2Color": "rgba(234, 112, 112, 0.22)" 8787 }, 8788 "gridPos": null, 8789 "height": null, 8790 "hideTimeOverride": false, 8791 "id": 68, 8792 "interval": null, 8793 "isNew": true, 8794 "legend": { 8795 "alignAsTable": false, 8796 "avg": false, 8797 "current": false, 8798 "hideEmpty": false, 8799 "hideZero": false, 8800 "max": false, 8801 "min": false, 8802 "rightSide": false, 8803 "show": true, 8804 "sideWidth": null, 8805 "sort": null, 8806 "sortDesc": false, 8807 "total": false, 8808 "values": false 8809 }, 8810 "lines": true, 8811 "linewidth": 2, 8812 "links": [], 8813 "maxDataPoints": 100, 8814 "maxPerRow": null, 8815 "minSpan": null, 8816 "nullPointMode": null, 8817 "options": { 8818 "alertThreshold": true, 8819 "dataLinks": [] 8820 }, 8821 "percentage": false, 8822 "pointradius": 5, 8823 "points": false, 8824 "renderer": "flot", 8825 "repeat": null, 8826 "repeatDirection": null, 8827 "seriesOverrides": [], 8828 "span": 12, 8829 "stack": false, 8830 "steppedLine": false, 8831 "targets": [ 8832 { 8833 "datasource": "$source", 8834 "expr": "sum(container_memory_usage_bytes{container!=\"\"}) by (container, instance)", 8835 "format": "time_series", 8836 "hide": false, 8837 "instant": false, 8838 "interval": "5s", 8839 "intervalFactor": 1, 8840 "legendFormat": "{{instance}}: {{container}}", 8841 "metric": "", 8842 "query": "sum(container_memory_usage_bytes{container!=\"\"}) by (container, instance)", 8843 "refId": "", 8844 "step": 10, 8845 "target": "" 8846 }, 8847 { 8848 "datasource": "$source", 8849 "expr": "machine_memory_bytes", 8850 "format": "time_series", 8851 "hide": false, 8852 "instant": false, 8853 "interval": "5s", 8854 "intervalFactor": 1, 8855 "legendFormat": "limit", 8856 "metric": "", 8857 "query": "machine_memory_bytes", 8858 "refId": "", 8859 "step": 10, 8860 "target": "" 8861 } 8862 ], 8863 "thresholds": [], 8864 "timeFrom": null, 8865 "timeShift": null, 8866 "title": "memory usage by container", 8867 "tooltip": { 8868 "msResolution": true, 8869 "shared": true, 8870 "sort": 2, 8871 "value_type": "cumulative" 8872 }, 8873 "transformations": [], 8874 "transparent": false, 8875 "type": "graph", 8876 "xaxis": { 8877 "mode": "time", 8878 "name": null, 8879 "show": true, 8880 "values": [] 8881 }, 8882 "yaxes": [ 8883 { 8884 "decimals": null, 8885 "format": "bytes", 8886 "label": null, 8887 "logBase": 1, 8888 "max": null, 8889 "min": null, 8890 "show": true 8891 }, 8892 { 8893 "decimals": null, 8894 "format": "short", 8895 "label": null, 8896 "logBase": 1, 8897 "max": null, 8898 "min": null, 8899 "show": true 8900 } 8901 ], 8902 "yaxis": { 8903 "align": false, 8904 "alignLevel": 0 8905 } 8906 }, 8907 { 8908 "aliasColors": {}, 8909 "bars": false, 8910 "cacheTimeout": null, 8911 "datasource": "$source", 8912 "description": null, 8913 "editable": true, 8914 "error": false, 8915 "fieldConfig": { 8916 "defaults": { 8917 "thresholds": { 8918 "mode": "absolute", 8919 "steps": [] 8920 } 8921 } 8922 }, 8923 "fill": 1, 8924 "grid": { 8925 "threshold1": null, 8926 "threshold1Color": "rgba(216, 200, 27, 0.27)", 8927 "threshold2": null, 8928 "threshold2Color": "rgba(234, 112, 112, 0.22)" 8929 }, 8930 "gridPos": null, 8931 "height": null, 8932 "hideTimeOverride": false, 8933 "id": 69, 8934 "interval": null, 8935 "isNew": true, 8936 "legend": { 8937 "alignAsTable": false, 8938 "avg": false, 8939 "current": false, 8940 "hideEmpty": false, 8941 "hideZero": false, 8942 "max": false, 8943 "min": false, 8944 "rightSide": false, 8945 "show": true, 8946 "sideWidth": null, 8947 "sort": null, 8948 "sortDesc": false, 8949 "total": false, 8950 "values": false 8951 }, 8952 "lines": true, 8953 "linewidth": 2, 8954 "links": [], 8955 "maxDataPoints": 100, 8956 "maxPerRow": null, 8957 "minSpan": null, 8958 "nullPointMode": null, 8959 "options": { 8960 "alertThreshold": true, 8961 "dataLinks": [] 8962 }, 8963 "percentage": false, 8964 "pointradius": 5, 8965 "points": false, 8966 "renderer": "flot", 8967 "repeat": null, 8968 "repeatDirection": null, 8969 "seriesOverrides": [], 8970 "span": 12, 8971 "stack": false, 8972 "steppedLine": false, 8973 "targets": [ 8974 { 8975 "datasource": "$source", 8976 "expr": "sum(container_memory_working_set_bytes{container!=\"\"}) by (container, instance)", 8977 "format": "time_series", 8978 "hide": false, 8979 "instant": false, 8980 "interval": "5s", 8981 "intervalFactor": 1, 8982 "legendFormat": "{{instance}}: {{container}}", 8983 "metric": "", 8984 "query": "sum(container_memory_working_set_bytes{container!=\"\"}) by (container, instance)", 8985 "refId": "", 8986 "step": 10, 8987 "target": "" 8988 }, 8989 { 8990 "datasource": "$source", 8991 "expr": "machine_memory_bytes", 8992 "format": "time_series", 8993 "hide": false, 8994 "instant": false, 8995 "interval": "5s", 8996 "intervalFactor": 1, 8997 "legendFormat": "limit", 8998 "metric": "", 8999 "query": "machine_memory_bytes", 9000 "refId": "", 9001 "step": 10, 9002 "target": "" 9003 } 9004 ], 9005 "thresholds": [], 9006 "timeFrom": null, 9007 "timeShift": null, 9008 "title": "memory working set by container", 9009 "tooltip": { 9010 "msResolution": true, 9011 "shared": true, 9012 "sort": 2, 9013 "value_type": "cumulative" 9014 }, 9015 "transformations": [], 9016 "transparent": false, 9017 "type": "graph", 9018 "xaxis": { 9019 "mode": "time", 9020 "name": null, 9021 "show": true, 9022 "values": [] 9023 }, 9024 "yaxes": [ 9025 { 9026 "decimals": null, 9027 "format": "bytes", 9028 "label": null, 9029 "logBase": 1, 9030 "max": null, 9031 "min": null, 9032 "show": true 9033 }, 9034 { 9035 "decimals": null, 9036 "format": "short", 9037 "label": null, 9038 "logBase": 1, 9039 "max": null, 9040 "min": null, 9041 "show": true 9042 } 9043 ], 9044 "yaxis": { 9045 "align": false, 9046 "alignLevel": 0 9047 } 9048 }, 9049 { 9050 "aliasColors": {}, 9051 "bars": false, 9052 "cacheTimeout": null, 9053 "datasource": "$source", 9054 "description": null, 9055 "editable": true, 9056 "error": false, 9057 "fieldConfig": { 9058 "defaults": { 9059 "thresholds": { 9060 "mode": "absolute", 9061 "steps": [] 9062 } 9063 } 9064 }, 9065 "fill": 1, 9066 "grid": { 9067 "threshold1": null, 9068 "threshold1Color": "rgba(216, 200, 27, 0.27)", 9069 "threshold2": null, 9070 "threshold2Color": "rgba(234, 112, 112, 0.22)" 9071 }, 9072 "gridPos": null, 9073 "height": null, 9074 "hideTimeOverride": false, 9075 "id": 70, 9076 "interval": null, 9077 "isNew": true, 9078 "legend": { 9079 "alignAsTable": false, 9080 "avg": false, 9081 "current": false, 9082 "hideEmpty": false, 9083 "hideZero": false, 9084 "max": false, 9085 "min": false, 9086 "rightSide": false, 9087 "show": true, 9088 "sideWidth": null, 9089 "sort": null, 9090 "sortDesc": false, 9091 "total": false, 9092 "values": false 9093 }, 9094 "lines": true, 9095 "linewidth": 2, 9096 "links": [], 9097 "maxDataPoints": 100, 9098 "maxPerRow": null, 9099 "minSpan": null, 9100 "nullPointMode": null, 9101 "options": { 9102 "alertThreshold": true, 9103 "dataLinks": [] 9104 }, 9105 "percentage": false, 9106 "pointradius": 5, 9107 "points": false, 9108 "renderer": "flot", 9109 "repeat": null, 9110 "repeatDirection": null, 9111 "seriesOverrides": [], 9112 "span": 12, 9113 "stack": false, 9114 "steppedLine": false, 9115 "targets": [ 9116 { 9117 "datasource": "$source", 9118 "expr": "rate(container_network_transmit_bytes_total{id=\"/\"}[1m])", 9119 "format": "time_series", 9120 "hide": false, 9121 "instant": false, 9122 "interval": "", 9123 "intervalFactor": 2, 9124 "legendFormat": "{{instance}} transmit", 9125 "metric": "", 9126 "query": "rate(container_network_transmit_bytes_total{id=\"/\"}[1m])", 9127 "refId": "", 9128 "step": 10, 9129 "target": "" 9130 }, 9131 { 9132 "datasource": "$source", 9133 "expr": "rate(container_network_receive_bytes_total{id=\"/\"}[1m])", 9134 "format": "time_series", 9135 "hide": false, 9136 "instant": false, 9137 "interval": "", 9138 "intervalFactor": 2, 9139 "legendFormat": "{{instance}} receive", 9140 "metric": "", 9141 "query": "rate(container_network_receive_bytes_total{id=\"/\"}[1m])", 9142 "refId": "", 9143 "step": 10, 9144 "target": "" 9145 } 9146 ], 9147 "thresholds": [], 9148 "timeFrom": null, 9149 "timeShift": null, 9150 "title": "Network usage (bytes)", 9151 "tooltip": { 9152 "msResolution": true, 9153 "shared": true, 9154 "sort": 2, 9155 "value_type": "cumulative" 9156 }, 9157 "transformations": [], 9158 "transparent": false, 9159 "type": "graph", 9160 "xaxis": { 9161 "mode": "time", 9162 "name": null, 9163 "show": true, 9164 "values": [] 9165 }, 9166 "yaxes": [ 9167 { 9168 "decimals": null, 9169 "format": "Bps", 9170 "label": null, 9171 "logBase": 1, 9172 "max": null, 9173 "min": null, 9174 "show": true 9175 }, 9176 { 9177 "decimals": null, 9178 "format": "short", 9179 "label": null, 9180 "logBase": 1, 9181 "max": null, 9182 "min": null, 9183 "show": true 9184 } 9185 ], 9186 "yaxis": { 9187 "align": false, 9188 "alignLevel": 0 9189 } 9190 }, 9191 { 9192 "aliasColors": {}, 9193 "bars": false, 9194 "cacheTimeout": null, 9195 "datasource": "$source", 9196 "description": null, 9197 "editable": true, 9198 "error": false, 9199 "fieldConfig": { 9200 "defaults": { 9201 "thresholds": { 9202 "mode": "absolute", 9203 "steps": [] 9204 } 9205 } 9206 }, 9207 "fill": 1, 9208 "grid": { 9209 "threshold1": null, 9210 "threshold1Color": "rgba(216, 200, 27, 0.27)", 9211 "threshold2": null, 9212 "threshold2Color": "rgba(234, 112, 112, 0.22)" 9213 }, 9214 "gridPos": null, 9215 "height": null, 9216 "hideTimeOverride": false, 9217 "id": 71, 9218 "interval": null, 9219 "isNew": true, 9220 "legend": { 9221 "alignAsTable": false, 9222 "avg": false, 9223 "current": false, 9224 "hideEmpty": false, 9225 "hideZero": false, 9226 "max": false, 9227 "min": false, 9228 "rightSide": false, 9229 "show": true, 9230 "sideWidth": null, 9231 "sort": null, 9232 "sortDesc": false, 9233 "total": false, 9234 "values": false 9235 }, 9236 "lines": true, 9237 "linewidth": 2, 9238 "links": [], 9239 "maxDataPoints": 100, 9240 "maxPerRow": null, 9241 "minSpan": null, 9242 "nullPointMode": null, 9243 "options": { 9244 "alertThreshold": true, 9245 "dataLinks": [] 9246 }, 9247 "percentage": false, 9248 "pointradius": 5, 9249 "points": false, 9250 "renderer": "flot", 9251 "repeat": null, 9252 "repeatDirection": null, 9253 "seriesOverrides": [], 9254 "span": 12, 9255 "stack": false, 9256 "steppedLine": false, 9257 "targets": [ 9258 { 9259 "datasource": "$source", 9260 "expr": "rate(container_network_transmit_packets_total{id=\"/\"}[1m])", 9261 "format": "time_series", 9262 "hide": false, 9263 "instant": false, 9264 "interval": "", 9265 "intervalFactor": 2, 9266 "legendFormat": "{{instance}} transmit", 9267 "metric": "", 9268 "query": "rate(container_network_transmit_packets_total{id=\"/\"}[1m])", 9269 "refId": "", 9270 "step": 10, 9271 "target": "" 9272 }, 9273 { 9274 "datasource": "$source", 9275 "expr": "rate(container_network_receive_packets_total{id=\"/\"}[1m])", 9276 "format": "time_series", 9277 "hide": false, 9278 "instant": false, 9279 "interval": "", 9280 "intervalFactor": 2, 9281 "legendFormat": "{{instance}} receive", 9282 "metric": "", 9283 "query": "rate(container_network_receive_packets_total{id=\"/\"}[1m])", 9284 "refId": "", 9285 "step": 10, 9286 "target": "" 9287 } 9288 ], 9289 "thresholds": [], 9290 "timeFrom": null, 9291 "timeShift": null, 9292 "title": "Network usage (packets)", 9293 "tooltip": { 9294 "msResolution": true, 9295 "shared": true, 9296 "sort": 2, 9297 "value_type": "cumulative" 9298 }, 9299 "transformations": [], 9300 "transparent": false, 9301 "type": "graph", 9302 "xaxis": { 9303 "mode": "time", 9304 "name": null, 9305 "show": true, 9306 "values": [] 9307 }, 9308 "yaxes": [ 9309 { 9310 "decimals": null, 9311 "format": "short", 9312 "label": null, 9313 "logBase": 1, 9314 "max": null, 9315 "min": null, 9316 "show": true 9317 }, 9318 { 9319 "decimals": null, 9320 "format": "short", 9321 "label": null, 9322 "logBase": 1, 9323 "max": null, 9324 "min": null, 9325 "show": true 9326 } 9327 ], 9328 "yaxis": { 9329 "align": false, 9330 "alignLevel": 0 9331 } 9332 }, 9333 { 9334 "aliasColors": {}, 9335 "bars": false, 9336 "cacheTimeout": null, 9337 "datasource": "$source", 9338 "description": null, 9339 "editable": true, 9340 "error": false, 9341 "fieldConfig": { 9342 "defaults": { 9343 "thresholds": { 9344 "mode": "absolute", 9345 "steps": [] 9346 } 9347 } 9348 }, 9349 "fill": 1, 9350 "grid": { 9351 "threshold1": null, 9352 "threshold1Color": "rgba(216, 200, 27, 0.27)", 9353 "threshold2": null, 9354 "threshold2Color": "rgba(234, 112, 112, 0.22)" 9355 }, 9356 "gridPos": null, 9357 "height": null, 9358 "hideTimeOverride": false, 9359 "id": 72, 9360 "interval": null, 9361 "isNew": true, 9362 "legend": { 9363 "alignAsTable": false, 9364 "avg": false, 9365 "current": false, 9366 "hideEmpty": false, 9367 "hideZero": false, 9368 "max": false, 9369 "min": false, 9370 "rightSide": false, 9371 "show": true, 9372 "sideWidth": null, 9373 "sort": null, 9374 "sortDesc": false, 9375 "total": false, 9376 "values": false 9377 }, 9378 "lines": true, 9379 "linewidth": 2, 9380 "links": [], 9381 "maxDataPoints": 100, 9382 "maxPerRow": null, 9383 "minSpan": null, 9384 "nullPointMode": null, 9385 "options": { 9386 "alertThreshold": true, 9387 "dataLinks": [] 9388 }, 9389 "percentage": false, 9390 "pointradius": 5, 9391 "points": false, 9392 "renderer": "flot", 9393 "repeat": null, 9394 "repeatDirection": null, 9395 "seriesOverrides": [], 9396 "span": 12, 9397 "stack": false, 9398 "steppedLine": false, 9399 "targets": [ 9400 { 9401 "datasource": "$source", 9402 "expr": "rate(container_network_transmit_bytes_total{id=\"/\"}[1m]) / rate(container_network_transmit_packets_total{id=\"/\"}[1m])", 9403 "format": "time_series", 9404 "hide": false, 9405 "instant": false, 9406 "interval": "", 9407 "intervalFactor": 2, 9408 "legendFormat": "{{instance}} transmit", 9409 "metric": "", 9410 "query": "rate(container_network_transmit_bytes_total{id=\"/\"}[1m]) / rate(container_network_transmit_packets_total{id=\"/\"}[1m])", 9411 "refId": "", 9412 "step": 10, 9413 "target": "" 9414 }, 9415 { 9416 "datasource": "$source", 9417 "expr": "rate(container_network_receive_bytes_total{id=\"/\"}[1m]) / rate(container_network_receive_packets_total{id=\"/\"}[1m])", 9418 "format": "time_series", 9419 "hide": false, 9420 "instant": false, 9421 "interval": "", 9422 "intervalFactor": 2, 9423 "legendFormat": "{{instance}} receive", 9424 "metric": "", 9425 "query": "rate(container_network_receive_bytes_total{id=\"/\"}[1m]) / rate(container_network_receive_packets_total{id=\"/\"}[1m])", 9426 "refId": "", 9427 "step": 10, 9428 "target": "" 9429 } 9430 ], 9431 "thresholds": [], 9432 "timeFrom": null, 9433 "timeShift": null, 9434 "title": "Network usage (avg packet size)", 9435 "tooltip": { 9436 "msResolution": true, 9437 "shared": true, 9438 "sort": 2, 9439 "value_type": "cumulative" 9440 }, 9441 "transformations": [], 9442 "transparent": false, 9443 "type": "graph", 9444 "xaxis": { 9445 "mode": "time", 9446 "name": null, 9447 "show": true, 9448 "values": [] 9449 }, 9450 "yaxes": [ 9451 { 9452 "decimals": null, 9453 "format": "bytes", 9454 "label": null, 9455 "logBase": 1, 9456 "max": null, 9457 "min": null, 9458 "show": true 9459 }, 9460 { 9461 "decimals": null, 9462 "format": "short", 9463 "label": null, 9464 "logBase": 1, 9465 "max": null, 9466 "min": null, 9467 "show": true 9468 } 9469 ], 9470 "yaxis": { 9471 "align": false, 9472 "alignLevel": 0 9473 } 9474 }, 9475 { 9476 "aliasColors": {}, 9477 "bars": false, 9478 "cacheTimeout": null, 9479 "datasource": "$source", 9480 "description": null, 9481 "editable": true, 9482 "error": false, 9483 "fieldConfig": { 9484 "defaults": { 9485 "thresholds": { 9486 "mode": "absolute", 9487 "steps": [] 9488 } 9489 } 9490 }, 9491 "fill": 1, 9492 "grid": { 9493 "threshold1": null, 9494 "threshold1Color": "rgba(216, 200, 27, 0.27)", 9495 "threshold2": null, 9496 "threshold2Color": "rgba(234, 112, 112, 0.22)" 9497 }, 9498 "gridPos": null, 9499 "height": null, 9500 "hideTimeOverride": false, 9501 "id": 73, 9502 "interval": null, 9503 "isNew": true, 9504 "legend": { 9505 "alignAsTable": false, 9506 "avg": false, 9507 "current": false, 9508 "hideEmpty": false, 9509 "hideZero": false, 9510 "max": false, 9511 "min": false, 9512 "rightSide": false, 9513 "show": true, 9514 "sideWidth": null, 9515 "sort": null, 9516 "sortDesc": false, 9517 "total": false, 9518 "values": false 9519 }, 9520 "lines": true, 9521 "linewidth": 2, 9522 "links": [], 9523 "maxDataPoints": 100, 9524 "maxPerRow": null, 9525 "minSpan": null, 9526 "nullPointMode": null, 9527 "options": { 9528 "alertThreshold": true, 9529 "dataLinks": [] 9530 }, 9531 "percentage": false, 9532 "pointradius": 5, 9533 "points": false, 9534 "renderer": "flot", 9535 "repeat": null, 9536 "repeatDirection": null, 9537 "seriesOverrides": [], 9538 "span": 12, 9539 "stack": false, 9540 "steppedLine": false, 9541 "targets": [ 9542 { 9543 "datasource": "$source", 9544 "expr": "sum(rate(node_netstat_Tcp_InSegs[1m])) by (instance)", 9545 "format": "time_series", 9546 "hide": false, 9547 "instant": false, 9548 "interval": "", 9549 "intervalFactor": 2, 9550 "legendFormat": "InSegs {{instance}}", 9551 "metric": "", 9552 "query": "sum(rate(node_netstat_Tcp_InSegs[1m])) by (instance)", 9553 "refId": "", 9554 "step": 10, 9555 "target": "" 9556 }, 9557 { 9558 "datasource": "$source", 9559 "expr": "sum(rate(node_netstat_Tcp_OutSegs[1m])) by (instance)", 9560 "format": "time_series", 9561 "hide": false, 9562 "instant": false, 9563 "interval": "", 9564 "intervalFactor": 2, 9565 "legendFormat": "OutSegs {{instance}}", 9566 "metric": "", 9567 "query": "sum(rate(node_netstat_Tcp_OutSegs[1m])) by (instance)", 9568 "refId": "", 9569 "step": 10, 9570 "target": "" 9571 }, 9572 { 9573 "datasource": "$source", 9574 "expr": "sum(rate(node_netstat_Tcp_RetransSegs[1m])) by (instance)", 9575 "format": "time_series", 9576 "hide": false, 9577 "instant": false, 9578 "interval": "", 9579 "intervalFactor": 2, 9580 "legendFormat": "RetransSegs {{instance}}", 9581 "metric": "", 9582 "query": "sum(rate(node_netstat_Tcp_RetransSegs[1m])) by (instance)", 9583 "refId": "", 9584 "step": 10, 9585 "target": "" 9586 } 9587 ], 9588 "thresholds": [], 9589 "timeFrom": null, 9590 "timeShift": null, 9591 "title": "Network tcp segments", 9592 "tooltip": { 9593 "msResolution": true, 9594 "shared": true, 9595 "sort": 2, 9596 "value_type": "cumulative" 9597 }, 9598 "transformations": [], 9599 "transparent": false, 9600 "type": "graph", 9601 "xaxis": { 9602 "mode": "time", 9603 "name": null, 9604 "show": true, 9605 "values": [] 9606 }, 9607 "yaxes": [ 9608 { 9609 "decimals": null, 9610 "format": "short", 9611 "label": null, 9612 "logBase": 10, 9613 "max": null, 9614 "min": null, 9615 "show": true 9616 }, 9617 { 9618 "decimals": null, 9619 "format": "short", 9620 "label": null, 9621 "logBase": 1, 9622 "max": null, 9623 "min": null, 9624 "show": true 9625 } 9626 ], 9627 "yaxis": { 9628 "align": false, 9629 "alignLevel": 0 9630 } 9631 } 9632 ], 9633 "repeat": null, 9634 "showTitle": true, 9635 "title": "Master VM" 9636 } 9637 ], 9638 "schemaVersion": 12, 9639 "sharedCrosshair": false, 9640 "style": "dark", 9641 "tags": [], 9642 "templating": { 9643 "list": [ 9644 { 9645 "allValue": null, 9646 "auto": false, 9647 "auto_count": 30, 9648 "auto_min": "10s", 9649 "current": { 9650 "selected": false, 9651 "tags": [], 9652 "text": null, 9653 "value": null 9654 }, 9655 "datasource": null, 9656 "hide": 0, 9657 "includeAll": false, 9658 "label": null, 9659 "multi": false, 9660 "name": "source", 9661 "options": [], 9662 "query": "prometheus", 9663 "refresh": 1, 9664 "regex": null, 9665 "sort": 1, 9666 "tagValuesQuery": null, 9667 "tagsQuery": null, 9668 "type": "datasource", 9669 "useTags": false 9670 }, 9671 { 9672 "allValue": null, 9673 "auto": false, 9674 "auto_count": 30, 9675 "auto_min": "10s", 9676 "current": { 9677 "selected": false, 9678 "tags": [], 9679 "text": null, 9680 "value": null 9681 }, 9682 "datasource": "$source", 9683 "hide": 0, 9684 "includeAll": true, 9685 "label": null, 9686 "multi": true, 9687 "name": "etcd_type", 9688 "options": [], 9689 "query": "label_values(etcd_request_duration_seconds_count, type)", 9690 "refresh": 2, 9691 "regex": "\\*\\[+\\]+(.*)", 9692 "sort": 1, 9693 "tagValuesQuery": null, 9694 "tagsQuery": null, 9695 "type": "query", 9696 "useTags": false 9697 }, 9698 { 9699 "allValue": null, 9700 "auto": false, 9701 "auto_count": 30, 9702 "auto_min": "10s", 9703 "current": { 9704 "selected": false, 9705 "tags": [], 9706 "text": null, 9707 "value": null 9708 }, 9709 "datasource": "$source", 9710 "hide": 0, 9711 "includeAll": true, 9712 "label": null, 9713 "multi": true, 9714 "name": "etcd_operation", 9715 "options": [], 9716 "query": "label_values(etcd_request_duration_seconds_count, operation)", 9717 "refresh": 2, 9718 "regex": null, 9719 "sort": 1, 9720 "tagValuesQuery": null, 9721 "tagsQuery": null, 9722 "type": "query", 9723 "useTags": false 9724 }, 9725 { 9726 "allValue": null, 9727 "auto": false, 9728 "auto_count": 30, 9729 "auto_min": "10s", 9730 "current": { 9731 "selected": false, 9732 "tags": [], 9733 "text": null, 9734 "value": null 9735 }, 9736 "datasource": "$source", 9737 "hide": 0, 9738 "includeAll": true, 9739 "label": null, 9740 "multi": true, 9741 "name": "verb", 9742 "options": [], 9743 "query": "label_values(apiserver_request_duration_seconds_count, verb)", 9744 "refresh": 2, 9745 "regex": null, 9746 "sort": 1, 9747 "tagValuesQuery": null, 9748 "tagsQuery": null, 9749 "type": "query", 9750 "useTags": false 9751 }, 9752 { 9753 "allValue": null, 9754 "auto": false, 9755 "auto_count": 30, 9756 "auto_min": "10s", 9757 "current": { 9758 "selected": false, 9759 "tags": [], 9760 "text": null, 9761 "value": null 9762 }, 9763 "datasource": "$source", 9764 "hide": 0, 9765 "includeAll": true, 9766 "label": null, 9767 "multi": true, 9768 "name": "resource", 9769 "options": [], 9770 "query": "label_values(apiserver_request_duration_seconds_count, resource)", 9771 "refresh": 2, 9772 "regex": "(.*)s", 9773 "sort": 1, 9774 "tagValuesQuery": null, 9775 "tagsQuery": null, 9776 "type": "query", 9777 "useTags": false 9778 }, 9779 { 9780 "allValue": null, 9781 "auto": false, 9782 "auto_count": 30, 9783 "auto_min": "10s", 9784 "current": { 9785 "selected": false, 9786 "tags": [], 9787 "text": null, 9788 "value": null 9789 }, 9790 "datasource": "$source", 9791 "hide": 0, 9792 "includeAll": true, 9793 "label": null, 9794 "multi": true, 9795 "name": "instance", 9796 "options": [], 9797 "query": "label_values(apiserver_request_duration_seconds_count, instance)", 9798 "refresh": 2, 9799 "regex": null, 9800 "sort": 1, 9801 "tagValuesQuery": null, 9802 "tagsQuery": null, 9803 "type": "query", 9804 "useTags": false 9805 } 9806 ] 9807 }, 9808 "time": { 9809 "from": "now-30d", 9810 "to": "now" 9811 }, 9812 "timepicker": { 9813 "hidden": false, 9814 "refresh_intervals": [ 9815 "5s", 9816 "10s", 9817 "30s", 9818 "1m", 9819 "5m", 9820 "15m", 9821 "30m", 9822 "1h", 9823 "2h", 9824 "1d" 9825 ], 9826 "time_options": [ 9827 "5m", 9828 "15m", 9829 "1h", 9830 "6h", 9831 "12h", 9832 "24h", 9833 "2d", 9834 "7d", 9835 "30d" 9836 ] 9837 }, 9838 "timezone": "utc", 9839 "title": "Master dashboard", 9840 "uid": null, 9841 "version": 0 9842 }