k8s.io/perf-tests/clusterloader2@v0.0.0-20240304094227-64bdb12da87e/pkg/prometheus/manifests/dashboards/master-comparison.json (about) 1 { 2 "__inputs": [], 3 "annotations": { 4 "list": [] 5 }, 6 "description": "", 7 "editable": true, 8 "gnetId": null, 9 "graphTooltip": 0, 10 "hideControls": false, 11 "id": null, 12 "links": [], 13 "panels": [], 14 "refresh": "", 15 "rows": [ 16 { 17 "collapse": false, 18 "editable": true, 19 "height": "300px", 20 "panels": [ 21 { 22 "aliasColors": {}, 23 "bars": false, 24 "cacheTimeout": null, 25 "datasource": "$source", 26 "description": null, 27 "editable": true, 28 "error": false, 29 "fieldConfig": { 30 "defaults": { 31 "thresholds": { 32 "mode": "absolute", 33 "steps": [] 34 } 35 } 36 }, 37 "fill": 1, 38 "grid": { 39 "threshold1": null, 40 "threshold1Color": "rgba(216, 200, 27, 0.27)", 41 "threshold2": null, 42 "threshold2Color": "rgba(234, 112, 112, 0.22)" 43 }, 44 "gridPos": null, 45 "height": null, 46 "hideTimeOverride": false, 47 "id": 1, 48 "interval": null, 49 "isNew": true, 50 "legend": { 51 "alignAsTable": false, 52 "avg": false, 53 "current": false, 54 "hideEmpty": false, 55 "hideZero": false, 56 "max": false, 57 "min": false, 58 "rightSide": false, 59 "show": true, 60 "sideWidth": null, 61 "sort": null, 62 "sortDesc": false, 63 "total": false, 64 "values": false 65 }, 66 "lines": true, 67 "linewidth": 2, 68 "links": [], 69 "maxDataPoints": 100, 70 "maxPerRow": null, 71 "minSpan": null, 72 "nullPointMode": null, 73 "options": { 74 "alertThreshold": true, 75 "dataLinks": [] 76 }, 77 "percentage": false, 78 "pointradius": 5, 79 "points": false, 80 "renderer": "flot", 81 "repeat": null, 82 "repeatDirection": null, 83 "seriesOverrides": [], 84 "span": 12, 85 "stack": false, 86 "steppedLine": false, 87 "targets": [ 88 { 89 "datasource": "$source", 90 "expr": "1", 91 "format": "time_series", 92 "hide": false, 93 "instant": false, 94 "interval": "", 95 "intervalFactor": 2, 96 "legendFormat": "threshold", 97 "metric": "", 98 "query": "1", 99 "refId": "", 100 "step": 10, 101 "target": "" 102 }, 103 { 104 "datasource": "$source", 105 "expr": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"GET\", scope=~\"resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 106 "format": "time_series", 107 "hide": false, 108 "instant": false, 109 "interval": "", 110 "intervalFactor": 2, 111 "legendFormat": "", 112 "metric": "", 113 "query": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"GET\", scope=~\"resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 114 "refId": "", 115 "step": 10, 116 "target": "" 117 } 118 ], 119 "thresholds": [], 120 "timeFrom": null, 121 "timeShift": null, 122 "title": "GET resource latency (percentaile=99, scope=resource, threshold=1s)", 123 "tooltip": { 124 "msResolution": true, 125 "shared": true, 126 "sort": 2, 127 "value_type": "cumulative" 128 }, 129 "transformations": [], 130 "transparent": false, 131 "type": "graph", 132 "xaxis": { 133 "mode": "time", 134 "name": null, 135 "show": true, 136 "values": [] 137 }, 138 "yaxes": [ 139 { 140 "decimals": null, 141 "format": "s", 142 "label": null, 143 "logBase": 1, 144 "max": null, 145 "min": null, 146 "show": true 147 }, 148 { 149 "decimals": null, 150 "format": "short", 151 "label": null, 152 "logBase": 1, 153 "max": null, 154 "min": null, 155 "show": true 156 } 157 ], 158 "yaxis": { 159 "align": false, 160 "alignLevel": 0 161 } 162 }, 163 { 164 "aliasColors": {}, 165 "bars": false, 166 "cacheTimeout": null, 167 "datasource": "$secondary_source", 168 "description": null, 169 "editable": true, 170 "error": false, 171 "fieldConfig": { 172 "defaults": { 173 "thresholds": { 174 "mode": "absolute", 175 "steps": [] 176 } 177 } 178 }, 179 "fill": 1, 180 "grid": { 181 "threshold1": null, 182 "threshold1Color": "rgba(216, 200, 27, 0.27)", 183 "threshold2": null, 184 "threshold2Color": "rgba(234, 112, 112, 0.22)" 185 }, 186 "gridPos": null, 187 "height": null, 188 "hideTimeOverride": false, 189 "id": 2, 190 "interval": null, 191 "isNew": true, 192 "legend": { 193 "alignAsTable": false, 194 "avg": false, 195 "current": false, 196 "hideEmpty": false, 197 "hideZero": false, 198 "max": false, 199 "min": false, 200 "rightSide": false, 201 "show": true, 202 "sideWidth": null, 203 "sort": null, 204 "sortDesc": false, 205 "total": false, 206 "values": false 207 }, 208 "lines": true, 209 "linewidth": 2, 210 "links": [], 211 "maxDataPoints": 100, 212 "maxPerRow": null, 213 "minSpan": null, 214 "nullPointMode": null, 215 "options": { 216 "alertThreshold": true, 217 "dataLinks": [] 218 }, 219 "percentage": false, 220 "pointradius": 5, 221 "points": false, 222 "renderer": "flot", 223 "repeat": null, 224 "repeatDirection": null, 225 "seriesOverrides": [], 226 "span": 12, 227 "stack": false, 228 "steppedLine": false, 229 "targets": [ 230 { 231 "datasource": "$source", 232 "expr": "1", 233 "format": "time_series", 234 "hide": false, 235 "instant": false, 236 "interval": "", 237 "intervalFactor": 2, 238 "legendFormat": "threshold", 239 "metric": "", 240 "query": "1", 241 "refId": "", 242 "step": 10, 243 "target": "" 244 }, 245 { 246 "datasource": "$source", 247 "expr": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"GET\", scope=~\"resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 248 "format": "time_series", 249 "hide": false, 250 "instant": false, 251 "interval": "", 252 "intervalFactor": 2, 253 "legendFormat": "", 254 "metric": "", 255 "query": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"GET\", scope=~\"resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 256 "refId": "", 257 "step": 10, 258 "target": "" 259 } 260 ], 261 "thresholds": [], 262 "timeFrom": null, 263 "timeShift": "$timeshift", 264 "title": "[SECONDARY] GET resource latency (percentaile=99, scope=resource, threshold=1s)", 265 "tooltip": { 266 "msResolution": true, 267 "shared": true, 268 "sort": 2, 269 "value_type": "cumulative" 270 }, 271 "transformations": [], 272 "transparent": false, 273 "type": "graph", 274 "xaxis": { 275 "mode": "time", 276 "name": null, 277 "show": true, 278 "values": [] 279 }, 280 "yaxes": [ 281 { 282 "decimals": null, 283 "format": "s", 284 "label": null, 285 "logBase": 1, 286 "max": null, 287 "min": null, 288 "show": true 289 }, 290 { 291 "decimals": null, 292 "format": "short", 293 "label": null, 294 "logBase": 1, 295 "max": null, 296 "min": null, 297 "show": true 298 } 299 ], 300 "yaxis": { 301 "align": false, 302 "alignLevel": 0 303 } 304 }, 305 { 306 "aliasColors": {}, 307 "bars": false, 308 "cacheTimeout": null, 309 "datasource": "$source", 310 "description": null, 311 "editable": true, 312 "error": false, 313 "fieldConfig": { 314 "defaults": { 315 "thresholds": { 316 "mode": "absolute", 317 "steps": [] 318 } 319 } 320 }, 321 "fill": 1, 322 "grid": { 323 "threshold1": null, 324 "threshold1Color": "rgba(216, 200, 27, 0.27)", 325 "threshold2": null, 326 "threshold2Color": "rgba(234, 112, 112, 0.22)" 327 }, 328 "gridPos": null, 329 "height": null, 330 "hideTimeOverride": false, 331 "id": 3, 332 "interval": null, 333 "isNew": true, 334 "legend": { 335 "alignAsTable": false, 336 "avg": false, 337 "current": false, 338 "hideEmpty": false, 339 "hideZero": false, 340 "max": false, 341 "min": false, 342 "rightSide": false, 343 "show": true, 344 "sideWidth": null, 345 "sort": null, 346 "sortDesc": false, 347 "total": false, 348 "values": false 349 }, 350 "lines": true, 351 "linewidth": 2, 352 "links": [], 353 "maxDataPoints": 100, 354 "maxPerRow": null, 355 "minSpan": null, 356 "nullPointMode": null, 357 "options": { 358 "alertThreshold": true, 359 "dataLinks": [] 360 }, 361 "percentage": false, 362 "pointradius": 5, 363 "points": false, 364 "renderer": "flot", 365 "repeat": null, 366 "repeatDirection": null, 367 "seriesOverrides": [], 368 "span": 12, 369 "stack": false, 370 "steppedLine": false, 371 "targets": [ 372 { 373 "datasource": "$source", 374 "expr": "5", 375 "format": "time_series", 376 "hide": false, 377 "instant": false, 378 "interval": "", 379 "intervalFactor": 2, 380 "legendFormat": "threshold", 381 "metric": "", 382 "query": "5", 383 "refId": "", 384 "step": 10, 385 "target": "" 386 }, 387 { 388 "datasource": "$source", 389 "expr": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"namespace\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 390 "format": "time_series", 391 "hide": false, 392 "instant": false, 393 "interval": "", 394 "intervalFactor": 2, 395 "legendFormat": "", 396 "metric": "", 397 "query": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"namespace\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 398 "refId": "", 399 "step": 10, 400 "target": "" 401 } 402 ], 403 "thresholds": [], 404 "timeFrom": null, 405 "timeShift": null, 406 "title": "LIST namespace latency (percentaile=99, scope=namespace, threshold=5s)", 407 "tooltip": { 408 "msResolution": true, 409 "shared": true, 410 "sort": 2, 411 "value_type": "cumulative" 412 }, 413 "transformations": [], 414 "transparent": false, 415 "type": "graph", 416 "xaxis": { 417 "mode": "time", 418 "name": null, 419 "show": true, 420 "values": [] 421 }, 422 "yaxes": [ 423 { 424 "decimals": null, 425 "format": "s", 426 "label": null, 427 "logBase": 1, 428 "max": null, 429 "min": null, 430 "show": true 431 }, 432 { 433 "decimals": null, 434 "format": "short", 435 "label": null, 436 "logBase": 1, 437 "max": null, 438 "min": null, 439 "show": true 440 } 441 ], 442 "yaxis": { 443 "align": false, 444 "alignLevel": 0 445 } 446 }, 447 { 448 "aliasColors": {}, 449 "bars": false, 450 "cacheTimeout": null, 451 "datasource": "$secondary_source", 452 "description": null, 453 "editable": true, 454 "error": false, 455 "fieldConfig": { 456 "defaults": { 457 "thresholds": { 458 "mode": "absolute", 459 "steps": [] 460 } 461 } 462 }, 463 "fill": 1, 464 "grid": { 465 "threshold1": null, 466 "threshold1Color": "rgba(216, 200, 27, 0.27)", 467 "threshold2": null, 468 "threshold2Color": "rgba(234, 112, 112, 0.22)" 469 }, 470 "gridPos": null, 471 "height": null, 472 "hideTimeOverride": false, 473 "id": 4, 474 "interval": null, 475 "isNew": true, 476 "legend": { 477 "alignAsTable": false, 478 "avg": false, 479 "current": false, 480 "hideEmpty": false, 481 "hideZero": false, 482 "max": false, 483 "min": false, 484 "rightSide": false, 485 "show": true, 486 "sideWidth": null, 487 "sort": null, 488 "sortDesc": false, 489 "total": false, 490 "values": false 491 }, 492 "lines": true, 493 "linewidth": 2, 494 "links": [], 495 "maxDataPoints": 100, 496 "maxPerRow": null, 497 "minSpan": null, 498 "nullPointMode": null, 499 "options": { 500 "alertThreshold": true, 501 "dataLinks": [] 502 }, 503 "percentage": false, 504 "pointradius": 5, 505 "points": false, 506 "renderer": "flot", 507 "repeat": null, 508 "repeatDirection": null, 509 "seriesOverrides": [], 510 "span": 12, 511 "stack": false, 512 "steppedLine": false, 513 "targets": [ 514 { 515 "datasource": "$source", 516 "expr": "5", 517 "format": "time_series", 518 "hide": false, 519 "instant": false, 520 "interval": "", 521 "intervalFactor": 2, 522 "legendFormat": "threshold", 523 "metric": "", 524 "query": "5", 525 "refId": "", 526 "step": 10, 527 "target": "" 528 }, 529 { 530 "datasource": "$source", 531 "expr": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"namespace\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 532 "format": "time_series", 533 "hide": false, 534 "instant": false, 535 "interval": "", 536 "intervalFactor": 2, 537 "legendFormat": "", 538 "metric": "", 539 "query": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"namespace\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 540 "refId": "", 541 "step": 10, 542 "target": "" 543 } 544 ], 545 "thresholds": [], 546 "timeFrom": null, 547 "timeShift": "$timeshift", 548 "title": "[SECONDARY] LIST namespace latency (percentaile=99, scope=namespace, threshold=5s)", 549 "tooltip": { 550 "msResolution": true, 551 "shared": true, 552 "sort": 2, 553 "value_type": "cumulative" 554 }, 555 "transformations": [], 556 "transparent": false, 557 "type": "graph", 558 "xaxis": { 559 "mode": "time", 560 "name": null, 561 "show": true, 562 "values": [] 563 }, 564 "yaxes": [ 565 { 566 "decimals": null, 567 "format": "s", 568 "label": null, 569 "logBase": 1, 570 "max": null, 571 "min": null, 572 "show": true 573 }, 574 { 575 "decimals": null, 576 "format": "short", 577 "label": null, 578 "logBase": 1, 579 "max": null, 580 "min": null, 581 "show": true 582 } 583 ], 584 "yaxis": { 585 "align": false, 586 "alignLevel": 0 587 } 588 }, 589 { 590 "aliasColors": {}, 591 "bars": false, 592 "cacheTimeout": null, 593 "datasource": "$source", 594 "description": null, 595 "editable": true, 596 "error": false, 597 "fieldConfig": { 598 "defaults": { 599 "thresholds": { 600 "mode": "absolute", 601 "steps": [] 602 } 603 } 604 }, 605 "fill": 1, 606 "grid": { 607 "threshold1": null, 608 "threshold1Color": "rgba(216, 200, 27, 0.27)", 609 "threshold2": null, 610 "threshold2Color": "rgba(234, 112, 112, 0.22)" 611 }, 612 "gridPos": null, 613 "height": null, 614 "hideTimeOverride": false, 615 "id": 5, 616 "interval": null, 617 "isNew": true, 618 "legend": { 619 "alignAsTable": false, 620 "avg": false, 621 "current": false, 622 "hideEmpty": false, 623 "hideZero": false, 624 "max": false, 625 "min": false, 626 "rightSide": false, 627 "show": true, 628 "sideWidth": null, 629 "sort": null, 630 "sortDesc": false, 631 "total": false, 632 "values": false 633 }, 634 "lines": true, 635 "linewidth": 2, 636 "links": [], 637 "maxDataPoints": 100, 638 "maxPerRow": null, 639 "minSpan": null, 640 "nullPointMode": null, 641 "options": { 642 "alertThreshold": true, 643 "dataLinks": [] 644 }, 645 "percentage": false, 646 "pointradius": 5, 647 "points": false, 648 "renderer": "flot", 649 "repeat": null, 650 "repeatDirection": null, 651 "seriesOverrides": [], 652 "span": 12, 653 "stack": false, 654 "steppedLine": false, 655 "targets": [ 656 { 657 "datasource": "$source", 658 "expr": "30", 659 "format": "time_series", 660 "hide": false, 661 "instant": false, 662 "interval": "", 663 "intervalFactor": 2, 664 "legendFormat": "threshold", 665 "metric": "", 666 "query": "30", 667 "refId": "", 668 "step": 10, 669 "target": "" 670 }, 671 { 672 "datasource": "$source", 673 "expr": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"cluster\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 674 "format": "time_series", 675 "hide": false, 676 "instant": false, 677 "interval": "", 678 "intervalFactor": 2, 679 "legendFormat": "", 680 "metric": "", 681 "query": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"cluster\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 682 "refId": "", 683 "step": 10, 684 "target": "" 685 } 686 ], 687 "thresholds": [], 688 "timeFrom": null, 689 "timeShift": null, 690 "title": "LIST cluster latency (percentaile=99, scope=cluster, threshold=30s)", 691 "tooltip": { 692 "msResolution": true, 693 "shared": true, 694 "sort": 2, 695 "value_type": "cumulative" 696 }, 697 "transformations": [], 698 "transparent": false, 699 "type": "graph", 700 "xaxis": { 701 "mode": "time", 702 "name": null, 703 "show": true, 704 "values": [] 705 }, 706 "yaxes": [ 707 { 708 "decimals": null, 709 "format": "s", 710 "label": null, 711 "logBase": 1, 712 "max": null, 713 "min": null, 714 "show": true 715 }, 716 { 717 "decimals": null, 718 "format": "short", 719 "label": null, 720 "logBase": 1, 721 "max": null, 722 "min": null, 723 "show": true 724 } 725 ], 726 "yaxis": { 727 "align": false, 728 "alignLevel": 0 729 } 730 }, 731 { 732 "aliasColors": {}, 733 "bars": false, 734 "cacheTimeout": null, 735 "datasource": "$secondary_source", 736 "description": null, 737 "editable": true, 738 "error": false, 739 "fieldConfig": { 740 "defaults": { 741 "thresholds": { 742 "mode": "absolute", 743 "steps": [] 744 } 745 } 746 }, 747 "fill": 1, 748 "grid": { 749 "threshold1": null, 750 "threshold1Color": "rgba(216, 200, 27, 0.27)", 751 "threshold2": null, 752 "threshold2Color": "rgba(234, 112, 112, 0.22)" 753 }, 754 "gridPos": null, 755 "height": null, 756 "hideTimeOverride": false, 757 "id": 6, 758 "interval": null, 759 "isNew": true, 760 "legend": { 761 "alignAsTable": false, 762 "avg": false, 763 "current": false, 764 "hideEmpty": false, 765 "hideZero": false, 766 "max": false, 767 "min": false, 768 "rightSide": false, 769 "show": true, 770 "sideWidth": null, 771 "sort": null, 772 "sortDesc": false, 773 "total": false, 774 "values": false 775 }, 776 "lines": true, 777 "linewidth": 2, 778 "links": [], 779 "maxDataPoints": 100, 780 "maxPerRow": null, 781 "minSpan": null, 782 "nullPointMode": null, 783 "options": { 784 "alertThreshold": true, 785 "dataLinks": [] 786 }, 787 "percentage": false, 788 "pointradius": 5, 789 "points": false, 790 "renderer": "flot", 791 "repeat": null, 792 "repeatDirection": null, 793 "seriesOverrides": [], 794 "span": 12, 795 "stack": false, 796 "steppedLine": false, 797 "targets": [ 798 { 799 "datasource": "$source", 800 "expr": "30", 801 "format": "time_series", 802 "hide": false, 803 "instant": false, 804 "interval": "", 805 "intervalFactor": 2, 806 "legendFormat": "threshold", 807 "metric": "", 808 "query": "30", 809 "refId": "", 810 "step": 10, 811 "target": "" 812 }, 813 { 814 "datasource": "$source", 815 "expr": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"cluster\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 816 "format": "time_series", 817 "hide": false, 818 "instant": false, 819 "interval": "", 820 "intervalFactor": 2, 821 "legendFormat": "", 822 "metric": "", 823 "query": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"cluster\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 824 "refId": "", 825 "step": 10, 826 "target": "" 827 } 828 ], 829 "thresholds": [], 830 "timeFrom": null, 831 "timeShift": "$timeshift", 832 "title": "[SECONDARY] LIST cluster latency (percentaile=99, scope=cluster, threshold=30s)", 833 "tooltip": { 834 "msResolution": true, 835 "shared": true, 836 "sort": 2, 837 "value_type": "cumulative" 838 }, 839 "transformations": [], 840 "transparent": false, 841 "type": "graph", 842 "xaxis": { 843 "mode": "time", 844 "name": null, 845 "show": true, 846 "values": [] 847 }, 848 "yaxes": [ 849 { 850 "decimals": null, 851 "format": "s", 852 "label": null, 853 "logBase": 1, 854 "max": null, 855 "min": null, 856 "show": true 857 }, 858 { 859 "decimals": null, 860 "format": "short", 861 "label": null, 862 "logBase": 1, 863 "max": null, 864 "min": null, 865 "show": true 866 } 867 ], 868 "yaxis": { 869 "align": false, 870 "alignLevel": 0 871 } 872 }, 873 { 874 "aliasColors": {}, 875 "bars": false, 876 "cacheTimeout": null, 877 "datasource": "$source", 878 "description": null, 879 "editable": true, 880 "error": false, 881 "fieldConfig": { 882 "defaults": { 883 "thresholds": { 884 "mode": "absolute", 885 "steps": [] 886 } 887 } 888 }, 889 "fill": 1, 890 "grid": { 891 "threshold1": null, 892 "threshold1Color": "rgba(216, 200, 27, 0.27)", 893 "threshold2": null, 894 "threshold2Color": "rgba(234, 112, 112, 0.22)" 895 }, 896 "gridPos": null, 897 "height": null, 898 "hideTimeOverride": false, 899 "id": 7, 900 "interval": null, 901 "isNew": true, 902 "legend": { 903 "alignAsTable": false, 904 "avg": false, 905 "current": false, 906 "hideEmpty": false, 907 "hideZero": false, 908 "max": false, 909 "min": false, 910 "rightSide": false, 911 "show": true, 912 "sideWidth": null, 913 "sort": null, 914 "sortDesc": false, 915 "total": false, 916 "values": false 917 }, 918 "lines": true, 919 "linewidth": 2, 920 "links": [], 921 "maxDataPoints": 100, 922 "maxPerRow": null, 923 "minSpan": null, 924 "nullPointMode": null, 925 "options": { 926 "alertThreshold": true, 927 "dataLinks": [] 928 }, 929 "percentage": false, 930 "pointradius": 5, 931 "points": false, 932 "renderer": "flot", 933 "repeat": null, 934 "repeatDirection": null, 935 "seriesOverrides": [], 936 "span": 12, 937 "stack": false, 938 "steppedLine": false, 939 "targets": [ 940 { 941 "datasource": "$source", 942 "expr": "1", 943 "format": "time_series", 944 "hide": false, 945 "instant": false, 946 "interval": "", 947 "intervalFactor": 2, 948 "legendFormat": "threshold", 949 "metric": "", 950 "query": "1", 951 "refId": "", 952 "step": 10, 953 "target": "" 954 }, 955 { 956 "datasource": "$source", 957 "expr": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"CREATE|DELETE|PATCH|POST|PUT\", scope=~\"namespace|cluster|resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 958 "format": "time_series", 959 "hide": false, 960 "instant": false, 961 "interval": "", 962 "intervalFactor": 2, 963 "legendFormat": "", 964 "metric": "", 965 "query": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"CREATE|DELETE|PATCH|POST|PUT\", scope=~\"namespace|cluster|resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 966 "refId": "", 967 "step": 10, 968 "target": "" 969 } 970 ], 971 "thresholds": [], 972 "timeFrom": null, 973 "timeShift": null, 974 "title": "Mutating API call latency (threshold=1s)", 975 "tooltip": { 976 "msResolution": true, 977 "shared": true, 978 "sort": 2, 979 "value_type": "cumulative" 980 }, 981 "transformations": [], 982 "transparent": false, 983 "type": "graph", 984 "xaxis": { 985 "mode": "time", 986 "name": null, 987 "show": true, 988 "values": [] 989 }, 990 "yaxes": [ 991 { 992 "decimals": null, 993 "format": "s", 994 "label": null, 995 "logBase": 1, 996 "max": null, 997 "min": null, 998 "show": true 999 }, 1000 { 1001 "decimals": null, 1002 "format": "short", 1003 "label": null, 1004 "logBase": 1, 1005 "max": null, 1006 "min": null, 1007 "show": true 1008 } 1009 ], 1010 "yaxis": { 1011 "align": false, 1012 "alignLevel": 0 1013 } 1014 }, 1015 { 1016 "aliasColors": {}, 1017 "bars": false, 1018 "cacheTimeout": null, 1019 "datasource": "$secondary_source", 1020 "description": null, 1021 "editable": true, 1022 "error": false, 1023 "fieldConfig": { 1024 "defaults": { 1025 "thresholds": { 1026 "mode": "absolute", 1027 "steps": [] 1028 } 1029 } 1030 }, 1031 "fill": 1, 1032 "grid": { 1033 "threshold1": null, 1034 "threshold1Color": "rgba(216, 200, 27, 0.27)", 1035 "threshold2": null, 1036 "threshold2Color": "rgba(234, 112, 112, 0.22)" 1037 }, 1038 "gridPos": null, 1039 "height": null, 1040 "hideTimeOverride": false, 1041 "id": 8, 1042 "interval": null, 1043 "isNew": true, 1044 "legend": { 1045 "alignAsTable": false, 1046 "avg": false, 1047 "current": false, 1048 "hideEmpty": false, 1049 "hideZero": false, 1050 "max": false, 1051 "min": false, 1052 "rightSide": false, 1053 "show": true, 1054 "sideWidth": null, 1055 "sort": null, 1056 "sortDesc": false, 1057 "total": false, 1058 "values": false 1059 }, 1060 "lines": true, 1061 "linewidth": 2, 1062 "links": [], 1063 "maxDataPoints": 100, 1064 "maxPerRow": null, 1065 "minSpan": null, 1066 "nullPointMode": null, 1067 "options": { 1068 "alertThreshold": true, 1069 "dataLinks": [] 1070 }, 1071 "percentage": false, 1072 "pointradius": 5, 1073 "points": false, 1074 "renderer": "flot", 1075 "repeat": null, 1076 "repeatDirection": null, 1077 "seriesOverrides": [], 1078 "span": 12, 1079 "stack": false, 1080 "steppedLine": false, 1081 "targets": [ 1082 { 1083 "datasource": "$source", 1084 "expr": "1", 1085 "format": "time_series", 1086 "hide": false, 1087 "instant": false, 1088 "interval": "", 1089 "intervalFactor": 2, 1090 "legendFormat": "threshold", 1091 "metric": "", 1092 "query": "1", 1093 "refId": "", 1094 "step": 10, 1095 "target": "" 1096 }, 1097 { 1098 "datasource": "$source", 1099 "expr": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"CREATE|DELETE|PATCH|POST|PUT\", scope=~\"namespace|cluster|resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 1100 "format": "time_series", 1101 "hide": false, 1102 "instant": false, 1103 "interval": "", 1104 "intervalFactor": 2, 1105 "legendFormat": "", 1106 "metric": "", 1107 "query": "apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"CREATE|DELETE|PATCH|POST|PUT\", scope=~\"namespace|cluster|resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }", 1108 "refId": "", 1109 "step": 10, 1110 "target": "" 1111 } 1112 ], 1113 "thresholds": [], 1114 "timeFrom": null, 1115 "timeShift": "$timeshift", 1116 "title": "[SECONDARY] Mutating API call latency (threshold=1s)", 1117 "tooltip": { 1118 "msResolution": true, 1119 "shared": true, 1120 "sort": 2, 1121 "value_type": "cumulative" 1122 }, 1123 "transformations": [], 1124 "transparent": false, 1125 "type": "graph", 1126 "xaxis": { 1127 "mode": "time", 1128 "name": null, 1129 "show": true, 1130 "values": [] 1131 }, 1132 "yaxes": [ 1133 { 1134 "decimals": null, 1135 "format": "s", 1136 "label": null, 1137 "logBase": 1, 1138 "max": null, 1139 "min": null, 1140 "show": true 1141 }, 1142 { 1143 "decimals": null, 1144 "format": "short", 1145 "label": null, 1146 "logBase": 1, 1147 "max": null, 1148 "min": null, 1149 "show": true 1150 } 1151 ], 1152 "yaxis": { 1153 "align": false, 1154 "alignLevel": 0 1155 } 1156 } 1157 ], 1158 "repeat": null, 1159 "showTitle": true, 1160 "title": "API call latency" 1161 }, 1162 { 1163 "collapse": true, 1164 "editable": true, 1165 "height": "300px", 1166 "panels": [ 1167 { 1168 "aliasColors": {}, 1169 "bars": false, 1170 "cacheTimeout": null, 1171 "datasource": "$source", 1172 "description": null, 1173 "editable": true, 1174 "error": false, 1175 "fieldConfig": { 1176 "defaults": { 1177 "thresholds": { 1178 "mode": "absolute", 1179 "steps": [] 1180 } 1181 } 1182 }, 1183 "fill": 1, 1184 "grid": { 1185 "threshold1": null, 1186 "threshold1Color": "rgba(216, 200, 27, 0.27)", 1187 "threshold2": null, 1188 "threshold2Color": "rgba(234, 112, 112, 0.22)" 1189 }, 1190 "gridPos": null, 1191 "height": null, 1192 "hideTimeOverride": false, 1193 "id": 9, 1194 "interval": null, 1195 "isNew": true, 1196 "legend": { 1197 "alignAsTable": false, 1198 "avg": false, 1199 "current": false, 1200 "hideEmpty": false, 1201 "hideZero": false, 1202 "max": false, 1203 "min": false, 1204 "rightSide": false, 1205 "show": true, 1206 "sideWidth": null, 1207 "sort": null, 1208 "sortDesc": false, 1209 "total": false, 1210 "values": false 1211 }, 1212 "lines": true, 1213 "linewidth": 2, 1214 "links": [], 1215 "maxDataPoints": 100, 1216 "maxPerRow": null, 1217 "minSpan": null, 1218 "nullPointMode": null, 1219 "options": { 1220 "alertThreshold": true, 1221 "dataLinks": [] 1222 }, 1223 "percentage": false, 1224 "pointradius": 5, 1225 "points": false, 1226 "renderer": "flot", 1227 "repeat": null, 1228 "repeatDirection": null, 1229 "seriesOverrides": [], 1230 "span": 12, 1231 "stack": false, 1232 "steppedLine": false, 1233 "targets": [ 1234 { 1235 "datasource": "$source", 1236 "expr": "1", 1237 "format": "time_series", 1238 "hide": false, 1239 "instant": false, 1240 "interval": "", 1241 "intervalFactor": 2, 1242 "legendFormat": "threshold", 1243 "metric": "", 1244 "query": "1", 1245 "refId": "", 1246 "step": 10, 1247 "target": "" 1248 }, 1249 { 1250 "datasource": "$source", 1251 "expr": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"GET\", scope=~\"resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 1252 "format": "time_series", 1253 "hide": false, 1254 "instant": false, 1255 "interval": "", 1256 "intervalFactor": 2, 1257 "legendFormat": "", 1258 "metric": "", 1259 "query": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"GET\", scope=~\"resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 1260 "refId": "", 1261 "step": 10, 1262 "target": "" 1263 } 1264 ], 1265 "thresholds": [], 1266 "timeFrom": null, 1267 "timeShift": null, 1268 "title": "GET resource latency (percentaile=99, scope=resource, threshold=1s)", 1269 "tooltip": { 1270 "msResolution": true, 1271 "shared": true, 1272 "sort": 2, 1273 "value_type": "cumulative" 1274 }, 1275 "transformations": [], 1276 "transparent": false, 1277 "type": "graph", 1278 "xaxis": { 1279 "mode": "time", 1280 "name": null, 1281 "show": true, 1282 "values": [] 1283 }, 1284 "yaxes": [ 1285 { 1286 "decimals": null, 1287 "format": "s", 1288 "label": null, 1289 "logBase": 1, 1290 "max": null, 1291 "min": null, 1292 "show": true 1293 }, 1294 { 1295 "decimals": null, 1296 "format": "short", 1297 "label": null, 1298 "logBase": 1, 1299 "max": null, 1300 "min": null, 1301 "show": true 1302 } 1303 ], 1304 "yaxis": { 1305 "align": false, 1306 "alignLevel": 0 1307 } 1308 }, 1309 { 1310 "aliasColors": {}, 1311 "bars": false, 1312 "cacheTimeout": null, 1313 "datasource": "$secondary_source", 1314 "description": null, 1315 "editable": true, 1316 "error": false, 1317 "fieldConfig": { 1318 "defaults": { 1319 "thresholds": { 1320 "mode": "absolute", 1321 "steps": [] 1322 } 1323 } 1324 }, 1325 "fill": 1, 1326 "grid": { 1327 "threshold1": null, 1328 "threshold1Color": "rgba(216, 200, 27, 0.27)", 1329 "threshold2": null, 1330 "threshold2Color": "rgba(234, 112, 112, 0.22)" 1331 }, 1332 "gridPos": null, 1333 "height": null, 1334 "hideTimeOverride": false, 1335 "id": 10, 1336 "interval": null, 1337 "isNew": true, 1338 "legend": { 1339 "alignAsTable": false, 1340 "avg": false, 1341 "current": false, 1342 "hideEmpty": false, 1343 "hideZero": false, 1344 "max": false, 1345 "min": false, 1346 "rightSide": false, 1347 "show": true, 1348 "sideWidth": null, 1349 "sort": null, 1350 "sortDesc": false, 1351 "total": false, 1352 "values": false 1353 }, 1354 "lines": true, 1355 "linewidth": 2, 1356 "links": [], 1357 "maxDataPoints": 100, 1358 "maxPerRow": null, 1359 "minSpan": null, 1360 "nullPointMode": null, 1361 "options": { 1362 "alertThreshold": true, 1363 "dataLinks": [] 1364 }, 1365 "percentage": false, 1366 "pointradius": 5, 1367 "points": false, 1368 "renderer": "flot", 1369 "repeat": null, 1370 "repeatDirection": null, 1371 "seriesOverrides": [], 1372 "span": 12, 1373 "stack": false, 1374 "steppedLine": false, 1375 "targets": [ 1376 { 1377 "datasource": "$source", 1378 "expr": "1", 1379 "format": "time_series", 1380 "hide": false, 1381 "instant": false, 1382 "interval": "", 1383 "intervalFactor": 2, 1384 "legendFormat": "threshold", 1385 "metric": "", 1386 "query": "1", 1387 "refId": "", 1388 "step": 10, 1389 "target": "" 1390 }, 1391 { 1392 "datasource": "$source", 1393 "expr": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"GET\", scope=~\"resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 1394 "format": "time_series", 1395 "hide": false, 1396 "instant": false, 1397 "interval": "", 1398 "intervalFactor": 2, 1399 "legendFormat": "", 1400 "metric": "", 1401 "query": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"GET\", scope=~\"resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 1402 "refId": "", 1403 "step": 10, 1404 "target": "" 1405 } 1406 ], 1407 "thresholds": [], 1408 "timeFrom": null, 1409 "timeShift": "$timeshift", 1410 "title": "[SECONDARY] GET resource latency (percentaile=99, scope=resource, threshold=1s)", 1411 "tooltip": { 1412 "msResolution": true, 1413 "shared": true, 1414 "sort": 2, 1415 "value_type": "cumulative" 1416 }, 1417 "transformations": [], 1418 "transparent": false, 1419 "type": "graph", 1420 "xaxis": { 1421 "mode": "time", 1422 "name": null, 1423 "show": true, 1424 "values": [] 1425 }, 1426 "yaxes": [ 1427 { 1428 "decimals": null, 1429 "format": "s", 1430 "label": null, 1431 "logBase": 1, 1432 "max": null, 1433 "min": null, 1434 "show": true 1435 }, 1436 { 1437 "decimals": null, 1438 "format": "short", 1439 "label": null, 1440 "logBase": 1, 1441 "max": null, 1442 "min": null, 1443 "show": true 1444 } 1445 ], 1446 "yaxis": { 1447 "align": false, 1448 "alignLevel": 0 1449 } 1450 }, 1451 { 1452 "aliasColors": {}, 1453 "bars": false, 1454 "cacheTimeout": null, 1455 "datasource": "$source", 1456 "description": null, 1457 "editable": true, 1458 "error": false, 1459 "fieldConfig": { 1460 "defaults": { 1461 "thresholds": { 1462 "mode": "absolute", 1463 "steps": [] 1464 } 1465 } 1466 }, 1467 "fill": 1, 1468 "grid": { 1469 "threshold1": null, 1470 "threshold1Color": "rgba(216, 200, 27, 0.27)", 1471 "threshold2": null, 1472 "threshold2Color": "rgba(234, 112, 112, 0.22)" 1473 }, 1474 "gridPos": null, 1475 "height": null, 1476 "hideTimeOverride": false, 1477 "id": 11, 1478 "interval": null, 1479 "isNew": true, 1480 "legend": { 1481 "alignAsTable": false, 1482 "avg": false, 1483 "current": false, 1484 "hideEmpty": false, 1485 "hideZero": false, 1486 "max": false, 1487 "min": false, 1488 "rightSide": false, 1489 "show": true, 1490 "sideWidth": null, 1491 "sort": null, 1492 "sortDesc": false, 1493 "total": false, 1494 "values": false 1495 }, 1496 "lines": true, 1497 "linewidth": 2, 1498 "links": [], 1499 "maxDataPoints": 100, 1500 "maxPerRow": null, 1501 "minSpan": null, 1502 "nullPointMode": null, 1503 "options": { 1504 "alertThreshold": true, 1505 "dataLinks": [] 1506 }, 1507 "percentage": false, 1508 "pointradius": 5, 1509 "points": false, 1510 "renderer": "flot", 1511 "repeat": null, 1512 "repeatDirection": null, 1513 "seriesOverrides": [], 1514 "span": 12, 1515 "stack": false, 1516 "steppedLine": false, 1517 "targets": [ 1518 { 1519 "datasource": "$source", 1520 "expr": "5", 1521 "format": "time_series", 1522 "hide": false, 1523 "instant": false, 1524 "interval": "", 1525 "intervalFactor": 2, 1526 "legendFormat": "threshold", 1527 "metric": "", 1528 "query": "5", 1529 "refId": "", 1530 "step": 10, 1531 "target": "" 1532 }, 1533 { 1534 "datasource": "$source", 1535 "expr": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"namespace\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 1536 "format": "time_series", 1537 "hide": false, 1538 "instant": false, 1539 "interval": "", 1540 "intervalFactor": 2, 1541 "legendFormat": "", 1542 "metric": "", 1543 "query": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"namespace\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 1544 "refId": "", 1545 "step": 10, 1546 "target": "" 1547 } 1548 ], 1549 "thresholds": [], 1550 "timeFrom": null, 1551 "timeShift": null, 1552 "title": "LIST namespace latency (percentaile=99, scope=namespace, threshold=5s)", 1553 "tooltip": { 1554 "msResolution": true, 1555 "shared": true, 1556 "sort": 2, 1557 "value_type": "cumulative" 1558 }, 1559 "transformations": [], 1560 "transparent": false, 1561 "type": "graph", 1562 "xaxis": { 1563 "mode": "time", 1564 "name": null, 1565 "show": true, 1566 "values": [] 1567 }, 1568 "yaxes": [ 1569 { 1570 "decimals": null, 1571 "format": "s", 1572 "label": null, 1573 "logBase": 1, 1574 "max": null, 1575 "min": null, 1576 "show": true 1577 }, 1578 { 1579 "decimals": null, 1580 "format": "short", 1581 "label": null, 1582 "logBase": 1, 1583 "max": null, 1584 "min": null, 1585 "show": true 1586 } 1587 ], 1588 "yaxis": { 1589 "align": false, 1590 "alignLevel": 0 1591 } 1592 }, 1593 { 1594 "aliasColors": {}, 1595 "bars": false, 1596 "cacheTimeout": null, 1597 "datasource": "$secondary_source", 1598 "description": null, 1599 "editable": true, 1600 "error": false, 1601 "fieldConfig": { 1602 "defaults": { 1603 "thresholds": { 1604 "mode": "absolute", 1605 "steps": [] 1606 } 1607 } 1608 }, 1609 "fill": 1, 1610 "grid": { 1611 "threshold1": null, 1612 "threshold1Color": "rgba(216, 200, 27, 0.27)", 1613 "threshold2": null, 1614 "threshold2Color": "rgba(234, 112, 112, 0.22)" 1615 }, 1616 "gridPos": null, 1617 "height": null, 1618 "hideTimeOverride": false, 1619 "id": 12, 1620 "interval": null, 1621 "isNew": true, 1622 "legend": { 1623 "alignAsTable": false, 1624 "avg": false, 1625 "current": false, 1626 "hideEmpty": false, 1627 "hideZero": false, 1628 "max": false, 1629 "min": false, 1630 "rightSide": false, 1631 "show": true, 1632 "sideWidth": null, 1633 "sort": null, 1634 "sortDesc": false, 1635 "total": false, 1636 "values": false 1637 }, 1638 "lines": true, 1639 "linewidth": 2, 1640 "links": [], 1641 "maxDataPoints": 100, 1642 "maxPerRow": null, 1643 "minSpan": null, 1644 "nullPointMode": null, 1645 "options": { 1646 "alertThreshold": true, 1647 "dataLinks": [] 1648 }, 1649 "percentage": false, 1650 "pointradius": 5, 1651 "points": false, 1652 "renderer": "flot", 1653 "repeat": null, 1654 "repeatDirection": null, 1655 "seriesOverrides": [], 1656 "span": 12, 1657 "stack": false, 1658 "steppedLine": false, 1659 "targets": [ 1660 { 1661 "datasource": "$source", 1662 "expr": "5", 1663 "format": "time_series", 1664 "hide": false, 1665 "instant": false, 1666 "interval": "", 1667 "intervalFactor": 2, 1668 "legendFormat": "threshold", 1669 "metric": "", 1670 "query": "5", 1671 "refId": "", 1672 "step": 10, 1673 "target": "" 1674 }, 1675 { 1676 "datasource": "$source", 1677 "expr": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"namespace\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 1678 "format": "time_series", 1679 "hide": false, 1680 "instant": false, 1681 "interval": "", 1682 "intervalFactor": 2, 1683 "legendFormat": "", 1684 "metric": "", 1685 "query": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"namespace\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 1686 "refId": "", 1687 "step": 10, 1688 "target": "" 1689 } 1690 ], 1691 "thresholds": [], 1692 "timeFrom": null, 1693 "timeShift": "$timeshift", 1694 "title": "[SECONDARY] LIST namespace latency (percentaile=99, scope=namespace, threshold=5s)", 1695 "tooltip": { 1696 "msResolution": true, 1697 "shared": true, 1698 "sort": 2, 1699 "value_type": "cumulative" 1700 }, 1701 "transformations": [], 1702 "transparent": false, 1703 "type": "graph", 1704 "xaxis": { 1705 "mode": "time", 1706 "name": null, 1707 "show": true, 1708 "values": [] 1709 }, 1710 "yaxes": [ 1711 { 1712 "decimals": null, 1713 "format": "s", 1714 "label": null, 1715 "logBase": 1, 1716 "max": null, 1717 "min": null, 1718 "show": true 1719 }, 1720 { 1721 "decimals": null, 1722 "format": "short", 1723 "label": null, 1724 "logBase": 1, 1725 "max": null, 1726 "min": null, 1727 "show": true 1728 } 1729 ], 1730 "yaxis": { 1731 "align": false, 1732 "alignLevel": 0 1733 } 1734 }, 1735 { 1736 "aliasColors": {}, 1737 "bars": false, 1738 "cacheTimeout": null, 1739 "datasource": "$source", 1740 "description": null, 1741 "editable": true, 1742 "error": false, 1743 "fieldConfig": { 1744 "defaults": { 1745 "thresholds": { 1746 "mode": "absolute", 1747 "steps": [] 1748 } 1749 } 1750 }, 1751 "fill": 1, 1752 "grid": { 1753 "threshold1": null, 1754 "threshold1Color": "rgba(216, 200, 27, 0.27)", 1755 "threshold2": null, 1756 "threshold2Color": "rgba(234, 112, 112, 0.22)" 1757 }, 1758 "gridPos": null, 1759 "height": null, 1760 "hideTimeOverride": false, 1761 "id": 13, 1762 "interval": null, 1763 "isNew": true, 1764 "legend": { 1765 "alignAsTable": false, 1766 "avg": false, 1767 "current": false, 1768 "hideEmpty": false, 1769 "hideZero": false, 1770 "max": false, 1771 "min": false, 1772 "rightSide": false, 1773 "show": true, 1774 "sideWidth": null, 1775 "sort": null, 1776 "sortDesc": false, 1777 "total": false, 1778 "values": false 1779 }, 1780 "lines": true, 1781 "linewidth": 2, 1782 "links": [], 1783 "maxDataPoints": 100, 1784 "maxPerRow": null, 1785 "minSpan": null, 1786 "nullPointMode": null, 1787 "options": { 1788 "alertThreshold": true, 1789 "dataLinks": [] 1790 }, 1791 "percentage": false, 1792 "pointradius": 5, 1793 "points": false, 1794 "renderer": "flot", 1795 "repeat": null, 1796 "repeatDirection": null, 1797 "seriesOverrides": [], 1798 "span": 12, 1799 "stack": false, 1800 "steppedLine": false, 1801 "targets": [ 1802 { 1803 "datasource": "$source", 1804 "expr": "30", 1805 "format": "time_series", 1806 "hide": false, 1807 "instant": false, 1808 "interval": "", 1809 "intervalFactor": 2, 1810 "legendFormat": "threshold", 1811 "metric": "", 1812 "query": "30", 1813 "refId": "", 1814 "step": 10, 1815 "target": "" 1816 }, 1817 { 1818 "datasource": "$source", 1819 "expr": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"cluster\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 1820 "format": "time_series", 1821 "hide": false, 1822 "instant": false, 1823 "interval": "", 1824 "intervalFactor": 2, 1825 "legendFormat": "", 1826 "metric": "", 1827 "query": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"cluster\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 1828 "refId": "", 1829 "step": 10, 1830 "target": "" 1831 } 1832 ], 1833 "thresholds": [], 1834 "timeFrom": null, 1835 "timeShift": null, 1836 "title": "LIST cluster latency (percentaile=99, scope=cluster, threshold=30s)", 1837 "tooltip": { 1838 "msResolution": true, 1839 "shared": true, 1840 "sort": 2, 1841 "value_type": "cumulative" 1842 }, 1843 "transformations": [], 1844 "transparent": false, 1845 "type": "graph", 1846 "xaxis": { 1847 "mode": "time", 1848 "name": null, 1849 "show": true, 1850 "values": [] 1851 }, 1852 "yaxes": [ 1853 { 1854 "decimals": null, 1855 "format": "s", 1856 "label": null, 1857 "logBase": 1, 1858 "max": null, 1859 "min": null, 1860 "show": true 1861 }, 1862 { 1863 "decimals": null, 1864 "format": "short", 1865 "label": null, 1866 "logBase": 1, 1867 "max": null, 1868 "min": null, 1869 "show": true 1870 } 1871 ], 1872 "yaxis": { 1873 "align": false, 1874 "alignLevel": 0 1875 } 1876 }, 1877 { 1878 "aliasColors": {}, 1879 "bars": false, 1880 "cacheTimeout": null, 1881 "datasource": "$secondary_source", 1882 "description": null, 1883 "editable": true, 1884 "error": false, 1885 "fieldConfig": { 1886 "defaults": { 1887 "thresholds": { 1888 "mode": "absolute", 1889 "steps": [] 1890 } 1891 } 1892 }, 1893 "fill": 1, 1894 "grid": { 1895 "threshold1": null, 1896 "threshold1Color": "rgba(216, 200, 27, 0.27)", 1897 "threshold2": null, 1898 "threshold2Color": "rgba(234, 112, 112, 0.22)" 1899 }, 1900 "gridPos": null, 1901 "height": null, 1902 "hideTimeOverride": false, 1903 "id": 14, 1904 "interval": null, 1905 "isNew": true, 1906 "legend": { 1907 "alignAsTable": false, 1908 "avg": false, 1909 "current": false, 1910 "hideEmpty": false, 1911 "hideZero": false, 1912 "max": false, 1913 "min": false, 1914 "rightSide": false, 1915 "show": true, 1916 "sideWidth": null, 1917 "sort": null, 1918 "sortDesc": false, 1919 "total": false, 1920 "values": false 1921 }, 1922 "lines": true, 1923 "linewidth": 2, 1924 "links": [], 1925 "maxDataPoints": 100, 1926 "maxPerRow": null, 1927 "minSpan": null, 1928 "nullPointMode": null, 1929 "options": { 1930 "alertThreshold": true, 1931 "dataLinks": [] 1932 }, 1933 "percentage": false, 1934 "pointradius": 5, 1935 "points": false, 1936 "renderer": "flot", 1937 "repeat": null, 1938 "repeatDirection": null, 1939 "seriesOverrides": [], 1940 "span": 12, 1941 "stack": false, 1942 "steppedLine": false, 1943 "targets": [ 1944 { 1945 "datasource": "$source", 1946 "expr": "30", 1947 "format": "time_series", 1948 "hide": false, 1949 "instant": false, 1950 "interval": "", 1951 "intervalFactor": 2, 1952 "legendFormat": "threshold", 1953 "metric": "", 1954 "query": "30", 1955 "refId": "", 1956 "step": 10, 1957 "target": "" 1958 }, 1959 { 1960 "datasource": "$source", 1961 "expr": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"cluster\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 1962 "format": "time_series", 1963 "hide": false, 1964 "instant": false, 1965 "interval": "", 1966 "intervalFactor": 2, 1967 "legendFormat": "", 1968 "metric": "", 1969 "query": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"LIST\", scope=~\"cluster\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 1970 "refId": "", 1971 "step": 10, 1972 "target": "" 1973 } 1974 ], 1975 "thresholds": [], 1976 "timeFrom": null, 1977 "timeShift": "$timeshift", 1978 "title": "[SECONDARY] LIST cluster latency (percentaile=99, scope=cluster, threshold=30s)", 1979 "tooltip": { 1980 "msResolution": true, 1981 "shared": true, 1982 "sort": 2, 1983 "value_type": "cumulative" 1984 }, 1985 "transformations": [], 1986 "transparent": false, 1987 "type": "graph", 1988 "xaxis": { 1989 "mode": "time", 1990 "name": null, 1991 "show": true, 1992 "values": [] 1993 }, 1994 "yaxes": [ 1995 { 1996 "decimals": null, 1997 "format": "s", 1998 "label": null, 1999 "logBase": 1, 2000 "max": null, 2001 "min": null, 2002 "show": true 2003 }, 2004 { 2005 "decimals": null, 2006 "format": "short", 2007 "label": null, 2008 "logBase": 1, 2009 "max": null, 2010 "min": null, 2011 "show": true 2012 } 2013 ], 2014 "yaxis": { 2015 "align": false, 2016 "alignLevel": 0 2017 } 2018 }, 2019 { 2020 "aliasColors": {}, 2021 "bars": false, 2022 "cacheTimeout": null, 2023 "datasource": "$source", 2024 "description": null, 2025 "editable": true, 2026 "error": false, 2027 "fieldConfig": { 2028 "defaults": { 2029 "thresholds": { 2030 "mode": "absolute", 2031 "steps": [] 2032 } 2033 } 2034 }, 2035 "fill": 1, 2036 "grid": { 2037 "threshold1": null, 2038 "threshold1Color": "rgba(216, 200, 27, 0.27)", 2039 "threshold2": null, 2040 "threshold2Color": "rgba(234, 112, 112, 0.22)" 2041 }, 2042 "gridPos": null, 2043 "height": null, 2044 "hideTimeOverride": false, 2045 "id": 15, 2046 "interval": null, 2047 "isNew": true, 2048 "legend": { 2049 "alignAsTable": false, 2050 "avg": false, 2051 "current": false, 2052 "hideEmpty": false, 2053 "hideZero": false, 2054 "max": false, 2055 "min": false, 2056 "rightSide": false, 2057 "show": true, 2058 "sideWidth": null, 2059 "sort": null, 2060 "sortDesc": false, 2061 "total": false, 2062 "values": false 2063 }, 2064 "lines": true, 2065 "linewidth": 2, 2066 "links": [], 2067 "maxDataPoints": 100, 2068 "maxPerRow": null, 2069 "minSpan": null, 2070 "nullPointMode": null, 2071 "options": { 2072 "alertThreshold": true, 2073 "dataLinks": [] 2074 }, 2075 "percentage": false, 2076 "pointradius": 5, 2077 "points": false, 2078 "renderer": "flot", 2079 "repeat": null, 2080 "repeatDirection": null, 2081 "seriesOverrides": [], 2082 "span": 12, 2083 "stack": false, 2084 "steppedLine": false, 2085 "targets": [ 2086 { 2087 "datasource": "$source", 2088 "expr": "1", 2089 "format": "time_series", 2090 "hide": false, 2091 "instant": false, 2092 "interval": "", 2093 "intervalFactor": 2, 2094 "legendFormat": "threshold", 2095 "metric": "", 2096 "query": "1", 2097 "refId": "", 2098 "step": 10, 2099 "target": "" 2100 }, 2101 { 2102 "datasource": "$source", 2103 "expr": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"CREATE|DELETE|PATCH|POST|PUT\", scope=~\"namespace|cluster|resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 2104 "format": "time_series", 2105 "hide": false, 2106 "instant": false, 2107 "interval": "", 2108 "intervalFactor": 2, 2109 "legendFormat": "", 2110 "metric": "", 2111 "query": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"CREATE|DELETE|PATCH|POST|PUT\", scope=~\"namespace|cluster|resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 2112 "refId": "", 2113 "step": 10, 2114 "target": "" 2115 } 2116 ], 2117 "thresholds": [], 2118 "timeFrom": null, 2119 "timeShift": null, 2120 "title": "Mutating API call latency (threshold=1s)", 2121 "tooltip": { 2122 "msResolution": true, 2123 "shared": true, 2124 "sort": 2, 2125 "value_type": "cumulative" 2126 }, 2127 "transformations": [], 2128 "transparent": false, 2129 "type": "graph", 2130 "xaxis": { 2131 "mode": "time", 2132 "name": null, 2133 "show": true, 2134 "values": [] 2135 }, 2136 "yaxes": [ 2137 { 2138 "decimals": null, 2139 "format": "s", 2140 "label": null, 2141 "logBase": 1, 2142 "max": null, 2143 "min": null, 2144 "show": true 2145 }, 2146 { 2147 "decimals": null, 2148 "format": "short", 2149 "label": null, 2150 "logBase": 1, 2151 "max": null, 2152 "min": null, 2153 "show": true 2154 } 2155 ], 2156 "yaxis": { 2157 "align": false, 2158 "alignLevel": 0 2159 } 2160 }, 2161 { 2162 "aliasColors": {}, 2163 "bars": false, 2164 "cacheTimeout": null, 2165 "datasource": "$secondary_source", 2166 "description": null, 2167 "editable": true, 2168 "error": false, 2169 "fieldConfig": { 2170 "defaults": { 2171 "thresholds": { 2172 "mode": "absolute", 2173 "steps": [] 2174 } 2175 } 2176 }, 2177 "fill": 1, 2178 "grid": { 2179 "threshold1": null, 2180 "threshold1Color": "rgba(216, 200, 27, 0.27)", 2181 "threshold2": null, 2182 "threshold2Color": "rgba(234, 112, 112, 0.22)" 2183 }, 2184 "gridPos": null, 2185 "height": null, 2186 "hideTimeOverride": false, 2187 "id": 16, 2188 "interval": null, 2189 "isNew": true, 2190 "legend": { 2191 "alignAsTable": false, 2192 "avg": false, 2193 "current": false, 2194 "hideEmpty": false, 2195 "hideZero": false, 2196 "max": false, 2197 "min": false, 2198 "rightSide": false, 2199 "show": true, 2200 "sideWidth": null, 2201 "sort": null, 2202 "sortDesc": false, 2203 "total": false, 2204 "values": false 2205 }, 2206 "lines": true, 2207 "linewidth": 2, 2208 "links": [], 2209 "maxDataPoints": 100, 2210 "maxPerRow": null, 2211 "minSpan": null, 2212 "nullPointMode": null, 2213 "options": { 2214 "alertThreshold": true, 2215 "dataLinks": [] 2216 }, 2217 "percentage": false, 2218 "pointradius": 5, 2219 "points": false, 2220 "renderer": "flot", 2221 "repeat": null, 2222 "repeatDirection": null, 2223 "seriesOverrides": [], 2224 "span": 12, 2225 "stack": false, 2226 "steppedLine": false, 2227 "targets": [ 2228 { 2229 "datasource": "$source", 2230 "expr": "1", 2231 "format": "time_series", 2232 "hide": false, 2233 "instant": false, 2234 "interval": "", 2235 "intervalFactor": 2, 2236 "legendFormat": "threshold", 2237 "metric": "", 2238 "query": "1", 2239 "refId": "", 2240 "step": 10, 2241 "target": "" 2242 }, 2243 { 2244 "datasource": "$source", 2245 "expr": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"CREATE|DELETE|PATCH|POST|PUT\", scope=~\"namespace|cluster|resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 2246 "format": "time_series", 2247 "hide": false, 2248 "instant": false, 2249 "interval": "", 2250 "intervalFactor": 2, 2251 "legendFormat": "", 2252 "metric": "", 2253 "query": "quantile_over_time(0.99, apiserver:apiserver_request_latency_1m:histogram_quantile{quantile=\"0.99\", verb=~\"CREATE|DELETE|PATCH|POST|PUT\", scope=~\"namespace|cluster|resource\", resource=~\"${resource:regex}s*\", subresource!~\"exec|proxy\", }[5d])", 2254 "refId": "", 2255 "step": 10, 2256 "target": "" 2257 } 2258 ], 2259 "thresholds": [], 2260 "timeFrom": null, 2261 "timeShift": "$timeshift", 2262 "title": "[SECONDARY] Mutating API call latency (threshold=1s)", 2263 "tooltip": { 2264 "msResolution": true, 2265 "shared": true, 2266 "sort": 2, 2267 "value_type": "cumulative" 2268 }, 2269 "transformations": [], 2270 "transparent": false, 2271 "type": "graph", 2272 "xaxis": { 2273 "mode": "time", 2274 "name": null, 2275 "show": true, 2276 "values": [] 2277 }, 2278 "yaxes": [ 2279 { 2280 "decimals": null, 2281 "format": "s", 2282 "label": null, 2283 "logBase": 1, 2284 "max": null, 2285 "min": null, 2286 "show": true 2287 }, 2288 { 2289 "decimals": null, 2290 "format": "short", 2291 "label": null, 2292 "logBase": 1, 2293 "max": null, 2294 "min": null, 2295 "show": true 2296 } 2297 ], 2298 "yaxis": { 2299 "align": false, 2300 "alignLevel": 0 2301 } 2302 } 2303 ], 2304 "repeat": null, 2305 "showTitle": true, 2306 "title": "API call latency aggregated with quantile" 2307 }, 2308 { 2309 "collapse": true, 2310 "editable": true, 2311 "height": "300px", 2312 "panels": [ 2313 { 2314 "aliasColors": {}, 2315 "bars": false, 2316 "cacheTimeout": null, 2317 "datasource": "$source", 2318 "description": null, 2319 "editable": true, 2320 "error": false, 2321 "fieldConfig": { 2322 "defaults": { 2323 "thresholds": { 2324 "mode": "absolute", 2325 "steps": [] 2326 } 2327 } 2328 }, 2329 "fill": 1, 2330 "grid": { 2331 "threshold1": null, 2332 "threshold1Color": "rgba(216, 200, 27, 0.27)", 2333 "threshold2": null, 2334 "threshold2Color": "rgba(234, 112, 112, 0.22)" 2335 }, 2336 "gridPos": null, 2337 "height": null, 2338 "hideTimeOverride": false, 2339 "id": 17, 2340 "interval": null, 2341 "isNew": true, 2342 "legend": { 2343 "alignAsTable": false, 2344 "avg": false, 2345 "current": false, 2346 "hideEmpty": false, 2347 "hideZero": false, 2348 "max": false, 2349 "min": false, 2350 "rightSide": false, 2351 "show": true, 2352 "sideWidth": null, 2353 "sort": null, 2354 "sortDesc": false, 2355 "total": false, 2356 "values": false 2357 }, 2358 "lines": true, 2359 "linewidth": 2, 2360 "links": [], 2361 "maxDataPoints": 100, 2362 "maxPerRow": null, 2363 "minSpan": null, 2364 "nullPointMode": null, 2365 "options": { 2366 "alertThreshold": true, 2367 "dataLinks": [] 2368 }, 2369 "percentage": false, 2370 "pointradius": 5, 2371 "points": false, 2372 "renderer": "flot", 2373 "repeat": null, 2374 "repeatDirection": null, 2375 "seriesOverrides": [], 2376 "span": 12, 2377 "stack": false, 2378 "steppedLine": false, 2379 "targets": [ 2380 { 2381 "datasource": "$source", 2382 "expr": "histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_wait_duration_seconds_bucket{instance=~\".*(${instance:pipe})\"}[1m])) by (le, instance, priority_level))", 2383 "format": "time_series", 2384 "hide": false, 2385 "instant": false, 2386 "interval": "5s", 2387 "intervalFactor": 1, 2388 "legendFormat": "{{instance}} {{priority_level}}", 2389 "metric": "", 2390 "query": "histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_wait_duration_seconds_bucket{instance=~\".*(${instance:pipe})\"}[1m])) by (le, instance, priority_level))", 2391 "refId": "", 2392 "step": 10, 2393 "target": "" 2394 } 2395 ], 2396 "thresholds": [], 2397 "timeFrom": null, 2398 "timeShift": null, 2399 "title": "Requests waiting time", 2400 "tooltip": { 2401 "msResolution": true, 2402 "shared": true, 2403 "sort": 2, 2404 "value_type": "cumulative" 2405 }, 2406 "transformations": [], 2407 "transparent": false, 2408 "type": "graph", 2409 "xaxis": { 2410 "mode": "time", 2411 "name": null, 2412 "show": true, 2413 "values": [] 2414 }, 2415 "yaxes": [ 2416 { 2417 "decimals": null, 2418 "format": "s", 2419 "label": null, 2420 "logBase": 1, 2421 "max": null, 2422 "min": null, 2423 "show": true 2424 }, 2425 { 2426 "decimals": null, 2427 "format": "short", 2428 "label": null, 2429 "logBase": 1, 2430 "max": null, 2431 "min": null, 2432 "show": true 2433 } 2434 ], 2435 "yaxis": { 2436 "align": false, 2437 "alignLevel": 0 2438 } 2439 }, 2440 { 2441 "aliasColors": {}, 2442 "bars": false, 2443 "cacheTimeout": null, 2444 "datasource": "$secondary_source", 2445 "description": null, 2446 "editable": true, 2447 "error": false, 2448 "fieldConfig": { 2449 "defaults": { 2450 "thresholds": { 2451 "mode": "absolute", 2452 "steps": [] 2453 } 2454 } 2455 }, 2456 "fill": 1, 2457 "grid": { 2458 "threshold1": null, 2459 "threshold1Color": "rgba(216, 200, 27, 0.27)", 2460 "threshold2": null, 2461 "threshold2Color": "rgba(234, 112, 112, 0.22)" 2462 }, 2463 "gridPos": null, 2464 "height": null, 2465 "hideTimeOverride": false, 2466 "id": 18, 2467 "interval": null, 2468 "isNew": true, 2469 "legend": { 2470 "alignAsTable": false, 2471 "avg": false, 2472 "current": false, 2473 "hideEmpty": false, 2474 "hideZero": false, 2475 "max": false, 2476 "min": false, 2477 "rightSide": false, 2478 "show": true, 2479 "sideWidth": null, 2480 "sort": null, 2481 "sortDesc": false, 2482 "total": false, 2483 "values": false 2484 }, 2485 "lines": true, 2486 "linewidth": 2, 2487 "links": [], 2488 "maxDataPoints": 100, 2489 "maxPerRow": null, 2490 "minSpan": null, 2491 "nullPointMode": null, 2492 "options": { 2493 "alertThreshold": true, 2494 "dataLinks": [] 2495 }, 2496 "percentage": false, 2497 "pointradius": 5, 2498 "points": false, 2499 "renderer": "flot", 2500 "repeat": null, 2501 "repeatDirection": null, 2502 "seriesOverrides": [], 2503 "span": 12, 2504 "stack": false, 2505 "steppedLine": false, 2506 "targets": [ 2507 { 2508 "datasource": "$source", 2509 "expr": "histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_wait_duration_seconds_bucket{instance=~\".*(${instance:pipe})\"}[1m])) by (le, instance, priority_level))", 2510 "format": "time_series", 2511 "hide": false, 2512 "instant": false, 2513 "interval": "5s", 2514 "intervalFactor": 1, 2515 "legendFormat": "{{instance}} {{priority_level}}", 2516 "metric": "", 2517 "query": "histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_wait_duration_seconds_bucket{instance=~\".*(${instance:pipe})\"}[1m])) by (le, instance, priority_level))", 2518 "refId": "", 2519 "step": 10, 2520 "target": "" 2521 } 2522 ], 2523 "thresholds": [], 2524 "timeFrom": null, 2525 "timeShift": "$timeshift", 2526 "title": "[SECONDARY] Requests waiting time", 2527 "tooltip": { 2528 "msResolution": true, 2529 "shared": true, 2530 "sort": 2, 2531 "value_type": "cumulative" 2532 }, 2533 "transformations": [], 2534 "transparent": false, 2535 "type": "graph", 2536 "xaxis": { 2537 "mode": "time", 2538 "name": null, 2539 "show": true, 2540 "values": [] 2541 }, 2542 "yaxes": [ 2543 { 2544 "decimals": null, 2545 "format": "s", 2546 "label": null, 2547 "logBase": 1, 2548 "max": null, 2549 "min": null, 2550 "show": true 2551 }, 2552 { 2553 "decimals": null, 2554 "format": "short", 2555 "label": null, 2556 "logBase": 1, 2557 "max": null, 2558 "min": null, 2559 "show": true 2560 } 2561 ], 2562 "yaxis": { 2563 "align": false, 2564 "alignLevel": 0 2565 } 2566 }, 2567 { 2568 "aliasColors": {}, 2569 "bars": false, 2570 "cacheTimeout": null, 2571 "datasource": "$source", 2572 "description": null, 2573 "editable": true, 2574 "error": false, 2575 "fieldConfig": { 2576 "defaults": { 2577 "thresholds": { 2578 "mode": "absolute", 2579 "steps": [] 2580 } 2581 } 2582 }, 2583 "fill": 1, 2584 "grid": { 2585 "threshold1": null, 2586 "threshold1Color": "rgba(216, 200, 27, 0.27)", 2587 "threshold2": null, 2588 "threshold2Color": "rgba(234, 112, 112, 0.22)" 2589 }, 2590 "gridPos": null, 2591 "height": null, 2592 "hideTimeOverride": false, 2593 "id": 19, 2594 "interval": null, 2595 "isNew": true, 2596 "legend": { 2597 "alignAsTable": false, 2598 "avg": false, 2599 "current": false, 2600 "hideEmpty": false, 2601 "hideZero": false, 2602 "max": false, 2603 "min": false, 2604 "rightSide": false, 2605 "show": true, 2606 "sideWidth": null, 2607 "sort": null, 2608 "sortDesc": false, 2609 "total": false, 2610 "values": false 2611 }, 2612 "lines": true, 2613 "linewidth": 2, 2614 "links": [], 2615 "maxDataPoints": 100, 2616 "maxPerRow": null, 2617 "minSpan": null, 2618 "nullPointMode": null, 2619 "options": { 2620 "alertThreshold": true, 2621 "dataLinks": [] 2622 }, 2623 "percentage": false, 2624 "pointradius": 5, 2625 "points": false, 2626 "renderer": "flot", 2627 "repeat": null, 2628 "repeatDirection": null, 2629 "seriesOverrides": [], 2630 "span": 12, 2631 "stack": false, 2632 "steppedLine": false, 2633 "targets": [ 2634 { 2635 "datasource": "$source", 2636 "expr": "histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_execution_seconds_bucket{instance=~\".*(${instance:pipe})\"}[1m])) by (le, instance, priority_level))", 2637 "format": "time_series", 2638 "hide": false, 2639 "instant": false, 2640 "interval": "5s", 2641 "intervalFactor": 1, 2642 "legendFormat": "{{instance}} {{priority_level}}", 2643 "metric": "", 2644 "query": "histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_execution_seconds_bucket{instance=~\".*(${instance:pipe})\"}[1m])) by (le, instance, priority_level))", 2645 "refId": "", 2646 "step": 10, 2647 "target": "" 2648 } 2649 ], 2650 "thresholds": [], 2651 "timeFrom": null, 2652 "timeShift": null, 2653 "title": "Execution time", 2654 "tooltip": { 2655 "msResolution": true, 2656 "shared": true, 2657 "sort": 2, 2658 "value_type": "cumulative" 2659 }, 2660 "transformations": [], 2661 "transparent": false, 2662 "type": "graph", 2663 "xaxis": { 2664 "mode": "time", 2665 "name": null, 2666 "show": true, 2667 "values": [] 2668 }, 2669 "yaxes": [ 2670 { 2671 "decimals": null, 2672 "format": "s", 2673 "label": null, 2674 "logBase": 1, 2675 "max": null, 2676 "min": null, 2677 "show": true 2678 }, 2679 { 2680 "decimals": null, 2681 "format": "short", 2682 "label": null, 2683 "logBase": 1, 2684 "max": null, 2685 "min": null, 2686 "show": true 2687 } 2688 ], 2689 "yaxis": { 2690 "align": false, 2691 "alignLevel": 0 2692 } 2693 }, 2694 { 2695 "aliasColors": {}, 2696 "bars": false, 2697 "cacheTimeout": null, 2698 "datasource": "$secondary_source", 2699 "description": null, 2700 "editable": true, 2701 "error": false, 2702 "fieldConfig": { 2703 "defaults": { 2704 "thresholds": { 2705 "mode": "absolute", 2706 "steps": [] 2707 } 2708 } 2709 }, 2710 "fill": 1, 2711 "grid": { 2712 "threshold1": null, 2713 "threshold1Color": "rgba(216, 200, 27, 0.27)", 2714 "threshold2": null, 2715 "threshold2Color": "rgba(234, 112, 112, 0.22)" 2716 }, 2717 "gridPos": null, 2718 "height": null, 2719 "hideTimeOverride": false, 2720 "id": 20, 2721 "interval": null, 2722 "isNew": true, 2723 "legend": { 2724 "alignAsTable": false, 2725 "avg": false, 2726 "current": false, 2727 "hideEmpty": false, 2728 "hideZero": false, 2729 "max": false, 2730 "min": false, 2731 "rightSide": false, 2732 "show": true, 2733 "sideWidth": null, 2734 "sort": null, 2735 "sortDesc": false, 2736 "total": false, 2737 "values": false 2738 }, 2739 "lines": true, 2740 "linewidth": 2, 2741 "links": [], 2742 "maxDataPoints": 100, 2743 "maxPerRow": null, 2744 "minSpan": null, 2745 "nullPointMode": null, 2746 "options": { 2747 "alertThreshold": true, 2748 "dataLinks": [] 2749 }, 2750 "percentage": false, 2751 "pointradius": 5, 2752 "points": false, 2753 "renderer": "flot", 2754 "repeat": null, 2755 "repeatDirection": null, 2756 "seriesOverrides": [], 2757 "span": 12, 2758 "stack": false, 2759 "steppedLine": false, 2760 "targets": [ 2761 { 2762 "datasource": "$source", 2763 "expr": "histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_execution_seconds_bucket{instance=~\".*(${instance:pipe})\"}[1m])) by (le, instance, priority_level))", 2764 "format": "time_series", 2765 "hide": false, 2766 "instant": false, 2767 "interval": "5s", 2768 "intervalFactor": 1, 2769 "legendFormat": "{{instance}} {{priority_level}}", 2770 "metric": "", 2771 "query": "histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_execution_seconds_bucket{instance=~\".*(${instance:pipe})\"}[1m])) by (le, instance, priority_level))", 2772 "refId": "", 2773 "step": 10, 2774 "target": "" 2775 } 2776 ], 2777 "thresholds": [], 2778 "timeFrom": null, 2779 "timeShift": "$timeshift", 2780 "title": "[SECONDARY] Execution time", 2781 "tooltip": { 2782 "msResolution": true, 2783 "shared": true, 2784 "sort": 2, 2785 "value_type": "cumulative" 2786 }, 2787 "transformations": [], 2788 "transparent": false, 2789 "type": "graph", 2790 "xaxis": { 2791 "mode": "time", 2792 "name": null, 2793 "show": true, 2794 "values": [] 2795 }, 2796 "yaxes": [ 2797 { 2798 "decimals": null, 2799 "format": "s", 2800 "label": null, 2801 "logBase": 1, 2802 "max": null, 2803 "min": null, 2804 "show": true 2805 }, 2806 { 2807 "decimals": null, 2808 "format": "short", 2809 "label": null, 2810 "logBase": 1, 2811 "max": null, 2812 "min": null, 2813 "show": true 2814 } 2815 ], 2816 "yaxis": { 2817 "align": false, 2818 "alignLevel": 0 2819 } 2820 }, 2821 { 2822 "aliasColors": {}, 2823 "bars": false, 2824 "cacheTimeout": null, 2825 "datasource": "$source", 2826 "description": null, 2827 "editable": true, 2828 "error": false, 2829 "fieldConfig": { 2830 "defaults": { 2831 "thresholds": { 2832 "mode": "absolute", 2833 "steps": [] 2834 } 2835 } 2836 }, 2837 "fill": 1, 2838 "grid": { 2839 "threshold1": null, 2840 "threshold1Color": "rgba(216, 200, 27, 0.27)", 2841 "threshold2": null, 2842 "threshold2Color": "rgba(234, 112, 112, 0.22)" 2843 }, 2844 "gridPos": null, 2845 "height": null, 2846 "hideTimeOverride": false, 2847 "id": 21, 2848 "interval": null, 2849 "isNew": true, 2850 "legend": { 2851 "alignAsTable": false, 2852 "avg": false, 2853 "current": false, 2854 "hideEmpty": false, 2855 "hideZero": false, 2856 "max": false, 2857 "min": false, 2858 "rightSide": false, 2859 "show": true, 2860 "sideWidth": null, 2861 "sort": null, 2862 "sortDesc": false, 2863 "total": false, 2864 "values": false 2865 }, 2866 "lines": true, 2867 "linewidth": 2, 2868 "links": [], 2869 "maxDataPoints": 100, 2870 "maxPerRow": null, 2871 "minSpan": null, 2872 "nullPointMode": null, 2873 "options": { 2874 "alertThreshold": true, 2875 "dataLinks": [] 2876 }, 2877 "percentage": false, 2878 "pointradius": 5, 2879 "points": false, 2880 "renderer": "flot", 2881 "repeat": null, 2882 "repeatDirection": null, 2883 "seriesOverrides": [], 2884 "span": 12, 2885 "stack": false, 2886 "steppedLine": false, 2887 "targets": [ 2888 { 2889 "datasource": "$source", 2890 "expr": "sum(irate(apiserver_flowcontrol_request_execution_seconds_sum{instance=~\".*(${instance:pipe})\"}[1m])) by (instance, priority_level)", 2891 "format": "time_series", 2892 "hide": false, 2893 "instant": false, 2894 "interval": "5s", 2895 "intervalFactor": 1, 2896 "legendFormat": "{{instance}} {{priority_level}}", 2897 "metric": "", 2898 "query": "sum(irate(apiserver_flowcontrol_request_execution_seconds_sum{instance=~\".*(${instance:pipe})\"}[1m])) by (instance, priority_level)", 2899 "refId": "", 2900 "step": 10, 2901 "target": "" 2902 } 2903 ], 2904 "thresholds": [], 2905 "timeFrom": null, 2906 "timeShift": null, 2907 "title": "Total execution time per second", 2908 "tooltip": { 2909 "msResolution": true, 2910 "shared": true, 2911 "sort": 2, 2912 "value_type": "cumulative" 2913 }, 2914 "transformations": [], 2915 "transparent": false, 2916 "type": "graph", 2917 "xaxis": { 2918 "mode": "time", 2919 "name": null, 2920 "show": true, 2921 "values": [] 2922 }, 2923 "yaxes": [ 2924 { 2925 "decimals": null, 2926 "format": "s", 2927 "label": null, 2928 "logBase": 1, 2929 "max": null, 2930 "min": null, 2931 "show": true 2932 }, 2933 { 2934 "decimals": null, 2935 "format": "short", 2936 "label": null, 2937 "logBase": 1, 2938 "max": null, 2939 "min": null, 2940 "show": true 2941 } 2942 ], 2943 "yaxis": { 2944 "align": false, 2945 "alignLevel": 0 2946 } 2947 }, 2948 { 2949 "aliasColors": {}, 2950 "bars": false, 2951 "cacheTimeout": null, 2952 "datasource": "$secondary_source", 2953 "description": null, 2954 "editable": true, 2955 "error": false, 2956 "fieldConfig": { 2957 "defaults": { 2958 "thresholds": { 2959 "mode": "absolute", 2960 "steps": [] 2961 } 2962 } 2963 }, 2964 "fill": 1, 2965 "grid": { 2966 "threshold1": null, 2967 "threshold1Color": "rgba(216, 200, 27, 0.27)", 2968 "threshold2": null, 2969 "threshold2Color": "rgba(234, 112, 112, 0.22)" 2970 }, 2971 "gridPos": null, 2972 "height": null, 2973 "hideTimeOverride": false, 2974 "id": 22, 2975 "interval": null, 2976 "isNew": true, 2977 "legend": { 2978 "alignAsTable": false, 2979 "avg": false, 2980 "current": false, 2981 "hideEmpty": false, 2982 "hideZero": false, 2983 "max": false, 2984 "min": false, 2985 "rightSide": false, 2986 "show": true, 2987 "sideWidth": null, 2988 "sort": null, 2989 "sortDesc": false, 2990 "total": false, 2991 "values": false 2992 }, 2993 "lines": true, 2994 "linewidth": 2, 2995 "links": [], 2996 "maxDataPoints": 100, 2997 "maxPerRow": null, 2998 "minSpan": null, 2999 "nullPointMode": null, 3000 "options": { 3001 "alertThreshold": true, 3002 "dataLinks": [] 3003 }, 3004 "percentage": false, 3005 "pointradius": 5, 3006 "points": false, 3007 "renderer": "flot", 3008 "repeat": null, 3009 "repeatDirection": null, 3010 "seriesOverrides": [], 3011 "span": 12, 3012 "stack": false, 3013 "steppedLine": false, 3014 "targets": [ 3015 { 3016 "datasource": "$source", 3017 "expr": "sum(irate(apiserver_flowcontrol_request_execution_seconds_sum{instance=~\".*(${instance:pipe})\"}[1m])) by (instance, priority_level)", 3018 "format": "time_series", 3019 "hide": false, 3020 "instant": false, 3021 "interval": "5s", 3022 "intervalFactor": 1, 3023 "legendFormat": "{{instance}} {{priority_level}}", 3024 "metric": "", 3025 "query": "sum(irate(apiserver_flowcontrol_request_execution_seconds_sum{instance=~\".*(${instance:pipe})\"}[1m])) by (instance, priority_level)", 3026 "refId": "", 3027 "step": 10, 3028 "target": "" 3029 } 3030 ], 3031 "thresholds": [], 3032 "timeFrom": null, 3033 "timeShift": "$timeshift", 3034 "title": "[SECONDARY] Total execution time per second", 3035 "tooltip": { 3036 "msResolution": true, 3037 "shared": true, 3038 "sort": 2, 3039 "value_type": "cumulative" 3040 }, 3041 "transformations": [], 3042 "transparent": false, 3043 "type": "graph", 3044 "xaxis": { 3045 "mode": "time", 3046 "name": null, 3047 "show": true, 3048 "values": [] 3049 }, 3050 "yaxes": [ 3051 { 3052 "decimals": null, 3053 "format": "s", 3054 "label": null, 3055 "logBase": 1, 3056 "max": null, 3057 "min": null, 3058 "show": true 3059 }, 3060 { 3061 "decimals": null, 3062 "format": "short", 3063 "label": null, 3064 "logBase": 1, 3065 "max": null, 3066 "min": null, 3067 "show": true 3068 } 3069 ], 3070 "yaxis": { 3071 "align": false, 3072 "alignLevel": 0 3073 } 3074 }, 3075 { 3076 "aliasColors": {}, 3077 "bars": false, 3078 "cacheTimeout": null, 3079 "datasource": "$source", 3080 "description": null, 3081 "editable": true, 3082 "error": false, 3083 "fieldConfig": { 3084 "defaults": { 3085 "thresholds": { 3086 "mode": "absolute", 3087 "steps": [] 3088 } 3089 } 3090 }, 3091 "fill": 1, 3092 "grid": { 3093 "threshold1": null, 3094 "threshold1Color": "rgba(216, 200, 27, 0.27)", 3095 "threshold2": null, 3096 "threshold2Color": "rgba(234, 112, 112, 0.22)" 3097 }, 3098 "gridPos": null, 3099 "height": null, 3100 "hideTimeOverride": false, 3101 "id": 23, 3102 "interval": null, 3103 "isNew": true, 3104 "legend": { 3105 "alignAsTable": false, 3106 "avg": false, 3107 "current": false, 3108 "hideEmpty": false, 3109 "hideZero": false, 3110 "max": false, 3111 "min": false, 3112 "rightSide": false, 3113 "show": true, 3114 "sideWidth": null, 3115 "sort": null, 3116 "sortDesc": false, 3117 "total": false, 3118 "values": false 3119 }, 3120 "lines": true, 3121 "linewidth": 2, 3122 "links": [], 3123 "maxDataPoints": 100, 3124 "maxPerRow": null, 3125 "minSpan": null, 3126 "nullPointMode": null, 3127 "options": { 3128 "alertThreshold": true, 3129 "dataLinks": [] 3130 }, 3131 "percentage": false, 3132 "pointradius": 5, 3133 "points": false, 3134 "renderer": "flot", 3135 "repeat": null, 3136 "repeatDirection": null, 3137 "seriesOverrides": [], 3138 "span": 12, 3139 "stack": false, 3140 "steppedLine": false, 3141 "targets": [ 3142 { 3143 "datasource": "$source", 3144 "expr": "sum(irate(apiserver_flowcontrol_dispatched_requests_total{instance=~\".*(${instance:pipe})\"}[1m])) by (instance, priority_level)", 3145 "format": "time_series", 3146 "hide": false, 3147 "instant": false, 3148 "interval": "5s", 3149 "intervalFactor": 1, 3150 "legendFormat": "{{instance}} {{priority_level}}", 3151 "metric": "", 3152 "query": "sum(irate(apiserver_flowcontrol_dispatched_requests_total{instance=~\".*(${instance:pipe})\"}[1m])) by (instance, priority_level)", 3153 "refId": "", 3154 "step": 10, 3155 "target": "" 3156 } 3157 ], 3158 "thresholds": [], 3159 "timeFrom": null, 3160 "timeShift": null, 3161 "title": "Requests rate by priority level", 3162 "tooltip": { 3163 "msResolution": true, 3164 "shared": true, 3165 "sort": 2, 3166 "value_type": "cumulative" 3167 }, 3168 "transformations": [], 3169 "transparent": false, 3170 "type": "graph", 3171 "xaxis": { 3172 "mode": "time", 3173 "name": null, 3174 "show": true, 3175 "values": [] 3176 }, 3177 "yaxes": [ 3178 { 3179 "decimals": null, 3180 "format": "ops", 3181 "label": null, 3182 "logBase": 1, 3183 "max": null, 3184 "min": null, 3185 "show": true 3186 }, 3187 { 3188 "decimals": null, 3189 "format": "short", 3190 "label": null, 3191 "logBase": 1, 3192 "max": null, 3193 "min": null, 3194 "show": true 3195 } 3196 ], 3197 "yaxis": { 3198 "align": false, 3199 "alignLevel": 0 3200 } 3201 }, 3202 { 3203 "aliasColors": {}, 3204 "bars": false, 3205 "cacheTimeout": null, 3206 "datasource": "$secondary_source", 3207 "description": null, 3208 "editable": true, 3209 "error": false, 3210 "fieldConfig": { 3211 "defaults": { 3212 "thresholds": { 3213 "mode": "absolute", 3214 "steps": [] 3215 } 3216 } 3217 }, 3218 "fill": 1, 3219 "grid": { 3220 "threshold1": null, 3221 "threshold1Color": "rgba(216, 200, 27, 0.27)", 3222 "threshold2": null, 3223 "threshold2Color": "rgba(234, 112, 112, 0.22)" 3224 }, 3225 "gridPos": null, 3226 "height": null, 3227 "hideTimeOverride": false, 3228 "id": 24, 3229 "interval": null, 3230 "isNew": true, 3231 "legend": { 3232 "alignAsTable": false, 3233 "avg": false, 3234 "current": false, 3235 "hideEmpty": false, 3236 "hideZero": false, 3237 "max": false, 3238 "min": false, 3239 "rightSide": false, 3240 "show": true, 3241 "sideWidth": null, 3242 "sort": null, 3243 "sortDesc": false, 3244 "total": false, 3245 "values": false 3246 }, 3247 "lines": true, 3248 "linewidth": 2, 3249 "links": [], 3250 "maxDataPoints": 100, 3251 "maxPerRow": null, 3252 "minSpan": null, 3253 "nullPointMode": null, 3254 "options": { 3255 "alertThreshold": true, 3256 "dataLinks": [] 3257 }, 3258 "percentage": false, 3259 "pointradius": 5, 3260 "points": false, 3261 "renderer": "flot", 3262 "repeat": null, 3263 "repeatDirection": null, 3264 "seriesOverrides": [], 3265 "span": 12, 3266 "stack": false, 3267 "steppedLine": false, 3268 "targets": [ 3269 { 3270 "datasource": "$source", 3271 "expr": "sum(irate(apiserver_flowcontrol_dispatched_requests_total{instance=~\".*(${instance:pipe})\"}[1m])) by (instance, priority_level)", 3272 "format": "time_series", 3273 "hide": false, 3274 "instant": false, 3275 "interval": "5s", 3276 "intervalFactor": 1, 3277 "legendFormat": "{{instance}} {{priority_level}}", 3278 "metric": "", 3279 "query": "sum(irate(apiserver_flowcontrol_dispatched_requests_total{instance=~\".*(${instance:pipe})\"}[1m])) by (instance, priority_level)", 3280 "refId": "", 3281 "step": 10, 3282 "target": "" 3283 } 3284 ], 3285 "thresholds": [], 3286 "timeFrom": null, 3287 "timeShift": "$timeshift", 3288 "title": "[SECONDARY] Requests rate by priority level", 3289 "tooltip": { 3290 "msResolution": true, 3291 "shared": true, 3292 "sort": 2, 3293 "value_type": "cumulative" 3294 }, 3295 "transformations": [], 3296 "transparent": false, 3297 "type": "graph", 3298 "xaxis": { 3299 "mode": "time", 3300 "name": null, 3301 "show": true, 3302 "values": [] 3303 }, 3304 "yaxes": [ 3305 { 3306 "decimals": null, 3307 "format": "ops", 3308 "label": null, 3309 "logBase": 1, 3310 "max": null, 3311 "min": null, 3312 "show": true 3313 }, 3314 { 3315 "decimals": null, 3316 "format": "short", 3317 "label": null, 3318 "logBase": 1, 3319 "max": null, 3320 "min": null, 3321 "show": true 3322 } 3323 ], 3324 "yaxis": { 3325 "align": false, 3326 "alignLevel": 0 3327 } 3328 }, 3329 { 3330 "aliasColors": {}, 3331 "bars": false, 3332 "cacheTimeout": null, 3333 "datasource": "$source", 3334 "description": null, 3335 "editable": true, 3336 "error": false, 3337 "fieldConfig": { 3338 "defaults": { 3339 "thresholds": { 3340 "mode": "absolute", 3341 "steps": [] 3342 } 3343 } 3344 }, 3345 "fill": 1, 3346 "grid": { 3347 "threshold1": null, 3348 "threshold1Color": "rgba(216, 200, 27, 0.27)", 3349 "threshold2": null, 3350 "threshold2Color": "rgba(234, 112, 112, 0.22)" 3351 }, 3352 "gridPos": null, 3353 "height": null, 3354 "hideTimeOverride": false, 3355 "id": 25, 3356 "interval": null, 3357 "isNew": true, 3358 "legend": { 3359 "alignAsTable": false, 3360 "avg": false, 3361 "current": false, 3362 "hideEmpty": false, 3363 "hideZero": false, 3364 "max": false, 3365 "min": false, 3366 "rightSide": false, 3367 "show": true, 3368 "sideWidth": null, 3369 "sort": null, 3370 "sortDesc": false, 3371 "total": false, 3372 "values": false 3373 }, 3374 "lines": true, 3375 "linewidth": 2, 3376 "links": [], 3377 "maxDataPoints": 100, 3378 "maxPerRow": null, 3379 "minSpan": null, 3380 "nullPointMode": null, 3381 "options": { 3382 "alertThreshold": true, 3383 "dataLinks": [] 3384 }, 3385 "percentage": false, 3386 "pointradius": 5, 3387 "points": false, 3388 "renderer": "flot", 3389 "repeat": null, 3390 "repeatDirection": null, 3391 "seriesOverrides": [], 3392 "span": 12, 3393 "stack": false, 3394 "steppedLine": false, 3395 "targets": [ 3396 { 3397 "datasource": "$source", 3398 "expr": "sum(apiserver_flowcontrol_request_concurrency_in_use{instance=~\".*(${instance:pipe})\"}) by (instance, priority_level)", 3399 "format": "time_series", 3400 "hide": false, 3401 "instant": false, 3402 "interval": "5s", 3403 "intervalFactor": 1, 3404 "legendFormat": "{{instance}} {{priority_level}}", 3405 "metric": "", 3406 "query": "sum(apiserver_flowcontrol_request_concurrency_in_use{instance=~\".*(${instance:pipe})\"}) by (instance, priority_level)", 3407 "refId": "", 3408 "step": 10, 3409 "target": "" 3410 } 3411 ], 3412 "thresholds": [], 3413 "timeFrom": null, 3414 "timeShift": null, 3415 "title": "Concurrency in use", 3416 "tooltip": { 3417 "msResolution": true, 3418 "shared": true, 3419 "sort": 2, 3420 "value_type": "cumulative" 3421 }, 3422 "transformations": [], 3423 "transparent": false, 3424 "type": "graph", 3425 "xaxis": { 3426 "mode": "time", 3427 "name": null, 3428 "show": true, 3429 "values": [] 3430 }, 3431 "yaxes": [ 3432 { 3433 "decimals": null, 3434 "format": "ops", 3435 "label": null, 3436 "logBase": 1, 3437 "max": null, 3438 "min": null, 3439 "show": true 3440 }, 3441 { 3442 "decimals": null, 3443 "format": "short", 3444 "label": null, 3445 "logBase": 1, 3446 "max": null, 3447 "min": null, 3448 "show": true 3449 } 3450 ], 3451 "yaxis": { 3452 "align": false, 3453 "alignLevel": 0 3454 } 3455 }, 3456 { 3457 "aliasColors": {}, 3458 "bars": false, 3459 "cacheTimeout": null, 3460 "datasource": "$secondary_source", 3461 "description": null, 3462 "editable": true, 3463 "error": false, 3464 "fieldConfig": { 3465 "defaults": { 3466 "thresholds": { 3467 "mode": "absolute", 3468 "steps": [] 3469 } 3470 } 3471 }, 3472 "fill": 1, 3473 "grid": { 3474 "threshold1": null, 3475 "threshold1Color": "rgba(216, 200, 27, 0.27)", 3476 "threshold2": null, 3477 "threshold2Color": "rgba(234, 112, 112, 0.22)" 3478 }, 3479 "gridPos": null, 3480 "height": null, 3481 "hideTimeOverride": false, 3482 "id": 26, 3483 "interval": null, 3484 "isNew": true, 3485 "legend": { 3486 "alignAsTable": false, 3487 "avg": false, 3488 "current": false, 3489 "hideEmpty": false, 3490 "hideZero": false, 3491 "max": false, 3492 "min": false, 3493 "rightSide": false, 3494 "show": true, 3495 "sideWidth": null, 3496 "sort": null, 3497 "sortDesc": false, 3498 "total": false, 3499 "values": false 3500 }, 3501 "lines": true, 3502 "linewidth": 2, 3503 "links": [], 3504 "maxDataPoints": 100, 3505 "maxPerRow": null, 3506 "minSpan": null, 3507 "nullPointMode": null, 3508 "options": { 3509 "alertThreshold": true, 3510 "dataLinks": [] 3511 }, 3512 "percentage": false, 3513 "pointradius": 5, 3514 "points": false, 3515 "renderer": "flot", 3516 "repeat": null, 3517 "repeatDirection": null, 3518 "seriesOverrides": [], 3519 "span": 12, 3520 "stack": false, 3521 "steppedLine": false, 3522 "targets": [ 3523 { 3524 "datasource": "$source", 3525 "expr": "sum(apiserver_flowcontrol_request_concurrency_in_use{instance=~\".*(${instance:pipe})\"}) by (instance, priority_level)", 3526 "format": "time_series", 3527 "hide": false, 3528 "instant": false, 3529 "interval": "5s", 3530 "intervalFactor": 1, 3531 "legendFormat": "{{instance}} {{priority_level}}", 3532 "metric": "", 3533 "query": "sum(apiserver_flowcontrol_request_concurrency_in_use{instance=~\".*(${instance:pipe})\"}) by (instance, priority_level)", 3534 "refId": "", 3535 "step": 10, 3536 "target": "" 3537 } 3538 ], 3539 "thresholds": [], 3540 "timeFrom": null, 3541 "timeShift": "$timeshift", 3542 "title": "[SECONDARY] Concurrency in use", 3543 "tooltip": { 3544 "msResolution": true, 3545 "shared": true, 3546 "sort": 2, 3547 "value_type": "cumulative" 3548 }, 3549 "transformations": [], 3550 "transparent": false, 3551 "type": "graph", 3552 "xaxis": { 3553 "mode": "time", 3554 "name": null, 3555 "show": true, 3556 "values": [] 3557 }, 3558 "yaxes": [ 3559 { 3560 "decimals": null, 3561 "format": "ops", 3562 "label": null, 3563 "logBase": 1, 3564 "max": null, 3565 "min": null, 3566 "show": true 3567 }, 3568 { 3569 "decimals": null, 3570 "format": "short", 3571 "label": null, 3572 "logBase": 1, 3573 "max": null, 3574 "min": null, 3575 "show": true 3576 } 3577 ], 3578 "yaxis": { 3579 "align": false, 3580 "alignLevel": 0 3581 } 3582 }, 3583 { 3584 "aliasColors": {}, 3585 "bars": false, 3586 "cacheTimeout": null, 3587 "datasource": "$source", 3588 "description": null, 3589 "editable": true, 3590 "error": false, 3591 "fieldConfig": { 3592 "defaults": { 3593 "thresholds": { 3594 "mode": "absolute", 3595 "steps": [] 3596 } 3597 } 3598 }, 3599 "fill": 1, 3600 "grid": { 3601 "threshold1": null, 3602 "threshold1Color": "rgba(216, 200, 27, 0.27)", 3603 "threshold2": null, 3604 "threshold2Color": "rgba(234, 112, 112, 0.22)" 3605 }, 3606 "gridPos": null, 3607 "height": null, 3608 "hideTimeOverride": false, 3609 "id": 27, 3610 "interval": null, 3611 "isNew": true, 3612 "legend": { 3613 "alignAsTable": false, 3614 "avg": false, 3615 "current": false, 3616 "hideEmpty": false, 3617 "hideZero": false, 3618 "max": false, 3619 "min": false, 3620 "rightSide": false, 3621 "show": true, 3622 "sideWidth": null, 3623 "sort": null, 3624 "sortDesc": false, 3625 "total": false, 3626 "values": false 3627 }, 3628 "lines": true, 3629 "linewidth": 2, 3630 "links": [], 3631 "maxDataPoints": 100, 3632 "maxPerRow": null, 3633 "minSpan": null, 3634 "nullPointMode": null, 3635 "options": { 3636 "alertThreshold": true, 3637 "dataLinks": [] 3638 }, 3639 "percentage": false, 3640 "pointradius": 5, 3641 "points": false, 3642 "renderer": "flot", 3643 "repeat": null, 3644 "repeatDirection": null, 3645 "seriesOverrides": [], 3646 "span": 12, 3647 "stack": false, 3648 "steppedLine": false, 3649 "targets": [ 3650 { 3651 "datasource": "$source", 3652 "expr": "sum(apiserver_flowcontrol_current_executing_requests{instance=~\".*(${instance:pipe})\"}) by (instance, priority_level)", 3653 "format": "time_series", 3654 "hide": false, 3655 "instant": false, 3656 "interval": "5s", 3657 "intervalFactor": 1, 3658 "legendFormat": "{{instance}} {{priority_level}}", 3659 "metric": "", 3660 "query": "sum(apiserver_flowcontrol_current_executing_requests{instance=~\".*(${instance:pipe})\"}) by (instance, priority_level)", 3661 "refId": "", 3662 "step": 10, 3663 "target": "" 3664 } 3665 ], 3666 "thresholds": [], 3667 "timeFrom": null, 3668 "timeShift": null, 3669 "title": "Current executing requests", 3670 "tooltip": { 3671 "msResolution": true, 3672 "shared": true, 3673 "sort": 2, 3674 "value_type": "cumulative" 3675 }, 3676 "transformations": [], 3677 "transparent": false, 3678 "type": "graph", 3679 "xaxis": { 3680 "mode": "time", 3681 "name": null, 3682 "show": true, 3683 "values": [] 3684 }, 3685 "yaxes": [ 3686 { 3687 "decimals": null, 3688 "format": "ops", 3689 "label": null, 3690 "logBase": 1, 3691 "max": null, 3692 "min": null, 3693 "show": true 3694 }, 3695 { 3696 "decimals": null, 3697 "format": "short", 3698 "label": null, 3699 "logBase": 1, 3700 "max": null, 3701 "min": null, 3702 "show": true 3703 } 3704 ], 3705 "yaxis": { 3706 "align": false, 3707 "alignLevel": 0 3708 } 3709 }, 3710 { 3711 "aliasColors": {}, 3712 "bars": false, 3713 "cacheTimeout": null, 3714 "datasource": "$secondary_source", 3715 "description": null, 3716 "editable": true, 3717 "error": false, 3718 "fieldConfig": { 3719 "defaults": { 3720 "thresholds": { 3721 "mode": "absolute", 3722 "steps": [] 3723 } 3724 } 3725 }, 3726 "fill": 1, 3727 "grid": { 3728 "threshold1": null, 3729 "threshold1Color": "rgba(216, 200, 27, 0.27)", 3730 "threshold2": null, 3731 "threshold2Color": "rgba(234, 112, 112, 0.22)" 3732 }, 3733 "gridPos": null, 3734 "height": null, 3735 "hideTimeOverride": false, 3736 "id": 28, 3737 "interval": null, 3738 "isNew": true, 3739 "legend": { 3740 "alignAsTable": false, 3741 "avg": false, 3742 "current": false, 3743 "hideEmpty": false, 3744 "hideZero": false, 3745 "max": false, 3746 "min": false, 3747 "rightSide": false, 3748 "show": true, 3749 "sideWidth": null, 3750 "sort": null, 3751 "sortDesc": false, 3752 "total": false, 3753 "values": false 3754 }, 3755 "lines": true, 3756 "linewidth": 2, 3757 "links": [], 3758 "maxDataPoints": 100, 3759 "maxPerRow": null, 3760 "minSpan": null, 3761 "nullPointMode": null, 3762 "options": { 3763 "alertThreshold": true, 3764 "dataLinks": [] 3765 }, 3766 "percentage": false, 3767 "pointradius": 5, 3768 "points": false, 3769 "renderer": "flot", 3770 "repeat": null, 3771 "repeatDirection": null, 3772 "seriesOverrides": [], 3773 "span": 12, 3774 "stack": false, 3775 "steppedLine": false, 3776 "targets": [ 3777 { 3778 "datasource": "$source", 3779 "expr": "sum(apiserver_flowcontrol_current_executing_requests{instance=~\".*(${instance:pipe})\"}) by (instance, priority_level)", 3780 "format": "time_series", 3781 "hide": false, 3782 "instant": false, 3783 "interval": "5s", 3784 "intervalFactor": 1, 3785 "legendFormat": "{{instance}} {{priority_level}}", 3786 "metric": "", 3787 "query": "sum(apiserver_flowcontrol_current_executing_requests{instance=~\".*(${instance:pipe})\"}) by (instance, priority_level)", 3788 "refId": "", 3789 "step": 10, 3790 "target": "" 3791 } 3792 ], 3793 "thresholds": [], 3794 "timeFrom": null, 3795 "timeShift": "$timeshift", 3796 "title": "[SECONDARY] Current executing requests", 3797 "tooltip": { 3798 "msResolution": true, 3799 "shared": true, 3800 "sort": 2, 3801 "value_type": "cumulative" 3802 }, 3803 "transformations": [], 3804 "transparent": false, 3805 "type": "graph", 3806 "xaxis": { 3807 "mode": "time", 3808 "name": null, 3809 "show": true, 3810 "values": [] 3811 }, 3812 "yaxes": [ 3813 { 3814 "decimals": null, 3815 "format": "ops", 3816 "label": null, 3817 "logBase": 1, 3818 "max": null, 3819 "min": null, 3820 "show": true 3821 }, 3822 { 3823 "decimals": null, 3824 "format": "short", 3825 "label": null, 3826 "logBase": 1, 3827 "max": null, 3828 "min": null, 3829 "show": true 3830 } 3831 ], 3832 "yaxis": { 3833 "align": false, 3834 "alignLevel": 0 3835 } 3836 }, 3837 { 3838 "aliasColors": {}, 3839 "bars": false, 3840 "cacheTimeout": null, 3841 "datasource": "$source", 3842 "description": null, 3843 "editable": true, 3844 "error": false, 3845 "fieldConfig": { 3846 "defaults": { 3847 "thresholds": { 3848 "mode": "absolute", 3849 "steps": [] 3850 } 3851 } 3852 }, 3853 "fill": 1, 3854 "grid": { 3855 "threshold1": null, 3856 "threshold1Color": "rgba(216, 200, 27, 0.27)", 3857 "threshold2": null, 3858 "threshold2Color": "rgba(234, 112, 112, 0.22)" 3859 }, 3860 "gridPos": null, 3861 "height": null, 3862 "hideTimeOverride": false, 3863 "id": 29, 3864 "interval": null, 3865 "isNew": true, 3866 "legend": { 3867 "alignAsTable": false, 3868 "avg": false, 3869 "current": false, 3870 "hideEmpty": false, 3871 "hideZero": false, 3872 "max": false, 3873 "min": false, 3874 "rightSide": false, 3875 "show": true, 3876 "sideWidth": null, 3877 "sort": null, 3878 "sortDesc": false, 3879 "total": false, 3880 "values": false 3881 }, 3882 "lines": true, 3883 "linewidth": 2, 3884 "links": [], 3885 "maxDataPoints": 100, 3886 "maxPerRow": null, 3887 "minSpan": null, 3888 "nullPointMode": null, 3889 "options": { 3890 "alertThreshold": true, 3891 "dataLinks": [] 3892 }, 3893 "percentage": false, 3894 "pointradius": 5, 3895 "points": false, 3896 "renderer": "flot", 3897 "repeat": null, 3898 "repeatDirection": null, 3899 "seriesOverrides": [], 3900 "span": 12, 3901 "stack": false, 3902 "steppedLine": false, 3903 "targets": [ 3904 { 3905 "datasource": "$source", 3906 "expr": "sum(apiserver_flowcontrol_current_inqueue_requests{instance=~\".*(${instance:pipe})\"}) by (instance, priority_level)", 3907 "format": "time_series", 3908 "hide": false, 3909 "instant": false, 3910 "interval": "5s", 3911 "intervalFactor": 1, 3912 "legendFormat": "{{instance}} {{priority_level}}", 3913 "metric": "", 3914 "query": "sum(apiserver_flowcontrol_current_inqueue_requests{instance=~\".*(${instance:pipe})\"}) by (instance, priority_level)", 3915 "refId": "", 3916 "step": 10, 3917 "target": "" 3918 } 3919 ], 3920 "thresholds": [], 3921 "timeFrom": null, 3922 "timeShift": null, 3923 "title": "Inqueue requests", 3924 "tooltip": { 3925 "msResolution": true, 3926 "shared": true, 3927 "sort": 2, 3928 "value_type": "cumulative" 3929 }, 3930 "transformations": [], 3931 "transparent": false, 3932 "type": "graph", 3933 "xaxis": { 3934 "mode": "time", 3935 "name": null, 3936 "show": true, 3937 "values": [] 3938 }, 3939 "yaxes": [ 3940 { 3941 "decimals": null, 3942 "format": "ops", 3943 "label": null, 3944 "logBase": 1, 3945 "max": null, 3946 "min": null, 3947 "show": true 3948 }, 3949 { 3950 "decimals": null, 3951 "format": "short", 3952 "label": null, 3953 "logBase": 1, 3954 "max": null, 3955 "min": null, 3956 "show": true 3957 } 3958 ], 3959 "yaxis": { 3960 "align": false, 3961 "alignLevel": 0 3962 } 3963 }, 3964 { 3965 "aliasColors": {}, 3966 "bars": false, 3967 "cacheTimeout": null, 3968 "datasource": "$secondary_source", 3969 "description": null, 3970 "editable": true, 3971 "error": false, 3972 "fieldConfig": { 3973 "defaults": { 3974 "thresholds": { 3975 "mode": "absolute", 3976 "steps": [] 3977 } 3978 } 3979 }, 3980 "fill": 1, 3981 "grid": { 3982 "threshold1": null, 3983 "threshold1Color": "rgba(216, 200, 27, 0.27)", 3984 "threshold2": null, 3985 "threshold2Color": "rgba(234, 112, 112, 0.22)" 3986 }, 3987 "gridPos": null, 3988 "height": null, 3989 "hideTimeOverride": false, 3990 "id": 30, 3991 "interval": null, 3992 "isNew": true, 3993 "legend": { 3994 "alignAsTable": false, 3995 "avg": false, 3996 "current": false, 3997 "hideEmpty": false, 3998 "hideZero": false, 3999 "max": false, 4000 "min": false, 4001 "rightSide": false, 4002 "show": true, 4003 "sideWidth": null, 4004 "sort": null, 4005 "sortDesc": false, 4006 "total": false, 4007 "values": false 4008 }, 4009 "lines": true, 4010 "linewidth": 2, 4011 "links": [], 4012 "maxDataPoints": 100, 4013 "maxPerRow": null, 4014 "minSpan": null, 4015 "nullPointMode": null, 4016 "options": { 4017 "alertThreshold": true, 4018 "dataLinks": [] 4019 }, 4020 "percentage": false, 4021 "pointradius": 5, 4022 "points": false, 4023 "renderer": "flot", 4024 "repeat": null, 4025 "repeatDirection": null, 4026 "seriesOverrides": [], 4027 "span": 12, 4028 "stack": false, 4029 "steppedLine": false, 4030 "targets": [ 4031 { 4032 "datasource": "$source", 4033 "expr": "sum(apiserver_flowcontrol_current_inqueue_requests{instance=~\".*(${instance:pipe})\"}) by (instance, priority_level)", 4034 "format": "time_series", 4035 "hide": false, 4036 "instant": false, 4037 "interval": "5s", 4038 "intervalFactor": 1, 4039 "legendFormat": "{{instance}} {{priority_level}}", 4040 "metric": "", 4041 "query": "sum(apiserver_flowcontrol_current_inqueue_requests{instance=~\".*(${instance:pipe})\"}) by (instance, priority_level)", 4042 "refId": "", 4043 "step": 10, 4044 "target": "" 4045 } 4046 ], 4047 "thresholds": [], 4048 "timeFrom": null, 4049 "timeShift": "$timeshift", 4050 "title": "[SECONDARY] Inqueue requests", 4051 "tooltip": { 4052 "msResolution": true, 4053 "shared": true, 4054 "sort": 2, 4055 "value_type": "cumulative" 4056 }, 4057 "transformations": [], 4058 "transparent": false, 4059 "type": "graph", 4060 "xaxis": { 4061 "mode": "time", 4062 "name": null, 4063 "show": true, 4064 "values": [] 4065 }, 4066 "yaxes": [ 4067 { 4068 "decimals": null, 4069 "format": "ops", 4070 "label": null, 4071 "logBase": 1, 4072 "max": null, 4073 "min": null, 4074 "show": true 4075 }, 4076 { 4077 "decimals": null, 4078 "format": "short", 4079 "label": null, 4080 "logBase": 1, 4081 "max": null, 4082 "min": null, 4083 "show": true 4084 } 4085 ], 4086 "yaxis": { 4087 "align": false, 4088 "alignLevel": 0 4089 } 4090 }, 4091 { 4092 "aliasColors": {}, 4093 "bars": false, 4094 "cacheTimeout": null, 4095 "datasource": "$source", 4096 "description": null, 4097 "editable": true, 4098 "error": false, 4099 "fieldConfig": { 4100 "defaults": { 4101 "thresholds": { 4102 "mode": "absolute", 4103 "steps": [] 4104 } 4105 } 4106 }, 4107 "fill": 1, 4108 "grid": { 4109 "threshold1": null, 4110 "threshold1Color": "rgba(216, 200, 27, 0.27)", 4111 "threshold2": null, 4112 "threshold2Color": "rgba(234, 112, 112, 0.22)" 4113 }, 4114 "gridPos": null, 4115 "height": null, 4116 "hideTimeOverride": false, 4117 "id": 31, 4118 "interval": null, 4119 "isNew": true, 4120 "legend": { 4121 "alignAsTable": false, 4122 "avg": false, 4123 "current": false, 4124 "hideEmpty": false, 4125 "hideZero": false, 4126 "max": false, 4127 "min": false, 4128 "rightSide": false, 4129 "show": true, 4130 "sideWidth": null, 4131 "sort": null, 4132 "sortDesc": false, 4133 "total": false, 4134 "values": false 4135 }, 4136 "lines": true, 4137 "linewidth": 2, 4138 "links": [], 4139 "maxDataPoints": 100, 4140 "maxPerRow": null, 4141 "minSpan": null, 4142 "nullPointMode": null, 4143 "options": { 4144 "alertThreshold": true, 4145 "dataLinks": [] 4146 }, 4147 "percentage": false, 4148 "pointradius": 5, 4149 "points": false, 4150 "renderer": "flot", 4151 "repeat": null, 4152 "repeatDirection": null, 4153 "seriesOverrides": [], 4154 "span": 12, 4155 "stack": false, 4156 "steppedLine": false, 4157 "targets": [ 4158 { 4159 "datasource": "$source", 4160 "expr": "avg(apiserver_flowcontrol_nominal_limit_seats{instance=~\".*(${instance:pipe})\"}) by (priority_level)", 4161 "format": "time_series", 4162 "hide": false, 4163 "instant": false, 4164 "interval": "5s", 4165 "intervalFactor": 1, 4166 "legendFormat": "{{priority_level}}", 4167 "metric": "", 4168 "query": "avg(apiserver_flowcontrol_nominal_limit_seats{instance=~\".*(${instance:pipe})\"}) by (priority_level)", 4169 "refId": "", 4170 "step": 10, 4171 "target": "" 4172 } 4173 ], 4174 "thresholds": [], 4175 "timeFrom": null, 4176 "timeShift": null, 4177 "title": "Nominal number of execution seats", 4178 "tooltip": { 4179 "msResolution": true, 4180 "shared": true, 4181 "sort": 2, 4182 "value_type": "cumulative" 4183 }, 4184 "transformations": [], 4185 "transparent": false, 4186 "type": "graph", 4187 "xaxis": { 4188 "mode": "time", 4189 "name": null, 4190 "show": true, 4191 "values": [] 4192 }, 4193 "yaxes": [ 4194 { 4195 "decimals": null, 4196 "format": "short", 4197 "label": null, 4198 "logBase": 1, 4199 "max": null, 4200 "min": null, 4201 "show": true 4202 }, 4203 { 4204 "decimals": null, 4205 "format": "short", 4206 "label": null, 4207 "logBase": 1, 4208 "max": null, 4209 "min": null, 4210 "show": true 4211 } 4212 ], 4213 "yaxis": { 4214 "align": false, 4215 "alignLevel": 0 4216 } 4217 }, 4218 { 4219 "aliasColors": {}, 4220 "bars": false, 4221 "cacheTimeout": null, 4222 "datasource": "$secondary_source", 4223 "description": null, 4224 "editable": true, 4225 "error": false, 4226 "fieldConfig": { 4227 "defaults": { 4228 "thresholds": { 4229 "mode": "absolute", 4230 "steps": [] 4231 } 4232 } 4233 }, 4234 "fill": 1, 4235 "grid": { 4236 "threshold1": null, 4237 "threshold1Color": "rgba(216, 200, 27, 0.27)", 4238 "threshold2": null, 4239 "threshold2Color": "rgba(234, 112, 112, 0.22)" 4240 }, 4241 "gridPos": null, 4242 "height": null, 4243 "hideTimeOverride": false, 4244 "id": 32, 4245 "interval": null, 4246 "isNew": true, 4247 "legend": { 4248 "alignAsTable": false, 4249 "avg": false, 4250 "current": false, 4251 "hideEmpty": false, 4252 "hideZero": false, 4253 "max": false, 4254 "min": false, 4255 "rightSide": false, 4256 "show": true, 4257 "sideWidth": null, 4258 "sort": null, 4259 "sortDesc": false, 4260 "total": false, 4261 "values": false 4262 }, 4263 "lines": true, 4264 "linewidth": 2, 4265 "links": [], 4266 "maxDataPoints": 100, 4267 "maxPerRow": null, 4268 "minSpan": null, 4269 "nullPointMode": null, 4270 "options": { 4271 "alertThreshold": true, 4272 "dataLinks": [] 4273 }, 4274 "percentage": false, 4275 "pointradius": 5, 4276 "points": false, 4277 "renderer": "flot", 4278 "repeat": null, 4279 "repeatDirection": null, 4280 "seriesOverrides": [], 4281 "span": 12, 4282 "stack": false, 4283 "steppedLine": false, 4284 "targets": [ 4285 { 4286 "datasource": "$source", 4287 "expr": "avg(apiserver_flowcontrol_nominal_limit_seats{instance=~\".*(${instance:pipe})\"}) by (priority_level)", 4288 "format": "time_series", 4289 "hide": false, 4290 "instant": false, 4291 "interval": "5s", 4292 "intervalFactor": 1, 4293 "legendFormat": "{{priority_level}}", 4294 "metric": "", 4295 "query": "avg(apiserver_flowcontrol_nominal_limit_seats{instance=~\".*(${instance:pipe})\"}) by (priority_level)", 4296 "refId": "", 4297 "step": 10, 4298 "target": "" 4299 } 4300 ], 4301 "thresholds": [], 4302 "timeFrom": null, 4303 "timeShift": "$timeshift", 4304 "title": "[SECONDARY] Nominal number of execution seats", 4305 "tooltip": { 4306 "msResolution": true, 4307 "shared": true, 4308 "sort": 2, 4309 "value_type": "cumulative" 4310 }, 4311 "transformations": [], 4312 "transparent": false, 4313 "type": "graph", 4314 "xaxis": { 4315 "mode": "time", 4316 "name": null, 4317 "show": true, 4318 "values": [] 4319 }, 4320 "yaxes": [ 4321 { 4322 "decimals": null, 4323 "format": "short", 4324 "label": null, 4325 "logBase": 1, 4326 "max": null, 4327 "min": null, 4328 "show": true 4329 }, 4330 { 4331 "decimals": null, 4332 "format": "short", 4333 "label": null, 4334 "logBase": 1, 4335 "max": null, 4336 "min": null, 4337 "show": true 4338 } 4339 ], 4340 "yaxis": { 4341 "align": false, 4342 "alignLevel": 0 4343 } 4344 }, 4345 { 4346 "aliasColors": {}, 4347 "bars": false, 4348 "cacheTimeout": null, 4349 "datasource": "$source", 4350 "description": null, 4351 "editable": true, 4352 "error": false, 4353 "fieldConfig": { 4354 "defaults": { 4355 "thresholds": { 4356 "mode": "absolute", 4357 "steps": [] 4358 } 4359 } 4360 }, 4361 "fill": 1, 4362 "grid": { 4363 "threshold1": null, 4364 "threshold1Color": "rgba(216, 200, 27, 0.27)", 4365 "threshold2": null, 4366 "threshold2Color": "rgba(234, 112, 112, 0.22)" 4367 }, 4368 "gridPos": null, 4369 "height": null, 4370 "hideTimeOverride": false, 4371 "id": 33, 4372 "interval": null, 4373 "isNew": true, 4374 "legend": { 4375 "alignAsTable": false, 4376 "avg": false, 4377 "current": false, 4378 "hideEmpty": false, 4379 "hideZero": false, 4380 "max": false, 4381 "min": false, 4382 "rightSide": false, 4383 "show": true, 4384 "sideWidth": null, 4385 "sort": null, 4386 "sortDesc": false, 4387 "total": false, 4388 "values": false 4389 }, 4390 "lines": true, 4391 "linewidth": 2, 4392 "links": [], 4393 "maxDataPoints": 100, 4394 "maxPerRow": null, 4395 "minSpan": null, 4396 "nullPointMode": null, 4397 "options": { 4398 "alertThreshold": true, 4399 "dataLinks": [] 4400 }, 4401 "percentage": false, 4402 "pointradius": 5, 4403 "points": false, 4404 "renderer": "flot", 4405 "repeat": null, 4406 "repeatDirection": null, 4407 "seriesOverrides": [], 4408 "span": 12, 4409 "stack": false, 4410 "steppedLine": false, 4411 "targets": [ 4412 { 4413 "datasource": "$source", 4414 "expr": "avg(apiserver_flowcontrol_lower_limit_seats{instance=~\".*(${instance:pipe})\"}) by (priority_level)", 4415 "format": "time_series", 4416 "hide": false, 4417 "instant": false, 4418 "interval": "5s", 4419 "intervalFactor": 1, 4420 "legendFormat": "{{priority_level}}", 4421 "metric": "", 4422 "query": "avg(apiserver_flowcontrol_lower_limit_seats{instance=~\".*(${instance:pipe})\"}) by (priority_level)", 4423 "refId": "", 4424 "step": 10, 4425 "target": "" 4426 } 4427 ], 4428 "thresholds": [], 4429 "timeFrom": null, 4430 "timeShift": null, 4431 "title": "Lower bound on number of execution seats", 4432 "tooltip": { 4433 "msResolution": true, 4434 "shared": true, 4435 "sort": 2, 4436 "value_type": "cumulative" 4437 }, 4438 "transformations": [], 4439 "transparent": false, 4440 "type": "graph", 4441 "xaxis": { 4442 "mode": "time", 4443 "name": null, 4444 "show": true, 4445 "values": [] 4446 }, 4447 "yaxes": [ 4448 { 4449 "decimals": null, 4450 "format": "short", 4451 "label": null, 4452 "logBase": 1, 4453 "max": null, 4454 "min": null, 4455 "show": true 4456 }, 4457 { 4458 "decimals": null, 4459 "format": "short", 4460 "label": null, 4461 "logBase": 1, 4462 "max": null, 4463 "min": null, 4464 "show": true 4465 } 4466 ], 4467 "yaxis": { 4468 "align": false, 4469 "alignLevel": 0 4470 } 4471 }, 4472 { 4473 "aliasColors": {}, 4474 "bars": false, 4475 "cacheTimeout": null, 4476 "datasource": "$secondary_source", 4477 "description": null, 4478 "editable": true, 4479 "error": false, 4480 "fieldConfig": { 4481 "defaults": { 4482 "thresholds": { 4483 "mode": "absolute", 4484 "steps": [] 4485 } 4486 } 4487 }, 4488 "fill": 1, 4489 "grid": { 4490 "threshold1": null, 4491 "threshold1Color": "rgba(216, 200, 27, 0.27)", 4492 "threshold2": null, 4493 "threshold2Color": "rgba(234, 112, 112, 0.22)" 4494 }, 4495 "gridPos": null, 4496 "height": null, 4497 "hideTimeOverride": false, 4498 "id": 34, 4499 "interval": null, 4500 "isNew": true, 4501 "legend": { 4502 "alignAsTable": false, 4503 "avg": false, 4504 "current": false, 4505 "hideEmpty": false, 4506 "hideZero": false, 4507 "max": false, 4508 "min": false, 4509 "rightSide": false, 4510 "show": true, 4511 "sideWidth": null, 4512 "sort": null, 4513 "sortDesc": false, 4514 "total": false, 4515 "values": false 4516 }, 4517 "lines": true, 4518 "linewidth": 2, 4519 "links": [], 4520 "maxDataPoints": 100, 4521 "maxPerRow": null, 4522 "minSpan": null, 4523 "nullPointMode": null, 4524 "options": { 4525 "alertThreshold": true, 4526 "dataLinks": [] 4527 }, 4528 "percentage": false, 4529 "pointradius": 5, 4530 "points": false, 4531 "renderer": "flot", 4532 "repeat": null, 4533 "repeatDirection": null, 4534 "seriesOverrides": [], 4535 "span": 12, 4536 "stack": false, 4537 "steppedLine": false, 4538 "targets": [ 4539 { 4540 "datasource": "$source", 4541 "expr": "avg(apiserver_flowcontrol_lower_limit_seats{instance=~\".*(${instance:pipe})\"}) by (priority_level)", 4542 "format": "time_series", 4543 "hide": false, 4544 "instant": false, 4545 "interval": "5s", 4546 "intervalFactor": 1, 4547 "legendFormat": "{{priority_level}}", 4548 "metric": "", 4549 "query": "avg(apiserver_flowcontrol_lower_limit_seats{instance=~\".*(${instance:pipe})\"}) by (priority_level)", 4550 "refId": "", 4551 "step": 10, 4552 "target": "" 4553 } 4554 ], 4555 "thresholds": [], 4556 "timeFrom": null, 4557 "timeShift": "$timeshift", 4558 "title": "[SECONDARY] Lower bound on number of execution seats", 4559 "tooltip": { 4560 "msResolution": true, 4561 "shared": true, 4562 "sort": 2, 4563 "value_type": "cumulative" 4564 }, 4565 "transformations": [], 4566 "transparent": false, 4567 "type": "graph", 4568 "xaxis": { 4569 "mode": "time", 4570 "name": null, 4571 "show": true, 4572 "values": [] 4573 }, 4574 "yaxes": [ 4575 { 4576 "decimals": null, 4577 "format": "short", 4578 "label": null, 4579 "logBase": 1, 4580 "max": null, 4581 "min": null, 4582 "show": true 4583 }, 4584 { 4585 "decimals": null, 4586 "format": "short", 4587 "label": null, 4588 "logBase": 1, 4589 "max": null, 4590 "min": null, 4591 "show": true 4592 } 4593 ], 4594 "yaxis": { 4595 "align": false, 4596 "alignLevel": 0 4597 } 4598 }, 4599 { 4600 "aliasColors": {}, 4601 "bars": false, 4602 "cacheTimeout": null, 4603 "datasource": "$source", 4604 "description": null, 4605 "editable": true, 4606 "error": false, 4607 "fieldConfig": { 4608 "defaults": { 4609 "thresholds": { 4610 "mode": "absolute", 4611 "steps": [] 4612 } 4613 } 4614 }, 4615 "fill": 1, 4616 "grid": { 4617 "threshold1": null, 4618 "threshold1Color": "rgba(216, 200, 27, 0.27)", 4619 "threshold2": null, 4620 "threshold2Color": "rgba(234, 112, 112, 0.22)" 4621 }, 4622 "gridPos": null, 4623 "height": null, 4624 "hideTimeOverride": false, 4625 "id": 35, 4626 "interval": null, 4627 "isNew": true, 4628 "legend": { 4629 "alignAsTable": false, 4630 "avg": false, 4631 "current": false, 4632 "hideEmpty": false, 4633 "hideZero": false, 4634 "max": false, 4635 "min": false, 4636 "rightSide": false, 4637 "show": true, 4638 "sideWidth": null, 4639 "sort": null, 4640 "sortDesc": false, 4641 "total": false, 4642 "values": false 4643 }, 4644 "lines": true, 4645 "linewidth": 2, 4646 "links": [], 4647 "maxDataPoints": 100, 4648 "maxPerRow": null, 4649 "minSpan": null, 4650 "nullPointMode": null, 4651 "options": { 4652 "alertThreshold": true, 4653 "dataLinks": [] 4654 }, 4655 "percentage": false, 4656 "pointradius": 5, 4657 "points": false, 4658 "renderer": "flot", 4659 "repeat": null, 4660 "repeatDirection": null, 4661 "seriesOverrides": [], 4662 "span": 12, 4663 "stack": false, 4664 "steppedLine": false, 4665 "targets": [ 4666 { 4667 "datasource": "$source", 4668 "expr": "avg(apiserver_flowcontrol_upper_limit_seats{instance=~\".*(${instance:pipe})\"}) by (priority_level)", 4669 "format": "time_series", 4670 "hide": false, 4671 "instant": false, 4672 "interval": "5s", 4673 "intervalFactor": 1, 4674 "legendFormat": "{{priority_level}}", 4675 "metric": "", 4676 "query": "avg(apiserver_flowcontrol_upper_limit_seats{instance=~\".*(${instance:pipe})\"}) by (priority_level)", 4677 "refId": "", 4678 "step": 10, 4679 "target": "" 4680 } 4681 ], 4682 "thresholds": [], 4683 "timeFrom": null, 4684 "timeShift": null, 4685 "title": "Upper bound on number of execution seats", 4686 "tooltip": { 4687 "msResolution": true, 4688 "shared": true, 4689 "sort": 2, 4690 "value_type": "cumulative" 4691 }, 4692 "transformations": [], 4693 "transparent": false, 4694 "type": "graph", 4695 "xaxis": { 4696 "mode": "time", 4697 "name": null, 4698 "show": true, 4699 "values": [] 4700 }, 4701 "yaxes": [ 4702 { 4703 "decimals": null, 4704 "format": "short", 4705 "label": null, 4706 "logBase": 1, 4707 "max": null, 4708 "min": null, 4709 "show": true 4710 }, 4711 { 4712 "decimals": null, 4713 "format": "short", 4714 "label": null, 4715 "logBase": 1, 4716 "max": null, 4717 "min": null, 4718 "show": true 4719 } 4720 ], 4721 "yaxis": { 4722 "align": false, 4723 "alignLevel": 0 4724 } 4725 }, 4726 { 4727 "aliasColors": {}, 4728 "bars": false, 4729 "cacheTimeout": null, 4730 "datasource": "$secondary_source", 4731 "description": null, 4732 "editable": true, 4733 "error": false, 4734 "fieldConfig": { 4735 "defaults": { 4736 "thresholds": { 4737 "mode": "absolute", 4738 "steps": [] 4739 } 4740 } 4741 }, 4742 "fill": 1, 4743 "grid": { 4744 "threshold1": null, 4745 "threshold1Color": "rgba(216, 200, 27, 0.27)", 4746 "threshold2": null, 4747 "threshold2Color": "rgba(234, 112, 112, 0.22)" 4748 }, 4749 "gridPos": null, 4750 "height": null, 4751 "hideTimeOverride": false, 4752 "id": 36, 4753 "interval": null, 4754 "isNew": true, 4755 "legend": { 4756 "alignAsTable": false, 4757 "avg": false, 4758 "current": false, 4759 "hideEmpty": false, 4760 "hideZero": false, 4761 "max": false, 4762 "min": false, 4763 "rightSide": false, 4764 "show": true, 4765 "sideWidth": null, 4766 "sort": null, 4767 "sortDesc": false, 4768 "total": false, 4769 "values": false 4770 }, 4771 "lines": true, 4772 "linewidth": 2, 4773 "links": [], 4774 "maxDataPoints": 100, 4775 "maxPerRow": null, 4776 "minSpan": null, 4777 "nullPointMode": null, 4778 "options": { 4779 "alertThreshold": true, 4780 "dataLinks": [] 4781 }, 4782 "percentage": false, 4783 "pointradius": 5, 4784 "points": false, 4785 "renderer": "flot", 4786 "repeat": null, 4787 "repeatDirection": null, 4788 "seriesOverrides": [], 4789 "span": 12, 4790 "stack": false, 4791 "steppedLine": false, 4792 "targets": [ 4793 { 4794 "datasource": "$source", 4795 "expr": "avg(apiserver_flowcontrol_upper_limit_seats{instance=~\".*(${instance:pipe})\"}) by (priority_level)", 4796 "format": "time_series", 4797 "hide": false, 4798 "instant": false, 4799 "interval": "5s", 4800 "intervalFactor": 1, 4801 "legendFormat": "{{priority_level}}", 4802 "metric": "", 4803 "query": "avg(apiserver_flowcontrol_upper_limit_seats{instance=~\".*(${instance:pipe})\"}) by (priority_level)", 4804 "refId": "", 4805 "step": 10, 4806 "target": "" 4807 } 4808 ], 4809 "thresholds": [], 4810 "timeFrom": null, 4811 "timeShift": "$timeshift", 4812 "title": "[SECONDARY] Upper bound on number of execution seats", 4813 "tooltip": { 4814 "msResolution": true, 4815 "shared": true, 4816 "sort": 2, 4817 "value_type": "cumulative" 4818 }, 4819 "transformations": [], 4820 "transparent": false, 4821 "type": "graph", 4822 "xaxis": { 4823 "mode": "time", 4824 "name": null, 4825 "show": true, 4826 "values": [] 4827 }, 4828 "yaxes": [ 4829 { 4830 "decimals": null, 4831 "format": "short", 4832 "label": null, 4833 "logBase": 1, 4834 "max": null, 4835 "min": null, 4836 "show": true 4837 }, 4838 { 4839 "decimals": null, 4840 "format": "short", 4841 "label": null, 4842 "logBase": 1, 4843 "max": null, 4844 "min": null, 4845 "show": true 4846 } 4847 ], 4848 "yaxis": { 4849 "align": false, 4850 "alignLevel": 0 4851 } 4852 }, 4853 { 4854 "aliasColors": {}, 4855 "bars": false, 4856 "cacheTimeout": null, 4857 "datasource": "$source", 4858 "description": null, 4859 "editable": true, 4860 "error": false, 4861 "fieldConfig": { 4862 "defaults": { 4863 "thresholds": { 4864 "mode": "absolute", 4865 "steps": [] 4866 } 4867 } 4868 }, 4869 "fill": 1, 4870 "grid": { 4871 "threshold1": null, 4872 "threshold1Color": "rgba(216, 200, 27, 0.27)", 4873 "threshold2": null, 4874 "threshold2Color": "rgba(234, 112, 112, 0.22)" 4875 }, 4876 "gridPos": null, 4877 "height": null, 4878 "hideTimeOverride": false, 4879 "id": 37, 4880 "interval": null, 4881 "isNew": true, 4882 "legend": { 4883 "alignAsTable": false, 4884 "avg": false, 4885 "current": false, 4886 "hideEmpty": false, 4887 "hideZero": false, 4888 "max": false, 4889 "min": false, 4890 "rightSide": false, 4891 "show": true, 4892 "sideWidth": null, 4893 "sort": null, 4894 "sortDesc": false, 4895 "total": false, 4896 "values": false 4897 }, 4898 "lines": true, 4899 "linewidth": 2, 4900 "links": [], 4901 "maxDataPoints": 100, 4902 "maxPerRow": null, 4903 "minSpan": null, 4904 "nullPointMode": null, 4905 "options": { 4906 "alertThreshold": true, 4907 "dataLinks": [] 4908 }, 4909 "percentage": false, 4910 "pointradius": 5, 4911 "points": false, 4912 "renderer": "flot", 4913 "repeat": null, 4914 "repeatDirection": null, 4915 "seriesOverrides": [], 4916 "span": 12, 4917 "stack": false, 4918 "steppedLine": false, 4919 "targets": [ 4920 { 4921 "datasource": "$source", 4922 "expr": "histogram_quantile(0.5, rate(apiserver_flowcontrol_demand_seats_bucket{instance=~\".*(${instance:pipe})\"}[10s]))", 4923 "format": "time_series", 4924 "hide": false, 4925 "instant": false, 4926 "interval": "5s", 4927 "intervalFactor": 1, 4928 "legendFormat": "{{instance}} {{priority_level}}", 4929 "metric": "", 4930 "query": "histogram_quantile(0.5, rate(apiserver_flowcontrol_demand_seats_bucket{instance=~\".*(${instance:pipe})\"}[10s]))", 4931 "refId": "", 4932 "step": 10, 4933 "target": "" 4934 } 4935 ], 4936 "thresholds": [], 4937 "timeFrom": null, 4938 "timeShift": null, 4939 "title": "Number of seats Priority Level could use divided by nominal seats (50th percentile)", 4940 "tooltip": { 4941 "msResolution": true, 4942 "shared": true, 4943 "sort": 2, 4944 "value_type": "cumulative" 4945 }, 4946 "transformations": [], 4947 "transparent": false, 4948 "type": "graph", 4949 "xaxis": { 4950 "mode": "time", 4951 "name": null, 4952 "show": true, 4953 "values": [] 4954 }, 4955 "yaxes": [ 4956 { 4957 "decimals": null, 4958 "format": "short", 4959 "label": null, 4960 "logBase": 1, 4961 "max": null, 4962 "min": null, 4963 "show": true 4964 }, 4965 { 4966 "decimals": null, 4967 "format": "short", 4968 "label": null, 4969 "logBase": 1, 4970 "max": null, 4971 "min": null, 4972 "show": true 4973 } 4974 ], 4975 "yaxis": { 4976 "align": false, 4977 "alignLevel": 0 4978 } 4979 }, 4980 { 4981 "aliasColors": {}, 4982 "bars": false, 4983 "cacheTimeout": null, 4984 "datasource": "$secondary_source", 4985 "description": null, 4986 "editable": true, 4987 "error": false, 4988 "fieldConfig": { 4989 "defaults": { 4990 "thresholds": { 4991 "mode": "absolute", 4992 "steps": [] 4993 } 4994 } 4995 }, 4996 "fill": 1, 4997 "grid": { 4998 "threshold1": null, 4999 "threshold1Color": "rgba(216, 200, 27, 0.27)", 5000 "threshold2": null, 5001 "threshold2Color": "rgba(234, 112, 112, 0.22)" 5002 }, 5003 "gridPos": null, 5004 "height": null, 5005 "hideTimeOverride": false, 5006 "id": 38, 5007 "interval": null, 5008 "isNew": true, 5009 "legend": { 5010 "alignAsTable": false, 5011 "avg": false, 5012 "current": false, 5013 "hideEmpty": false, 5014 "hideZero": false, 5015 "max": false, 5016 "min": false, 5017 "rightSide": false, 5018 "show": true, 5019 "sideWidth": null, 5020 "sort": null, 5021 "sortDesc": false, 5022 "total": false, 5023 "values": false 5024 }, 5025 "lines": true, 5026 "linewidth": 2, 5027 "links": [], 5028 "maxDataPoints": 100, 5029 "maxPerRow": null, 5030 "minSpan": null, 5031 "nullPointMode": null, 5032 "options": { 5033 "alertThreshold": true, 5034 "dataLinks": [] 5035 }, 5036 "percentage": false, 5037 "pointradius": 5, 5038 "points": false, 5039 "renderer": "flot", 5040 "repeat": null, 5041 "repeatDirection": null, 5042 "seriesOverrides": [], 5043 "span": 12, 5044 "stack": false, 5045 "steppedLine": false, 5046 "targets": [ 5047 { 5048 "datasource": "$source", 5049 "expr": "histogram_quantile(0.5, rate(apiserver_flowcontrol_demand_seats_bucket{instance=~\".*(${instance:pipe})\"}[10s]))", 5050 "format": "time_series", 5051 "hide": false, 5052 "instant": false, 5053 "interval": "5s", 5054 "intervalFactor": 1, 5055 "legendFormat": "{{instance}} {{priority_level}}", 5056 "metric": "", 5057 "query": "histogram_quantile(0.5, rate(apiserver_flowcontrol_demand_seats_bucket{instance=~\".*(${instance:pipe})\"}[10s]))", 5058 "refId": "", 5059 "step": 10, 5060 "target": "" 5061 } 5062 ], 5063 "thresholds": [], 5064 "timeFrom": null, 5065 "timeShift": "$timeshift", 5066 "title": "[SECONDARY] Number of seats Priority Level could use divided by nominal seats (50th percentile)", 5067 "tooltip": { 5068 "msResolution": true, 5069 "shared": true, 5070 "sort": 2, 5071 "value_type": "cumulative" 5072 }, 5073 "transformations": [], 5074 "transparent": false, 5075 "type": "graph", 5076 "xaxis": { 5077 "mode": "time", 5078 "name": null, 5079 "show": true, 5080 "values": [] 5081 }, 5082 "yaxes": [ 5083 { 5084 "decimals": null, 5085 "format": "short", 5086 "label": null, 5087 "logBase": 1, 5088 "max": null, 5089 "min": null, 5090 "show": true 5091 }, 5092 { 5093 "decimals": null, 5094 "format": "short", 5095 "label": null, 5096 "logBase": 1, 5097 "max": null, 5098 "min": null, 5099 "show": true 5100 } 5101 ], 5102 "yaxis": { 5103 "align": false, 5104 "alignLevel": 0 5105 } 5106 }, 5107 { 5108 "aliasColors": {}, 5109 "bars": false, 5110 "cacheTimeout": null, 5111 "datasource": "$source", 5112 "description": null, 5113 "editable": true, 5114 "error": false, 5115 "fieldConfig": { 5116 "defaults": { 5117 "thresholds": { 5118 "mode": "absolute", 5119 "steps": [] 5120 } 5121 } 5122 }, 5123 "fill": 1, 5124 "grid": { 5125 "threshold1": null, 5126 "threshold1Color": "rgba(216, 200, 27, 0.27)", 5127 "threshold2": null, 5128 "threshold2Color": "rgba(234, 112, 112, 0.22)" 5129 }, 5130 "gridPos": null, 5131 "height": null, 5132 "hideTimeOverride": false, 5133 "id": 39, 5134 "interval": null, 5135 "isNew": true, 5136 "legend": { 5137 "alignAsTable": false, 5138 "avg": false, 5139 "current": false, 5140 "hideEmpty": false, 5141 "hideZero": false, 5142 "max": false, 5143 "min": false, 5144 "rightSide": false, 5145 "show": true, 5146 "sideWidth": null, 5147 "sort": null, 5148 "sortDesc": false, 5149 "total": false, 5150 "values": false 5151 }, 5152 "lines": true, 5153 "linewidth": 2, 5154 "links": [], 5155 "maxDataPoints": 100, 5156 "maxPerRow": null, 5157 "minSpan": null, 5158 "nullPointMode": null, 5159 "options": { 5160 "alertThreshold": true, 5161 "dataLinks": [] 5162 }, 5163 "percentage": false, 5164 "pointradius": 5, 5165 "points": false, 5166 "renderer": "flot", 5167 "repeat": null, 5168 "repeatDirection": null, 5169 "seriesOverrides": [], 5170 "span": 12, 5171 "stack": false, 5172 "steppedLine": false, 5173 "targets": [ 5174 { 5175 "datasource": "$source", 5176 "expr": "apiserver_flowcontrol_demand_seats_high_watermark{instance=~\".*(${instance:pipe})\"}", 5177 "format": "time_series", 5178 "hide": false, 5179 "instant": false, 5180 "interval": "5s", 5181 "intervalFactor": 1, 5182 "legendFormat": "{{instance}} {{priority_level}}", 5183 "metric": "", 5184 "query": "apiserver_flowcontrol_demand_seats_high_watermark{instance=~\".*(${instance:pipe})\"}", 5185 "refId": "", 5186 "step": 10, 5187 "target": "" 5188 } 5189 ], 5190 "thresholds": [], 5191 "timeFrom": null, 5192 "timeShift": null, 5193 "title": "High watermark of demand seats over last adjustment period", 5194 "tooltip": { 5195 "msResolution": true, 5196 "shared": true, 5197 "sort": 2, 5198 "value_type": "cumulative" 5199 }, 5200 "transformations": [], 5201 "transparent": false, 5202 "type": "graph", 5203 "xaxis": { 5204 "mode": "time", 5205 "name": null, 5206 "show": true, 5207 "values": [] 5208 }, 5209 "yaxes": [ 5210 { 5211 "decimals": null, 5212 "format": "short", 5213 "label": null, 5214 "logBase": 1, 5215 "max": null, 5216 "min": null, 5217 "show": true 5218 }, 5219 { 5220 "decimals": null, 5221 "format": "short", 5222 "label": null, 5223 "logBase": 1, 5224 "max": null, 5225 "min": null, 5226 "show": true 5227 } 5228 ], 5229 "yaxis": { 5230 "align": false, 5231 "alignLevel": 0 5232 } 5233 }, 5234 { 5235 "aliasColors": {}, 5236 "bars": false, 5237 "cacheTimeout": null, 5238 "datasource": "$secondary_source", 5239 "description": null, 5240 "editable": true, 5241 "error": false, 5242 "fieldConfig": { 5243 "defaults": { 5244 "thresholds": { 5245 "mode": "absolute", 5246 "steps": [] 5247 } 5248 } 5249 }, 5250 "fill": 1, 5251 "grid": { 5252 "threshold1": null, 5253 "threshold1Color": "rgba(216, 200, 27, 0.27)", 5254 "threshold2": null, 5255 "threshold2Color": "rgba(234, 112, 112, 0.22)" 5256 }, 5257 "gridPos": null, 5258 "height": null, 5259 "hideTimeOverride": false, 5260 "id": 40, 5261 "interval": null, 5262 "isNew": true, 5263 "legend": { 5264 "alignAsTable": false, 5265 "avg": false, 5266 "current": false, 5267 "hideEmpty": false, 5268 "hideZero": false, 5269 "max": false, 5270 "min": false, 5271 "rightSide": false, 5272 "show": true, 5273 "sideWidth": null, 5274 "sort": null, 5275 "sortDesc": false, 5276 "total": false, 5277 "values": false 5278 }, 5279 "lines": true, 5280 "linewidth": 2, 5281 "links": [], 5282 "maxDataPoints": 100, 5283 "maxPerRow": null, 5284 "minSpan": null, 5285 "nullPointMode": null, 5286 "options": { 5287 "alertThreshold": true, 5288 "dataLinks": [] 5289 }, 5290 "percentage": false, 5291 "pointradius": 5, 5292 "points": false, 5293 "renderer": "flot", 5294 "repeat": null, 5295 "repeatDirection": null, 5296 "seriesOverrides": [], 5297 "span": 12, 5298 "stack": false, 5299 "steppedLine": false, 5300 "targets": [ 5301 { 5302 "datasource": "$source", 5303 "expr": "apiserver_flowcontrol_demand_seats_high_watermark{instance=~\".*(${instance:pipe})\"}", 5304 "format": "time_series", 5305 "hide": false, 5306 "instant": false, 5307 "interval": "5s", 5308 "intervalFactor": 1, 5309 "legendFormat": "{{instance}} {{priority_level}}", 5310 "metric": "", 5311 "query": "apiserver_flowcontrol_demand_seats_high_watermark{instance=~\".*(${instance:pipe})\"}", 5312 "refId": "", 5313 "step": 10, 5314 "target": "" 5315 } 5316 ], 5317 "thresholds": [], 5318 "timeFrom": null, 5319 "timeShift": "$timeshift", 5320 "title": "[SECONDARY] High watermark of demand seats over last adjustment period", 5321 "tooltip": { 5322 "msResolution": true, 5323 "shared": true, 5324 "sort": 2, 5325 "value_type": "cumulative" 5326 }, 5327 "transformations": [], 5328 "transparent": false, 5329 "type": "graph", 5330 "xaxis": { 5331 "mode": "time", 5332 "name": null, 5333 "show": true, 5334 "values": [] 5335 }, 5336 "yaxes": [ 5337 { 5338 "decimals": null, 5339 "format": "short", 5340 "label": null, 5341 "logBase": 1, 5342 "max": null, 5343 "min": null, 5344 "show": true 5345 }, 5346 { 5347 "decimals": null, 5348 "format": "short", 5349 "label": null, 5350 "logBase": 1, 5351 "max": null, 5352 "min": null, 5353 "show": true 5354 } 5355 ], 5356 "yaxis": { 5357 "align": false, 5358 "alignLevel": 0 5359 } 5360 }, 5361 { 5362 "aliasColors": {}, 5363 "bars": false, 5364 "cacheTimeout": null, 5365 "datasource": "$source", 5366 "description": null, 5367 "editable": true, 5368 "error": false, 5369 "fieldConfig": { 5370 "defaults": { 5371 "thresholds": { 5372 "mode": "absolute", 5373 "steps": [] 5374 } 5375 } 5376 }, 5377 "fill": 1, 5378 "grid": { 5379 "threshold1": null, 5380 "threshold1Color": "rgba(216, 200, 27, 0.27)", 5381 "threshold2": null, 5382 "threshold2Color": "rgba(234, 112, 112, 0.22)" 5383 }, 5384 "gridPos": null, 5385 "height": null, 5386 "hideTimeOverride": false, 5387 "id": 41, 5388 "interval": null, 5389 "isNew": true, 5390 "legend": { 5391 "alignAsTable": false, 5392 "avg": false, 5393 "current": false, 5394 "hideEmpty": false, 5395 "hideZero": false, 5396 "max": false, 5397 "min": false, 5398 "rightSide": false, 5399 "show": true, 5400 "sideWidth": null, 5401 "sort": null, 5402 "sortDesc": false, 5403 "total": false, 5404 "values": false 5405 }, 5406 "lines": true, 5407 "linewidth": 2, 5408 "links": [], 5409 "maxDataPoints": 100, 5410 "maxPerRow": null, 5411 "minSpan": null, 5412 "nullPointMode": null, 5413 "options": { 5414 "alertThreshold": true, 5415 "dataLinks": [] 5416 }, 5417 "percentage": false, 5418 "pointradius": 5, 5419 "points": false, 5420 "renderer": "flot", 5421 "repeat": null, 5422 "repeatDirection": null, 5423 "seriesOverrides": [], 5424 "span": 12, 5425 "stack": false, 5426 "steppedLine": false, 5427 "targets": [ 5428 { 5429 "datasource": "$source", 5430 "expr": "apiserver_flowcontrol_demand_seats_smoothed{instance=~\".*(${instance:pipe})\"}", 5431 "format": "time_series", 5432 "hide": false, 5433 "instant": false, 5434 "interval": "5s", 5435 "intervalFactor": 1, 5436 "legendFormat": "{{instance}} {{priority_level}}", 5437 "metric": "", 5438 "query": "apiserver_flowcontrol_demand_seats_smoothed{instance=~\".*(${instance:pipe})\"}", 5439 "refId": "", 5440 "step": 10, 5441 "target": "" 5442 } 5443 ], 5444 "thresholds": [], 5445 "timeFrom": null, 5446 "timeShift": null, 5447 "title": "Smoothed seat demands", 5448 "tooltip": { 5449 "msResolution": true, 5450 "shared": true, 5451 "sort": 2, 5452 "value_type": "cumulative" 5453 }, 5454 "transformations": [], 5455 "transparent": false, 5456 "type": "graph", 5457 "xaxis": { 5458 "mode": "time", 5459 "name": null, 5460 "show": true, 5461 "values": [] 5462 }, 5463 "yaxes": [ 5464 { 5465 "decimals": null, 5466 "format": "short", 5467 "label": null, 5468 "logBase": 1, 5469 "max": null, 5470 "min": null, 5471 "show": true 5472 }, 5473 { 5474 "decimals": null, 5475 "format": "short", 5476 "label": null, 5477 "logBase": 1, 5478 "max": null, 5479 "min": null, 5480 "show": true 5481 } 5482 ], 5483 "yaxis": { 5484 "align": false, 5485 "alignLevel": 0 5486 } 5487 }, 5488 { 5489 "aliasColors": {}, 5490 "bars": false, 5491 "cacheTimeout": null, 5492 "datasource": "$secondary_source", 5493 "description": null, 5494 "editable": true, 5495 "error": false, 5496 "fieldConfig": { 5497 "defaults": { 5498 "thresholds": { 5499 "mode": "absolute", 5500 "steps": [] 5501 } 5502 } 5503 }, 5504 "fill": 1, 5505 "grid": { 5506 "threshold1": null, 5507 "threshold1Color": "rgba(216, 200, 27, 0.27)", 5508 "threshold2": null, 5509 "threshold2Color": "rgba(234, 112, 112, 0.22)" 5510 }, 5511 "gridPos": null, 5512 "height": null, 5513 "hideTimeOverride": false, 5514 "id": 42, 5515 "interval": null, 5516 "isNew": true, 5517 "legend": { 5518 "alignAsTable": false, 5519 "avg": false, 5520 "current": false, 5521 "hideEmpty": false, 5522 "hideZero": false, 5523 "max": false, 5524 "min": false, 5525 "rightSide": false, 5526 "show": true, 5527 "sideWidth": null, 5528 "sort": null, 5529 "sortDesc": false, 5530 "total": false, 5531 "values": false 5532 }, 5533 "lines": true, 5534 "linewidth": 2, 5535 "links": [], 5536 "maxDataPoints": 100, 5537 "maxPerRow": null, 5538 "minSpan": null, 5539 "nullPointMode": null, 5540 "options": { 5541 "alertThreshold": true, 5542 "dataLinks": [] 5543 }, 5544 "percentage": false, 5545 "pointradius": 5, 5546 "points": false, 5547 "renderer": "flot", 5548 "repeat": null, 5549 "repeatDirection": null, 5550 "seriesOverrides": [], 5551 "span": 12, 5552 "stack": false, 5553 "steppedLine": false, 5554 "targets": [ 5555 { 5556 "datasource": "$source", 5557 "expr": "apiserver_flowcontrol_demand_seats_smoothed{instance=~\".*(${instance:pipe})\"}", 5558 "format": "time_series", 5559 "hide": false, 5560 "instant": false, 5561 "interval": "5s", 5562 "intervalFactor": 1, 5563 "legendFormat": "{{instance}} {{priority_level}}", 5564 "metric": "", 5565 "query": "apiserver_flowcontrol_demand_seats_smoothed{instance=~\".*(${instance:pipe})\"}", 5566 "refId": "", 5567 "step": 10, 5568 "target": "" 5569 } 5570 ], 5571 "thresholds": [], 5572 "timeFrom": null, 5573 "timeShift": "$timeshift", 5574 "title": "[SECONDARY] Smoothed seat demands", 5575 "tooltip": { 5576 "msResolution": true, 5577 "shared": true, 5578 "sort": 2, 5579 "value_type": "cumulative" 5580 }, 5581 "transformations": [], 5582 "transparent": false, 5583 "type": "graph", 5584 "xaxis": { 5585 "mode": "time", 5586 "name": null, 5587 "show": true, 5588 "values": [] 5589 }, 5590 "yaxes": [ 5591 { 5592 "decimals": null, 5593 "format": "short", 5594 "label": null, 5595 "logBase": 1, 5596 "max": null, 5597 "min": null, 5598 "show": true 5599 }, 5600 { 5601 "decimals": null, 5602 "format": "short", 5603 "label": null, 5604 "logBase": 1, 5605 "max": null, 5606 "min": null, 5607 "show": true 5608 } 5609 ], 5610 "yaxis": { 5611 "align": false, 5612 "alignLevel": 0 5613 } 5614 }, 5615 { 5616 "aliasColors": {}, 5617 "bars": false, 5618 "cacheTimeout": null, 5619 "datasource": "$source", 5620 "description": null, 5621 "editable": true, 5622 "error": false, 5623 "fieldConfig": { 5624 "defaults": { 5625 "thresholds": { 5626 "mode": "absolute", 5627 "steps": [] 5628 } 5629 } 5630 }, 5631 "fill": 1, 5632 "grid": { 5633 "threshold1": null, 5634 "threshold1Color": "rgba(216, 200, 27, 0.27)", 5635 "threshold2": null, 5636 "threshold2Color": "rgba(234, 112, 112, 0.22)" 5637 }, 5638 "gridPos": null, 5639 "height": null, 5640 "hideTimeOverride": false, 5641 "id": 43, 5642 "interval": null, 5643 "isNew": true, 5644 "legend": { 5645 "alignAsTable": false, 5646 "avg": false, 5647 "current": false, 5648 "hideEmpty": false, 5649 "hideZero": false, 5650 "max": false, 5651 "min": false, 5652 "rightSide": false, 5653 "show": true, 5654 "sideWidth": null, 5655 "sort": null, 5656 "sortDesc": false, 5657 "total": false, 5658 "values": false 5659 }, 5660 "lines": true, 5661 "linewidth": 2, 5662 "links": [], 5663 "maxDataPoints": 100, 5664 "maxPerRow": null, 5665 "minSpan": null, 5666 "nullPointMode": null, 5667 "options": { 5668 "alertThreshold": true, 5669 "dataLinks": [] 5670 }, 5671 "percentage": false, 5672 "pointradius": 5, 5673 "points": false, 5674 "renderer": "flot", 5675 "repeat": null, 5676 "repeatDirection": null, 5677 "seriesOverrides": [], 5678 "span": 12, 5679 "stack": false, 5680 "steppedLine": false, 5681 "targets": [ 5682 { 5683 "datasource": "$source", 5684 "expr": "apiserver_flowcontrol_current_limit_seats{instance=~\".*(${instance:pipe})\"}", 5685 "format": "time_series", 5686 "hide": false, 5687 "instant": false, 5688 "interval": "5s", 5689 "intervalFactor": 1, 5690 "legendFormat": "{{instance}} {{priority_level}}", 5691 "metric": "", 5692 "query": "apiserver_flowcontrol_current_limit_seats{instance=~\".*(${instance:pipe})\"}", 5693 "refId": "", 5694 "step": 10, 5695 "target": "" 5696 } 5697 ], 5698 "thresholds": [], 5699 "timeFrom": null, 5700 "timeShift": null, 5701 "title": "Current seat limit for each Priority Level", 5702 "tooltip": { 5703 "msResolution": true, 5704 "shared": true, 5705 "sort": 2, 5706 "value_type": "cumulative" 5707 }, 5708 "transformations": [], 5709 "transparent": false, 5710 "type": "graph", 5711 "xaxis": { 5712 "mode": "time", 5713 "name": null, 5714 "show": true, 5715 "values": [] 5716 }, 5717 "yaxes": [ 5718 { 5719 "decimals": null, 5720 "format": "short", 5721 "label": null, 5722 "logBase": 1, 5723 "max": null, 5724 "min": null, 5725 "show": true 5726 }, 5727 { 5728 "decimals": null, 5729 "format": "short", 5730 "label": null, 5731 "logBase": 1, 5732 "max": null, 5733 "min": null, 5734 "show": true 5735 } 5736 ], 5737 "yaxis": { 5738 "align": false, 5739 "alignLevel": 0 5740 } 5741 }, 5742 { 5743 "aliasColors": {}, 5744 "bars": false, 5745 "cacheTimeout": null, 5746 "datasource": "$secondary_source", 5747 "description": null, 5748 "editable": true, 5749 "error": false, 5750 "fieldConfig": { 5751 "defaults": { 5752 "thresholds": { 5753 "mode": "absolute", 5754 "steps": [] 5755 } 5756 } 5757 }, 5758 "fill": 1, 5759 "grid": { 5760 "threshold1": null, 5761 "threshold1Color": "rgba(216, 200, 27, 0.27)", 5762 "threshold2": null, 5763 "threshold2Color": "rgba(234, 112, 112, 0.22)" 5764 }, 5765 "gridPos": null, 5766 "height": null, 5767 "hideTimeOverride": false, 5768 "id": 44, 5769 "interval": null, 5770 "isNew": true, 5771 "legend": { 5772 "alignAsTable": false, 5773 "avg": false, 5774 "current": false, 5775 "hideEmpty": false, 5776 "hideZero": false, 5777 "max": false, 5778 "min": false, 5779 "rightSide": false, 5780 "show": true, 5781 "sideWidth": null, 5782 "sort": null, 5783 "sortDesc": false, 5784 "total": false, 5785 "values": false 5786 }, 5787 "lines": true, 5788 "linewidth": 2, 5789 "links": [], 5790 "maxDataPoints": 100, 5791 "maxPerRow": null, 5792 "minSpan": null, 5793 "nullPointMode": null, 5794 "options": { 5795 "alertThreshold": true, 5796 "dataLinks": [] 5797 }, 5798 "percentage": false, 5799 "pointradius": 5, 5800 "points": false, 5801 "renderer": "flot", 5802 "repeat": null, 5803 "repeatDirection": null, 5804 "seriesOverrides": [], 5805 "span": 12, 5806 "stack": false, 5807 "steppedLine": false, 5808 "targets": [ 5809 { 5810 "datasource": "$source", 5811 "expr": "apiserver_flowcontrol_current_limit_seats{instance=~\".*(${instance:pipe})\"}", 5812 "format": "time_series", 5813 "hide": false, 5814 "instant": false, 5815 "interval": "5s", 5816 "intervalFactor": 1, 5817 "legendFormat": "{{instance}} {{priority_level}}", 5818 "metric": "", 5819 "query": "apiserver_flowcontrol_current_limit_seats{instance=~\".*(${instance:pipe})\"}", 5820 "refId": "", 5821 "step": 10, 5822 "target": "" 5823 } 5824 ], 5825 "thresholds": [], 5826 "timeFrom": null, 5827 "timeShift": "$timeshift", 5828 "title": "[SECONDARY] Current seat limit for each Priority Level", 5829 "tooltip": { 5830 "msResolution": true, 5831 "shared": true, 5832 "sort": 2, 5833 "value_type": "cumulative" 5834 }, 5835 "transformations": [], 5836 "transparent": false, 5837 "type": "graph", 5838 "xaxis": { 5839 "mode": "time", 5840 "name": null, 5841 "show": true, 5842 "values": [] 5843 }, 5844 "yaxes": [ 5845 { 5846 "decimals": null, 5847 "format": "short", 5848 "label": null, 5849 "logBase": 1, 5850 "max": null, 5851 "min": null, 5852 "show": true 5853 }, 5854 { 5855 "decimals": null, 5856 "format": "short", 5857 "label": null, 5858 "logBase": 1, 5859 "max": null, 5860 "min": null, 5861 "show": true 5862 } 5863 ], 5864 "yaxis": { 5865 "align": false, 5866 "alignLevel": 0 5867 } 5868 } 5869 ], 5870 "repeat": null, 5871 "showTitle": true, 5872 "title": "P&F metrics" 5873 }, 5874 { 5875 "collapse": true, 5876 "editable": true, 5877 "height": "300px", 5878 "panels": [ 5879 { 5880 "aliasColors": {}, 5881 "bars": false, 5882 "cacheTimeout": null, 5883 "datasource": "$source", 5884 "description": null, 5885 "editable": true, 5886 "error": false, 5887 "fieldConfig": { 5888 "defaults": { 5889 "thresholds": { 5890 "mode": "absolute", 5891 "steps": [] 5892 } 5893 } 5894 }, 5895 "fill": 1, 5896 "grid": { 5897 "threshold1": null, 5898 "threshold1Color": "rgba(216, 200, 27, 0.27)", 5899 "threshold2": null, 5900 "threshold2Color": "rgba(234, 112, 112, 0.22)" 5901 }, 5902 "gridPos": null, 5903 "height": null, 5904 "hideTimeOverride": false, 5905 "id": 45, 5906 "interval": null, 5907 "isNew": true, 5908 "legend": { 5909 "alignAsTable": false, 5910 "avg": false, 5911 "current": false, 5912 "hideEmpty": false, 5913 "hideZero": false, 5914 "max": false, 5915 "min": false, 5916 "rightSide": false, 5917 "show": true, 5918 "sideWidth": null, 5919 "sort": null, 5920 "sortDesc": false, 5921 "total": false, 5922 "values": false 5923 }, 5924 "lines": true, 5925 "linewidth": 2, 5926 "links": [], 5927 "maxDataPoints": 100, 5928 "maxPerRow": null, 5929 "minSpan": null, 5930 "nullPointMode": null, 5931 "options": { 5932 "alertThreshold": true, 5933 "dataLinks": [] 5934 }, 5935 "percentage": false, 5936 "pointradius": 5, 5937 "points": false, 5938 "renderer": "flot", 5939 "repeat": null, 5940 "repeatDirection": null, 5941 "seriesOverrides": [], 5942 "span": 12, 5943 "stack": false, 5944 "steppedLine": false, 5945 "targets": [ 5946 { 5947 "datasource": "$source", 5948 "expr": "sum(node_collector_unhealthy_nodes_in_zone) by (zone)", 5949 "format": "time_series", 5950 "hide": false, 5951 "instant": false, 5952 "interval": "5s", 5953 "intervalFactor": 1, 5954 "legendFormat": "{{zone}}", 5955 "metric": "", 5956 "query": "sum(node_collector_unhealthy_nodes_in_zone) by (zone)", 5957 "refId": "", 5958 "step": 10, 5959 "target": "" 5960 } 5961 ], 5962 "thresholds": [], 5963 "timeFrom": null, 5964 "timeShift": null, 5965 "title": "Unhealthy nodes", 5966 "tooltip": { 5967 "msResolution": true, 5968 "shared": true, 5969 "sort": 2, 5970 "value_type": "cumulative" 5971 }, 5972 "transformations": [], 5973 "transparent": false, 5974 "type": "graph", 5975 "xaxis": { 5976 "mode": "time", 5977 "name": null, 5978 "show": true, 5979 "values": [] 5980 }, 5981 "yaxes": [ 5982 { 5983 "decimals": null, 5984 "format": "short", 5985 "label": null, 5986 "logBase": 1, 5987 "max": null, 5988 "min": null, 5989 "show": true 5990 }, 5991 { 5992 "decimals": null, 5993 "format": "short", 5994 "label": null, 5995 "logBase": 1, 5996 "max": null, 5997 "min": null, 5998 "show": true 5999 } 6000 ], 6001 "yaxis": { 6002 "align": false, 6003 "alignLevel": 0 6004 } 6005 }, 6006 { 6007 "aliasColors": {}, 6008 "bars": false, 6009 "cacheTimeout": null, 6010 "datasource": "$secondary_source", 6011 "description": null, 6012 "editable": true, 6013 "error": false, 6014 "fieldConfig": { 6015 "defaults": { 6016 "thresholds": { 6017 "mode": "absolute", 6018 "steps": [] 6019 } 6020 } 6021 }, 6022 "fill": 1, 6023 "grid": { 6024 "threshold1": null, 6025 "threshold1Color": "rgba(216, 200, 27, 0.27)", 6026 "threshold2": null, 6027 "threshold2Color": "rgba(234, 112, 112, 0.22)" 6028 }, 6029 "gridPos": null, 6030 "height": null, 6031 "hideTimeOverride": false, 6032 "id": 46, 6033 "interval": null, 6034 "isNew": true, 6035 "legend": { 6036 "alignAsTable": false, 6037 "avg": false, 6038 "current": false, 6039 "hideEmpty": false, 6040 "hideZero": false, 6041 "max": false, 6042 "min": false, 6043 "rightSide": false, 6044 "show": true, 6045 "sideWidth": null, 6046 "sort": null, 6047 "sortDesc": false, 6048 "total": false, 6049 "values": false 6050 }, 6051 "lines": true, 6052 "linewidth": 2, 6053 "links": [], 6054 "maxDataPoints": 100, 6055 "maxPerRow": null, 6056 "minSpan": null, 6057 "nullPointMode": null, 6058 "options": { 6059 "alertThreshold": true, 6060 "dataLinks": [] 6061 }, 6062 "percentage": false, 6063 "pointradius": 5, 6064 "points": false, 6065 "renderer": "flot", 6066 "repeat": null, 6067 "repeatDirection": null, 6068 "seriesOverrides": [], 6069 "span": 12, 6070 "stack": false, 6071 "steppedLine": false, 6072 "targets": [ 6073 { 6074 "datasource": "$source", 6075 "expr": "sum(node_collector_unhealthy_nodes_in_zone) by (zone)", 6076 "format": "time_series", 6077 "hide": false, 6078 "instant": false, 6079 "interval": "5s", 6080 "intervalFactor": 1, 6081 "legendFormat": "{{zone}}", 6082 "metric": "", 6083 "query": "sum(node_collector_unhealthy_nodes_in_zone) by (zone)", 6084 "refId": "", 6085 "step": 10, 6086 "target": "" 6087 } 6088 ], 6089 "thresholds": [], 6090 "timeFrom": null, 6091 "timeShift": "$timeshift", 6092 "title": "[SECONDARY] Unhealthy nodes", 6093 "tooltip": { 6094 "msResolution": true, 6095 "shared": true, 6096 "sort": 2, 6097 "value_type": "cumulative" 6098 }, 6099 "transformations": [], 6100 "transparent": false, 6101 "type": "graph", 6102 "xaxis": { 6103 "mode": "time", 6104 "name": null, 6105 "show": true, 6106 "values": [] 6107 }, 6108 "yaxes": [ 6109 { 6110 "decimals": null, 6111 "format": "short", 6112 "label": null, 6113 "logBase": 1, 6114 "max": null, 6115 "min": null, 6116 "show": true 6117 }, 6118 { 6119 "decimals": null, 6120 "format": "short", 6121 "label": null, 6122 "logBase": 1, 6123 "max": null, 6124 "min": null, 6125 "show": true 6126 } 6127 ], 6128 "yaxis": { 6129 "align": false, 6130 "alignLevel": 0 6131 } 6132 }, 6133 { 6134 "aliasColors": {}, 6135 "bars": false, 6136 "cacheTimeout": null, 6137 "datasource": "$source", 6138 "description": null, 6139 "editable": true, 6140 "error": false, 6141 "fieldConfig": { 6142 "defaults": { 6143 "thresholds": { 6144 "mode": "absolute", 6145 "steps": [] 6146 } 6147 } 6148 }, 6149 "fill": 1, 6150 "grid": { 6151 "threshold1": null, 6152 "threshold1Color": "rgba(216, 200, 27, 0.27)", 6153 "threshold2": null, 6154 "threshold2Color": "rgba(234, 112, 112, 0.22)" 6155 }, 6156 "gridPos": null, 6157 "height": null, 6158 "hideTimeOverride": false, 6159 "id": 47, 6160 "interval": null, 6161 "isNew": true, 6162 "legend": { 6163 "alignAsTable": false, 6164 "avg": false, 6165 "current": false, 6166 "hideEmpty": false, 6167 "hideZero": false, 6168 "max": false, 6169 "min": false, 6170 "rightSide": false, 6171 "show": true, 6172 "sideWidth": null, 6173 "sort": null, 6174 "sortDesc": false, 6175 "total": false, 6176 "values": false 6177 }, 6178 "lines": true, 6179 "linewidth": 2, 6180 "links": [], 6181 "maxDataPoints": 100, 6182 "maxPerRow": null, 6183 "minSpan": null, 6184 "nullPointMode": null, 6185 "options": { 6186 "alertThreshold": true, 6187 "dataLinks": [] 6188 }, 6189 "percentage": false, 6190 "pointradius": 5, 6191 "points": false, 6192 "renderer": "flot", 6193 "repeat": null, 6194 "repeatDirection": null, 6195 "seriesOverrides": [], 6196 "span": 12, 6197 "stack": false, 6198 "steppedLine": false, 6199 "targets": [ 6200 { 6201 "datasource": "$source", 6202 "expr": "sum(irate(apiserver_request_total{verb=\"POST\", resource=\"pods\", subresource=\"\"}[1m]))", 6203 "format": "time_series", 6204 "hide": false, 6205 "instant": false, 6206 "interval": "5s", 6207 "intervalFactor": 1, 6208 "legendFormat": "", 6209 "metric": "", 6210 "query": "sum(irate(apiserver_request_total{verb=\"POST\", resource=\"pods\", subresource=\"\"}[1m]))", 6211 "refId": "", 6212 "step": 10, 6213 "target": "" 6214 } 6215 ], 6216 "thresholds": [], 6217 "timeFrom": null, 6218 "timeShift": null, 6219 "title": "Pod creations", 6220 "tooltip": { 6221 "msResolution": true, 6222 "shared": true, 6223 "sort": 2, 6224 "value_type": "cumulative" 6225 }, 6226 "transformations": [], 6227 "transparent": false, 6228 "type": "graph", 6229 "xaxis": { 6230 "mode": "time", 6231 "name": null, 6232 "show": true, 6233 "values": [] 6234 }, 6235 "yaxes": [ 6236 { 6237 "decimals": null, 6238 "format": "ops", 6239 "label": null, 6240 "logBase": 1, 6241 "max": null, 6242 "min": null, 6243 "show": true 6244 }, 6245 { 6246 "decimals": null, 6247 "format": "short", 6248 "label": null, 6249 "logBase": 1, 6250 "max": null, 6251 "min": null, 6252 "show": true 6253 } 6254 ], 6255 "yaxis": { 6256 "align": false, 6257 "alignLevel": 0 6258 } 6259 }, 6260 { 6261 "aliasColors": {}, 6262 "bars": false, 6263 "cacheTimeout": null, 6264 "datasource": "$secondary_source", 6265 "description": null, 6266 "editable": true, 6267 "error": false, 6268 "fieldConfig": { 6269 "defaults": { 6270 "thresholds": { 6271 "mode": "absolute", 6272 "steps": [] 6273 } 6274 } 6275 }, 6276 "fill": 1, 6277 "grid": { 6278 "threshold1": null, 6279 "threshold1Color": "rgba(216, 200, 27, 0.27)", 6280 "threshold2": null, 6281 "threshold2Color": "rgba(234, 112, 112, 0.22)" 6282 }, 6283 "gridPos": null, 6284 "height": null, 6285 "hideTimeOverride": false, 6286 "id": 48, 6287 "interval": null, 6288 "isNew": true, 6289 "legend": { 6290 "alignAsTable": false, 6291 "avg": false, 6292 "current": false, 6293 "hideEmpty": false, 6294 "hideZero": false, 6295 "max": false, 6296 "min": false, 6297 "rightSide": false, 6298 "show": true, 6299 "sideWidth": null, 6300 "sort": null, 6301 "sortDesc": false, 6302 "total": false, 6303 "values": false 6304 }, 6305 "lines": true, 6306 "linewidth": 2, 6307 "links": [], 6308 "maxDataPoints": 100, 6309 "maxPerRow": null, 6310 "minSpan": null, 6311 "nullPointMode": null, 6312 "options": { 6313 "alertThreshold": true, 6314 "dataLinks": [] 6315 }, 6316 "percentage": false, 6317 "pointradius": 5, 6318 "points": false, 6319 "renderer": "flot", 6320 "repeat": null, 6321 "repeatDirection": null, 6322 "seriesOverrides": [], 6323 "span": 12, 6324 "stack": false, 6325 "steppedLine": false, 6326 "targets": [ 6327 { 6328 "datasource": "$source", 6329 "expr": "sum(irate(apiserver_request_total{verb=\"POST\", resource=\"pods\", subresource=\"\"}[1m]))", 6330 "format": "time_series", 6331 "hide": false, 6332 "instant": false, 6333 "interval": "5s", 6334 "intervalFactor": 1, 6335 "legendFormat": "", 6336 "metric": "", 6337 "query": "sum(irate(apiserver_request_total{verb=\"POST\", resource=\"pods\", subresource=\"\"}[1m]))", 6338 "refId": "", 6339 "step": 10, 6340 "target": "" 6341 } 6342 ], 6343 "thresholds": [], 6344 "timeFrom": null, 6345 "timeShift": "$timeshift", 6346 "title": "[SECONDARY] Pod creations", 6347 "tooltip": { 6348 "msResolution": true, 6349 "shared": true, 6350 "sort": 2, 6351 "value_type": "cumulative" 6352 }, 6353 "transformations": [], 6354 "transparent": false, 6355 "type": "graph", 6356 "xaxis": { 6357 "mode": "time", 6358 "name": null, 6359 "show": true, 6360 "values": [] 6361 }, 6362 "yaxes": [ 6363 { 6364 "decimals": null, 6365 "format": "ops", 6366 "label": null, 6367 "logBase": 1, 6368 "max": null, 6369 "min": null, 6370 "show": true 6371 }, 6372 { 6373 "decimals": null, 6374 "format": "short", 6375 "label": null, 6376 "logBase": 1, 6377 "max": null, 6378 "min": null, 6379 "show": true 6380 } 6381 ], 6382 "yaxis": { 6383 "align": false, 6384 "alignLevel": 0 6385 } 6386 }, 6387 { 6388 "aliasColors": {}, 6389 "bars": false, 6390 "cacheTimeout": null, 6391 "datasource": "$source", 6392 "description": null, 6393 "editable": true, 6394 "error": false, 6395 "fieldConfig": { 6396 "defaults": { 6397 "thresholds": { 6398 "mode": "absolute", 6399 "steps": [] 6400 } 6401 } 6402 }, 6403 "fill": 1, 6404 "grid": { 6405 "threshold1": null, 6406 "threshold1Color": "rgba(216, 200, 27, 0.27)", 6407 "threshold2": null, 6408 "threshold2Color": "rgba(234, 112, 112, 0.22)" 6409 }, 6410 "gridPos": null, 6411 "height": null, 6412 "hideTimeOverride": false, 6413 "id": 49, 6414 "interval": null, 6415 "isNew": true, 6416 "legend": { 6417 "alignAsTable": false, 6418 "avg": false, 6419 "current": false, 6420 "hideEmpty": false, 6421 "hideZero": false, 6422 "max": false, 6423 "min": false, 6424 "rightSide": false, 6425 "show": true, 6426 "sideWidth": null, 6427 "sort": null, 6428 "sortDesc": false, 6429 "total": false, 6430 "values": false 6431 }, 6432 "lines": true, 6433 "linewidth": 2, 6434 "links": [], 6435 "maxDataPoints": 100, 6436 "maxPerRow": null, 6437 "minSpan": null, 6438 "nullPointMode": null, 6439 "options": { 6440 "alertThreshold": true, 6441 "dataLinks": [] 6442 }, 6443 "percentage": false, 6444 "pointradius": 5, 6445 "points": false, 6446 "renderer": "flot", 6447 "repeat": null, 6448 "repeatDirection": null, 6449 "seriesOverrides": [], 6450 "span": 12, 6451 "stack": false, 6452 "steppedLine": false, 6453 "targets": [ 6454 { 6455 "datasource": "$source", 6456 "expr": "sum(irate(apiserver_request_total{verb=\"POST\", resource=\"pods\", subresource=\"binding\"}[1m]))", 6457 "format": "time_series", 6458 "hide": false, 6459 "instant": false, 6460 "interval": "5s", 6461 "intervalFactor": 1, 6462 "legendFormat": "", 6463 "metric": "", 6464 "query": "sum(irate(apiserver_request_total{verb=\"POST\", resource=\"pods\", subresource=\"binding\"}[1m]))", 6465 "refId": "", 6466 "step": 10, 6467 "target": "" 6468 } 6469 ], 6470 "thresholds": [], 6471 "timeFrom": null, 6472 "timeShift": null, 6473 "title": "Pod bindings", 6474 "tooltip": { 6475 "msResolution": true, 6476 "shared": true, 6477 "sort": 2, 6478 "value_type": "cumulative" 6479 }, 6480 "transformations": [], 6481 "transparent": false, 6482 "type": "graph", 6483 "xaxis": { 6484 "mode": "time", 6485 "name": null, 6486 "show": true, 6487 "values": [] 6488 }, 6489 "yaxes": [ 6490 { 6491 "decimals": null, 6492 "format": "ops", 6493 "label": null, 6494 "logBase": 1, 6495 "max": null, 6496 "min": null, 6497 "show": true 6498 }, 6499 { 6500 "decimals": null, 6501 "format": "short", 6502 "label": null, 6503 "logBase": 1, 6504 "max": null, 6505 "min": null, 6506 "show": true 6507 } 6508 ], 6509 "yaxis": { 6510 "align": false, 6511 "alignLevel": 0 6512 } 6513 }, 6514 { 6515 "aliasColors": {}, 6516 "bars": false, 6517 "cacheTimeout": null, 6518 "datasource": "$secondary_source", 6519 "description": null, 6520 "editable": true, 6521 "error": false, 6522 "fieldConfig": { 6523 "defaults": { 6524 "thresholds": { 6525 "mode": "absolute", 6526 "steps": [] 6527 } 6528 } 6529 }, 6530 "fill": 1, 6531 "grid": { 6532 "threshold1": null, 6533 "threshold1Color": "rgba(216, 200, 27, 0.27)", 6534 "threshold2": null, 6535 "threshold2Color": "rgba(234, 112, 112, 0.22)" 6536 }, 6537 "gridPos": null, 6538 "height": null, 6539 "hideTimeOverride": false, 6540 "id": 50, 6541 "interval": null, 6542 "isNew": true, 6543 "legend": { 6544 "alignAsTable": false, 6545 "avg": false, 6546 "current": false, 6547 "hideEmpty": false, 6548 "hideZero": false, 6549 "max": false, 6550 "min": false, 6551 "rightSide": false, 6552 "show": true, 6553 "sideWidth": null, 6554 "sort": null, 6555 "sortDesc": false, 6556 "total": false, 6557 "values": false 6558 }, 6559 "lines": true, 6560 "linewidth": 2, 6561 "links": [], 6562 "maxDataPoints": 100, 6563 "maxPerRow": null, 6564 "minSpan": null, 6565 "nullPointMode": null, 6566 "options": { 6567 "alertThreshold": true, 6568 "dataLinks": [] 6569 }, 6570 "percentage": false, 6571 "pointradius": 5, 6572 "points": false, 6573 "renderer": "flot", 6574 "repeat": null, 6575 "repeatDirection": null, 6576 "seriesOverrides": [], 6577 "span": 12, 6578 "stack": false, 6579 "steppedLine": false, 6580 "targets": [ 6581 { 6582 "datasource": "$source", 6583 "expr": "sum(irate(apiserver_request_total{verb=\"POST\", resource=\"pods\", subresource=\"binding\"}[1m]))", 6584 "format": "time_series", 6585 "hide": false, 6586 "instant": false, 6587 "interval": "5s", 6588 "intervalFactor": 1, 6589 "legendFormat": "", 6590 "metric": "", 6591 "query": "sum(irate(apiserver_request_total{verb=\"POST\", resource=\"pods\", subresource=\"binding\"}[1m]))", 6592 "refId": "", 6593 "step": 10, 6594 "target": "" 6595 } 6596 ], 6597 "thresholds": [], 6598 "timeFrom": null, 6599 "timeShift": "$timeshift", 6600 "title": "[SECONDARY] Pod bindings", 6601 "tooltip": { 6602 "msResolution": true, 6603 "shared": true, 6604 "sort": 2, 6605 "value_type": "cumulative" 6606 }, 6607 "transformations": [], 6608 "transparent": false, 6609 "type": "graph", 6610 "xaxis": { 6611 "mode": "time", 6612 "name": null, 6613 "show": true, 6614 "values": [] 6615 }, 6616 "yaxes": [ 6617 { 6618 "decimals": null, 6619 "format": "ops", 6620 "label": null, 6621 "logBase": 1, 6622 "max": null, 6623 "min": null, 6624 "show": true 6625 }, 6626 { 6627 "decimals": null, 6628 "format": "short", 6629 "label": null, 6630 "logBase": 1, 6631 "max": null, 6632 "min": null, 6633 "show": true 6634 } 6635 ], 6636 "yaxis": { 6637 "align": false, 6638 "alignLevel": 0 6639 } 6640 }, 6641 { 6642 "aliasColors": {}, 6643 "bars": false, 6644 "cacheTimeout": null, 6645 "datasource": "$source", 6646 "description": null, 6647 "editable": true, 6648 "error": false, 6649 "fieldConfig": { 6650 "defaults": { 6651 "thresholds": { 6652 "mode": "absolute", 6653 "steps": [] 6654 } 6655 } 6656 }, 6657 "fill": 1, 6658 "grid": { 6659 "threshold1": null, 6660 "threshold1Color": "rgba(216, 200, 27, 0.27)", 6661 "threshold2": null, 6662 "threshold2Color": "rgba(234, 112, 112, 0.22)" 6663 }, 6664 "gridPos": null, 6665 "height": null, 6666 "hideTimeOverride": false, 6667 "id": 51, 6668 "interval": null, 6669 "isNew": true, 6670 "legend": { 6671 "alignAsTable": false, 6672 "avg": false, 6673 "current": false, 6674 "hideEmpty": false, 6675 "hideZero": false, 6676 "max": false, 6677 "min": false, 6678 "rightSide": false, 6679 "show": true, 6680 "sideWidth": null, 6681 "sort": null, 6682 "sortDesc": false, 6683 "total": false, 6684 "values": false 6685 }, 6686 "lines": true, 6687 "linewidth": 2, 6688 "links": [], 6689 "maxDataPoints": 100, 6690 "maxPerRow": null, 6691 "minSpan": null, 6692 "nullPointMode": null, 6693 "options": { 6694 "alertThreshold": true, 6695 "dataLinks": [] 6696 }, 6697 "percentage": false, 6698 "pointradius": 5, 6699 "points": false, 6700 "renderer": "flot", 6701 "repeat": null, 6702 "repeatDirection": null, 6703 "seriesOverrides": [], 6704 "span": 12, 6705 "stack": false, 6706 "steppedLine": false, 6707 "targets": [ 6708 { 6709 "datasource": "$source", 6710 "expr": "sum(rate(process_start_time_seconds[1m]) > bool 0) by (job, endpoint)", 6711 "format": "time_series", 6712 "hide": false, 6713 "instant": false, 6714 "interval": "5s", 6715 "intervalFactor": 1, 6716 "legendFormat": "", 6717 "metric": "", 6718 "query": "sum(rate(process_start_time_seconds[1m]) > bool 0) by (job, endpoint)", 6719 "refId": "", 6720 "step": 10, 6721 "target": "" 6722 } 6723 ], 6724 "thresholds": [], 6725 "timeFrom": null, 6726 "timeShift": null, 6727 "title": "Component restarts", 6728 "tooltip": { 6729 "msResolution": true, 6730 "shared": true, 6731 "sort": 2, 6732 "value_type": "cumulative" 6733 }, 6734 "transformations": [], 6735 "transparent": false, 6736 "type": "graph", 6737 "xaxis": { 6738 "mode": "time", 6739 "name": null, 6740 "show": true, 6741 "values": [] 6742 }, 6743 "yaxes": [ 6744 { 6745 "decimals": null, 6746 "format": "short", 6747 "label": null, 6748 "logBase": 1, 6749 "max": null, 6750 "min": null, 6751 "show": true 6752 }, 6753 { 6754 "decimals": null, 6755 "format": "short", 6756 "label": null, 6757 "logBase": 1, 6758 "max": null, 6759 "min": null, 6760 "show": true 6761 } 6762 ], 6763 "yaxis": { 6764 "align": false, 6765 "alignLevel": 0 6766 } 6767 }, 6768 { 6769 "aliasColors": {}, 6770 "bars": false, 6771 "cacheTimeout": null, 6772 "datasource": "$secondary_source", 6773 "description": null, 6774 "editable": true, 6775 "error": false, 6776 "fieldConfig": { 6777 "defaults": { 6778 "thresholds": { 6779 "mode": "absolute", 6780 "steps": [] 6781 } 6782 } 6783 }, 6784 "fill": 1, 6785 "grid": { 6786 "threshold1": null, 6787 "threshold1Color": "rgba(216, 200, 27, 0.27)", 6788 "threshold2": null, 6789 "threshold2Color": "rgba(234, 112, 112, 0.22)" 6790 }, 6791 "gridPos": null, 6792 "height": null, 6793 "hideTimeOverride": false, 6794 "id": 52, 6795 "interval": null, 6796 "isNew": true, 6797 "legend": { 6798 "alignAsTable": false, 6799 "avg": false, 6800 "current": false, 6801 "hideEmpty": false, 6802 "hideZero": false, 6803 "max": false, 6804 "min": false, 6805 "rightSide": false, 6806 "show": true, 6807 "sideWidth": null, 6808 "sort": null, 6809 "sortDesc": false, 6810 "total": false, 6811 "values": false 6812 }, 6813 "lines": true, 6814 "linewidth": 2, 6815 "links": [], 6816 "maxDataPoints": 100, 6817 "maxPerRow": null, 6818 "minSpan": null, 6819 "nullPointMode": null, 6820 "options": { 6821 "alertThreshold": true, 6822 "dataLinks": [] 6823 }, 6824 "percentage": false, 6825 "pointradius": 5, 6826 "points": false, 6827 "renderer": "flot", 6828 "repeat": null, 6829 "repeatDirection": null, 6830 "seriesOverrides": [], 6831 "span": 12, 6832 "stack": false, 6833 "steppedLine": false, 6834 "targets": [ 6835 { 6836 "datasource": "$source", 6837 "expr": "sum(rate(process_start_time_seconds[1m]) > bool 0) by (job, endpoint)", 6838 "format": "time_series", 6839 "hide": false, 6840 "instant": false, 6841 "interval": "5s", 6842 "intervalFactor": 1, 6843 "legendFormat": "", 6844 "metric": "", 6845 "query": "sum(rate(process_start_time_seconds[1m]) > bool 0) by (job, endpoint)", 6846 "refId": "", 6847 "step": 10, 6848 "target": "" 6849 } 6850 ], 6851 "thresholds": [], 6852 "timeFrom": null, 6853 "timeShift": "$timeshift", 6854 "title": "[SECONDARY] Component restarts", 6855 "tooltip": { 6856 "msResolution": true, 6857 "shared": true, 6858 "sort": 2, 6859 "value_type": "cumulative" 6860 }, 6861 "transformations": [], 6862 "transparent": false, 6863 "type": "graph", 6864 "xaxis": { 6865 "mode": "time", 6866 "name": null, 6867 "show": true, 6868 "values": [] 6869 }, 6870 "yaxes": [ 6871 { 6872 "decimals": null, 6873 "format": "short", 6874 "label": null, 6875 "logBase": 1, 6876 "max": null, 6877 "min": null, 6878 "show": true 6879 }, 6880 { 6881 "decimals": null, 6882 "format": "short", 6883 "label": null, 6884 "logBase": 1, 6885 "max": null, 6886 "min": null, 6887 "show": true 6888 } 6889 ], 6890 "yaxis": { 6891 "align": false, 6892 "alignLevel": 0 6893 } 6894 }, 6895 { 6896 "aliasColors": {}, 6897 "bars": false, 6898 "cacheTimeout": null, 6899 "datasource": "$source", 6900 "description": null, 6901 "editable": true, 6902 "error": false, 6903 "fieldConfig": { 6904 "defaults": { 6905 "thresholds": { 6906 "mode": "absolute", 6907 "steps": [] 6908 } 6909 } 6910 }, 6911 "fill": 1, 6912 "grid": { 6913 "threshold1": null, 6914 "threshold1Color": "rgba(216, 200, 27, 0.27)", 6915 "threshold2": null, 6916 "threshold2Color": "rgba(234, 112, 112, 0.22)" 6917 }, 6918 "gridPos": null, 6919 "height": null, 6920 "hideTimeOverride": false, 6921 "id": 53, 6922 "interval": null, 6923 "isNew": true, 6924 "legend": { 6925 "alignAsTable": false, 6926 "avg": false, 6927 "current": false, 6928 "hideEmpty": false, 6929 "hideZero": false, 6930 "max": false, 6931 "min": false, 6932 "rightSide": false, 6933 "show": true, 6934 "sideWidth": null, 6935 "sort": null, 6936 "sortDesc": false, 6937 "total": false, 6938 "values": false 6939 }, 6940 "lines": true, 6941 "linewidth": 2, 6942 "links": [], 6943 "maxDataPoints": 100, 6944 "maxPerRow": null, 6945 "minSpan": null, 6946 "nullPointMode": null, 6947 "options": { 6948 "alertThreshold": true, 6949 "dataLinks": [] 6950 }, 6951 "percentage": false, 6952 "pointradius": 5, 6953 "points": false, 6954 "renderer": "flot", 6955 "repeat": null, 6956 "repeatDirection": null, 6957 "seriesOverrides": [], 6958 "span": 12, 6959 "stack": false, 6960 "steppedLine": false, 6961 "targets": [ 6962 { 6963 "datasource": "$source", 6964 "expr": "sum(min_over_time(container_start_time_seconds{container!=\"\",container!=\"POD\"}[2m])) by (container)", 6965 "format": "time_series", 6966 "hide": false, 6967 "instant": false, 6968 "interval": "5s", 6969 "intervalFactor": 1, 6970 "legendFormat": "", 6971 "metric": "", 6972 "query": "sum(min_over_time(container_start_time_seconds{container!=\"\",container!=\"POD\"}[2m])) by (container)", 6973 "refId": "", 6974 "step": 10, 6975 "target": "" 6976 } 6977 ], 6978 "thresholds": [], 6979 "timeFrom": null, 6980 "timeShift": null, 6981 "title": "Component restarts 2", 6982 "tooltip": { 6983 "msResolution": true, 6984 "shared": true, 6985 "sort": 2, 6986 "value_type": "cumulative" 6987 }, 6988 "transformations": [], 6989 "transparent": false, 6990 "type": "graph", 6991 "xaxis": { 6992 "mode": "time", 6993 "name": null, 6994 "show": true, 6995 "values": [] 6996 }, 6997 "yaxes": [ 6998 { 6999 "decimals": null, 7000 "format": "short", 7001 "label": null, 7002 "logBase": 1, 7003 "max": null, 7004 "min": null, 7005 "show": true 7006 }, 7007 { 7008 "decimals": null, 7009 "format": "short", 7010 "label": null, 7011 "logBase": 1, 7012 "max": null, 7013 "min": null, 7014 "show": true 7015 } 7016 ], 7017 "yaxis": { 7018 "align": false, 7019 "alignLevel": 0 7020 } 7021 }, 7022 { 7023 "aliasColors": {}, 7024 "bars": false, 7025 "cacheTimeout": null, 7026 "datasource": "$secondary_source", 7027 "description": null, 7028 "editable": true, 7029 "error": false, 7030 "fieldConfig": { 7031 "defaults": { 7032 "thresholds": { 7033 "mode": "absolute", 7034 "steps": [] 7035 } 7036 } 7037 }, 7038 "fill": 1, 7039 "grid": { 7040 "threshold1": null, 7041 "threshold1Color": "rgba(216, 200, 27, 0.27)", 7042 "threshold2": null, 7043 "threshold2Color": "rgba(234, 112, 112, 0.22)" 7044 }, 7045 "gridPos": null, 7046 "height": null, 7047 "hideTimeOverride": false, 7048 "id": 54, 7049 "interval": null, 7050 "isNew": true, 7051 "legend": { 7052 "alignAsTable": false, 7053 "avg": false, 7054 "current": false, 7055 "hideEmpty": false, 7056 "hideZero": false, 7057 "max": false, 7058 "min": false, 7059 "rightSide": false, 7060 "show": true, 7061 "sideWidth": null, 7062 "sort": null, 7063 "sortDesc": false, 7064 "total": false, 7065 "values": false 7066 }, 7067 "lines": true, 7068 "linewidth": 2, 7069 "links": [], 7070 "maxDataPoints": 100, 7071 "maxPerRow": null, 7072 "minSpan": null, 7073 "nullPointMode": null, 7074 "options": { 7075 "alertThreshold": true, 7076 "dataLinks": [] 7077 }, 7078 "percentage": false, 7079 "pointradius": 5, 7080 "points": false, 7081 "renderer": "flot", 7082 "repeat": null, 7083 "repeatDirection": null, 7084 "seriesOverrides": [], 7085 "span": 12, 7086 "stack": false, 7087 "steppedLine": false, 7088 "targets": [ 7089 { 7090 "datasource": "$source", 7091 "expr": "sum(min_over_time(container_start_time_seconds{container!=\"\",container!=\"POD\"}[2m])) by (container)", 7092 "format": "time_series", 7093 "hide": false, 7094 "instant": false, 7095 "interval": "5s", 7096 "intervalFactor": 1, 7097 "legendFormat": "", 7098 "metric": "", 7099 "query": "sum(min_over_time(container_start_time_seconds{container!=\"\",container!=\"POD\"}[2m])) by (container)", 7100 "refId": "", 7101 "step": 10, 7102 "target": "" 7103 } 7104 ], 7105 "thresholds": [], 7106 "timeFrom": null, 7107 "timeShift": "$timeshift", 7108 "title": "[SECONDARY] Component restarts 2", 7109 "tooltip": { 7110 "msResolution": true, 7111 "shared": true, 7112 "sort": 2, 7113 "value_type": "cumulative" 7114 }, 7115 "transformations": [], 7116 "transparent": false, 7117 "type": "graph", 7118 "xaxis": { 7119 "mode": "time", 7120 "name": null, 7121 "show": true, 7122 "values": [] 7123 }, 7124 "yaxes": [ 7125 { 7126 "decimals": null, 7127 "format": "short", 7128 "label": null, 7129 "logBase": 1, 7130 "max": null, 7131 "min": null, 7132 "show": true 7133 }, 7134 { 7135 "decimals": null, 7136 "format": "short", 7137 "label": null, 7138 "logBase": 1, 7139 "max": null, 7140 "min": null, 7141 "show": true 7142 } 7143 ], 7144 "yaxis": { 7145 "align": false, 7146 "alignLevel": 0 7147 } 7148 }, 7149 { 7150 "aliasColors": {}, 7151 "bars": false, 7152 "cacheTimeout": null, 7153 "datasource": "$source", 7154 "description": null, 7155 "editable": true, 7156 "error": false, 7157 "fieldConfig": { 7158 "defaults": { 7159 "thresholds": { 7160 "mode": "absolute", 7161 "steps": [] 7162 } 7163 } 7164 }, 7165 "fill": 1, 7166 "grid": { 7167 "threshold1": null, 7168 "threshold1Color": "rgba(216, 200, 27, 0.27)", 7169 "threshold2": null, 7170 "threshold2Color": "rgba(234, 112, 112, 0.22)" 7171 }, 7172 "gridPos": null, 7173 "height": null, 7174 "hideTimeOverride": false, 7175 "id": 55, 7176 "interval": null, 7177 "isNew": true, 7178 "legend": { 7179 "alignAsTable": false, 7180 "avg": false, 7181 "current": false, 7182 "hideEmpty": false, 7183 "hideZero": false, 7184 "max": false, 7185 "min": false, 7186 "rightSide": false, 7187 "show": true, 7188 "sideWidth": null, 7189 "sort": null, 7190 "sortDesc": false, 7191 "total": false, 7192 "values": false 7193 }, 7194 "lines": true, 7195 "linewidth": 2, 7196 "links": [], 7197 "maxDataPoints": 100, 7198 "maxPerRow": null, 7199 "minSpan": null, 7200 "nullPointMode": null, 7201 "options": { 7202 "alertThreshold": true, 7203 "dataLinks": [] 7204 }, 7205 "percentage": false, 7206 "pointradius": 5, 7207 "points": false, 7208 "renderer": "flot", 7209 "repeat": null, 7210 "repeatDirection": null, 7211 "seriesOverrides": [], 7212 "span": 12, 7213 "stack": false, 7214 "steppedLine": false, 7215 "targets": [ 7216 { 7217 "datasource": "$source", 7218 "expr": "sum(leader_election_master_status) by (name, instance)", 7219 "format": "time_series", 7220 "hide": false, 7221 "instant": false, 7222 "interval": "5s", 7223 "intervalFactor": 1, 7224 "legendFormat": "", 7225 "metric": "", 7226 "query": "sum(leader_election_master_status) by (name, instance)", 7227 "refId": "", 7228 "step": 10, 7229 "target": "" 7230 } 7231 ], 7232 "thresholds": [], 7233 "timeFrom": null, 7234 "timeShift": null, 7235 "title": "Active component", 7236 "tooltip": { 7237 "msResolution": true, 7238 "shared": true, 7239 "sort": 2, 7240 "value_type": "cumulative" 7241 }, 7242 "transformations": [], 7243 "transparent": false, 7244 "type": "graph", 7245 "xaxis": { 7246 "mode": "time", 7247 "name": null, 7248 "show": true, 7249 "values": [] 7250 }, 7251 "yaxes": [ 7252 { 7253 "decimals": null, 7254 "format": "short", 7255 "label": null, 7256 "logBase": 1, 7257 "max": null, 7258 "min": null, 7259 "show": true 7260 }, 7261 { 7262 "decimals": null, 7263 "format": "short", 7264 "label": null, 7265 "logBase": 1, 7266 "max": null, 7267 "min": null, 7268 "show": true 7269 } 7270 ], 7271 "yaxis": { 7272 "align": false, 7273 "alignLevel": 0 7274 } 7275 }, 7276 { 7277 "aliasColors": {}, 7278 "bars": false, 7279 "cacheTimeout": null, 7280 "datasource": "$secondary_source", 7281 "description": null, 7282 "editable": true, 7283 "error": false, 7284 "fieldConfig": { 7285 "defaults": { 7286 "thresholds": { 7287 "mode": "absolute", 7288 "steps": [] 7289 } 7290 } 7291 }, 7292 "fill": 1, 7293 "grid": { 7294 "threshold1": null, 7295 "threshold1Color": "rgba(216, 200, 27, 0.27)", 7296 "threshold2": null, 7297 "threshold2Color": "rgba(234, 112, 112, 0.22)" 7298 }, 7299 "gridPos": null, 7300 "height": null, 7301 "hideTimeOverride": false, 7302 "id": 56, 7303 "interval": null, 7304 "isNew": true, 7305 "legend": { 7306 "alignAsTable": false, 7307 "avg": false, 7308 "current": false, 7309 "hideEmpty": false, 7310 "hideZero": false, 7311 "max": false, 7312 "min": false, 7313 "rightSide": false, 7314 "show": true, 7315 "sideWidth": null, 7316 "sort": null, 7317 "sortDesc": false, 7318 "total": false, 7319 "values": false 7320 }, 7321 "lines": true, 7322 "linewidth": 2, 7323 "links": [], 7324 "maxDataPoints": 100, 7325 "maxPerRow": null, 7326 "minSpan": null, 7327 "nullPointMode": null, 7328 "options": { 7329 "alertThreshold": true, 7330 "dataLinks": [] 7331 }, 7332 "percentage": false, 7333 "pointradius": 5, 7334 "points": false, 7335 "renderer": "flot", 7336 "repeat": null, 7337 "repeatDirection": null, 7338 "seriesOverrides": [], 7339 "span": 12, 7340 "stack": false, 7341 "steppedLine": false, 7342 "targets": [ 7343 { 7344 "datasource": "$source", 7345 "expr": "sum(leader_election_master_status) by (name, instance)", 7346 "format": "time_series", 7347 "hide": false, 7348 "instant": false, 7349 "interval": "5s", 7350 "intervalFactor": 1, 7351 "legendFormat": "", 7352 "metric": "", 7353 "query": "sum(leader_election_master_status) by (name, instance)", 7354 "refId": "", 7355 "step": 10, 7356 "target": "" 7357 } 7358 ], 7359 "thresholds": [], 7360 "timeFrom": null, 7361 "timeShift": "$timeshift", 7362 "title": "[SECONDARY] Active component", 7363 "tooltip": { 7364 "msResolution": true, 7365 "shared": true, 7366 "sort": 2, 7367 "value_type": "cumulative" 7368 }, 7369 "transformations": [], 7370 "transparent": false, 7371 "type": "graph", 7372 "xaxis": { 7373 "mode": "time", 7374 "name": null, 7375 "show": true, 7376 "values": [] 7377 }, 7378 "yaxes": [ 7379 { 7380 "decimals": null, 7381 "format": "short", 7382 "label": null, 7383 "logBase": 1, 7384 "max": null, 7385 "min": null, 7386 "show": true 7387 }, 7388 { 7389 "decimals": null, 7390 "format": "short", 7391 "label": null, 7392 "logBase": 1, 7393 "max": null, 7394 "min": null, 7395 "show": true 7396 } 7397 ], 7398 "yaxis": { 7399 "align": false, 7400 "alignLevel": 0 7401 } 7402 } 7403 ], 7404 "repeat": null, 7405 "showTitle": true, 7406 "title": "Overall cluster health" 7407 }, 7408 { 7409 "collapse": true, 7410 "editable": true, 7411 "height": "300px", 7412 "panels": [ 7413 { 7414 "aliasColors": {}, 7415 "bars": false, 7416 "cacheTimeout": null, 7417 "datasource": "$source", 7418 "description": null, 7419 "editable": true, 7420 "error": false, 7421 "fieldConfig": { 7422 "defaults": { 7423 "thresholds": { 7424 "mode": "absolute", 7425 "steps": [] 7426 } 7427 } 7428 }, 7429 "fill": 1, 7430 "grid": { 7431 "threshold1": null, 7432 "threshold1Color": "rgba(216, 200, 27, 0.27)", 7433 "threshold2": null, 7434 "threshold2Color": "rgba(234, 112, 112, 0.22)" 7435 }, 7436 "gridPos": null, 7437 "height": null, 7438 "hideTimeOverride": false, 7439 "id": 57, 7440 "interval": null, 7441 "isNew": true, 7442 "legend": { 7443 "alignAsTable": false, 7444 "avg": false, 7445 "current": false, 7446 "hideEmpty": false, 7447 "hideZero": false, 7448 "max": false, 7449 "min": false, 7450 "rightSide": false, 7451 "show": true, 7452 "sideWidth": null, 7453 "sort": null, 7454 "sortDesc": false, 7455 "total": false, 7456 "values": false 7457 }, 7458 "lines": true, 7459 "linewidth": 2, 7460 "links": [], 7461 "maxDataPoints": 100, 7462 "maxPerRow": null, 7463 "minSpan": null, 7464 "nullPointMode": null, 7465 "options": { 7466 "alertThreshold": true, 7467 "dataLinks": [] 7468 }, 7469 "percentage": false, 7470 "pointradius": 5, 7471 "points": false, 7472 "renderer": "flot", 7473 "repeat": null, 7474 "repeatDirection": null, 7475 "seriesOverrides": [], 7476 "span": 12, 7477 "stack": false, 7478 "steppedLine": false, 7479 "targets": [ 7480 { 7481 "datasource": "$source", 7482 "expr": "etcd_server_is_leader", 7483 "format": "time_series", 7484 "hide": false, 7485 "instant": false, 7486 "interval": "5s", 7487 "intervalFactor": 1, 7488 "legendFormat": "{{instance}}", 7489 "metric": "", 7490 "query": "etcd_server_is_leader", 7491 "refId": "", 7492 "step": 10, 7493 "target": "" 7494 } 7495 ], 7496 "thresholds": [], 7497 "timeFrom": null, 7498 "timeShift": null, 7499 "title": "etcd leader", 7500 "tooltip": { 7501 "msResolution": true, 7502 "shared": true, 7503 "sort": 2, 7504 "value_type": "cumulative" 7505 }, 7506 "transformations": [], 7507 "transparent": false, 7508 "type": "graph", 7509 "xaxis": { 7510 "mode": "time", 7511 "name": null, 7512 "show": true, 7513 "values": [] 7514 }, 7515 "yaxes": [ 7516 { 7517 "decimals": null, 7518 "format": "short", 7519 "label": null, 7520 "logBase": 1, 7521 "max": null, 7522 "min": null, 7523 "show": true 7524 }, 7525 { 7526 "decimals": null, 7527 "format": "short", 7528 "label": null, 7529 "logBase": 1, 7530 "max": null, 7531 "min": null, 7532 "show": true 7533 } 7534 ], 7535 "yaxis": { 7536 "align": false, 7537 "alignLevel": 0 7538 } 7539 }, 7540 { 7541 "aliasColors": {}, 7542 "bars": false, 7543 "cacheTimeout": null, 7544 "datasource": "$secondary_source", 7545 "description": null, 7546 "editable": true, 7547 "error": false, 7548 "fieldConfig": { 7549 "defaults": { 7550 "thresholds": { 7551 "mode": "absolute", 7552 "steps": [] 7553 } 7554 } 7555 }, 7556 "fill": 1, 7557 "grid": { 7558 "threshold1": null, 7559 "threshold1Color": "rgba(216, 200, 27, 0.27)", 7560 "threshold2": null, 7561 "threshold2Color": "rgba(234, 112, 112, 0.22)" 7562 }, 7563 "gridPos": null, 7564 "height": null, 7565 "hideTimeOverride": false, 7566 "id": 58, 7567 "interval": null, 7568 "isNew": true, 7569 "legend": { 7570 "alignAsTable": false, 7571 "avg": false, 7572 "current": false, 7573 "hideEmpty": false, 7574 "hideZero": false, 7575 "max": false, 7576 "min": false, 7577 "rightSide": false, 7578 "show": true, 7579 "sideWidth": null, 7580 "sort": null, 7581 "sortDesc": false, 7582 "total": false, 7583 "values": false 7584 }, 7585 "lines": true, 7586 "linewidth": 2, 7587 "links": [], 7588 "maxDataPoints": 100, 7589 "maxPerRow": null, 7590 "minSpan": null, 7591 "nullPointMode": null, 7592 "options": { 7593 "alertThreshold": true, 7594 "dataLinks": [] 7595 }, 7596 "percentage": false, 7597 "pointradius": 5, 7598 "points": false, 7599 "renderer": "flot", 7600 "repeat": null, 7601 "repeatDirection": null, 7602 "seriesOverrides": [], 7603 "span": 12, 7604 "stack": false, 7605 "steppedLine": false, 7606 "targets": [ 7607 { 7608 "datasource": "$source", 7609 "expr": "etcd_server_is_leader", 7610 "format": "time_series", 7611 "hide": false, 7612 "instant": false, 7613 "interval": "5s", 7614 "intervalFactor": 1, 7615 "legendFormat": "{{instance}}", 7616 "metric": "", 7617 "query": "etcd_server_is_leader", 7618 "refId": "", 7619 "step": 10, 7620 "target": "" 7621 } 7622 ], 7623 "thresholds": [], 7624 "timeFrom": null, 7625 "timeShift": "$timeshift", 7626 "title": "[SECONDARY] etcd leader", 7627 "tooltip": { 7628 "msResolution": true, 7629 "shared": true, 7630 "sort": 2, 7631 "value_type": "cumulative" 7632 }, 7633 "transformations": [], 7634 "transparent": false, 7635 "type": "graph", 7636 "xaxis": { 7637 "mode": "time", 7638 "name": null, 7639 "show": true, 7640 "values": [] 7641 }, 7642 "yaxes": [ 7643 { 7644 "decimals": null, 7645 "format": "short", 7646 "label": null, 7647 "logBase": 1, 7648 "max": null, 7649 "min": null, 7650 "show": true 7651 }, 7652 { 7653 "decimals": null, 7654 "format": "short", 7655 "label": null, 7656 "logBase": 1, 7657 "max": null, 7658 "min": null, 7659 "show": true 7660 } 7661 ], 7662 "yaxis": { 7663 "align": false, 7664 "alignLevel": 0 7665 } 7666 }, 7667 { 7668 "aliasColors": {}, 7669 "bars": false, 7670 "cacheTimeout": null, 7671 "datasource": "$source", 7672 "description": null, 7673 "editable": true, 7674 "error": false, 7675 "fieldConfig": { 7676 "defaults": { 7677 "thresholds": { 7678 "mode": "absolute", 7679 "steps": [] 7680 } 7681 } 7682 }, 7683 "fill": 1, 7684 "grid": { 7685 "threshold1": null, 7686 "threshold1Color": "rgba(216, 200, 27, 0.27)", 7687 "threshold2": null, 7688 "threshold2Color": "rgba(234, 112, 112, 0.22)" 7689 }, 7690 "gridPos": null, 7691 "height": null, 7692 "hideTimeOverride": false, 7693 "id": 59, 7694 "interval": null, 7695 "isNew": true, 7696 "legend": { 7697 "alignAsTable": false, 7698 "avg": false, 7699 "current": false, 7700 "hideEmpty": false, 7701 "hideZero": false, 7702 "max": false, 7703 "min": false, 7704 "rightSide": false, 7705 "show": true, 7706 "sideWidth": null, 7707 "sort": null, 7708 "sortDesc": false, 7709 "total": false, 7710 "values": false 7711 }, 7712 "lines": true, 7713 "linewidth": 2, 7714 "links": [], 7715 "maxDataPoints": 100, 7716 "maxPerRow": null, 7717 "minSpan": null, 7718 "nullPointMode": null, 7719 "options": { 7720 "alertThreshold": true, 7721 "dataLinks": [] 7722 }, 7723 "percentage": false, 7724 "pointradius": 5, 7725 "points": false, 7726 "renderer": "flot", 7727 "repeat": null, 7728 "repeatDirection": null, 7729 "seriesOverrides": [], 7730 "span": 12, 7731 "stack": false, 7732 "steppedLine": false, 7733 "targets": [ 7734 { 7735 "datasource": "$source", 7736 "expr": "rate(etcd_network_client_grpc_sent_bytes_total[1m])", 7737 "format": "time_series", 7738 "hide": false, 7739 "instant": false, 7740 "interval": "5s", 7741 "intervalFactor": 1, 7742 "legendFormat": "{{instance}}", 7743 "metric": "", 7744 "query": "rate(etcd_network_client_grpc_sent_bytes_total[1m])", 7745 "refId": "", 7746 "step": 10, 7747 "target": "" 7748 } 7749 ], 7750 "thresholds": [], 7751 "timeFrom": null, 7752 "timeShift": null, 7753 "title": "etcd bytes sent", 7754 "tooltip": { 7755 "msResolution": true, 7756 "shared": true, 7757 "sort": 2, 7758 "value_type": "cumulative" 7759 }, 7760 "transformations": [], 7761 "transparent": false, 7762 "type": "graph", 7763 "xaxis": { 7764 "mode": "time", 7765 "name": null, 7766 "show": true, 7767 "values": [] 7768 }, 7769 "yaxes": [ 7770 { 7771 "decimals": null, 7772 "format": "Bps", 7773 "label": null, 7774 "logBase": 1, 7775 "max": null, 7776 "min": null, 7777 "show": true 7778 }, 7779 { 7780 "decimals": null, 7781 "format": "short", 7782 "label": null, 7783 "logBase": 1, 7784 "max": null, 7785 "min": null, 7786 "show": true 7787 } 7788 ], 7789 "yaxis": { 7790 "align": false, 7791 "alignLevel": 0 7792 } 7793 }, 7794 { 7795 "aliasColors": {}, 7796 "bars": false, 7797 "cacheTimeout": null, 7798 "datasource": "$secondary_source", 7799 "description": null, 7800 "editable": true, 7801 "error": false, 7802 "fieldConfig": { 7803 "defaults": { 7804 "thresholds": { 7805 "mode": "absolute", 7806 "steps": [] 7807 } 7808 } 7809 }, 7810 "fill": 1, 7811 "grid": { 7812 "threshold1": null, 7813 "threshold1Color": "rgba(216, 200, 27, 0.27)", 7814 "threshold2": null, 7815 "threshold2Color": "rgba(234, 112, 112, 0.22)" 7816 }, 7817 "gridPos": null, 7818 "height": null, 7819 "hideTimeOverride": false, 7820 "id": 60, 7821 "interval": null, 7822 "isNew": true, 7823 "legend": { 7824 "alignAsTable": false, 7825 "avg": false, 7826 "current": false, 7827 "hideEmpty": false, 7828 "hideZero": false, 7829 "max": false, 7830 "min": false, 7831 "rightSide": false, 7832 "show": true, 7833 "sideWidth": null, 7834 "sort": null, 7835 "sortDesc": false, 7836 "total": false, 7837 "values": false 7838 }, 7839 "lines": true, 7840 "linewidth": 2, 7841 "links": [], 7842 "maxDataPoints": 100, 7843 "maxPerRow": null, 7844 "minSpan": null, 7845 "nullPointMode": null, 7846 "options": { 7847 "alertThreshold": true, 7848 "dataLinks": [] 7849 }, 7850 "percentage": false, 7851 "pointradius": 5, 7852 "points": false, 7853 "renderer": "flot", 7854 "repeat": null, 7855 "repeatDirection": null, 7856 "seriesOverrides": [], 7857 "span": 12, 7858 "stack": false, 7859 "steppedLine": false, 7860 "targets": [ 7861 { 7862 "datasource": "$source", 7863 "expr": "rate(etcd_network_client_grpc_sent_bytes_total[1m])", 7864 "format": "time_series", 7865 "hide": false, 7866 "instant": false, 7867 "interval": "5s", 7868 "intervalFactor": 1, 7869 "legendFormat": "{{instance}}", 7870 "metric": "", 7871 "query": "rate(etcd_network_client_grpc_sent_bytes_total[1m])", 7872 "refId": "", 7873 "step": 10, 7874 "target": "" 7875 } 7876 ], 7877 "thresholds": [], 7878 "timeFrom": null, 7879 "timeShift": "$timeshift", 7880 "title": "[SECONDARY] etcd bytes sent", 7881 "tooltip": { 7882 "msResolution": true, 7883 "shared": true, 7884 "sort": 2, 7885 "value_type": "cumulative" 7886 }, 7887 "transformations": [], 7888 "transparent": false, 7889 "type": "graph", 7890 "xaxis": { 7891 "mode": "time", 7892 "name": null, 7893 "show": true, 7894 "values": [] 7895 }, 7896 "yaxes": [ 7897 { 7898 "decimals": null, 7899 "format": "Bps", 7900 "label": null, 7901 "logBase": 1, 7902 "max": null, 7903 "min": null, 7904 "show": true 7905 }, 7906 { 7907 "decimals": null, 7908 "format": "short", 7909 "label": null, 7910 "logBase": 1, 7911 "max": null, 7912 "min": null, 7913 "show": true 7914 } 7915 ], 7916 "yaxis": { 7917 "align": false, 7918 "alignLevel": 0 7919 } 7920 }, 7921 { 7922 "aliasColors": {}, 7923 "bars": false, 7924 "cacheTimeout": null, 7925 "datasource": "$source", 7926 "description": null, 7927 "editable": true, 7928 "error": false, 7929 "fieldConfig": { 7930 "defaults": { 7931 "thresholds": { 7932 "mode": "absolute", 7933 "steps": [] 7934 } 7935 } 7936 }, 7937 "fill": 1, 7938 "grid": { 7939 "threshold1": null, 7940 "threshold1Color": "rgba(216, 200, 27, 0.27)", 7941 "threshold2": null, 7942 "threshold2Color": "rgba(234, 112, 112, 0.22)" 7943 }, 7944 "gridPos": null, 7945 "height": null, 7946 "hideTimeOverride": false, 7947 "id": 61, 7948 "interval": null, 7949 "isNew": true, 7950 "legend": { 7951 "alignAsTable": false, 7952 "avg": false, 7953 "current": false, 7954 "hideEmpty": false, 7955 "hideZero": false, 7956 "max": false, 7957 "min": false, 7958 "rightSide": false, 7959 "show": true, 7960 "sideWidth": null, 7961 "sort": null, 7962 "sortDesc": false, 7963 "total": false, 7964 "values": false 7965 }, 7966 "lines": true, 7967 "linewidth": 2, 7968 "links": [], 7969 "maxDataPoints": 100, 7970 "maxPerRow": null, 7971 "minSpan": null, 7972 "nullPointMode": null, 7973 "options": { 7974 "alertThreshold": true, 7975 "dataLinks": [] 7976 }, 7977 "percentage": false, 7978 "pointradius": 5, 7979 "points": false, 7980 "renderer": "flot", 7981 "repeat": null, 7982 "repeatDirection": null, 7983 "seriesOverrides": [], 7984 "span": 12, 7985 "stack": false, 7986 "steppedLine": false, 7987 "targets": [ 7988 { 7989 "datasource": "$source", 7990 "expr": "sum(rate(etcd_request_duration_seconds_count{operation=~\"${etcd_operation:regex}\", type=~\".*(${etcd_type:pipe})\"}[1m])) by (operation, type)", 7991 "format": "time_series", 7992 "hide": false, 7993 "instant": false, 7994 "interval": "5s", 7995 "intervalFactor": 1, 7996 "legendFormat": "{{operation}} {{type}}", 7997 "metric": "", 7998 "query": "sum(rate(etcd_request_duration_seconds_count{operation=~\"${etcd_operation:regex}\", type=~\".*(${etcd_type:pipe})\"}[1m])) by (operation, type)", 7999 "refId": "", 8000 "step": 10, 8001 "target": "" 8002 } 8003 ], 8004 "thresholds": [], 8005 "timeFrom": null, 8006 "timeShift": null, 8007 "title": "etcd operations rate", 8008 "tooltip": { 8009 "msResolution": true, 8010 "shared": true, 8011 "sort": 2, 8012 "value_type": "cumulative" 8013 }, 8014 "transformations": [], 8015 "transparent": false, 8016 "type": "graph", 8017 "xaxis": { 8018 "mode": "time", 8019 "name": null, 8020 "show": true, 8021 "values": [] 8022 }, 8023 "yaxes": [ 8024 { 8025 "decimals": null, 8026 "format": "ops", 8027 "label": null, 8028 "logBase": 1, 8029 "max": null, 8030 "min": null, 8031 "show": true 8032 }, 8033 { 8034 "decimals": null, 8035 "format": "short", 8036 "label": null, 8037 "logBase": 1, 8038 "max": null, 8039 "min": null, 8040 "show": true 8041 } 8042 ], 8043 "yaxis": { 8044 "align": false, 8045 "alignLevel": 0 8046 } 8047 }, 8048 { 8049 "aliasColors": {}, 8050 "bars": false, 8051 "cacheTimeout": null, 8052 "datasource": "$secondary_source", 8053 "description": null, 8054 "editable": true, 8055 "error": false, 8056 "fieldConfig": { 8057 "defaults": { 8058 "thresholds": { 8059 "mode": "absolute", 8060 "steps": [] 8061 } 8062 } 8063 }, 8064 "fill": 1, 8065 "grid": { 8066 "threshold1": null, 8067 "threshold1Color": "rgba(216, 200, 27, 0.27)", 8068 "threshold2": null, 8069 "threshold2Color": "rgba(234, 112, 112, 0.22)" 8070 }, 8071 "gridPos": null, 8072 "height": null, 8073 "hideTimeOverride": false, 8074 "id": 62, 8075 "interval": null, 8076 "isNew": true, 8077 "legend": { 8078 "alignAsTable": false, 8079 "avg": false, 8080 "current": false, 8081 "hideEmpty": false, 8082 "hideZero": false, 8083 "max": false, 8084 "min": false, 8085 "rightSide": false, 8086 "show": true, 8087 "sideWidth": null, 8088 "sort": null, 8089 "sortDesc": false, 8090 "total": false, 8091 "values": false 8092 }, 8093 "lines": true, 8094 "linewidth": 2, 8095 "links": [], 8096 "maxDataPoints": 100, 8097 "maxPerRow": null, 8098 "minSpan": null, 8099 "nullPointMode": null, 8100 "options": { 8101 "alertThreshold": true, 8102 "dataLinks": [] 8103 }, 8104 "percentage": false, 8105 "pointradius": 5, 8106 "points": false, 8107 "renderer": "flot", 8108 "repeat": null, 8109 "repeatDirection": null, 8110 "seriesOverrides": [], 8111 "span": 12, 8112 "stack": false, 8113 "steppedLine": false, 8114 "targets": [ 8115 { 8116 "datasource": "$source", 8117 "expr": "sum(rate(etcd_request_duration_seconds_count{operation=~\"${etcd_operation:regex}\", type=~\".*(${etcd_type:pipe})\"}[1m])) by (operation, type)", 8118 "format": "time_series", 8119 "hide": false, 8120 "instant": false, 8121 "interval": "5s", 8122 "intervalFactor": 1, 8123 "legendFormat": "{{operation}} {{type}}", 8124 "metric": "", 8125 "query": "sum(rate(etcd_request_duration_seconds_count{operation=~\"${etcd_operation:regex}\", type=~\".*(${etcd_type:pipe})\"}[1m])) by (operation, type)", 8126 "refId": "", 8127 "step": 10, 8128 "target": "" 8129 } 8130 ], 8131 "thresholds": [], 8132 "timeFrom": null, 8133 "timeShift": "$timeshift", 8134 "title": "[SECONDARY] etcd operations rate", 8135 "tooltip": { 8136 "msResolution": true, 8137 "shared": true, 8138 "sort": 2, 8139 "value_type": "cumulative" 8140 }, 8141 "transformations": [], 8142 "transparent": false, 8143 "type": "graph", 8144 "xaxis": { 8145 "mode": "time", 8146 "name": null, 8147 "show": true, 8148 "values": [] 8149 }, 8150 "yaxes": [ 8151 { 8152 "decimals": null, 8153 "format": "ops", 8154 "label": null, 8155 "logBase": 1, 8156 "max": null, 8157 "min": null, 8158 "show": true 8159 }, 8160 { 8161 "decimals": null, 8162 "format": "short", 8163 "label": null, 8164 "logBase": 1, 8165 "max": null, 8166 "min": null, 8167 "show": true 8168 } 8169 ], 8170 "yaxis": { 8171 "align": false, 8172 "alignLevel": 0 8173 } 8174 }, 8175 { 8176 "aliasColors": {}, 8177 "bars": false, 8178 "cacheTimeout": null, 8179 "datasource": "$source", 8180 "description": null, 8181 "editable": true, 8182 "error": false, 8183 "fieldConfig": { 8184 "defaults": { 8185 "thresholds": { 8186 "mode": "absolute", 8187 "steps": [] 8188 } 8189 } 8190 }, 8191 "fill": 1, 8192 "grid": { 8193 "threshold1": null, 8194 "threshold1Color": "rgba(216, 200, 27, 0.27)", 8195 "threshold2": null, 8196 "threshold2Color": "rgba(234, 112, 112, 0.22)" 8197 }, 8198 "gridPos": null, 8199 "height": null, 8200 "hideTimeOverride": false, 8201 "id": 63, 8202 "interval": null, 8203 "isNew": true, 8204 "legend": { 8205 "alignAsTable": false, 8206 "avg": false, 8207 "current": false, 8208 "hideEmpty": false, 8209 "hideZero": false, 8210 "max": false, 8211 "min": false, 8212 "rightSide": false, 8213 "show": true, 8214 "sideWidth": null, 8215 "sort": null, 8216 "sortDesc": false, 8217 "total": false, 8218 "values": false 8219 }, 8220 "lines": true, 8221 "linewidth": 2, 8222 "links": [], 8223 "maxDataPoints": 100, 8224 "maxPerRow": null, 8225 "minSpan": null, 8226 "nullPointMode": null, 8227 "options": { 8228 "alertThreshold": true, 8229 "dataLinks": [] 8230 }, 8231 "percentage": false, 8232 "pointradius": 5, 8233 "points": false, 8234 "renderer": "flot", 8235 "repeat": null, 8236 "repeatDirection": null, 8237 "seriesOverrides": [], 8238 "span": 12, 8239 "stack": false, 8240 "steppedLine": false, 8241 "targets": [ 8242 { 8243 "datasource": "$source", 8244 "expr": "histogram_quantile(0.99, sum(rate(etcd_request_duration_seconds_bucket{operation=~\"${etcd_operation:regex}\", type=~\".*(${etcd_type:pipe})\"}[1m])) by (le, operation, type, instance))", 8245 "format": "time_series", 8246 "hide": false, 8247 "instant": false, 8248 "interval": "5s", 8249 "intervalFactor": 1, 8250 "legendFormat": "{{operation}} {{type}} on {{instance}}", 8251 "metric": "", 8252 "query": "histogram_quantile(0.99, sum(rate(etcd_request_duration_seconds_bucket{operation=~\"${etcd_operation:regex}\", type=~\".*(${etcd_type:pipe})\"}[1m])) by (le, operation, type, instance))", 8253 "refId": "", 8254 "step": 10, 8255 "target": "" 8256 } 8257 ], 8258 "thresholds": [], 8259 "timeFrom": null, 8260 "timeShift": null, 8261 "title": "etcd get latency by type (99th percentile)", 8262 "tooltip": { 8263 "msResolution": true, 8264 "shared": true, 8265 "sort": 2, 8266 "value_type": "cumulative" 8267 }, 8268 "transformations": [], 8269 "transparent": false, 8270 "type": "graph", 8271 "xaxis": { 8272 "mode": "time", 8273 "name": null, 8274 "show": true, 8275 "values": [] 8276 }, 8277 "yaxes": [ 8278 { 8279 "decimals": null, 8280 "format": "s", 8281 "label": null, 8282 "logBase": 1, 8283 "max": null, 8284 "min": null, 8285 "show": true 8286 }, 8287 { 8288 "decimals": null, 8289 "format": "short", 8290 "label": null, 8291 "logBase": 1, 8292 "max": null, 8293 "min": null, 8294 "show": true 8295 } 8296 ], 8297 "yaxis": { 8298 "align": false, 8299 "alignLevel": 0 8300 } 8301 }, 8302 { 8303 "aliasColors": {}, 8304 "bars": false, 8305 "cacheTimeout": null, 8306 "datasource": "$secondary_source", 8307 "description": null, 8308 "editable": true, 8309 "error": false, 8310 "fieldConfig": { 8311 "defaults": { 8312 "thresholds": { 8313 "mode": "absolute", 8314 "steps": [] 8315 } 8316 } 8317 }, 8318 "fill": 1, 8319 "grid": { 8320 "threshold1": null, 8321 "threshold1Color": "rgba(216, 200, 27, 0.27)", 8322 "threshold2": null, 8323 "threshold2Color": "rgba(234, 112, 112, 0.22)" 8324 }, 8325 "gridPos": null, 8326 "height": null, 8327 "hideTimeOverride": false, 8328 "id": 64, 8329 "interval": null, 8330 "isNew": true, 8331 "legend": { 8332 "alignAsTable": false, 8333 "avg": false, 8334 "current": false, 8335 "hideEmpty": false, 8336 "hideZero": false, 8337 "max": false, 8338 "min": false, 8339 "rightSide": false, 8340 "show": true, 8341 "sideWidth": null, 8342 "sort": null, 8343 "sortDesc": false, 8344 "total": false, 8345 "values": false 8346 }, 8347 "lines": true, 8348 "linewidth": 2, 8349 "links": [], 8350 "maxDataPoints": 100, 8351 "maxPerRow": null, 8352 "minSpan": null, 8353 "nullPointMode": null, 8354 "options": { 8355 "alertThreshold": true, 8356 "dataLinks": [] 8357 }, 8358 "percentage": false, 8359 "pointradius": 5, 8360 "points": false, 8361 "renderer": "flot", 8362 "repeat": null, 8363 "repeatDirection": null, 8364 "seriesOverrides": [], 8365 "span": 12, 8366 "stack": false, 8367 "steppedLine": false, 8368 "targets": [ 8369 { 8370 "datasource": "$source", 8371 "expr": "histogram_quantile(0.99, sum(rate(etcd_request_duration_seconds_bucket{operation=~\"${etcd_operation:regex}\", type=~\".*(${etcd_type:pipe})\"}[1m])) by (le, operation, type, instance))", 8372 "format": "time_series", 8373 "hide": false, 8374 "instant": false, 8375 "interval": "5s", 8376 "intervalFactor": 1, 8377 "legendFormat": "{{operation}} {{type}} on {{instance}}", 8378 "metric": "", 8379 "query": "histogram_quantile(0.99, sum(rate(etcd_request_duration_seconds_bucket{operation=~\"${etcd_operation:regex}\", type=~\".*(${etcd_type:pipe})\"}[1m])) by (le, operation, type, instance))", 8380 "refId": "", 8381 "step": 10, 8382 "target": "" 8383 } 8384 ], 8385 "thresholds": [], 8386 "timeFrom": null, 8387 "timeShift": "$timeshift", 8388 "title": "[SECONDARY] etcd get latency by type (99th percentile)", 8389 "tooltip": { 8390 "msResolution": true, 8391 "shared": true, 8392 "sort": 2, 8393 "value_type": "cumulative" 8394 }, 8395 "transformations": [], 8396 "transparent": false, 8397 "type": "graph", 8398 "xaxis": { 8399 "mode": "time", 8400 "name": null, 8401 "show": true, 8402 "values": [] 8403 }, 8404 "yaxes": [ 8405 { 8406 "decimals": null, 8407 "format": "s", 8408 "label": null, 8409 "logBase": 1, 8410 "max": null, 8411 "min": null, 8412 "show": true 8413 }, 8414 { 8415 "decimals": null, 8416 "format": "short", 8417 "label": null, 8418 "logBase": 1, 8419 "max": null, 8420 "min": null, 8421 "show": true 8422 } 8423 ], 8424 "yaxis": { 8425 "align": false, 8426 "alignLevel": 0 8427 } 8428 }, 8429 { 8430 "aliasColors": {}, 8431 "bars": false, 8432 "cacheTimeout": null, 8433 "datasource": "$source", 8434 "description": null, 8435 "editable": true, 8436 "error": false, 8437 "fieldConfig": { 8438 "defaults": { 8439 "thresholds": { 8440 "mode": "absolute", 8441 "steps": [] 8442 } 8443 } 8444 }, 8445 "fill": 1, 8446 "grid": { 8447 "threshold1": null, 8448 "threshold1Color": "rgba(216, 200, 27, 0.27)", 8449 "threshold2": null, 8450 "threshold2Color": "rgba(234, 112, 112, 0.22)" 8451 }, 8452 "gridPos": null, 8453 "height": null, 8454 "hideTimeOverride": false, 8455 "id": 65, 8456 "interval": null, 8457 "isNew": true, 8458 "legend": { 8459 "alignAsTable": false, 8460 "avg": false, 8461 "current": false, 8462 "hideEmpty": false, 8463 "hideZero": false, 8464 "max": false, 8465 "min": false, 8466 "rightSide": false, 8467 "show": true, 8468 "sideWidth": null, 8469 "sort": null, 8470 "sortDesc": false, 8471 "total": false, 8472 "values": false 8473 }, 8474 "lines": true, 8475 "linewidth": 2, 8476 "links": [], 8477 "maxDataPoints": 100, 8478 "maxPerRow": null, 8479 "minSpan": null, 8480 "nullPointMode": null, 8481 "options": { 8482 "alertThreshold": true, 8483 "dataLinks": [] 8484 }, 8485 "percentage": false, 8486 "pointradius": 5, 8487 "points": false, 8488 "renderer": "flot", 8489 "repeat": null, 8490 "repeatDirection": null, 8491 "seriesOverrides": [], 8492 "span": 12, 8493 "stack": false, 8494 "steppedLine": false, 8495 "targets": [ 8496 { 8497 "datasource": "$source", 8498 "expr": "histogram_quantile(0.50, sum(rate(etcd_request_duration_seconds_bucket{operation=~\"${etcd_operation:regex}\", type=~\".*(${etcd_type:pipe})\"}[1m])) by (le, operation, type, instance))", 8499 "format": "time_series", 8500 "hide": false, 8501 "instant": false, 8502 "interval": "5s", 8503 "intervalFactor": 1, 8504 "legendFormat": "", 8505 "metric": "", 8506 "query": "histogram_quantile(0.50, sum(rate(etcd_request_duration_seconds_bucket{operation=~\"${etcd_operation:regex}\", type=~\".*(${etcd_type:pipe})\"}[1m])) by (le, operation, type, instance))", 8507 "refId": "", 8508 "step": 10, 8509 "target": "" 8510 } 8511 ], 8512 "thresholds": [], 8513 "timeFrom": null, 8514 "timeShift": null, 8515 "title": "etcd get latency by type (50th percentile)", 8516 "tooltip": { 8517 "msResolution": true, 8518 "shared": true, 8519 "sort": 2, 8520 "value_type": "cumulative" 8521 }, 8522 "transformations": [], 8523 "transparent": false, 8524 "type": "graph", 8525 "xaxis": { 8526 "mode": "time", 8527 "name": null, 8528 "show": true, 8529 "values": [] 8530 }, 8531 "yaxes": [ 8532 { 8533 "decimals": null, 8534 "format": "s", 8535 "label": null, 8536 "logBase": 1, 8537 "max": null, 8538 "min": null, 8539 "show": true 8540 }, 8541 { 8542 "decimals": null, 8543 "format": "short", 8544 "label": null, 8545 "logBase": 1, 8546 "max": null, 8547 "min": null, 8548 "show": true 8549 } 8550 ], 8551 "yaxis": { 8552 "align": false, 8553 "alignLevel": 0 8554 } 8555 }, 8556 { 8557 "aliasColors": {}, 8558 "bars": false, 8559 "cacheTimeout": null, 8560 "datasource": "$secondary_source", 8561 "description": null, 8562 "editable": true, 8563 "error": false, 8564 "fieldConfig": { 8565 "defaults": { 8566 "thresholds": { 8567 "mode": "absolute", 8568 "steps": [] 8569 } 8570 } 8571 }, 8572 "fill": 1, 8573 "grid": { 8574 "threshold1": null, 8575 "threshold1Color": "rgba(216, 200, 27, 0.27)", 8576 "threshold2": null, 8577 "threshold2Color": "rgba(234, 112, 112, 0.22)" 8578 }, 8579 "gridPos": null, 8580 "height": null, 8581 "hideTimeOverride": false, 8582 "id": 66, 8583 "interval": null, 8584 "isNew": true, 8585 "legend": { 8586 "alignAsTable": false, 8587 "avg": false, 8588 "current": false, 8589 "hideEmpty": false, 8590 "hideZero": false, 8591 "max": false, 8592 "min": false, 8593 "rightSide": false, 8594 "show": true, 8595 "sideWidth": null, 8596 "sort": null, 8597 "sortDesc": false, 8598 "total": false, 8599 "values": false 8600 }, 8601 "lines": true, 8602 "linewidth": 2, 8603 "links": [], 8604 "maxDataPoints": 100, 8605 "maxPerRow": null, 8606 "minSpan": null, 8607 "nullPointMode": null, 8608 "options": { 8609 "alertThreshold": true, 8610 "dataLinks": [] 8611 }, 8612 "percentage": false, 8613 "pointradius": 5, 8614 "points": false, 8615 "renderer": "flot", 8616 "repeat": null, 8617 "repeatDirection": null, 8618 "seriesOverrides": [], 8619 "span": 12, 8620 "stack": false, 8621 "steppedLine": false, 8622 "targets": [ 8623 { 8624 "datasource": "$source", 8625 "expr": "histogram_quantile(0.50, sum(rate(etcd_request_duration_seconds_bucket{operation=~\"${etcd_operation:regex}\", type=~\".*(${etcd_type:pipe})\"}[1m])) by (le, operation, type, instance))", 8626 "format": "time_series", 8627 "hide": false, 8628 "instant": false, 8629 "interval": "5s", 8630 "intervalFactor": 1, 8631 "legendFormat": "", 8632 "metric": "", 8633 "query": "histogram_quantile(0.50, sum(rate(etcd_request_duration_seconds_bucket{operation=~\"${etcd_operation:regex}\", type=~\".*(${etcd_type:pipe})\"}[1m])) by (le, operation, type, instance))", 8634 "refId": "", 8635 "step": 10, 8636 "target": "" 8637 } 8638 ], 8639 "thresholds": [], 8640 "timeFrom": null, 8641 "timeShift": "$timeshift", 8642 "title": "[SECONDARY] etcd get latency by type (50th percentile)", 8643 "tooltip": { 8644 "msResolution": true, 8645 "shared": true, 8646 "sort": 2, 8647 "value_type": "cumulative" 8648 }, 8649 "transformations": [], 8650 "transparent": false, 8651 "type": "graph", 8652 "xaxis": { 8653 "mode": "time", 8654 "name": null, 8655 "show": true, 8656 "values": [] 8657 }, 8658 "yaxes": [ 8659 { 8660 "decimals": null, 8661 "format": "s", 8662 "label": null, 8663 "logBase": 1, 8664 "max": null, 8665 "min": null, 8666 "show": true 8667 }, 8668 { 8669 "decimals": null, 8670 "format": "short", 8671 "label": null, 8672 "logBase": 1, 8673 "max": null, 8674 "min": null, 8675 "show": true 8676 } 8677 ], 8678 "yaxis": { 8679 "align": false, 8680 "alignLevel": 0 8681 } 8682 }, 8683 { 8684 "aliasColors": {}, 8685 "bars": false, 8686 "cacheTimeout": null, 8687 "datasource": "$source", 8688 "description": null, 8689 "editable": true, 8690 "error": false, 8691 "fieldConfig": { 8692 "defaults": { 8693 "thresholds": { 8694 "mode": "absolute", 8695 "steps": [] 8696 } 8697 } 8698 }, 8699 "fill": 1, 8700 "grid": { 8701 "threshold1": null, 8702 "threshold1Color": "rgba(216, 200, 27, 0.27)", 8703 "threshold2": null, 8704 "threshold2Color": "rgba(234, 112, 112, 0.22)" 8705 }, 8706 "gridPos": null, 8707 "height": null, 8708 "hideTimeOverride": false, 8709 "id": 67, 8710 "interval": null, 8711 "isNew": true, 8712 "legend": { 8713 "alignAsTable": false, 8714 "avg": false, 8715 "current": false, 8716 "hideEmpty": false, 8717 "hideZero": false, 8718 "max": false, 8719 "min": false, 8720 "rightSide": false, 8721 "show": true, 8722 "sideWidth": null, 8723 "sort": null, 8724 "sortDesc": false, 8725 "total": false, 8726 "values": false 8727 }, 8728 "lines": true, 8729 "linewidth": 2, 8730 "links": [], 8731 "maxDataPoints": 100, 8732 "maxPerRow": null, 8733 "minSpan": null, 8734 "nullPointMode": null, 8735 "options": { 8736 "alertThreshold": true, 8737 "dataLinks": [] 8738 }, 8739 "percentage": false, 8740 "pointradius": 5, 8741 "points": false, 8742 "renderer": "flot", 8743 "repeat": null, 8744 "repeatDirection": null, 8745 "seriesOverrides": [], 8746 "span": 12, 8747 "stack": false, 8748 "steppedLine": false, 8749 "targets": [ 8750 { 8751 "datasource": "$source", 8752 "expr": "sum(etcd_server_id) by (instance, server_id)", 8753 "format": "time_series", 8754 "hide": false, 8755 "instant": false, 8756 "interval": "5s", 8757 "intervalFactor": 1, 8758 "legendFormat": "", 8759 "metric": "", 8760 "query": "sum(etcd_server_id) by (instance, server_id)", 8761 "refId": "", 8762 "step": 10, 8763 "target": "" 8764 } 8765 ], 8766 "thresholds": [], 8767 "timeFrom": null, 8768 "timeShift": null, 8769 "title": "etcd instance id", 8770 "tooltip": { 8771 "msResolution": true, 8772 "shared": true, 8773 "sort": 2, 8774 "value_type": "cumulative" 8775 }, 8776 "transformations": [], 8777 "transparent": false, 8778 "type": "graph", 8779 "xaxis": { 8780 "mode": "time", 8781 "name": null, 8782 "show": true, 8783 "values": [] 8784 }, 8785 "yaxes": [ 8786 { 8787 "decimals": null, 8788 "format": "short", 8789 "label": null, 8790 "logBase": 1, 8791 "max": null, 8792 "min": null, 8793 "show": true 8794 }, 8795 { 8796 "decimals": null, 8797 "format": "short", 8798 "label": null, 8799 "logBase": 1, 8800 "max": null, 8801 "min": null, 8802 "show": true 8803 } 8804 ], 8805 "yaxis": { 8806 "align": false, 8807 "alignLevel": 0 8808 } 8809 }, 8810 { 8811 "aliasColors": {}, 8812 "bars": false, 8813 "cacheTimeout": null, 8814 "datasource": "$secondary_source", 8815 "description": null, 8816 "editable": true, 8817 "error": false, 8818 "fieldConfig": { 8819 "defaults": { 8820 "thresholds": { 8821 "mode": "absolute", 8822 "steps": [] 8823 } 8824 } 8825 }, 8826 "fill": 1, 8827 "grid": { 8828 "threshold1": null, 8829 "threshold1Color": "rgba(216, 200, 27, 0.27)", 8830 "threshold2": null, 8831 "threshold2Color": "rgba(234, 112, 112, 0.22)" 8832 }, 8833 "gridPos": null, 8834 "height": null, 8835 "hideTimeOverride": false, 8836 "id": 68, 8837 "interval": null, 8838 "isNew": true, 8839 "legend": { 8840 "alignAsTable": false, 8841 "avg": false, 8842 "current": false, 8843 "hideEmpty": false, 8844 "hideZero": false, 8845 "max": false, 8846 "min": false, 8847 "rightSide": false, 8848 "show": true, 8849 "sideWidth": null, 8850 "sort": null, 8851 "sortDesc": false, 8852 "total": false, 8853 "values": false 8854 }, 8855 "lines": true, 8856 "linewidth": 2, 8857 "links": [], 8858 "maxDataPoints": 100, 8859 "maxPerRow": null, 8860 "minSpan": null, 8861 "nullPointMode": null, 8862 "options": { 8863 "alertThreshold": true, 8864 "dataLinks": [] 8865 }, 8866 "percentage": false, 8867 "pointradius": 5, 8868 "points": false, 8869 "renderer": "flot", 8870 "repeat": null, 8871 "repeatDirection": null, 8872 "seriesOverrides": [], 8873 "span": 12, 8874 "stack": false, 8875 "steppedLine": false, 8876 "targets": [ 8877 { 8878 "datasource": "$source", 8879 "expr": "sum(etcd_server_id) by (instance, server_id)", 8880 "format": "time_series", 8881 "hide": false, 8882 "instant": false, 8883 "interval": "5s", 8884 "intervalFactor": 1, 8885 "legendFormat": "", 8886 "metric": "", 8887 "query": "sum(etcd_server_id) by (instance, server_id)", 8888 "refId": "", 8889 "step": 10, 8890 "target": "" 8891 } 8892 ], 8893 "thresholds": [], 8894 "timeFrom": null, 8895 "timeShift": "$timeshift", 8896 "title": "[SECONDARY] etcd instance id", 8897 "tooltip": { 8898 "msResolution": true, 8899 "shared": true, 8900 "sort": 2, 8901 "value_type": "cumulative" 8902 }, 8903 "transformations": [], 8904 "transparent": false, 8905 "type": "graph", 8906 "xaxis": { 8907 "mode": "time", 8908 "name": null, 8909 "show": true, 8910 "values": [] 8911 }, 8912 "yaxes": [ 8913 { 8914 "decimals": null, 8915 "format": "short", 8916 "label": null, 8917 "logBase": 1, 8918 "max": null, 8919 "min": null, 8920 "show": true 8921 }, 8922 { 8923 "decimals": null, 8924 "format": "short", 8925 "label": null, 8926 "logBase": 1, 8927 "max": null, 8928 "min": null, 8929 "show": true 8930 } 8931 ], 8932 "yaxis": { 8933 "align": false, 8934 "alignLevel": 0 8935 } 8936 }, 8937 { 8938 "aliasColors": {}, 8939 "bars": false, 8940 "cacheTimeout": null, 8941 "datasource": "$source", 8942 "description": null, 8943 "editable": true, 8944 "error": false, 8945 "fieldConfig": { 8946 "defaults": { 8947 "thresholds": { 8948 "mode": "absolute", 8949 "steps": [] 8950 } 8951 } 8952 }, 8953 "fill": 1, 8954 "grid": { 8955 "threshold1": null, 8956 "threshold1Color": "rgba(216, 200, 27, 0.27)", 8957 "threshold2": null, 8958 "threshold2Color": "rgba(234, 112, 112, 0.22)" 8959 }, 8960 "gridPos": null, 8961 "height": null, 8962 "hideTimeOverride": false, 8963 "id": 69, 8964 "interval": null, 8965 "isNew": true, 8966 "legend": { 8967 "alignAsTable": false, 8968 "avg": false, 8969 "current": false, 8970 "hideEmpty": false, 8971 "hideZero": false, 8972 "max": false, 8973 "min": false, 8974 "rightSide": false, 8975 "show": true, 8976 "sideWidth": null, 8977 "sort": null, 8978 "sortDesc": false, 8979 "total": false, 8980 "values": false 8981 }, 8982 "lines": true, 8983 "linewidth": 2, 8984 "links": [], 8985 "maxDataPoints": 100, 8986 "maxPerRow": null, 8987 "minSpan": null, 8988 "nullPointMode": null, 8989 "options": { 8990 "alertThreshold": true, 8991 "dataLinks": [] 8992 }, 8993 "percentage": false, 8994 "pointradius": 5, 8995 "points": false, 8996 "renderer": "flot", 8997 "repeat": null, 8998 "repeatDirection": null, 8999 "seriesOverrides": [], 9000 "span": 12, 9001 "stack": false, 9002 "steppedLine": false, 9003 "targets": [ 9004 { 9005 "datasource": "$source", 9006 "expr": "histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket[1m])) by (le, instance, To))", 9007 "format": "time_series", 9008 "hide": false, 9009 "instant": false, 9010 "interval": "5s", 9011 "intervalFactor": 1, 9012 "legendFormat": "", 9013 "metric": "", 9014 "query": "histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket[1m])) by (le, instance, To))", 9015 "refId": "", 9016 "step": 10, 9017 "target": "" 9018 } 9019 ], 9020 "thresholds": [], 9021 "timeFrom": null, 9022 "timeShift": null, 9023 "title": "etcd network latency (99th percentile)", 9024 "tooltip": { 9025 "msResolution": true, 9026 "shared": true, 9027 "sort": 2, 9028 "value_type": "cumulative" 9029 }, 9030 "transformations": [], 9031 "transparent": false, 9032 "type": "graph", 9033 "xaxis": { 9034 "mode": "time", 9035 "name": null, 9036 "show": true, 9037 "values": [] 9038 }, 9039 "yaxes": [ 9040 { 9041 "decimals": null, 9042 "format": "s", 9043 "label": null, 9044 "logBase": 1, 9045 "max": null, 9046 "min": null, 9047 "show": true 9048 }, 9049 { 9050 "decimals": null, 9051 "format": "short", 9052 "label": null, 9053 "logBase": 1, 9054 "max": null, 9055 "min": null, 9056 "show": true 9057 } 9058 ], 9059 "yaxis": { 9060 "align": false, 9061 "alignLevel": 0 9062 } 9063 }, 9064 { 9065 "aliasColors": {}, 9066 "bars": false, 9067 "cacheTimeout": null, 9068 "datasource": "$secondary_source", 9069 "description": null, 9070 "editable": true, 9071 "error": false, 9072 "fieldConfig": { 9073 "defaults": { 9074 "thresholds": { 9075 "mode": "absolute", 9076 "steps": [] 9077 } 9078 } 9079 }, 9080 "fill": 1, 9081 "grid": { 9082 "threshold1": null, 9083 "threshold1Color": "rgba(216, 200, 27, 0.27)", 9084 "threshold2": null, 9085 "threshold2Color": "rgba(234, 112, 112, 0.22)" 9086 }, 9087 "gridPos": null, 9088 "height": null, 9089 "hideTimeOverride": false, 9090 "id": 70, 9091 "interval": null, 9092 "isNew": true, 9093 "legend": { 9094 "alignAsTable": false, 9095 "avg": false, 9096 "current": false, 9097 "hideEmpty": false, 9098 "hideZero": false, 9099 "max": false, 9100 "min": false, 9101 "rightSide": false, 9102 "show": true, 9103 "sideWidth": null, 9104 "sort": null, 9105 "sortDesc": false, 9106 "total": false, 9107 "values": false 9108 }, 9109 "lines": true, 9110 "linewidth": 2, 9111 "links": [], 9112 "maxDataPoints": 100, 9113 "maxPerRow": null, 9114 "minSpan": null, 9115 "nullPointMode": null, 9116 "options": { 9117 "alertThreshold": true, 9118 "dataLinks": [] 9119 }, 9120 "percentage": false, 9121 "pointradius": 5, 9122 "points": false, 9123 "renderer": "flot", 9124 "repeat": null, 9125 "repeatDirection": null, 9126 "seriesOverrides": [], 9127 "span": 12, 9128 "stack": false, 9129 "steppedLine": false, 9130 "targets": [ 9131 { 9132 "datasource": "$source", 9133 "expr": "histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket[1m])) by (le, instance, To))", 9134 "format": "time_series", 9135 "hide": false, 9136 "instant": false, 9137 "interval": "5s", 9138 "intervalFactor": 1, 9139 "legendFormat": "", 9140 "metric": "", 9141 "query": "histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket[1m])) by (le, instance, To))", 9142 "refId": "", 9143 "step": 10, 9144 "target": "" 9145 } 9146 ], 9147 "thresholds": [], 9148 "timeFrom": null, 9149 "timeShift": "$timeshift", 9150 "title": "[SECONDARY] etcd network latency (99th percentile)", 9151 "tooltip": { 9152 "msResolution": true, 9153 "shared": true, 9154 "sort": 2, 9155 "value_type": "cumulative" 9156 }, 9157 "transformations": [], 9158 "transparent": false, 9159 "type": "graph", 9160 "xaxis": { 9161 "mode": "time", 9162 "name": null, 9163 "show": true, 9164 "values": [] 9165 }, 9166 "yaxes": [ 9167 { 9168 "decimals": null, 9169 "format": "s", 9170 "label": null, 9171 "logBase": 1, 9172 "max": null, 9173 "min": null, 9174 "show": true 9175 }, 9176 { 9177 "decimals": null, 9178 "format": "short", 9179 "label": null, 9180 "logBase": 1, 9181 "max": null, 9182 "min": null, 9183 "show": true 9184 } 9185 ], 9186 "yaxis": { 9187 "align": false, 9188 "alignLevel": 0 9189 } 9190 }, 9191 { 9192 "aliasColors": {}, 9193 "bars": false, 9194 "cacheTimeout": null, 9195 "datasource": "$source", 9196 "description": null, 9197 "editable": true, 9198 "error": false, 9199 "fieldConfig": { 9200 "defaults": { 9201 "thresholds": { 9202 "mode": "absolute", 9203 "steps": [] 9204 } 9205 } 9206 }, 9207 "fill": 1, 9208 "grid": { 9209 "threshold1": null, 9210 "threshold1Color": "rgba(216, 200, 27, 0.27)", 9211 "threshold2": null, 9212 "threshold2Color": "rgba(234, 112, 112, 0.22)" 9213 }, 9214 "gridPos": null, 9215 "height": null, 9216 "hideTimeOverride": false, 9217 "id": 71, 9218 "interval": null, 9219 "isNew": true, 9220 "legend": { 9221 "alignAsTable": false, 9222 "avg": false, 9223 "current": false, 9224 "hideEmpty": false, 9225 "hideZero": false, 9226 "max": false, 9227 "min": false, 9228 "rightSide": false, 9229 "show": true, 9230 "sideWidth": null, 9231 "sort": null, 9232 "sortDesc": false, 9233 "total": false, 9234 "values": false 9235 }, 9236 "lines": true, 9237 "linewidth": 2, 9238 "links": [], 9239 "maxDataPoints": 100, 9240 "maxPerRow": null, 9241 "minSpan": null, 9242 "nullPointMode": null, 9243 "options": { 9244 "alertThreshold": true, 9245 "dataLinks": [] 9246 }, 9247 "percentage": false, 9248 "pointradius": 5, 9249 "points": false, 9250 "renderer": "flot", 9251 "repeat": null, 9252 "repeatDirection": null, 9253 "seriesOverrides": [], 9254 "span": 12, 9255 "stack": false, 9256 "steppedLine": false, 9257 "targets": [ 9258 { 9259 "datasource": "$source", 9260 "expr": "delta(etcd_debugging_mvcc_db_compaction_keys_total[1m])", 9261 "format": "time_series", 9262 "hide": false, 9263 "instant": false, 9264 "interval": "5s", 9265 "intervalFactor": 1, 9266 "legendFormat": "", 9267 "metric": "", 9268 "query": "delta(etcd_debugging_mvcc_db_compaction_keys_total[1m])", 9269 "refId": "", 9270 "step": 10, 9271 "target": "" 9272 } 9273 ], 9274 "thresholds": [], 9275 "timeFrom": null, 9276 "timeShift": null, 9277 "title": "etcd compaction keys", 9278 "tooltip": { 9279 "msResolution": true, 9280 "shared": true, 9281 "sort": 2, 9282 "value_type": "cumulative" 9283 }, 9284 "transformations": [], 9285 "transparent": false, 9286 "type": "graph", 9287 "xaxis": { 9288 "mode": "time", 9289 "name": null, 9290 "show": true, 9291 "values": [] 9292 }, 9293 "yaxes": [ 9294 { 9295 "decimals": null, 9296 "format": "short", 9297 "label": null, 9298 "logBase": 1, 9299 "max": null, 9300 "min": null, 9301 "show": true 9302 }, 9303 { 9304 "decimals": null, 9305 "format": "short", 9306 "label": null, 9307 "logBase": 1, 9308 "max": null, 9309 "min": null, 9310 "show": true 9311 } 9312 ], 9313 "yaxis": { 9314 "align": false, 9315 "alignLevel": 0 9316 } 9317 }, 9318 { 9319 "aliasColors": {}, 9320 "bars": false, 9321 "cacheTimeout": null, 9322 "datasource": "$secondary_source", 9323 "description": null, 9324 "editable": true, 9325 "error": false, 9326 "fieldConfig": { 9327 "defaults": { 9328 "thresholds": { 9329 "mode": "absolute", 9330 "steps": [] 9331 } 9332 } 9333 }, 9334 "fill": 1, 9335 "grid": { 9336 "threshold1": null, 9337 "threshold1Color": "rgba(216, 200, 27, 0.27)", 9338 "threshold2": null, 9339 "threshold2Color": "rgba(234, 112, 112, 0.22)" 9340 }, 9341 "gridPos": null, 9342 "height": null, 9343 "hideTimeOverride": false, 9344 "id": 72, 9345 "interval": null, 9346 "isNew": true, 9347 "legend": { 9348 "alignAsTable": false, 9349 "avg": false, 9350 "current": false, 9351 "hideEmpty": false, 9352 "hideZero": false, 9353 "max": false, 9354 "min": false, 9355 "rightSide": false, 9356 "show": true, 9357 "sideWidth": null, 9358 "sort": null, 9359 "sortDesc": false, 9360 "total": false, 9361 "values": false 9362 }, 9363 "lines": true, 9364 "linewidth": 2, 9365 "links": [], 9366 "maxDataPoints": 100, 9367 "maxPerRow": null, 9368 "minSpan": null, 9369 "nullPointMode": null, 9370 "options": { 9371 "alertThreshold": true, 9372 "dataLinks": [] 9373 }, 9374 "percentage": false, 9375 "pointradius": 5, 9376 "points": false, 9377 "renderer": "flot", 9378 "repeat": null, 9379 "repeatDirection": null, 9380 "seriesOverrides": [], 9381 "span": 12, 9382 "stack": false, 9383 "steppedLine": false, 9384 "targets": [ 9385 { 9386 "datasource": "$source", 9387 "expr": "delta(etcd_debugging_mvcc_db_compaction_keys_total[1m])", 9388 "format": "time_series", 9389 "hide": false, 9390 "instant": false, 9391 "interval": "5s", 9392 "intervalFactor": 1, 9393 "legendFormat": "", 9394 "metric": "", 9395 "query": "delta(etcd_debugging_mvcc_db_compaction_keys_total[1m])", 9396 "refId": "", 9397 "step": 10, 9398 "target": "" 9399 } 9400 ], 9401 "thresholds": [], 9402 "timeFrom": null, 9403 "timeShift": "$timeshift", 9404 "title": "[SECONDARY] etcd compaction keys", 9405 "tooltip": { 9406 "msResolution": true, 9407 "shared": true, 9408 "sort": 2, 9409 "value_type": "cumulative" 9410 }, 9411 "transformations": [], 9412 "transparent": false, 9413 "type": "graph", 9414 "xaxis": { 9415 "mode": "time", 9416 "name": null, 9417 "show": true, 9418 "values": [] 9419 }, 9420 "yaxes": [ 9421 { 9422 "decimals": null, 9423 "format": "short", 9424 "label": null, 9425 "logBase": 1, 9426 "max": null, 9427 "min": null, 9428 "show": true 9429 }, 9430 { 9431 "decimals": null, 9432 "format": "short", 9433 "label": null, 9434 "logBase": 1, 9435 "max": null, 9436 "min": null, 9437 "show": true 9438 } 9439 ], 9440 "yaxis": { 9441 "align": false, 9442 "alignLevel": 0 9443 } 9444 }, 9445 { 9446 "aliasColors": {}, 9447 "bars": false, 9448 "cacheTimeout": null, 9449 "datasource": "$source", 9450 "description": null, 9451 "editable": true, 9452 "error": false, 9453 "fieldConfig": { 9454 "defaults": { 9455 "thresholds": { 9456 "mode": "absolute", 9457 "steps": [] 9458 } 9459 } 9460 }, 9461 "fill": 1, 9462 "grid": { 9463 "threshold1": null, 9464 "threshold1Color": "rgba(216, 200, 27, 0.27)", 9465 "threshold2": null, 9466 "threshold2Color": "rgba(234, 112, 112, 0.22)" 9467 }, 9468 "gridPos": null, 9469 "height": null, 9470 "hideTimeOverride": false, 9471 "id": 73, 9472 "interval": null, 9473 "isNew": true, 9474 "legend": { 9475 "alignAsTable": false, 9476 "avg": false, 9477 "current": false, 9478 "hideEmpty": false, 9479 "hideZero": false, 9480 "max": false, 9481 "min": false, 9482 "rightSide": false, 9483 "show": true, 9484 "sideWidth": null, 9485 "sort": null, 9486 "sortDesc": false, 9487 "total": false, 9488 "values": false 9489 }, 9490 "lines": true, 9491 "linewidth": 2, 9492 "links": [], 9493 "maxDataPoints": 100, 9494 "maxPerRow": null, 9495 "minSpan": null, 9496 "nullPointMode": null, 9497 "options": { 9498 "alertThreshold": true, 9499 "dataLinks": [] 9500 }, 9501 "percentage": false, 9502 "pointradius": 5, 9503 "points": false, 9504 "renderer": "flot", 9505 "repeat": null, 9506 "repeatDirection": null, 9507 "seriesOverrides": [], 9508 "span": 12, 9509 "stack": false, 9510 "steppedLine": false, 9511 "targets": [ 9512 { 9513 "datasource": "$source", 9514 "expr": "delta(etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds_sum[1m])", 9515 "format": "time_series", 9516 "hide": false, 9517 "instant": false, 9518 "interval": "5s", 9519 "intervalFactor": 1, 9520 "legendFormat": "", 9521 "metric": "", 9522 "query": "delta(etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds_sum[1m])", 9523 "refId": "", 9524 "step": 10, 9525 "target": "" 9526 } 9527 ], 9528 "thresholds": [], 9529 "timeFrom": null, 9530 "timeShift": null, 9531 "title": "etcd compaction pause sum duration", 9532 "tooltip": { 9533 "msResolution": true, 9534 "shared": true, 9535 "sort": 2, 9536 "value_type": "cumulative" 9537 }, 9538 "transformations": [], 9539 "transparent": false, 9540 "type": "graph", 9541 "xaxis": { 9542 "mode": "time", 9543 "name": null, 9544 "show": true, 9545 "values": [] 9546 }, 9547 "yaxes": [ 9548 { 9549 "decimals": null, 9550 "format": "ms", 9551 "label": null, 9552 "logBase": 1, 9553 "max": null, 9554 "min": null, 9555 "show": true 9556 }, 9557 { 9558 "decimals": null, 9559 "format": "short", 9560 "label": null, 9561 "logBase": 1, 9562 "max": null, 9563 "min": null, 9564 "show": true 9565 } 9566 ], 9567 "yaxis": { 9568 "align": false, 9569 "alignLevel": 0 9570 } 9571 }, 9572 { 9573 "aliasColors": {}, 9574 "bars": false, 9575 "cacheTimeout": null, 9576 "datasource": "$secondary_source", 9577 "description": null, 9578 "editable": true, 9579 "error": false, 9580 "fieldConfig": { 9581 "defaults": { 9582 "thresholds": { 9583 "mode": "absolute", 9584 "steps": [] 9585 } 9586 } 9587 }, 9588 "fill": 1, 9589 "grid": { 9590 "threshold1": null, 9591 "threshold1Color": "rgba(216, 200, 27, 0.27)", 9592 "threshold2": null, 9593 "threshold2Color": "rgba(234, 112, 112, 0.22)" 9594 }, 9595 "gridPos": null, 9596 "height": null, 9597 "hideTimeOverride": false, 9598 "id": 74, 9599 "interval": null, 9600 "isNew": true, 9601 "legend": { 9602 "alignAsTable": false, 9603 "avg": false, 9604 "current": false, 9605 "hideEmpty": false, 9606 "hideZero": false, 9607 "max": false, 9608 "min": false, 9609 "rightSide": false, 9610 "show": true, 9611 "sideWidth": null, 9612 "sort": null, 9613 "sortDesc": false, 9614 "total": false, 9615 "values": false 9616 }, 9617 "lines": true, 9618 "linewidth": 2, 9619 "links": [], 9620 "maxDataPoints": 100, 9621 "maxPerRow": null, 9622 "minSpan": null, 9623 "nullPointMode": null, 9624 "options": { 9625 "alertThreshold": true, 9626 "dataLinks": [] 9627 }, 9628 "percentage": false, 9629 "pointradius": 5, 9630 "points": false, 9631 "renderer": "flot", 9632 "repeat": null, 9633 "repeatDirection": null, 9634 "seriesOverrides": [], 9635 "span": 12, 9636 "stack": false, 9637 "steppedLine": false, 9638 "targets": [ 9639 { 9640 "datasource": "$source", 9641 "expr": "delta(etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds_sum[1m])", 9642 "format": "time_series", 9643 "hide": false, 9644 "instant": false, 9645 "interval": "5s", 9646 "intervalFactor": 1, 9647 "legendFormat": "", 9648 "metric": "", 9649 "query": "delta(etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds_sum[1m])", 9650 "refId": "", 9651 "step": 10, 9652 "target": "" 9653 } 9654 ], 9655 "thresholds": [], 9656 "timeFrom": null, 9657 "timeShift": "$timeshift", 9658 "title": "[SECONDARY] etcd compaction pause sum duration", 9659 "tooltip": { 9660 "msResolution": true, 9661 "shared": true, 9662 "sort": 2, 9663 "value_type": "cumulative" 9664 }, 9665 "transformations": [], 9666 "transparent": false, 9667 "type": "graph", 9668 "xaxis": { 9669 "mode": "time", 9670 "name": null, 9671 "show": true, 9672 "values": [] 9673 }, 9674 "yaxes": [ 9675 { 9676 "decimals": null, 9677 "format": "ms", 9678 "label": null, 9679 "logBase": 1, 9680 "max": null, 9681 "min": null, 9682 "show": true 9683 }, 9684 { 9685 "decimals": null, 9686 "format": "short", 9687 "label": null, 9688 "logBase": 1, 9689 "max": null, 9690 "min": null, 9691 "show": true 9692 } 9693 ], 9694 "yaxis": { 9695 "align": false, 9696 "alignLevel": 0 9697 } 9698 }, 9699 { 9700 "aliasColors": {}, 9701 "bars": false, 9702 "cacheTimeout": null, 9703 "datasource": "$source", 9704 "description": null, 9705 "editable": true, 9706 "error": false, 9707 "fieldConfig": { 9708 "defaults": { 9709 "thresholds": { 9710 "mode": "absolute", 9711 "steps": [] 9712 } 9713 } 9714 }, 9715 "fill": 1, 9716 "grid": { 9717 "threshold1": null, 9718 "threshold1Color": "rgba(216, 200, 27, 0.27)", 9719 "threshold2": null, 9720 "threshold2Color": "rgba(234, 112, 112, 0.22)" 9721 }, 9722 "gridPos": null, 9723 "height": null, 9724 "hideTimeOverride": false, 9725 "id": 75, 9726 "interval": null, 9727 "isNew": true, 9728 "legend": { 9729 "alignAsTable": false, 9730 "avg": false, 9731 "current": false, 9732 "hideEmpty": false, 9733 "hideZero": false, 9734 "max": false, 9735 "min": false, 9736 "rightSide": false, 9737 "show": true, 9738 "sideWidth": null, 9739 "sort": null, 9740 "sortDesc": false, 9741 "total": false, 9742 "values": false 9743 }, 9744 "lines": true, 9745 "linewidth": 2, 9746 "links": [], 9747 "maxDataPoints": 100, 9748 "maxPerRow": null, 9749 "minSpan": null, 9750 "nullPointMode": null, 9751 "options": { 9752 "alertThreshold": true, 9753 "dataLinks": [] 9754 }, 9755 "percentage": false, 9756 "pointradius": 5, 9757 "points": false, 9758 "renderer": "flot", 9759 "repeat": null, 9760 "repeatDirection": null, 9761 "seriesOverrides": [], 9762 "span": 12, 9763 "stack": false, 9764 "steppedLine": false, 9765 "targets": [ 9766 { 9767 "datasource": "$source", 9768 "expr": "delta(etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds_count[1m])", 9769 "format": "time_series", 9770 "hide": false, 9771 "instant": false, 9772 "interval": "5s", 9773 "intervalFactor": 1, 9774 "legendFormat": "", 9775 "metric": "", 9776 "query": "delta(etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds_count[1m])", 9777 "refId": "", 9778 "step": 10, 9779 "target": "" 9780 } 9781 ], 9782 "thresholds": [], 9783 "timeFrom": null, 9784 "timeShift": null, 9785 "title": "etcd compaction pause num chunks", 9786 "tooltip": { 9787 "msResolution": true, 9788 "shared": true, 9789 "sort": 2, 9790 "value_type": "cumulative" 9791 }, 9792 "transformations": [], 9793 "transparent": false, 9794 "type": "graph", 9795 "xaxis": { 9796 "mode": "time", 9797 "name": null, 9798 "show": true, 9799 "values": [] 9800 }, 9801 "yaxes": [ 9802 { 9803 "decimals": null, 9804 "format": "short", 9805 "label": null, 9806 "logBase": 1, 9807 "max": null, 9808 "min": null, 9809 "show": true 9810 }, 9811 { 9812 "decimals": null, 9813 "format": "short", 9814 "label": null, 9815 "logBase": 1, 9816 "max": null, 9817 "min": null, 9818 "show": true 9819 } 9820 ], 9821 "yaxis": { 9822 "align": false, 9823 "alignLevel": 0 9824 } 9825 }, 9826 { 9827 "aliasColors": {}, 9828 "bars": false, 9829 "cacheTimeout": null, 9830 "datasource": "$secondary_source", 9831 "description": null, 9832 "editable": true, 9833 "error": false, 9834 "fieldConfig": { 9835 "defaults": { 9836 "thresholds": { 9837 "mode": "absolute", 9838 "steps": [] 9839 } 9840 } 9841 }, 9842 "fill": 1, 9843 "grid": { 9844 "threshold1": null, 9845 "threshold1Color": "rgba(216, 200, 27, 0.27)", 9846 "threshold2": null, 9847 "threshold2Color": "rgba(234, 112, 112, 0.22)" 9848 }, 9849 "gridPos": null, 9850 "height": null, 9851 "hideTimeOverride": false, 9852 "id": 76, 9853 "interval": null, 9854 "isNew": true, 9855 "legend": { 9856 "alignAsTable": false, 9857 "avg": false, 9858 "current": false, 9859 "hideEmpty": false, 9860 "hideZero": false, 9861 "max": false, 9862 "min": false, 9863 "rightSide": false, 9864 "show": true, 9865 "sideWidth": null, 9866 "sort": null, 9867 "sortDesc": false, 9868 "total": false, 9869 "values": false 9870 }, 9871 "lines": true, 9872 "linewidth": 2, 9873 "links": [], 9874 "maxDataPoints": 100, 9875 "maxPerRow": null, 9876 "minSpan": null, 9877 "nullPointMode": null, 9878 "options": { 9879 "alertThreshold": true, 9880 "dataLinks": [] 9881 }, 9882 "percentage": false, 9883 "pointradius": 5, 9884 "points": false, 9885 "renderer": "flot", 9886 "repeat": null, 9887 "repeatDirection": null, 9888 "seriesOverrides": [], 9889 "span": 12, 9890 "stack": false, 9891 "steppedLine": false, 9892 "targets": [ 9893 { 9894 "datasource": "$source", 9895 "expr": "delta(etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds_count[1m])", 9896 "format": "time_series", 9897 "hide": false, 9898 "instant": false, 9899 "interval": "5s", 9900 "intervalFactor": 1, 9901 "legendFormat": "", 9902 "metric": "", 9903 "query": "delta(etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds_count[1m])", 9904 "refId": "", 9905 "step": 10, 9906 "target": "" 9907 } 9908 ], 9909 "thresholds": [], 9910 "timeFrom": null, 9911 "timeShift": "$timeshift", 9912 "title": "[SECONDARY] etcd compaction pause num chunks", 9913 "tooltip": { 9914 "msResolution": true, 9915 "shared": true, 9916 "sort": 2, 9917 "value_type": "cumulative" 9918 }, 9919 "transformations": [], 9920 "transparent": false, 9921 "type": "graph", 9922 "xaxis": { 9923 "mode": "time", 9924 "name": null, 9925 "show": true, 9926 "values": [] 9927 }, 9928 "yaxes": [ 9929 { 9930 "decimals": null, 9931 "format": "short", 9932 "label": null, 9933 "logBase": 1, 9934 "max": null, 9935 "min": null, 9936 "show": true 9937 }, 9938 { 9939 "decimals": null, 9940 "format": "short", 9941 "label": null, 9942 "logBase": 1, 9943 "max": null, 9944 "min": null, 9945 "show": true 9946 } 9947 ], 9948 "yaxis": { 9949 "align": false, 9950 "alignLevel": 0 9951 } 9952 }, 9953 { 9954 "aliasColors": {}, 9955 "bars": false, 9956 "cacheTimeout": null, 9957 "datasource": "$source", 9958 "description": null, 9959 "editable": true, 9960 "error": false, 9961 "fieldConfig": { 9962 "defaults": { 9963 "thresholds": { 9964 "mode": "absolute", 9965 "steps": [] 9966 } 9967 } 9968 }, 9969 "fill": 1, 9970 "grid": { 9971 "threshold1": null, 9972 "threshold1Color": "rgba(216, 200, 27, 0.27)", 9973 "threshold2": null, 9974 "threshold2Color": "rgba(234, 112, 112, 0.22)" 9975 }, 9976 "gridPos": null, 9977 "height": null, 9978 "hideTimeOverride": false, 9979 "id": 77, 9980 "interval": null, 9981 "isNew": true, 9982 "legend": { 9983 "alignAsTable": false, 9984 "avg": false, 9985 "current": false, 9986 "hideEmpty": false, 9987 "hideZero": false, 9988 "max": false, 9989 "min": false, 9990 "rightSide": false, 9991 "show": true, 9992 "sideWidth": null, 9993 "sort": null, 9994 "sortDesc": false, 9995 "total": false, 9996 "values": false 9997 }, 9998 "lines": true, 9999 "linewidth": 2, 10000 "links": [], 10001 "maxDataPoints": 100, 10002 "maxPerRow": null, 10003 "minSpan": null, 10004 "nullPointMode": null, 10005 "options": { 10006 "alertThreshold": true, 10007 "dataLinks": [] 10008 }, 10009 "percentage": false, 10010 "pointradius": 5, 10011 "points": false, 10012 "renderer": "flot", 10013 "repeat": null, 10014 "repeatDirection": null, 10015 "seriesOverrides": [], 10016 "span": 12, 10017 "stack": false, 10018 "steppedLine": false, 10019 "targets": [ 10020 { 10021 "datasource": "$source", 10022 "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[1m])) by (le, instance))", 10023 "format": "time_series", 10024 "hide": false, 10025 "instant": false, 10026 "interval": "5s", 10027 "intervalFactor": 1, 10028 "legendFormat": "", 10029 "metric": "", 10030 "query": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[1m])) by (le, instance))", 10031 "refId": "", 10032 "step": 10, 10033 "target": "" 10034 } 10035 ], 10036 "thresholds": [], 10037 "timeFrom": null, 10038 "timeShift": null, 10039 "title": "etcd_disk_backend_commit_duration_seconds", 10040 "tooltip": { 10041 "msResolution": true, 10042 "shared": true, 10043 "sort": 2, 10044 "value_type": "cumulative" 10045 }, 10046 "transformations": [], 10047 "transparent": false, 10048 "type": "graph", 10049 "xaxis": { 10050 "mode": "time", 10051 "name": null, 10052 "show": true, 10053 "values": [] 10054 }, 10055 "yaxes": [ 10056 { 10057 "decimals": null, 10058 "format": "s", 10059 "label": null, 10060 "logBase": 1, 10061 "max": null, 10062 "min": null, 10063 "show": true 10064 }, 10065 { 10066 "decimals": null, 10067 "format": "short", 10068 "label": null, 10069 "logBase": 1, 10070 "max": null, 10071 "min": null, 10072 "show": true 10073 } 10074 ], 10075 "yaxis": { 10076 "align": false, 10077 "alignLevel": 0 10078 } 10079 }, 10080 { 10081 "aliasColors": {}, 10082 "bars": false, 10083 "cacheTimeout": null, 10084 "datasource": "$secondary_source", 10085 "description": null, 10086 "editable": true, 10087 "error": false, 10088 "fieldConfig": { 10089 "defaults": { 10090 "thresholds": { 10091 "mode": "absolute", 10092 "steps": [] 10093 } 10094 } 10095 }, 10096 "fill": 1, 10097 "grid": { 10098 "threshold1": null, 10099 "threshold1Color": "rgba(216, 200, 27, 0.27)", 10100 "threshold2": null, 10101 "threshold2Color": "rgba(234, 112, 112, 0.22)" 10102 }, 10103 "gridPos": null, 10104 "height": null, 10105 "hideTimeOverride": false, 10106 "id": 78, 10107 "interval": null, 10108 "isNew": true, 10109 "legend": { 10110 "alignAsTable": false, 10111 "avg": false, 10112 "current": false, 10113 "hideEmpty": false, 10114 "hideZero": false, 10115 "max": false, 10116 "min": false, 10117 "rightSide": false, 10118 "show": true, 10119 "sideWidth": null, 10120 "sort": null, 10121 "sortDesc": false, 10122 "total": false, 10123 "values": false 10124 }, 10125 "lines": true, 10126 "linewidth": 2, 10127 "links": [], 10128 "maxDataPoints": 100, 10129 "maxPerRow": null, 10130 "minSpan": null, 10131 "nullPointMode": null, 10132 "options": { 10133 "alertThreshold": true, 10134 "dataLinks": [] 10135 }, 10136 "percentage": false, 10137 "pointradius": 5, 10138 "points": false, 10139 "renderer": "flot", 10140 "repeat": null, 10141 "repeatDirection": null, 10142 "seriesOverrides": [], 10143 "span": 12, 10144 "stack": false, 10145 "steppedLine": false, 10146 "targets": [ 10147 { 10148 "datasource": "$source", 10149 "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[1m])) by (le, instance))", 10150 "format": "time_series", 10151 "hide": false, 10152 "instant": false, 10153 "interval": "5s", 10154 "intervalFactor": 1, 10155 "legendFormat": "", 10156 "metric": "", 10157 "query": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[1m])) by (le, instance))", 10158 "refId": "", 10159 "step": 10, 10160 "target": "" 10161 } 10162 ], 10163 "thresholds": [], 10164 "timeFrom": null, 10165 "timeShift": "$timeshift", 10166 "title": "[SECONDARY] etcd_disk_backend_commit_duration_seconds", 10167 "tooltip": { 10168 "msResolution": true, 10169 "shared": true, 10170 "sort": 2, 10171 "value_type": "cumulative" 10172 }, 10173 "transformations": [], 10174 "transparent": false, 10175 "type": "graph", 10176 "xaxis": { 10177 "mode": "time", 10178 "name": null, 10179 "show": true, 10180 "values": [] 10181 }, 10182 "yaxes": [ 10183 { 10184 "decimals": null, 10185 "format": "s", 10186 "label": null, 10187 "logBase": 1, 10188 "max": null, 10189 "min": null, 10190 "show": true 10191 }, 10192 { 10193 "decimals": null, 10194 "format": "short", 10195 "label": null, 10196 "logBase": 1, 10197 "max": null, 10198 "min": null, 10199 "show": true 10200 } 10201 ], 10202 "yaxis": { 10203 "align": false, 10204 "alignLevel": 0 10205 } 10206 }, 10207 { 10208 "aliasColors": {}, 10209 "bars": false, 10210 "cacheTimeout": null, 10211 "datasource": "$source", 10212 "description": null, 10213 "editable": true, 10214 "error": false, 10215 "fieldConfig": { 10216 "defaults": { 10217 "thresholds": { 10218 "mode": "absolute", 10219 "steps": [] 10220 } 10221 } 10222 }, 10223 "fill": 1, 10224 "grid": { 10225 "threshold1": null, 10226 "threshold1Color": "rgba(216, 200, 27, 0.27)", 10227 "threshold2": null, 10228 "threshold2Color": "rgba(234, 112, 112, 0.22)" 10229 }, 10230 "gridPos": null, 10231 "height": null, 10232 "hideTimeOverride": false, 10233 "id": 79, 10234 "interval": null, 10235 "isNew": true, 10236 "legend": { 10237 "alignAsTable": false, 10238 "avg": false, 10239 "current": false, 10240 "hideEmpty": false, 10241 "hideZero": false, 10242 "max": false, 10243 "min": false, 10244 "rightSide": false, 10245 "show": true, 10246 "sideWidth": null, 10247 "sort": null, 10248 "sortDesc": false, 10249 "total": false, 10250 "values": false 10251 }, 10252 "lines": true, 10253 "linewidth": 2, 10254 "links": [], 10255 "maxDataPoints": 100, 10256 "maxPerRow": null, 10257 "minSpan": null, 10258 "nullPointMode": null, 10259 "options": { 10260 "alertThreshold": true, 10261 "dataLinks": [] 10262 }, 10263 "percentage": false, 10264 "pointradius": 5, 10265 "points": false, 10266 "renderer": "flot", 10267 "repeat": null, 10268 "repeatDirection": null, 10269 "seriesOverrides": [], 10270 "span": 12, 10271 "stack": false, 10272 "steppedLine": false, 10273 "targets": [ 10274 { 10275 "datasource": "$source", 10276 "expr": "histogram_quantile(1.0, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[1m])) by (le, endpoint))", 10277 "format": "time_series", 10278 "hide": false, 10279 "instant": false, 10280 "interval": "5s", 10281 "intervalFactor": 1, 10282 "legendFormat": "", 10283 "metric": "", 10284 "query": "histogram_quantile(1.0, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[1m])) by (le, endpoint))", 10285 "refId": "", 10286 "step": 10, 10287 "target": "" 10288 } 10289 ], 10290 "thresholds": [], 10291 "timeFrom": null, 10292 "timeShift": null, 10293 "title": "etcd wal fsync duration", 10294 "tooltip": { 10295 "msResolution": true, 10296 "shared": true, 10297 "sort": 2, 10298 "value_type": "cumulative" 10299 }, 10300 "transformations": [], 10301 "transparent": false, 10302 "type": "graph", 10303 "xaxis": { 10304 "mode": "time", 10305 "name": null, 10306 "show": true, 10307 "values": [] 10308 }, 10309 "yaxes": [ 10310 { 10311 "decimals": null, 10312 "format": "s", 10313 "label": null, 10314 "logBase": 1, 10315 "max": null, 10316 "min": null, 10317 "show": true 10318 }, 10319 { 10320 "decimals": null, 10321 "format": "short", 10322 "label": null, 10323 "logBase": 1, 10324 "max": null, 10325 "min": null, 10326 "show": true 10327 } 10328 ], 10329 "yaxis": { 10330 "align": false, 10331 "alignLevel": 0 10332 } 10333 }, 10334 { 10335 "aliasColors": {}, 10336 "bars": false, 10337 "cacheTimeout": null, 10338 "datasource": "$secondary_source", 10339 "description": null, 10340 "editable": true, 10341 "error": false, 10342 "fieldConfig": { 10343 "defaults": { 10344 "thresholds": { 10345 "mode": "absolute", 10346 "steps": [] 10347 } 10348 } 10349 }, 10350 "fill": 1, 10351 "grid": { 10352 "threshold1": null, 10353 "threshold1Color": "rgba(216, 200, 27, 0.27)", 10354 "threshold2": null, 10355 "threshold2Color": "rgba(234, 112, 112, 0.22)" 10356 }, 10357 "gridPos": null, 10358 "height": null, 10359 "hideTimeOverride": false, 10360 "id": 80, 10361 "interval": null, 10362 "isNew": true, 10363 "legend": { 10364 "alignAsTable": false, 10365 "avg": false, 10366 "current": false, 10367 "hideEmpty": false, 10368 "hideZero": false, 10369 "max": false, 10370 "min": false, 10371 "rightSide": false, 10372 "show": true, 10373 "sideWidth": null, 10374 "sort": null, 10375 "sortDesc": false, 10376 "total": false, 10377 "values": false 10378 }, 10379 "lines": true, 10380 "linewidth": 2, 10381 "links": [], 10382 "maxDataPoints": 100, 10383 "maxPerRow": null, 10384 "minSpan": null, 10385 "nullPointMode": null, 10386 "options": { 10387 "alertThreshold": true, 10388 "dataLinks": [] 10389 }, 10390 "percentage": false, 10391 "pointradius": 5, 10392 "points": false, 10393 "renderer": "flot", 10394 "repeat": null, 10395 "repeatDirection": null, 10396 "seriesOverrides": [], 10397 "span": 12, 10398 "stack": false, 10399 "steppedLine": false, 10400 "targets": [ 10401 { 10402 "datasource": "$source", 10403 "expr": "histogram_quantile(1.0, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[1m])) by (le, endpoint))", 10404 "format": "time_series", 10405 "hide": false, 10406 "instant": false, 10407 "interval": "5s", 10408 "intervalFactor": 1, 10409 "legendFormat": "", 10410 "metric": "", 10411 "query": "histogram_quantile(1.0, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[1m])) by (le, endpoint))", 10412 "refId": "", 10413 "step": 10, 10414 "target": "" 10415 } 10416 ], 10417 "thresholds": [], 10418 "timeFrom": null, 10419 "timeShift": "$timeshift", 10420 "title": "[SECONDARY] etcd wal fsync duration", 10421 "tooltip": { 10422 "msResolution": true, 10423 "shared": true, 10424 "sort": 2, 10425 "value_type": "cumulative" 10426 }, 10427 "transformations": [], 10428 "transparent": false, 10429 "type": "graph", 10430 "xaxis": { 10431 "mode": "time", 10432 "name": null, 10433 "show": true, 10434 "values": [] 10435 }, 10436 "yaxes": [ 10437 { 10438 "decimals": null, 10439 "format": "s", 10440 "label": null, 10441 "logBase": 1, 10442 "max": null, 10443 "min": null, 10444 "show": true 10445 }, 10446 { 10447 "decimals": null, 10448 "format": "short", 10449 "label": null, 10450 "logBase": 1, 10451 "max": null, 10452 "min": null, 10453 "show": true 10454 } 10455 ], 10456 "yaxis": { 10457 "align": false, 10458 "alignLevel": 0 10459 } 10460 }, 10461 { 10462 "aliasColors": {}, 10463 "bars": false, 10464 "cacheTimeout": null, 10465 "datasource": "$source", 10466 "description": null, 10467 "editable": true, 10468 "error": false, 10469 "fieldConfig": { 10470 "defaults": { 10471 "thresholds": { 10472 "mode": "absolute", 10473 "steps": [] 10474 } 10475 } 10476 }, 10477 "fill": 1, 10478 "grid": { 10479 "threshold1": null, 10480 "threshold1Color": "rgba(216, 200, 27, 0.27)", 10481 "threshold2": null, 10482 "threshold2Color": "rgba(234, 112, 112, 0.22)" 10483 }, 10484 "gridPos": null, 10485 "height": null, 10486 "hideTimeOverride": false, 10487 "id": 81, 10488 "interval": null, 10489 "isNew": true, 10490 "legend": { 10491 "alignAsTable": false, 10492 "avg": false, 10493 "current": false, 10494 "hideEmpty": false, 10495 "hideZero": false, 10496 "max": false, 10497 "min": false, 10498 "rightSide": false, 10499 "show": true, 10500 "sideWidth": null, 10501 "sort": null, 10502 "sortDesc": false, 10503 "total": false, 10504 "values": false 10505 }, 10506 "lines": false, 10507 "linewidth": 2, 10508 "links": [], 10509 "maxDataPoints": 100, 10510 "maxPerRow": null, 10511 "minSpan": null, 10512 "nullPointMode": null, 10513 "options": { 10514 "alertThreshold": true, 10515 "dataLinks": [] 10516 }, 10517 "percentage": false, 10518 "pointradius": 5, 10519 "points": true, 10520 "renderer": "flot", 10521 "repeat": null, 10522 "repeatDirection": null, 10523 "seriesOverrides": [], 10524 "span": 12, 10525 "stack": false, 10526 "steppedLine": false, 10527 "targets": [ 10528 { 10529 "datasource": "$source", 10530 "expr": "histogram_quantile(1.0, sum(rate(etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds_bucket[1m])) by (le, instance))", 10531 "format": "time_series", 10532 "hide": false, 10533 "instant": false, 10534 "interval": "", 10535 "intervalFactor": 2, 10536 "legendFormat": "", 10537 "metric": "", 10538 "query": "histogram_quantile(1.0, sum(rate(etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds_bucket[1m])) by (le, instance))", 10539 "refId": "", 10540 "step": 10, 10541 "target": "" 10542 } 10543 ], 10544 "thresholds": [], 10545 "timeFrom": null, 10546 "timeShift": null, 10547 "title": "etcd compaction max pause", 10548 "tooltip": { 10549 "msResolution": true, 10550 "shared": true, 10551 "sort": 2, 10552 "value_type": "cumulative" 10553 }, 10554 "transformations": [], 10555 "transparent": false, 10556 "type": "graph", 10557 "xaxis": { 10558 "mode": "time", 10559 "name": null, 10560 "show": true, 10561 "values": [] 10562 }, 10563 "yaxes": [ 10564 { 10565 "decimals": null, 10566 "format": "ms", 10567 "label": null, 10568 "logBase": 1, 10569 "max": null, 10570 "min": null, 10571 "show": true 10572 }, 10573 { 10574 "decimals": null, 10575 "format": "short", 10576 "label": null, 10577 "logBase": 1, 10578 "max": null, 10579 "min": null, 10580 "show": true 10581 } 10582 ], 10583 "yaxis": { 10584 "align": false, 10585 "alignLevel": 0 10586 } 10587 }, 10588 { 10589 "aliasColors": {}, 10590 "bars": false, 10591 "cacheTimeout": null, 10592 "datasource": "$secondary_source", 10593 "description": null, 10594 "editable": true, 10595 "error": false, 10596 "fieldConfig": { 10597 "defaults": { 10598 "thresholds": { 10599 "mode": "absolute", 10600 "steps": [] 10601 } 10602 } 10603 }, 10604 "fill": 1, 10605 "grid": { 10606 "threshold1": null, 10607 "threshold1Color": "rgba(216, 200, 27, 0.27)", 10608 "threshold2": null, 10609 "threshold2Color": "rgba(234, 112, 112, 0.22)" 10610 }, 10611 "gridPos": null, 10612 "height": null, 10613 "hideTimeOverride": false, 10614 "id": 82, 10615 "interval": null, 10616 "isNew": true, 10617 "legend": { 10618 "alignAsTable": false, 10619 "avg": false, 10620 "current": false, 10621 "hideEmpty": false, 10622 "hideZero": false, 10623 "max": false, 10624 "min": false, 10625 "rightSide": false, 10626 "show": true, 10627 "sideWidth": null, 10628 "sort": null, 10629 "sortDesc": false, 10630 "total": false, 10631 "values": false 10632 }, 10633 "lines": false, 10634 "linewidth": 2, 10635 "links": [], 10636 "maxDataPoints": 100, 10637 "maxPerRow": null, 10638 "minSpan": null, 10639 "nullPointMode": null, 10640 "options": { 10641 "alertThreshold": true, 10642 "dataLinks": [] 10643 }, 10644 "percentage": false, 10645 "pointradius": 5, 10646 "points": true, 10647 "renderer": "flot", 10648 "repeat": null, 10649 "repeatDirection": null, 10650 "seriesOverrides": [], 10651 "span": 12, 10652 "stack": false, 10653 "steppedLine": false, 10654 "targets": [ 10655 { 10656 "datasource": "$source", 10657 "expr": "histogram_quantile(1.0, sum(rate(etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds_bucket[1m])) by (le, instance))", 10658 "format": "time_series", 10659 "hide": false, 10660 "instant": false, 10661 "interval": "", 10662 "intervalFactor": 2, 10663 "legendFormat": "", 10664 "metric": "", 10665 "query": "histogram_quantile(1.0, sum(rate(etcd_debugging_mvcc_db_compaction_pause_duration_milliseconds_bucket[1m])) by (le, instance))", 10666 "refId": "", 10667 "step": 10, 10668 "target": "" 10669 } 10670 ], 10671 "thresholds": [], 10672 "timeFrom": null, 10673 "timeShift": "$timeshift", 10674 "title": "[SECONDARY] etcd compaction max pause", 10675 "tooltip": { 10676 "msResolution": true, 10677 "shared": true, 10678 "sort": 2, 10679 "value_type": "cumulative" 10680 }, 10681 "transformations": [], 10682 "transparent": false, 10683 "type": "graph", 10684 "xaxis": { 10685 "mode": "time", 10686 "name": null, 10687 "show": true, 10688 "values": [] 10689 }, 10690 "yaxes": [ 10691 { 10692 "decimals": null, 10693 "format": "ms", 10694 "label": null, 10695 "logBase": 1, 10696 "max": null, 10697 "min": null, 10698 "show": true 10699 }, 10700 { 10701 "decimals": null, 10702 "format": "short", 10703 "label": null, 10704 "logBase": 1, 10705 "max": null, 10706 "min": null, 10707 "show": true 10708 } 10709 ], 10710 "yaxis": { 10711 "align": false, 10712 "alignLevel": 0 10713 } 10714 }, 10715 { 10716 "aliasColors": {}, 10717 "bars": false, 10718 "cacheTimeout": null, 10719 "datasource": "$source", 10720 "description": null, 10721 "editable": true, 10722 "error": false, 10723 "fieldConfig": { 10724 "defaults": { 10725 "thresholds": { 10726 "mode": "absolute", 10727 "steps": [] 10728 } 10729 } 10730 }, 10731 "fill": 1, 10732 "grid": { 10733 "threshold1": null, 10734 "threshold1Color": "rgba(216, 200, 27, 0.27)", 10735 "threshold2": null, 10736 "threshold2Color": "rgba(234, 112, 112, 0.22)" 10737 }, 10738 "gridPos": null, 10739 "height": null, 10740 "hideTimeOverride": false, 10741 "id": 83, 10742 "interval": null, 10743 "isNew": true, 10744 "legend": { 10745 "alignAsTable": false, 10746 "avg": false, 10747 "current": false, 10748 "hideEmpty": false, 10749 "hideZero": false, 10750 "max": false, 10751 "min": false, 10752 "rightSide": false, 10753 "show": true, 10754 "sideWidth": null, 10755 "sort": null, 10756 "sortDesc": false, 10757 "total": false, 10758 "values": false 10759 }, 10760 "lines": true, 10761 "linewidth": 2, 10762 "links": [], 10763 "maxDataPoints": 100, 10764 "maxPerRow": null, 10765 "minSpan": null, 10766 "nullPointMode": null, 10767 "options": { 10768 "alertThreshold": true, 10769 "dataLinks": [] 10770 }, 10771 "percentage": false, 10772 "pointradius": 5, 10773 "points": false, 10774 "renderer": "flot", 10775 "repeat": null, 10776 "repeatDirection": null, 10777 "seriesOverrides": [], 10778 "span": 12, 10779 "stack": false, 10780 "steppedLine": false, 10781 "targets": [ 10782 { 10783 "datasource": "$source", 10784 "expr": "sum(etcd_object_counts) by (resource, instance)", 10785 "format": "time_series", 10786 "hide": false, 10787 "instant": false, 10788 "interval": "5s", 10789 "intervalFactor": 1, 10790 "legendFormat": "{{instance}}: {{resource}}", 10791 "metric": "", 10792 "query": "sum(etcd_object_counts) by (resource, instance)", 10793 "refId": "", 10794 "step": 10, 10795 "target": "" 10796 } 10797 ], 10798 "thresholds": [], 10799 "timeFrom": null, 10800 "timeShift": null, 10801 "title": "etcd objects", 10802 "tooltip": { 10803 "msResolution": true, 10804 "shared": true, 10805 "sort": 2, 10806 "value_type": "cumulative" 10807 }, 10808 "transformations": [], 10809 "transparent": false, 10810 "type": "graph", 10811 "xaxis": { 10812 "mode": "time", 10813 "name": null, 10814 "show": true, 10815 "values": [] 10816 }, 10817 "yaxes": [ 10818 { 10819 "decimals": null, 10820 "format": "short", 10821 "label": null, 10822 "logBase": 1, 10823 "max": null, 10824 "min": null, 10825 "show": true 10826 }, 10827 { 10828 "decimals": null, 10829 "format": "short", 10830 "label": null, 10831 "logBase": 1, 10832 "max": null, 10833 "min": null, 10834 "show": true 10835 } 10836 ], 10837 "yaxis": { 10838 "align": false, 10839 "alignLevel": 0 10840 } 10841 }, 10842 { 10843 "aliasColors": {}, 10844 "bars": false, 10845 "cacheTimeout": null, 10846 "datasource": "$secondary_source", 10847 "description": null, 10848 "editable": true, 10849 "error": false, 10850 "fieldConfig": { 10851 "defaults": { 10852 "thresholds": { 10853 "mode": "absolute", 10854 "steps": [] 10855 } 10856 } 10857 }, 10858 "fill": 1, 10859 "grid": { 10860 "threshold1": null, 10861 "threshold1Color": "rgba(216, 200, 27, 0.27)", 10862 "threshold2": null, 10863 "threshold2Color": "rgba(234, 112, 112, 0.22)" 10864 }, 10865 "gridPos": null, 10866 "height": null, 10867 "hideTimeOverride": false, 10868 "id": 84, 10869 "interval": null, 10870 "isNew": true, 10871 "legend": { 10872 "alignAsTable": false, 10873 "avg": false, 10874 "current": false, 10875 "hideEmpty": false, 10876 "hideZero": false, 10877 "max": false, 10878 "min": false, 10879 "rightSide": false, 10880 "show": true, 10881 "sideWidth": null, 10882 "sort": null, 10883 "sortDesc": false, 10884 "total": false, 10885 "values": false 10886 }, 10887 "lines": true, 10888 "linewidth": 2, 10889 "links": [], 10890 "maxDataPoints": 100, 10891 "maxPerRow": null, 10892 "minSpan": null, 10893 "nullPointMode": null, 10894 "options": { 10895 "alertThreshold": true, 10896 "dataLinks": [] 10897 }, 10898 "percentage": false, 10899 "pointradius": 5, 10900 "points": false, 10901 "renderer": "flot", 10902 "repeat": null, 10903 "repeatDirection": null, 10904 "seriesOverrides": [], 10905 "span": 12, 10906 "stack": false, 10907 "steppedLine": false, 10908 "targets": [ 10909 { 10910 "datasource": "$source", 10911 "expr": "sum(etcd_object_counts) by (resource, instance)", 10912 "format": "time_series", 10913 "hide": false, 10914 "instant": false, 10915 "interval": "5s", 10916 "intervalFactor": 1, 10917 "legendFormat": "{{instance}}: {{resource}}", 10918 "metric": "", 10919 "query": "sum(etcd_object_counts) by (resource, instance)", 10920 "refId": "", 10921 "step": 10, 10922 "target": "" 10923 } 10924 ], 10925 "thresholds": [], 10926 "timeFrom": null, 10927 "timeShift": "$timeshift", 10928 "title": "[SECONDARY] etcd objects", 10929 "tooltip": { 10930 "msResolution": true, 10931 "shared": true, 10932 "sort": 2, 10933 "value_type": "cumulative" 10934 }, 10935 "transformations": [], 10936 "transparent": false, 10937 "type": "graph", 10938 "xaxis": { 10939 "mode": "time", 10940 "name": null, 10941 "show": true, 10942 "values": [] 10943 }, 10944 "yaxes": [ 10945 { 10946 "decimals": null, 10947 "format": "short", 10948 "label": null, 10949 "logBase": 1, 10950 "max": null, 10951 "min": null, 10952 "show": true 10953 }, 10954 { 10955 "decimals": null, 10956 "format": "short", 10957 "label": null, 10958 "logBase": 1, 10959 "max": null, 10960 "min": null, 10961 "show": true 10962 } 10963 ], 10964 "yaxis": { 10965 "align": false, 10966 "alignLevel": 0 10967 } 10968 }, 10969 { 10970 "aliasColors": {}, 10971 "bars": false, 10972 "cacheTimeout": null, 10973 "datasource": "$source", 10974 "description": null, 10975 "editable": true, 10976 "error": false, 10977 "fieldConfig": { 10978 "defaults": { 10979 "thresholds": { 10980 "mode": "absolute", 10981 "steps": [] 10982 } 10983 } 10984 }, 10985 "fill": 1, 10986 "grid": { 10987 "threshold1": null, 10988 "threshold1Color": "rgba(216, 200, 27, 0.27)", 10989 "threshold2": null, 10990 "threshold2Color": "rgba(234, 112, 112, 0.22)" 10991 }, 10992 "gridPos": null, 10993 "height": null, 10994 "hideTimeOverride": false, 10995 "id": 85, 10996 "interval": null, 10997 "isNew": true, 10998 "legend": { 10999 "alignAsTable": false, 11000 "avg": false, 11001 "current": false, 11002 "hideEmpty": false, 11003 "hideZero": false, 11004 "max": false, 11005 "min": false, 11006 "rightSide": false, 11007 "show": true, 11008 "sideWidth": null, 11009 "sort": null, 11010 "sortDesc": false, 11011 "total": false, 11012 "values": false 11013 }, 11014 "lines": true, 11015 "linewidth": 2, 11016 "links": [], 11017 "maxDataPoints": 100, 11018 "maxPerRow": null, 11019 "minSpan": null, 11020 "nullPointMode": null, 11021 "options": { 11022 "alertThreshold": true, 11023 "dataLinks": [] 11024 }, 11025 "percentage": false, 11026 "pointradius": 5, 11027 "points": false, 11028 "renderer": "flot", 11029 "repeat": null, 11030 "repeatDirection": null, 11031 "seriesOverrides": [], 11032 "span": 12, 11033 "stack": false, 11034 "steppedLine": false, 11035 "targets": [ 11036 { 11037 "datasource": "$source", 11038 "expr": "etcd_mvcc_db_total_size_in_bytes", 11039 "format": "time_series", 11040 "hide": false, 11041 "instant": false, 11042 "interval": "5s", 11043 "intervalFactor": 1, 11044 "legendFormat": "", 11045 "metric": "", 11046 "query": "etcd_mvcc_db_total_size_in_bytes", 11047 "refId": "", 11048 "step": 10, 11049 "target": "" 11050 }, 11051 { 11052 "datasource": "$source", 11053 "expr": "etcd_mvcc_db_total_size_in_use_in_bytes", 11054 "format": "time_series", 11055 "hide": false, 11056 "instant": false, 11057 "interval": "5s", 11058 "intervalFactor": 1, 11059 "legendFormat": "", 11060 "metric": "", 11061 "query": "etcd_mvcc_db_total_size_in_use_in_bytes", 11062 "refId": "", 11063 "step": 10, 11064 "target": "" 11065 }, 11066 { 11067 "datasource": "$source", 11068 "expr": "etcd_server_quota_backend_bytes", 11069 "format": "time_series", 11070 "hide": false, 11071 "instant": false, 11072 "interval": "5s", 11073 "intervalFactor": 1, 11074 "legendFormat": "", 11075 "metric": "", 11076 "query": "etcd_server_quota_backend_bytes", 11077 "refId": "", 11078 "step": 10, 11079 "target": "" 11080 } 11081 ], 11082 "thresholds": [], 11083 "timeFrom": null, 11084 "timeShift": null, 11085 "title": "etcd db size", 11086 "tooltip": { 11087 "msResolution": true, 11088 "shared": true, 11089 "sort": 2, 11090 "value_type": "cumulative" 11091 }, 11092 "transformations": [], 11093 "transparent": false, 11094 "type": "graph", 11095 "xaxis": { 11096 "mode": "time", 11097 "name": null, 11098 "show": true, 11099 "values": [] 11100 }, 11101 "yaxes": [ 11102 { 11103 "decimals": null, 11104 "format": "bytes", 11105 "label": null, 11106 "logBase": 1, 11107 "max": null, 11108 "min": null, 11109 "show": true 11110 }, 11111 { 11112 "decimals": null, 11113 "format": "short", 11114 "label": null, 11115 "logBase": 1, 11116 "max": null, 11117 "min": null, 11118 "show": true 11119 } 11120 ], 11121 "yaxis": { 11122 "align": false, 11123 "alignLevel": 0 11124 } 11125 }, 11126 { 11127 "aliasColors": {}, 11128 "bars": false, 11129 "cacheTimeout": null, 11130 "datasource": "$secondary_source", 11131 "description": null, 11132 "editable": true, 11133 "error": false, 11134 "fieldConfig": { 11135 "defaults": { 11136 "thresholds": { 11137 "mode": "absolute", 11138 "steps": [] 11139 } 11140 } 11141 }, 11142 "fill": 1, 11143 "grid": { 11144 "threshold1": null, 11145 "threshold1Color": "rgba(216, 200, 27, 0.27)", 11146 "threshold2": null, 11147 "threshold2Color": "rgba(234, 112, 112, 0.22)" 11148 }, 11149 "gridPos": null, 11150 "height": null, 11151 "hideTimeOverride": false, 11152 "id": 86, 11153 "interval": null, 11154 "isNew": true, 11155 "legend": { 11156 "alignAsTable": false, 11157 "avg": false, 11158 "current": false, 11159 "hideEmpty": false, 11160 "hideZero": false, 11161 "max": false, 11162 "min": false, 11163 "rightSide": false, 11164 "show": true, 11165 "sideWidth": null, 11166 "sort": null, 11167 "sortDesc": false, 11168 "total": false, 11169 "values": false 11170 }, 11171 "lines": true, 11172 "linewidth": 2, 11173 "links": [], 11174 "maxDataPoints": 100, 11175 "maxPerRow": null, 11176 "minSpan": null, 11177 "nullPointMode": null, 11178 "options": { 11179 "alertThreshold": true, 11180 "dataLinks": [] 11181 }, 11182 "percentage": false, 11183 "pointradius": 5, 11184 "points": false, 11185 "renderer": "flot", 11186 "repeat": null, 11187 "repeatDirection": null, 11188 "seriesOverrides": [], 11189 "span": 12, 11190 "stack": false, 11191 "steppedLine": false, 11192 "targets": [ 11193 { 11194 "datasource": "$source", 11195 "expr": "etcd_mvcc_db_total_size_in_bytes", 11196 "format": "time_series", 11197 "hide": false, 11198 "instant": false, 11199 "interval": "5s", 11200 "intervalFactor": 1, 11201 "legendFormat": "", 11202 "metric": "", 11203 "query": "etcd_mvcc_db_total_size_in_bytes", 11204 "refId": "", 11205 "step": 10, 11206 "target": "" 11207 }, 11208 { 11209 "datasource": "$source", 11210 "expr": "etcd_mvcc_db_total_size_in_use_in_bytes", 11211 "format": "time_series", 11212 "hide": false, 11213 "instant": false, 11214 "interval": "5s", 11215 "intervalFactor": 1, 11216 "legendFormat": "", 11217 "metric": "", 11218 "query": "etcd_mvcc_db_total_size_in_use_in_bytes", 11219 "refId": "", 11220 "step": 10, 11221 "target": "" 11222 }, 11223 { 11224 "datasource": "$source", 11225 "expr": "etcd_server_quota_backend_bytes", 11226 "format": "time_series", 11227 "hide": false, 11228 "instant": false, 11229 "interval": "5s", 11230 "intervalFactor": 1, 11231 "legendFormat": "", 11232 "metric": "", 11233 "query": "etcd_server_quota_backend_bytes", 11234 "refId": "", 11235 "step": 10, 11236 "target": "" 11237 } 11238 ], 11239 "thresholds": [], 11240 "timeFrom": null, 11241 "timeShift": "$timeshift", 11242 "title": "[SECONDARY] etcd db size", 11243 "tooltip": { 11244 "msResolution": true, 11245 "shared": true, 11246 "sort": 2, 11247 "value_type": "cumulative" 11248 }, 11249 "transformations": [], 11250 "transparent": false, 11251 "type": "graph", 11252 "xaxis": { 11253 "mode": "time", 11254 "name": null, 11255 "show": true, 11256 "values": [] 11257 }, 11258 "yaxes": [ 11259 { 11260 "decimals": null, 11261 "format": "bytes", 11262 "label": null, 11263 "logBase": 1, 11264 "max": null, 11265 "min": null, 11266 "show": true 11267 }, 11268 { 11269 "decimals": null, 11270 "format": "short", 11271 "label": null, 11272 "logBase": 1, 11273 "max": null, 11274 "min": null, 11275 "show": true 11276 } 11277 ], 11278 "yaxis": { 11279 "align": false, 11280 "alignLevel": 0 11281 } 11282 } 11283 ], 11284 "repeat": null, 11285 "showTitle": true, 11286 "title": "etcd" 11287 }, 11288 { 11289 "collapse": true, 11290 "editable": true, 11291 "height": "300px", 11292 "panels": [ 11293 { 11294 "aliasColors": {}, 11295 "bars": false, 11296 "cacheTimeout": null, 11297 "datasource": "$source", 11298 "description": null, 11299 "editable": true, 11300 "error": false, 11301 "fieldConfig": { 11302 "defaults": { 11303 "thresholds": { 11304 "mode": "absolute", 11305 "steps": [] 11306 } 11307 } 11308 }, 11309 "fill": 1, 11310 "grid": { 11311 "threshold1": null, 11312 "threshold1Color": "rgba(216, 200, 27, 0.27)", 11313 "threshold2": null, 11314 "threshold2Color": "rgba(234, 112, 112, 0.22)" 11315 }, 11316 "gridPos": null, 11317 "height": null, 11318 "hideTimeOverride": false, 11319 "id": 87, 11320 "interval": null, 11321 "isNew": true, 11322 "legend": { 11323 "alignAsTable": false, 11324 "avg": false, 11325 "current": false, 11326 "hideEmpty": false, 11327 "hideZero": false, 11328 "max": false, 11329 "min": false, 11330 "rightSide": false, 11331 "show": true, 11332 "sideWidth": null, 11333 "sort": null, 11334 "sortDesc": false, 11335 "total": false, 11336 "values": false 11337 }, 11338 "lines": true, 11339 "linewidth": 2, 11340 "links": [], 11341 "maxDataPoints": 100, 11342 "maxPerRow": null, 11343 "minSpan": null, 11344 "nullPointMode": null, 11345 "options": { 11346 "alertThreshold": true, 11347 "dataLinks": [] 11348 }, 11349 "percentage": false, 11350 "pointradius": 5, 11351 "points": false, 11352 "renderer": "flot", 11353 "repeat": null, 11354 "repeatDirection": null, 11355 "seriesOverrides": [], 11356 "span": 12, 11357 "stack": false, 11358 "steppedLine": false, 11359 "targets": [ 11360 { 11361 "datasource": "$source", 11362 "expr": "go_goroutines{job=\"master\", endpoint=\"apiserver\"}", 11363 "format": "time_series", 11364 "hide": false, 11365 "instant": false, 11366 "interval": "5s", 11367 "intervalFactor": 1, 11368 "legendFormat": "{{instance}}", 11369 "metric": "", 11370 "query": "go_goroutines{job=\"master\", endpoint=\"apiserver\"}", 11371 "refId": "", 11372 "step": 10, 11373 "target": "" 11374 } 11375 ], 11376 "thresholds": [], 11377 "timeFrom": null, 11378 "timeShift": null, 11379 "title": "goroutines", 11380 "tooltip": { 11381 "msResolution": true, 11382 "shared": true, 11383 "sort": 2, 11384 "value_type": "cumulative" 11385 }, 11386 "transformations": [], 11387 "transparent": false, 11388 "type": "graph", 11389 "xaxis": { 11390 "mode": "time", 11391 "name": null, 11392 "show": true, 11393 "values": [] 11394 }, 11395 "yaxes": [ 11396 { 11397 "decimals": null, 11398 "format": "short", 11399 "label": null, 11400 "logBase": 1, 11401 "max": null, 11402 "min": null, 11403 "show": true 11404 }, 11405 { 11406 "decimals": null, 11407 "format": "short", 11408 "label": null, 11409 "logBase": 1, 11410 "max": null, 11411 "min": null, 11412 "show": true 11413 } 11414 ], 11415 "yaxis": { 11416 "align": false, 11417 "alignLevel": 0 11418 } 11419 }, 11420 { 11421 "aliasColors": {}, 11422 "bars": false, 11423 "cacheTimeout": null, 11424 "datasource": "$secondary_source", 11425 "description": null, 11426 "editable": true, 11427 "error": false, 11428 "fieldConfig": { 11429 "defaults": { 11430 "thresholds": { 11431 "mode": "absolute", 11432 "steps": [] 11433 } 11434 } 11435 }, 11436 "fill": 1, 11437 "grid": { 11438 "threshold1": null, 11439 "threshold1Color": "rgba(216, 200, 27, 0.27)", 11440 "threshold2": null, 11441 "threshold2Color": "rgba(234, 112, 112, 0.22)" 11442 }, 11443 "gridPos": null, 11444 "height": null, 11445 "hideTimeOverride": false, 11446 "id": 88, 11447 "interval": null, 11448 "isNew": true, 11449 "legend": { 11450 "alignAsTable": false, 11451 "avg": false, 11452 "current": false, 11453 "hideEmpty": false, 11454 "hideZero": false, 11455 "max": false, 11456 "min": false, 11457 "rightSide": false, 11458 "show": true, 11459 "sideWidth": null, 11460 "sort": null, 11461 "sortDesc": false, 11462 "total": false, 11463 "values": false 11464 }, 11465 "lines": true, 11466 "linewidth": 2, 11467 "links": [], 11468 "maxDataPoints": 100, 11469 "maxPerRow": null, 11470 "minSpan": null, 11471 "nullPointMode": null, 11472 "options": { 11473 "alertThreshold": true, 11474 "dataLinks": [] 11475 }, 11476 "percentage": false, 11477 "pointradius": 5, 11478 "points": false, 11479 "renderer": "flot", 11480 "repeat": null, 11481 "repeatDirection": null, 11482 "seriesOverrides": [], 11483 "span": 12, 11484 "stack": false, 11485 "steppedLine": false, 11486 "targets": [ 11487 { 11488 "datasource": "$source", 11489 "expr": "go_goroutines{job=\"master\", endpoint=\"apiserver\"}", 11490 "format": "time_series", 11491 "hide": false, 11492 "instant": false, 11493 "interval": "5s", 11494 "intervalFactor": 1, 11495 "legendFormat": "{{instance}}", 11496 "metric": "", 11497 "query": "go_goroutines{job=\"master\", endpoint=\"apiserver\"}", 11498 "refId": "", 11499 "step": 10, 11500 "target": "" 11501 } 11502 ], 11503 "thresholds": [], 11504 "timeFrom": null, 11505 "timeShift": "$timeshift", 11506 "title": "[SECONDARY] goroutines", 11507 "tooltip": { 11508 "msResolution": true, 11509 "shared": true, 11510 "sort": 2, 11511 "value_type": "cumulative" 11512 }, 11513 "transformations": [], 11514 "transparent": false, 11515 "type": "graph", 11516 "xaxis": { 11517 "mode": "time", 11518 "name": null, 11519 "show": true, 11520 "values": [] 11521 }, 11522 "yaxes": [ 11523 { 11524 "decimals": null, 11525 "format": "short", 11526 "label": null, 11527 "logBase": 1, 11528 "max": null, 11529 "min": null, 11530 "show": true 11531 }, 11532 { 11533 "decimals": null, 11534 "format": "short", 11535 "label": null, 11536 "logBase": 1, 11537 "max": null, 11538 "min": null, 11539 "show": true 11540 } 11541 ], 11542 "yaxis": { 11543 "align": false, 11544 "alignLevel": 0 11545 } 11546 }, 11547 { 11548 "aliasColors": {}, 11549 "bars": false, 11550 "cacheTimeout": null, 11551 "datasource": "$source", 11552 "description": null, 11553 "editable": true, 11554 "error": false, 11555 "fieldConfig": { 11556 "defaults": { 11557 "thresholds": { 11558 "mode": "absolute", 11559 "steps": [] 11560 } 11561 } 11562 }, 11563 "fill": 1, 11564 "grid": { 11565 "threshold1": null, 11566 "threshold1Color": "rgba(216, 200, 27, 0.27)", 11567 "threshold2": null, 11568 "threshold2Color": "rgba(234, 112, 112, 0.22)" 11569 }, 11570 "gridPos": null, 11571 "height": null, 11572 "hideTimeOverride": false, 11573 "id": 89, 11574 "interval": null, 11575 "isNew": true, 11576 "legend": { 11577 "alignAsTable": false, 11578 "avg": false, 11579 "current": false, 11580 "hideEmpty": false, 11581 "hideZero": false, 11582 "max": false, 11583 "min": false, 11584 "rightSide": false, 11585 "show": true, 11586 "sideWidth": null, 11587 "sort": null, 11588 "sortDesc": false, 11589 "total": false, 11590 "values": false 11591 }, 11592 "lines": true, 11593 "linewidth": 2, 11594 "links": [], 11595 "maxDataPoints": 100, 11596 "maxPerRow": null, 11597 "minSpan": null, 11598 "nullPointMode": null, 11599 "options": { 11600 "alertThreshold": true, 11601 "dataLinks": [] 11602 }, 11603 "percentage": false, 11604 "pointradius": 5, 11605 "points": false, 11606 "renderer": "flot", 11607 "repeat": null, 11608 "repeatDirection": null, 11609 "seriesOverrides": [], 11610 "span": 12, 11611 "stack": false, 11612 "steppedLine": false, 11613 "targets": [ 11614 { 11615 "datasource": "$source", 11616 "expr": "rate(go_gc_duration_seconds_count{job=\"master\", endpoint=\"apiserver\"}[1m])", 11617 "format": "time_series", 11618 "hide": false, 11619 "instant": false, 11620 "interval": "5s", 11621 "intervalFactor": 1, 11622 "legendFormat": "{{instance}}", 11623 "metric": "", 11624 "query": "rate(go_gc_duration_seconds_count{job=\"master\", endpoint=\"apiserver\"}[1m])", 11625 "refId": "", 11626 "step": 10, 11627 "target": "" 11628 } 11629 ], 11630 "thresholds": [], 11631 "timeFrom": null, 11632 "timeShift": null, 11633 "title": "gc rate", 11634 "tooltip": { 11635 "msResolution": true, 11636 "shared": true, 11637 "sort": 2, 11638 "value_type": "cumulative" 11639 }, 11640 "transformations": [], 11641 "transparent": false, 11642 "type": "graph", 11643 "xaxis": { 11644 "mode": "time", 11645 "name": null, 11646 "show": true, 11647 "values": [] 11648 }, 11649 "yaxes": [ 11650 { 11651 "decimals": null, 11652 "format": "short", 11653 "label": null, 11654 "logBase": 1, 11655 "max": null, 11656 "min": null, 11657 "show": true 11658 }, 11659 { 11660 "decimals": null, 11661 "format": "short", 11662 "label": null, 11663 "logBase": 1, 11664 "max": null, 11665 "min": null, 11666 "show": true 11667 } 11668 ], 11669 "yaxis": { 11670 "align": false, 11671 "alignLevel": 0 11672 } 11673 }, 11674 { 11675 "aliasColors": {}, 11676 "bars": false, 11677 "cacheTimeout": null, 11678 "datasource": "$secondary_source", 11679 "description": null, 11680 "editable": true, 11681 "error": false, 11682 "fieldConfig": { 11683 "defaults": { 11684 "thresholds": { 11685 "mode": "absolute", 11686 "steps": [] 11687 } 11688 } 11689 }, 11690 "fill": 1, 11691 "grid": { 11692 "threshold1": null, 11693 "threshold1Color": "rgba(216, 200, 27, 0.27)", 11694 "threshold2": null, 11695 "threshold2Color": "rgba(234, 112, 112, 0.22)" 11696 }, 11697 "gridPos": null, 11698 "height": null, 11699 "hideTimeOverride": false, 11700 "id": 90, 11701 "interval": null, 11702 "isNew": true, 11703 "legend": { 11704 "alignAsTable": false, 11705 "avg": false, 11706 "current": false, 11707 "hideEmpty": false, 11708 "hideZero": false, 11709 "max": false, 11710 "min": false, 11711 "rightSide": false, 11712 "show": true, 11713 "sideWidth": null, 11714 "sort": null, 11715 "sortDesc": false, 11716 "total": false, 11717 "values": false 11718 }, 11719 "lines": true, 11720 "linewidth": 2, 11721 "links": [], 11722 "maxDataPoints": 100, 11723 "maxPerRow": null, 11724 "minSpan": null, 11725 "nullPointMode": null, 11726 "options": { 11727 "alertThreshold": true, 11728 "dataLinks": [] 11729 }, 11730 "percentage": false, 11731 "pointradius": 5, 11732 "points": false, 11733 "renderer": "flot", 11734 "repeat": null, 11735 "repeatDirection": null, 11736 "seriesOverrides": [], 11737 "span": 12, 11738 "stack": false, 11739 "steppedLine": false, 11740 "targets": [ 11741 { 11742 "datasource": "$source", 11743 "expr": "rate(go_gc_duration_seconds_count{job=\"master\", endpoint=\"apiserver\"}[1m])", 11744 "format": "time_series", 11745 "hide": false, 11746 "instant": false, 11747 "interval": "5s", 11748 "intervalFactor": 1, 11749 "legendFormat": "{{instance}}", 11750 "metric": "", 11751 "query": "rate(go_gc_duration_seconds_count{job=\"master\", endpoint=\"apiserver\"}[1m])", 11752 "refId": "", 11753 "step": 10, 11754 "target": "" 11755 } 11756 ], 11757 "thresholds": [], 11758 "timeFrom": null, 11759 "timeShift": "$timeshift", 11760 "title": "[SECONDARY] gc rate", 11761 "tooltip": { 11762 "msResolution": true, 11763 "shared": true, 11764 "sort": 2, 11765 "value_type": "cumulative" 11766 }, 11767 "transformations": [], 11768 "transparent": false, 11769 "type": "graph", 11770 "xaxis": { 11771 "mode": "time", 11772 "name": null, 11773 "show": true, 11774 "values": [] 11775 }, 11776 "yaxes": [ 11777 { 11778 "decimals": null, 11779 "format": "short", 11780 "label": null, 11781 "logBase": 1, 11782 "max": null, 11783 "min": null, 11784 "show": true 11785 }, 11786 { 11787 "decimals": null, 11788 "format": "short", 11789 "label": null, 11790 "logBase": 1, 11791 "max": null, 11792 "min": null, 11793 "show": true 11794 } 11795 ], 11796 "yaxis": { 11797 "align": false, 11798 "alignLevel": 0 11799 } 11800 }, 11801 { 11802 "aliasColors": {}, 11803 "bars": false, 11804 "cacheTimeout": null, 11805 "datasource": "$source", 11806 "description": null, 11807 "editable": true, 11808 "error": false, 11809 "fieldConfig": { 11810 "defaults": { 11811 "thresholds": { 11812 "mode": "absolute", 11813 "steps": [] 11814 } 11815 } 11816 }, 11817 "fill": 1, 11818 "grid": { 11819 "threshold1": null, 11820 "threshold1Color": "rgba(216, 200, 27, 0.27)", 11821 "threshold2": null, 11822 "threshold2Color": "rgba(234, 112, 112, 0.22)" 11823 }, 11824 "gridPos": null, 11825 "height": null, 11826 "hideTimeOverride": false, 11827 "id": 91, 11828 "interval": null, 11829 "isNew": true, 11830 "legend": { 11831 "alignAsTable": false, 11832 "avg": false, 11833 "current": false, 11834 "hideEmpty": false, 11835 "hideZero": false, 11836 "max": false, 11837 "min": false, 11838 "rightSide": false, 11839 "show": true, 11840 "sideWidth": null, 11841 "sort": null, 11842 "sortDesc": false, 11843 "total": false, 11844 "values": false 11845 }, 11846 "lines": true, 11847 "linewidth": 2, 11848 "links": [], 11849 "maxDataPoints": 100, 11850 "maxPerRow": null, 11851 "minSpan": null, 11852 "nullPointMode": null, 11853 "options": { 11854 "alertThreshold": true, 11855 "dataLinks": [] 11856 }, 11857 "percentage": false, 11858 "pointradius": 5, 11859 "points": false, 11860 "renderer": "flot", 11861 "repeat": null, 11862 "repeatDirection": null, 11863 "seriesOverrides": [], 11864 "span": 12, 11865 "stack": false, 11866 "steppedLine": false, 11867 "targets": [ 11868 { 11869 "datasource": "$source", 11870 "expr": "rate(go_memstats_alloc_bytes_total{job=\"master\", endpoint=\"apiserver\"}[1m])", 11871 "format": "time_series", 11872 "hide": false, 11873 "instant": false, 11874 "interval": "5s", 11875 "intervalFactor": 1, 11876 "legendFormat": "{{instance}}", 11877 "metric": "", 11878 "query": "rate(go_memstats_alloc_bytes_total{job=\"master\", endpoint=\"apiserver\"}[1m])", 11879 "refId": "", 11880 "step": 10, 11881 "target": "" 11882 } 11883 ], 11884 "thresholds": [], 11885 "timeFrom": null, 11886 "timeShift": null, 11887 "title": "alloc rate", 11888 "tooltip": { 11889 "msResolution": true, 11890 "shared": true, 11891 "sort": 2, 11892 "value_type": "cumulative" 11893 }, 11894 "transformations": [], 11895 "transparent": false, 11896 "type": "graph", 11897 "xaxis": { 11898 "mode": "time", 11899 "name": null, 11900 "show": true, 11901 "values": [] 11902 }, 11903 "yaxes": [ 11904 { 11905 "decimals": null, 11906 "format": "Bps", 11907 "label": null, 11908 "logBase": 1, 11909 "max": null, 11910 "min": null, 11911 "show": true 11912 }, 11913 { 11914 "decimals": null, 11915 "format": "short", 11916 "label": null, 11917 "logBase": 1, 11918 "max": null, 11919 "min": null, 11920 "show": true 11921 } 11922 ], 11923 "yaxis": { 11924 "align": false, 11925 "alignLevel": 0 11926 } 11927 }, 11928 { 11929 "aliasColors": {}, 11930 "bars": false, 11931 "cacheTimeout": null, 11932 "datasource": "$secondary_source", 11933 "description": null, 11934 "editable": true, 11935 "error": false, 11936 "fieldConfig": { 11937 "defaults": { 11938 "thresholds": { 11939 "mode": "absolute", 11940 "steps": [] 11941 } 11942 } 11943 }, 11944 "fill": 1, 11945 "grid": { 11946 "threshold1": null, 11947 "threshold1Color": "rgba(216, 200, 27, 0.27)", 11948 "threshold2": null, 11949 "threshold2Color": "rgba(234, 112, 112, 0.22)" 11950 }, 11951 "gridPos": null, 11952 "height": null, 11953 "hideTimeOverride": false, 11954 "id": 92, 11955 "interval": null, 11956 "isNew": true, 11957 "legend": { 11958 "alignAsTable": false, 11959 "avg": false, 11960 "current": false, 11961 "hideEmpty": false, 11962 "hideZero": false, 11963 "max": false, 11964 "min": false, 11965 "rightSide": false, 11966 "show": true, 11967 "sideWidth": null, 11968 "sort": null, 11969 "sortDesc": false, 11970 "total": false, 11971 "values": false 11972 }, 11973 "lines": true, 11974 "linewidth": 2, 11975 "links": [], 11976 "maxDataPoints": 100, 11977 "maxPerRow": null, 11978 "minSpan": null, 11979 "nullPointMode": null, 11980 "options": { 11981 "alertThreshold": true, 11982 "dataLinks": [] 11983 }, 11984 "percentage": false, 11985 "pointradius": 5, 11986 "points": false, 11987 "renderer": "flot", 11988 "repeat": null, 11989 "repeatDirection": null, 11990 "seriesOverrides": [], 11991 "span": 12, 11992 "stack": false, 11993 "steppedLine": false, 11994 "targets": [ 11995 { 11996 "datasource": "$source", 11997 "expr": "rate(go_memstats_alloc_bytes_total{job=\"master\", endpoint=\"apiserver\"}[1m])", 11998 "format": "time_series", 11999 "hide": false, 12000 "instant": false, 12001 "interval": "5s", 12002 "intervalFactor": 1, 12003 "legendFormat": "{{instance}}", 12004 "metric": "", 12005 "query": "rate(go_memstats_alloc_bytes_total{job=\"master\", endpoint=\"apiserver\"}[1m])", 12006 "refId": "", 12007 "step": 10, 12008 "target": "" 12009 } 12010 ], 12011 "thresholds": [], 12012 "timeFrom": null, 12013 "timeShift": "$timeshift", 12014 "title": "[SECONDARY] alloc rate", 12015 "tooltip": { 12016 "msResolution": true, 12017 "shared": true, 12018 "sort": 2, 12019 "value_type": "cumulative" 12020 }, 12021 "transformations": [], 12022 "transparent": false, 12023 "type": "graph", 12024 "xaxis": { 12025 "mode": "time", 12026 "name": null, 12027 "show": true, 12028 "values": [] 12029 }, 12030 "yaxes": [ 12031 { 12032 "decimals": null, 12033 "format": "Bps", 12034 "label": null, 12035 "logBase": 1, 12036 "max": null, 12037 "min": null, 12038 "show": true 12039 }, 12040 { 12041 "decimals": null, 12042 "format": "short", 12043 "label": null, 12044 "logBase": 1, 12045 "max": null, 12046 "min": null, 12047 "show": true 12048 } 12049 ], 12050 "yaxis": { 12051 "align": false, 12052 "alignLevel": 0 12053 } 12054 }, 12055 { 12056 "aliasColors": {}, 12057 "bars": false, 12058 "cacheTimeout": null, 12059 "datasource": "$source", 12060 "description": null, 12061 "editable": true, 12062 "error": false, 12063 "fieldConfig": { 12064 "defaults": { 12065 "thresholds": { 12066 "mode": "absolute", 12067 "steps": [] 12068 } 12069 } 12070 }, 12071 "fill": 1, 12072 "grid": { 12073 "threshold1": null, 12074 "threshold1Color": "rgba(216, 200, 27, 0.27)", 12075 "threshold2": null, 12076 "threshold2Color": "rgba(234, 112, 112, 0.22)" 12077 }, 12078 "gridPos": null, 12079 "height": null, 12080 "hideTimeOverride": false, 12081 "id": 93, 12082 "interval": null, 12083 "isNew": true, 12084 "legend": { 12085 "alignAsTable": false, 12086 "avg": false, 12087 "current": false, 12088 "hideEmpty": false, 12089 "hideZero": false, 12090 "max": false, 12091 "min": false, 12092 "rightSide": false, 12093 "show": true, 12094 "sideWidth": null, 12095 "sort": null, 12096 "sortDesc": false, 12097 "total": false, 12098 "values": false 12099 }, 12100 "lines": true, 12101 "linewidth": 2, 12102 "links": [], 12103 "maxDataPoints": 100, 12104 "maxPerRow": null, 12105 "minSpan": null, 12106 "nullPointMode": null, 12107 "options": { 12108 "alertThreshold": true, 12109 "dataLinks": [] 12110 }, 12111 "percentage": false, 12112 "pointradius": 5, 12113 "points": false, 12114 "renderer": "flot", 12115 "repeat": null, 12116 "repeatDirection": null, 12117 "seriesOverrides": [], 12118 "span": 12, 12119 "stack": false, 12120 "steppedLine": false, 12121 "targets": [ 12122 { 12123 "datasource": "$source", 12124 "expr": "sum(apiserver_registered_watchers{kind=~\"(?i:(${resource:regex}))s*\"}) by (instance, group, version, kind)", 12125 "format": "time_series", 12126 "hide": false, 12127 "instant": false, 12128 "interval": "5s", 12129 "intervalFactor": 1, 12130 "legendFormat": "{{instance}}: {{version}}.{{group}}.{{kind}}", 12131 "metric": "", 12132 "query": "sum(apiserver_registered_watchers{kind=~\"(?i:(${resource:regex}))s*\"}) by (instance, group, version, kind)", 12133 "refId": "", 12134 "step": 10, 12135 "target": "" 12136 } 12137 ], 12138 "thresholds": [], 12139 "timeFrom": null, 12140 "timeShift": null, 12141 "title": "Number of active watches", 12142 "tooltip": { 12143 "msResolution": true, 12144 "shared": true, 12145 "sort": 2, 12146 "value_type": "cumulative" 12147 }, 12148 "transformations": [], 12149 "transparent": false, 12150 "type": "graph", 12151 "xaxis": { 12152 "mode": "time", 12153 "name": null, 12154 "show": true, 12155 "values": [] 12156 }, 12157 "yaxes": [ 12158 { 12159 "decimals": null, 12160 "format": "short", 12161 "label": null, 12162 "logBase": 1, 12163 "max": null, 12164 "min": null, 12165 "show": true 12166 }, 12167 { 12168 "decimals": null, 12169 "format": "short", 12170 "label": null, 12171 "logBase": 1, 12172 "max": null, 12173 "min": null, 12174 "show": true 12175 } 12176 ], 12177 "yaxis": { 12178 "align": false, 12179 "alignLevel": 0 12180 } 12181 }, 12182 { 12183 "aliasColors": {}, 12184 "bars": false, 12185 "cacheTimeout": null, 12186 "datasource": "$secondary_source", 12187 "description": null, 12188 "editable": true, 12189 "error": false, 12190 "fieldConfig": { 12191 "defaults": { 12192 "thresholds": { 12193 "mode": "absolute", 12194 "steps": [] 12195 } 12196 } 12197 }, 12198 "fill": 1, 12199 "grid": { 12200 "threshold1": null, 12201 "threshold1Color": "rgba(216, 200, 27, 0.27)", 12202 "threshold2": null, 12203 "threshold2Color": "rgba(234, 112, 112, 0.22)" 12204 }, 12205 "gridPos": null, 12206 "height": null, 12207 "hideTimeOverride": false, 12208 "id": 94, 12209 "interval": null, 12210 "isNew": true, 12211 "legend": { 12212 "alignAsTable": false, 12213 "avg": false, 12214 "current": false, 12215 "hideEmpty": false, 12216 "hideZero": false, 12217 "max": false, 12218 "min": false, 12219 "rightSide": false, 12220 "show": true, 12221 "sideWidth": null, 12222 "sort": null, 12223 "sortDesc": false, 12224 "total": false, 12225 "values": false 12226 }, 12227 "lines": true, 12228 "linewidth": 2, 12229 "links": [], 12230 "maxDataPoints": 100, 12231 "maxPerRow": null, 12232 "minSpan": null, 12233 "nullPointMode": null, 12234 "options": { 12235 "alertThreshold": true, 12236 "dataLinks": [] 12237 }, 12238 "percentage": false, 12239 "pointradius": 5, 12240 "points": false, 12241 "renderer": "flot", 12242 "repeat": null, 12243 "repeatDirection": null, 12244 "seriesOverrides": [], 12245 "span": 12, 12246 "stack": false, 12247 "steppedLine": false, 12248 "targets": [ 12249 { 12250 "datasource": "$source", 12251 "expr": "sum(apiserver_registered_watchers{kind=~\"(?i:(${resource:regex}))s*\"}) by (instance, group, version, kind)", 12252 "format": "time_series", 12253 "hide": false, 12254 "instant": false, 12255 "interval": "5s", 12256 "intervalFactor": 1, 12257 "legendFormat": "{{instance}}: {{version}}.{{group}}.{{kind}}", 12258 "metric": "", 12259 "query": "sum(apiserver_registered_watchers{kind=~\"(?i:(${resource:regex}))s*\"}) by (instance, group, version, kind)", 12260 "refId": "", 12261 "step": 10, 12262 "target": "" 12263 } 12264 ], 12265 "thresholds": [], 12266 "timeFrom": null, 12267 "timeShift": "$timeshift", 12268 "title": "[SECONDARY] Number of active watches", 12269 "tooltip": { 12270 "msResolution": true, 12271 "shared": true, 12272 "sort": 2, 12273 "value_type": "cumulative" 12274 }, 12275 "transformations": [], 12276 "transparent": false, 12277 "type": "graph", 12278 "xaxis": { 12279 "mode": "time", 12280 "name": null, 12281 "show": true, 12282 "values": [] 12283 }, 12284 "yaxes": [ 12285 { 12286 "decimals": null, 12287 "format": "short", 12288 "label": null, 12289 "logBase": 1, 12290 "max": null, 12291 "min": null, 12292 "show": true 12293 }, 12294 { 12295 "decimals": null, 12296 "format": "short", 12297 "label": null, 12298 "logBase": 1, 12299 "max": null, 12300 "min": null, 12301 "show": true 12302 } 12303 ], 12304 "yaxis": { 12305 "align": false, 12306 "alignLevel": 0 12307 } 12308 }, 12309 { 12310 "aliasColors": {}, 12311 "bars": false, 12312 "cacheTimeout": null, 12313 "datasource": "$source", 12314 "description": null, 12315 "editable": true, 12316 "error": false, 12317 "fieldConfig": { 12318 "defaults": { 12319 "thresholds": { 12320 "mode": "absolute", 12321 "steps": [] 12322 } 12323 } 12324 }, 12325 "fill": 1, 12326 "grid": { 12327 "threshold1": null, 12328 "threshold1Color": "rgba(216, 200, 27, 0.27)", 12329 "threshold2": null, 12330 "threshold2Color": "rgba(234, 112, 112, 0.22)" 12331 }, 12332 "gridPos": null, 12333 "height": null, 12334 "hideTimeOverride": false, 12335 "id": 95, 12336 "interval": null, 12337 "isNew": true, 12338 "legend": { 12339 "alignAsTable": false, 12340 "avg": false, 12341 "current": false, 12342 "hideEmpty": false, 12343 "hideZero": false, 12344 "max": false, 12345 "min": false, 12346 "rightSide": false, 12347 "show": true, 12348 "sideWidth": null, 12349 "sort": null, 12350 "sortDesc": false, 12351 "total": false, 12352 "values": false 12353 }, 12354 "lines": true, 12355 "linewidth": 2, 12356 "links": [], 12357 "maxDataPoints": 100, 12358 "maxPerRow": null, 12359 "minSpan": null, 12360 "nullPointMode": null, 12361 "options": { 12362 "alertThreshold": true, 12363 "dataLinks": [] 12364 }, 12365 "percentage": false, 12366 "pointradius": 5, 12367 "points": false, 12368 "renderer": "flot", 12369 "repeat": null, 12370 "repeatDirection": null, 12371 "seriesOverrides": [], 12372 "span": 12, 12373 "stack": false, 12374 "steppedLine": false, 12375 "targets": [ 12376 { 12377 "datasource": "$source", 12378 "expr": "sum(irate(apiserver_watch_events_total{kind=~\"(?i:(${resource:regex}))s*\"}[1m])) by (instance, group, version, kind)", 12379 "format": "time_series", 12380 "hide": false, 12381 "instant": false, 12382 "interval": "5s", 12383 "intervalFactor": 1, 12384 "legendFormat": "{{instance}}: {{version}}.{{group}}.{{kind}}", 12385 "metric": "", 12386 "query": "sum(irate(apiserver_watch_events_total{kind=~\"(?i:(${resource:regex}))s*\"}[1m])) by (instance, group, version, kind)", 12387 "refId": "", 12388 "step": 10, 12389 "target": "" 12390 } 12391 ], 12392 "thresholds": [], 12393 "timeFrom": null, 12394 "timeShift": null, 12395 "title": "Watch events rate", 12396 "tooltip": { 12397 "msResolution": true, 12398 "shared": true, 12399 "sort": 2, 12400 "value_type": "cumulative" 12401 }, 12402 "transformations": [], 12403 "transparent": false, 12404 "type": "graph", 12405 "xaxis": { 12406 "mode": "time", 12407 "name": null, 12408 "show": true, 12409 "values": [] 12410 }, 12411 "yaxes": [ 12412 { 12413 "decimals": null, 12414 "format": "short", 12415 "label": null, 12416 "logBase": 1, 12417 "max": null, 12418 "min": null, 12419 "show": true 12420 }, 12421 { 12422 "decimals": null, 12423 "format": "short", 12424 "label": null, 12425 "logBase": 1, 12426 "max": null, 12427 "min": null, 12428 "show": true 12429 } 12430 ], 12431 "yaxis": { 12432 "align": false, 12433 "alignLevel": 0 12434 } 12435 }, 12436 { 12437 "aliasColors": {}, 12438 "bars": false, 12439 "cacheTimeout": null, 12440 "datasource": "$secondary_source", 12441 "description": null, 12442 "editable": true, 12443 "error": false, 12444 "fieldConfig": { 12445 "defaults": { 12446 "thresholds": { 12447 "mode": "absolute", 12448 "steps": [] 12449 } 12450 } 12451 }, 12452 "fill": 1, 12453 "grid": { 12454 "threshold1": null, 12455 "threshold1Color": "rgba(216, 200, 27, 0.27)", 12456 "threshold2": null, 12457 "threshold2Color": "rgba(234, 112, 112, 0.22)" 12458 }, 12459 "gridPos": null, 12460 "height": null, 12461 "hideTimeOverride": false, 12462 "id": 96, 12463 "interval": null, 12464 "isNew": true, 12465 "legend": { 12466 "alignAsTable": false, 12467 "avg": false, 12468 "current": false, 12469 "hideEmpty": false, 12470 "hideZero": false, 12471 "max": false, 12472 "min": false, 12473 "rightSide": false, 12474 "show": true, 12475 "sideWidth": null, 12476 "sort": null, 12477 "sortDesc": false, 12478 "total": false, 12479 "values": false 12480 }, 12481 "lines": true, 12482 "linewidth": 2, 12483 "links": [], 12484 "maxDataPoints": 100, 12485 "maxPerRow": null, 12486 "minSpan": null, 12487 "nullPointMode": null, 12488 "options": { 12489 "alertThreshold": true, 12490 "dataLinks": [] 12491 }, 12492 "percentage": false, 12493 "pointradius": 5, 12494 "points": false, 12495 "renderer": "flot", 12496 "repeat": null, 12497 "repeatDirection": null, 12498 "seriesOverrides": [], 12499 "span": 12, 12500 "stack": false, 12501 "steppedLine": false, 12502 "targets": [ 12503 { 12504 "datasource": "$source", 12505 "expr": "sum(irate(apiserver_watch_events_total{kind=~\"(?i:(${resource:regex}))s*\"}[1m])) by (instance, group, version, kind)", 12506 "format": "time_series", 12507 "hide": false, 12508 "instant": false, 12509 "interval": "5s", 12510 "intervalFactor": 1, 12511 "legendFormat": "{{instance}}: {{version}}.{{group}}.{{kind}}", 12512 "metric": "", 12513 "query": "sum(irate(apiserver_watch_events_total{kind=~\"(?i:(${resource:regex}))s*\"}[1m])) by (instance, group, version, kind)", 12514 "refId": "", 12515 "step": 10, 12516 "target": "" 12517 } 12518 ], 12519 "thresholds": [], 12520 "timeFrom": null, 12521 "timeShift": "$timeshift", 12522 "title": "[SECONDARY] Watch events rate", 12523 "tooltip": { 12524 "msResolution": true, 12525 "shared": true, 12526 "sort": 2, 12527 "value_type": "cumulative" 12528 }, 12529 "transformations": [], 12530 "transparent": false, 12531 "type": "graph", 12532 "xaxis": { 12533 "mode": "time", 12534 "name": null, 12535 "show": true, 12536 "values": [] 12537 }, 12538 "yaxes": [ 12539 { 12540 "decimals": null, 12541 "format": "short", 12542 "label": null, 12543 "logBase": 1, 12544 "max": null, 12545 "min": null, 12546 "show": true 12547 }, 12548 { 12549 "decimals": null, 12550 "format": "short", 12551 "label": null, 12552 "logBase": 1, 12553 "max": null, 12554 "min": null, 12555 "show": true 12556 } 12557 ], 12558 "yaxis": { 12559 "align": false, 12560 "alignLevel": 0 12561 } 12562 }, 12563 { 12564 "aliasColors": {}, 12565 "bars": false, 12566 "cacheTimeout": null, 12567 "datasource": "$source", 12568 "description": null, 12569 "editable": true, 12570 "error": false, 12571 "fieldConfig": { 12572 "defaults": { 12573 "thresholds": { 12574 "mode": "absolute", 12575 "steps": [] 12576 } 12577 } 12578 }, 12579 "fill": 1, 12580 "grid": { 12581 "threshold1": null, 12582 "threshold1Color": "rgba(216, 200, 27, 0.27)", 12583 "threshold2": null, 12584 "threshold2Color": "rgba(234, 112, 112, 0.22)" 12585 }, 12586 "gridPos": null, 12587 "height": null, 12588 "hideTimeOverride": false, 12589 "id": 97, 12590 "interval": null, 12591 "isNew": true, 12592 "legend": { 12593 "alignAsTable": false, 12594 "avg": false, 12595 "current": false, 12596 "hideEmpty": false, 12597 "hideZero": false, 12598 "max": false, 12599 "min": false, 12600 "rightSide": false, 12601 "show": true, 12602 "sideWidth": null, 12603 "sort": null, 12604 "sortDesc": false, 12605 "total": false, 12606 "values": false 12607 }, 12608 "lines": true, 12609 "linewidth": 2, 12610 "links": [], 12611 "maxDataPoints": 100, 12612 "maxPerRow": null, 12613 "minSpan": null, 12614 "nullPointMode": null, 12615 "options": { 12616 "alertThreshold": true, 12617 "dataLinks": [] 12618 }, 12619 "percentage": false, 12620 "pointradius": 5, 12621 "points": false, 12622 "renderer": "flot", 12623 "repeat": null, 12624 "repeatDirection": null, 12625 "seriesOverrides": [], 12626 "span": 12, 12627 "stack": false, 12628 "steppedLine": false, 12629 "targets": [ 12630 { 12631 "datasource": "$source", 12632 "expr": "sum(irate(apiserver_watch_events_sizes_sum{kind=~\"(?i:(${resource:regex}))s*\"}[1m])) by (instance, group, version, kind)", 12633 "format": "time_series", 12634 "hide": false, 12635 "instant": false, 12636 "interval": "5s", 12637 "intervalFactor": 1, 12638 "legendFormat": "{{instance}}: {{version}}.{{group}}.{{kind}}", 12639 "metric": "", 12640 "query": "sum(irate(apiserver_watch_events_sizes_sum{kind=~\"(?i:(${resource:regex}))s*\"}[1m])) by (instance, group, version, kind)", 12641 "refId": "", 12642 "step": 10, 12643 "target": "" 12644 } 12645 ], 12646 "thresholds": [], 12647 "timeFrom": null, 12648 "timeShift": null, 12649 "title": "Watch events traffic", 12650 "tooltip": { 12651 "msResolution": true, 12652 "shared": true, 12653 "sort": 2, 12654 "value_type": "cumulative" 12655 }, 12656 "transformations": [], 12657 "transparent": false, 12658 "type": "graph", 12659 "xaxis": { 12660 "mode": "time", 12661 "name": null, 12662 "show": true, 12663 "values": [] 12664 }, 12665 "yaxes": [ 12666 { 12667 "decimals": null, 12668 "format": "Bps", 12669 "label": null, 12670 "logBase": 1, 12671 "max": null, 12672 "min": null, 12673 "show": true 12674 }, 12675 { 12676 "decimals": null, 12677 "format": "short", 12678 "label": null, 12679 "logBase": 1, 12680 "max": null, 12681 "min": null, 12682 "show": true 12683 } 12684 ], 12685 "yaxis": { 12686 "align": false, 12687 "alignLevel": 0 12688 } 12689 }, 12690 { 12691 "aliasColors": {}, 12692 "bars": false, 12693 "cacheTimeout": null, 12694 "datasource": "$secondary_source", 12695 "description": null, 12696 "editable": true, 12697 "error": false, 12698 "fieldConfig": { 12699 "defaults": { 12700 "thresholds": { 12701 "mode": "absolute", 12702 "steps": [] 12703 } 12704 } 12705 }, 12706 "fill": 1, 12707 "grid": { 12708 "threshold1": null, 12709 "threshold1Color": "rgba(216, 200, 27, 0.27)", 12710 "threshold2": null, 12711 "threshold2Color": "rgba(234, 112, 112, 0.22)" 12712 }, 12713 "gridPos": null, 12714 "height": null, 12715 "hideTimeOverride": false, 12716 "id": 98, 12717 "interval": null, 12718 "isNew": true, 12719 "legend": { 12720 "alignAsTable": false, 12721 "avg": false, 12722 "current": false, 12723 "hideEmpty": false, 12724 "hideZero": false, 12725 "max": false, 12726 "min": false, 12727 "rightSide": false, 12728 "show": true, 12729 "sideWidth": null, 12730 "sort": null, 12731 "sortDesc": false, 12732 "total": false, 12733 "values": false 12734 }, 12735 "lines": true, 12736 "linewidth": 2, 12737 "links": [], 12738 "maxDataPoints": 100, 12739 "maxPerRow": null, 12740 "minSpan": null, 12741 "nullPointMode": null, 12742 "options": { 12743 "alertThreshold": true, 12744 "dataLinks": [] 12745 }, 12746 "percentage": false, 12747 "pointradius": 5, 12748 "points": false, 12749 "renderer": "flot", 12750 "repeat": null, 12751 "repeatDirection": null, 12752 "seriesOverrides": [], 12753 "span": 12, 12754 "stack": false, 12755 "steppedLine": false, 12756 "targets": [ 12757 { 12758 "datasource": "$source", 12759 "expr": "sum(irate(apiserver_watch_events_sizes_sum{kind=~\"(?i:(${resource:regex}))s*\"}[1m])) by (instance, group, version, kind)", 12760 "format": "time_series", 12761 "hide": false, 12762 "instant": false, 12763 "interval": "5s", 12764 "intervalFactor": 1, 12765 "legendFormat": "{{instance}}: {{version}}.{{group}}.{{kind}}", 12766 "metric": "", 12767 "query": "sum(irate(apiserver_watch_events_sizes_sum{kind=~\"(?i:(${resource:regex}))s*\"}[1m])) by (instance, group, version, kind)", 12768 "refId": "", 12769 "step": 10, 12770 "target": "" 12771 } 12772 ], 12773 "thresholds": [], 12774 "timeFrom": null, 12775 "timeShift": "$timeshift", 12776 "title": "[SECONDARY] Watch events traffic", 12777 "tooltip": { 12778 "msResolution": true, 12779 "shared": true, 12780 "sort": 2, 12781 "value_type": "cumulative" 12782 }, 12783 "transformations": [], 12784 "transparent": false, 12785 "type": "graph", 12786 "xaxis": { 12787 "mode": "time", 12788 "name": null, 12789 "show": true, 12790 "values": [] 12791 }, 12792 "yaxes": [ 12793 { 12794 "decimals": null, 12795 "format": "Bps", 12796 "label": null, 12797 "logBase": 1, 12798 "max": null, 12799 "min": null, 12800 "show": true 12801 }, 12802 { 12803 "decimals": null, 12804 "format": "short", 12805 "label": null, 12806 "logBase": 1, 12807 "max": null, 12808 "min": null, 12809 "show": true 12810 } 12811 ], 12812 "yaxis": { 12813 "align": false, 12814 "alignLevel": 0 12815 } 12816 }, 12817 { 12818 "aliasColors": {}, 12819 "bars": false, 12820 "cacheTimeout": null, 12821 "datasource": "$source", 12822 "description": null, 12823 "editable": true, 12824 "error": false, 12825 "fieldConfig": { 12826 "defaults": { 12827 "thresholds": { 12828 "mode": "absolute", 12829 "steps": [] 12830 } 12831 } 12832 }, 12833 "fill": 1, 12834 "grid": { 12835 "threshold1": null, 12836 "threshold1Color": "rgba(216, 200, 27, 0.27)", 12837 "threshold2": null, 12838 "threshold2Color": "rgba(234, 112, 112, 0.22)" 12839 }, 12840 "gridPos": null, 12841 "height": null, 12842 "hideTimeOverride": false, 12843 "id": 99, 12844 "interval": null, 12845 "isNew": true, 12846 "legend": { 12847 "alignAsTable": false, 12848 "avg": false, 12849 "current": false, 12850 "hideEmpty": false, 12851 "hideZero": false, 12852 "max": false, 12853 "min": false, 12854 "rightSide": false, 12855 "show": true, 12856 "sideWidth": null, 12857 "sort": null, 12858 "sortDesc": false, 12859 "total": false, 12860 "values": false 12861 }, 12862 "lines": true, 12863 "linewidth": 2, 12864 "links": [], 12865 "maxDataPoints": 100, 12866 "maxPerRow": null, 12867 "minSpan": null, 12868 "nullPointMode": null, 12869 "options": { 12870 "alertThreshold": true, 12871 "dataLinks": [] 12872 }, 12873 "percentage": false, 12874 "pointradius": 5, 12875 "points": false, 12876 "renderer": "flot", 12877 "repeat": null, 12878 "repeatDirection": null, 12879 "seriesOverrides": [], 12880 "span": 12, 12881 "stack": false, 12882 "steppedLine": false, 12883 "targets": [ 12884 { 12885 "datasource": "$source", 12886 "expr": "sum(rate(apiserver_watch_events_sizes_sum{kind=~\"(?i:(${resource:regex}))s*\"}[1m])/rate(apiserver_watch_events_sizes_count{kind=~\"(?i:(${resource:regex}))s*\"}[1m])) by (instance, group, version, kind)", 12887 "format": "time_series", 12888 "hide": false, 12889 "instant": false, 12890 "interval": "5s", 12891 "intervalFactor": 1, 12892 "legendFormat": "{{instance}}: {{version}}.{{group}}.{{kind}}", 12893 "metric": "", 12894 "query": "sum(rate(apiserver_watch_events_sizes_sum{kind=~\"(?i:(${resource:regex}))s*\"}[1m])/rate(apiserver_watch_events_sizes_count{kind=~\"(?i:(${resource:regex}))s*\"}[1m])) by (instance, group, version, kind)", 12895 "refId": "", 12896 "step": 10, 12897 "target": "" 12898 } 12899 ], 12900 "thresholds": [], 12901 "timeFrom": null, 12902 "timeShift": null, 12903 "title": "Watch event avg size", 12904 "tooltip": { 12905 "msResolution": true, 12906 "shared": true, 12907 "sort": 2, 12908 "value_type": "cumulative" 12909 }, 12910 "transformations": [], 12911 "transparent": false, 12912 "type": "graph", 12913 "xaxis": { 12914 "mode": "time", 12915 "name": null, 12916 "show": true, 12917 "values": [] 12918 }, 12919 "yaxes": [ 12920 { 12921 "decimals": null, 12922 "format": "short", 12923 "label": null, 12924 "logBase": 1, 12925 "max": null, 12926 "min": null, 12927 "show": true 12928 }, 12929 { 12930 "decimals": null, 12931 "format": "short", 12932 "label": null, 12933 "logBase": 1, 12934 "max": null, 12935 "min": null, 12936 "show": true 12937 } 12938 ], 12939 "yaxis": { 12940 "align": false, 12941 "alignLevel": 0 12942 } 12943 }, 12944 { 12945 "aliasColors": {}, 12946 "bars": false, 12947 "cacheTimeout": null, 12948 "datasource": "$secondary_source", 12949 "description": null, 12950 "editable": true, 12951 "error": false, 12952 "fieldConfig": { 12953 "defaults": { 12954 "thresholds": { 12955 "mode": "absolute", 12956 "steps": [] 12957 } 12958 } 12959 }, 12960 "fill": 1, 12961 "grid": { 12962 "threshold1": null, 12963 "threshold1Color": "rgba(216, 200, 27, 0.27)", 12964 "threshold2": null, 12965 "threshold2Color": "rgba(234, 112, 112, 0.22)" 12966 }, 12967 "gridPos": null, 12968 "height": null, 12969 "hideTimeOverride": false, 12970 "id": 100, 12971 "interval": null, 12972 "isNew": true, 12973 "legend": { 12974 "alignAsTable": false, 12975 "avg": false, 12976 "current": false, 12977 "hideEmpty": false, 12978 "hideZero": false, 12979 "max": false, 12980 "min": false, 12981 "rightSide": false, 12982 "show": true, 12983 "sideWidth": null, 12984 "sort": null, 12985 "sortDesc": false, 12986 "total": false, 12987 "values": false 12988 }, 12989 "lines": true, 12990 "linewidth": 2, 12991 "links": [], 12992 "maxDataPoints": 100, 12993 "maxPerRow": null, 12994 "minSpan": null, 12995 "nullPointMode": null, 12996 "options": { 12997 "alertThreshold": true, 12998 "dataLinks": [] 12999 }, 13000 "percentage": false, 13001 "pointradius": 5, 13002 "points": false, 13003 "renderer": "flot", 13004 "repeat": null, 13005 "repeatDirection": null, 13006 "seriesOverrides": [], 13007 "span": 12, 13008 "stack": false, 13009 "steppedLine": false, 13010 "targets": [ 13011 { 13012 "datasource": "$source", 13013 "expr": "sum(rate(apiserver_watch_events_sizes_sum{kind=~\"(?i:(${resource:regex}))s*\"}[1m])/rate(apiserver_watch_events_sizes_count{kind=~\"(?i:(${resource:regex}))s*\"}[1m])) by (instance, group, version, kind)", 13014 "format": "time_series", 13015 "hide": false, 13016 "instant": false, 13017 "interval": "5s", 13018 "intervalFactor": 1, 13019 "legendFormat": "{{instance}}: {{version}}.{{group}}.{{kind}}", 13020 "metric": "", 13021 "query": "sum(rate(apiserver_watch_events_sizes_sum{kind=~\"(?i:(${resource:regex}))s*\"}[1m])/rate(apiserver_watch_events_sizes_count{kind=~\"(?i:(${resource:regex}))s*\"}[1m])) by (instance, group, version, kind)", 13022 "refId": "", 13023 "step": 10, 13024 "target": "" 13025 } 13026 ], 13027 "thresholds": [], 13028 "timeFrom": null, 13029 "timeShift": "$timeshift", 13030 "title": "[SECONDARY] Watch event avg size", 13031 "tooltip": { 13032 "msResolution": true, 13033 "shared": true, 13034 "sort": 2, 13035 "value_type": "cumulative" 13036 }, 13037 "transformations": [], 13038 "transparent": false, 13039 "type": "graph", 13040 "xaxis": { 13041 "mode": "time", 13042 "name": null, 13043 "show": true, 13044 "values": [] 13045 }, 13046 "yaxes": [ 13047 { 13048 "decimals": null, 13049 "format": "short", 13050 "label": null, 13051 "logBase": 1, 13052 "max": null, 13053 "min": null, 13054 "show": true 13055 }, 13056 { 13057 "decimals": null, 13058 "format": "short", 13059 "label": null, 13060 "logBase": 1, 13061 "max": null, 13062 "min": null, 13063 "show": true 13064 } 13065 ], 13066 "yaxis": { 13067 "align": false, 13068 "alignLevel": 0 13069 } 13070 }, 13071 { 13072 "aliasColors": {}, 13073 "bars": false, 13074 "cacheTimeout": null, 13075 "datasource": "$source", 13076 "description": null, 13077 "editable": true, 13078 "error": false, 13079 "fieldConfig": { 13080 "defaults": { 13081 "thresholds": { 13082 "mode": "absolute", 13083 "steps": [] 13084 } 13085 } 13086 }, 13087 "fill": 1, 13088 "grid": { 13089 "threshold1": null, 13090 "threshold1Color": "rgba(216, 200, 27, 0.27)", 13091 "threshold2": null, 13092 "threshold2Color": "rgba(234, 112, 112, 0.22)" 13093 }, 13094 "gridPos": null, 13095 "height": null, 13096 "hideTimeOverride": false, 13097 "id": 101, 13098 "interval": null, 13099 "isNew": true, 13100 "legend": { 13101 "alignAsTable": false, 13102 "avg": false, 13103 "current": false, 13104 "hideEmpty": false, 13105 "hideZero": false, 13106 "max": false, 13107 "min": false, 13108 "rightSide": false, 13109 "show": true, 13110 "sideWidth": null, 13111 "sort": null, 13112 "sortDesc": false, 13113 "total": false, 13114 "values": false 13115 }, 13116 "lines": true, 13117 "linewidth": 2, 13118 "links": [], 13119 "maxDataPoints": 100, 13120 "maxPerRow": null, 13121 "minSpan": null, 13122 "nullPointMode": null, 13123 "options": { 13124 "alertThreshold": true, 13125 "dataLinks": [] 13126 }, 13127 "percentage": false, 13128 "pointradius": 5, 13129 "points": false, 13130 "renderer": "flot", 13131 "repeat": null, 13132 "repeatDirection": null, 13133 "seriesOverrides": [], 13134 "span": 12, 13135 "stack": false, 13136 "steppedLine": false, 13137 "targets": [ 13138 { 13139 "datasource": "$source", 13140 "expr": "sum(rate(apiserver_terminated_watchers_total{}[1m])) by (resource, instance)", 13141 "format": "time_series", 13142 "hide": false, 13143 "instant": false, 13144 "interval": "5s", 13145 "intervalFactor": 1, 13146 "legendFormat": "{{instance}}: {{resource}}", 13147 "metric": "", 13148 "query": "sum(rate(apiserver_terminated_watchers_total{}[1m])) by (resource, instance)", 13149 "refId": "", 13150 "step": 10, 13151 "target": "" 13152 } 13153 ], 13154 "thresholds": [], 13155 "timeFrom": null, 13156 "timeShift": null, 13157 "title": "Watch terminated total", 13158 "tooltip": { 13159 "msResolution": true, 13160 "shared": true, 13161 "sort": 2, 13162 "value_type": "cumulative" 13163 }, 13164 "transformations": [], 13165 "transparent": false, 13166 "type": "graph", 13167 "xaxis": { 13168 "mode": "time", 13169 "name": null, 13170 "show": true, 13171 "values": [] 13172 }, 13173 "yaxes": [ 13174 { 13175 "decimals": null, 13176 "format": "short", 13177 "label": null, 13178 "logBase": 1, 13179 "max": null, 13180 "min": null, 13181 "show": true 13182 }, 13183 { 13184 "decimals": null, 13185 "format": "short", 13186 "label": null, 13187 "logBase": 1, 13188 "max": null, 13189 "min": null, 13190 "show": true 13191 } 13192 ], 13193 "yaxis": { 13194 "align": false, 13195 "alignLevel": 0 13196 } 13197 }, 13198 { 13199 "aliasColors": {}, 13200 "bars": false, 13201 "cacheTimeout": null, 13202 "datasource": "$secondary_source", 13203 "description": null, 13204 "editable": true, 13205 "error": false, 13206 "fieldConfig": { 13207 "defaults": { 13208 "thresholds": { 13209 "mode": "absolute", 13210 "steps": [] 13211 } 13212 } 13213 }, 13214 "fill": 1, 13215 "grid": { 13216 "threshold1": null, 13217 "threshold1Color": "rgba(216, 200, 27, 0.27)", 13218 "threshold2": null, 13219 "threshold2Color": "rgba(234, 112, 112, 0.22)" 13220 }, 13221 "gridPos": null, 13222 "height": null, 13223 "hideTimeOverride": false, 13224 "id": 102, 13225 "interval": null, 13226 "isNew": true, 13227 "legend": { 13228 "alignAsTable": false, 13229 "avg": false, 13230 "current": false, 13231 "hideEmpty": false, 13232 "hideZero": false, 13233 "max": false, 13234 "min": false, 13235 "rightSide": false, 13236 "show": true, 13237 "sideWidth": null, 13238 "sort": null, 13239 "sortDesc": false, 13240 "total": false, 13241 "values": false 13242 }, 13243 "lines": true, 13244 "linewidth": 2, 13245 "links": [], 13246 "maxDataPoints": 100, 13247 "maxPerRow": null, 13248 "minSpan": null, 13249 "nullPointMode": null, 13250 "options": { 13251 "alertThreshold": true, 13252 "dataLinks": [] 13253 }, 13254 "percentage": false, 13255 "pointradius": 5, 13256 "points": false, 13257 "renderer": "flot", 13258 "repeat": null, 13259 "repeatDirection": null, 13260 "seriesOverrides": [], 13261 "span": 12, 13262 "stack": false, 13263 "steppedLine": false, 13264 "targets": [ 13265 { 13266 "datasource": "$source", 13267 "expr": "sum(rate(apiserver_terminated_watchers_total{}[1m])) by (resource, instance)", 13268 "format": "time_series", 13269 "hide": false, 13270 "instant": false, 13271 "interval": "5s", 13272 "intervalFactor": 1, 13273 "legendFormat": "{{instance}}: {{resource}}", 13274 "metric": "", 13275 "query": "sum(rate(apiserver_terminated_watchers_total{}[1m])) by (resource, instance)", 13276 "refId": "", 13277 "step": 10, 13278 "target": "" 13279 } 13280 ], 13281 "thresholds": [], 13282 "timeFrom": null, 13283 "timeShift": "$timeshift", 13284 "title": "[SECONDARY] Watch terminated total", 13285 "tooltip": { 13286 "msResolution": true, 13287 "shared": true, 13288 "sort": 2, 13289 "value_type": "cumulative" 13290 }, 13291 "transformations": [], 13292 "transparent": false, 13293 "type": "graph", 13294 "xaxis": { 13295 "mode": "time", 13296 "name": null, 13297 "show": true, 13298 "values": [] 13299 }, 13300 "yaxes": [ 13301 { 13302 "decimals": null, 13303 "format": "short", 13304 "label": null, 13305 "logBase": 1, 13306 "max": null, 13307 "min": null, 13308 "show": true 13309 }, 13310 { 13311 "decimals": null, 13312 "format": "short", 13313 "label": null, 13314 "logBase": 1, 13315 "max": null, 13316 "min": null, 13317 "show": true 13318 } 13319 ], 13320 "yaxis": { 13321 "align": false, 13322 "alignLevel": 0 13323 } 13324 }, 13325 { 13326 "aliasColors": {}, 13327 "bars": false, 13328 "cacheTimeout": null, 13329 "datasource": "$source", 13330 "description": null, 13331 "editable": true, 13332 "error": false, 13333 "fieldConfig": { 13334 "defaults": { 13335 "thresholds": { 13336 "mode": "absolute", 13337 "steps": [] 13338 } 13339 } 13340 }, 13341 "fill": 1, 13342 "grid": { 13343 "threshold1": null, 13344 "threshold1Color": "rgba(216, 200, 27, 0.27)", 13345 "threshold2": null, 13346 "threshold2Color": "rgba(234, 112, 112, 0.22)" 13347 }, 13348 "gridPos": null, 13349 "height": null, 13350 "hideTimeOverride": false, 13351 "id": 103, 13352 "interval": null, 13353 "isNew": true, 13354 "legend": { 13355 "alignAsTable": false, 13356 "avg": false, 13357 "current": false, 13358 "hideEmpty": false, 13359 "hideZero": false, 13360 "max": false, 13361 "min": false, 13362 "rightSide": false, 13363 "show": true, 13364 "sideWidth": null, 13365 "sort": null, 13366 "sortDesc": false, 13367 "total": false, 13368 "values": false 13369 }, 13370 "lines": true, 13371 "linewidth": 2, 13372 "links": [], 13373 "maxDataPoints": 100, 13374 "maxPerRow": null, 13375 "minSpan": null, 13376 "nullPointMode": null, 13377 "options": { 13378 "alertThreshold": true, 13379 "dataLinks": [] 13380 }, 13381 "percentage": false, 13382 "pointradius": 5, 13383 "points": false, 13384 "renderer": "flot", 13385 "repeat": null, 13386 "repeatDirection": null, 13387 "seriesOverrides": [], 13388 "span": 12, 13389 "stack": false, 13390 "steppedLine": false, 13391 "targets": [ 13392 { 13393 "datasource": "$source", 13394 "expr": "sum(apiserver_current_inflight_requests) by (requestKind, instance)", 13395 "format": "time_series", 13396 "hide": false, 13397 "instant": false, 13398 "interval": "5s", 13399 "intervalFactor": 1, 13400 "legendFormat": "{{instance}}: {{requestKind}}", 13401 "metric": "", 13402 "query": "sum(apiserver_current_inflight_requests) by (requestKind, instance)", 13403 "refId": "", 13404 "step": 10, 13405 "target": "" 13406 } 13407 ], 13408 "thresholds": [], 13409 "timeFrom": null, 13410 "timeShift": null, 13411 "title": "Inflight requests", 13412 "tooltip": { 13413 "msResolution": true, 13414 "shared": true, 13415 "sort": 2, 13416 "value_type": "cumulative" 13417 }, 13418 "transformations": [], 13419 "transparent": false, 13420 "type": "graph", 13421 "xaxis": { 13422 "mode": "time", 13423 "name": null, 13424 "show": true, 13425 "values": [] 13426 }, 13427 "yaxes": [ 13428 { 13429 "decimals": null, 13430 "format": "short", 13431 "label": null, 13432 "logBase": 1, 13433 "max": null, 13434 "min": null, 13435 "show": true 13436 }, 13437 { 13438 "decimals": null, 13439 "format": "short", 13440 "label": null, 13441 "logBase": 1, 13442 "max": null, 13443 "min": null, 13444 "show": true 13445 } 13446 ], 13447 "yaxis": { 13448 "align": false, 13449 "alignLevel": 0 13450 } 13451 }, 13452 { 13453 "aliasColors": {}, 13454 "bars": false, 13455 "cacheTimeout": null, 13456 "datasource": "$secondary_source", 13457 "description": null, 13458 "editable": true, 13459 "error": false, 13460 "fieldConfig": { 13461 "defaults": { 13462 "thresholds": { 13463 "mode": "absolute", 13464 "steps": [] 13465 } 13466 } 13467 }, 13468 "fill": 1, 13469 "grid": { 13470 "threshold1": null, 13471 "threshold1Color": "rgba(216, 200, 27, 0.27)", 13472 "threshold2": null, 13473 "threshold2Color": "rgba(234, 112, 112, 0.22)" 13474 }, 13475 "gridPos": null, 13476 "height": null, 13477 "hideTimeOverride": false, 13478 "id": 104, 13479 "interval": null, 13480 "isNew": true, 13481 "legend": { 13482 "alignAsTable": false, 13483 "avg": false, 13484 "current": false, 13485 "hideEmpty": false, 13486 "hideZero": false, 13487 "max": false, 13488 "min": false, 13489 "rightSide": false, 13490 "show": true, 13491 "sideWidth": null, 13492 "sort": null, 13493 "sortDesc": false, 13494 "total": false, 13495 "values": false 13496 }, 13497 "lines": true, 13498 "linewidth": 2, 13499 "links": [], 13500 "maxDataPoints": 100, 13501 "maxPerRow": null, 13502 "minSpan": null, 13503 "nullPointMode": null, 13504 "options": { 13505 "alertThreshold": true, 13506 "dataLinks": [] 13507 }, 13508 "percentage": false, 13509 "pointradius": 5, 13510 "points": false, 13511 "renderer": "flot", 13512 "repeat": null, 13513 "repeatDirection": null, 13514 "seriesOverrides": [], 13515 "span": 12, 13516 "stack": false, 13517 "steppedLine": false, 13518 "targets": [ 13519 { 13520 "datasource": "$source", 13521 "expr": "sum(apiserver_current_inflight_requests) by (requestKind, instance)", 13522 "format": "time_series", 13523 "hide": false, 13524 "instant": false, 13525 "interval": "5s", 13526 "intervalFactor": 1, 13527 "legendFormat": "{{instance}}: {{requestKind}}", 13528 "metric": "", 13529 "query": "sum(apiserver_current_inflight_requests) by (requestKind, instance)", 13530 "refId": "", 13531 "step": 10, 13532 "target": "" 13533 } 13534 ], 13535 "thresholds": [], 13536 "timeFrom": null, 13537 "timeShift": "$timeshift", 13538 "title": "[SECONDARY] Inflight requests", 13539 "tooltip": { 13540 "msResolution": true, 13541 "shared": true, 13542 "sort": 2, 13543 "value_type": "cumulative" 13544 }, 13545 "transformations": [], 13546 "transparent": false, 13547 "type": "graph", 13548 "xaxis": { 13549 "mode": "time", 13550 "name": null, 13551 "show": true, 13552 "values": [] 13553 }, 13554 "yaxes": [ 13555 { 13556 "decimals": null, 13557 "format": "short", 13558 "label": null, 13559 "logBase": 1, 13560 "max": null, 13561 "min": null, 13562 "show": true 13563 }, 13564 { 13565 "decimals": null, 13566 "format": "short", 13567 "label": null, 13568 "logBase": 1, 13569 "max": null, 13570 "min": null, 13571 "show": true 13572 } 13573 ], 13574 "yaxis": { 13575 "align": false, 13576 "alignLevel": 0 13577 } 13578 }, 13579 { 13580 "aliasColors": {}, 13581 "bars": false, 13582 "cacheTimeout": null, 13583 "datasource": "$source", 13584 "description": null, 13585 "editable": true, 13586 "error": false, 13587 "fieldConfig": { 13588 "defaults": { 13589 "thresholds": { 13590 "mode": "absolute", 13591 "steps": [] 13592 } 13593 } 13594 }, 13595 "fill": 1, 13596 "grid": { 13597 "threshold1": null, 13598 "threshold1Color": "rgba(216, 200, 27, 0.27)", 13599 "threshold2": null, 13600 "threshold2Color": "rgba(234, 112, 112, 0.22)" 13601 }, 13602 "gridPos": null, 13603 "height": null, 13604 "hideTimeOverride": false, 13605 "id": 105, 13606 "interval": null, 13607 "isNew": true, 13608 "legend": { 13609 "alignAsTable": false, 13610 "avg": false, 13611 "current": false, 13612 "hideEmpty": false, 13613 "hideZero": false, 13614 "max": false, 13615 "min": false, 13616 "rightSide": false, 13617 "show": true, 13618 "sideWidth": null, 13619 "sort": null, 13620 "sortDesc": false, 13621 "total": false, 13622 "values": false 13623 }, 13624 "lines": true, 13625 "linewidth": 2, 13626 "links": [], 13627 "maxDataPoints": 100, 13628 "maxPerRow": null, 13629 "minSpan": null, 13630 "nullPointMode": null, 13631 "options": { 13632 "alertThreshold": true, 13633 "dataLinks": [] 13634 }, 13635 "percentage": false, 13636 "pointradius": 5, 13637 "points": false, 13638 "renderer": "flot", 13639 "repeat": null, 13640 "repeatDirection": null, 13641 "seriesOverrides": [], 13642 "span": 12, 13643 "stack": false, 13644 "steppedLine": false, 13645 "targets": [ 13646 { 13647 "datasource": "$source", 13648 "expr": "sum(rate(apiserver_request_total{verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}[1m])) by (verb, resource, subresource, instance)", 13649 "format": "time_series", 13650 "hide": false, 13651 "instant": false, 13652 "interval": "5s", 13653 "intervalFactor": 1, 13654 "legendFormat": "", 13655 "metric": "", 13656 "query": "sum(rate(apiserver_request_total{verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}[1m])) by (verb, resource, subresource, instance)", 13657 "refId": "", 13658 "step": 10, 13659 "target": "" 13660 } 13661 ], 13662 "thresholds": [], 13663 "timeFrom": null, 13664 "timeShift": null, 13665 "title": "Request rate", 13666 "tooltip": { 13667 "msResolution": true, 13668 "shared": true, 13669 "sort": 2, 13670 "value_type": "cumulative" 13671 }, 13672 "transformations": [], 13673 "transparent": false, 13674 "type": "graph", 13675 "xaxis": { 13676 "mode": "time", 13677 "name": null, 13678 "show": true, 13679 "values": [] 13680 }, 13681 "yaxes": [ 13682 { 13683 "decimals": null, 13684 "format": "short", 13685 "label": null, 13686 "logBase": 1, 13687 "max": null, 13688 "min": null, 13689 "show": true 13690 }, 13691 { 13692 "decimals": null, 13693 "format": "short", 13694 "label": null, 13695 "logBase": 1, 13696 "max": null, 13697 "min": null, 13698 "show": true 13699 } 13700 ], 13701 "yaxis": { 13702 "align": false, 13703 "alignLevel": 0 13704 } 13705 }, 13706 { 13707 "aliasColors": {}, 13708 "bars": false, 13709 "cacheTimeout": null, 13710 "datasource": "$secondary_source", 13711 "description": null, 13712 "editable": true, 13713 "error": false, 13714 "fieldConfig": { 13715 "defaults": { 13716 "thresholds": { 13717 "mode": "absolute", 13718 "steps": [] 13719 } 13720 } 13721 }, 13722 "fill": 1, 13723 "grid": { 13724 "threshold1": null, 13725 "threshold1Color": "rgba(216, 200, 27, 0.27)", 13726 "threshold2": null, 13727 "threshold2Color": "rgba(234, 112, 112, 0.22)" 13728 }, 13729 "gridPos": null, 13730 "height": null, 13731 "hideTimeOverride": false, 13732 "id": 106, 13733 "interval": null, 13734 "isNew": true, 13735 "legend": { 13736 "alignAsTable": false, 13737 "avg": false, 13738 "current": false, 13739 "hideEmpty": false, 13740 "hideZero": false, 13741 "max": false, 13742 "min": false, 13743 "rightSide": false, 13744 "show": true, 13745 "sideWidth": null, 13746 "sort": null, 13747 "sortDesc": false, 13748 "total": false, 13749 "values": false 13750 }, 13751 "lines": true, 13752 "linewidth": 2, 13753 "links": [], 13754 "maxDataPoints": 100, 13755 "maxPerRow": null, 13756 "minSpan": null, 13757 "nullPointMode": null, 13758 "options": { 13759 "alertThreshold": true, 13760 "dataLinks": [] 13761 }, 13762 "percentage": false, 13763 "pointradius": 5, 13764 "points": false, 13765 "renderer": "flot", 13766 "repeat": null, 13767 "repeatDirection": null, 13768 "seriesOverrides": [], 13769 "span": 12, 13770 "stack": false, 13771 "steppedLine": false, 13772 "targets": [ 13773 { 13774 "datasource": "$source", 13775 "expr": "sum(rate(apiserver_request_total{verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}[1m])) by (verb, resource, subresource, instance)", 13776 "format": "time_series", 13777 "hide": false, 13778 "instant": false, 13779 "interval": "5s", 13780 "intervalFactor": 1, 13781 "legendFormat": "", 13782 "metric": "", 13783 "query": "sum(rate(apiserver_request_total{verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}[1m])) by (verb, resource, subresource, instance)", 13784 "refId": "", 13785 "step": 10, 13786 "target": "" 13787 } 13788 ], 13789 "thresholds": [], 13790 "timeFrom": null, 13791 "timeShift": "$timeshift", 13792 "title": "[SECONDARY] Request rate", 13793 "tooltip": { 13794 "msResolution": true, 13795 "shared": true, 13796 "sort": 2, 13797 "value_type": "cumulative" 13798 }, 13799 "transformations": [], 13800 "transparent": false, 13801 "type": "graph", 13802 "xaxis": { 13803 "mode": "time", 13804 "name": null, 13805 "show": true, 13806 "values": [] 13807 }, 13808 "yaxes": [ 13809 { 13810 "decimals": null, 13811 "format": "short", 13812 "label": null, 13813 "logBase": 1, 13814 "max": null, 13815 "min": null, 13816 "show": true 13817 }, 13818 { 13819 "decimals": null, 13820 "format": "short", 13821 "label": null, 13822 "logBase": 1, 13823 "max": null, 13824 "min": null, 13825 "show": true 13826 } 13827 ], 13828 "yaxis": { 13829 "align": false, 13830 "alignLevel": 0 13831 } 13832 }, 13833 { 13834 "aliasColors": {}, 13835 "bars": false, 13836 "cacheTimeout": null, 13837 "datasource": "$source", 13838 "description": null, 13839 "editable": true, 13840 "error": false, 13841 "fieldConfig": { 13842 "defaults": { 13843 "thresholds": { 13844 "mode": "absolute", 13845 "steps": [] 13846 } 13847 } 13848 }, 13849 "fill": 1, 13850 "grid": { 13851 "threshold1": null, 13852 "threshold1Color": "rgba(216, 200, 27, 0.27)", 13853 "threshold2": null, 13854 "threshold2Color": "rgba(234, 112, 112, 0.22)" 13855 }, 13856 "gridPos": null, 13857 "height": null, 13858 "hideTimeOverride": false, 13859 "id": 107, 13860 "interval": null, 13861 "isNew": true, 13862 "legend": { 13863 "alignAsTable": false, 13864 "avg": false, 13865 "current": false, 13866 "hideEmpty": false, 13867 "hideZero": false, 13868 "max": false, 13869 "min": false, 13870 "rightSide": false, 13871 "show": true, 13872 "sideWidth": null, 13873 "sort": null, 13874 "sortDesc": false, 13875 "total": false, 13876 "values": false 13877 }, 13878 "lines": true, 13879 "linewidth": 2, 13880 "links": [], 13881 "maxDataPoints": 100, 13882 "maxPerRow": null, 13883 "minSpan": null, 13884 "nullPointMode": null, 13885 "options": { 13886 "alertThreshold": true, 13887 "dataLinks": [] 13888 }, 13889 "percentage": false, 13890 "pointradius": 5, 13891 "points": false, 13892 "renderer": "flot", 13893 "repeat": null, 13894 "repeatDirection": null, 13895 "seriesOverrides": [], 13896 "span": 12, 13897 "stack": false, 13898 "steppedLine": false, 13899 "targets": [ 13900 { 13901 "datasource": "$source", 13902 "expr": "sum(rate(apiserver_request_total[1m])) by (code, instance)", 13903 "format": "time_series", 13904 "hide": false, 13905 "instant": false, 13906 "interval": "5s", 13907 "intervalFactor": 1, 13908 "legendFormat": "{{instance}}: {{code}}", 13909 "metric": "", 13910 "query": "sum(rate(apiserver_request_total[1m])) by (code, instance)", 13911 "refId": "", 13912 "step": 10, 13913 "target": "" 13914 } 13915 ], 13916 "thresholds": [], 13917 "timeFrom": null, 13918 "timeShift": null, 13919 "title": "Request rate by code", 13920 "tooltip": { 13921 "msResolution": true, 13922 "shared": true, 13923 "sort": 2, 13924 "value_type": "cumulative" 13925 }, 13926 "transformations": [], 13927 "transparent": false, 13928 "type": "graph", 13929 "xaxis": { 13930 "mode": "time", 13931 "name": null, 13932 "show": true, 13933 "values": [] 13934 }, 13935 "yaxes": [ 13936 { 13937 "decimals": null, 13938 "format": "short", 13939 "label": null, 13940 "logBase": 1, 13941 "max": null, 13942 "min": null, 13943 "show": true 13944 }, 13945 { 13946 "decimals": null, 13947 "format": "short", 13948 "label": null, 13949 "logBase": 1, 13950 "max": null, 13951 "min": null, 13952 "show": true 13953 } 13954 ], 13955 "yaxis": { 13956 "align": false, 13957 "alignLevel": 0 13958 } 13959 }, 13960 { 13961 "aliasColors": {}, 13962 "bars": false, 13963 "cacheTimeout": null, 13964 "datasource": "$secondary_source", 13965 "description": null, 13966 "editable": true, 13967 "error": false, 13968 "fieldConfig": { 13969 "defaults": { 13970 "thresholds": { 13971 "mode": "absolute", 13972 "steps": [] 13973 } 13974 } 13975 }, 13976 "fill": 1, 13977 "grid": { 13978 "threshold1": null, 13979 "threshold1Color": "rgba(216, 200, 27, 0.27)", 13980 "threshold2": null, 13981 "threshold2Color": "rgba(234, 112, 112, 0.22)" 13982 }, 13983 "gridPos": null, 13984 "height": null, 13985 "hideTimeOverride": false, 13986 "id": 108, 13987 "interval": null, 13988 "isNew": true, 13989 "legend": { 13990 "alignAsTable": false, 13991 "avg": false, 13992 "current": false, 13993 "hideEmpty": false, 13994 "hideZero": false, 13995 "max": false, 13996 "min": false, 13997 "rightSide": false, 13998 "show": true, 13999 "sideWidth": null, 14000 "sort": null, 14001 "sortDesc": false, 14002 "total": false, 14003 "values": false 14004 }, 14005 "lines": true, 14006 "linewidth": 2, 14007 "links": [], 14008 "maxDataPoints": 100, 14009 "maxPerRow": null, 14010 "minSpan": null, 14011 "nullPointMode": null, 14012 "options": { 14013 "alertThreshold": true, 14014 "dataLinks": [] 14015 }, 14016 "percentage": false, 14017 "pointradius": 5, 14018 "points": false, 14019 "renderer": "flot", 14020 "repeat": null, 14021 "repeatDirection": null, 14022 "seriesOverrides": [], 14023 "span": 12, 14024 "stack": false, 14025 "steppedLine": false, 14026 "targets": [ 14027 { 14028 "datasource": "$source", 14029 "expr": "sum(rate(apiserver_request_total[1m])) by (code, instance)", 14030 "format": "time_series", 14031 "hide": false, 14032 "instant": false, 14033 "interval": "5s", 14034 "intervalFactor": 1, 14035 "legendFormat": "{{instance}}: {{code}}", 14036 "metric": "", 14037 "query": "sum(rate(apiserver_request_total[1m])) by (code, instance)", 14038 "refId": "", 14039 "step": 10, 14040 "target": "" 14041 } 14042 ], 14043 "thresholds": [], 14044 "timeFrom": null, 14045 "timeShift": "$timeshift", 14046 "title": "[SECONDARY] Request rate by code", 14047 "tooltip": { 14048 "msResolution": true, 14049 "shared": true, 14050 "sort": 2, 14051 "value_type": "cumulative" 14052 }, 14053 "transformations": [], 14054 "transparent": false, 14055 "type": "graph", 14056 "xaxis": { 14057 "mode": "time", 14058 "name": null, 14059 "show": true, 14060 "values": [] 14061 }, 14062 "yaxes": [ 14063 { 14064 "decimals": null, 14065 "format": "short", 14066 "label": null, 14067 "logBase": 1, 14068 "max": null, 14069 "min": null, 14070 "show": true 14071 }, 14072 { 14073 "decimals": null, 14074 "format": "short", 14075 "label": null, 14076 "logBase": 1, 14077 "max": null, 14078 "min": null, 14079 "show": true 14080 } 14081 ], 14082 "yaxis": { 14083 "align": false, 14084 "alignLevel": 0 14085 } 14086 }, 14087 { 14088 "aliasColors": {}, 14089 "bars": false, 14090 "cacheTimeout": null, 14091 "datasource": "$source", 14092 "description": null, 14093 "editable": true, 14094 "error": false, 14095 "fieldConfig": { 14096 "defaults": { 14097 "thresholds": { 14098 "mode": "absolute", 14099 "steps": [] 14100 } 14101 } 14102 }, 14103 "fill": 1, 14104 "grid": { 14105 "threshold1": null, 14106 "threshold1Color": "rgba(216, 200, 27, 0.27)", 14107 "threshold2": null, 14108 "threshold2Color": "rgba(234, 112, 112, 0.22)" 14109 }, 14110 "gridPos": null, 14111 "height": null, 14112 "hideTimeOverride": false, 14113 "id": 109, 14114 "interval": null, 14115 "isNew": true, 14116 "legend": { 14117 "alignAsTable": false, 14118 "avg": false, 14119 "current": false, 14120 "hideEmpty": false, 14121 "hideZero": false, 14122 "max": false, 14123 "min": false, 14124 "rightSide": false, 14125 "show": true, 14126 "sideWidth": null, 14127 "sort": null, 14128 "sortDesc": false, 14129 "total": false, 14130 "values": false 14131 }, 14132 "lines": true, 14133 "linewidth": 2, 14134 "links": [], 14135 "maxDataPoints": 100, 14136 "maxPerRow": null, 14137 "minSpan": null, 14138 "nullPointMode": null, 14139 "options": { 14140 "alertThreshold": true, 14141 "dataLinks": [] 14142 }, 14143 "percentage": false, 14144 "pointradius": 5, 14145 "points": false, 14146 "renderer": "flot", 14147 "repeat": null, 14148 "repeatDirection": null, 14149 "seriesOverrides": [], 14150 "span": 12, 14151 "stack": false, 14152 "steppedLine": false, 14153 "targets": [ 14154 { 14155 "datasource": "$source", 14156 "expr": "apiserver:apiserver_request_latency:histogram_quantile{quantile=\"0.50\", verb!=\"WATCH\", verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}", 14157 "format": "time_series", 14158 "hide": false, 14159 "instant": false, 14160 "interval": "5s", 14161 "intervalFactor": 1, 14162 "legendFormat": "", 14163 "metric": "", 14164 "query": "apiserver:apiserver_request_latency:histogram_quantile{quantile=\"0.50\", verb!=\"WATCH\", verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}", 14165 "refId": "", 14166 "step": 10, 14167 "target": "" 14168 } 14169 ], 14170 "thresholds": [], 14171 "timeFrom": null, 14172 "timeShift": null, 14173 "title": "Request latency (50th percentile) (excl. WATCH)", 14174 "tooltip": { 14175 "msResolution": true, 14176 "shared": true, 14177 "sort": 2, 14178 "value_type": "cumulative" 14179 }, 14180 "transformations": [], 14181 "transparent": false, 14182 "type": "graph", 14183 "xaxis": { 14184 "mode": "time", 14185 "name": null, 14186 "show": true, 14187 "values": [] 14188 }, 14189 "yaxes": [ 14190 { 14191 "decimals": null, 14192 "format": "s", 14193 "label": null, 14194 "logBase": 1, 14195 "max": null, 14196 "min": null, 14197 "show": true 14198 }, 14199 { 14200 "decimals": null, 14201 "format": "short", 14202 "label": null, 14203 "logBase": 1, 14204 "max": null, 14205 "min": null, 14206 "show": true 14207 } 14208 ], 14209 "yaxis": { 14210 "align": false, 14211 "alignLevel": 0 14212 } 14213 }, 14214 { 14215 "aliasColors": {}, 14216 "bars": false, 14217 "cacheTimeout": null, 14218 "datasource": "$secondary_source", 14219 "description": null, 14220 "editable": true, 14221 "error": false, 14222 "fieldConfig": { 14223 "defaults": { 14224 "thresholds": { 14225 "mode": "absolute", 14226 "steps": [] 14227 } 14228 } 14229 }, 14230 "fill": 1, 14231 "grid": { 14232 "threshold1": null, 14233 "threshold1Color": "rgba(216, 200, 27, 0.27)", 14234 "threshold2": null, 14235 "threshold2Color": "rgba(234, 112, 112, 0.22)" 14236 }, 14237 "gridPos": null, 14238 "height": null, 14239 "hideTimeOverride": false, 14240 "id": 110, 14241 "interval": null, 14242 "isNew": true, 14243 "legend": { 14244 "alignAsTable": false, 14245 "avg": false, 14246 "current": false, 14247 "hideEmpty": false, 14248 "hideZero": false, 14249 "max": false, 14250 "min": false, 14251 "rightSide": false, 14252 "show": true, 14253 "sideWidth": null, 14254 "sort": null, 14255 "sortDesc": false, 14256 "total": false, 14257 "values": false 14258 }, 14259 "lines": true, 14260 "linewidth": 2, 14261 "links": [], 14262 "maxDataPoints": 100, 14263 "maxPerRow": null, 14264 "minSpan": null, 14265 "nullPointMode": null, 14266 "options": { 14267 "alertThreshold": true, 14268 "dataLinks": [] 14269 }, 14270 "percentage": false, 14271 "pointradius": 5, 14272 "points": false, 14273 "renderer": "flot", 14274 "repeat": null, 14275 "repeatDirection": null, 14276 "seriesOverrides": [], 14277 "span": 12, 14278 "stack": false, 14279 "steppedLine": false, 14280 "targets": [ 14281 { 14282 "datasource": "$source", 14283 "expr": "apiserver:apiserver_request_latency:histogram_quantile{quantile=\"0.50\", verb!=\"WATCH\", verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}", 14284 "format": "time_series", 14285 "hide": false, 14286 "instant": false, 14287 "interval": "5s", 14288 "intervalFactor": 1, 14289 "legendFormat": "", 14290 "metric": "", 14291 "query": "apiserver:apiserver_request_latency:histogram_quantile{quantile=\"0.50\", verb!=\"WATCH\", verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}", 14292 "refId": "", 14293 "step": 10, 14294 "target": "" 14295 } 14296 ], 14297 "thresholds": [], 14298 "timeFrom": null, 14299 "timeShift": "$timeshift", 14300 "title": "[SECONDARY] Request latency (50th percentile) (excl. WATCH)", 14301 "tooltip": { 14302 "msResolution": true, 14303 "shared": true, 14304 "sort": 2, 14305 "value_type": "cumulative" 14306 }, 14307 "transformations": [], 14308 "transparent": false, 14309 "type": "graph", 14310 "xaxis": { 14311 "mode": "time", 14312 "name": null, 14313 "show": true, 14314 "values": [] 14315 }, 14316 "yaxes": [ 14317 { 14318 "decimals": null, 14319 "format": "s", 14320 "label": null, 14321 "logBase": 1, 14322 "max": null, 14323 "min": null, 14324 "show": true 14325 }, 14326 { 14327 "decimals": null, 14328 "format": "short", 14329 "label": null, 14330 "logBase": 1, 14331 "max": null, 14332 "min": null, 14333 "show": true 14334 } 14335 ], 14336 "yaxis": { 14337 "align": false, 14338 "alignLevel": 0 14339 } 14340 }, 14341 { 14342 "aliasColors": {}, 14343 "bars": false, 14344 "cacheTimeout": null, 14345 "datasource": "$source", 14346 "description": null, 14347 "editable": true, 14348 "error": false, 14349 "fieldConfig": { 14350 "defaults": { 14351 "thresholds": { 14352 "mode": "absolute", 14353 "steps": [] 14354 } 14355 } 14356 }, 14357 "fill": 1, 14358 "grid": { 14359 "threshold1": null, 14360 "threshold1Color": "rgba(216, 200, 27, 0.27)", 14361 "threshold2": null, 14362 "threshold2Color": "rgba(234, 112, 112, 0.22)" 14363 }, 14364 "gridPos": null, 14365 "height": null, 14366 "hideTimeOverride": false, 14367 "id": 111, 14368 "interval": null, 14369 "isNew": true, 14370 "legend": { 14371 "alignAsTable": false, 14372 "avg": false, 14373 "current": false, 14374 "hideEmpty": false, 14375 "hideZero": false, 14376 "max": false, 14377 "min": false, 14378 "rightSide": false, 14379 "show": true, 14380 "sideWidth": null, 14381 "sort": null, 14382 "sortDesc": false, 14383 "total": false, 14384 "values": false 14385 }, 14386 "lines": true, 14387 "linewidth": 2, 14388 "links": [], 14389 "maxDataPoints": 100, 14390 "maxPerRow": null, 14391 "minSpan": null, 14392 "nullPointMode": null, 14393 "options": { 14394 "alertThreshold": true, 14395 "dataLinks": [] 14396 }, 14397 "percentage": false, 14398 "pointradius": 5, 14399 "points": false, 14400 "renderer": "flot", 14401 "repeat": null, 14402 "repeatDirection": null, 14403 "seriesOverrides": [], 14404 "span": 12, 14405 "stack": false, 14406 "steppedLine": false, 14407 "targets": [ 14408 { 14409 "datasource": "$source", 14410 "expr": "apiserver:apiserver_request_latency:histogram_quantile{quantile=\"0.99\", verb!=\"WATCH\", verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}", 14411 "format": "time_series", 14412 "hide": false, 14413 "instant": false, 14414 "interval": "5s", 14415 "intervalFactor": 1, 14416 "legendFormat": "", 14417 "metric": "", 14418 "query": "apiserver:apiserver_request_latency:histogram_quantile{quantile=\"0.99\", verb!=\"WATCH\", verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}", 14419 "refId": "", 14420 "step": 10, 14421 "target": "" 14422 } 14423 ], 14424 "thresholds": [], 14425 "timeFrom": null, 14426 "timeShift": null, 14427 "title": "Request latency (99th percentile) (excl. WATCH)", 14428 "tooltip": { 14429 "msResolution": true, 14430 "shared": true, 14431 "sort": 2, 14432 "value_type": "cumulative" 14433 }, 14434 "transformations": [], 14435 "transparent": false, 14436 "type": "graph", 14437 "xaxis": { 14438 "mode": "time", 14439 "name": null, 14440 "show": true, 14441 "values": [] 14442 }, 14443 "yaxes": [ 14444 { 14445 "decimals": null, 14446 "format": "s", 14447 "label": null, 14448 "logBase": 1, 14449 "max": null, 14450 "min": null, 14451 "show": true 14452 }, 14453 { 14454 "decimals": null, 14455 "format": "short", 14456 "label": null, 14457 "logBase": 1, 14458 "max": null, 14459 "min": null, 14460 "show": true 14461 } 14462 ], 14463 "yaxis": { 14464 "align": false, 14465 "alignLevel": 0 14466 } 14467 }, 14468 { 14469 "aliasColors": {}, 14470 "bars": false, 14471 "cacheTimeout": null, 14472 "datasource": "$secondary_source", 14473 "description": null, 14474 "editable": true, 14475 "error": false, 14476 "fieldConfig": { 14477 "defaults": { 14478 "thresholds": { 14479 "mode": "absolute", 14480 "steps": [] 14481 } 14482 } 14483 }, 14484 "fill": 1, 14485 "grid": { 14486 "threshold1": null, 14487 "threshold1Color": "rgba(216, 200, 27, 0.27)", 14488 "threshold2": null, 14489 "threshold2Color": "rgba(234, 112, 112, 0.22)" 14490 }, 14491 "gridPos": null, 14492 "height": null, 14493 "hideTimeOverride": false, 14494 "id": 112, 14495 "interval": null, 14496 "isNew": true, 14497 "legend": { 14498 "alignAsTable": false, 14499 "avg": false, 14500 "current": false, 14501 "hideEmpty": false, 14502 "hideZero": false, 14503 "max": false, 14504 "min": false, 14505 "rightSide": false, 14506 "show": true, 14507 "sideWidth": null, 14508 "sort": null, 14509 "sortDesc": false, 14510 "total": false, 14511 "values": false 14512 }, 14513 "lines": true, 14514 "linewidth": 2, 14515 "links": [], 14516 "maxDataPoints": 100, 14517 "maxPerRow": null, 14518 "minSpan": null, 14519 "nullPointMode": null, 14520 "options": { 14521 "alertThreshold": true, 14522 "dataLinks": [] 14523 }, 14524 "percentage": false, 14525 "pointradius": 5, 14526 "points": false, 14527 "renderer": "flot", 14528 "repeat": null, 14529 "repeatDirection": null, 14530 "seriesOverrides": [], 14531 "span": 12, 14532 "stack": false, 14533 "steppedLine": false, 14534 "targets": [ 14535 { 14536 "datasource": "$source", 14537 "expr": "apiserver:apiserver_request_latency:histogram_quantile{quantile=\"0.99\", verb!=\"WATCH\", verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}", 14538 "format": "time_series", 14539 "hide": false, 14540 "instant": false, 14541 "interval": "5s", 14542 "intervalFactor": 1, 14543 "legendFormat": "", 14544 "metric": "", 14545 "query": "apiserver:apiserver_request_latency:histogram_quantile{quantile=\"0.99\", verb!=\"WATCH\", verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}", 14546 "refId": "", 14547 "step": 10, 14548 "target": "" 14549 } 14550 ], 14551 "thresholds": [], 14552 "timeFrom": null, 14553 "timeShift": "$timeshift", 14554 "title": "[SECONDARY] Request latency (99th percentile) (excl. WATCH)", 14555 "tooltip": { 14556 "msResolution": true, 14557 "shared": true, 14558 "sort": 2, 14559 "value_type": "cumulative" 14560 }, 14561 "transformations": [], 14562 "transparent": false, 14563 "type": "graph", 14564 "xaxis": { 14565 "mode": "time", 14566 "name": null, 14567 "show": true, 14568 "values": [] 14569 }, 14570 "yaxes": [ 14571 { 14572 "decimals": null, 14573 "format": "s", 14574 "label": null, 14575 "logBase": 1, 14576 "max": null, 14577 "min": null, 14578 "show": true 14579 }, 14580 { 14581 "decimals": null, 14582 "format": "short", 14583 "label": null, 14584 "logBase": 1, 14585 "max": null, 14586 "min": null, 14587 "show": true 14588 } 14589 ], 14590 "yaxis": { 14591 "align": false, 14592 "alignLevel": 0 14593 } 14594 }, 14595 { 14596 "aliasColors": {}, 14597 "bars": false, 14598 "cacheTimeout": null, 14599 "datasource": "$source", 14600 "description": null, 14601 "editable": true, 14602 "error": false, 14603 "fieldConfig": { 14604 "defaults": { 14605 "thresholds": { 14606 "mode": "absolute", 14607 "steps": [] 14608 } 14609 } 14610 }, 14611 "fill": 1, 14612 "grid": { 14613 "threshold1": null, 14614 "threshold1Color": "rgba(216, 200, 27, 0.27)", 14615 "threshold2": null, 14616 "threshold2Color": "rgba(234, 112, 112, 0.22)" 14617 }, 14618 "gridPos": null, 14619 "height": null, 14620 "hideTimeOverride": false, 14621 "id": 113, 14622 "interval": null, 14623 "isNew": true, 14624 "legend": { 14625 "alignAsTable": false, 14626 "avg": false, 14627 "current": false, 14628 "hideEmpty": false, 14629 "hideZero": false, 14630 "max": false, 14631 "min": false, 14632 "rightSide": false, 14633 "show": true, 14634 "sideWidth": null, 14635 "sort": null, 14636 "sortDesc": false, 14637 "total": false, 14638 "values": false 14639 }, 14640 "lines": true, 14641 "linewidth": 2, 14642 "links": [], 14643 "maxDataPoints": 100, 14644 "maxPerRow": null, 14645 "minSpan": null, 14646 "nullPointMode": null, 14647 "options": { 14648 "alertThreshold": true, 14649 "dataLinks": [] 14650 }, 14651 "percentage": false, 14652 "pointradius": 5, 14653 "points": false, 14654 "renderer": "flot", 14655 "repeat": null, 14656 "repeatDirection": null, 14657 "seriesOverrides": [], 14658 "span": 12, 14659 "stack": false, 14660 "steppedLine": false, 14661 "targets": [ 14662 { 14663 "datasource": "$source", 14664 "expr": "sum(rate(apiserver_response_sizes_sum{verb!=\"WATCH\", verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}[1m])) by (verb, version, resource, subresource, scope, instance)", 14665 "format": "time_series", 14666 "hide": false, 14667 "instant": false, 14668 "interval": "5s", 14669 "intervalFactor": 1, 14670 "legendFormat": "", 14671 "metric": "", 14672 "query": "sum(rate(apiserver_response_sizes_sum{verb!=\"WATCH\", verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}[1m])) by (verb, version, resource, subresource, scope, instance)", 14673 "refId": "", 14674 "step": 10, 14675 "target": "" 14676 } 14677 ], 14678 "thresholds": [], 14679 "timeFrom": null, 14680 "timeShift": null, 14681 "title": "Traffic (excl. WATCH)", 14682 "tooltip": { 14683 "msResolution": true, 14684 "shared": true, 14685 "sort": 2, 14686 "value_type": "cumulative" 14687 }, 14688 "transformations": [], 14689 "transparent": false, 14690 "type": "graph", 14691 "xaxis": { 14692 "mode": "time", 14693 "name": null, 14694 "show": true, 14695 "values": [] 14696 }, 14697 "yaxes": [ 14698 { 14699 "decimals": null, 14700 "format": "Bps", 14701 "label": null, 14702 "logBase": 1, 14703 "max": null, 14704 "min": null, 14705 "show": true 14706 }, 14707 { 14708 "decimals": null, 14709 "format": "short", 14710 "label": null, 14711 "logBase": 1, 14712 "max": null, 14713 "min": null, 14714 "show": true 14715 } 14716 ], 14717 "yaxis": { 14718 "align": false, 14719 "alignLevel": 0 14720 } 14721 }, 14722 { 14723 "aliasColors": {}, 14724 "bars": false, 14725 "cacheTimeout": null, 14726 "datasource": "$secondary_source", 14727 "description": null, 14728 "editable": true, 14729 "error": false, 14730 "fieldConfig": { 14731 "defaults": { 14732 "thresholds": { 14733 "mode": "absolute", 14734 "steps": [] 14735 } 14736 } 14737 }, 14738 "fill": 1, 14739 "grid": { 14740 "threshold1": null, 14741 "threshold1Color": "rgba(216, 200, 27, 0.27)", 14742 "threshold2": null, 14743 "threshold2Color": "rgba(234, 112, 112, 0.22)" 14744 }, 14745 "gridPos": null, 14746 "height": null, 14747 "hideTimeOverride": false, 14748 "id": 114, 14749 "interval": null, 14750 "isNew": true, 14751 "legend": { 14752 "alignAsTable": false, 14753 "avg": false, 14754 "current": false, 14755 "hideEmpty": false, 14756 "hideZero": false, 14757 "max": false, 14758 "min": false, 14759 "rightSide": false, 14760 "show": true, 14761 "sideWidth": null, 14762 "sort": null, 14763 "sortDesc": false, 14764 "total": false, 14765 "values": false 14766 }, 14767 "lines": true, 14768 "linewidth": 2, 14769 "links": [], 14770 "maxDataPoints": 100, 14771 "maxPerRow": null, 14772 "minSpan": null, 14773 "nullPointMode": null, 14774 "options": { 14775 "alertThreshold": true, 14776 "dataLinks": [] 14777 }, 14778 "percentage": false, 14779 "pointradius": 5, 14780 "points": false, 14781 "renderer": "flot", 14782 "repeat": null, 14783 "repeatDirection": null, 14784 "seriesOverrides": [], 14785 "span": 12, 14786 "stack": false, 14787 "steppedLine": false, 14788 "targets": [ 14789 { 14790 "datasource": "$source", 14791 "expr": "sum(rate(apiserver_response_sizes_sum{verb!=\"WATCH\", verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}[1m])) by (verb, version, resource, subresource, scope, instance)", 14792 "format": "time_series", 14793 "hide": false, 14794 "instant": false, 14795 "interval": "5s", 14796 "intervalFactor": 1, 14797 "legendFormat": "", 14798 "metric": "", 14799 "query": "sum(rate(apiserver_response_sizes_sum{verb!=\"WATCH\", verb=~\"${verb:regex}\", resource=~\"${resource:regex}s*\"}[1m])) by (verb, version, resource, subresource, scope, instance)", 14800 "refId": "", 14801 "step": 10, 14802 "target": "" 14803 } 14804 ], 14805 "thresholds": [], 14806 "timeFrom": null, 14807 "timeShift": "$timeshift", 14808 "title": "[SECONDARY] Traffic (excl. WATCH)", 14809 "tooltip": { 14810 "msResolution": true, 14811 "shared": true, 14812 "sort": 2, 14813 "value_type": "cumulative" 14814 }, 14815 "transformations": [], 14816 "transparent": false, 14817 "type": "graph", 14818 "xaxis": { 14819 "mode": "time", 14820 "name": null, 14821 "show": true, 14822 "values": [] 14823 }, 14824 "yaxes": [ 14825 { 14826 "decimals": null, 14827 "format": "Bps", 14828 "label": null, 14829 "logBase": 1, 14830 "max": null, 14831 "min": null, 14832 "show": true 14833 }, 14834 { 14835 "decimals": null, 14836 "format": "short", 14837 "label": null, 14838 "logBase": 1, 14839 "max": null, 14840 "min": null, 14841 "show": true 14842 } 14843 ], 14844 "yaxis": { 14845 "align": false, 14846 "alignLevel": 0 14847 } 14848 }, 14849 { 14850 "aliasColors": {}, 14851 "bars": false, 14852 "cacheTimeout": null, 14853 "datasource": "$source", 14854 "description": null, 14855 "editable": true, 14856 "error": false, 14857 "fieldConfig": { 14858 "defaults": { 14859 "thresholds": { 14860 "mode": "absolute", 14861 "steps": [] 14862 } 14863 } 14864 }, 14865 "fill": 1, 14866 "grid": { 14867 "threshold1": null, 14868 "threshold1Color": "rgba(216, 200, 27, 0.27)", 14869 "threshold2": null, 14870 "threshold2Color": "rgba(234, 112, 112, 0.22)" 14871 }, 14872 "gridPos": null, 14873 "height": null, 14874 "hideTimeOverride": false, 14875 "id": 115, 14876 "interval": null, 14877 "isNew": true, 14878 "legend": { 14879 "alignAsTable": false, 14880 "avg": false, 14881 "current": false, 14882 "hideEmpty": false, 14883 "hideZero": false, 14884 "max": false, 14885 "min": false, 14886 "rightSide": false, 14887 "show": true, 14888 "sideWidth": null, 14889 "sort": null, 14890 "sortDesc": false, 14891 "total": false, 14892 "values": false 14893 }, 14894 "lines": true, 14895 "linewidth": 2, 14896 "links": [], 14897 "maxDataPoints": 100, 14898 "maxPerRow": null, 14899 "minSpan": null, 14900 "nullPointMode": null, 14901 "options": { 14902 "alertThreshold": true, 14903 "dataLinks": [] 14904 }, 14905 "percentage": false, 14906 "pointradius": 5, 14907 "points": false, 14908 "renderer": "flot", 14909 "repeat": null, 14910 "repeatDirection": null, 14911 "seriesOverrides": [], 14912 "span": 12, 14913 "stack": false, 14914 "steppedLine": false, 14915 "targets": [ 14916 { 14917 "datasource": "$source", 14918 "expr": "histogram_quantile(0.99, sum(rate(apiserver_admission_webhook_admission_duration_seconds_bucket[1m])) by (le, type, name))", 14919 "format": "time_series", 14920 "hide": false, 14921 "instant": false, 14922 "interval": "5s", 14923 "intervalFactor": 1, 14924 "legendFormat": "{{type}}: {{name}}", 14925 "metric": "", 14926 "query": "histogram_quantile(0.99, sum(rate(apiserver_admission_webhook_admission_duration_seconds_bucket[1m])) by (le, type, name))", 14927 "refId": "", 14928 "step": 10, 14929 "target": "" 14930 } 14931 ], 14932 "thresholds": [], 14933 "timeFrom": null, 14934 "timeShift": null, 14935 "title": "Webhook admission duration (99th percentile)", 14936 "tooltip": { 14937 "msResolution": true, 14938 "shared": true, 14939 "sort": 2, 14940 "value_type": "cumulative" 14941 }, 14942 "transformations": [], 14943 "transparent": false, 14944 "type": "graph", 14945 "xaxis": { 14946 "mode": "time", 14947 "name": null, 14948 "show": true, 14949 "values": [] 14950 }, 14951 "yaxes": [ 14952 { 14953 "decimals": null, 14954 "format": "s", 14955 "label": null, 14956 "logBase": 1, 14957 "max": null, 14958 "min": null, 14959 "show": true 14960 }, 14961 { 14962 "decimals": null, 14963 "format": "short", 14964 "label": null, 14965 "logBase": 1, 14966 "max": null, 14967 "min": null, 14968 "show": true 14969 } 14970 ], 14971 "yaxis": { 14972 "align": false, 14973 "alignLevel": 0 14974 } 14975 }, 14976 { 14977 "aliasColors": {}, 14978 "bars": false, 14979 "cacheTimeout": null, 14980 "datasource": "$secondary_source", 14981 "description": null, 14982 "editable": true, 14983 "error": false, 14984 "fieldConfig": { 14985 "defaults": { 14986 "thresholds": { 14987 "mode": "absolute", 14988 "steps": [] 14989 } 14990 } 14991 }, 14992 "fill": 1, 14993 "grid": { 14994 "threshold1": null, 14995 "threshold1Color": "rgba(216, 200, 27, 0.27)", 14996 "threshold2": null, 14997 "threshold2Color": "rgba(234, 112, 112, 0.22)" 14998 }, 14999 "gridPos": null, 15000 "height": null, 15001 "hideTimeOverride": false, 15002 "id": 116, 15003 "interval": null, 15004 "isNew": true, 15005 "legend": { 15006 "alignAsTable": false, 15007 "avg": false, 15008 "current": false, 15009 "hideEmpty": false, 15010 "hideZero": false, 15011 "max": false, 15012 "min": false, 15013 "rightSide": false, 15014 "show": true, 15015 "sideWidth": null, 15016 "sort": null, 15017 "sortDesc": false, 15018 "total": false, 15019 "values": false 15020 }, 15021 "lines": true, 15022 "linewidth": 2, 15023 "links": [], 15024 "maxDataPoints": 100, 15025 "maxPerRow": null, 15026 "minSpan": null, 15027 "nullPointMode": null, 15028 "options": { 15029 "alertThreshold": true, 15030 "dataLinks": [] 15031 }, 15032 "percentage": false, 15033 "pointradius": 5, 15034 "points": false, 15035 "renderer": "flot", 15036 "repeat": null, 15037 "repeatDirection": null, 15038 "seriesOverrides": [], 15039 "span": 12, 15040 "stack": false, 15041 "steppedLine": false, 15042 "targets": [ 15043 { 15044 "datasource": "$source", 15045 "expr": "histogram_quantile(0.99, sum(rate(apiserver_admission_webhook_admission_duration_seconds_bucket[1m])) by (le, type, name))", 15046 "format": "time_series", 15047 "hide": false, 15048 "instant": false, 15049 "interval": "5s", 15050 "intervalFactor": 1, 15051 "legendFormat": "{{type}}: {{name}}", 15052 "metric": "", 15053 "query": "histogram_quantile(0.99, sum(rate(apiserver_admission_webhook_admission_duration_seconds_bucket[1m])) by (le, type, name))", 15054 "refId": "", 15055 "step": 10, 15056 "target": "" 15057 } 15058 ], 15059 "thresholds": [], 15060 "timeFrom": null, 15061 "timeShift": "$timeshift", 15062 "title": "[SECONDARY] Webhook admission duration (99th percentile)", 15063 "tooltip": { 15064 "msResolution": true, 15065 "shared": true, 15066 "sort": 2, 15067 "value_type": "cumulative" 15068 }, 15069 "transformations": [], 15070 "transparent": false, 15071 "type": "graph", 15072 "xaxis": { 15073 "mode": "time", 15074 "name": null, 15075 "show": true, 15076 "values": [] 15077 }, 15078 "yaxes": [ 15079 { 15080 "decimals": null, 15081 "format": "s", 15082 "label": null, 15083 "logBase": 1, 15084 "max": null, 15085 "min": null, 15086 "show": true 15087 }, 15088 { 15089 "decimals": null, 15090 "format": "short", 15091 "label": null, 15092 "logBase": 1, 15093 "max": null, 15094 "min": null, 15095 "show": true 15096 } 15097 ], 15098 "yaxis": { 15099 "align": false, 15100 "alignLevel": 0 15101 } 15102 }, 15103 { 15104 "aliasColors": {}, 15105 "bars": false, 15106 "cacheTimeout": null, 15107 "datasource": "$source", 15108 "description": null, 15109 "editable": true, 15110 "error": false, 15111 "fieldConfig": { 15112 "defaults": { 15113 "thresholds": { 15114 "mode": "absolute", 15115 "steps": [] 15116 } 15117 } 15118 }, 15119 "fill": 1, 15120 "grid": { 15121 "threshold1": null, 15122 "threshold1Color": "rgba(216, 200, 27, 0.27)", 15123 "threshold2": null, 15124 "threshold2Color": "rgba(234, 112, 112, 0.22)" 15125 }, 15126 "gridPos": null, 15127 "height": null, 15128 "hideTimeOverride": false, 15129 "id": 117, 15130 "interval": null, 15131 "isNew": true, 15132 "legend": { 15133 "alignAsTable": false, 15134 "avg": false, 15135 "current": false, 15136 "hideEmpty": false, 15137 "hideZero": false, 15138 "max": false, 15139 "min": false, 15140 "rightSide": false, 15141 "show": true, 15142 "sideWidth": null, 15143 "sort": null, 15144 "sortDesc": false, 15145 "total": false, 15146 "values": false 15147 }, 15148 "lines": true, 15149 "linewidth": 2, 15150 "links": [], 15151 "maxDataPoints": 100, 15152 "maxPerRow": null, 15153 "minSpan": null, 15154 "nullPointMode": null, 15155 "options": { 15156 "alertThreshold": true, 15157 "dataLinks": [] 15158 }, 15159 "percentage": false, 15160 "pointradius": 5, 15161 "points": false, 15162 "renderer": "flot", 15163 "repeat": null, 15164 "repeatDirection": null, 15165 "seriesOverrides": [], 15166 "span": 12, 15167 "stack": false, 15168 "steppedLine": false, 15169 "targets": [ 15170 { 15171 "datasource": "$source", 15172 "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_filter_duration_seconds_bucket[1m])) by (le, filter))", 15173 "format": "time_series", 15174 "hide": false, 15175 "instant": false, 15176 "interval": "5s", 15177 "intervalFactor": 1, 15178 "legendFormat": "{{filter}}", 15179 "metric": "", 15180 "query": "histogram_quantile(0.99, sum(rate(apiserver_request_filter_duration_seconds_bucket[1m])) by (le, filter))", 15181 "refId": "", 15182 "step": 10, 15183 "target": "" 15184 } 15185 ], 15186 "thresholds": [], 15187 "timeFrom": null, 15188 "timeShift": null, 15189 "title": "Request filter latency for each filter type (99th percentile)", 15190 "tooltip": { 15191 "msResolution": true, 15192 "shared": true, 15193 "sort": 2, 15194 "value_type": "cumulative" 15195 }, 15196 "transformations": [], 15197 "transparent": false, 15198 "type": "graph", 15199 "xaxis": { 15200 "mode": "time", 15201 "name": null, 15202 "show": true, 15203 "values": [] 15204 }, 15205 "yaxes": [ 15206 { 15207 "decimals": null, 15208 "format": "s", 15209 "label": null, 15210 "logBase": 1, 15211 "max": null, 15212 "min": null, 15213 "show": true 15214 }, 15215 { 15216 "decimals": null, 15217 "format": "short", 15218 "label": null, 15219 "logBase": 1, 15220 "max": null, 15221 "min": null, 15222 "show": true 15223 } 15224 ], 15225 "yaxis": { 15226 "align": false, 15227 "alignLevel": 0 15228 } 15229 }, 15230 { 15231 "aliasColors": {}, 15232 "bars": false, 15233 "cacheTimeout": null, 15234 "datasource": "$secondary_source", 15235 "description": null, 15236 "editable": true, 15237 "error": false, 15238 "fieldConfig": { 15239 "defaults": { 15240 "thresholds": { 15241 "mode": "absolute", 15242 "steps": [] 15243 } 15244 } 15245 }, 15246 "fill": 1, 15247 "grid": { 15248 "threshold1": null, 15249 "threshold1Color": "rgba(216, 200, 27, 0.27)", 15250 "threshold2": null, 15251 "threshold2Color": "rgba(234, 112, 112, 0.22)" 15252 }, 15253 "gridPos": null, 15254 "height": null, 15255 "hideTimeOverride": false, 15256 "id": 118, 15257 "interval": null, 15258 "isNew": true, 15259 "legend": { 15260 "alignAsTable": false, 15261 "avg": false, 15262 "current": false, 15263 "hideEmpty": false, 15264 "hideZero": false, 15265 "max": false, 15266 "min": false, 15267 "rightSide": false, 15268 "show": true, 15269 "sideWidth": null, 15270 "sort": null, 15271 "sortDesc": false, 15272 "total": false, 15273 "values": false 15274 }, 15275 "lines": true, 15276 "linewidth": 2, 15277 "links": [], 15278 "maxDataPoints": 100, 15279 "maxPerRow": null, 15280 "minSpan": null, 15281 "nullPointMode": null, 15282 "options": { 15283 "alertThreshold": true, 15284 "dataLinks": [] 15285 }, 15286 "percentage": false, 15287 "pointradius": 5, 15288 "points": false, 15289 "renderer": "flot", 15290 "repeat": null, 15291 "repeatDirection": null, 15292 "seriesOverrides": [], 15293 "span": 12, 15294 "stack": false, 15295 "steppedLine": false, 15296 "targets": [ 15297 { 15298 "datasource": "$source", 15299 "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_filter_duration_seconds_bucket[1m])) by (le, filter))", 15300 "format": "time_series", 15301 "hide": false, 15302 "instant": false, 15303 "interval": "5s", 15304 "intervalFactor": 1, 15305 "legendFormat": "{{filter}}", 15306 "metric": "", 15307 "query": "histogram_quantile(0.99, sum(rate(apiserver_request_filter_duration_seconds_bucket[1m])) by (le, filter))", 15308 "refId": "", 15309 "step": 10, 15310 "target": "" 15311 } 15312 ], 15313 "thresholds": [], 15314 "timeFrom": null, 15315 "timeShift": "$timeshift", 15316 "title": "[SECONDARY] Request filter latency for each filter type (99th percentile)", 15317 "tooltip": { 15318 "msResolution": true, 15319 "shared": true, 15320 "sort": 2, 15321 "value_type": "cumulative" 15322 }, 15323 "transformations": [], 15324 "transparent": false, 15325 "type": "graph", 15326 "xaxis": { 15327 "mode": "time", 15328 "name": null, 15329 "show": true, 15330 "values": [] 15331 }, 15332 "yaxes": [ 15333 { 15334 "decimals": null, 15335 "format": "s", 15336 "label": null, 15337 "logBase": 1, 15338 "max": null, 15339 "min": null, 15340 "show": true 15341 }, 15342 { 15343 "decimals": null, 15344 "format": "short", 15345 "label": null, 15346 "logBase": 1, 15347 "max": null, 15348 "min": null, 15349 "show": true 15350 } 15351 ], 15352 "yaxis": { 15353 "align": false, 15354 "alignLevel": 0 15355 } 15356 }, 15357 { 15358 "aliasColors": {}, 15359 "bars": false, 15360 "cacheTimeout": null, 15361 "datasource": "$source", 15362 "description": null, 15363 "editable": true, 15364 "error": false, 15365 "fieldConfig": { 15366 "defaults": { 15367 "thresholds": { 15368 "mode": "absolute", 15369 "steps": [] 15370 } 15371 } 15372 }, 15373 "fill": 1, 15374 "grid": { 15375 "threshold1": null, 15376 "threshold1Color": "rgba(216, 200, 27, 0.27)", 15377 "threshold2": null, 15378 "threshold2Color": "rgba(234, 112, 112, 0.22)" 15379 }, 15380 "gridPos": null, 15381 "height": null, 15382 "hideTimeOverride": false, 15383 "id": 119, 15384 "interval": null, 15385 "isNew": true, 15386 "legend": { 15387 "alignAsTable": false, 15388 "avg": false, 15389 "current": false, 15390 "hideEmpty": false, 15391 "hideZero": false, 15392 "max": false, 15393 "min": false, 15394 "rightSide": false, 15395 "show": true, 15396 "sideWidth": null, 15397 "sort": null, 15398 "sortDesc": false, 15399 "total": false, 15400 "values": false 15401 }, 15402 "lines": true, 15403 "linewidth": 2, 15404 "links": [], 15405 "maxDataPoints": 100, 15406 "maxPerRow": null, 15407 "minSpan": null, 15408 "nullPointMode": null, 15409 "options": { 15410 "alertThreshold": true, 15411 "dataLinks": [] 15412 }, 15413 "percentage": false, 15414 "pointradius": 5, 15415 "points": false, 15416 "renderer": "flot", 15417 "repeat": null, 15418 "repeatDirection": null, 15419 "seriesOverrides": [], 15420 "span": 12, 15421 "stack": false, 15422 "steppedLine": false, 15423 "targets": [ 15424 { 15425 "datasource": "$source", 15426 "expr": "sum(rate(rest_client_requests_total{endpoint=\"apiserver\", code!=\"200\", host!=\"[::1]:443\"}[1m])) by (code, instance, method)", 15427 "format": "time_series", 15428 "hide": false, 15429 "instant": false, 15430 "interval": "5s", 15431 "intervalFactor": 1, 15432 "legendFormat": "{{instance}}: {{code}} {{method}}", 15433 "metric": "", 15434 "query": "sum(rate(rest_client_requests_total{endpoint=\"apiserver\", code!=\"200\", host!=\"[::1]:443\"}[1m])) by (code, instance, method)", 15435 "refId": "", 15436 "step": 10, 15437 "target": "" 15438 } 15439 ], 15440 "thresholds": [], 15441 "timeFrom": null, 15442 "timeShift": null, 15443 "title": "Failed external requests", 15444 "tooltip": { 15445 "msResolution": true, 15446 "shared": true, 15447 "sort": 2, 15448 "value_type": "cumulative" 15449 }, 15450 "transformations": [], 15451 "transparent": false, 15452 "type": "graph", 15453 "xaxis": { 15454 "mode": "time", 15455 "name": null, 15456 "show": true, 15457 "values": [] 15458 }, 15459 "yaxes": [ 15460 { 15461 "decimals": null, 15462 "format": "short", 15463 "label": null, 15464 "logBase": 1, 15465 "max": null, 15466 "min": null, 15467 "show": true 15468 }, 15469 { 15470 "decimals": null, 15471 "format": "short", 15472 "label": null, 15473 "logBase": 1, 15474 "max": null, 15475 "min": null, 15476 "show": true 15477 } 15478 ], 15479 "yaxis": { 15480 "align": false, 15481 "alignLevel": 0 15482 } 15483 }, 15484 { 15485 "aliasColors": {}, 15486 "bars": false, 15487 "cacheTimeout": null, 15488 "datasource": "$secondary_source", 15489 "description": null, 15490 "editable": true, 15491 "error": false, 15492 "fieldConfig": { 15493 "defaults": { 15494 "thresholds": { 15495 "mode": "absolute", 15496 "steps": [] 15497 } 15498 } 15499 }, 15500 "fill": 1, 15501 "grid": { 15502 "threshold1": null, 15503 "threshold1Color": "rgba(216, 200, 27, 0.27)", 15504 "threshold2": null, 15505 "threshold2Color": "rgba(234, 112, 112, 0.22)" 15506 }, 15507 "gridPos": null, 15508 "height": null, 15509 "hideTimeOverride": false, 15510 "id": 120, 15511 "interval": null, 15512 "isNew": true, 15513 "legend": { 15514 "alignAsTable": false, 15515 "avg": false, 15516 "current": false, 15517 "hideEmpty": false, 15518 "hideZero": false, 15519 "max": false, 15520 "min": false, 15521 "rightSide": false, 15522 "show": true, 15523 "sideWidth": null, 15524 "sort": null, 15525 "sortDesc": false, 15526 "total": false, 15527 "values": false 15528 }, 15529 "lines": true, 15530 "linewidth": 2, 15531 "links": [], 15532 "maxDataPoints": 100, 15533 "maxPerRow": null, 15534 "minSpan": null, 15535 "nullPointMode": null, 15536 "options": { 15537 "alertThreshold": true, 15538 "dataLinks": [] 15539 }, 15540 "percentage": false, 15541 "pointradius": 5, 15542 "points": false, 15543 "renderer": "flot", 15544 "repeat": null, 15545 "repeatDirection": null, 15546 "seriesOverrides": [], 15547 "span": 12, 15548 "stack": false, 15549 "steppedLine": false, 15550 "targets": [ 15551 { 15552 "datasource": "$source", 15553 "expr": "sum(rate(rest_client_requests_total{endpoint=\"apiserver\", code!=\"200\", host!=\"[::1]:443\"}[1m])) by (code, instance, method)", 15554 "format": "time_series", 15555 "hide": false, 15556 "instant": false, 15557 "interval": "5s", 15558 "intervalFactor": 1, 15559 "legendFormat": "{{instance}}: {{code}} {{method}}", 15560 "metric": "", 15561 "query": "sum(rate(rest_client_requests_total{endpoint=\"apiserver\", code!=\"200\", host!=\"[::1]:443\"}[1m])) by (code, instance, method)", 15562 "refId": "", 15563 "step": 10, 15564 "target": "" 15565 } 15566 ], 15567 "thresholds": [], 15568 "timeFrom": null, 15569 "timeShift": "$timeshift", 15570 "title": "[SECONDARY] Failed external requests", 15571 "tooltip": { 15572 "msResolution": true, 15573 "shared": true, 15574 "sort": 2, 15575 "value_type": "cumulative" 15576 }, 15577 "transformations": [], 15578 "transparent": false, 15579 "type": "graph", 15580 "xaxis": { 15581 "mode": "time", 15582 "name": null, 15583 "show": true, 15584 "values": [] 15585 }, 15586 "yaxes": [ 15587 { 15588 "decimals": null, 15589 "format": "short", 15590 "label": null, 15591 "logBase": 1, 15592 "max": null, 15593 "min": null, 15594 "show": true 15595 }, 15596 { 15597 "decimals": null, 15598 "format": "short", 15599 "label": null, 15600 "logBase": 1, 15601 "max": null, 15602 "min": null, 15603 "show": true 15604 } 15605 ], 15606 "yaxis": { 15607 "align": false, 15608 "alignLevel": 0 15609 } 15610 }, 15611 { 15612 "aliasColors": {}, 15613 "bars": false, 15614 "cacheTimeout": null, 15615 "datasource": "$source", 15616 "description": null, 15617 "editable": true, 15618 "error": false, 15619 "fieldConfig": { 15620 "defaults": { 15621 "thresholds": { 15622 "mode": "absolute", 15623 "steps": [] 15624 } 15625 } 15626 }, 15627 "fill": 1, 15628 "grid": { 15629 "threshold1": null, 15630 "threshold1Color": "rgba(216, 200, 27, 0.27)", 15631 "threshold2": null, 15632 "threshold2Color": "rgba(234, 112, 112, 0.22)" 15633 }, 15634 "gridPos": null, 15635 "height": null, 15636 "hideTimeOverride": false, 15637 "id": 121, 15638 "interval": null, 15639 "isNew": true, 15640 "legend": { 15641 "alignAsTable": false, 15642 "avg": false, 15643 "current": false, 15644 "hideEmpty": false, 15645 "hideZero": false, 15646 "max": false, 15647 "min": false, 15648 "rightSide": false, 15649 "show": true, 15650 "sideWidth": null, 15651 "sort": null, 15652 "sortDesc": false, 15653 "total": false, 15654 "values": false 15655 }, 15656 "lines": true, 15657 "linewidth": 2, 15658 "links": [], 15659 "maxDataPoints": 100, 15660 "maxPerRow": null, 15661 "minSpan": null, 15662 "nullPointMode": null, 15663 "options": { 15664 "alertThreshold": true, 15665 "dataLinks": [] 15666 }, 15667 "percentage": false, 15668 "pointradius": 5, 15669 "points": false, 15670 "renderer": "flot", 15671 "repeat": null, 15672 "repeatDirection": null, 15673 "seriesOverrides": [], 15674 "span": 12, 15675 "stack": false, 15676 "steppedLine": false, 15677 "targets": [ 15678 { 15679 "datasource": "$source", 15680 "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{endpoint=\"apiserver\", host!=\"[::1]:443\"}[1m])) by (verb, host, instance, le))", 15681 "format": "time_series", 15682 "hide": false, 15683 "instant": false, 15684 "interval": "5s", 15685 "intervalFactor": 1, 15686 "legendFormat": "{{instance}}: {{verb}} {{host}}", 15687 "metric": "", 15688 "query": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{endpoint=\"apiserver\", host!=\"[::1]:443\"}[1m])) by (verb, host, instance, le))", 15689 "refId": "", 15690 "step": 10, 15691 "target": "" 15692 } 15693 ], 15694 "thresholds": [], 15695 "timeFrom": null, 15696 "timeShift": null, 15697 "title": "Extrernal requests latency (99th percentile)", 15698 "tooltip": { 15699 "msResolution": true, 15700 "shared": true, 15701 "sort": 2, 15702 "value_type": "cumulative" 15703 }, 15704 "transformations": [], 15705 "transparent": false, 15706 "type": "graph", 15707 "xaxis": { 15708 "mode": "time", 15709 "name": null, 15710 "show": true, 15711 "values": [] 15712 }, 15713 "yaxes": [ 15714 { 15715 "decimals": null, 15716 "format": "s", 15717 "label": null, 15718 "logBase": 1, 15719 "max": null, 15720 "min": null, 15721 "show": true 15722 }, 15723 { 15724 "decimals": null, 15725 "format": "short", 15726 "label": null, 15727 "logBase": 1, 15728 "max": null, 15729 "min": null, 15730 "show": true 15731 } 15732 ], 15733 "yaxis": { 15734 "align": false, 15735 "alignLevel": 0 15736 } 15737 }, 15738 { 15739 "aliasColors": {}, 15740 "bars": false, 15741 "cacheTimeout": null, 15742 "datasource": "$secondary_source", 15743 "description": null, 15744 "editable": true, 15745 "error": false, 15746 "fieldConfig": { 15747 "defaults": { 15748 "thresholds": { 15749 "mode": "absolute", 15750 "steps": [] 15751 } 15752 } 15753 }, 15754 "fill": 1, 15755 "grid": { 15756 "threshold1": null, 15757 "threshold1Color": "rgba(216, 200, 27, 0.27)", 15758 "threshold2": null, 15759 "threshold2Color": "rgba(234, 112, 112, 0.22)" 15760 }, 15761 "gridPos": null, 15762 "height": null, 15763 "hideTimeOverride": false, 15764 "id": 122, 15765 "interval": null, 15766 "isNew": true, 15767 "legend": { 15768 "alignAsTable": false, 15769 "avg": false, 15770 "current": false, 15771 "hideEmpty": false, 15772 "hideZero": false, 15773 "max": false, 15774 "min": false, 15775 "rightSide": false, 15776 "show": true, 15777 "sideWidth": null, 15778 "sort": null, 15779 "sortDesc": false, 15780 "total": false, 15781 "values": false 15782 }, 15783 "lines": true, 15784 "linewidth": 2, 15785 "links": [], 15786 "maxDataPoints": 100, 15787 "maxPerRow": null, 15788 "minSpan": null, 15789 "nullPointMode": null, 15790 "options": { 15791 "alertThreshold": true, 15792 "dataLinks": [] 15793 }, 15794 "percentage": false, 15795 "pointradius": 5, 15796 "points": false, 15797 "renderer": "flot", 15798 "repeat": null, 15799 "repeatDirection": null, 15800 "seriesOverrides": [], 15801 "span": 12, 15802 "stack": false, 15803 "steppedLine": false, 15804 "targets": [ 15805 { 15806 "datasource": "$source", 15807 "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{endpoint=\"apiserver\", host!=\"[::1]:443\"}[1m])) by (verb, host, instance, le))", 15808 "format": "time_series", 15809 "hide": false, 15810 "instant": false, 15811 "interval": "5s", 15812 "intervalFactor": 1, 15813 "legendFormat": "{{instance}}: {{verb}} {{host}}", 15814 "metric": "", 15815 "query": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{endpoint=\"apiserver\", host!=\"[::1]:443\"}[1m])) by (verb, host, instance, le))", 15816 "refId": "", 15817 "step": 10, 15818 "target": "" 15819 } 15820 ], 15821 "thresholds": [], 15822 "timeFrom": null, 15823 "timeShift": "$timeshift", 15824 "title": "[SECONDARY] Extrernal requests latency (99th percentile)", 15825 "tooltip": { 15826 "msResolution": true, 15827 "shared": true, 15828 "sort": 2, 15829 "value_type": "cumulative" 15830 }, 15831 "transformations": [], 15832 "transparent": false, 15833 "type": "graph", 15834 "xaxis": { 15835 "mode": "time", 15836 "name": null, 15837 "show": true, 15838 "values": [] 15839 }, 15840 "yaxes": [ 15841 { 15842 "decimals": null, 15843 "format": "s", 15844 "label": null, 15845 "logBase": 1, 15846 "max": null, 15847 "min": null, 15848 "show": true 15849 }, 15850 { 15851 "decimals": null, 15852 "format": "short", 15853 "label": null, 15854 "logBase": 1, 15855 "max": null, 15856 "min": null, 15857 "show": true 15858 } 15859 ], 15860 "yaxis": { 15861 "align": false, 15862 "alignLevel": 0 15863 } 15864 } 15865 ], 15866 "repeat": null, 15867 "showTitle": true, 15868 "title": "kube-apiserver" 15869 }, 15870 { 15871 "collapse": true, 15872 "editable": true, 15873 "height": "300px", 15874 "panels": [ 15875 { 15876 "aliasColors": {}, 15877 "bars": false, 15878 "cacheTimeout": null, 15879 "datasource": "$source", 15880 "description": null, 15881 "editable": true, 15882 "error": false, 15883 "fieldConfig": { 15884 "defaults": { 15885 "thresholds": { 15886 "mode": "absolute", 15887 "steps": [] 15888 } 15889 } 15890 }, 15891 "fill": 1, 15892 "grid": { 15893 "threshold1": null, 15894 "threshold1Color": "rgba(216, 200, 27, 0.27)", 15895 "threshold2": null, 15896 "threshold2Color": "rgba(234, 112, 112, 0.22)" 15897 }, 15898 "gridPos": null, 15899 "height": null, 15900 "hideTimeOverride": false, 15901 "id": 123, 15902 "interval": null, 15903 "isNew": true, 15904 "legend": { 15905 "alignAsTable": false, 15906 "avg": false, 15907 "current": false, 15908 "hideEmpty": false, 15909 "hideZero": false, 15910 "max": false, 15911 "min": false, 15912 "rightSide": false, 15913 "show": true, 15914 "sideWidth": null, 15915 "sort": null, 15916 "sortDesc": false, 15917 "total": false, 15918 "values": false 15919 }, 15920 "lines": true, 15921 "linewidth": 2, 15922 "links": [], 15923 "maxDataPoints": 100, 15924 "maxPerRow": null, 15925 "minSpan": null, 15926 "nullPointMode": null, 15927 "options": { 15928 "alertThreshold": true, 15929 "dataLinks": [] 15930 }, 15931 "percentage": false, 15932 "pointradius": 5, 15933 "points": false, 15934 "renderer": "flot", 15935 "repeat": null, 15936 "repeatDirection": null, 15937 "seriesOverrides": [], 15938 "span": 12, 15939 "stack": false, 15940 "steppedLine": false, 15941 "targets": [ 15942 { 15943 "datasource": "$source", 15944 "expr": "workqueue_depth{endpoint=\"kube-controller-manager\"}", 15945 "format": "time_series", 15946 "hide": false, 15947 "instant": false, 15948 "interval": "5s", 15949 "intervalFactor": 1, 15950 "legendFormat": "{{name}}", 15951 "metric": "", 15952 "query": "workqueue_depth{endpoint=\"kube-controller-manager\"}", 15953 "refId": "", 15954 "step": 10, 15955 "target": "" 15956 } 15957 ], 15958 "thresholds": [], 15959 "timeFrom": null, 15960 "timeShift": null, 15961 "title": "Workqueue depths", 15962 "tooltip": { 15963 "msResolution": true, 15964 "shared": true, 15965 "sort": 2, 15966 "value_type": "cumulative" 15967 }, 15968 "transformations": [], 15969 "transparent": false, 15970 "type": "graph", 15971 "xaxis": { 15972 "mode": "time", 15973 "name": null, 15974 "show": true, 15975 "values": [] 15976 }, 15977 "yaxes": [ 15978 { 15979 "decimals": null, 15980 "format": "short", 15981 "label": null, 15982 "logBase": 1, 15983 "max": null, 15984 "min": null, 15985 "show": true 15986 }, 15987 { 15988 "decimals": null, 15989 "format": "short", 15990 "label": null, 15991 "logBase": 1, 15992 "max": null, 15993 "min": null, 15994 "show": true 15995 } 15996 ], 15997 "yaxis": { 15998 "align": false, 15999 "alignLevel": 0 16000 } 16001 }, 16002 { 16003 "aliasColors": {}, 16004 "bars": false, 16005 "cacheTimeout": null, 16006 "datasource": "$secondary_source", 16007 "description": null, 16008 "editable": true, 16009 "error": false, 16010 "fieldConfig": { 16011 "defaults": { 16012 "thresholds": { 16013 "mode": "absolute", 16014 "steps": [] 16015 } 16016 } 16017 }, 16018 "fill": 1, 16019 "grid": { 16020 "threshold1": null, 16021 "threshold1Color": "rgba(216, 200, 27, 0.27)", 16022 "threshold2": null, 16023 "threshold2Color": "rgba(234, 112, 112, 0.22)" 16024 }, 16025 "gridPos": null, 16026 "height": null, 16027 "hideTimeOverride": false, 16028 "id": 124, 16029 "interval": null, 16030 "isNew": true, 16031 "legend": { 16032 "alignAsTable": false, 16033 "avg": false, 16034 "current": false, 16035 "hideEmpty": false, 16036 "hideZero": false, 16037 "max": false, 16038 "min": false, 16039 "rightSide": false, 16040 "show": true, 16041 "sideWidth": null, 16042 "sort": null, 16043 "sortDesc": false, 16044 "total": false, 16045 "values": false 16046 }, 16047 "lines": true, 16048 "linewidth": 2, 16049 "links": [], 16050 "maxDataPoints": 100, 16051 "maxPerRow": null, 16052 "minSpan": null, 16053 "nullPointMode": null, 16054 "options": { 16055 "alertThreshold": true, 16056 "dataLinks": [] 16057 }, 16058 "percentage": false, 16059 "pointradius": 5, 16060 "points": false, 16061 "renderer": "flot", 16062 "repeat": null, 16063 "repeatDirection": null, 16064 "seriesOverrides": [], 16065 "span": 12, 16066 "stack": false, 16067 "steppedLine": false, 16068 "targets": [ 16069 { 16070 "datasource": "$source", 16071 "expr": "workqueue_depth{endpoint=\"kube-controller-manager\"}", 16072 "format": "time_series", 16073 "hide": false, 16074 "instant": false, 16075 "interval": "5s", 16076 "intervalFactor": 1, 16077 "legendFormat": "{{name}}", 16078 "metric": "", 16079 "query": "workqueue_depth{endpoint=\"kube-controller-manager\"}", 16080 "refId": "", 16081 "step": 10, 16082 "target": "" 16083 } 16084 ], 16085 "thresholds": [], 16086 "timeFrom": null, 16087 "timeShift": "$timeshift", 16088 "title": "[SECONDARY] Workqueue depths", 16089 "tooltip": { 16090 "msResolution": true, 16091 "shared": true, 16092 "sort": 2, 16093 "value_type": "cumulative" 16094 }, 16095 "transformations": [], 16096 "transparent": false, 16097 "type": "graph", 16098 "xaxis": { 16099 "mode": "time", 16100 "name": null, 16101 "show": true, 16102 "values": [] 16103 }, 16104 "yaxes": [ 16105 { 16106 "decimals": null, 16107 "format": "short", 16108 "label": null, 16109 "logBase": 1, 16110 "max": null, 16111 "min": null, 16112 "show": true 16113 }, 16114 { 16115 "decimals": null, 16116 "format": "short", 16117 "label": null, 16118 "logBase": 1, 16119 "max": null, 16120 "min": null, 16121 "show": true 16122 } 16123 ], 16124 "yaxis": { 16125 "align": false, 16126 "alignLevel": 0 16127 } 16128 } 16129 ], 16130 "repeat": null, 16131 "showTitle": true, 16132 "title": "kube-controller-manager" 16133 }, 16134 { 16135 "collapse": true, 16136 "editable": true, 16137 "height": "300px", 16138 "panels": [ 16139 { 16140 "aliasColors": {}, 16141 "bars": false, 16142 "cacheTimeout": null, 16143 "datasource": "$source", 16144 "description": null, 16145 "editable": true, 16146 "error": false, 16147 "fieldConfig": { 16148 "defaults": { 16149 "thresholds": { 16150 "mode": "absolute", 16151 "steps": [] 16152 } 16153 } 16154 }, 16155 "fill": 1, 16156 "grid": { 16157 "threshold1": null, 16158 "threshold1Color": "rgba(216, 200, 27, 0.27)", 16159 "threshold2": null, 16160 "threshold2Color": "rgba(234, 112, 112, 0.22)" 16161 }, 16162 "gridPos": null, 16163 "height": null, 16164 "hideTimeOverride": false, 16165 "id": 125, 16166 "interval": null, 16167 "isNew": true, 16168 "legend": { 16169 "alignAsTable": false, 16170 "avg": false, 16171 "current": false, 16172 "hideEmpty": false, 16173 "hideZero": false, 16174 "max": false, 16175 "min": false, 16176 "rightSide": false, 16177 "show": true, 16178 "sideWidth": null, 16179 "sort": null, 16180 "sortDesc": false, 16181 "total": false, 16182 "values": false 16183 }, 16184 "lines": true, 16185 "linewidth": 2, 16186 "links": [], 16187 "maxDataPoints": 100, 16188 "maxPerRow": null, 16189 "minSpan": null, 16190 "nullPointMode": null, 16191 "options": { 16192 "alertThreshold": true, 16193 "dataLinks": [] 16194 }, 16195 "percentage": false, 16196 "pointradius": 5, 16197 "points": false, 16198 "renderer": "flot", 16199 "repeat": null, 16200 "repeatDirection": null, 16201 "seriesOverrides": [], 16202 "span": 12, 16203 "stack": false, 16204 "steppedLine": false, 16205 "targets": [ 16206 { 16207 "datasource": "$source", 16208 "expr": "sum(rate(container_fs_reads_bytes_total[1m])) by (container, instance)", 16209 "format": "time_series", 16210 "hide": false, 16211 "instant": false, 16212 "interval": "5s", 16213 "intervalFactor": 1, 16214 "legendFormat": "{{instance}}: {{container}}", 16215 "metric": "", 16216 "query": "sum(rate(container_fs_reads_bytes_total[1m])) by (container, instance)", 16217 "refId": "", 16218 "step": 10, 16219 "target": "" 16220 } 16221 ], 16222 "thresholds": [], 16223 "timeFrom": null, 16224 "timeShift": null, 16225 "title": "fs bytes reads by container", 16226 "tooltip": { 16227 "msResolution": true, 16228 "shared": true, 16229 "sort": 2, 16230 "value_type": "cumulative" 16231 }, 16232 "transformations": [], 16233 "transparent": false, 16234 "type": "graph", 16235 "xaxis": { 16236 "mode": "time", 16237 "name": null, 16238 "show": true, 16239 "values": [] 16240 }, 16241 "yaxes": [ 16242 { 16243 "decimals": null, 16244 "format": "bytes", 16245 "label": null, 16246 "logBase": 1, 16247 "max": null, 16248 "min": null, 16249 "show": true 16250 }, 16251 { 16252 "decimals": null, 16253 "format": "short", 16254 "label": null, 16255 "logBase": 1, 16256 "max": null, 16257 "min": null, 16258 "show": true 16259 } 16260 ], 16261 "yaxis": { 16262 "align": false, 16263 "alignLevel": 0 16264 } 16265 }, 16266 { 16267 "aliasColors": {}, 16268 "bars": false, 16269 "cacheTimeout": null, 16270 "datasource": "$secondary_source", 16271 "description": null, 16272 "editable": true, 16273 "error": false, 16274 "fieldConfig": { 16275 "defaults": { 16276 "thresholds": { 16277 "mode": "absolute", 16278 "steps": [] 16279 } 16280 } 16281 }, 16282 "fill": 1, 16283 "grid": { 16284 "threshold1": null, 16285 "threshold1Color": "rgba(216, 200, 27, 0.27)", 16286 "threshold2": null, 16287 "threshold2Color": "rgba(234, 112, 112, 0.22)" 16288 }, 16289 "gridPos": null, 16290 "height": null, 16291 "hideTimeOverride": false, 16292 "id": 126, 16293 "interval": null, 16294 "isNew": true, 16295 "legend": { 16296 "alignAsTable": false, 16297 "avg": false, 16298 "current": false, 16299 "hideEmpty": false, 16300 "hideZero": false, 16301 "max": false, 16302 "min": false, 16303 "rightSide": false, 16304 "show": true, 16305 "sideWidth": null, 16306 "sort": null, 16307 "sortDesc": false, 16308 "total": false, 16309 "values": false 16310 }, 16311 "lines": true, 16312 "linewidth": 2, 16313 "links": [], 16314 "maxDataPoints": 100, 16315 "maxPerRow": null, 16316 "minSpan": null, 16317 "nullPointMode": null, 16318 "options": { 16319 "alertThreshold": true, 16320 "dataLinks": [] 16321 }, 16322 "percentage": false, 16323 "pointradius": 5, 16324 "points": false, 16325 "renderer": "flot", 16326 "repeat": null, 16327 "repeatDirection": null, 16328 "seriesOverrides": [], 16329 "span": 12, 16330 "stack": false, 16331 "steppedLine": false, 16332 "targets": [ 16333 { 16334 "datasource": "$source", 16335 "expr": "sum(rate(container_fs_reads_bytes_total[1m])) by (container, instance)", 16336 "format": "time_series", 16337 "hide": false, 16338 "instant": false, 16339 "interval": "5s", 16340 "intervalFactor": 1, 16341 "legendFormat": "{{instance}}: {{container}}", 16342 "metric": "", 16343 "query": "sum(rate(container_fs_reads_bytes_total[1m])) by (container, instance)", 16344 "refId": "", 16345 "step": 10, 16346 "target": "" 16347 } 16348 ], 16349 "thresholds": [], 16350 "timeFrom": null, 16351 "timeShift": "$timeshift", 16352 "title": "[SECONDARY] fs bytes reads by container", 16353 "tooltip": { 16354 "msResolution": true, 16355 "shared": true, 16356 "sort": 2, 16357 "value_type": "cumulative" 16358 }, 16359 "transformations": [], 16360 "transparent": false, 16361 "type": "graph", 16362 "xaxis": { 16363 "mode": "time", 16364 "name": null, 16365 "show": true, 16366 "values": [] 16367 }, 16368 "yaxes": [ 16369 { 16370 "decimals": null, 16371 "format": "bytes", 16372 "label": null, 16373 "logBase": 1, 16374 "max": null, 16375 "min": null, 16376 "show": true 16377 }, 16378 { 16379 "decimals": null, 16380 "format": "short", 16381 "label": null, 16382 "logBase": 1, 16383 "max": null, 16384 "min": null, 16385 "show": true 16386 } 16387 ], 16388 "yaxis": { 16389 "align": false, 16390 "alignLevel": 0 16391 } 16392 }, 16393 { 16394 "aliasColors": {}, 16395 "bars": false, 16396 "cacheTimeout": null, 16397 "datasource": "$source", 16398 "description": null, 16399 "editable": true, 16400 "error": false, 16401 "fieldConfig": { 16402 "defaults": { 16403 "thresholds": { 16404 "mode": "absolute", 16405 "steps": [] 16406 } 16407 } 16408 }, 16409 "fill": 1, 16410 "grid": { 16411 "threshold1": null, 16412 "threshold1Color": "rgba(216, 200, 27, 0.27)", 16413 "threshold2": null, 16414 "threshold2Color": "rgba(234, 112, 112, 0.22)" 16415 }, 16416 "gridPos": null, 16417 "height": null, 16418 "hideTimeOverride": false, 16419 "id": 127, 16420 "interval": null, 16421 "isNew": true, 16422 "legend": { 16423 "alignAsTable": false, 16424 "avg": false, 16425 "current": false, 16426 "hideEmpty": false, 16427 "hideZero": false, 16428 "max": false, 16429 "min": false, 16430 "rightSide": false, 16431 "show": true, 16432 "sideWidth": null, 16433 "sort": null, 16434 "sortDesc": false, 16435 "total": false, 16436 "values": false 16437 }, 16438 "lines": true, 16439 "linewidth": 2, 16440 "links": [], 16441 "maxDataPoints": 100, 16442 "maxPerRow": null, 16443 "minSpan": null, 16444 "nullPointMode": null, 16445 "options": { 16446 "alertThreshold": true, 16447 "dataLinks": [] 16448 }, 16449 "percentage": false, 16450 "pointradius": 5, 16451 "points": false, 16452 "renderer": "flot", 16453 "repeat": null, 16454 "repeatDirection": null, 16455 "seriesOverrides": [], 16456 "span": 12, 16457 "stack": false, 16458 "steppedLine": false, 16459 "targets": [ 16460 { 16461 "datasource": "$source", 16462 "expr": "sum(rate(container_fs_reads_total[1m])) by (container, instance)", 16463 "format": "time_series", 16464 "hide": false, 16465 "instant": false, 16466 "interval": "5s", 16467 "intervalFactor": 1, 16468 "legendFormat": "{{instance}}: {{container}}", 16469 "metric": "", 16470 "query": "sum(rate(container_fs_reads_total[1m])) by (container, instance)", 16471 "refId": "", 16472 "step": 10, 16473 "target": "" 16474 } 16475 ], 16476 "thresholds": [], 16477 "timeFrom": null, 16478 "timeShift": null, 16479 "title": "fs reads by container", 16480 "tooltip": { 16481 "msResolution": true, 16482 "shared": true, 16483 "sort": 2, 16484 "value_type": "cumulative" 16485 }, 16486 "transformations": [], 16487 "transparent": false, 16488 "type": "graph", 16489 "xaxis": { 16490 "mode": "time", 16491 "name": null, 16492 "show": true, 16493 "values": [] 16494 }, 16495 "yaxes": [ 16496 { 16497 "decimals": null, 16498 "format": "short", 16499 "label": null, 16500 "logBase": 1, 16501 "max": null, 16502 "min": null, 16503 "show": true 16504 }, 16505 { 16506 "decimals": null, 16507 "format": "short", 16508 "label": null, 16509 "logBase": 1, 16510 "max": null, 16511 "min": null, 16512 "show": true 16513 } 16514 ], 16515 "yaxis": { 16516 "align": false, 16517 "alignLevel": 0 16518 } 16519 }, 16520 { 16521 "aliasColors": {}, 16522 "bars": false, 16523 "cacheTimeout": null, 16524 "datasource": "$secondary_source", 16525 "description": null, 16526 "editable": true, 16527 "error": false, 16528 "fieldConfig": { 16529 "defaults": { 16530 "thresholds": { 16531 "mode": "absolute", 16532 "steps": [] 16533 } 16534 } 16535 }, 16536 "fill": 1, 16537 "grid": { 16538 "threshold1": null, 16539 "threshold1Color": "rgba(216, 200, 27, 0.27)", 16540 "threshold2": null, 16541 "threshold2Color": "rgba(234, 112, 112, 0.22)" 16542 }, 16543 "gridPos": null, 16544 "height": null, 16545 "hideTimeOverride": false, 16546 "id": 128, 16547 "interval": null, 16548 "isNew": true, 16549 "legend": { 16550 "alignAsTable": false, 16551 "avg": false, 16552 "current": false, 16553 "hideEmpty": false, 16554 "hideZero": false, 16555 "max": false, 16556 "min": false, 16557 "rightSide": false, 16558 "show": true, 16559 "sideWidth": null, 16560 "sort": null, 16561 "sortDesc": false, 16562 "total": false, 16563 "values": false 16564 }, 16565 "lines": true, 16566 "linewidth": 2, 16567 "links": [], 16568 "maxDataPoints": 100, 16569 "maxPerRow": null, 16570 "minSpan": null, 16571 "nullPointMode": null, 16572 "options": { 16573 "alertThreshold": true, 16574 "dataLinks": [] 16575 }, 16576 "percentage": false, 16577 "pointradius": 5, 16578 "points": false, 16579 "renderer": "flot", 16580 "repeat": null, 16581 "repeatDirection": null, 16582 "seriesOverrides": [], 16583 "span": 12, 16584 "stack": false, 16585 "steppedLine": false, 16586 "targets": [ 16587 { 16588 "datasource": "$source", 16589 "expr": "sum(rate(container_fs_reads_total[1m])) by (container, instance)", 16590 "format": "time_series", 16591 "hide": false, 16592 "instant": false, 16593 "interval": "5s", 16594 "intervalFactor": 1, 16595 "legendFormat": "{{instance}}: {{container}}", 16596 "metric": "", 16597 "query": "sum(rate(container_fs_reads_total[1m])) by (container, instance)", 16598 "refId": "", 16599 "step": 10, 16600 "target": "" 16601 } 16602 ], 16603 "thresholds": [], 16604 "timeFrom": null, 16605 "timeShift": "$timeshift", 16606 "title": "[SECONDARY] fs reads by container", 16607 "tooltip": { 16608 "msResolution": true, 16609 "shared": true, 16610 "sort": 2, 16611 "value_type": "cumulative" 16612 }, 16613 "transformations": [], 16614 "transparent": false, 16615 "type": "graph", 16616 "xaxis": { 16617 "mode": "time", 16618 "name": null, 16619 "show": true, 16620 "values": [] 16621 }, 16622 "yaxes": [ 16623 { 16624 "decimals": null, 16625 "format": "short", 16626 "label": null, 16627 "logBase": 1, 16628 "max": null, 16629 "min": null, 16630 "show": true 16631 }, 16632 { 16633 "decimals": null, 16634 "format": "short", 16635 "label": null, 16636 "logBase": 1, 16637 "max": null, 16638 "min": null, 16639 "show": true 16640 } 16641 ], 16642 "yaxis": { 16643 "align": false, 16644 "alignLevel": 0 16645 } 16646 }, 16647 { 16648 "aliasColors": {}, 16649 "bars": false, 16650 "cacheTimeout": null, 16651 "datasource": "$source", 16652 "description": null, 16653 "editable": true, 16654 "error": false, 16655 "fieldConfig": { 16656 "defaults": { 16657 "thresholds": { 16658 "mode": "absolute", 16659 "steps": [] 16660 } 16661 } 16662 }, 16663 "fill": 1, 16664 "grid": { 16665 "threshold1": null, 16666 "threshold1Color": "rgba(216, 200, 27, 0.27)", 16667 "threshold2": null, 16668 "threshold2Color": "rgba(234, 112, 112, 0.22)" 16669 }, 16670 "gridPos": null, 16671 "height": null, 16672 "hideTimeOverride": false, 16673 "id": 129, 16674 "interval": null, 16675 "isNew": true, 16676 "legend": { 16677 "alignAsTable": false, 16678 "avg": false, 16679 "current": false, 16680 "hideEmpty": false, 16681 "hideZero": false, 16682 "max": false, 16683 "min": false, 16684 "rightSide": false, 16685 "show": true, 16686 "sideWidth": null, 16687 "sort": null, 16688 "sortDesc": false, 16689 "total": false, 16690 "values": false 16691 }, 16692 "lines": true, 16693 "linewidth": 2, 16694 "links": [], 16695 "maxDataPoints": 100, 16696 "maxPerRow": null, 16697 "minSpan": null, 16698 "nullPointMode": null, 16699 "options": { 16700 "alertThreshold": true, 16701 "dataLinks": [] 16702 }, 16703 "percentage": false, 16704 "pointradius": 5, 16705 "points": false, 16706 "renderer": "flot", 16707 "repeat": null, 16708 "repeatDirection": null, 16709 "seriesOverrides": [], 16710 "span": 12, 16711 "stack": false, 16712 "steppedLine": false, 16713 "targets": [ 16714 { 16715 "datasource": "$source", 16716 "expr": "sum(rate(container_fs_writes_bytes_total[1m])) by (container, instance)", 16717 "format": "time_series", 16718 "hide": false, 16719 "instant": false, 16720 "interval": "5s", 16721 "intervalFactor": 1, 16722 "legendFormat": "{{instance}}: {{container}}", 16723 "metric": "", 16724 "query": "sum(rate(container_fs_writes_bytes_total[1m])) by (container, instance)", 16725 "refId": "", 16726 "step": 10, 16727 "target": "" 16728 } 16729 ], 16730 "thresholds": [], 16731 "timeFrom": null, 16732 "timeShift": null, 16733 "title": "fs bytes writes by container", 16734 "tooltip": { 16735 "msResolution": true, 16736 "shared": true, 16737 "sort": 2, 16738 "value_type": "cumulative" 16739 }, 16740 "transformations": [], 16741 "transparent": false, 16742 "type": "graph", 16743 "xaxis": { 16744 "mode": "time", 16745 "name": null, 16746 "show": true, 16747 "values": [] 16748 }, 16749 "yaxes": [ 16750 { 16751 "decimals": null, 16752 "format": "bytes", 16753 "label": null, 16754 "logBase": 1, 16755 "max": null, 16756 "min": null, 16757 "show": true 16758 }, 16759 { 16760 "decimals": null, 16761 "format": "short", 16762 "label": null, 16763 "logBase": 1, 16764 "max": null, 16765 "min": null, 16766 "show": true 16767 } 16768 ], 16769 "yaxis": { 16770 "align": false, 16771 "alignLevel": 0 16772 } 16773 }, 16774 { 16775 "aliasColors": {}, 16776 "bars": false, 16777 "cacheTimeout": null, 16778 "datasource": "$secondary_source", 16779 "description": null, 16780 "editable": true, 16781 "error": false, 16782 "fieldConfig": { 16783 "defaults": { 16784 "thresholds": { 16785 "mode": "absolute", 16786 "steps": [] 16787 } 16788 } 16789 }, 16790 "fill": 1, 16791 "grid": { 16792 "threshold1": null, 16793 "threshold1Color": "rgba(216, 200, 27, 0.27)", 16794 "threshold2": null, 16795 "threshold2Color": "rgba(234, 112, 112, 0.22)" 16796 }, 16797 "gridPos": null, 16798 "height": null, 16799 "hideTimeOverride": false, 16800 "id": 130, 16801 "interval": null, 16802 "isNew": true, 16803 "legend": { 16804 "alignAsTable": false, 16805 "avg": false, 16806 "current": false, 16807 "hideEmpty": false, 16808 "hideZero": false, 16809 "max": false, 16810 "min": false, 16811 "rightSide": false, 16812 "show": true, 16813 "sideWidth": null, 16814 "sort": null, 16815 "sortDesc": false, 16816 "total": false, 16817 "values": false 16818 }, 16819 "lines": true, 16820 "linewidth": 2, 16821 "links": [], 16822 "maxDataPoints": 100, 16823 "maxPerRow": null, 16824 "minSpan": null, 16825 "nullPointMode": null, 16826 "options": { 16827 "alertThreshold": true, 16828 "dataLinks": [] 16829 }, 16830 "percentage": false, 16831 "pointradius": 5, 16832 "points": false, 16833 "renderer": "flot", 16834 "repeat": null, 16835 "repeatDirection": null, 16836 "seriesOverrides": [], 16837 "span": 12, 16838 "stack": false, 16839 "steppedLine": false, 16840 "targets": [ 16841 { 16842 "datasource": "$source", 16843 "expr": "sum(rate(container_fs_writes_bytes_total[1m])) by (container, instance)", 16844 "format": "time_series", 16845 "hide": false, 16846 "instant": false, 16847 "interval": "5s", 16848 "intervalFactor": 1, 16849 "legendFormat": "{{instance}}: {{container}}", 16850 "metric": "", 16851 "query": "sum(rate(container_fs_writes_bytes_total[1m])) by (container, instance)", 16852 "refId": "", 16853 "step": 10, 16854 "target": "" 16855 } 16856 ], 16857 "thresholds": [], 16858 "timeFrom": null, 16859 "timeShift": "$timeshift", 16860 "title": "[SECONDARY] fs bytes writes by container", 16861 "tooltip": { 16862 "msResolution": true, 16863 "shared": true, 16864 "sort": 2, 16865 "value_type": "cumulative" 16866 }, 16867 "transformations": [], 16868 "transparent": false, 16869 "type": "graph", 16870 "xaxis": { 16871 "mode": "time", 16872 "name": null, 16873 "show": true, 16874 "values": [] 16875 }, 16876 "yaxes": [ 16877 { 16878 "decimals": null, 16879 "format": "bytes", 16880 "label": null, 16881 "logBase": 1, 16882 "max": null, 16883 "min": null, 16884 "show": true 16885 }, 16886 { 16887 "decimals": null, 16888 "format": "short", 16889 "label": null, 16890 "logBase": 1, 16891 "max": null, 16892 "min": null, 16893 "show": true 16894 } 16895 ], 16896 "yaxis": { 16897 "align": false, 16898 "alignLevel": 0 16899 } 16900 }, 16901 { 16902 "aliasColors": {}, 16903 "bars": false, 16904 "cacheTimeout": null, 16905 "datasource": "$source", 16906 "description": null, 16907 "editable": true, 16908 "error": false, 16909 "fieldConfig": { 16910 "defaults": { 16911 "thresholds": { 16912 "mode": "absolute", 16913 "steps": [] 16914 } 16915 } 16916 }, 16917 "fill": 1, 16918 "grid": { 16919 "threshold1": null, 16920 "threshold1Color": "rgba(216, 200, 27, 0.27)", 16921 "threshold2": null, 16922 "threshold2Color": "rgba(234, 112, 112, 0.22)" 16923 }, 16924 "gridPos": null, 16925 "height": null, 16926 "hideTimeOverride": false, 16927 "id": 131, 16928 "interval": null, 16929 "isNew": true, 16930 "legend": { 16931 "alignAsTable": false, 16932 "avg": false, 16933 "current": false, 16934 "hideEmpty": false, 16935 "hideZero": false, 16936 "max": false, 16937 "min": false, 16938 "rightSide": false, 16939 "show": true, 16940 "sideWidth": null, 16941 "sort": null, 16942 "sortDesc": false, 16943 "total": false, 16944 "values": false 16945 }, 16946 "lines": true, 16947 "linewidth": 2, 16948 "links": [], 16949 "maxDataPoints": 100, 16950 "maxPerRow": null, 16951 "minSpan": null, 16952 "nullPointMode": null, 16953 "options": { 16954 "alertThreshold": true, 16955 "dataLinks": [] 16956 }, 16957 "percentage": false, 16958 "pointradius": 5, 16959 "points": false, 16960 "renderer": "flot", 16961 "repeat": null, 16962 "repeatDirection": null, 16963 "seriesOverrides": [], 16964 "span": 12, 16965 "stack": false, 16966 "steppedLine": false, 16967 "targets": [ 16968 { 16969 "datasource": "$source", 16970 "expr": "sum(rate(container_fs_writes_total[1m])) by (container, instance)", 16971 "format": "time_series", 16972 "hide": false, 16973 "instant": false, 16974 "interval": "5s", 16975 "intervalFactor": 1, 16976 "legendFormat": "{{instance}}: {{container}}", 16977 "metric": "", 16978 "query": "sum(rate(container_fs_writes_total[1m])) by (container, instance)", 16979 "refId": "", 16980 "step": 10, 16981 "target": "" 16982 } 16983 ], 16984 "thresholds": [], 16985 "timeFrom": null, 16986 "timeShift": null, 16987 "title": "fs writes by container", 16988 "tooltip": { 16989 "msResolution": true, 16990 "shared": true, 16991 "sort": 2, 16992 "value_type": "cumulative" 16993 }, 16994 "transformations": [], 16995 "transparent": false, 16996 "type": "graph", 16997 "xaxis": { 16998 "mode": "time", 16999 "name": null, 17000 "show": true, 17001 "values": [] 17002 }, 17003 "yaxes": [ 17004 { 17005 "decimals": null, 17006 "format": "short", 17007 "label": null, 17008 "logBase": 1, 17009 "max": null, 17010 "min": null, 17011 "show": true 17012 }, 17013 { 17014 "decimals": null, 17015 "format": "short", 17016 "label": null, 17017 "logBase": 1, 17018 "max": null, 17019 "min": null, 17020 "show": true 17021 } 17022 ], 17023 "yaxis": { 17024 "align": false, 17025 "alignLevel": 0 17026 } 17027 }, 17028 { 17029 "aliasColors": {}, 17030 "bars": false, 17031 "cacheTimeout": null, 17032 "datasource": "$secondary_source", 17033 "description": null, 17034 "editable": true, 17035 "error": false, 17036 "fieldConfig": { 17037 "defaults": { 17038 "thresholds": { 17039 "mode": "absolute", 17040 "steps": [] 17041 } 17042 } 17043 }, 17044 "fill": 1, 17045 "grid": { 17046 "threshold1": null, 17047 "threshold1Color": "rgba(216, 200, 27, 0.27)", 17048 "threshold2": null, 17049 "threshold2Color": "rgba(234, 112, 112, 0.22)" 17050 }, 17051 "gridPos": null, 17052 "height": null, 17053 "hideTimeOverride": false, 17054 "id": 132, 17055 "interval": null, 17056 "isNew": true, 17057 "legend": { 17058 "alignAsTable": false, 17059 "avg": false, 17060 "current": false, 17061 "hideEmpty": false, 17062 "hideZero": false, 17063 "max": false, 17064 "min": false, 17065 "rightSide": false, 17066 "show": true, 17067 "sideWidth": null, 17068 "sort": null, 17069 "sortDesc": false, 17070 "total": false, 17071 "values": false 17072 }, 17073 "lines": true, 17074 "linewidth": 2, 17075 "links": [], 17076 "maxDataPoints": 100, 17077 "maxPerRow": null, 17078 "minSpan": null, 17079 "nullPointMode": null, 17080 "options": { 17081 "alertThreshold": true, 17082 "dataLinks": [] 17083 }, 17084 "percentage": false, 17085 "pointradius": 5, 17086 "points": false, 17087 "renderer": "flot", 17088 "repeat": null, 17089 "repeatDirection": null, 17090 "seriesOverrides": [], 17091 "span": 12, 17092 "stack": false, 17093 "steppedLine": false, 17094 "targets": [ 17095 { 17096 "datasource": "$source", 17097 "expr": "sum(rate(container_fs_writes_total[1m])) by (container, instance)", 17098 "format": "time_series", 17099 "hide": false, 17100 "instant": false, 17101 "interval": "5s", 17102 "intervalFactor": 1, 17103 "legendFormat": "{{instance}}: {{container}}", 17104 "metric": "", 17105 "query": "sum(rate(container_fs_writes_total[1m])) by (container, instance)", 17106 "refId": "", 17107 "step": 10, 17108 "target": "" 17109 } 17110 ], 17111 "thresholds": [], 17112 "timeFrom": null, 17113 "timeShift": "$timeshift", 17114 "title": "[SECONDARY] fs writes by container", 17115 "tooltip": { 17116 "msResolution": true, 17117 "shared": true, 17118 "sort": 2, 17119 "value_type": "cumulative" 17120 }, 17121 "transformations": [], 17122 "transparent": false, 17123 "type": "graph", 17124 "xaxis": { 17125 "mode": "time", 17126 "name": null, 17127 "show": true, 17128 "values": [] 17129 }, 17130 "yaxes": [ 17131 { 17132 "decimals": null, 17133 "format": "short", 17134 "label": null, 17135 "logBase": 1, 17136 "max": null, 17137 "min": null, 17138 "show": true 17139 }, 17140 { 17141 "decimals": null, 17142 "format": "short", 17143 "label": null, 17144 "logBase": 1, 17145 "max": null, 17146 "min": null, 17147 "show": true 17148 } 17149 ], 17150 "yaxis": { 17151 "align": false, 17152 "alignLevel": 0 17153 } 17154 }, 17155 { 17156 "aliasColors": {}, 17157 "bars": false, 17158 "cacheTimeout": null, 17159 "datasource": "$source", 17160 "description": null, 17161 "editable": true, 17162 "error": false, 17163 "fieldConfig": { 17164 "defaults": { 17165 "thresholds": { 17166 "mode": "absolute", 17167 "steps": [] 17168 } 17169 } 17170 }, 17171 "fill": 1, 17172 "grid": { 17173 "threshold1": null, 17174 "threshold1Color": "rgba(216, 200, 27, 0.27)", 17175 "threshold2": null, 17176 "threshold2Color": "rgba(234, 112, 112, 0.22)" 17177 }, 17178 "gridPos": null, 17179 "height": null, 17180 "hideTimeOverride": false, 17181 "id": 133, 17182 "interval": null, 17183 "isNew": true, 17184 "legend": { 17185 "alignAsTable": false, 17186 "avg": false, 17187 "current": false, 17188 "hideEmpty": false, 17189 "hideZero": false, 17190 "max": false, 17191 "min": false, 17192 "rightSide": false, 17193 "show": true, 17194 "sideWidth": null, 17195 "sort": null, 17196 "sortDesc": false, 17197 "total": false, 17198 "values": false 17199 }, 17200 "lines": true, 17201 "linewidth": 2, 17202 "links": [], 17203 "maxDataPoints": 100, 17204 "maxPerRow": null, 17205 "minSpan": null, 17206 "nullPointMode": null, 17207 "options": { 17208 "alertThreshold": true, 17209 "dataLinks": [] 17210 }, 17211 "percentage": false, 17212 "pointradius": 5, 17213 "points": false, 17214 "renderer": "flot", 17215 "repeat": null, 17216 "repeatDirection": null, 17217 "seriesOverrides": [], 17218 "span": 12, 17219 "stack": false, 17220 "steppedLine": false, 17221 "targets": [ 17222 { 17223 "datasource": "$source", 17224 "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"\"}[1m])) by (container, instance)", 17225 "format": "time_series", 17226 "hide": false, 17227 "instant": false, 17228 "interval": "5s", 17229 "intervalFactor": 1, 17230 "legendFormat": "{{instance}}: {{container}}", 17231 "metric": "", 17232 "query": "sum(rate(container_cpu_usage_seconds_total{container!=\"\"}[1m])) by (container, instance)", 17233 "refId": "", 17234 "step": 10, 17235 "target": "" 17236 }, 17237 { 17238 "datasource": "$source", 17239 "expr": "machine_cpu_cores", 17240 "format": "time_series", 17241 "hide": false, 17242 "instant": false, 17243 "interval": "5s", 17244 "intervalFactor": 1, 17245 "legendFormat": "limit", 17246 "metric": "", 17247 "query": "machine_cpu_cores", 17248 "refId": "", 17249 "step": 10, 17250 "target": "" 17251 } 17252 ], 17253 "thresholds": [], 17254 "timeFrom": null, 17255 "timeShift": null, 17256 "title": "CPU usage by container", 17257 "tooltip": { 17258 "msResolution": true, 17259 "shared": true, 17260 "sort": 2, 17261 "value_type": "cumulative" 17262 }, 17263 "transformations": [], 17264 "transparent": false, 17265 "type": "graph", 17266 "xaxis": { 17267 "mode": "time", 17268 "name": null, 17269 "show": true, 17270 "values": [] 17271 }, 17272 "yaxes": [ 17273 { 17274 "decimals": null, 17275 "format": "short", 17276 "label": null, 17277 "logBase": 1, 17278 "max": null, 17279 "min": null, 17280 "show": true 17281 }, 17282 { 17283 "decimals": null, 17284 "format": "short", 17285 "label": null, 17286 "logBase": 1, 17287 "max": null, 17288 "min": null, 17289 "show": true 17290 } 17291 ], 17292 "yaxis": { 17293 "align": false, 17294 "alignLevel": 0 17295 } 17296 }, 17297 { 17298 "aliasColors": {}, 17299 "bars": false, 17300 "cacheTimeout": null, 17301 "datasource": "$secondary_source", 17302 "description": null, 17303 "editable": true, 17304 "error": false, 17305 "fieldConfig": { 17306 "defaults": { 17307 "thresholds": { 17308 "mode": "absolute", 17309 "steps": [] 17310 } 17311 } 17312 }, 17313 "fill": 1, 17314 "grid": { 17315 "threshold1": null, 17316 "threshold1Color": "rgba(216, 200, 27, 0.27)", 17317 "threshold2": null, 17318 "threshold2Color": "rgba(234, 112, 112, 0.22)" 17319 }, 17320 "gridPos": null, 17321 "height": null, 17322 "hideTimeOverride": false, 17323 "id": 134, 17324 "interval": null, 17325 "isNew": true, 17326 "legend": { 17327 "alignAsTable": false, 17328 "avg": false, 17329 "current": false, 17330 "hideEmpty": false, 17331 "hideZero": false, 17332 "max": false, 17333 "min": false, 17334 "rightSide": false, 17335 "show": true, 17336 "sideWidth": null, 17337 "sort": null, 17338 "sortDesc": false, 17339 "total": false, 17340 "values": false 17341 }, 17342 "lines": true, 17343 "linewidth": 2, 17344 "links": [], 17345 "maxDataPoints": 100, 17346 "maxPerRow": null, 17347 "minSpan": null, 17348 "nullPointMode": null, 17349 "options": { 17350 "alertThreshold": true, 17351 "dataLinks": [] 17352 }, 17353 "percentage": false, 17354 "pointradius": 5, 17355 "points": false, 17356 "renderer": "flot", 17357 "repeat": null, 17358 "repeatDirection": null, 17359 "seriesOverrides": [], 17360 "span": 12, 17361 "stack": false, 17362 "steppedLine": false, 17363 "targets": [ 17364 { 17365 "datasource": "$source", 17366 "expr": "sum(rate(container_cpu_usage_seconds_total{container!=\"\"}[1m])) by (container, instance)", 17367 "format": "time_series", 17368 "hide": false, 17369 "instant": false, 17370 "interval": "5s", 17371 "intervalFactor": 1, 17372 "legendFormat": "{{instance}}: {{container}}", 17373 "metric": "", 17374 "query": "sum(rate(container_cpu_usage_seconds_total{container!=\"\"}[1m])) by (container, instance)", 17375 "refId": "", 17376 "step": 10, 17377 "target": "" 17378 }, 17379 { 17380 "datasource": "$source", 17381 "expr": "machine_cpu_cores", 17382 "format": "time_series", 17383 "hide": false, 17384 "instant": false, 17385 "interval": "5s", 17386 "intervalFactor": 1, 17387 "legendFormat": "limit", 17388 "metric": "", 17389 "query": "machine_cpu_cores", 17390 "refId": "", 17391 "step": 10, 17392 "target": "" 17393 } 17394 ], 17395 "thresholds": [], 17396 "timeFrom": null, 17397 "timeShift": "$timeshift", 17398 "title": "[SECONDARY] CPU usage by container", 17399 "tooltip": { 17400 "msResolution": true, 17401 "shared": true, 17402 "sort": 2, 17403 "value_type": "cumulative" 17404 }, 17405 "transformations": [], 17406 "transparent": false, 17407 "type": "graph", 17408 "xaxis": { 17409 "mode": "time", 17410 "name": null, 17411 "show": true, 17412 "values": [] 17413 }, 17414 "yaxes": [ 17415 { 17416 "decimals": null, 17417 "format": "short", 17418 "label": null, 17419 "logBase": 1, 17420 "max": null, 17421 "min": null, 17422 "show": true 17423 }, 17424 { 17425 "decimals": null, 17426 "format": "short", 17427 "label": null, 17428 "logBase": 1, 17429 "max": null, 17430 "min": null, 17431 "show": true 17432 } 17433 ], 17434 "yaxis": { 17435 "align": false, 17436 "alignLevel": 0 17437 } 17438 }, 17439 { 17440 "aliasColors": {}, 17441 "bars": false, 17442 "cacheTimeout": null, 17443 "datasource": "$source", 17444 "description": null, 17445 "editable": true, 17446 "error": false, 17447 "fieldConfig": { 17448 "defaults": { 17449 "thresholds": { 17450 "mode": "absolute", 17451 "steps": [] 17452 } 17453 } 17454 }, 17455 "fill": 1, 17456 "grid": { 17457 "threshold1": null, 17458 "threshold1Color": "rgba(216, 200, 27, 0.27)", 17459 "threshold2": null, 17460 "threshold2Color": "rgba(234, 112, 112, 0.22)" 17461 }, 17462 "gridPos": null, 17463 "height": null, 17464 "hideTimeOverride": false, 17465 "id": 135, 17466 "interval": null, 17467 "isNew": true, 17468 "legend": { 17469 "alignAsTable": false, 17470 "avg": false, 17471 "current": false, 17472 "hideEmpty": false, 17473 "hideZero": false, 17474 "max": false, 17475 "min": false, 17476 "rightSide": false, 17477 "show": true, 17478 "sideWidth": null, 17479 "sort": null, 17480 "sortDesc": false, 17481 "total": false, 17482 "values": false 17483 }, 17484 "lines": true, 17485 "linewidth": 2, 17486 "links": [], 17487 "maxDataPoints": 100, 17488 "maxPerRow": null, 17489 "minSpan": null, 17490 "nullPointMode": null, 17491 "options": { 17492 "alertThreshold": true, 17493 "dataLinks": [] 17494 }, 17495 "percentage": false, 17496 "pointradius": 5, 17497 "points": false, 17498 "renderer": "flot", 17499 "repeat": null, 17500 "repeatDirection": null, 17501 "seriesOverrides": [], 17502 "span": 12, 17503 "stack": false, 17504 "steppedLine": false, 17505 "targets": [ 17506 { 17507 "datasource": "$source", 17508 "expr": "sum(container_memory_usage_bytes{container!=\"\"}) by (container, instance)", 17509 "format": "time_series", 17510 "hide": false, 17511 "instant": false, 17512 "interval": "5s", 17513 "intervalFactor": 1, 17514 "legendFormat": "{{instance}}: {{container}}", 17515 "metric": "", 17516 "query": "sum(container_memory_usage_bytes{container!=\"\"}) by (container, instance)", 17517 "refId": "", 17518 "step": 10, 17519 "target": "" 17520 }, 17521 { 17522 "datasource": "$source", 17523 "expr": "machine_memory_bytes", 17524 "format": "time_series", 17525 "hide": false, 17526 "instant": false, 17527 "interval": "5s", 17528 "intervalFactor": 1, 17529 "legendFormat": "limit", 17530 "metric": "", 17531 "query": "machine_memory_bytes", 17532 "refId": "", 17533 "step": 10, 17534 "target": "" 17535 } 17536 ], 17537 "thresholds": [], 17538 "timeFrom": null, 17539 "timeShift": null, 17540 "title": "memory usage by container", 17541 "tooltip": { 17542 "msResolution": true, 17543 "shared": true, 17544 "sort": 2, 17545 "value_type": "cumulative" 17546 }, 17547 "transformations": [], 17548 "transparent": false, 17549 "type": "graph", 17550 "xaxis": { 17551 "mode": "time", 17552 "name": null, 17553 "show": true, 17554 "values": [] 17555 }, 17556 "yaxes": [ 17557 { 17558 "decimals": null, 17559 "format": "bytes", 17560 "label": null, 17561 "logBase": 1, 17562 "max": null, 17563 "min": null, 17564 "show": true 17565 }, 17566 { 17567 "decimals": null, 17568 "format": "short", 17569 "label": null, 17570 "logBase": 1, 17571 "max": null, 17572 "min": null, 17573 "show": true 17574 } 17575 ], 17576 "yaxis": { 17577 "align": false, 17578 "alignLevel": 0 17579 } 17580 }, 17581 { 17582 "aliasColors": {}, 17583 "bars": false, 17584 "cacheTimeout": null, 17585 "datasource": "$secondary_source", 17586 "description": null, 17587 "editable": true, 17588 "error": false, 17589 "fieldConfig": { 17590 "defaults": { 17591 "thresholds": { 17592 "mode": "absolute", 17593 "steps": [] 17594 } 17595 } 17596 }, 17597 "fill": 1, 17598 "grid": { 17599 "threshold1": null, 17600 "threshold1Color": "rgba(216, 200, 27, 0.27)", 17601 "threshold2": null, 17602 "threshold2Color": "rgba(234, 112, 112, 0.22)" 17603 }, 17604 "gridPos": null, 17605 "height": null, 17606 "hideTimeOverride": false, 17607 "id": 136, 17608 "interval": null, 17609 "isNew": true, 17610 "legend": { 17611 "alignAsTable": false, 17612 "avg": false, 17613 "current": false, 17614 "hideEmpty": false, 17615 "hideZero": false, 17616 "max": false, 17617 "min": false, 17618 "rightSide": false, 17619 "show": true, 17620 "sideWidth": null, 17621 "sort": null, 17622 "sortDesc": false, 17623 "total": false, 17624 "values": false 17625 }, 17626 "lines": true, 17627 "linewidth": 2, 17628 "links": [], 17629 "maxDataPoints": 100, 17630 "maxPerRow": null, 17631 "minSpan": null, 17632 "nullPointMode": null, 17633 "options": { 17634 "alertThreshold": true, 17635 "dataLinks": [] 17636 }, 17637 "percentage": false, 17638 "pointradius": 5, 17639 "points": false, 17640 "renderer": "flot", 17641 "repeat": null, 17642 "repeatDirection": null, 17643 "seriesOverrides": [], 17644 "span": 12, 17645 "stack": false, 17646 "steppedLine": false, 17647 "targets": [ 17648 { 17649 "datasource": "$source", 17650 "expr": "sum(container_memory_usage_bytes{container!=\"\"}) by (container, instance)", 17651 "format": "time_series", 17652 "hide": false, 17653 "instant": false, 17654 "interval": "5s", 17655 "intervalFactor": 1, 17656 "legendFormat": "{{instance}}: {{container}}", 17657 "metric": "", 17658 "query": "sum(container_memory_usage_bytes{container!=\"\"}) by (container, instance)", 17659 "refId": "", 17660 "step": 10, 17661 "target": "" 17662 }, 17663 { 17664 "datasource": "$source", 17665 "expr": "machine_memory_bytes", 17666 "format": "time_series", 17667 "hide": false, 17668 "instant": false, 17669 "interval": "5s", 17670 "intervalFactor": 1, 17671 "legendFormat": "limit", 17672 "metric": "", 17673 "query": "machine_memory_bytes", 17674 "refId": "", 17675 "step": 10, 17676 "target": "" 17677 } 17678 ], 17679 "thresholds": [], 17680 "timeFrom": null, 17681 "timeShift": "$timeshift", 17682 "title": "[SECONDARY] memory usage by container", 17683 "tooltip": { 17684 "msResolution": true, 17685 "shared": true, 17686 "sort": 2, 17687 "value_type": "cumulative" 17688 }, 17689 "transformations": [], 17690 "transparent": false, 17691 "type": "graph", 17692 "xaxis": { 17693 "mode": "time", 17694 "name": null, 17695 "show": true, 17696 "values": [] 17697 }, 17698 "yaxes": [ 17699 { 17700 "decimals": null, 17701 "format": "bytes", 17702 "label": null, 17703 "logBase": 1, 17704 "max": null, 17705 "min": null, 17706 "show": true 17707 }, 17708 { 17709 "decimals": null, 17710 "format": "short", 17711 "label": null, 17712 "logBase": 1, 17713 "max": null, 17714 "min": null, 17715 "show": true 17716 } 17717 ], 17718 "yaxis": { 17719 "align": false, 17720 "alignLevel": 0 17721 } 17722 }, 17723 { 17724 "aliasColors": {}, 17725 "bars": false, 17726 "cacheTimeout": null, 17727 "datasource": "$source", 17728 "description": null, 17729 "editable": true, 17730 "error": false, 17731 "fieldConfig": { 17732 "defaults": { 17733 "thresholds": { 17734 "mode": "absolute", 17735 "steps": [] 17736 } 17737 } 17738 }, 17739 "fill": 1, 17740 "grid": { 17741 "threshold1": null, 17742 "threshold1Color": "rgba(216, 200, 27, 0.27)", 17743 "threshold2": null, 17744 "threshold2Color": "rgba(234, 112, 112, 0.22)" 17745 }, 17746 "gridPos": null, 17747 "height": null, 17748 "hideTimeOverride": false, 17749 "id": 137, 17750 "interval": null, 17751 "isNew": true, 17752 "legend": { 17753 "alignAsTable": false, 17754 "avg": false, 17755 "current": false, 17756 "hideEmpty": false, 17757 "hideZero": false, 17758 "max": false, 17759 "min": false, 17760 "rightSide": false, 17761 "show": true, 17762 "sideWidth": null, 17763 "sort": null, 17764 "sortDesc": false, 17765 "total": false, 17766 "values": false 17767 }, 17768 "lines": true, 17769 "linewidth": 2, 17770 "links": [], 17771 "maxDataPoints": 100, 17772 "maxPerRow": null, 17773 "minSpan": null, 17774 "nullPointMode": null, 17775 "options": { 17776 "alertThreshold": true, 17777 "dataLinks": [] 17778 }, 17779 "percentage": false, 17780 "pointradius": 5, 17781 "points": false, 17782 "renderer": "flot", 17783 "repeat": null, 17784 "repeatDirection": null, 17785 "seriesOverrides": [], 17786 "span": 12, 17787 "stack": false, 17788 "steppedLine": false, 17789 "targets": [ 17790 { 17791 "datasource": "$source", 17792 "expr": "sum(container_memory_working_set_bytes{container!=\"\"}) by (container, instance)", 17793 "format": "time_series", 17794 "hide": false, 17795 "instant": false, 17796 "interval": "5s", 17797 "intervalFactor": 1, 17798 "legendFormat": "{{instance}}: {{container}}", 17799 "metric": "", 17800 "query": "sum(container_memory_working_set_bytes{container!=\"\"}) by (container, instance)", 17801 "refId": "", 17802 "step": 10, 17803 "target": "" 17804 }, 17805 { 17806 "datasource": "$source", 17807 "expr": "machine_memory_bytes", 17808 "format": "time_series", 17809 "hide": false, 17810 "instant": false, 17811 "interval": "5s", 17812 "intervalFactor": 1, 17813 "legendFormat": "limit", 17814 "metric": "", 17815 "query": "machine_memory_bytes", 17816 "refId": "", 17817 "step": 10, 17818 "target": "" 17819 } 17820 ], 17821 "thresholds": [], 17822 "timeFrom": null, 17823 "timeShift": null, 17824 "title": "memory working set by container", 17825 "tooltip": { 17826 "msResolution": true, 17827 "shared": true, 17828 "sort": 2, 17829 "value_type": "cumulative" 17830 }, 17831 "transformations": [], 17832 "transparent": false, 17833 "type": "graph", 17834 "xaxis": { 17835 "mode": "time", 17836 "name": null, 17837 "show": true, 17838 "values": [] 17839 }, 17840 "yaxes": [ 17841 { 17842 "decimals": null, 17843 "format": "bytes", 17844 "label": null, 17845 "logBase": 1, 17846 "max": null, 17847 "min": null, 17848 "show": true 17849 }, 17850 { 17851 "decimals": null, 17852 "format": "short", 17853 "label": null, 17854 "logBase": 1, 17855 "max": null, 17856 "min": null, 17857 "show": true 17858 } 17859 ], 17860 "yaxis": { 17861 "align": false, 17862 "alignLevel": 0 17863 } 17864 }, 17865 { 17866 "aliasColors": {}, 17867 "bars": false, 17868 "cacheTimeout": null, 17869 "datasource": "$secondary_source", 17870 "description": null, 17871 "editable": true, 17872 "error": false, 17873 "fieldConfig": { 17874 "defaults": { 17875 "thresholds": { 17876 "mode": "absolute", 17877 "steps": [] 17878 } 17879 } 17880 }, 17881 "fill": 1, 17882 "grid": { 17883 "threshold1": null, 17884 "threshold1Color": "rgba(216, 200, 27, 0.27)", 17885 "threshold2": null, 17886 "threshold2Color": "rgba(234, 112, 112, 0.22)" 17887 }, 17888 "gridPos": null, 17889 "height": null, 17890 "hideTimeOverride": false, 17891 "id": 138, 17892 "interval": null, 17893 "isNew": true, 17894 "legend": { 17895 "alignAsTable": false, 17896 "avg": false, 17897 "current": false, 17898 "hideEmpty": false, 17899 "hideZero": false, 17900 "max": false, 17901 "min": false, 17902 "rightSide": false, 17903 "show": true, 17904 "sideWidth": null, 17905 "sort": null, 17906 "sortDesc": false, 17907 "total": false, 17908 "values": false 17909 }, 17910 "lines": true, 17911 "linewidth": 2, 17912 "links": [], 17913 "maxDataPoints": 100, 17914 "maxPerRow": null, 17915 "minSpan": null, 17916 "nullPointMode": null, 17917 "options": { 17918 "alertThreshold": true, 17919 "dataLinks": [] 17920 }, 17921 "percentage": false, 17922 "pointradius": 5, 17923 "points": false, 17924 "renderer": "flot", 17925 "repeat": null, 17926 "repeatDirection": null, 17927 "seriesOverrides": [], 17928 "span": 12, 17929 "stack": false, 17930 "steppedLine": false, 17931 "targets": [ 17932 { 17933 "datasource": "$source", 17934 "expr": "sum(container_memory_working_set_bytes{container!=\"\"}) by (container, instance)", 17935 "format": "time_series", 17936 "hide": false, 17937 "instant": false, 17938 "interval": "5s", 17939 "intervalFactor": 1, 17940 "legendFormat": "{{instance}}: {{container}}", 17941 "metric": "", 17942 "query": "sum(container_memory_working_set_bytes{container!=\"\"}) by (container, instance)", 17943 "refId": "", 17944 "step": 10, 17945 "target": "" 17946 }, 17947 { 17948 "datasource": "$source", 17949 "expr": "machine_memory_bytes", 17950 "format": "time_series", 17951 "hide": false, 17952 "instant": false, 17953 "interval": "5s", 17954 "intervalFactor": 1, 17955 "legendFormat": "limit", 17956 "metric": "", 17957 "query": "machine_memory_bytes", 17958 "refId": "", 17959 "step": 10, 17960 "target": "" 17961 } 17962 ], 17963 "thresholds": [], 17964 "timeFrom": null, 17965 "timeShift": "$timeshift", 17966 "title": "[SECONDARY] memory working set by container", 17967 "tooltip": { 17968 "msResolution": true, 17969 "shared": true, 17970 "sort": 2, 17971 "value_type": "cumulative" 17972 }, 17973 "transformations": [], 17974 "transparent": false, 17975 "type": "graph", 17976 "xaxis": { 17977 "mode": "time", 17978 "name": null, 17979 "show": true, 17980 "values": [] 17981 }, 17982 "yaxes": [ 17983 { 17984 "decimals": null, 17985 "format": "bytes", 17986 "label": null, 17987 "logBase": 1, 17988 "max": null, 17989 "min": null, 17990 "show": true 17991 }, 17992 { 17993 "decimals": null, 17994 "format": "short", 17995 "label": null, 17996 "logBase": 1, 17997 "max": null, 17998 "min": null, 17999 "show": true 18000 } 18001 ], 18002 "yaxis": { 18003 "align": false, 18004 "alignLevel": 0 18005 } 18006 }, 18007 { 18008 "aliasColors": {}, 18009 "bars": false, 18010 "cacheTimeout": null, 18011 "datasource": "$source", 18012 "description": null, 18013 "editable": true, 18014 "error": false, 18015 "fieldConfig": { 18016 "defaults": { 18017 "thresholds": { 18018 "mode": "absolute", 18019 "steps": [] 18020 } 18021 } 18022 }, 18023 "fill": 1, 18024 "grid": { 18025 "threshold1": null, 18026 "threshold1Color": "rgba(216, 200, 27, 0.27)", 18027 "threshold2": null, 18028 "threshold2Color": "rgba(234, 112, 112, 0.22)" 18029 }, 18030 "gridPos": null, 18031 "height": null, 18032 "hideTimeOverride": false, 18033 "id": 139, 18034 "interval": null, 18035 "isNew": true, 18036 "legend": { 18037 "alignAsTable": false, 18038 "avg": false, 18039 "current": false, 18040 "hideEmpty": false, 18041 "hideZero": false, 18042 "max": false, 18043 "min": false, 18044 "rightSide": false, 18045 "show": true, 18046 "sideWidth": null, 18047 "sort": null, 18048 "sortDesc": false, 18049 "total": false, 18050 "values": false 18051 }, 18052 "lines": true, 18053 "linewidth": 2, 18054 "links": [], 18055 "maxDataPoints": 100, 18056 "maxPerRow": null, 18057 "minSpan": null, 18058 "nullPointMode": null, 18059 "options": { 18060 "alertThreshold": true, 18061 "dataLinks": [] 18062 }, 18063 "percentage": false, 18064 "pointradius": 5, 18065 "points": false, 18066 "renderer": "flot", 18067 "repeat": null, 18068 "repeatDirection": null, 18069 "seriesOverrides": [], 18070 "span": 12, 18071 "stack": false, 18072 "steppedLine": false, 18073 "targets": [ 18074 { 18075 "datasource": "$source", 18076 "expr": "rate(container_network_transmit_bytes_total{id=\"/\"}[1m])", 18077 "format": "time_series", 18078 "hide": false, 18079 "instant": false, 18080 "interval": "", 18081 "intervalFactor": 2, 18082 "legendFormat": "{{instance}} transmit", 18083 "metric": "", 18084 "query": "rate(container_network_transmit_bytes_total{id=\"/\"}[1m])", 18085 "refId": "", 18086 "step": 10, 18087 "target": "" 18088 }, 18089 { 18090 "datasource": "$source", 18091 "expr": "rate(container_network_receive_bytes_total{id=\"/\"}[1m])", 18092 "format": "time_series", 18093 "hide": false, 18094 "instant": false, 18095 "interval": "", 18096 "intervalFactor": 2, 18097 "legendFormat": "{{instance}} receive", 18098 "metric": "", 18099 "query": "rate(container_network_receive_bytes_total{id=\"/\"}[1m])", 18100 "refId": "", 18101 "step": 10, 18102 "target": "" 18103 } 18104 ], 18105 "thresholds": [], 18106 "timeFrom": null, 18107 "timeShift": null, 18108 "title": "Network usage (bytes)", 18109 "tooltip": { 18110 "msResolution": true, 18111 "shared": true, 18112 "sort": 2, 18113 "value_type": "cumulative" 18114 }, 18115 "transformations": [], 18116 "transparent": false, 18117 "type": "graph", 18118 "xaxis": { 18119 "mode": "time", 18120 "name": null, 18121 "show": true, 18122 "values": [] 18123 }, 18124 "yaxes": [ 18125 { 18126 "decimals": null, 18127 "format": "Bps", 18128 "label": null, 18129 "logBase": 1, 18130 "max": null, 18131 "min": null, 18132 "show": true 18133 }, 18134 { 18135 "decimals": null, 18136 "format": "short", 18137 "label": null, 18138 "logBase": 1, 18139 "max": null, 18140 "min": null, 18141 "show": true 18142 } 18143 ], 18144 "yaxis": { 18145 "align": false, 18146 "alignLevel": 0 18147 } 18148 }, 18149 { 18150 "aliasColors": {}, 18151 "bars": false, 18152 "cacheTimeout": null, 18153 "datasource": "$secondary_source", 18154 "description": null, 18155 "editable": true, 18156 "error": false, 18157 "fieldConfig": { 18158 "defaults": { 18159 "thresholds": { 18160 "mode": "absolute", 18161 "steps": [] 18162 } 18163 } 18164 }, 18165 "fill": 1, 18166 "grid": { 18167 "threshold1": null, 18168 "threshold1Color": "rgba(216, 200, 27, 0.27)", 18169 "threshold2": null, 18170 "threshold2Color": "rgba(234, 112, 112, 0.22)" 18171 }, 18172 "gridPos": null, 18173 "height": null, 18174 "hideTimeOverride": false, 18175 "id": 140, 18176 "interval": null, 18177 "isNew": true, 18178 "legend": { 18179 "alignAsTable": false, 18180 "avg": false, 18181 "current": false, 18182 "hideEmpty": false, 18183 "hideZero": false, 18184 "max": false, 18185 "min": false, 18186 "rightSide": false, 18187 "show": true, 18188 "sideWidth": null, 18189 "sort": null, 18190 "sortDesc": false, 18191 "total": false, 18192 "values": false 18193 }, 18194 "lines": true, 18195 "linewidth": 2, 18196 "links": [], 18197 "maxDataPoints": 100, 18198 "maxPerRow": null, 18199 "minSpan": null, 18200 "nullPointMode": null, 18201 "options": { 18202 "alertThreshold": true, 18203 "dataLinks": [] 18204 }, 18205 "percentage": false, 18206 "pointradius": 5, 18207 "points": false, 18208 "renderer": "flot", 18209 "repeat": null, 18210 "repeatDirection": null, 18211 "seriesOverrides": [], 18212 "span": 12, 18213 "stack": false, 18214 "steppedLine": false, 18215 "targets": [ 18216 { 18217 "datasource": "$source", 18218 "expr": "rate(container_network_transmit_bytes_total{id=\"/\"}[1m])", 18219 "format": "time_series", 18220 "hide": false, 18221 "instant": false, 18222 "interval": "", 18223 "intervalFactor": 2, 18224 "legendFormat": "{{instance}} transmit", 18225 "metric": "", 18226 "query": "rate(container_network_transmit_bytes_total{id=\"/\"}[1m])", 18227 "refId": "", 18228 "step": 10, 18229 "target": "" 18230 }, 18231 { 18232 "datasource": "$source", 18233 "expr": "rate(container_network_receive_bytes_total{id=\"/\"}[1m])", 18234 "format": "time_series", 18235 "hide": false, 18236 "instant": false, 18237 "interval": "", 18238 "intervalFactor": 2, 18239 "legendFormat": "{{instance}} receive", 18240 "metric": "", 18241 "query": "rate(container_network_receive_bytes_total{id=\"/\"}[1m])", 18242 "refId": "", 18243 "step": 10, 18244 "target": "" 18245 } 18246 ], 18247 "thresholds": [], 18248 "timeFrom": null, 18249 "timeShift": "$timeshift", 18250 "title": "[SECONDARY] Network usage (bytes)", 18251 "tooltip": { 18252 "msResolution": true, 18253 "shared": true, 18254 "sort": 2, 18255 "value_type": "cumulative" 18256 }, 18257 "transformations": [], 18258 "transparent": false, 18259 "type": "graph", 18260 "xaxis": { 18261 "mode": "time", 18262 "name": null, 18263 "show": true, 18264 "values": [] 18265 }, 18266 "yaxes": [ 18267 { 18268 "decimals": null, 18269 "format": "Bps", 18270 "label": null, 18271 "logBase": 1, 18272 "max": null, 18273 "min": null, 18274 "show": true 18275 }, 18276 { 18277 "decimals": null, 18278 "format": "short", 18279 "label": null, 18280 "logBase": 1, 18281 "max": null, 18282 "min": null, 18283 "show": true 18284 } 18285 ], 18286 "yaxis": { 18287 "align": false, 18288 "alignLevel": 0 18289 } 18290 }, 18291 { 18292 "aliasColors": {}, 18293 "bars": false, 18294 "cacheTimeout": null, 18295 "datasource": "$source", 18296 "description": null, 18297 "editable": true, 18298 "error": false, 18299 "fieldConfig": { 18300 "defaults": { 18301 "thresholds": { 18302 "mode": "absolute", 18303 "steps": [] 18304 } 18305 } 18306 }, 18307 "fill": 1, 18308 "grid": { 18309 "threshold1": null, 18310 "threshold1Color": "rgba(216, 200, 27, 0.27)", 18311 "threshold2": null, 18312 "threshold2Color": "rgba(234, 112, 112, 0.22)" 18313 }, 18314 "gridPos": null, 18315 "height": null, 18316 "hideTimeOverride": false, 18317 "id": 141, 18318 "interval": null, 18319 "isNew": true, 18320 "legend": { 18321 "alignAsTable": false, 18322 "avg": false, 18323 "current": false, 18324 "hideEmpty": false, 18325 "hideZero": false, 18326 "max": false, 18327 "min": false, 18328 "rightSide": false, 18329 "show": true, 18330 "sideWidth": null, 18331 "sort": null, 18332 "sortDesc": false, 18333 "total": false, 18334 "values": false 18335 }, 18336 "lines": true, 18337 "linewidth": 2, 18338 "links": [], 18339 "maxDataPoints": 100, 18340 "maxPerRow": null, 18341 "minSpan": null, 18342 "nullPointMode": null, 18343 "options": { 18344 "alertThreshold": true, 18345 "dataLinks": [] 18346 }, 18347 "percentage": false, 18348 "pointradius": 5, 18349 "points": false, 18350 "renderer": "flot", 18351 "repeat": null, 18352 "repeatDirection": null, 18353 "seriesOverrides": [], 18354 "span": 12, 18355 "stack": false, 18356 "steppedLine": false, 18357 "targets": [ 18358 { 18359 "datasource": "$source", 18360 "expr": "rate(container_network_transmit_packets_total{id=\"/\"}[1m])", 18361 "format": "time_series", 18362 "hide": false, 18363 "instant": false, 18364 "interval": "", 18365 "intervalFactor": 2, 18366 "legendFormat": "{{instance}} transmit", 18367 "metric": "", 18368 "query": "rate(container_network_transmit_packets_total{id=\"/\"}[1m])", 18369 "refId": "", 18370 "step": 10, 18371 "target": "" 18372 }, 18373 { 18374 "datasource": "$source", 18375 "expr": "rate(container_network_receive_packets_total{id=\"/\"}[1m])", 18376 "format": "time_series", 18377 "hide": false, 18378 "instant": false, 18379 "interval": "", 18380 "intervalFactor": 2, 18381 "legendFormat": "{{instance}} receive", 18382 "metric": "", 18383 "query": "rate(container_network_receive_packets_total{id=\"/\"}[1m])", 18384 "refId": "", 18385 "step": 10, 18386 "target": "" 18387 } 18388 ], 18389 "thresholds": [], 18390 "timeFrom": null, 18391 "timeShift": null, 18392 "title": "Network usage (packets)", 18393 "tooltip": { 18394 "msResolution": true, 18395 "shared": true, 18396 "sort": 2, 18397 "value_type": "cumulative" 18398 }, 18399 "transformations": [], 18400 "transparent": false, 18401 "type": "graph", 18402 "xaxis": { 18403 "mode": "time", 18404 "name": null, 18405 "show": true, 18406 "values": [] 18407 }, 18408 "yaxes": [ 18409 { 18410 "decimals": null, 18411 "format": "short", 18412 "label": null, 18413 "logBase": 1, 18414 "max": null, 18415 "min": null, 18416 "show": true 18417 }, 18418 { 18419 "decimals": null, 18420 "format": "short", 18421 "label": null, 18422 "logBase": 1, 18423 "max": null, 18424 "min": null, 18425 "show": true 18426 } 18427 ], 18428 "yaxis": { 18429 "align": false, 18430 "alignLevel": 0 18431 } 18432 }, 18433 { 18434 "aliasColors": {}, 18435 "bars": false, 18436 "cacheTimeout": null, 18437 "datasource": "$secondary_source", 18438 "description": null, 18439 "editable": true, 18440 "error": false, 18441 "fieldConfig": { 18442 "defaults": { 18443 "thresholds": { 18444 "mode": "absolute", 18445 "steps": [] 18446 } 18447 } 18448 }, 18449 "fill": 1, 18450 "grid": { 18451 "threshold1": null, 18452 "threshold1Color": "rgba(216, 200, 27, 0.27)", 18453 "threshold2": null, 18454 "threshold2Color": "rgba(234, 112, 112, 0.22)" 18455 }, 18456 "gridPos": null, 18457 "height": null, 18458 "hideTimeOverride": false, 18459 "id": 142, 18460 "interval": null, 18461 "isNew": true, 18462 "legend": { 18463 "alignAsTable": false, 18464 "avg": false, 18465 "current": false, 18466 "hideEmpty": false, 18467 "hideZero": false, 18468 "max": false, 18469 "min": false, 18470 "rightSide": false, 18471 "show": true, 18472 "sideWidth": null, 18473 "sort": null, 18474 "sortDesc": false, 18475 "total": false, 18476 "values": false 18477 }, 18478 "lines": true, 18479 "linewidth": 2, 18480 "links": [], 18481 "maxDataPoints": 100, 18482 "maxPerRow": null, 18483 "minSpan": null, 18484 "nullPointMode": null, 18485 "options": { 18486 "alertThreshold": true, 18487 "dataLinks": [] 18488 }, 18489 "percentage": false, 18490 "pointradius": 5, 18491 "points": false, 18492 "renderer": "flot", 18493 "repeat": null, 18494 "repeatDirection": null, 18495 "seriesOverrides": [], 18496 "span": 12, 18497 "stack": false, 18498 "steppedLine": false, 18499 "targets": [ 18500 { 18501 "datasource": "$source", 18502 "expr": "rate(container_network_transmit_packets_total{id=\"/\"}[1m])", 18503 "format": "time_series", 18504 "hide": false, 18505 "instant": false, 18506 "interval": "", 18507 "intervalFactor": 2, 18508 "legendFormat": "{{instance}} transmit", 18509 "metric": "", 18510 "query": "rate(container_network_transmit_packets_total{id=\"/\"}[1m])", 18511 "refId": "", 18512 "step": 10, 18513 "target": "" 18514 }, 18515 { 18516 "datasource": "$source", 18517 "expr": "rate(container_network_receive_packets_total{id=\"/\"}[1m])", 18518 "format": "time_series", 18519 "hide": false, 18520 "instant": false, 18521 "interval": "", 18522 "intervalFactor": 2, 18523 "legendFormat": "{{instance}} receive", 18524 "metric": "", 18525 "query": "rate(container_network_receive_packets_total{id=\"/\"}[1m])", 18526 "refId": "", 18527 "step": 10, 18528 "target": "" 18529 } 18530 ], 18531 "thresholds": [], 18532 "timeFrom": null, 18533 "timeShift": "$timeshift", 18534 "title": "[SECONDARY] Network usage (packets)", 18535 "tooltip": { 18536 "msResolution": true, 18537 "shared": true, 18538 "sort": 2, 18539 "value_type": "cumulative" 18540 }, 18541 "transformations": [], 18542 "transparent": false, 18543 "type": "graph", 18544 "xaxis": { 18545 "mode": "time", 18546 "name": null, 18547 "show": true, 18548 "values": [] 18549 }, 18550 "yaxes": [ 18551 { 18552 "decimals": null, 18553 "format": "short", 18554 "label": null, 18555 "logBase": 1, 18556 "max": null, 18557 "min": null, 18558 "show": true 18559 }, 18560 { 18561 "decimals": null, 18562 "format": "short", 18563 "label": null, 18564 "logBase": 1, 18565 "max": null, 18566 "min": null, 18567 "show": true 18568 } 18569 ], 18570 "yaxis": { 18571 "align": false, 18572 "alignLevel": 0 18573 } 18574 }, 18575 { 18576 "aliasColors": {}, 18577 "bars": false, 18578 "cacheTimeout": null, 18579 "datasource": "$source", 18580 "description": null, 18581 "editable": true, 18582 "error": false, 18583 "fieldConfig": { 18584 "defaults": { 18585 "thresholds": { 18586 "mode": "absolute", 18587 "steps": [] 18588 } 18589 } 18590 }, 18591 "fill": 1, 18592 "grid": { 18593 "threshold1": null, 18594 "threshold1Color": "rgba(216, 200, 27, 0.27)", 18595 "threshold2": null, 18596 "threshold2Color": "rgba(234, 112, 112, 0.22)" 18597 }, 18598 "gridPos": null, 18599 "height": null, 18600 "hideTimeOverride": false, 18601 "id": 143, 18602 "interval": null, 18603 "isNew": true, 18604 "legend": { 18605 "alignAsTable": false, 18606 "avg": false, 18607 "current": false, 18608 "hideEmpty": false, 18609 "hideZero": false, 18610 "max": false, 18611 "min": false, 18612 "rightSide": false, 18613 "show": true, 18614 "sideWidth": null, 18615 "sort": null, 18616 "sortDesc": false, 18617 "total": false, 18618 "values": false 18619 }, 18620 "lines": true, 18621 "linewidth": 2, 18622 "links": [], 18623 "maxDataPoints": 100, 18624 "maxPerRow": null, 18625 "minSpan": null, 18626 "nullPointMode": null, 18627 "options": { 18628 "alertThreshold": true, 18629 "dataLinks": [] 18630 }, 18631 "percentage": false, 18632 "pointradius": 5, 18633 "points": false, 18634 "renderer": "flot", 18635 "repeat": null, 18636 "repeatDirection": null, 18637 "seriesOverrides": [], 18638 "span": 12, 18639 "stack": false, 18640 "steppedLine": false, 18641 "targets": [ 18642 { 18643 "datasource": "$source", 18644 "expr": "rate(container_network_transmit_bytes_total{id=\"/\"}[1m]) / rate(container_network_transmit_packets_total{id=\"/\"}[1m])", 18645 "format": "time_series", 18646 "hide": false, 18647 "instant": false, 18648 "interval": "", 18649 "intervalFactor": 2, 18650 "legendFormat": "{{instance}} transmit", 18651 "metric": "", 18652 "query": "rate(container_network_transmit_bytes_total{id=\"/\"}[1m]) / rate(container_network_transmit_packets_total{id=\"/\"}[1m])", 18653 "refId": "", 18654 "step": 10, 18655 "target": "" 18656 }, 18657 { 18658 "datasource": "$source", 18659 "expr": "rate(container_network_receive_bytes_total{id=\"/\"}[1m]) / rate(container_network_receive_packets_total{id=\"/\"}[1m])", 18660 "format": "time_series", 18661 "hide": false, 18662 "instant": false, 18663 "interval": "", 18664 "intervalFactor": 2, 18665 "legendFormat": "{{instance}} receive", 18666 "metric": "", 18667 "query": "rate(container_network_receive_bytes_total{id=\"/\"}[1m]) / rate(container_network_receive_packets_total{id=\"/\"}[1m])", 18668 "refId": "", 18669 "step": 10, 18670 "target": "" 18671 } 18672 ], 18673 "thresholds": [], 18674 "timeFrom": null, 18675 "timeShift": null, 18676 "title": "Network usage (avg packet size)", 18677 "tooltip": { 18678 "msResolution": true, 18679 "shared": true, 18680 "sort": 2, 18681 "value_type": "cumulative" 18682 }, 18683 "transformations": [], 18684 "transparent": false, 18685 "type": "graph", 18686 "xaxis": { 18687 "mode": "time", 18688 "name": null, 18689 "show": true, 18690 "values": [] 18691 }, 18692 "yaxes": [ 18693 { 18694 "decimals": null, 18695 "format": "bytes", 18696 "label": null, 18697 "logBase": 1, 18698 "max": null, 18699 "min": null, 18700 "show": true 18701 }, 18702 { 18703 "decimals": null, 18704 "format": "short", 18705 "label": null, 18706 "logBase": 1, 18707 "max": null, 18708 "min": null, 18709 "show": true 18710 } 18711 ], 18712 "yaxis": { 18713 "align": false, 18714 "alignLevel": 0 18715 } 18716 }, 18717 { 18718 "aliasColors": {}, 18719 "bars": false, 18720 "cacheTimeout": null, 18721 "datasource": "$secondary_source", 18722 "description": null, 18723 "editable": true, 18724 "error": false, 18725 "fieldConfig": { 18726 "defaults": { 18727 "thresholds": { 18728 "mode": "absolute", 18729 "steps": [] 18730 } 18731 } 18732 }, 18733 "fill": 1, 18734 "grid": { 18735 "threshold1": null, 18736 "threshold1Color": "rgba(216, 200, 27, 0.27)", 18737 "threshold2": null, 18738 "threshold2Color": "rgba(234, 112, 112, 0.22)" 18739 }, 18740 "gridPos": null, 18741 "height": null, 18742 "hideTimeOverride": false, 18743 "id": 144, 18744 "interval": null, 18745 "isNew": true, 18746 "legend": { 18747 "alignAsTable": false, 18748 "avg": false, 18749 "current": false, 18750 "hideEmpty": false, 18751 "hideZero": false, 18752 "max": false, 18753 "min": false, 18754 "rightSide": false, 18755 "show": true, 18756 "sideWidth": null, 18757 "sort": null, 18758 "sortDesc": false, 18759 "total": false, 18760 "values": false 18761 }, 18762 "lines": true, 18763 "linewidth": 2, 18764 "links": [], 18765 "maxDataPoints": 100, 18766 "maxPerRow": null, 18767 "minSpan": null, 18768 "nullPointMode": null, 18769 "options": { 18770 "alertThreshold": true, 18771 "dataLinks": [] 18772 }, 18773 "percentage": false, 18774 "pointradius": 5, 18775 "points": false, 18776 "renderer": "flot", 18777 "repeat": null, 18778 "repeatDirection": null, 18779 "seriesOverrides": [], 18780 "span": 12, 18781 "stack": false, 18782 "steppedLine": false, 18783 "targets": [ 18784 { 18785 "datasource": "$source", 18786 "expr": "rate(container_network_transmit_bytes_total{id=\"/\"}[1m]) / rate(container_network_transmit_packets_total{id=\"/\"}[1m])", 18787 "format": "time_series", 18788 "hide": false, 18789 "instant": false, 18790 "interval": "", 18791 "intervalFactor": 2, 18792 "legendFormat": "{{instance}} transmit", 18793 "metric": "", 18794 "query": "rate(container_network_transmit_bytes_total{id=\"/\"}[1m]) / rate(container_network_transmit_packets_total{id=\"/\"}[1m])", 18795 "refId": "", 18796 "step": 10, 18797 "target": "" 18798 }, 18799 { 18800 "datasource": "$source", 18801 "expr": "rate(container_network_receive_bytes_total{id=\"/\"}[1m]) / rate(container_network_receive_packets_total{id=\"/\"}[1m])", 18802 "format": "time_series", 18803 "hide": false, 18804 "instant": false, 18805 "interval": "", 18806 "intervalFactor": 2, 18807 "legendFormat": "{{instance}} receive", 18808 "metric": "", 18809 "query": "rate(container_network_receive_bytes_total{id=\"/\"}[1m]) / rate(container_network_receive_packets_total{id=\"/\"}[1m])", 18810 "refId": "", 18811 "step": 10, 18812 "target": "" 18813 } 18814 ], 18815 "thresholds": [], 18816 "timeFrom": null, 18817 "timeShift": "$timeshift", 18818 "title": "[SECONDARY] Network usage (avg packet size)", 18819 "tooltip": { 18820 "msResolution": true, 18821 "shared": true, 18822 "sort": 2, 18823 "value_type": "cumulative" 18824 }, 18825 "transformations": [], 18826 "transparent": false, 18827 "type": "graph", 18828 "xaxis": { 18829 "mode": "time", 18830 "name": null, 18831 "show": true, 18832 "values": [] 18833 }, 18834 "yaxes": [ 18835 { 18836 "decimals": null, 18837 "format": "bytes", 18838 "label": null, 18839 "logBase": 1, 18840 "max": null, 18841 "min": null, 18842 "show": true 18843 }, 18844 { 18845 "decimals": null, 18846 "format": "short", 18847 "label": null, 18848 "logBase": 1, 18849 "max": null, 18850 "min": null, 18851 "show": true 18852 } 18853 ], 18854 "yaxis": { 18855 "align": false, 18856 "alignLevel": 0 18857 } 18858 }, 18859 { 18860 "aliasColors": {}, 18861 "bars": false, 18862 "cacheTimeout": null, 18863 "datasource": "$source", 18864 "description": null, 18865 "editable": true, 18866 "error": false, 18867 "fieldConfig": { 18868 "defaults": { 18869 "thresholds": { 18870 "mode": "absolute", 18871 "steps": [] 18872 } 18873 } 18874 }, 18875 "fill": 1, 18876 "grid": { 18877 "threshold1": null, 18878 "threshold1Color": "rgba(216, 200, 27, 0.27)", 18879 "threshold2": null, 18880 "threshold2Color": "rgba(234, 112, 112, 0.22)" 18881 }, 18882 "gridPos": null, 18883 "height": null, 18884 "hideTimeOverride": false, 18885 "id": 145, 18886 "interval": null, 18887 "isNew": true, 18888 "legend": { 18889 "alignAsTable": false, 18890 "avg": false, 18891 "current": false, 18892 "hideEmpty": false, 18893 "hideZero": false, 18894 "max": false, 18895 "min": false, 18896 "rightSide": false, 18897 "show": true, 18898 "sideWidth": null, 18899 "sort": null, 18900 "sortDesc": false, 18901 "total": false, 18902 "values": false 18903 }, 18904 "lines": true, 18905 "linewidth": 2, 18906 "links": [], 18907 "maxDataPoints": 100, 18908 "maxPerRow": null, 18909 "minSpan": null, 18910 "nullPointMode": null, 18911 "options": { 18912 "alertThreshold": true, 18913 "dataLinks": [] 18914 }, 18915 "percentage": false, 18916 "pointradius": 5, 18917 "points": false, 18918 "renderer": "flot", 18919 "repeat": null, 18920 "repeatDirection": null, 18921 "seriesOverrides": [], 18922 "span": 12, 18923 "stack": false, 18924 "steppedLine": false, 18925 "targets": [ 18926 { 18927 "datasource": "$source", 18928 "expr": "sum(rate(node_netstat_Tcp_InSegs[1m])) by (instance)", 18929 "format": "time_series", 18930 "hide": false, 18931 "instant": false, 18932 "interval": "", 18933 "intervalFactor": 2, 18934 "legendFormat": "InSegs {{instance}}", 18935 "metric": "", 18936 "query": "sum(rate(node_netstat_Tcp_InSegs[1m])) by (instance)", 18937 "refId": "", 18938 "step": 10, 18939 "target": "" 18940 }, 18941 { 18942 "datasource": "$source", 18943 "expr": "sum(rate(node_netstat_Tcp_OutSegs[1m])) by (instance)", 18944 "format": "time_series", 18945 "hide": false, 18946 "instant": false, 18947 "interval": "", 18948 "intervalFactor": 2, 18949 "legendFormat": "OutSegs {{instance}}", 18950 "metric": "", 18951 "query": "sum(rate(node_netstat_Tcp_OutSegs[1m])) by (instance)", 18952 "refId": "", 18953 "step": 10, 18954 "target": "" 18955 }, 18956 { 18957 "datasource": "$source", 18958 "expr": "sum(rate(node_netstat_Tcp_RetransSegs[1m])) by (instance)", 18959 "format": "time_series", 18960 "hide": false, 18961 "instant": false, 18962 "interval": "", 18963 "intervalFactor": 2, 18964 "legendFormat": "RetransSegs {{instance}}", 18965 "metric": "", 18966 "query": "sum(rate(node_netstat_Tcp_RetransSegs[1m])) by (instance)", 18967 "refId": "", 18968 "step": 10, 18969 "target": "" 18970 } 18971 ], 18972 "thresholds": [], 18973 "timeFrom": null, 18974 "timeShift": null, 18975 "title": "Network tcp segments", 18976 "tooltip": { 18977 "msResolution": true, 18978 "shared": true, 18979 "sort": 2, 18980 "value_type": "cumulative" 18981 }, 18982 "transformations": [], 18983 "transparent": false, 18984 "type": "graph", 18985 "xaxis": { 18986 "mode": "time", 18987 "name": null, 18988 "show": true, 18989 "values": [] 18990 }, 18991 "yaxes": [ 18992 { 18993 "decimals": null, 18994 "format": "short", 18995 "label": null, 18996 "logBase": 10, 18997 "max": null, 18998 "min": null, 18999 "show": true 19000 }, 19001 { 19002 "decimals": null, 19003 "format": "short", 19004 "label": null, 19005 "logBase": 1, 19006 "max": null, 19007 "min": null, 19008 "show": true 19009 } 19010 ], 19011 "yaxis": { 19012 "align": false, 19013 "alignLevel": 0 19014 } 19015 }, 19016 { 19017 "aliasColors": {}, 19018 "bars": false, 19019 "cacheTimeout": null, 19020 "datasource": "$secondary_source", 19021 "description": null, 19022 "editable": true, 19023 "error": false, 19024 "fieldConfig": { 19025 "defaults": { 19026 "thresholds": { 19027 "mode": "absolute", 19028 "steps": [] 19029 } 19030 } 19031 }, 19032 "fill": 1, 19033 "grid": { 19034 "threshold1": null, 19035 "threshold1Color": "rgba(216, 200, 27, 0.27)", 19036 "threshold2": null, 19037 "threshold2Color": "rgba(234, 112, 112, 0.22)" 19038 }, 19039 "gridPos": null, 19040 "height": null, 19041 "hideTimeOverride": false, 19042 "id": 146, 19043 "interval": null, 19044 "isNew": true, 19045 "legend": { 19046 "alignAsTable": false, 19047 "avg": false, 19048 "current": false, 19049 "hideEmpty": false, 19050 "hideZero": false, 19051 "max": false, 19052 "min": false, 19053 "rightSide": false, 19054 "show": true, 19055 "sideWidth": null, 19056 "sort": null, 19057 "sortDesc": false, 19058 "total": false, 19059 "values": false 19060 }, 19061 "lines": true, 19062 "linewidth": 2, 19063 "links": [], 19064 "maxDataPoints": 100, 19065 "maxPerRow": null, 19066 "minSpan": null, 19067 "nullPointMode": null, 19068 "options": { 19069 "alertThreshold": true, 19070 "dataLinks": [] 19071 }, 19072 "percentage": false, 19073 "pointradius": 5, 19074 "points": false, 19075 "renderer": "flot", 19076 "repeat": null, 19077 "repeatDirection": null, 19078 "seriesOverrides": [], 19079 "span": 12, 19080 "stack": false, 19081 "steppedLine": false, 19082 "targets": [ 19083 { 19084 "datasource": "$source", 19085 "expr": "sum(rate(node_netstat_Tcp_InSegs[1m])) by (instance)", 19086 "format": "time_series", 19087 "hide": false, 19088 "instant": false, 19089 "interval": "", 19090 "intervalFactor": 2, 19091 "legendFormat": "InSegs {{instance}}", 19092 "metric": "", 19093 "query": "sum(rate(node_netstat_Tcp_InSegs[1m])) by (instance)", 19094 "refId": "", 19095 "step": 10, 19096 "target": "" 19097 }, 19098 { 19099 "datasource": "$source", 19100 "expr": "sum(rate(node_netstat_Tcp_OutSegs[1m])) by (instance)", 19101 "format": "time_series", 19102 "hide": false, 19103 "instant": false, 19104 "interval": "", 19105 "intervalFactor": 2, 19106 "legendFormat": "OutSegs {{instance}}", 19107 "metric": "", 19108 "query": "sum(rate(node_netstat_Tcp_OutSegs[1m])) by (instance)", 19109 "refId": "", 19110 "step": 10, 19111 "target": "" 19112 }, 19113 { 19114 "datasource": "$source", 19115 "expr": "sum(rate(node_netstat_Tcp_RetransSegs[1m])) by (instance)", 19116 "format": "time_series", 19117 "hide": false, 19118 "instant": false, 19119 "interval": "", 19120 "intervalFactor": 2, 19121 "legendFormat": "RetransSegs {{instance}}", 19122 "metric": "", 19123 "query": "sum(rate(node_netstat_Tcp_RetransSegs[1m])) by (instance)", 19124 "refId": "", 19125 "step": 10, 19126 "target": "" 19127 } 19128 ], 19129 "thresholds": [], 19130 "timeFrom": null, 19131 "timeShift": "$timeshift", 19132 "title": "[SECONDARY] Network tcp segments", 19133 "tooltip": { 19134 "msResolution": true, 19135 "shared": true, 19136 "sort": 2, 19137 "value_type": "cumulative" 19138 }, 19139 "transformations": [], 19140 "transparent": false, 19141 "type": "graph", 19142 "xaxis": { 19143 "mode": "time", 19144 "name": null, 19145 "show": true, 19146 "values": [] 19147 }, 19148 "yaxes": [ 19149 { 19150 "decimals": null, 19151 "format": "short", 19152 "label": null, 19153 "logBase": 10, 19154 "max": null, 19155 "min": null, 19156 "show": true 19157 }, 19158 { 19159 "decimals": null, 19160 "format": "short", 19161 "label": null, 19162 "logBase": 1, 19163 "max": null, 19164 "min": null, 19165 "show": true 19166 } 19167 ], 19168 "yaxis": { 19169 "align": false, 19170 "alignLevel": 0 19171 } 19172 } 19173 ], 19174 "repeat": null, 19175 "showTitle": true, 19176 "title": "Master VM" 19177 } 19178 ], 19179 "schemaVersion": 12, 19180 "sharedCrosshair": false, 19181 "style": "dark", 19182 "tags": [], 19183 "templating": { 19184 "list": [ 19185 { 19186 "allValue": null, 19187 "auto": false, 19188 "auto_count": 30, 19189 "auto_min": "10s", 19190 "current": { 19191 "selected": false, 19192 "tags": [], 19193 "text": null, 19194 "value": null 19195 }, 19196 "datasource": null, 19197 "hide": 0, 19198 "includeAll": false, 19199 "label": null, 19200 "multi": false, 19201 "name": "source", 19202 "options": [], 19203 "query": "prometheus", 19204 "refresh": 1, 19205 "regex": null, 19206 "sort": 1, 19207 "tagValuesQuery": null, 19208 "tagsQuery": null, 19209 "type": "datasource", 19210 "useTags": false 19211 }, 19212 { 19213 "allValue": null, 19214 "auto": false, 19215 "auto_count": 30, 19216 "auto_min": "10s", 19217 "current": { 19218 "selected": false, 19219 "tags": [], 19220 "text": null, 19221 "value": null 19222 }, 19223 "datasource": null, 19224 "hide": 0, 19225 "includeAll": false, 19226 "label": null, 19227 "multi": false, 19228 "name": "secondary_source", 19229 "options": [], 19230 "query": "prometheus", 19231 "refresh": 1, 19232 "regex": null, 19233 "sort": 1, 19234 "tagValuesQuery": null, 19235 "tagsQuery": null, 19236 "type": "datasource", 19237 "useTags": false 19238 }, 19239 { 19240 "allValue": null, 19241 "auto": false, 19242 "auto_count": 30, 19243 "auto_min": "10s", 19244 "current": { 19245 "selected": false, 19246 "tags": [], 19247 "text": null, 19248 "value": null 19249 }, 19250 "datasource": null, 19251 "hide": 0, 19252 "includeAll": false, 19253 "label": null, 19254 "multi": false, 19255 "name": "timeshift", 19256 "options": [], 19257 "query": "", 19258 "refresh": 1, 19259 "regex": null, 19260 "sort": 1, 19261 "tagValuesQuery": null, 19262 "tagsQuery": null, 19263 "type": "interval", 19264 "useTags": false 19265 }, 19266 { 19267 "allValue": null, 19268 "auto": false, 19269 "auto_count": 30, 19270 "auto_min": "10s", 19271 "current": { 19272 "selected": false, 19273 "tags": [], 19274 "text": null, 19275 "value": null 19276 }, 19277 "datasource": "$source", 19278 "hide": 0, 19279 "includeAll": true, 19280 "label": null, 19281 "multi": true, 19282 "name": "etcd_type", 19283 "options": [], 19284 "query": "label_values(etcd_request_duration_seconds_count, type)", 19285 "refresh": 2, 19286 "regex": "\\*\\[+\\]+(.*)", 19287 "sort": 1, 19288 "tagValuesQuery": null, 19289 "tagsQuery": null, 19290 "type": "query", 19291 "useTags": false 19292 }, 19293 { 19294 "allValue": null, 19295 "auto": false, 19296 "auto_count": 30, 19297 "auto_min": "10s", 19298 "current": { 19299 "selected": false, 19300 "tags": [], 19301 "text": null, 19302 "value": null 19303 }, 19304 "datasource": "$source", 19305 "hide": 0, 19306 "includeAll": true, 19307 "label": null, 19308 "multi": true, 19309 "name": "etcd_operation", 19310 "options": [], 19311 "query": "label_values(etcd_request_duration_seconds_count, operation)", 19312 "refresh": 2, 19313 "regex": null, 19314 "sort": 1, 19315 "tagValuesQuery": null, 19316 "tagsQuery": null, 19317 "type": "query", 19318 "useTags": false 19319 }, 19320 { 19321 "allValue": null, 19322 "auto": false, 19323 "auto_count": 30, 19324 "auto_min": "10s", 19325 "current": { 19326 "selected": false, 19327 "tags": [], 19328 "text": null, 19329 "value": null 19330 }, 19331 "datasource": "$source", 19332 "hide": 0, 19333 "includeAll": true, 19334 "label": null, 19335 "multi": true, 19336 "name": "verb", 19337 "options": [], 19338 "query": "label_values(apiserver_request_duration_seconds_count, verb)", 19339 "refresh": 2, 19340 "regex": null, 19341 "sort": 1, 19342 "tagValuesQuery": null, 19343 "tagsQuery": null, 19344 "type": "query", 19345 "useTags": false 19346 }, 19347 { 19348 "allValue": null, 19349 "auto": false, 19350 "auto_count": 30, 19351 "auto_min": "10s", 19352 "current": { 19353 "selected": false, 19354 "tags": [], 19355 "text": null, 19356 "value": null 19357 }, 19358 "datasource": "$source", 19359 "hide": 0, 19360 "includeAll": true, 19361 "label": null, 19362 "multi": true, 19363 "name": "resource", 19364 "options": [], 19365 "query": "label_values(apiserver_request_duration_seconds_count, resource)", 19366 "refresh": 2, 19367 "regex": "(.*)s", 19368 "sort": 1, 19369 "tagValuesQuery": null, 19370 "tagsQuery": null, 19371 "type": "query", 19372 "useTags": false 19373 } 19374 ] 19375 }, 19376 "time": { 19377 "from": "now-30d", 19378 "to": "now" 19379 }, 19380 "timepicker": { 19381 "hidden": false, 19382 "refresh_intervals": [ 19383 "5s", 19384 "10s", 19385 "30s", 19386 "1m", 19387 "5m", 19388 "15m", 19389 "30m", 19390 "1h", 19391 "2h", 19392 "1d" 19393 ], 19394 "time_options": [ 19395 "5m", 19396 "15m", 19397 "1h", 19398 "6h", 19399 "12h", 19400 "24h", 19401 "2d", 19402 "7d", 19403 "30d" 19404 ] 19405 }, 19406 "timezone": "utc", 19407 "title": "Comparison Master dashboard", 19408 "uid": null, 19409 "version": 0 19410 }