sigs.k8s.io/cluster-api@v1.7.1/hack/observability/grafana/dashboards/controller-runtime.json (about) 1 { 2 "annotations": { 3 "list": [ 4 { 5 "builtIn": 1, 6 "datasource": { 7 "type": "grafana", 8 "uid": "-- Grafana --" 9 }, 10 "enable": true, 11 "hide": true, 12 "iconColor": "rgba(0, 211, 255, 1)", 13 "name": "Annotations & Alerts", 14 "type": "dashboard" 15 } 16 ] 17 }, 18 "description": "", 19 "editable": true, 20 "fiscalYearStartMonth": 0, 21 "graphTooltip": 0, 22 "id": 2, 23 "links": [], 24 "liveNow": false, 25 "panels": [ 26 { 27 "datasource": { 28 "type": "prometheus", 29 "uid": "prometheus" 30 }, 31 "fieldConfig": { 32 "defaults": { 33 "color": { 34 "mode": "thresholds" 35 }, 36 "custom": { 37 "align": "auto", 38 "cellOptions": { 39 "type": "auto" 40 }, 41 "filterable": true, 42 "inspect": false 43 }, 44 "mappings": [], 45 "thresholds": { 46 "mode": "absolute", 47 "steps": [ 48 { 49 "color": "green", 50 "value": null 51 }, 52 { 53 "color": "red", 54 "value": 80 55 } 56 ] 57 } 58 }, 59 "overrides": [] 60 }, 61 "gridPos": { 62 "h": 5, 63 "w": 24, 64 "x": 0, 65 "y": 0 66 }, 67 "id": 17, 68 "options": { 69 "cellHeight": "sm", 70 "footer": { 71 "countRows": false, 72 "fields": [], 73 "reducer": [ 74 "sum" 75 ], 76 "show": false 77 }, 78 "showHeader": true 79 }, 80 "pluginVersion": "9.5.2", 81 "targets": [ 82 { 83 "datasource": { 84 "type": "prometheus", 85 "uid": "prometheus" 86 }, 87 "editorMode": "code", 88 "exemplar": false, 89 "expr": "kube_pod_info{pod=~\"$Pod\"}", 90 "format": "table", 91 "instant": true, 92 "legendFormat": "__auto", 93 "range": false, 94 "refId": "A" 95 }, 96 { 97 "datasource": { 98 "type": "prometheus", 99 "uid": "prometheus" 100 }, 101 "editorMode": "code", 102 "exemplar": false, 103 "expr": "kube_pod_container_status_restarts_total{pod=~\"$Pod\"}", 104 "format": "table", 105 "hide": false, 106 "instant": true, 107 "legendFormat": "__auto", 108 "range": false, 109 "refId": "B" 110 }, 111 { 112 "datasource": { 113 "type": "prometheus", 114 "uid": "prometheus" 115 }, 116 "editorMode": "code", 117 "exemplar": false, 118 "expr": "kube_pod_status_phase{pod=~\"$Pod\"}==1", 119 "format": "table", 120 "hide": false, 121 "instant": true, 122 "legendFormat": "__auto", 123 "range": false, 124 "refId": "D" 125 } 126 ], 127 "title": "Pods", 128 "transformations": [ 129 { 130 "id": "joinByField", 131 "options": { 132 "byField": "pod", 133 "mode": "outer" 134 } 135 }, 136 { 137 "id": "organize", 138 "options": { 139 "excludeByName": { 140 "Time": true, 141 "Time 1": true, 142 "Time 2": true, 143 "Time 3": true, 144 "Time 4": true, 145 "Value": true, 146 "Value #A": true, 147 "Value #C": true, 148 "Value #D": true, 149 "__name__": true, 150 "__name__ 1": true, 151 "__name__ 2": true, 152 "__name__ 3": true, 153 "app_kubernetes_io_component": true, 154 "app_kubernetes_io_component 1": true, 155 "app_kubernetes_io_component 2": true, 156 "app_kubernetes_io_component 3": true, 157 "app_kubernetes_io_instance": true, 158 "app_kubernetes_io_instance 1": true, 159 "app_kubernetes_io_instance 2": true, 160 "app_kubernetes_io_instance 3": true, 161 "app_kubernetes_io_managed_by": true, 162 "app_kubernetes_io_managed_by 1": true, 163 "app_kubernetes_io_managed_by 2": true, 164 "app_kubernetes_io_managed_by 3": true, 165 "app_kubernetes_io_name": true, 166 "app_kubernetes_io_name 1": true, 167 "app_kubernetes_io_name 2": true, 168 "app_kubernetes_io_name 3": true, 169 "app_kubernetes_io_part_of": true, 170 "app_kubernetes_io_part_of 1": true, 171 "app_kubernetes_io_part_of 2": true, 172 "app_kubernetes_io_part_of 3": true, 173 "app_kubernetes_io_version": true, 174 "app_kubernetes_io_version 1": true, 175 "app_kubernetes_io_version 2": true, 176 "app_kubernetes_io_version 3": true, 177 "container": true, 178 "container 1": true, 179 "container 2": true, 180 "created_by_kind": true, 181 "created_by_name": true, 182 "helm_sh_chart": true, 183 "helm_sh_chart 1": true, 184 "helm_sh_chart 2": true, 185 "helm_sh_chart 3": true, 186 "host_ip": true, 187 "host_network": true, 188 "instance": true, 189 "instance 1": true, 190 "instance 2": true, 191 "instance 3": true, 192 "job": true, 193 "job 1": true, 194 "job 2": true, 195 "job 3": true, 196 "namespace 2": true, 197 "namespace 3": true, 198 "node 2": true, 199 "node 3": true, 200 "priority_class": true, 201 "service": true, 202 "service 1": true, 203 "service 2": true, 204 "service 3": true, 205 "uid": true, 206 "uid 1": true, 207 "uid 2": true, 208 "uid 3": true 209 }, 210 "indexByName": { 211 "Time 1": 11, 212 "Time 2": 25, 213 "Time 3": 40, 214 "Time 4": 42, 215 "Value #A": 24, 216 "Value #B": 5, 217 "Value #C": 57, 218 "Value #D": 58, 219 "__name__ 1": 12, 220 "__name__ 2": 26, 221 "__name__ 3": 43, 222 "app_kubernetes_io_component 1": 13, 223 "app_kubernetes_io_component 2": 27, 224 "app_kubernetes_io_component 3": 44, 225 "app_kubernetes_io_instance 1": 14, 226 "app_kubernetes_io_instance 2": 28, 227 "app_kubernetes_io_instance 3": 45, 228 "app_kubernetes_io_managed_by 1": 15, 229 "app_kubernetes_io_managed_by 2": 29, 230 "app_kubernetes_io_managed_by 3": 46, 231 "app_kubernetes_io_name 1": 16, 232 "app_kubernetes_io_name 2": 30, 233 "app_kubernetes_io_name 3": 47, 234 "app_kubernetes_io_part_of 1": 17, 235 "app_kubernetes_io_part_of 2": 31, 236 "app_kubernetes_io_part_of 3": 48, 237 "app_kubernetes_io_version 1": 18, 238 "app_kubernetes_io_version 2": 32, 239 "app_kubernetes_io_version 3": 49, 240 "container 1": 6, 241 "container 2": 41, 242 "created_by_kind": 7, 243 "created_by_name": 8, 244 "helm_sh_chart 1": 19, 245 "helm_sh_chart 2": 33, 246 "helm_sh_chart 3": 50, 247 "host_ip": 9, 248 "host_network": 10, 249 "instance 1": 20, 250 "instance 2": 34, 251 "instance 3": 51, 252 "job 1": 21, 253 "job 2": 35, 254 "job 3": 52, 255 "namespace 1": 1, 256 "namespace 2": 36, 257 "namespace 3": 53, 258 "node 1": 0, 259 "node 2": 37, 260 "node 3": 54, 261 "phase": 4, 262 "pod": 2, 263 "pod_ip": 3, 264 "service 1": 22, 265 "service 2": 38, 266 "service 3": 55, 267 "uid 1": 23, 268 "uid 2": 39, 269 "uid 3": 56 270 }, 271 "renameByName": { 272 "Value #B": "Pod Restarts", 273 "Value #C": "", 274 "container 1": "Container", 275 "host_ip": "", 276 "namespace 1": "Namespace", 277 "node 1": "Node", 278 "phase": "Pod Phase", 279 "pod": "Pod", 280 "pod_ip": "Pod IP" 281 } 282 } 283 } 284 ], 285 "type": "table" 286 }, 287 { 288 "collapsed": true, 289 "gridPos": { 290 "h": 1, 291 "w": 24, 292 "x": 0, 293 "y": 5 294 }, 295 "id": 14, 296 "panels": [ 297 { 298 "datasource": { 299 "type": "prometheus", 300 "uid": "prometheus" 301 }, 302 "fieldConfig": { 303 "defaults": { 304 "color": { 305 "mode": "palette-classic" 306 }, 307 "custom": { 308 "axisCenteredZero": false, 309 "axisColorMode": "text", 310 "axisLabel": "", 311 "axisPlacement": "auto", 312 "barAlignment": 0, 313 "drawStyle": "line", 314 "fillOpacity": 0, 315 "gradientMode": "none", 316 "hideFrom": { 317 "legend": false, 318 "tooltip": false, 319 "viz": false 320 }, 321 "lineInterpolation": "linear", 322 "lineWidth": 1, 323 "pointSize": 5, 324 "scaleDistribution": { 325 "type": "linear" 326 }, 327 "showPoints": "auto", 328 "spanNulls": false, 329 "stacking": { 330 "group": "A", 331 "mode": "none" 332 }, 333 "thresholdsStyle": { 334 "mode": "off" 335 } 336 }, 337 "mappings": [], 338 "thresholds": { 339 "mode": "absolute", 340 "steps": [ 341 { 342 "color": "green" 343 }, 344 { 345 "color": "red", 346 "value": 80 347 } 348 ] 349 }, 350 "unit": "s" 351 }, 352 "overrides": [] 353 }, 354 "gridPos": { 355 "h": 6, 356 "w": 12, 357 "x": 0, 358 "y": 6 359 }, 360 "id": 13, 361 "options": { 362 "legend": { 363 "calcs": [], 364 "displayMode": "list", 365 "placement": "bottom", 366 "showLegend": true 367 }, 368 "tooltip": { 369 "mode": "multi", 370 "sort": "desc" 371 } 372 }, 373 "targets": [ 374 { 375 "datasource": { 376 "type": "prometheus", 377 "uid": "prometheus" 378 }, 379 "editorMode": "code", 380 "expr": "avg(rate(container_cpu_usage_seconds_total{pod=~\"$Pod\",container=~\"$Container\",container!=\"\"}[5m])) by (pod,container)\n", 381 "format": "time_series", 382 "legendFormat": "{{pod}}: {{container}}", 383 "range": true, 384 "refId": "A" 385 } 386 ], 387 "title": "CPU Usage", 388 "type": "timeseries" 389 } 390 ], 391 "title": "CPU", 392 "type": "row" 393 }, 394 { 395 "collapsed": true, 396 "gridPos": { 397 "h": 1, 398 "w": 24, 399 "x": 0, 400 "y": 6 401 }, 402 "id": 5, 403 "panels": [ 404 { 405 "datasource": { 406 "type": "prometheus", 407 "uid": "prometheus" 408 }, 409 "fieldConfig": { 410 "defaults": { 411 "color": { 412 "mode": "palette-classic" 413 }, 414 "custom": { 415 "axisCenteredZero": false, 416 "axisColorMode": "text", 417 "axisLabel": "", 418 "axisPlacement": "auto", 419 "barAlignment": 0, 420 "drawStyle": "line", 421 "fillOpacity": 0, 422 "gradientMode": "none", 423 "hideFrom": { 424 "legend": false, 425 "tooltip": false, 426 "viz": false 427 }, 428 "lineInterpolation": "linear", 429 "lineWidth": 1, 430 "pointSize": 5, 431 "scaleDistribution": { 432 "type": "linear" 433 }, 434 "showPoints": "auto", 435 "spanNulls": false, 436 "stacking": { 437 "group": "A", 438 "mode": "none" 439 }, 440 "thresholdsStyle": { 441 "mode": "off" 442 } 443 }, 444 "mappings": [], 445 "thresholds": { 446 "mode": "absolute", 447 "steps": [ 448 { 449 "color": "green" 450 }, 451 { 452 "color": "red", 453 "value": 80 454 } 455 ] 456 }, 457 "unit": "decbytes" 458 }, 459 "overrides": [] 460 }, 461 "gridPos": { 462 "h": 6, 463 "w": 12, 464 "x": 0, 465 "y": 7 466 }, 467 "id": 6, 468 "options": { 469 "legend": { 470 "calcs": [], 471 "displayMode": "list", 472 "placement": "bottom", 473 "showLegend": true 474 }, 475 "tooltip": { 476 "mode": "multi", 477 "sort": "desc" 478 } 479 }, 480 "targets": [ 481 { 482 "datasource": { 483 "type": "prometheus", 484 "uid": "prometheus" 485 }, 486 "editorMode": "code", 487 "expr": "avg(container_memory_working_set_bytes{pod=~\"$Pod\",container=~\"$Container\",container!=\"\"}) by (pod,container)", 488 "format": "time_series", 489 "legendFormat": "{{pod}}: {{container}}", 490 "range": true, 491 "refId": "A" 492 } 493 ], 494 "title": "Memory Working Set Bytes", 495 "type": "timeseries" 496 }, 497 { 498 "datasource": { 499 "type": "prometheus", 500 "uid": "prometheus" 501 }, 502 "fieldConfig": { 503 "defaults": { 504 "color": { 505 "mode": "palette-classic" 506 }, 507 "custom": { 508 "axisCenteredZero": false, 509 "axisColorMode": "text", 510 "axisLabel": "", 511 "axisPlacement": "auto", 512 "barAlignment": 0, 513 "drawStyle": "line", 514 "fillOpacity": 0, 515 "gradientMode": "none", 516 "hideFrom": { 517 "legend": false, 518 "tooltip": false, 519 "viz": false 520 }, 521 "lineInterpolation": "linear", 522 "lineWidth": 1, 523 "pointSize": 5, 524 "scaleDistribution": { 525 "type": "linear" 526 }, 527 "showPoints": "auto", 528 "spanNulls": false, 529 "stacking": { 530 "group": "A", 531 "mode": "none" 532 }, 533 "thresholdsStyle": { 534 "mode": "off" 535 } 536 }, 537 "mappings": [], 538 "thresholds": { 539 "mode": "absolute", 540 "steps": [ 541 { 542 "color": "green" 543 }, 544 { 545 "color": "red", 546 "value": 80 547 } 548 ] 549 }, 550 "unit": "decbytes" 551 }, 552 "overrides": [] 553 }, 554 "gridPos": { 555 "h": 6, 556 "w": 12, 557 "x": 12, 558 "y": 7 559 }, 560 "id": 7, 561 "options": { 562 "legend": { 563 "calcs": [], 564 "displayMode": "list", 565 "placement": "bottom", 566 "showLegend": true 567 }, 568 "tooltip": { 569 "mode": "multi", 570 "sort": "desc" 571 } 572 }, 573 "targets": [ 574 { 575 "datasource": { 576 "type": "prometheus", 577 "uid": "prometheus" 578 }, 579 "editorMode": "code", 580 "expr": "avg(container_memory_cache{pod=~\"$Pod\",container=~\"$Container\",container!=\"\"}) by (pod,container)", 581 "format": "time_series", 582 "legendFormat": "{{pod}}: {{container}}", 583 "range": true, 584 "refId": "A" 585 } 586 ], 587 "title": "Memory Cached", 588 "type": "timeseries" 589 } 590 ], 591 "title": "Memory", 592 "type": "row" 593 }, 594 { 595 "collapsed": true, 596 "gridPos": { 597 "h": 1, 598 "w": 24, 599 "x": 0, 600 "y": 7 601 }, 602 "id": 10, 603 "panels": [ 604 { 605 "datasource": { 606 "type": "prometheus", 607 "uid": "prometheus" 608 }, 609 "fieldConfig": { 610 "defaults": { 611 "color": { 612 "mode": "palette-classic" 613 }, 614 "custom": { 615 "axisCenteredZero": false, 616 "axisColorMode": "text", 617 "axisLabel": "", 618 "axisPlacement": "auto", 619 "barAlignment": 0, 620 "drawStyle": "line", 621 "fillOpacity": 0, 622 "gradientMode": "none", 623 "hideFrom": { 624 "legend": false, 625 "tooltip": false, 626 "viz": false 627 }, 628 "lineInterpolation": "linear", 629 "lineWidth": 1, 630 "pointSize": 5, 631 "scaleDistribution": { 632 "type": "linear" 633 }, 634 "showPoints": "auto", 635 "spanNulls": false, 636 "stacking": { 637 "group": "A", 638 "mode": "none" 639 }, 640 "thresholdsStyle": { 641 "mode": "off" 642 } 643 }, 644 "mappings": [], 645 "thresholds": { 646 "mode": "absolute", 647 "steps": [ 648 { 649 "color": "green" 650 }, 651 { 652 "color": "red", 653 "value": 80 654 } 655 ] 656 }, 657 "unit": "decbytes" 658 }, 659 "overrides": [] 660 }, 661 "gridPos": { 662 "h": 6, 663 "w": 12, 664 "x": 0, 665 "y": 14 666 }, 667 "id": 9, 668 "options": { 669 "legend": { 670 "calcs": [], 671 "displayMode": "list", 672 "placement": "bottom", 673 "showLegend": true 674 }, 675 "tooltip": { 676 "mode": "multi", 677 "sort": "desc" 678 } 679 }, 680 "targets": [ 681 { 682 "datasource": { 683 "type": "prometheus", 684 "uid": "prometheus" 685 }, 686 "editorMode": "code", 687 "expr": "avg(rate(container_network_transmit_bytes_total{pod=~\"$Pod\"}[5m])) by (pod)", 688 "format": "time_series", 689 "legendFormat": "{{pod}}", 690 "range": true, 691 "refId": "A" 692 } 693 ], 694 "title": "Network Transmit Bytes", 695 "type": "timeseries" 696 }, 697 { 698 "datasource": { 699 "type": "prometheus", 700 "uid": "prometheus" 701 }, 702 "fieldConfig": { 703 "defaults": { 704 "color": { 705 "mode": "palette-classic" 706 }, 707 "custom": { 708 "axisCenteredZero": false, 709 "axisColorMode": "text", 710 "axisLabel": "", 711 "axisPlacement": "auto", 712 "barAlignment": 0, 713 "drawStyle": "line", 714 "fillOpacity": 0, 715 "gradientMode": "none", 716 "hideFrom": { 717 "legend": false, 718 "tooltip": false, 719 "viz": false 720 }, 721 "lineInterpolation": "linear", 722 "lineWidth": 1, 723 "pointSize": 5, 724 "scaleDistribution": { 725 "type": "linear" 726 }, 727 "showPoints": "auto", 728 "spanNulls": false, 729 "stacking": { 730 "group": "A", 731 "mode": "none" 732 }, 733 "thresholdsStyle": { 734 "mode": "off" 735 } 736 }, 737 "mappings": [], 738 "thresholds": { 739 "mode": "absolute", 740 "steps": [ 741 { 742 "color": "green" 743 }, 744 { 745 "color": "red", 746 "value": 80 747 } 748 ] 749 }, 750 "unit": "decbytes" 751 }, 752 "overrides": [] 753 }, 754 "gridPos": { 755 "h": 6, 756 "w": 12, 757 "x": 12, 758 "y": 14 759 }, 760 "id": 11, 761 "options": { 762 "legend": { 763 "calcs": [], 764 "displayMode": "list", 765 "placement": "bottom", 766 "showLegend": true 767 }, 768 "tooltip": { 769 "mode": "multi", 770 "sort": "desc" 771 } 772 }, 773 "targets": [ 774 { 775 "datasource": { 776 "type": "prometheus", 777 "uid": "prometheus" 778 }, 779 "editorMode": "code", 780 "expr": "avg(rate(container_network_receive_bytes_total{pod=~\"$Pod\"}[5m])) by (pod)", 781 "format": "time_series", 782 "legendFormat": "{{pod}}", 783 "range": true, 784 "refId": "A" 785 } 786 ], 787 "title": "Network Receive Bytes", 788 "type": "timeseries" 789 }, 790 { 791 "datasource": { 792 "type": "prometheus", 793 "uid": "prometheus" 794 }, 795 "fieldConfig": { 796 "defaults": { 797 "color": { 798 "mode": "palette-classic" 799 }, 800 "custom": { 801 "axisCenteredZero": false, 802 "axisColorMode": "text", 803 "axisLabel": "", 804 "axisPlacement": "auto", 805 "barAlignment": 0, 806 "drawStyle": "line", 807 "fillOpacity": 0, 808 "gradientMode": "none", 809 "hideFrom": { 810 "legend": false, 811 "tooltip": false, 812 "viz": false 813 }, 814 "lineInterpolation": "linear", 815 "lineWidth": 1, 816 "pointSize": 5, 817 "scaleDistribution": { 818 "type": "linear" 819 }, 820 "showPoints": "auto", 821 "spanNulls": false, 822 "stacking": { 823 "group": "A", 824 "mode": "none" 825 }, 826 "thresholdsStyle": { 827 "mode": "off" 828 } 829 }, 830 "mappings": [], 831 "thresholds": { 832 "mode": "absolute", 833 "steps": [ 834 { 835 "color": "green" 836 }, 837 { 838 "color": "red", 839 "value": 80 840 } 841 ] 842 }, 843 "unit": "none" 844 }, 845 "overrides": [] 846 }, 847 "gridPos": { 848 "h": 6, 849 "w": 12, 850 "x": 0, 851 "y": 20 852 }, 853 "id": 60, 854 "options": { 855 "legend": { 856 "calcs": [], 857 "displayMode": "list", 858 "placement": "bottom", 859 "showLegend": true 860 }, 861 "tooltip": { 862 "mode": "multi", 863 "sort": "desc" 864 } 865 }, 866 "targets": [ 867 { 868 "datasource": { 869 "type": "prometheus", 870 "uid": "prometheus" 871 }, 872 "editorMode": "code", 873 "expr": "avg(rate(container_network_transmit_errors_total{pod=~\"$Pod\"}[5m])) by (pod)", 874 "format": "time_series", 875 "legendFormat": "{{pod}}", 876 "range": true, 877 "refId": "A" 878 } 879 ], 880 "title": "Network Transmit Errors", 881 "type": "timeseries" 882 }, 883 { 884 "datasource": { 885 "type": "prometheus", 886 "uid": "prometheus" 887 }, 888 "fieldConfig": { 889 "defaults": { 890 "color": { 891 "mode": "palette-classic" 892 }, 893 "custom": { 894 "axisCenteredZero": false, 895 "axisColorMode": "text", 896 "axisLabel": "", 897 "axisPlacement": "auto", 898 "barAlignment": 0, 899 "drawStyle": "line", 900 "fillOpacity": 0, 901 "gradientMode": "none", 902 "hideFrom": { 903 "legend": false, 904 "tooltip": false, 905 "viz": false 906 }, 907 "lineInterpolation": "linear", 908 "lineWidth": 1, 909 "pointSize": 5, 910 "scaleDistribution": { 911 "type": "linear" 912 }, 913 "showPoints": "auto", 914 "spanNulls": false, 915 "stacking": { 916 "group": "A", 917 "mode": "none" 918 }, 919 "thresholdsStyle": { 920 "mode": "off" 921 } 922 }, 923 "mappings": [], 924 "thresholds": { 925 "mode": "absolute", 926 "steps": [ 927 { 928 "color": "green" 929 }, 930 { 931 "color": "red", 932 "value": 80 933 } 934 ] 935 }, 936 "unit": "none" 937 }, 938 "overrides": [] 939 }, 940 "gridPos": { 941 "h": 6, 942 "w": 12, 943 "x": 12, 944 "y": 20 945 }, 946 "id": 61, 947 "options": { 948 "legend": { 949 "calcs": [], 950 "displayMode": "list", 951 "placement": "bottom", 952 "showLegend": true 953 }, 954 "tooltip": { 955 "mode": "multi", 956 "sort": "desc" 957 } 958 }, 959 "targets": [ 960 { 961 "datasource": { 962 "type": "prometheus", 963 "uid": "prometheus" 964 }, 965 "editorMode": "code", 966 "expr": "avg(rate(container_network_receive_errors_total{pod=~\"$Pod\"}[5m])) by (pod)", 967 "format": "time_series", 968 "legendFormat": "{{pod}}", 969 "range": true, 970 "refId": "A" 971 } 972 ], 973 "title": "Network Receive Errors", 974 "type": "timeseries" 975 } 976 ], 977 "title": "Network", 978 "type": "row" 979 }, 980 { 981 "collapsed": true, 982 "gridPos": { 983 "h": 1, 984 "w": 24, 985 "x": 0, 986 "y": 8 987 }, 988 "id": 16, 989 "panels": [ 990 { 991 "datasource": { 992 "type": "prometheus", 993 "uid": "prometheus" 994 }, 995 "fieldConfig": { 996 "defaults": { 997 "color": { 998 "mode": "palette-classic" 999 }, 1000 "custom": { 1001 "axisCenteredZero": false, 1002 "axisColorMode": "text", 1003 "axisLabel": "", 1004 "axisPlacement": "auto", 1005 "barAlignment": 0, 1006 "drawStyle": "line", 1007 "fillOpacity": 0, 1008 "gradientMode": "none", 1009 "hideFrom": { 1010 "legend": false, 1011 "tooltip": false, 1012 "viz": false 1013 }, 1014 "lineInterpolation": "linear", 1015 "lineWidth": 1, 1016 "pointSize": 5, 1017 "scaleDistribution": { 1018 "type": "linear" 1019 }, 1020 "showPoints": "auto", 1021 "spanNulls": false, 1022 "stacking": { 1023 "group": "A", 1024 "mode": "none" 1025 }, 1026 "thresholdsStyle": { 1027 "mode": "off" 1028 } 1029 }, 1030 "mappings": [], 1031 "thresholds": { 1032 "mode": "absolute", 1033 "steps": [ 1034 { 1035 "color": "green" 1036 }, 1037 { 1038 "color": "red", 1039 "value": 80 1040 } 1041 ] 1042 } 1043 }, 1044 "overrides": [] 1045 }, 1046 "gridPos": { 1047 "h": 8, 1048 "w": 12, 1049 "x": 0, 1050 "y": 25 1051 }, 1052 "id": 15, 1053 "options": { 1054 "legend": { 1055 "calcs": [], 1056 "displayMode": "list", 1057 "placement": "bottom", 1058 "showLegend": true 1059 }, 1060 "tooltip": { 1061 "mode": "multi", 1062 "sort": "desc" 1063 } 1064 }, 1065 "targets": [ 1066 { 1067 "datasource": { 1068 "type": "prometheus", 1069 "uid": "prometheus" 1070 }, 1071 "editorMode": "code", 1072 "expr": "sum(controller_runtime_active_workers{pod=~\"$Pod\",controller=~\"$Controller\"}) by (pod,controller)", 1073 "legendFormat": "{{pod}}: {{controller}}", 1074 "range": true, 1075 "refId": "A" 1076 } 1077 ], 1078 "title": "Controller Active Workers", 1079 "type": "timeseries" 1080 }, 1081 { 1082 "datasource": { 1083 "type": "prometheus", 1084 "uid": "prometheus" 1085 }, 1086 "fieldConfig": { 1087 "defaults": { 1088 "color": { 1089 "mode": "palette-classic" 1090 }, 1091 "custom": { 1092 "axisCenteredZero": false, 1093 "axisColorMode": "text", 1094 "axisLabel": "", 1095 "axisPlacement": "auto", 1096 "barAlignment": 0, 1097 "drawStyle": "line", 1098 "fillOpacity": 0, 1099 "gradientMode": "none", 1100 "hideFrom": { 1101 "legend": false, 1102 "tooltip": false, 1103 "viz": false 1104 }, 1105 "lineInterpolation": "linear", 1106 "lineWidth": 1, 1107 "pointSize": 5, 1108 "scaleDistribution": { 1109 "type": "linear" 1110 }, 1111 "showPoints": "auto", 1112 "spanNulls": false, 1113 "stacking": { 1114 "group": "A", 1115 "mode": "none" 1116 }, 1117 "thresholdsStyle": { 1118 "mode": "off" 1119 } 1120 }, 1121 "mappings": [], 1122 "thresholds": { 1123 "mode": "absolute", 1124 "steps": [ 1125 { 1126 "color": "green" 1127 }, 1128 { 1129 "color": "red", 1130 "value": 80 1131 } 1132 ] 1133 }, 1134 "unit": "percentunit" 1135 }, 1136 "overrides": [] 1137 }, 1138 "gridPos": { 1139 "h": 8, 1140 "w": 12, 1141 "x": 12, 1142 "y": 25 1143 }, 1144 "id": 18, 1145 "options": { 1146 "legend": { 1147 "calcs": [], 1148 "displayMode": "list", 1149 "placement": "bottom", 1150 "showLegend": true 1151 }, 1152 "tooltip": { 1153 "mode": "multi", 1154 "sort": "desc" 1155 } 1156 }, 1157 "targets": [ 1158 { 1159 "datasource": { 1160 "type": "prometheus", 1161 "uid": "prometheus" 1162 }, 1163 "editorMode": "code", 1164 "expr": "sum(controller_runtime_active_workers{pod=~\"$Pod\",controller=~\"$Controller\"}) by (pod,controller) / sum(controller_runtime_max_concurrent_reconciles{pod=~\"$Pod\",controller=~\"$Controller\"}) by (pod,controller)", 1165 "legendFormat": "{{pod}}: {{controller}}", 1166 "range": true, 1167 "refId": "A" 1168 } 1169 ], 1170 "title": "Controller Worker Utilization", 1171 "type": "timeseries" 1172 } 1173 ], 1174 "title": "Controller Workers", 1175 "type": "row" 1176 }, 1177 { 1178 "collapsed": true, 1179 "gridPos": { 1180 "h": 1, 1181 "w": 24, 1182 "x": 0, 1183 "y": 9 1184 }, 1185 "id": 25, 1186 "panels": [ 1187 { 1188 "datasource": { 1189 "type": "prometheus", 1190 "uid": "prometheus" 1191 }, 1192 "fieldConfig": { 1193 "defaults": { 1194 "color": { 1195 "mode": "palette-classic" 1196 }, 1197 "custom": { 1198 "axisCenteredZero": false, 1199 "axisColorMode": "text", 1200 "axisLabel": "", 1201 "axisPlacement": "auto", 1202 "barAlignment": 0, 1203 "drawStyle": "line", 1204 "fillOpacity": 0, 1205 "gradientMode": "none", 1206 "hideFrom": { 1207 "legend": false, 1208 "tooltip": false, 1209 "viz": false 1210 }, 1211 "lineInterpolation": "linear", 1212 "lineWidth": 1, 1213 "pointSize": 5, 1214 "scaleDistribution": { 1215 "type": "linear" 1216 }, 1217 "showPoints": "auto", 1218 "spanNulls": false, 1219 "stacking": { 1220 "group": "A", 1221 "mode": "none" 1222 }, 1223 "thresholdsStyle": { 1224 "mode": "off" 1225 } 1226 }, 1227 "mappings": [], 1228 "thresholds": { 1229 "mode": "absolute", 1230 "steps": [ 1231 { 1232 "color": "green", 1233 "value": null 1234 }, 1235 { 1236 "color": "red", 1237 "value": 80 1238 } 1239 ] 1240 }, 1241 "unit": "none" 1242 }, 1243 "overrides": [] 1244 }, 1245 "gridPos": { 1246 "h": 8, 1247 "w": 12, 1248 "x": 0, 1249 "y": 10 1250 }, 1251 "id": 19, 1252 "options": { 1253 "legend": { 1254 "calcs": [], 1255 "displayMode": "list", 1256 "placement": "bottom", 1257 "showLegend": true 1258 }, 1259 "tooltip": { 1260 "mode": "multi", 1261 "sort": "desc" 1262 } 1263 }, 1264 "targets": [ 1265 { 1266 "datasource": { 1267 "type": "prometheus", 1268 "uid": "prometheus" 1269 }, 1270 "editorMode": "code", 1271 "expr": "sum(rate(controller_runtime_reconcile_total{pod=~\"$Pod\",controller=~\"$Controller\"}[5m])) by (pod)", 1272 "legendFormat": "{{pod}}", 1273 "range": true, 1274 "refId": "A" 1275 } 1276 ], 1277 "title": "Reconcile Total", 1278 "type": "timeseries" 1279 }, 1280 { 1281 "datasource": { 1282 "type": "prometheus", 1283 "uid": "prometheus" 1284 }, 1285 "fieldConfig": { 1286 "defaults": { 1287 "color": { 1288 "mode": "palette-classic" 1289 }, 1290 "custom": { 1291 "axisCenteredZero": false, 1292 "axisColorMode": "text", 1293 "axisLabel": "", 1294 "axisPlacement": "auto", 1295 "barAlignment": 0, 1296 "drawStyle": "line", 1297 "fillOpacity": 0, 1298 "gradientMode": "none", 1299 "hideFrom": { 1300 "legend": false, 1301 "tooltip": false, 1302 "viz": false 1303 }, 1304 "lineInterpolation": "linear", 1305 "lineWidth": 1, 1306 "pointSize": 5, 1307 "scaleDistribution": { 1308 "type": "linear" 1309 }, 1310 "showPoints": "auto", 1311 "spanNulls": false, 1312 "stacking": { 1313 "group": "A", 1314 "mode": "none" 1315 }, 1316 "thresholdsStyle": { 1317 "mode": "off" 1318 } 1319 }, 1320 "mappings": [], 1321 "thresholds": { 1322 "mode": "absolute", 1323 "steps": [ 1324 { 1325 "color": "green", 1326 "value": null 1327 }, 1328 { 1329 "color": "red", 1330 "value": 80 1331 } 1332 ] 1333 }, 1334 "unit": "none" 1335 }, 1336 "overrides": [] 1337 }, 1338 "gridPos": { 1339 "h": 8, 1340 "w": 12, 1341 "x": 12, 1342 "y": 10 1343 }, 1344 "id": 20, 1345 "options": { 1346 "legend": { 1347 "calcs": [], 1348 "displayMode": "list", 1349 "placement": "bottom", 1350 "showLegend": true 1351 }, 1352 "tooltip": { 1353 "mode": "multi", 1354 "sort": "desc" 1355 } 1356 }, 1357 "targets": [ 1358 { 1359 "datasource": { 1360 "type": "prometheus", 1361 "uid": "prometheus" 1362 }, 1363 "editorMode": "code", 1364 "expr": "sum(rate(controller_runtime_reconcile_total{pod=~\"$Pod\",controller=~\"$Controller\"}[5m])) by (pod,controller)", 1365 "legendFormat": "{{pod}}: {{controller}}", 1366 "range": true, 1367 "refId": "A" 1368 } 1369 ], 1370 "title": "Reconcile Total by Controller", 1371 "type": "timeseries" 1372 }, 1373 { 1374 "datasource": { 1375 "type": "prometheus", 1376 "uid": "prometheus" 1377 }, 1378 "fieldConfig": { 1379 "defaults": { 1380 "color": { 1381 "mode": "palette-classic" 1382 }, 1383 "custom": { 1384 "axisCenteredZero": false, 1385 "axisColorMode": "text", 1386 "axisLabel": "", 1387 "axisPlacement": "auto", 1388 "barAlignment": 0, 1389 "drawStyle": "line", 1390 "fillOpacity": 0, 1391 "gradientMode": "none", 1392 "hideFrom": { 1393 "legend": false, 1394 "tooltip": false, 1395 "viz": false 1396 }, 1397 "lineInterpolation": "linear", 1398 "lineWidth": 1, 1399 "pointSize": 5, 1400 "scaleDistribution": { 1401 "type": "linear" 1402 }, 1403 "showPoints": "auto", 1404 "spanNulls": false, 1405 "stacking": { 1406 "group": "A", 1407 "mode": "none" 1408 }, 1409 "thresholdsStyle": { 1410 "mode": "off" 1411 } 1412 }, 1413 "mappings": [], 1414 "thresholds": { 1415 "mode": "absolute", 1416 "steps": [ 1417 { 1418 "color": "green", 1419 "value": null 1420 }, 1421 { 1422 "color": "red", 1423 "value": 80 1424 } 1425 ] 1426 }, 1427 "unit": "none" 1428 }, 1429 "overrides": [] 1430 }, 1431 "gridPos": { 1432 "h": 8, 1433 "w": 12, 1434 "x": 0, 1435 "y": 18 1436 }, 1437 "id": 21, 1438 "options": { 1439 "legend": { 1440 "calcs": [], 1441 "displayMode": "list", 1442 "placement": "bottom", 1443 "showLegend": true 1444 }, 1445 "tooltip": { 1446 "mode": "multi", 1447 "sort": "desc" 1448 } 1449 }, 1450 "targets": [ 1451 { 1452 "datasource": { 1453 "type": "prometheus", 1454 "uid": "prometheus" 1455 }, 1456 "editorMode": "code", 1457 "expr": "sum(rate(controller_runtime_reconcile_total{pod=~\"$Pod\",controller=~\"$Controller\"}[5m])) by (pod,result)", 1458 "legendFormat": "{{pod}}: {{result}}", 1459 "range": true, 1460 "refId": "A" 1461 } 1462 ], 1463 "title": "Reconcile Total by Result", 1464 "type": "timeseries" 1465 }, 1466 { 1467 "datasource": { 1468 "type": "prometheus", 1469 "uid": "prometheus" 1470 }, 1471 "fieldConfig": { 1472 "defaults": { 1473 "color": { 1474 "mode": "palette-classic" 1475 }, 1476 "custom": { 1477 "axisCenteredZero": false, 1478 "axisColorMode": "text", 1479 "axisLabel": "", 1480 "axisPlacement": "auto", 1481 "barAlignment": 0, 1482 "drawStyle": "line", 1483 "fillOpacity": 0, 1484 "gradientMode": "none", 1485 "hideFrom": { 1486 "legend": false, 1487 "tooltip": false, 1488 "viz": false 1489 }, 1490 "lineInterpolation": "linear", 1491 "lineWidth": 1, 1492 "pointSize": 5, 1493 "scaleDistribution": { 1494 "type": "linear" 1495 }, 1496 "showPoints": "auto", 1497 "spanNulls": false, 1498 "stacking": { 1499 "group": "A", 1500 "mode": "none" 1501 }, 1502 "thresholdsStyle": { 1503 "mode": "off" 1504 } 1505 }, 1506 "mappings": [], 1507 "thresholds": { 1508 "mode": "absolute", 1509 "steps": [ 1510 { 1511 "color": "green", 1512 "value": null 1513 }, 1514 { 1515 "color": "red", 1516 "value": 80 1517 } 1518 ] 1519 }, 1520 "unit": "none" 1521 }, 1522 "overrides": [] 1523 }, 1524 "gridPos": { 1525 "h": 8, 1526 "w": 12, 1527 "x": 12, 1528 "y": 18 1529 }, 1530 "id": 22, 1531 "options": { 1532 "legend": { 1533 "calcs": [], 1534 "displayMode": "list", 1535 "placement": "bottom", 1536 "showLegend": true 1537 }, 1538 "tooltip": { 1539 "mode": "multi", 1540 "sort": "desc" 1541 } 1542 }, 1543 "targets": [ 1544 { 1545 "datasource": { 1546 "type": "prometheus", 1547 "uid": "prometheus" 1548 }, 1549 "editorMode": "code", 1550 "expr": "sum(rate(controller_runtime_reconcile_total{pod=~\"$Pod\",controller=~\"$Controller\"}[5m])) by (pod,controller,result)", 1551 "legendFormat": "{{pod}}: {{controller}}: {{result}}", 1552 "range": true, 1553 "refId": "A" 1554 } 1555 ], 1556 "title": "Reconcile Total by Controller, Result", 1557 "type": "timeseries" 1558 }, 1559 { 1560 "datasource": { 1561 "type": "prometheus", 1562 "uid": "prometheus" 1563 }, 1564 "fieldConfig": { 1565 "defaults": { 1566 "color": { 1567 "mode": "palette-classic" 1568 }, 1569 "custom": { 1570 "axisCenteredZero": false, 1571 "axisColorMode": "text", 1572 "axisLabel": "", 1573 "axisPlacement": "auto", 1574 "barAlignment": 0, 1575 "drawStyle": "line", 1576 "fillOpacity": 0, 1577 "gradientMode": "none", 1578 "hideFrom": { 1579 "legend": false, 1580 "tooltip": false, 1581 "viz": false 1582 }, 1583 "lineInterpolation": "linear", 1584 "lineWidth": 1, 1585 "pointSize": 5, 1586 "scaleDistribution": { 1587 "type": "linear" 1588 }, 1589 "showPoints": "auto", 1590 "spanNulls": false, 1591 "stacking": { 1592 "group": "A", 1593 "mode": "none" 1594 }, 1595 "thresholdsStyle": { 1596 "mode": "off" 1597 } 1598 }, 1599 "mappings": [], 1600 "thresholds": { 1601 "mode": "absolute", 1602 "steps": [ 1603 { 1604 "color": "green", 1605 "value": null 1606 }, 1607 { 1608 "color": "red", 1609 "value": 80 1610 } 1611 ] 1612 }, 1613 "unit": "none" 1614 }, 1615 "overrides": [] 1616 }, 1617 "gridPos": { 1618 "h": 8, 1619 "w": 12, 1620 "x": 0, 1621 "y": 26 1622 }, 1623 "id": 23, 1624 "options": { 1625 "legend": { 1626 "calcs": [], 1627 "displayMode": "list", 1628 "placement": "bottom", 1629 "showLegend": true 1630 }, 1631 "tooltip": { 1632 "mode": "multi", 1633 "sort": "desc" 1634 } 1635 }, 1636 "targets": [ 1637 { 1638 "datasource": { 1639 "type": "prometheus", 1640 "uid": "prometheus" 1641 }, 1642 "editorMode": "code", 1643 "expr": "sum(rate(controller_runtime_reconcile_errors_total{pod=~\"$Pod\",controller=~\"$Controller\"}[5m])) by (pod)", 1644 "legendFormat": "{{pod}}", 1645 "range": true, 1646 "refId": "A" 1647 } 1648 ], 1649 "title": "Reconcile Errors", 1650 "type": "timeseries" 1651 }, 1652 { 1653 "datasource": { 1654 "type": "prometheus", 1655 "uid": "prometheus" 1656 }, 1657 "fieldConfig": { 1658 "defaults": { 1659 "color": { 1660 "mode": "palette-classic" 1661 }, 1662 "custom": { 1663 "axisCenteredZero": false, 1664 "axisColorMode": "text", 1665 "axisLabel": "", 1666 "axisPlacement": "auto", 1667 "barAlignment": 0, 1668 "drawStyle": "line", 1669 "fillOpacity": 0, 1670 "gradientMode": "none", 1671 "hideFrom": { 1672 "legend": false, 1673 "tooltip": false, 1674 "viz": false 1675 }, 1676 "lineInterpolation": "linear", 1677 "lineWidth": 1, 1678 "pointSize": 5, 1679 "scaleDistribution": { 1680 "type": "linear" 1681 }, 1682 "showPoints": "auto", 1683 "spanNulls": false, 1684 "stacking": { 1685 "group": "A", 1686 "mode": "none" 1687 }, 1688 "thresholdsStyle": { 1689 "mode": "off" 1690 } 1691 }, 1692 "mappings": [], 1693 "thresholds": { 1694 "mode": "absolute", 1695 "steps": [ 1696 { 1697 "color": "green", 1698 "value": null 1699 }, 1700 { 1701 "color": "red", 1702 "value": 80 1703 } 1704 ] 1705 }, 1706 "unit": "none" 1707 }, 1708 "overrides": [] 1709 }, 1710 "gridPos": { 1711 "h": 8, 1712 "w": 12, 1713 "x": 12, 1714 "y": 26 1715 }, 1716 "id": 24, 1717 "options": { 1718 "legend": { 1719 "calcs": [], 1720 "displayMode": "list", 1721 "placement": "bottom", 1722 "showLegend": true 1723 }, 1724 "tooltip": { 1725 "mode": "multi", 1726 "sort": "desc" 1727 } 1728 }, 1729 "targets": [ 1730 { 1731 "datasource": { 1732 "type": "prometheus", 1733 "uid": "prometheus" 1734 }, 1735 "editorMode": "code", 1736 "expr": "sum(rate(controller_runtime_reconcile_errors_total{pod=~\"$Pod\",controller=~\"$Controller\"}[5m])) by (pod,controller)", 1737 "legendFormat": "{{pod}}: {{controller}}", 1738 "range": true, 1739 "refId": "A" 1740 } 1741 ], 1742 "title": "Reconcile Errors by Controller", 1743 "type": "timeseries" 1744 } 1745 ], 1746 "title": "Reconcile Results", 1747 "type": "row" 1748 }, 1749 { 1750 "collapsed": true, 1751 "gridPos": { 1752 "h": 1, 1753 "w": 24, 1754 "x": 0, 1755 "y": 10 1756 }, 1757 "id": 29, 1758 "panels": [ 1759 { 1760 "datasource": { 1761 "type": "prometheus", 1762 "uid": "prometheus" 1763 }, 1764 "fieldConfig": { 1765 "defaults": { 1766 "color": { 1767 "mode": "palette-classic" 1768 }, 1769 "custom": { 1770 "axisCenteredZero": false, 1771 "axisColorMode": "text", 1772 "axisLabel": "", 1773 "axisPlacement": "auto", 1774 "barAlignment": 0, 1775 "drawStyle": "line", 1776 "fillOpacity": 0, 1777 "gradientMode": "none", 1778 "hideFrom": { 1779 "legend": false, 1780 "tooltip": false, 1781 "viz": false 1782 }, 1783 "lineInterpolation": "linear", 1784 "lineWidth": 1, 1785 "pointSize": 5, 1786 "scaleDistribution": { 1787 "type": "linear" 1788 }, 1789 "showPoints": "auto", 1790 "spanNulls": false, 1791 "stacking": { 1792 "group": "A", 1793 "mode": "none" 1794 }, 1795 "thresholdsStyle": { 1796 "mode": "off" 1797 } 1798 }, 1799 "mappings": [], 1800 "thresholds": { 1801 "mode": "absolute", 1802 "steps": [ 1803 { 1804 "color": "green", 1805 "value": null 1806 }, 1807 { 1808 "color": "red", 1809 "value": 80 1810 } 1811 ] 1812 }, 1813 "unit": "s" 1814 }, 1815 "overrides": [] 1816 }, 1817 "gridPos": { 1818 "h": 8, 1819 "w": 24, 1820 "x": 0, 1821 "y": 11 1822 }, 1823 "id": 27, 1824 "options": { 1825 "legend": { 1826 "calcs": [], 1827 "displayMode": "list", 1828 "placement": "bottom", 1829 "showLegend": true 1830 }, 1831 "tooltip": { 1832 "mode": "multi", 1833 "sort": "desc" 1834 } 1835 }, 1836 "targets": [ 1837 { 1838 "datasource": { 1839 "type": "prometheus", 1840 "uid": "prometheus" 1841 }, 1842 "editorMode": "code", 1843 "expr": "sum(rate(controller_runtime_reconcile_time_seconds_sum{pod=~\"$Pod\"}[5m])) by (pod) / sum(rate(controller_runtime_reconcile_time_seconds_count{pod=~\"$Pod\"}[5m])) by (pod)", 1844 "legendFormat": "{{pod}}", 1845 "range": true, 1846 "refId": "A" 1847 } 1848 ], 1849 "title": "Reconcile Duration", 1850 "type": "timeseries" 1851 }, 1852 { 1853 "datasource": { 1854 "type": "prometheus", 1855 "uid": "prometheus" 1856 }, 1857 "description": "", 1858 "fieldConfig": { 1859 "defaults": { 1860 "color": { 1861 "mode": "palette-classic" 1862 }, 1863 "custom": { 1864 "axisCenteredZero": false, 1865 "axisColorMode": "text", 1866 "axisLabel": "", 1867 "axisPlacement": "auto", 1868 "barAlignment": 0, 1869 "drawStyle": "line", 1870 "fillOpacity": 0, 1871 "gradientMode": "none", 1872 "hideFrom": { 1873 "legend": false, 1874 "tooltip": false, 1875 "viz": false 1876 }, 1877 "lineInterpolation": "linear", 1878 "lineWidth": 1, 1879 "pointSize": 5, 1880 "scaleDistribution": { 1881 "type": "linear" 1882 }, 1883 "showPoints": "auto", 1884 "spanNulls": false, 1885 "stacking": { 1886 "group": "A", 1887 "mode": "none" 1888 }, 1889 "thresholdsStyle": { 1890 "mode": "off" 1891 } 1892 }, 1893 "mappings": [], 1894 "thresholds": { 1895 "mode": "absolute", 1896 "steps": [ 1897 { 1898 "color": "green", 1899 "value": null 1900 }, 1901 { 1902 "color": "red", 1903 "value": 80 1904 } 1905 ] 1906 }, 1907 "unit": "s" 1908 }, 1909 "overrides": [] 1910 }, 1911 "gridPos": { 1912 "h": 9, 1913 "w": 24, 1914 "x": 0, 1915 "y": 19 1916 }, 1917 "id": 65, 1918 "options": { 1919 "legend": { 1920 "calcs": [], 1921 "displayMode": "list", 1922 "placement": "bottom", 1923 "showLegend": true 1924 }, 1925 "tooltip": { 1926 "mode": "multi", 1927 "sort": "desc" 1928 } 1929 }, 1930 "targets": [ 1931 { 1932 "datasource": { 1933 "type": "prometheus", 1934 "uid": "prometheus" 1935 }, 1936 "editorMode": "code", 1937 "expr": "histogram_quantile(0.50, sum(rate(controller_runtime_reconcile_time_seconds_bucket{pod=~\"$Pod\"}[5m])) by (pod, le))", 1938 "hide": false, 1939 "legendFormat": "P50 {{pod}}", 1940 "range": true, 1941 "refId": "B" 1942 }, 1943 { 1944 "datasource": { 1945 "type": "prometheus", 1946 "uid": "prometheus" 1947 }, 1948 "editorMode": "code", 1949 "expr": "histogram_quantile(0.90, sum(rate(controller_runtime_reconcile_time_seconds_bucket{pod=~\"$Pod\"}[5m])) by (pod, le))", 1950 "hide": false, 1951 "legendFormat": "P90 {{pod}}", 1952 "range": true, 1953 "refId": "A" 1954 }, 1955 { 1956 "datasource": { 1957 "type": "prometheus", 1958 "uid": "prometheus" 1959 }, 1960 "editorMode": "code", 1961 "expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{pod=~\"$Pod\"}[5m])) by (pod, le))", 1962 "hide": false, 1963 "legendFormat": "P99 {{pod}}", 1964 "range": true, 1965 "refId": "C" 1966 } 1967 ], 1968 "title": "Reconcile Duration Quantils", 1969 "type": "timeseries" 1970 }, 1971 { 1972 "datasource": { 1973 "type": "prometheus", 1974 "uid": "prometheus" 1975 }, 1976 "fieldConfig": { 1977 "defaults": { 1978 "color": { 1979 "mode": "palette-classic" 1980 }, 1981 "custom": { 1982 "axisCenteredZero": false, 1983 "axisColorMode": "text", 1984 "axisLabel": "", 1985 "axisPlacement": "auto", 1986 "barAlignment": 0, 1987 "drawStyle": "line", 1988 "fillOpacity": 0, 1989 "gradientMode": "none", 1990 "hideFrom": { 1991 "legend": false, 1992 "tooltip": false, 1993 "viz": false 1994 }, 1995 "lineInterpolation": "linear", 1996 "lineWidth": 1, 1997 "pointSize": 5, 1998 "scaleDistribution": { 1999 "type": "linear" 2000 }, 2001 "showPoints": "auto", 2002 "spanNulls": false, 2003 "stacking": { 2004 "group": "A", 2005 "mode": "none" 2006 }, 2007 "thresholdsStyle": { 2008 "mode": "off" 2009 } 2010 }, 2011 "mappings": [], 2012 "thresholds": { 2013 "mode": "absolute", 2014 "steps": [ 2015 { 2016 "color": "green", 2017 "value": null 2018 }, 2019 { 2020 "color": "red", 2021 "value": 80 2022 } 2023 ] 2024 }, 2025 "unit": "s" 2026 }, 2027 "overrides": [] 2028 }, 2029 "gridPos": { 2030 "h": 8, 2031 "w": 24, 2032 "x": 0, 2033 "y": 28 2034 }, 2035 "id": 62, 2036 "options": { 2037 "legend": { 2038 "calcs": [], 2039 "displayMode": "list", 2040 "placement": "bottom", 2041 "showLegend": true 2042 }, 2043 "tooltip": { 2044 "mode": "multi", 2045 "sort": "desc" 2046 } 2047 }, 2048 "targets": [ 2049 { 2050 "datasource": { 2051 "type": "prometheus", 2052 "uid": "prometheus" 2053 }, 2054 "editorMode": "code", 2055 "expr": "sum(rate(controller_runtime_reconcile_time_seconds_sum{pod=~\"$Pod\",controller=~\"$Controller\"}[5m])) by (pod,controller) / sum(rate(controller_runtime_reconcile_time_seconds_count{pod=~\"$Pod\",controller=~\"$Controller\"}[5m])) by (pod,controller)", 2056 "legendFormat": "{{pod}}: {{controller}}", 2057 "range": true, 2058 "refId": "A" 2059 } 2060 ], 2061 "title": "Reconcile Duration by Controller", 2062 "type": "timeseries" 2063 }, 2064 { 2065 "datasource": { 2066 "type": "prometheus", 2067 "uid": "prometheus" 2068 }, 2069 "description": "", 2070 "fieldConfig": { 2071 "defaults": { 2072 "color": { 2073 "mode": "palette-classic" 2074 }, 2075 "custom": { 2076 "axisCenteredZero": false, 2077 "axisColorMode": "text", 2078 "axisLabel": "", 2079 "axisPlacement": "auto", 2080 "barAlignment": 0, 2081 "drawStyle": "line", 2082 "fillOpacity": 0, 2083 "gradientMode": "none", 2084 "hideFrom": { 2085 "legend": false, 2086 "tooltip": false, 2087 "viz": false 2088 }, 2089 "lineInterpolation": "linear", 2090 "lineWidth": 1, 2091 "pointSize": 5, 2092 "scaleDistribution": { 2093 "type": "linear" 2094 }, 2095 "showPoints": "auto", 2096 "spanNulls": false, 2097 "stacking": { 2098 "group": "A", 2099 "mode": "none" 2100 }, 2101 "thresholdsStyle": { 2102 "mode": "off" 2103 } 2104 }, 2105 "mappings": [], 2106 "thresholds": { 2107 "mode": "absolute", 2108 "steps": [ 2109 { 2110 "color": "green", 2111 "value": null 2112 }, 2113 { 2114 "color": "red", 2115 "value": 80 2116 } 2117 ] 2118 }, 2119 "unit": "s" 2120 }, 2121 "overrides": [] 2122 }, 2123 "gridPos": { 2124 "h": 9, 2125 "w": 24, 2126 "x": 0, 2127 "y": 36 2128 }, 2129 "id": 66, 2130 "options": { 2131 "legend": { 2132 "calcs": [], 2133 "displayMode": "list", 2134 "placement": "bottom", 2135 "showLegend": true 2136 }, 2137 "tooltip": { 2138 "mode": "multi", 2139 "sort": "desc" 2140 } 2141 }, 2142 "targets": [ 2143 { 2144 "datasource": { 2145 "type": "prometheus", 2146 "uid": "prometheus" 2147 }, 2148 "editorMode": "code", 2149 "expr": "histogram_quantile(0.50, sum(rate(controller_runtime_reconcile_time_seconds_bucket{pod=~\"$Pod\",controller=~\"$Controller\"}[5m])) by (pod, controller, le))", 2150 "hide": false, 2151 "legendFormat": "P50 {{pod}} {{controller}} ", 2152 "range": true, 2153 "refId": "B" 2154 }, 2155 { 2156 "datasource": { 2157 "type": "prometheus", 2158 "uid": "prometheus" 2159 }, 2160 "editorMode": "code", 2161 "expr": "histogram_quantile(0.90, sum(rate(controller_runtime_reconcile_time_seconds_bucket{pod=~\"$Pod\",controller=~\"$Controller\"}[5m])) by (pod, controller, le))", 2162 "hide": false, 2163 "legendFormat": "P90 {{pod}} {{controller}} ", 2164 "range": true, 2165 "refId": "A" 2166 }, 2167 { 2168 "datasource": { 2169 "type": "prometheus", 2170 "uid": "prometheus" 2171 }, 2172 "editorMode": "code", 2173 "expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{pod=~\"$Pod\",controller=~\"$Controller\"}[5m])) by (pod, controller, le))", 2174 "hide": false, 2175 "legendFormat": "P99 {{pod}} {{controller}} ", 2176 "range": true, 2177 "refId": "C" 2178 } 2179 ], 2180 "title": "Reconcile Duration Quantils by Controller", 2181 "type": "timeseries" 2182 }, 2183 { 2184 "datasource": { 2185 "type": "prometheus", 2186 "uid": "prometheus" 2187 }, 2188 "fieldConfig": { 2189 "defaults": { 2190 "custom": { 2191 "hideFrom": { 2192 "legend": false, 2193 "tooltip": false, 2194 "viz": false 2195 }, 2196 "scaleDistribution": { 2197 "type": "linear" 2198 } 2199 } 2200 }, 2201 "overrides": [] 2202 }, 2203 "gridPos": { 2204 "h": 13, 2205 "w": 24, 2206 "x": 0, 2207 "y": 45 2208 }, 2209 "id": 26, 2210 "options": { 2211 "calculate": false, 2212 "cellGap": 2, 2213 "color": { 2214 "exponent": 0.5, 2215 "fill": "dark-orange", 2216 "mode": "scheme", 2217 "reverse": true, 2218 "scale": "exponential", 2219 "scheme": "Oranges", 2220 "steps": 128 2221 }, 2222 "exemplars": { 2223 "color": "rgba(255,0,255,0.7)" 2224 }, 2225 "filterValues": { 2226 "le": 1e-9 2227 }, 2228 "legend": { 2229 "show": true 2230 }, 2231 "rowsFrame": { 2232 "layout": "auto" 2233 }, 2234 "tooltip": { 2235 "show": true, 2236 "yHistogram": false 2237 }, 2238 "yAxis": { 2239 "axisPlacement": "left", 2240 "reverse": false, 2241 "unit": "short" 2242 } 2243 }, 2244 "pluginVersion": "9.5.2", 2245 "targets": [ 2246 { 2247 "datasource": { 2248 "type": "prometheus", 2249 "uid": "prometheus" 2250 }, 2251 "editorMode": "code", 2252 "exemplar": true, 2253 "expr": "rate(controller_runtime_reconcile_time_seconds_bucket{pod=~\"$Pod\", controller=~\"$Controller\"}[$__rate_interval])", 2254 "format": "heatmap", 2255 "legendFormat": "{{le}}", 2256 "range": true, 2257 "refId": "A" 2258 } 2259 ], 2260 "title": "Reconcile Duration Buckets", 2261 "type": "heatmap" 2262 } 2263 ], 2264 "title": "Reconcile Durations", 2265 "type": "row" 2266 }, 2267 { 2268 "collapsed": true, 2269 "gridPos": { 2270 "h": 1, 2271 "w": 24, 2272 "x": 0, 2273 "y": 11 2274 }, 2275 "id": 30, 2276 "panels": [ 2277 { 2278 "datasource": { 2279 "type": "prometheus", 2280 "uid": "prometheus" 2281 }, 2282 "fieldConfig": { 2283 "defaults": { 2284 "color": { 2285 "mode": "palette-classic" 2286 }, 2287 "custom": { 2288 "axisCenteredZero": false, 2289 "axisColorMode": "text", 2290 "axisLabel": "", 2291 "axisPlacement": "auto", 2292 "barAlignment": 0, 2293 "drawStyle": "line", 2294 "fillOpacity": 0, 2295 "gradientMode": "none", 2296 "hideFrom": { 2297 "legend": false, 2298 "tooltip": false, 2299 "viz": false 2300 }, 2301 "lineInterpolation": "linear", 2302 "lineWidth": 1, 2303 "pointSize": 5, 2304 "scaleDistribution": { 2305 "type": "linear" 2306 }, 2307 "showPoints": "auto", 2308 "spanNulls": false, 2309 "stacking": { 2310 "group": "A", 2311 "mode": "none" 2312 }, 2313 "thresholdsStyle": { 2314 "mode": "off" 2315 } 2316 }, 2317 "mappings": [], 2318 "thresholds": { 2319 "mode": "absolute", 2320 "steps": [ 2321 { 2322 "color": "green", 2323 "value": null 2324 }, 2325 { 2326 "color": "red", 2327 "value": 80 2328 } 2329 ] 2330 }, 2331 "unit": "none" 2332 }, 2333 "overrides": [] 2334 }, 2335 "gridPos": { 2336 "h": 8, 2337 "w": 12, 2338 "x": 0, 2339 "y": 12 2340 }, 2341 "id": 31, 2342 "options": { 2343 "legend": { 2344 "calcs": [], 2345 "displayMode": "list", 2346 "placement": "bottom", 2347 "showLegend": true 2348 }, 2349 "tooltip": { 2350 "mode": "multi", 2351 "sort": "desc" 2352 } 2353 }, 2354 "targets": [ 2355 { 2356 "datasource": { 2357 "type": "prometheus", 2358 "uid": "prometheus" 2359 }, 2360 "editorMode": "code", 2361 "expr": "workqueue_depth{pod=~\"$Pod\",name=~\"$Controller\"}", 2362 "legendFormat": "{{pod}}: {{name}}", 2363 "range": true, 2364 "refId": "A" 2365 } 2366 ], 2367 "title": "Workqueue Depth", 2368 "type": "timeseries" 2369 }, 2370 { 2371 "datasource": { 2372 "type": "prometheus", 2373 "uid": "prometheus" 2374 }, 2375 "fieldConfig": { 2376 "defaults": { 2377 "color": { 2378 "mode": "palette-classic" 2379 }, 2380 "custom": { 2381 "axisCenteredZero": false, 2382 "axisColorMode": "text", 2383 "axisLabel": "", 2384 "axisPlacement": "auto", 2385 "barAlignment": 0, 2386 "drawStyle": "line", 2387 "fillOpacity": 0, 2388 "gradientMode": "none", 2389 "hideFrom": { 2390 "legend": false, 2391 "tooltip": false, 2392 "viz": false 2393 }, 2394 "lineInterpolation": "linear", 2395 "lineWidth": 1, 2396 "pointSize": 5, 2397 "scaleDistribution": { 2398 "type": "linear" 2399 }, 2400 "showPoints": "auto", 2401 "spanNulls": false, 2402 "stacking": { 2403 "group": "A", 2404 "mode": "none" 2405 }, 2406 "thresholdsStyle": { 2407 "mode": "off" 2408 } 2409 }, 2410 "mappings": [], 2411 "thresholds": { 2412 "mode": "absolute", 2413 "steps": [ 2414 { 2415 "color": "green", 2416 "value": null 2417 }, 2418 { 2419 "color": "red", 2420 "value": 80 2421 } 2422 ] 2423 }, 2424 "unit": "none" 2425 }, 2426 "overrides": [] 2427 }, 2428 "gridPos": { 2429 "h": 8, 2430 "w": 12, 2431 "x": 12, 2432 "y": 12 2433 }, 2434 "id": 32, 2435 "options": { 2436 "legend": { 2437 "calcs": [], 2438 "displayMode": "list", 2439 "placement": "bottom", 2440 "showLegend": true 2441 }, 2442 "tooltip": { 2443 "mode": "multi", 2444 "sort": "desc" 2445 } 2446 }, 2447 "targets": [ 2448 { 2449 "datasource": { 2450 "type": "prometheus", 2451 "uid": "prometheus" 2452 }, 2453 "editorMode": "code", 2454 "expr": "rate(workqueue_adds_total{pod=~\"$Pod\",name=~\"$Controller\"}[5m])", 2455 "legendFormat": "{{pod}}: {{name}}", 2456 "range": true, 2457 "refId": "A" 2458 } 2459 ], 2460 "title": "Workqueue Adds", 2461 "type": "timeseries" 2462 }, 2463 { 2464 "datasource": { 2465 "type": "prometheus", 2466 "uid": "prometheus" 2467 }, 2468 "description": "Total number of retries handled by workqueue. In controller-runtime the retry metric is observed when an item is added to the queue because of RequeueAfter.", 2469 "fieldConfig": { 2470 "defaults": { 2471 "color": { 2472 "mode": "palette-classic" 2473 }, 2474 "custom": { 2475 "axisCenteredZero": false, 2476 "axisColorMode": "text", 2477 "axisLabel": "", 2478 "axisPlacement": "auto", 2479 "barAlignment": 0, 2480 "drawStyle": "line", 2481 "fillOpacity": 0, 2482 "gradientMode": "none", 2483 "hideFrom": { 2484 "legend": false, 2485 "tooltip": false, 2486 "viz": false 2487 }, 2488 "lineInterpolation": "linear", 2489 "lineWidth": 1, 2490 "pointSize": 5, 2491 "scaleDistribution": { 2492 "type": "linear" 2493 }, 2494 "showPoints": "auto", 2495 "spanNulls": false, 2496 "stacking": { 2497 "group": "A", 2498 "mode": "none" 2499 }, 2500 "thresholdsStyle": { 2501 "mode": "off" 2502 } 2503 }, 2504 "mappings": [], 2505 "thresholds": { 2506 "mode": "absolute", 2507 "steps": [ 2508 { 2509 "color": "green", 2510 "value": null 2511 }, 2512 { 2513 "color": "red", 2514 "value": 80 2515 } 2516 ] 2517 }, 2518 "unit": "none" 2519 }, 2520 "overrides": [] 2521 }, 2522 "gridPos": { 2523 "h": 8, 2524 "w": 12, 2525 "x": 0, 2526 "y": 20 2527 }, 2528 "id": 33, 2529 "options": { 2530 "legend": { 2531 "calcs": [], 2532 "displayMode": "list", 2533 "placement": "bottom", 2534 "showLegend": true 2535 }, 2536 "tooltip": { 2537 "mode": "multi", 2538 "sort": "desc" 2539 } 2540 }, 2541 "targets": [ 2542 { 2543 "datasource": { 2544 "type": "prometheus", 2545 "uid": "prometheus" 2546 }, 2547 "editorMode": "code", 2548 "expr": "rate(workqueue_retries_total{pod=~\"$Pod\",name=~\"$Controller\"}[5m])", 2549 "legendFormat": "{{pod}}: {{name}}", 2550 "range": true, 2551 "refId": "A" 2552 } 2553 ], 2554 "title": "Workqueue Retries", 2555 "type": "timeseries" 2556 }, 2557 { 2558 "datasource": { 2559 "type": "prometheus", 2560 "uid": "prometheus" 2561 }, 2562 "description": "How many seconds has the longest running processor for workqueue been running.", 2563 "fieldConfig": { 2564 "defaults": { 2565 "color": { 2566 "mode": "palette-classic" 2567 }, 2568 "custom": { 2569 "axisCenteredZero": false, 2570 "axisColorMode": "text", 2571 "axisLabel": "", 2572 "axisPlacement": "auto", 2573 "barAlignment": 0, 2574 "drawStyle": "line", 2575 "fillOpacity": 0, 2576 "gradientMode": "none", 2577 "hideFrom": { 2578 "legend": false, 2579 "tooltip": false, 2580 "viz": false 2581 }, 2582 "lineInterpolation": "linear", 2583 "lineWidth": 1, 2584 "pointSize": 5, 2585 "scaleDistribution": { 2586 "type": "linear" 2587 }, 2588 "showPoints": "auto", 2589 "spanNulls": false, 2590 "stacking": { 2591 "group": "A", 2592 "mode": "none" 2593 }, 2594 "thresholdsStyle": { 2595 "mode": "off" 2596 } 2597 }, 2598 "mappings": [], 2599 "thresholds": { 2600 "mode": "absolute", 2601 "steps": [ 2602 { 2603 "color": "green", 2604 "value": null 2605 }, 2606 { 2607 "color": "red", 2608 "value": 80 2609 } 2610 ] 2611 }, 2612 "unit": "s" 2613 }, 2614 "overrides": [] 2615 }, 2616 "gridPos": { 2617 "h": 8, 2618 "w": 12, 2619 "x": 12, 2620 "y": 20 2621 }, 2622 "id": 34, 2623 "options": { 2624 "legend": { 2625 "calcs": [], 2626 "displayMode": "list", 2627 "placement": "bottom", 2628 "showLegend": true 2629 }, 2630 "tooltip": { 2631 "mode": "multi", 2632 "sort": "desc" 2633 } 2634 }, 2635 "targets": [ 2636 { 2637 "datasource": { 2638 "type": "prometheus", 2639 "uid": "prometheus" 2640 }, 2641 "editorMode": "code", 2642 "expr": "workqueue_longest_running_processor_seconds{pod=~\"$Pod\",name=~\"$Controller\"}", 2643 "legendFormat": "{{pod}}: {{name}}", 2644 "range": true, 2645 "refId": "A" 2646 } 2647 ], 2648 "title": "Workqueue Longest Running Processor", 2649 "type": "timeseries" 2650 }, 2651 { 2652 "datasource": { 2653 "type": "prometheus", 2654 "uid": "prometheus" 2655 }, 2656 "description": "How many seconds of work has been done that is in progress and hasn't been observed by work_duration. Large values indicate stuck threads. One can deduce the number of stuck threads by observing the rate at which this increases.", 2657 "fieldConfig": { 2658 "defaults": { 2659 "color": { 2660 "mode": "palette-classic" 2661 }, 2662 "custom": { 2663 "axisCenteredZero": false, 2664 "axisColorMode": "text", 2665 "axisLabel": "", 2666 "axisPlacement": "auto", 2667 "barAlignment": 0, 2668 "drawStyle": "line", 2669 "fillOpacity": 0, 2670 "gradientMode": "none", 2671 "hideFrom": { 2672 "legend": false, 2673 "tooltip": false, 2674 "viz": false 2675 }, 2676 "lineInterpolation": "linear", 2677 "lineWidth": 1, 2678 "pointSize": 5, 2679 "scaleDistribution": { 2680 "type": "linear" 2681 }, 2682 "showPoints": "auto", 2683 "spanNulls": false, 2684 "stacking": { 2685 "group": "A", 2686 "mode": "none" 2687 }, 2688 "thresholdsStyle": { 2689 "mode": "off" 2690 } 2691 }, 2692 "mappings": [], 2693 "thresholds": { 2694 "mode": "absolute", 2695 "steps": [ 2696 { 2697 "color": "green", 2698 "value": null 2699 }, 2700 { 2701 "color": "red", 2702 "value": 80 2703 } 2704 ] 2705 }, 2706 "unit": "s" 2707 }, 2708 "overrides": [] 2709 }, 2710 "gridPos": { 2711 "h": 8, 2712 "w": 12, 2713 "x": 0, 2714 "y": 28 2715 }, 2716 "id": 35, 2717 "options": { 2718 "legend": { 2719 "calcs": [], 2720 "displayMode": "list", 2721 "placement": "bottom", 2722 "showLegend": true 2723 }, 2724 "tooltip": { 2725 "mode": "multi", 2726 "sort": "desc" 2727 } 2728 }, 2729 "targets": [ 2730 { 2731 "datasource": { 2732 "type": "prometheus", 2733 "uid": "prometheus" 2734 }, 2735 "editorMode": "code", 2736 "expr": "workqueue_unfinished_work_seconds{pod=~\"$Pod\",name=~\"$Controller\"}", 2737 "legendFormat": "{{pod}}: {{name}}", 2738 "range": true, 2739 "refId": "A" 2740 } 2741 ], 2742 "title": "Workqueue Unfinished Work", 2743 "type": "timeseries" 2744 }, 2745 { 2746 "datasource": { 2747 "type": "prometheus", 2748 "uid": "prometheus" 2749 }, 2750 "description": "How long in seconds processing an item from workqueue takes", 2751 "fieldConfig": { 2752 "defaults": { 2753 "color": { 2754 "mode": "palette-classic" 2755 }, 2756 "custom": { 2757 "axisCenteredZero": false, 2758 "axisColorMode": "text", 2759 "axisLabel": "", 2760 "axisPlacement": "auto", 2761 "barAlignment": 0, 2762 "drawStyle": "line", 2763 "fillOpacity": 0, 2764 "gradientMode": "none", 2765 "hideFrom": { 2766 "legend": false, 2767 "tooltip": false, 2768 "viz": false 2769 }, 2770 "lineInterpolation": "linear", 2771 "lineWidth": 1, 2772 "pointSize": 5, 2773 "scaleDistribution": { 2774 "type": "linear" 2775 }, 2776 "showPoints": "auto", 2777 "spanNulls": false, 2778 "stacking": { 2779 "group": "A", 2780 "mode": "none" 2781 }, 2782 "thresholdsStyle": { 2783 "mode": "off" 2784 } 2785 }, 2786 "mappings": [], 2787 "thresholds": { 2788 "mode": "absolute", 2789 "steps": [ 2790 { 2791 "color": "green", 2792 "value": null 2793 }, 2794 { 2795 "color": "red", 2796 "value": 80 2797 } 2798 ] 2799 }, 2800 "unit": "s" 2801 }, 2802 "overrides": [] 2803 }, 2804 "gridPos": { 2805 "h": 9, 2806 "w": 24, 2807 "x": 0, 2808 "y": 36 2809 }, 2810 "id": 64, 2811 "options": { 2812 "legend": { 2813 "calcs": [], 2814 "displayMode": "list", 2815 "placement": "bottom", 2816 "showLegend": true 2817 }, 2818 "tooltip": { 2819 "mode": "multi", 2820 "sort": "desc" 2821 } 2822 }, 2823 "targets": [ 2824 { 2825 "datasource": { 2826 "type": "prometheus", 2827 "uid": "prometheus" 2828 }, 2829 "editorMode": "code", 2830 "expr": "histogram_quantile(0.50, sum(rate(workqueue_work_duration_seconds_bucket{pod=~\"$Pod\",name=~\"$Controller\"}[5m])) by (pod, name, le))", 2831 "hide": false, 2832 "legendFormat": "P50 {{pod}} {{name}} ", 2833 "range": true, 2834 "refId": "B" 2835 }, 2836 { 2837 "datasource": { 2838 "type": "prometheus", 2839 "uid": "prometheus" 2840 }, 2841 "editorMode": "code", 2842 "expr": "histogram_quantile(0.90, sum(rate(workqueue_work_duration_seconds_bucket{pod=~\"$Pod\",name=~\"$Controller\"}[5m])) by (pod, name, le))", 2843 "hide": false, 2844 "legendFormat": "P90 {{pod}} {{name}} ", 2845 "range": true, 2846 "refId": "A" 2847 }, 2848 { 2849 "datasource": { 2850 "type": "prometheus", 2851 "uid": "prometheus" 2852 }, 2853 "editorMode": "code", 2854 "expr": "histogram_quantile(0.99, sum(rate(workqueue_work_duration_seconds_bucket{pod=~\"$Pod\",name=~\"$Controller\"}[5m])) by (pod, name, le))", 2855 "hide": false, 2856 "legendFormat": "P99 {{pod}} {{name}} ", 2857 "range": true, 2858 "refId": "C" 2859 } 2860 ], 2861 "title": "Workqueue Work Duration Quantils", 2862 "type": "timeseries" 2863 }, 2864 { 2865 "datasource": { 2866 "type": "prometheus", 2867 "uid": "prometheus" 2868 }, 2869 "description": "How long in seconds an item stays in workqueue before being requested", 2870 "fieldConfig": { 2871 "defaults": { 2872 "color": { 2873 "mode": "palette-classic" 2874 }, 2875 "custom": { 2876 "axisCenteredZero": false, 2877 "axisColorMode": "text", 2878 "axisLabel": "", 2879 "axisPlacement": "auto", 2880 "barAlignment": 0, 2881 "drawStyle": "line", 2882 "fillOpacity": 0, 2883 "gradientMode": "none", 2884 "hideFrom": { 2885 "legend": false, 2886 "tooltip": false, 2887 "viz": false 2888 }, 2889 "lineInterpolation": "linear", 2890 "lineWidth": 1, 2891 "pointSize": 5, 2892 "scaleDistribution": { 2893 "type": "linear" 2894 }, 2895 "showPoints": "auto", 2896 "spanNulls": false, 2897 "stacking": { 2898 "group": "A", 2899 "mode": "none" 2900 }, 2901 "thresholdsStyle": { 2902 "mode": "off" 2903 } 2904 }, 2905 "mappings": [], 2906 "thresholds": { 2907 "mode": "absolute", 2908 "steps": [ 2909 { 2910 "color": "green", 2911 "value": null 2912 }, 2913 { 2914 "color": "red", 2915 "value": 80 2916 } 2917 ] 2918 }, 2919 "unit": "s" 2920 }, 2921 "overrides": [] 2922 }, 2923 "gridPos": { 2924 "h": 9, 2925 "w": 24, 2926 "x": 0, 2927 "y": 45 2928 }, 2929 "id": 63, 2930 "options": { 2931 "legend": { 2932 "calcs": [], 2933 "displayMode": "list", 2934 "placement": "bottom", 2935 "showLegend": true 2936 }, 2937 "tooltip": { 2938 "mode": "multi", 2939 "sort": "desc" 2940 } 2941 }, 2942 "targets": [ 2943 { 2944 "datasource": { 2945 "type": "prometheus", 2946 "uid": "prometheus" 2947 }, 2948 "editorMode": "code", 2949 "expr": "histogram_quantile(0.50, sum(rate(workqueue_queue_duration_seconds_bucket{pod=~\"$Pod\",name=~\"$Controller\"}[5m])) by (pod, name, le))", 2950 "hide": false, 2951 "legendFormat": "P50 {{pod}} {{name}} ", 2952 "range": true, 2953 "refId": "B" 2954 }, 2955 { 2956 "datasource": { 2957 "type": "prometheus", 2958 "uid": "prometheus" 2959 }, 2960 "editorMode": "code", 2961 "expr": "histogram_quantile(0.90, sum(rate(workqueue_queue_duration_seconds_bucket{pod=~\"$Pod\",name=~\"$Controller\"}[5m])) by (pod, name, le))", 2962 "hide": false, 2963 "legendFormat": "P90 {{pod}} {{name}} ", 2964 "range": true, 2965 "refId": "A" 2966 }, 2967 { 2968 "datasource": { 2969 "type": "prometheus", 2970 "uid": "prometheus" 2971 }, 2972 "editorMode": "code", 2973 "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{pod=~\"$Pod\",name=~\"$Controller\"}[5m])) by (pod, name, le))", 2974 "hide": false, 2975 "legendFormat": "P99 {{pod}} {{name}} ", 2976 "range": true, 2977 "refId": "C" 2978 } 2979 ], 2980 "title": "Workqueue Queue Duration Quantils", 2981 "type": "timeseries" 2982 }, 2983 { 2984 "datasource": { 2985 "type": "prometheus", 2986 "uid": "prometheus" 2987 }, 2988 "description": "How long in seconds processing an item from workqueue takes", 2989 "fieldConfig": { 2990 "defaults": { 2991 "custom": { 2992 "hideFrom": { 2993 "legend": false, 2994 "tooltip": false, 2995 "viz": false 2996 }, 2997 "scaleDistribution": { 2998 "type": "linear" 2999 } 3000 } 3001 }, 3002 "overrides": [] 3003 }, 3004 "gridPos": { 3005 "h": 13, 3006 "w": 24, 3007 "x": 0, 3008 "y": 54 3009 }, 3010 "id": 37, 3011 "options": { 3012 "calculate": false, 3013 "cellGap": 2, 3014 "color": { 3015 "exponent": 0.5, 3016 "fill": "dark-orange", 3017 "mode": "scheme", 3018 "reverse": true, 3019 "scale": "exponential", 3020 "scheme": "Oranges", 3021 "steps": 128 3022 }, 3023 "exemplars": { 3024 "color": "rgba(255,0,255,0.7)" 3025 }, 3026 "filterValues": { 3027 "le": 1e-9 3028 }, 3029 "legend": { 3030 "show": true 3031 }, 3032 "rowsFrame": { 3033 "layout": "auto" 3034 }, 3035 "tooltip": { 3036 "show": true, 3037 "yHistogram": false 3038 }, 3039 "yAxis": { 3040 "axisPlacement": "left", 3041 "reverse": false, 3042 "unit": "short" 3043 } 3044 }, 3045 "pluginVersion": "9.5.2", 3046 "targets": [ 3047 { 3048 "datasource": { 3049 "type": "prometheus", 3050 "uid": "prometheus" 3051 }, 3052 "editorMode": "code", 3053 "exemplar": true, 3054 "expr": "rate(workqueue_work_duration_seconds_bucket{pod=~\"$Pod\", name=~\"$Controller\"}[$__rate_interval])", 3055 "format": "heatmap", 3056 "legendFormat": "{{le}}", 3057 "range": true, 3058 "refId": "A" 3059 } 3060 ], 3061 "title": "Workqueue Work Duration", 3062 "type": "heatmap" 3063 }, 3064 { 3065 "datasource": { 3066 "type": "prometheus", 3067 "uid": "prometheus" 3068 }, 3069 "description": "How long in seconds an item stays in workqueue before being requested", 3070 "fieldConfig": { 3071 "defaults": { 3072 "custom": { 3073 "hideFrom": { 3074 "legend": false, 3075 "tooltip": false, 3076 "viz": false 3077 }, 3078 "scaleDistribution": { 3079 "type": "linear" 3080 } 3081 } 3082 }, 3083 "overrides": [] 3084 }, 3085 "gridPos": { 3086 "h": 13, 3087 "w": 24, 3088 "x": 0, 3089 "y": 67 3090 }, 3091 "id": 36, 3092 "options": { 3093 "calculate": false, 3094 "cellGap": 2, 3095 "color": { 3096 "exponent": 0.5, 3097 "fill": "dark-orange", 3098 "mode": "scheme", 3099 "reverse": true, 3100 "scale": "exponential", 3101 "scheme": "Oranges", 3102 "steps": 128 3103 }, 3104 "exemplars": { 3105 "color": "rgba(255,0,255,0.7)" 3106 }, 3107 "filterValues": { 3108 "le": 1e-9 3109 }, 3110 "legend": { 3111 "show": true 3112 }, 3113 "rowsFrame": { 3114 "layout": "auto" 3115 }, 3116 "tooltip": { 3117 "show": true, 3118 "yHistogram": false 3119 }, 3120 "yAxis": { 3121 "axisPlacement": "left", 3122 "reverse": false, 3123 "unit": "short" 3124 } 3125 }, 3126 "pluginVersion": "9.5.2", 3127 "targets": [ 3128 { 3129 "datasource": { 3130 "type": "prometheus", 3131 "uid": "prometheus" 3132 }, 3133 "editorMode": "code", 3134 "exemplar": true, 3135 "expr": "rate(workqueue_queue_duration_seconds_bucket{pod=~\"$Pod\", name=~\"$Controller\"}[$__rate_interval])", 3136 "format": "heatmap", 3137 "legendFormat": "{{le}}", 3138 "range": true, 3139 "refId": "A" 3140 } 3141 ], 3142 "title": "Workqueue Queue Duration", 3143 "type": "heatmap" 3144 } 3145 ], 3146 "title": "Workqueue", 3147 "type": "row" 3148 }, 3149 { 3150 "collapsed": true, 3151 "gridPos": { 3152 "h": 1, 3153 "w": 24, 3154 "x": 0, 3155 "y": 12 3156 }, 3157 "id": 38, 3158 "panels": [ 3159 { 3160 "datasource": { 3161 "type": "prometheus", 3162 "uid": "prometheus" 3163 }, 3164 "fieldConfig": { 3165 "defaults": { 3166 "color": { 3167 "mode": "palette-classic" 3168 }, 3169 "custom": { 3170 "axisCenteredZero": false, 3171 "axisColorMode": "text", 3172 "axisLabel": "", 3173 "axisPlacement": "auto", 3174 "barAlignment": 0, 3175 "drawStyle": "line", 3176 "fillOpacity": 0, 3177 "gradientMode": "none", 3178 "hideFrom": { 3179 "legend": false, 3180 "tooltip": false, 3181 "viz": false 3182 }, 3183 "lineInterpolation": "linear", 3184 "lineWidth": 1, 3185 "pointSize": 5, 3186 "scaleDistribution": { 3187 "type": "linear" 3188 }, 3189 "showPoints": "auto", 3190 "spanNulls": false, 3191 "stacking": { 3192 "group": "A", 3193 "mode": "none" 3194 }, 3195 "thresholdsStyle": { 3196 "mode": "off" 3197 } 3198 }, 3199 "mappings": [], 3200 "thresholds": { 3201 "mode": "absolute", 3202 "steps": [ 3203 { 3204 "color": "green" 3205 }, 3206 { 3207 "color": "red", 3208 "value": 80 3209 } 3210 ] 3211 }, 3212 "unit": "none" 3213 }, 3214 "overrides": [] 3215 }, 3216 "gridPos": { 3217 "h": 8, 3218 "w": 24, 3219 "x": 0, 3220 "y": 21 3221 }, 3222 "id": 39, 3223 "options": { 3224 "legend": { 3225 "calcs": [], 3226 "displayMode": "list", 3227 "placement": "bottom", 3228 "showLegend": true 3229 }, 3230 "tooltip": { 3231 "mode": "multi", 3232 "sort": "desc" 3233 } 3234 }, 3235 "targets": [ 3236 { 3237 "datasource": { 3238 "type": "prometheus", 3239 "uid": "prometheus" 3240 }, 3241 "editorMode": "code", 3242 "expr": "sum(rate(controller_runtime_webhook_requests_total{pod=~\"$Pod\",webhook=~\"/mutate-.*\"}[5m])) by (pod)", 3243 "legendFormat": "{{pod}}: Mutate", 3244 "range": true, 3245 "refId": "A" 3246 }, 3247 { 3248 "datasource": { 3249 "type": "prometheus", 3250 "uid": "prometheus" 3251 }, 3252 "editorMode": "code", 3253 "expr": "sum(rate(controller_runtime_webhook_requests_total{pod=~\"$Pod\",webhook=~\"/validate-.*\"}[5m])) by (pod)", 3254 "hide": false, 3255 "legendFormat": "{{pod}}: Validate", 3256 "range": true, 3257 "refId": "B" 3258 }, 3259 { 3260 "datasource": { 3261 "type": "prometheus", 3262 "uid": "prometheus" 3263 }, 3264 "editorMode": "code", 3265 "expr": "sum(rate(controller_runtime_webhook_requests_total{pod=~\"$Pod\",webhook=~\"/convert\"}[5m])) by (pod)", 3266 "hide": false, 3267 "legendFormat": "{{pod}}: Convert", 3268 "range": true, 3269 "refId": "C" 3270 } 3271 ], 3272 "title": "Webhook Total by Path", 3273 "type": "timeseries" 3274 }, 3275 { 3276 "datasource": { 3277 "type": "prometheus", 3278 "uid": "prometheus" 3279 }, 3280 "fieldConfig": { 3281 "defaults": { 3282 "color": { 3283 "mode": "palette-classic" 3284 }, 3285 "custom": { 3286 "axisCenteredZero": false, 3287 "axisColorMode": "text", 3288 "axisLabel": "", 3289 "axisPlacement": "auto", 3290 "barAlignment": 0, 3291 "drawStyle": "line", 3292 "fillOpacity": 0, 3293 "gradientMode": "none", 3294 "hideFrom": { 3295 "legend": false, 3296 "tooltip": false, 3297 "viz": false 3298 }, 3299 "lineInterpolation": "linear", 3300 "lineWidth": 1, 3301 "pointSize": 5, 3302 "scaleDistribution": { 3303 "type": "linear" 3304 }, 3305 "showPoints": "auto", 3306 "spanNulls": false, 3307 "stacking": { 3308 "group": "A", 3309 "mode": "none" 3310 }, 3311 "thresholdsStyle": { 3312 "mode": "off" 3313 } 3314 }, 3315 "mappings": [], 3316 "thresholds": { 3317 "mode": "absolute", 3318 "steps": [ 3319 { 3320 "color": "green" 3321 }, 3322 { 3323 "color": "red", 3324 "value": 80 3325 } 3326 ] 3327 }, 3328 "unit": "none" 3329 }, 3330 "overrides": [] 3331 }, 3332 "gridPos": { 3333 "h": 8, 3334 "w": 24, 3335 "x": 0, 3336 "y": 29 3337 }, 3338 "id": 40, 3339 "options": { 3340 "legend": { 3341 "calcs": [], 3342 "displayMode": "list", 3343 "placement": "bottom", 3344 "showLegend": true 3345 }, 3346 "tooltip": { 3347 "mode": "multi", 3348 "sort": "desc" 3349 } 3350 }, 3351 "targets": [ 3352 { 3353 "datasource": { 3354 "type": "prometheus", 3355 "uid": "prometheus" 3356 }, 3357 "editorMode": "code", 3358 "expr": "sum(rate(controller_runtime_webhook_requests_total{pod=~\"$Pod\"}[5m])) by (pod,code,webhook) > 0", 3359 "legendFormat": "{{pod}}: {{code}}: {{webhook}}", 3360 "range": true, 3361 "refId": "A" 3362 } 3363 ], 3364 "title": "Webhook Total by Code", 3365 "type": "timeseries" 3366 }, 3367 { 3368 "datasource": { 3369 "type": "prometheus", 3370 "uid": "prometheus" 3371 }, 3372 "fieldConfig": { 3373 "defaults": { 3374 "color": { 3375 "mode": "palette-classic" 3376 }, 3377 "custom": { 3378 "axisCenteredZero": false, 3379 "axisColorMode": "text", 3380 "axisLabel": "", 3381 "axisPlacement": "auto", 3382 "barAlignment": 0, 3383 "drawStyle": "line", 3384 "fillOpacity": 0, 3385 "gradientMode": "none", 3386 "hideFrom": { 3387 "legend": false, 3388 "tooltip": false, 3389 "viz": false 3390 }, 3391 "lineInterpolation": "linear", 3392 "lineWidth": 1, 3393 "pointSize": 5, 3394 "scaleDistribution": { 3395 "type": "linear" 3396 }, 3397 "showPoints": "auto", 3398 "spanNulls": false, 3399 "stacking": { 3400 "group": "A", 3401 "mode": "none" 3402 }, 3403 "thresholdsStyle": { 3404 "mode": "off" 3405 } 3406 }, 3407 "mappings": [], 3408 "thresholds": { 3409 "mode": "absolute", 3410 "steps": [ 3411 { 3412 "color": "green" 3413 }, 3414 { 3415 "color": "red", 3416 "value": 80 3417 } 3418 ] 3419 }, 3420 "unit": "none" 3421 }, 3422 "overrides": [] 3423 }, 3424 "gridPos": { 3425 "h": 8, 3426 "w": 24, 3427 "x": 0, 3428 "y": 37 3429 }, 3430 "id": 41, 3431 "options": { 3432 "legend": { 3433 "calcs": [], 3434 "displayMode": "list", 3435 "placement": "bottom", 3436 "showLegend": true 3437 }, 3438 "tooltip": { 3439 "mode": "multi", 3440 "sort": "desc" 3441 } 3442 }, 3443 "targets": [ 3444 { 3445 "datasource": { 3446 "type": "prometheus", 3447 "uid": "prometheus" 3448 }, 3449 "editorMode": "code", 3450 "expr": "sum(rate(controller_runtime_webhook_requests_total{pod=~\"$Pod\",code!=\"200\"}[5m])) by (pod,code,webhook) > 0", 3451 "legendFormat": "{{pod}}: {{code}}: {{webhook}}", 3452 "range": true, 3453 "refId": "A" 3454 } 3455 ], 3456 "title": "Webhook Total by Code != 200", 3457 "type": "timeseries" 3458 }, 3459 { 3460 "datasource": { 3461 "type": "prometheus", 3462 "uid": "prometheus" 3463 }, 3464 "fieldConfig": { 3465 "defaults": { 3466 "color": { 3467 "mode": "palette-classic" 3468 }, 3469 "custom": { 3470 "axisCenteredZero": false, 3471 "axisColorMode": "text", 3472 "axisLabel": "", 3473 "axisPlacement": "auto", 3474 "barAlignment": 0, 3475 "drawStyle": "line", 3476 "fillOpacity": 0, 3477 "gradientMode": "none", 3478 "hideFrom": { 3479 "legend": false, 3480 "tooltip": false, 3481 "viz": false 3482 }, 3483 "lineInterpolation": "linear", 3484 "lineWidth": 1, 3485 "pointSize": 5, 3486 "scaleDistribution": { 3487 "type": "linear" 3488 }, 3489 "showPoints": "auto", 3490 "spanNulls": false, 3491 "stacking": { 3492 "group": "A", 3493 "mode": "none" 3494 }, 3495 "thresholdsStyle": { 3496 "mode": "off" 3497 } 3498 }, 3499 "mappings": [], 3500 "thresholds": { 3501 "mode": "absolute", 3502 "steps": [ 3503 { 3504 "color": "green" 3505 }, 3506 { 3507 "color": "red", 3508 "value": 80 3509 } 3510 ] 3511 }, 3512 "unit": "s" 3513 }, 3514 "overrides": [] 3515 }, 3516 "gridPos": { 3517 "h": 8, 3518 "w": 24, 3519 "x": 0, 3520 "y": 45 3521 }, 3522 "id": 42, 3523 "options": { 3524 "legend": { 3525 "calcs": [], 3526 "displayMode": "list", 3527 "placement": "bottom", 3528 "showLegend": true 3529 }, 3530 "tooltip": { 3531 "mode": "multi", 3532 "sort": "desc" 3533 } 3534 }, 3535 "targets": [ 3536 { 3537 "datasource": { 3538 "type": "prometheus", 3539 "uid": "prometheus" 3540 }, 3541 "editorMode": "code", 3542 "expr": "sum(rate(controller_runtime_webhook_latency_seconds_sum{pod=~\"$Pod\"}[5m])) by (pod,webhook) / sum(rate(controller_runtime_webhook_latency_seconds_count{pod=~\"$Pod\"}[5m])) by (pod,webhook) > 0", 3543 "legendFormat": "{{pod}}: {{code}}: {{webhook}}", 3544 "range": true, 3545 "refId": "A" 3546 } 3547 ], 3548 "title": "Webhook Total by Path", 3549 "type": "timeseries" 3550 } 3551 ], 3552 "title": "Webhook", 3553 "type": "row" 3554 }, 3555 { 3556 "collapsed": true, 3557 "gridPos": { 3558 "h": 1, 3559 "w": 24, 3560 "x": 0, 3561 "y": 13 3562 }, 3563 "id": 43, 3564 "panels": [ 3565 { 3566 "datasource": { 3567 "type": "prometheus", 3568 "uid": "prometheus" 3569 }, 3570 "fieldConfig": { 3571 "defaults": { 3572 "color": { 3573 "mode": "palette-classic" 3574 }, 3575 "custom": { 3576 "axisCenteredZero": false, 3577 "axisColorMode": "text", 3578 "axisLabel": "", 3579 "axisPlacement": "auto", 3580 "barAlignment": 0, 3581 "drawStyle": "line", 3582 "fillOpacity": 0, 3583 "gradientMode": "none", 3584 "hideFrom": { 3585 "legend": false, 3586 "tooltip": false, 3587 "viz": false 3588 }, 3589 "lineInterpolation": "linear", 3590 "lineWidth": 1, 3591 "pointSize": 5, 3592 "scaleDistribution": { 3593 "type": "linear" 3594 }, 3595 "showPoints": "auto", 3596 "spanNulls": false, 3597 "stacking": { 3598 "group": "A", 3599 "mode": "none" 3600 }, 3601 "thresholdsStyle": { 3602 "mode": "off" 3603 } 3604 }, 3605 "mappings": [], 3606 "thresholds": { 3607 "mode": "absolute", 3608 "steps": [ 3609 { 3610 "color": "green" 3611 }, 3612 { 3613 "color": "red", 3614 "value": 80 3615 } 3616 ] 3617 }, 3618 "unit": "none" 3619 }, 3620 "overrides": [] 3621 }, 3622 "gridPos": { 3623 "h": 8, 3624 "w": 24, 3625 "x": 0, 3626 "y": 38 3627 }, 3628 "id": 44, 3629 "options": { 3630 "legend": { 3631 "calcs": [], 3632 "displayMode": "list", 3633 "placement": "bottom", 3634 "showLegend": true 3635 }, 3636 "tooltip": { 3637 "mode": "multi", 3638 "sort": "desc" 3639 } 3640 }, 3641 "targets": [ 3642 { 3643 "datasource": { 3644 "type": "prometheus", 3645 "uid": "prometheus" 3646 }, 3647 "editorMode": "code", 3648 "expr": "sum(rate(rest_client_requests_total{pod=~\"$Pod\"}[5m])) by (pod, code, method) > 0", 3649 "legendFormat": "{{pod}}: {{method}}: {{code}}", 3650 "range": true, 3651 "refId": "A" 3652 } 3653 ], 3654 "title": "Client Requests by Method, Code", 3655 "type": "timeseries" 3656 } 3657 ], 3658 "title": "Client Requests", 3659 "type": "row" 3660 }, 3661 { 3662 "collapsed": true, 3663 "gridPos": { 3664 "h": 1, 3665 "w": 24, 3666 "x": 0, 3667 "y": 14 3668 }, 3669 "id": 45, 3670 "panels": [ 3671 { 3672 "datasource": { 3673 "type": "prometheus", 3674 "uid": "prometheus" 3675 }, 3676 "fieldConfig": { 3677 "defaults": { 3678 "color": { 3679 "mode": "palette-classic" 3680 }, 3681 "custom": { 3682 "axisCenteredZero": false, 3683 "axisColorMode": "text", 3684 "axisLabel": "", 3685 "axisPlacement": "auto", 3686 "barAlignment": 0, 3687 "drawStyle": "line", 3688 "fillOpacity": 0, 3689 "gradientMode": "none", 3690 "hideFrom": { 3691 "legend": false, 3692 "tooltip": false, 3693 "viz": false 3694 }, 3695 "lineInterpolation": "linear", 3696 "lineWidth": 1, 3697 "pointSize": 5, 3698 "scaleDistribution": { 3699 "type": "linear" 3700 }, 3701 "showPoints": "auto", 3702 "spanNulls": false, 3703 "stacking": { 3704 "group": "A", 3705 "mode": "none" 3706 }, 3707 "thresholdsStyle": { 3708 "mode": "off" 3709 } 3710 }, 3711 "mappings": [], 3712 "thresholds": { 3713 "mode": "absolute", 3714 "steps": [ 3715 { 3716 "color": "green" 3717 }, 3718 { 3719 "color": "red", 3720 "value": 80 3721 } 3722 ] 3723 }, 3724 "unit": "none" 3725 }, 3726 "overrides": [] 3727 }, 3728 "gridPos": { 3729 "h": 7, 3730 "w": 24, 3731 "x": 0, 3732 "y": 15 3733 }, 3734 "id": 47, 3735 "options": { 3736 "legend": { 3737 "calcs": [], 3738 "displayMode": "list", 3739 "placement": "bottom", 3740 "showLegend": true 3741 }, 3742 "tooltip": { 3743 "mode": "multi", 3744 "sort": "desc" 3745 } 3746 }, 3747 "targets": [ 3748 { 3749 "datasource": { 3750 "type": "prometheus", 3751 "uid": "prometheus" 3752 }, 3753 "editorMode": "code", 3754 "expr": "process_open_fds{pod=~\"$Pod\"}", 3755 "legendFormat": "{{pod}}", 3756 "range": true, 3757 "refId": "A" 3758 } 3759 ], 3760 "title": "Proccess Open File Descriptors", 3761 "type": "timeseries" 3762 }, 3763 { 3764 "datasource": { 3765 "type": "prometheus", 3766 "uid": "prometheus" 3767 }, 3768 "fieldConfig": { 3769 "defaults": { 3770 "color": { 3771 "mode": "palette-classic" 3772 }, 3773 "custom": { 3774 "axisCenteredZero": false, 3775 "axisColorMode": "text", 3776 "axisLabel": "", 3777 "axisPlacement": "auto", 3778 "barAlignment": 0, 3779 "drawStyle": "line", 3780 "fillOpacity": 0, 3781 "gradientMode": "none", 3782 "hideFrom": { 3783 "legend": false, 3784 "tooltip": false, 3785 "viz": false 3786 }, 3787 "lineInterpolation": "linear", 3788 "lineWidth": 1, 3789 "pointSize": 5, 3790 "scaleDistribution": { 3791 "type": "linear" 3792 }, 3793 "showPoints": "auto", 3794 "spanNulls": false, 3795 "stacking": { 3796 "group": "A", 3797 "mode": "none" 3798 }, 3799 "thresholdsStyle": { 3800 "mode": "off" 3801 } 3802 }, 3803 "mappings": [], 3804 "thresholds": { 3805 "mode": "absolute", 3806 "steps": [ 3807 { 3808 "color": "green" 3809 }, 3810 { 3811 "color": "red", 3812 "value": 80 3813 } 3814 ] 3815 }, 3816 "unit": "none" 3817 }, 3818 "overrides": [] 3819 }, 3820 "gridPos": { 3821 "h": 7, 3822 "w": 24, 3823 "x": 0, 3824 "y": 22 3825 }, 3826 "id": 59, 3827 "options": { 3828 "legend": { 3829 "calcs": [], 3830 "displayMode": "list", 3831 "placement": "bottom", 3832 "showLegend": true 3833 }, 3834 "tooltip": { 3835 "mode": "multi", 3836 "sort": "desc" 3837 } 3838 }, 3839 "targets": [ 3840 { 3841 "datasource": { 3842 "type": "prometheus", 3843 "uid": "prometheus" 3844 }, 3845 "editorMode": "code", 3846 "expr": "deriv(process_open_fds{pod=~\"$Pod\"}[5m])", 3847 "legendFormat": "{{pod}}", 3848 "range": true, 3849 "refId": "A" 3850 } 3851 ], 3852 "title": "Proccess Open File Descriptors Derivative", 3853 "type": "timeseries" 3854 } 3855 ], 3856 "title": "Process", 3857 "type": "row" 3858 }, 3859 { 3860 "collapsed": true, 3861 "gridPos": { 3862 "h": 1, 3863 "w": 24, 3864 "x": 0, 3865 "y": 15 3866 }, 3867 "id": 46, 3868 "panels": [ 3869 { 3870 "datasource": { 3871 "type": "prometheus", 3872 "uid": "prometheus" 3873 }, 3874 "description": "", 3875 "fieldConfig": { 3876 "defaults": { 3877 "color": { 3878 "mode": "palette-classic" 3879 }, 3880 "custom": { 3881 "axisCenteredZero": false, 3882 "axisColorMode": "text", 3883 "axisLabel": "", 3884 "axisPlacement": "auto", 3885 "barAlignment": 0, 3886 "drawStyle": "line", 3887 "fillOpacity": 0, 3888 "gradientMode": "none", 3889 "hideFrom": { 3890 "legend": false, 3891 "tooltip": false, 3892 "viz": false 3893 }, 3894 "lineInterpolation": "linear", 3895 "lineWidth": 1, 3896 "pointSize": 5, 3897 "scaleDistribution": { 3898 "type": "linear" 3899 }, 3900 "showPoints": "auto", 3901 "spanNulls": false, 3902 "stacking": { 3903 "group": "A", 3904 "mode": "none" 3905 }, 3906 "thresholdsStyle": { 3907 "mode": "off" 3908 } 3909 }, 3910 "mappings": [], 3911 "thresholds": { 3912 "mode": "absolute", 3913 "steps": [ 3914 { 3915 "color": "green" 3916 }, 3917 { 3918 "color": "red", 3919 "value": 80 3920 } 3921 ] 3922 }, 3923 "unit": "none" 3924 }, 3925 "overrides": [] 3926 }, 3927 "gridPos": { 3928 "h": 8, 3929 "w": 12, 3930 "x": 0, 3931 "y": 16 3932 }, 3933 "id": 55, 3934 "options": { 3935 "legend": { 3936 "calcs": [], 3937 "displayMode": "list", 3938 "placement": "bottom", 3939 "showLegend": true 3940 }, 3941 "tooltip": { 3942 "mode": "multi", 3943 "sort": "desc" 3944 } 3945 }, 3946 "targets": [ 3947 { 3948 "datasource": { 3949 "type": "prometheus", 3950 "uid": "prometheus" 3951 }, 3952 "editorMode": "code", 3953 "expr": "go_goroutines{pod=~\"$Pod\"}", 3954 "legendFormat": "{{pod}}", 3955 "range": true, 3956 "refId": "A" 3957 } 3958 ], 3959 "title": "Goroutines", 3960 "type": "timeseries" 3961 } 3962 ], 3963 "title": "Go", 3964 "type": "row" 3965 }, 3966 { 3967 "collapsed": true, 3968 "gridPos": { 3969 "h": 1, 3970 "w": 24, 3971 "x": 0, 3972 "y": 16 3973 }, 3974 "id": 58, 3975 "panels": [ 3976 { 3977 "datasource": { 3978 "type": "prometheus", 3979 "uid": "prometheus" 3980 }, 3981 "description": "", 3982 "fieldConfig": { 3983 "defaults": { 3984 "color": { 3985 "mode": "palette-classic" 3986 }, 3987 "custom": { 3988 "axisCenteredZero": false, 3989 "axisColorMode": "text", 3990 "axisLabel": "", 3991 "axisPlacement": "auto", 3992 "barAlignment": 0, 3993 "drawStyle": "line", 3994 "fillOpacity": 0, 3995 "gradientMode": "none", 3996 "hideFrom": { 3997 "legend": false, 3998 "tooltip": false, 3999 "viz": false 4000 }, 4001 "lineInterpolation": "linear", 4002 "lineWidth": 1, 4003 "pointSize": 5, 4004 "scaleDistribution": { 4005 "type": "linear" 4006 }, 4007 "showPoints": "auto", 4008 "spanNulls": false, 4009 "stacking": { 4010 "group": "A", 4011 "mode": "none" 4012 }, 4013 "thresholdsStyle": { 4014 "mode": "off" 4015 } 4016 }, 4017 "mappings": [], 4018 "thresholds": { 4019 "mode": "absolute", 4020 "steps": [ 4021 { 4022 "color": "green" 4023 }, 4024 { 4025 "color": "red", 4026 "value": 80 4027 } 4028 ] 4029 }, 4030 "unit": "decbytes" 4031 }, 4032 "overrides": [] 4033 }, 4034 "gridPos": { 4035 "h": 8, 4036 "w": 12, 4037 "x": 0, 4038 "y": 17 4039 }, 4040 "id": 51, 4041 "options": { 4042 "legend": { 4043 "calcs": [], 4044 "displayMode": "list", 4045 "placement": "bottom", 4046 "showLegend": true 4047 }, 4048 "tooltip": { 4049 "mode": "multi", 4050 "sort": "desc" 4051 } 4052 }, 4053 "targets": [ 4054 { 4055 "datasource": { 4056 "type": "prometheus", 4057 "uid": "prometheus" 4058 }, 4059 "editorMode": "code", 4060 "expr": "go_memstats_sys_bytes{pod=~\"$Pod\"}", 4061 "legendFormat": "{{pod}}: memstats sys", 4062 "range": true, 4063 "refId": "A" 4064 } 4065 ], 4066 "title": "Total Used Memory", 4067 "type": "timeseries" 4068 }, 4069 { 4070 "datasource": { 4071 "type": "prometheus", 4072 "uid": "prometheus" 4073 }, 4074 "description": "", 4075 "fieldConfig": { 4076 "defaults": { 4077 "color": { 4078 "mode": "palette-classic" 4079 }, 4080 "custom": { 4081 "axisCenteredZero": false, 4082 "axisColorMode": "text", 4083 "axisLabel": "", 4084 "axisPlacement": "auto", 4085 "barAlignment": 0, 4086 "drawStyle": "line", 4087 "fillOpacity": 0, 4088 "gradientMode": "none", 4089 "hideFrom": { 4090 "legend": false, 4091 "tooltip": false, 4092 "viz": false 4093 }, 4094 "lineInterpolation": "linear", 4095 "lineWidth": 1, 4096 "pointSize": 5, 4097 "scaleDistribution": { 4098 "type": "linear" 4099 }, 4100 "showPoints": "auto", 4101 "spanNulls": false, 4102 "stacking": { 4103 "group": "A", 4104 "mode": "none" 4105 }, 4106 "thresholdsStyle": { 4107 "mode": "off" 4108 } 4109 }, 4110 "mappings": [], 4111 "thresholds": { 4112 "mode": "absolute", 4113 "steps": [ 4114 { 4115 "color": "green" 4116 }, 4117 { 4118 "color": "red", 4119 "value": 80 4120 } 4121 ] 4122 }, 4123 "unit": "decbytes" 4124 }, 4125 "overrides": [] 4126 }, 4127 "gridPos": { 4128 "h": 8, 4129 "w": 12, 4130 "x": 12, 4131 "y": 17 4132 }, 4133 "id": 50, 4134 "options": { 4135 "legend": { 4136 "calcs": [], 4137 "displayMode": "list", 4138 "placement": "bottom", 4139 "showLegend": true 4140 }, 4141 "tooltip": { 4142 "mode": "multi", 4143 "sort": "desc" 4144 } 4145 }, 4146 "targets": [ 4147 { 4148 "datasource": { 4149 "type": "prometheus", 4150 "uid": "prometheus" 4151 }, 4152 "editorMode": "code", 4153 "expr": "go_memstats_stack_inuse_bytes{pod=~\"$Pod\"}", 4154 "legendFormat": "{{pod}}: stack inuse", 4155 "range": true, 4156 "refId": "A" 4157 }, 4158 { 4159 "datasource": { 4160 "type": "prometheus", 4161 "uid": "prometheus" 4162 }, 4163 "editorMode": "code", 4164 "expr": "go_memstats_stack_sys_bytes{pod=~\"$Pod\"}", 4165 "hide": false, 4166 "legendFormat": "{{pod}}: stack sys", 4167 "range": true, 4168 "refId": "B" 4169 } 4170 ], 4171 "title": "Memory in Stack", 4172 "type": "timeseries" 4173 }, 4174 { 4175 "datasource": { 4176 "type": "prometheus", 4177 "uid": "prometheus" 4178 }, 4179 "description": "", 4180 "fieldConfig": { 4181 "defaults": { 4182 "color": { 4183 "mode": "palette-classic" 4184 }, 4185 "custom": { 4186 "axisCenteredZero": false, 4187 "axisColorMode": "text", 4188 "axisLabel": "", 4189 "axisPlacement": "auto", 4190 "barAlignment": 0, 4191 "drawStyle": "line", 4192 "fillOpacity": 0, 4193 "gradientMode": "none", 4194 "hideFrom": { 4195 "legend": false, 4196 "tooltip": false, 4197 "viz": false 4198 }, 4199 "lineInterpolation": "linear", 4200 "lineWidth": 1, 4201 "pointSize": 5, 4202 "scaleDistribution": { 4203 "type": "linear" 4204 }, 4205 "showPoints": "auto", 4206 "spanNulls": false, 4207 "stacking": { 4208 "group": "A", 4209 "mode": "none" 4210 }, 4211 "thresholdsStyle": { 4212 "mode": "off" 4213 } 4214 }, 4215 "mappings": [], 4216 "thresholds": { 4217 "mode": "absolute", 4218 "steps": [ 4219 { 4220 "color": "green" 4221 }, 4222 { 4223 "color": "red", 4224 "value": 80 4225 } 4226 ] 4227 }, 4228 "unit": "decbytes" 4229 }, 4230 "overrides": [] 4231 }, 4232 "gridPos": { 4233 "h": 8, 4234 "w": 12, 4235 "x": 0, 4236 "y": 25 4237 }, 4238 "id": 49, 4239 "options": { 4240 "legend": { 4241 "calcs": [], 4242 "displayMode": "list", 4243 "placement": "bottom", 4244 "showLegend": true 4245 }, 4246 "tooltip": { 4247 "mode": "multi", 4248 "sort": "desc" 4249 } 4250 }, 4251 "targets": [ 4252 { 4253 "datasource": { 4254 "type": "prometheus", 4255 "uid": "prometheus" 4256 }, 4257 "editorMode": "code", 4258 "expr": "go_memstats_heap_alloc_bytes{pod=~\"$Pod\"}", 4259 "legendFormat": "{{pod}}: heap alloc", 4260 "range": true, 4261 "refId": "A" 4262 }, 4263 { 4264 "datasource": { 4265 "type": "prometheus", 4266 "uid": "prometheus" 4267 }, 4268 "editorMode": "code", 4269 "expr": "go_memstats_heap_sys_bytes{pod=~\"$Pod\"}", 4270 "hide": false, 4271 "legendFormat": "{{pod}}: heap sys", 4272 "range": true, 4273 "refId": "B" 4274 }, 4275 { 4276 "datasource": { 4277 "type": "prometheus", 4278 "uid": "prometheus" 4279 }, 4280 "editorMode": "code", 4281 "expr": "go_memstats_heap_idle_bytes{pod=~\"$Pod\"}", 4282 "hide": false, 4283 "legendFormat": "{{pod}}: heap idle", 4284 "range": true, 4285 "refId": "C" 4286 }, 4287 { 4288 "datasource": { 4289 "type": "prometheus", 4290 "uid": "prometheus" 4291 }, 4292 "editorMode": "code", 4293 "expr": "go_memstats_heap_inuse_bytes{pod=~\"$Pod\"}", 4294 "hide": false, 4295 "legendFormat": "{{pod}}: heap inuse", 4296 "range": true, 4297 "refId": "D" 4298 }, 4299 { 4300 "datasource": { 4301 "type": "prometheus", 4302 "uid": "prometheus" 4303 }, 4304 "editorMode": "code", 4305 "expr": "go_memstats_heap_released_bytes{pod=~\"$Pod\"}", 4306 "hide": false, 4307 "legendFormat": "{{pod}}: heap released", 4308 "range": true, 4309 "refId": "E" 4310 } 4311 ], 4312 "title": "Memory in Heap", 4313 "type": "timeseries" 4314 }, 4315 { 4316 "datasource": { 4317 "type": "prometheus", 4318 "uid": "prometheus" 4319 }, 4320 "description": "", 4321 "fieldConfig": { 4322 "defaults": { 4323 "color": { 4324 "mode": "palette-classic" 4325 }, 4326 "custom": { 4327 "axisCenteredZero": false, 4328 "axisColorMode": "text", 4329 "axisLabel": "", 4330 "axisPlacement": "auto", 4331 "barAlignment": 0, 4332 "drawStyle": "line", 4333 "fillOpacity": 0, 4334 "gradientMode": "none", 4335 "hideFrom": { 4336 "legend": false, 4337 "tooltip": false, 4338 "viz": false 4339 }, 4340 "lineInterpolation": "linear", 4341 "lineWidth": 1, 4342 "pointSize": 5, 4343 "scaleDistribution": { 4344 "type": "linear" 4345 }, 4346 "showPoints": "auto", 4347 "spanNulls": false, 4348 "stacking": { 4349 "group": "A", 4350 "mode": "none" 4351 }, 4352 "thresholdsStyle": { 4353 "mode": "off" 4354 } 4355 }, 4356 "mappings": [], 4357 "thresholds": { 4358 "mode": "absolute", 4359 "steps": [ 4360 { 4361 "color": "green" 4362 }, 4363 { 4364 "color": "red", 4365 "value": 80 4366 } 4367 ] 4368 }, 4369 "unit": "decbytes" 4370 }, 4371 "overrides": [] 4372 }, 4373 "gridPos": { 4374 "h": 8, 4375 "w": 12, 4376 "x": 12, 4377 "y": 25 4378 }, 4379 "id": 48, 4380 "options": { 4381 "legend": { 4382 "calcs": [], 4383 "displayMode": "list", 4384 "placement": "bottom", 4385 "showLegend": true 4386 }, 4387 "tooltip": { 4388 "mode": "multi", 4389 "sort": "desc" 4390 } 4391 }, 4392 "targets": [ 4393 { 4394 "datasource": { 4395 "type": "prometheus", 4396 "uid": "prometheus" 4397 }, 4398 "editorMode": "code", 4399 "expr": "go_memstats_mspan_inuse_bytes{pod=~\"$Pod\"}", 4400 "legendFormat": "{{pod}}: mspan inuse", 4401 "range": true, 4402 "refId": "A" 4403 }, 4404 { 4405 "datasource": { 4406 "type": "prometheus", 4407 "uid": "prometheus" 4408 }, 4409 "editorMode": "code", 4410 "expr": "go_memstats_mspan_sys_bytes{pod=~\"$Pod\"}", 4411 "hide": false, 4412 "legendFormat": "{{pod}}: mspan sys", 4413 "range": true, 4414 "refId": "B" 4415 }, 4416 { 4417 "datasource": { 4418 "type": "prometheus", 4419 "uid": "prometheus" 4420 }, 4421 "editorMode": "code", 4422 "expr": "go_memstats_mcache_inuse_bytes{pod=~\"$Pod\"}", 4423 "hide": false, 4424 "legendFormat": "{{pod}}: mcache inuse", 4425 "range": true, 4426 "refId": "C" 4427 }, 4428 { 4429 "datasource": { 4430 "type": "prometheus", 4431 "uid": "prometheus" 4432 }, 4433 "editorMode": "code", 4434 "expr": "go_memstats_mcache_sys_bytes{pod=~\"$Pod\"}", 4435 "hide": false, 4436 "legendFormat": "{{pod}}: mcache sys", 4437 "range": true, 4438 "refId": "D" 4439 }, 4440 { 4441 "datasource": { 4442 "type": "prometheus", 4443 "uid": "prometheus" 4444 }, 4445 "editorMode": "code", 4446 "expr": "go_memstats_buck_hash_sys_bytes{pod=~\"$Pod\"}", 4447 "hide": false, 4448 "legendFormat": "{{pod}}: bucket hash sys", 4449 "range": true, 4450 "refId": "E" 4451 }, 4452 { 4453 "datasource": { 4454 "type": "prometheus", 4455 "uid": "prometheus" 4456 }, 4457 "editorMode": "code", 4458 "expr": "go_memstats_gc_sys_bytes{pod=~\"$Pod\"}", 4459 "hide": false, 4460 "legendFormat": "{{pod}}: gc sys", 4461 "range": true, 4462 "refId": "F" 4463 }, 4464 { 4465 "datasource": { 4466 "type": "prometheus", 4467 "uid": "prometheus" 4468 }, 4469 "editorMode": "code", 4470 "expr": "go_memstats_other_sys_bytes{pod=~\"$Pod\"}", 4471 "hide": false, 4472 "legendFormat": "{{pod}}: other sys", 4473 "range": true, 4474 "refId": "G" 4475 }, 4476 { 4477 "datasource": { 4478 "type": "prometheus", 4479 "uid": "prometheus" 4480 }, 4481 "editorMode": "code", 4482 "expr": "go_memstats_next_gc_bytes{pod=~\"$Pod\"}", 4483 "hide": false, 4484 "legendFormat": "{{pod}}: next gc", 4485 "range": true, 4486 "refId": "H" 4487 } 4488 ], 4489 "title": "Memory in Off-Heap", 4490 "type": "timeseries" 4491 }, 4492 { 4493 "datasource": { 4494 "type": "prometheus", 4495 "uid": "prometheus" 4496 }, 4497 "description": "", 4498 "fieldConfig": { 4499 "defaults": { 4500 "color": { 4501 "mode": "palette-classic" 4502 }, 4503 "custom": { 4504 "axisCenteredZero": false, 4505 "axisColorMode": "text", 4506 "axisLabel": "", 4507 "axisPlacement": "auto", 4508 "barAlignment": 0, 4509 "drawStyle": "line", 4510 "fillOpacity": 0, 4511 "gradientMode": "none", 4512 "hideFrom": { 4513 "legend": false, 4514 "tooltip": false, 4515 "viz": false 4516 }, 4517 "lineInterpolation": "linear", 4518 "lineWidth": 1, 4519 "pointSize": 5, 4520 "scaleDistribution": { 4521 "type": "linear" 4522 }, 4523 "showPoints": "auto", 4524 "spanNulls": false, 4525 "stacking": { 4526 "group": "A", 4527 "mode": "none" 4528 }, 4529 "thresholdsStyle": { 4530 "mode": "off" 4531 } 4532 }, 4533 "mappings": [], 4534 "thresholds": { 4535 "mode": "absolute", 4536 "steps": [ 4537 { 4538 "color": "green" 4539 }, 4540 { 4541 "color": "red", 4542 "value": 80 4543 } 4544 ] 4545 }, 4546 "unit": "none" 4547 }, 4548 "overrides": [] 4549 }, 4550 "gridPos": { 4551 "h": 8, 4552 "w": 12, 4553 "x": 0, 4554 "y": 33 4555 }, 4556 "id": 52, 4557 "options": { 4558 "legend": { 4559 "calcs": [], 4560 "displayMode": "list", 4561 "placement": "bottom", 4562 "showLegend": true 4563 }, 4564 "tooltip": { 4565 "mode": "multi", 4566 "sort": "desc" 4567 } 4568 }, 4569 "targets": [ 4570 { 4571 "datasource": { 4572 "type": "prometheus", 4573 "uid": "prometheus" 4574 }, 4575 "editorMode": "code", 4576 "expr": "go_memstats_mallocs_total{pod=~\"$Pod\"} - go_memstats_frees_total{pod=~\"$Pod\"}", 4577 "legendFormat": "{{pod}}", 4578 "range": true, 4579 "refId": "A" 4580 } 4581 ], 4582 "title": "Number of Live Objects", 4583 "type": "timeseries" 4584 }, 4585 { 4586 "datasource": { 4587 "type": "prometheus", 4588 "uid": "prometheus" 4589 }, 4590 "description": "", 4591 "fieldConfig": { 4592 "defaults": { 4593 "color": { 4594 "mode": "palette-classic" 4595 }, 4596 "custom": { 4597 "axisCenteredZero": false, 4598 "axisColorMode": "text", 4599 "axisLabel": "", 4600 "axisPlacement": "auto", 4601 "barAlignment": 0, 4602 "drawStyle": "line", 4603 "fillOpacity": 0, 4604 "gradientMode": "none", 4605 "hideFrom": { 4606 "legend": false, 4607 "tooltip": false, 4608 "viz": false 4609 }, 4610 "lineInterpolation": "linear", 4611 "lineWidth": 1, 4612 "pointSize": 5, 4613 "scaleDistribution": { 4614 "type": "linear" 4615 }, 4616 "showPoints": "auto", 4617 "spanNulls": false, 4618 "stacking": { 4619 "group": "A", 4620 "mode": "none" 4621 }, 4622 "thresholdsStyle": { 4623 "mode": "off" 4624 } 4625 }, 4626 "mappings": [], 4627 "thresholds": { 4628 "mode": "absolute", 4629 "steps": [ 4630 { 4631 "color": "green" 4632 }, 4633 { 4634 "color": "red", 4635 "value": 80 4636 } 4637 ] 4638 }, 4639 "unit": "none" 4640 }, 4641 "overrides": [] 4642 }, 4643 "gridPos": { 4644 "h": 8, 4645 "w": 12, 4646 "x": 12, 4647 "y": 33 4648 }, 4649 "id": 54, 4650 "options": { 4651 "legend": { 4652 "calcs": [], 4653 "displayMode": "list", 4654 "placement": "bottom", 4655 "showLegend": true 4656 }, 4657 "tooltip": { 4658 "mode": "multi", 4659 "sort": "desc" 4660 } 4661 }, 4662 "targets": [ 4663 { 4664 "datasource": { 4665 "type": "prometheus", 4666 "uid": "prometheus" 4667 }, 4668 "editorMode": "code", 4669 "expr": "rate(go_memstats_lookups_total{pod=~\"$Pod\"}[5m])", 4670 "legendFormat": "{{pod}}", 4671 "range": true, 4672 "refId": "A" 4673 } 4674 ], 4675 "title": "Rate of Pointer Dereferences", 4676 "type": "timeseries" 4677 }, 4678 { 4679 "datasource": { 4680 "type": "prometheus", 4681 "uid": "prometheus" 4682 }, 4683 "description": "", 4684 "fieldConfig": { 4685 "defaults": { 4686 "color": { 4687 "mode": "palette-classic" 4688 }, 4689 "custom": { 4690 "axisCenteredZero": false, 4691 "axisColorMode": "text", 4692 "axisLabel": "", 4693 "axisPlacement": "auto", 4694 "barAlignment": 0, 4695 "drawStyle": "line", 4696 "fillOpacity": 0, 4697 "gradientMode": "none", 4698 "hideFrom": { 4699 "legend": false, 4700 "tooltip": false, 4701 "viz": false 4702 }, 4703 "lineInterpolation": "linear", 4704 "lineWidth": 1, 4705 "pointSize": 5, 4706 "scaleDistribution": { 4707 "type": "linear" 4708 }, 4709 "showPoints": "auto", 4710 "spanNulls": false, 4711 "stacking": { 4712 "group": "A", 4713 "mode": "none" 4714 }, 4715 "thresholdsStyle": { 4716 "mode": "off" 4717 } 4718 }, 4719 "mappings": [], 4720 "thresholds": { 4721 "mode": "absolute", 4722 "steps": [ 4723 { 4724 "color": "green" 4725 }, 4726 { 4727 "color": "red", 4728 "value": 80 4729 } 4730 ] 4731 }, 4732 "unit": "none" 4733 }, 4734 "overrides": [] 4735 }, 4736 "gridPos": { 4737 "h": 8, 4738 "w": 12, 4739 "x": 0, 4740 "y": 41 4741 }, 4742 "id": 53, 4743 "options": { 4744 "legend": { 4745 "calcs": [], 4746 "displayMode": "list", 4747 "placement": "bottom", 4748 "showLegend": true 4749 }, 4750 "tooltip": { 4751 "mode": "multi", 4752 "sort": "desc" 4753 } 4754 }, 4755 "targets": [ 4756 { 4757 "datasource": { 4758 "type": "prometheus", 4759 "uid": "prometheus" 4760 }, 4761 "editorMode": "code", 4762 "expr": "rate(go_memstats_mallocs_total{pod=~\"$Pod\"}[5m])", 4763 "legendFormat": "{{pod}}", 4764 "range": true, 4765 "refId": "A" 4766 } 4767 ], 4768 "title": "Rate of Objects Allocated", 4769 "type": "timeseries" 4770 }, 4771 { 4772 "datasource": { 4773 "type": "prometheus", 4774 "uid": "prometheus" 4775 }, 4776 "description": "", 4777 "fieldConfig": { 4778 "defaults": { 4779 "color": { 4780 "mode": "palette-classic" 4781 }, 4782 "custom": { 4783 "axisCenteredZero": false, 4784 "axisColorMode": "text", 4785 "axisLabel": "", 4786 "axisPlacement": "auto", 4787 "barAlignment": 0, 4788 "drawStyle": "line", 4789 "fillOpacity": 0, 4790 "gradientMode": "none", 4791 "hideFrom": { 4792 "legend": false, 4793 "tooltip": false, 4794 "viz": false 4795 }, 4796 "lineInterpolation": "linear", 4797 "lineWidth": 1, 4798 "pointSize": 5, 4799 "scaleDistribution": { 4800 "type": "linear" 4801 }, 4802 "showPoints": "auto", 4803 "spanNulls": false, 4804 "stacking": { 4805 "group": "A", 4806 "mode": "none" 4807 }, 4808 "thresholdsStyle": { 4809 "mode": "off" 4810 } 4811 }, 4812 "mappings": [], 4813 "thresholds": { 4814 "mode": "absolute", 4815 "steps": [ 4816 { 4817 "color": "green" 4818 }, 4819 { 4820 "color": "red", 4821 "value": 80 4822 } 4823 ] 4824 }, 4825 "unit": "decbytes" 4826 }, 4827 "overrides": [] 4828 }, 4829 "gridPos": { 4830 "h": 8, 4831 "w": 12, 4832 "x": 12, 4833 "y": 41 4834 }, 4835 "id": 56, 4836 "options": { 4837 "legend": { 4838 "calcs": [], 4839 "displayMode": "list", 4840 "placement": "bottom", 4841 "showLegend": true 4842 }, 4843 "tooltip": { 4844 "mode": "multi", 4845 "sort": "desc" 4846 } 4847 }, 4848 "targets": [ 4849 { 4850 "datasource": { 4851 "type": "prometheus", 4852 "uid": "prometheus" 4853 }, 4854 "editorMode": "code", 4855 "expr": "rate(go_memstats_alloc_bytes_total{pod=~\"$Pod\"}[5m])", 4856 "legendFormat": "{{pod}}", 4857 "range": true, 4858 "refId": "A" 4859 } 4860 ], 4861 "title": "Rate of Bytes Allocated", 4862 "type": "timeseries" 4863 }, 4864 { 4865 "datasource": { 4866 "type": "prometheus", 4867 "uid": "prometheus" 4868 }, 4869 "description": "", 4870 "fieldConfig": { 4871 "defaults": { 4872 "color": { 4873 "mode": "palette-classic" 4874 }, 4875 "custom": { 4876 "axisCenteredZero": false, 4877 "axisColorMode": "text", 4878 "axisLabel": "", 4879 "axisPlacement": "auto", 4880 "barAlignment": 0, 4881 "drawStyle": "line", 4882 "fillOpacity": 0, 4883 "gradientMode": "none", 4884 "hideFrom": { 4885 "legend": false, 4886 "tooltip": false, 4887 "viz": false 4888 }, 4889 "lineInterpolation": "linear", 4890 "lineWidth": 1, 4891 "pointSize": 5, 4892 "scaleDistribution": { 4893 "type": "linear" 4894 }, 4895 "showPoints": "auto", 4896 "spanNulls": false, 4897 "stacking": { 4898 "group": "A", 4899 "mode": "none" 4900 }, 4901 "thresholdsStyle": { 4902 "mode": "off" 4903 } 4904 }, 4905 "mappings": [], 4906 "thresholds": { 4907 "mode": "absolute", 4908 "steps": [ 4909 { 4910 "color": "green" 4911 }, 4912 { 4913 "color": "red", 4914 "value": 80 4915 } 4916 ] 4917 }, 4918 "unit": "s" 4919 }, 4920 "overrides": [] 4921 }, 4922 "gridPos": { 4923 "h": 8, 4924 "w": 12, 4925 "x": 0, 4926 "y": 49 4927 }, 4928 "id": 57, 4929 "options": { 4930 "legend": { 4931 "calcs": [], 4932 "displayMode": "list", 4933 "placement": "bottom", 4934 "showLegend": true 4935 }, 4936 "tooltip": { 4937 "mode": "multi", 4938 "sort": "desc" 4939 } 4940 }, 4941 "targets": [ 4942 { 4943 "datasource": { 4944 "type": "prometheus", 4945 "uid": "prometheus" 4946 }, 4947 "editorMode": "code", 4948 "expr": "go_gc_duration_seconds{pod=~\"$Pod\"}", 4949 "legendFormat": "{{pod}}: {{quantile}}", 4950 "range": true, 4951 "refId": "A" 4952 } 4953 ], 4954 "title": "GC duration quantile", 4955 "type": "timeseries" 4956 } 4957 ], 4958 "title": "Go Memory", 4959 "type": "row" 4960 } 4961 ], 4962 "refresh": "", 4963 "schemaVersion": 38, 4964 "style": "dark", 4965 "tags": [], 4966 "templating": { 4967 "list": [ 4968 { 4969 "allValue": ".*", 4970 "current": { 4971 "selected": true, 4972 "text": [ 4973 "All" 4974 ], 4975 "value": [ 4976 "$__all" 4977 ] 4978 }, 4979 "datasource": { 4980 "type": "prometheus", 4981 "uid": "prometheus" 4982 }, 4983 "definition": "label_values(kubernetes_io_hostname)", 4984 "hide": 0, 4985 "includeAll": true, 4986 "label": "Node", 4987 "multi": true, 4988 "name": "Node", 4989 "options": [], 4990 "query": { 4991 "query": "label_values(kubernetes_io_hostname)", 4992 "refId": "PrometheusVariableQueryEditor-VariableQuery" 4993 }, 4994 "refresh": 1, 4995 "regex": "", 4996 "skipUrlSync": false, 4997 "sort": 1, 4998 "type": "query" 4999 }, 5000 { 5001 "allValue": ".*", 5002 "current": { 5003 "selected": false, 5004 "text": "kindnet-whq29", 5005 "value": "kindnet-whq29" 5006 }, 5007 "datasource": { 5008 "type": "prometheus", 5009 "uid": "prometheus" 5010 }, 5011 "definition": "label_values(kube_pod_info{node=~\"^$Node$\"},pod)", 5012 "hide": 0, 5013 "includeAll": false, 5014 "label": "Pod", 5015 "multi": true, 5016 "name": "Pod", 5017 "options": [], 5018 "query": { 5019 "query": "label_values(kube_pod_info{node=~\"^$Node$\"},pod)", 5020 "refId": "PrometheusVariableQueryEditor-VariableQuery" 5021 }, 5022 "refresh": 1, 5023 "regex": "", 5024 "skipUrlSync": false, 5025 "sort": 0, 5026 "type": "query" 5027 }, 5028 { 5029 "allValue": ".*", 5030 "current": { 5031 "selected": false, 5032 "text": "All", 5033 "value": "$__all" 5034 }, 5035 "datasource": { 5036 "type": "prometheus", 5037 "uid": "prometheus" 5038 }, 5039 "definition": "label_values(kube_pod_container_info{node=~\"^$Node$\",pod=~\"^$Pod$\"},container)", 5040 "hide": 0, 5041 "includeAll": true, 5042 "label": "Container", 5043 "multi": true, 5044 "name": "Container", 5045 "options": [], 5046 "query": { 5047 "query": "label_values(kube_pod_container_info{node=~\"^$Node$\",pod=~\"^$Pod$\"},container)", 5048 "refId": "PrometheusVariableQueryEditor-VariableQuery" 5049 }, 5050 "refresh": 1, 5051 "regex": "", 5052 "skipUrlSync": false, 5053 "sort": 1, 5054 "type": "query" 5055 }, 5056 { 5057 "allValue": ".*", 5058 "current": { 5059 "selected": true, 5060 "text": [ 5061 "All" 5062 ], 5063 "value": [ 5064 "$__all" 5065 ] 5066 }, 5067 "datasource": { 5068 "type": "prometheus", 5069 "uid": "prometheus" 5070 }, 5071 "definition": "label_values(controller_runtime_active_workers,controller)", 5072 "hide": 0, 5073 "includeAll": true, 5074 "label": "Controller", 5075 "multi": true, 5076 "name": "Controller", 5077 "options": [], 5078 "query": { 5079 "query": "label_values(controller_runtime_active_workers,controller)", 5080 "refId": "PrometheusVariableQueryEditor-VariableQuery" 5081 }, 5082 "refresh": 1, 5083 "regex": "", 5084 "skipUrlSync": false, 5085 "sort": 0, 5086 "type": "query" 5087 }, 5088 { 5089 "datasource": { 5090 "type": "prometheus", 5091 "uid": "prometheus" 5092 }, 5093 "filters": [], 5094 "hide": 0, 5095 "name": "Filters", 5096 "skipUrlSync": false, 5097 "type": "adhoc" 5098 } 5099 ] 5100 }, 5101 "time": { 5102 "from": "now-30m", 5103 "to": "now" 5104 }, 5105 "timepicker": {}, 5106 "timezone": "", 5107 "title": "Controller Runtime", 5108 "uid": "abe29aa7-e44a-4eef-9474-970f95f08ee6", 5109 "version": 1, 5110 "weekStart": "" 5111 }