github.com/grafana/pyroscope@v1.18.0/operations/monitoring/helm/pyroscope-monitoring/templates/_dashboard_operational.yaml (about) 1 {{- define "pyroscope-monitoring.dashboards.operational" -}} 2 {{- $dashboard := "operational" -}} 3 annotations: 4 list: 5 - builtIn: 1 6 datasource: 7 type: datasource 8 uid: grafana 9 enable: true 10 hide: true 11 iconColor: rgba(0, 211, 255, 1) 12 name: Annotations & Alerts 13 target: 14 limit: 100 15 matchAny: false 16 tags: [] 17 type: dashboard 18 type: dashboard 19 - datasource: 20 uid: $loki_datasource 21 enable: true 22 expr: >- 23 {cluster=~"$cluster", container="kube-diff-logger"} | json | 24 namespace_extracted=~"$namespace" | notes!~".*Replicas.*" 25 hide: false 26 iconColor: rgba(255, 96, 96, 1) 27 name: K8s Changes 28 showIn: 0 29 target: {} 30 - datasource: 31 uid: $loki_datasource 32 enable: true 33 expr: >- 34 {cluster=~"$cluster", container="kube-diff-logger"} | json | 35 namespace_extracted=~"$namespace" | notes=~".*Replicas.*" 36 hide: false 37 iconColor: rgba(255, 96, 96, 1) 38 name: K8s Changes(Replicas) 39 showIn: 0 40 target: {} 41 - datasource: 42 type: loki 43 uid: ${loki_datasource} 44 enable: false 45 expr: >- 46 {namespace=~"$namespace",cluster=~"$cluster"} |= "head successfully 47 written to block" | logfmt 48 hide: false 49 iconColor: yellow 50 instant: false 51 name: Flush Events 52 tagKeys: block_path,tenant 53 textFormat: head successfully written to block 54 titleFormat: Head Flush 55 description: Bird eyes view of Pyroscope clusters 56 editable: true 57 fiscalYearStartMonth: 0 58 graphTooltip: 0 59 id: 9933 60 links: 61 {{ .Values.dashboards.links.global | toYaml | nindent 2}} 62 {{- with (dig $dashboard (list) .Values.dashboards.links.perDashboard) }} 63 {{ . | toYaml | nindent 2}} 64 {{- end }} 65 panels: 66 - collapsed: false 67 gridPos: 68 h: 1 69 w: 24 70 x: 0 71 'y': 0 72 id: 4 73 panels: [] 74 title: Global 75 type: row 76 - datasource: 77 type: prometheus 78 uid: ${datasource} 79 fieldConfig: 80 defaults: 81 color: 82 mode: palette-classic 83 custom: 84 axisBorderShow: false 85 axisCenteredZero: false 86 axisColorMode: text 87 axisLabel: '' 88 axisPlacement: auto 89 barAlignment: 0 90 barWidthFactor: 0.6 91 drawStyle: line 92 fillOpacity: 10 93 gradientMode: none 94 hideFrom: 95 legend: false 96 tooltip: false 97 viz: false 98 insertNulls: false 99 lineInterpolation: linear 100 lineWidth: 1 101 pointSize: 5 102 scaleDistribution: 103 type: linear 104 showPoints: auto 105 spanNulls: false 106 stacking: 107 group: A 108 mode: normal 109 thresholdsStyle: 110 mode: 'off' 111 mappings: [] 112 thresholds: 113 mode: absolute 114 steps: 115 - color: green 116 - color: red 117 value: 80 118 unit: binBps 119 overrides: [] 120 gridPos: 121 h: 6 122 w: 7 123 x: 0 124 'y': 1 125 id: 29 126 options: 127 legend: 128 calcs: [] 129 displayMode: table 130 placement: right 131 showLegend: true 132 tooltip: 133 hideZeros: false 134 mode: single 135 sort: none 136 pluginVersion: 12.1.0-88106 137 targets: 138 - datasource: 139 type: prometheus 140 uid: ${datasource} 141 editorMode: code 142 expr: >- 143 sum by (tenant) 144 (rate(pyroscope_distributor_received_decompressed_bytes_sum{cluster=~"$cluster",namespace=~"$namespace"}[$__rate_interval])) 145 hide: false 146 legendFormat: __auto 147 range: true 148 refId: A 149 title: MBs per Tenant (Decompressed) 150 type: timeseries 151 - datasource: 152 type: prometheus 153 uid: ${datasource} 154 fieldConfig: 155 defaults: 156 color: 157 mode: palette-classic 158 custom: 159 axisBorderShow: false 160 axisCenteredZero: false 161 axisColorMode: text 162 axisLabel: '' 163 axisPlacement: auto 164 barAlignment: 0 165 barWidthFactor: 0.6 166 drawStyle: line 167 fillOpacity: 10 168 gradientMode: none 169 hideFrom: 170 legend: false 171 tooltip: false 172 viz: false 173 insertNulls: false 174 lineInterpolation: linear 175 lineWidth: 1 176 pointSize: 5 177 scaleDistribution: 178 type: linear 179 showPoints: auto 180 spanNulls: false 181 stacking: 182 group: A 183 mode: normal 184 thresholdsStyle: 185 mode: 'off' 186 mappings: [] 187 thresholds: 188 mode: absolute 189 steps: 190 - color: green 191 - color: red 192 value: 80 193 unit: cps 194 overrides: [] 195 gridPos: 196 h: 6 197 w: 6 198 x: 7 199 'y': 1 200 id: 32 201 options: 202 legend: 203 calcs: [] 204 displayMode: table 205 placement: right 206 showLegend: true 207 tooltip: 208 hideZeros: false 209 mode: single 210 sort: none 211 pluginVersion: 12.1.0-88106 212 targets: 213 - datasource: 214 type: prometheus 215 uid: ${datasource} 216 editorMode: code 217 expr: >- 218 sum by (tenant) 219 (rate(pyroscope_distributor_received_decompressed_bytes_count{cluster=~"$cluster",namespace=~"$namespace"}[$__rate_interval])) 220 legendFormat: __auto 221 range: true 222 refId: A 223 title: Profiles/s per Tenant 224 type: timeseries 225 - datasource: 226 type: prometheus 227 uid: ${datasource} 228 fieldConfig: 229 defaults: 230 color: 231 mode: palette-classic 232 custom: 233 axisBorderShow: false 234 axisCenteredZero: false 235 axisColorMode: text 236 axisLabel: '' 237 axisPlacement: auto 238 barAlignment: 0 239 barWidthFactor: 0.6 240 drawStyle: line 241 fillOpacity: 10 242 gradientMode: none 243 hideFrom: 244 legend: false 245 tooltip: false 246 viz: false 247 insertNulls: false 248 lineInterpolation: linear 249 lineWidth: 1 250 pointSize: 5 251 scaleDistribution: 252 type: linear 253 showPoints: never 254 spanNulls: false 255 stacking: 256 group: A 257 mode: none 258 thresholdsStyle: 259 mode: 'off' 260 mappings: [] 261 thresholds: 262 mode: absolute 263 steps: 264 - color: green 265 - color: red 266 value: 80 267 unit: short 268 overrides: [] 269 gridPos: 270 h: 6 271 w: 5 272 x: 13 273 'y': 1 274 id: 24 275 options: 276 dataLinks: [] 277 legend: 278 calcs: [] 279 displayMode: list 280 placement: bottom 281 showLegend: false 282 tooltip: 283 hideZeros: false 284 mode: multi 285 sort: desc 286 pluginVersion: 12.1.0-88106 287 targets: 288 - datasource: 289 type: prometheus 290 uid: ${datasource} 291 expr: >- 292 increase(kube_pod_container_status_restarts_total{cluster=~"$cluster", 293 namespace=~"$namespace"}[10m]) > 0 294 hide: false 295 interval: '' 296 legendFormat: '{{"{{container}}-{{pod}}"}}' 297 refId: B 298 title: Container Restarts 299 type: timeseries 300 - datasource: 301 type: prometheus 302 uid: ${datasource} 303 fieldConfig: 304 defaults: 305 color: 306 mode: palette-classic 307 custom: 308 hideFrom: 309 legend: false 310 tooltip: false 311 viz: false 312 mappings: [] 313 overrides: [] 314 gridPos: 315 h: 6 316 w: 3 317 x: 18 318 'y': 1 319 id: 34 320 options: 321 legend: 322 displayMode: list 323 placement: bottom 324 showLegend: true 325 pieType: pie 326 reduceOptions: 327 calcs: 328 - lastNotNull 329 fields: '' 330 values: false 331 tooltip: 332 hideZeros: false 333 mode: single 334 sort: none 335 pluginVersion: 12.1.0-88106 336 targets: 337 - datasource: 338 type: prometheus 339 uid: ${datasource} 340 editorMode: code 341 exemplar: false 342 expr: >- 343 sum by (state) (pyroscope_ring_members{cluster=~"$cluster", 344 namespace=~"$namespace", name="ingester"}) / count by 345 (state)(pyroscope_ring_members{cluster=~"$cluster", 346 namespace=~"$namespace", name="ingester"}) 347 format: time_series 348 instant: true 349 legendFormat: __auto 350 range: false 351 refId: A 352 title: Ingester Ring Status 353 type: piechart 354 - datasource: 355 type: prometheus 356 uid: ${datasource} 357 description: Distributor Ring is used for rate limiting 358 fieldConfig: 359 defaults: 360 color: 361 mode: palette-classic 362 custom: 363 hideFrom: 364 legend: false 365 tooltip: false 366 viz: false 367 mappings: [] 368 overrides: [] 369 gridPos: 370 h: 6 371 w: 3 372 x: 21 373 'y': 1 374 id: 35 375 options: 376 legend: 377 displayMode: list 378 placement: bottom 379 showLegend: true 380 pieType: pie 381 reduceOptions: 382 calcs: 383 - lastNotNull 384 fields: '' 385 values: false 386 tooltip: 387 hideZeros: false 388 mode: single 389 sort: none 390 pluginVersion: 12.1.0-88106 391 targets: 392 - datasource: 393 type: prometheus 394 uid: ${datasource} 395 editorMode: code 396 exemplar: false 397 expr: >- 398 sum by (state) (pyroscope_ring_members{cluster=~"$cluster", 399 namespace=~"$namespace", name="distributor"}) / count by 400 (state)(pyroscope_ring_members{cluster=~"$cluster", 401 namespace=~"$namespace", name="distributor"}) 402 format: time_series 403 instant: true 404 legendFormat: __auto 405 range: false 406 refId: A 407 title: Distributor Ring Status 408 type: piechart 409 - datasource: 410 type: prometheus 411 uid: ${datasource} 412 fieldConfig: 413 defaults: 414 color: 415 mode: thresholds 416 custom: 417 align: left 418 cellOptions: 419 type: auto 420 filterable: true 421 inspect: false 422 mappings: [] 423 thresholds: 424 mode: absolute 425 steps: 426 - color: green 427 - color: red 428 value: 80 429 overrides: 430 - matcher: 431 id: byName 432 options: Value 433 properties: 434 - id: unit 435 value: decbytes 436 - matcher: 437 id: byName 438 options: tenant 439 properties: 440 - id: custom.width 441 value: 78 442 - matcher: 443 id: byName 444 options: slug 445 properties: 446 - id: custom.width 447 value: 132 448 - matcher: 449 id: byName 450 options: org_name 451 properties: 452 - id: custom.width 453 value: 142 454 gridPos: 455 h: 10 456 w: 5 457 x: 0 458 'y': 7 459 id: 85 460 options: 461 cellHeight: sm 462 footer: 463 countRows: false 464 enablePagination: true 465 fields: '' 466 reducer: 467 - count 468 show: false 469 showHeader: true 470 sortBy: 471 - desc: true 472 displayName: Value 473 pluginVersion: 12.1.0-88106 474 targets: 475 - datasource: 476 type: prometheus 477 uid: ${datasource} 478 editorMode: code 479 exemplar: false 480 expr: | 481 {{ .Values.dashboards.tenantQuery | nindent 10 }} 482 format: table 483 instant: true 484 legendFormat: __auto 485 range: false 486 refId: A 487 title: Tenant id / name mapping 488 transformations: 489 - id: organize 490 options: 491 excludeByName: 492 Time: true 493 Value: false 494 cluster: true 495 includeByName: {} 496 indexByName: 497 Time: 0 498 Value: 5 499 cluster: 1 500 org_name: 4 501 slug: 3 502 tenant: 2 503 renameByName: {} 504 type: table 505 - datasource: 506 type: prometheus 507 uid: $datasource 508 fieldConfig: 509 defaults: 510 color: 511 mode: palette-classic 512 custom: 513 axisBorderShow: false 514 axisCenteredZero: false 515 axisColorMode: text 516 axisLabel: '' 517 axisPlacement: auto 518 barAlignment: 0 519 barWidthFactor: 0.6 520 drawStyle: line 521 fillOpacity: 40 522 gradientMode: none 523 hideFrom: 524 legend: false 525 tooltip: false 526 viz: false 527 insertNulls: false 528 lineInterpolation: linear 529 lineWidth: 1 530 pointSize: 5 531 scaleDistribution: 532 type: linear 533 showPoints: never 534 spanNulls: false 535 stacking: 536 group: A 537 mode: normal 538 thresholdsStyle: 539 mode: 'off' 540 mappings: [] 541 thresholds: 542 mode: absolute 543 steps: 544 - color: green 545 - color: red 546 value: 80 547 unit: reqps 548 overrides: 549 - matcher: 550 id: byName 551 options: '5xx ' 552 properties: 553 - id: color 554 value: 555 fixedColor: red 556 mode: fixed 557 - matcher: 558 id: byName 559 options: '2xx ' 560 properties: 561 - id: color 562 value: 563 fixedColor: green 564 mode: fixed 565 seriesBy: min 566 - matcher: 567 id: byName 568 options: '4xx ' 569 properties: 570 - id: color 571 value: 572 fixedColor: yellow 573 mode: fixed 574 gridPos: 575 h: 5 576 w: 6 577 x: 5 578 'y': 7 579 id: 7 580 options: 581 dataLinks: [] 582 legend: 583 calcs: [] 584 displayMode: list 585 placement: bottom 586 showLegend: true 587 tooltip: 588 hideZeros: false 589 mode: multi 590 sort: none 591 pluginVersion: 12.1.0-88106 592 targets: 593 - datasource: 594 uid: $datasource 595 editorMode: code 596 expr: |- 597 sum by (status) ( 598 label_replace( 599 label_replace( 600 rate(pyroscope_request_duration_seconds_count{cluster=~"$cluster", {{ include "pyroscope-monitoring.dashboards-ingest-selector" . }}, namespace=~"$namespace", route=~".*ingest.*|.*pusher.*"}[$__rate_interval]), 601 "status", "${1}xx", "status_code", "([0-9]).."), 602 "status", "${1}", "status_code", "([a-z]+)") 603 ) 604 legendFormat: '{{"{{status}}"}} ' 605 range: true 606 refId: A 607 title: Pushes/Second 608 type: timeseries 609 - datasource: 610 type: prometheus 611 uid: ${datasource} 612 fieldConfig: 613 defaults: 614 color: 615 mode: palette-classic 616 custom: 617 axisBorderShow: false 618 axisCenteredZero: false 619 axisColorMode: text 620 axisLabel: '' 621 axisPlacement: auto 622 barAlignment: 0 623 barWidthFactor: 0.6 624 drawStyle: line 625 fillOpacity: 0 626 gradientMode: none 627 hideFrom: 628 legend: false 629 tooltip: false 630 viz: false 631 insertNulls: false 632 lineInterpolation: linear 633 lineWidth: 1 634 pointSize: 5 635 scaleDistribution: 636 type: linear 637 showPoints: auto 638 spanNulls: false 639 stacking: 640 group: A 641 mode: none 642 thresholdsStyle: 643 mode: 'off' 644 mappings: [] 645 thresholds: 646 mode: absolute 647 steps: 648 - color: green 649 - color: red 650 value: 80 651 unit: s 652 overrides: [] 653 gridPos: 654 h: 5 655 w: 7 656 x: 11 657 'y': 7 658 id: 27 659 options: 660 legend: 661 calcs: [] 662 displayMode: table 663 placement: right 664 showLegend: true 665 tooltip: 666 hideZeros: false 667 mode: single 668 sort: none 669 pluginVersion: 12.1.0-88106 670 targets: 671 - datasource: 672 type: prometheus 673 uid: ${datasource} 674 editorMode: code 675 expr: >- 676 histogram_quantile(0.99, sum by (route) ( 677 rate(pyroscope_request_duration_seconds{{ "{" }}{{ include "pyroscope-monitoring.dashboards-ingest-selector" . }}, namespace=~"$namespace", 678 cluster=~"$cluster",route=~".*ingest.*|.*pusher.*"}[$__rate_interval]))) 679 legendFormat: .99 {{"{{route}}"}} 680 range: true 681 refId: A 682 - datasource: 683 type: prometheus 684 uid: ${datasource} 685 editorMode: code 686 expr: >- 687 histogram_quantile(0.75, sum by (route) ( 688 rate(pyroscope_request_duration_seconds{{ "{" }}{{ include "pyroscope-monitoring.dashboards-ingest-selector" . }}, namespace=~"$namespace", 689 cluster=~"$cluster",route=~".*ingest.*|.*pusher.*"}[$__rate_interval]))) 690 hide: false 691 legendFormat: .75 {{"{{route}}"}} 692 range: true 693 refId: B 694 - datasource: 695 type: prometheus 696 uid: ${datasource} 697 editorMode: code 698 expr: >- 699 histogram_quantile(0.5, sum by (route) ( 700 rate(pyroscope_request_duration_seconds{{ "{" }}{{ include "pyroscope-monitoring.dashboards-ingest-selector" . }}, namespace=~"$namespace", 701 cluster=~"$cluster",route=~".*ingest.*|.*pusher.*"}[$__rate_interval]))) 702 hide: false 703 legendFormat: .5 {{"{{route}}"}} 704 range: true 705 refId: C 706 title: Push Latencies 707 type: timeseries 708 - datasource: 709 type: prometheus 710 uid: ${datasource} 711 fieldConfig: 712 defaults: 713 color: 714 mode: thresholds 715 custom: 716 align: center 717 cellOptions: 718 type: auto 719 filterable: true 720 inspect: false 721 mappings: [] 722 thresholds: 723 mode: absolute 724 steps: 725 - color: green 726 - color: red 727 value: 80 728 overrides: [] 729 gridPos: 730 h: 10 731 w: 6 732 x: 18 733 'y': 7 734 id: 31 735 options: 736 cellHeight: sm 737 footer: 738 countRows: false 739 enablePagination: false 740 fields: '' 741 reducer: 742 - sum 743 show: true 744 showHeader: true 745 showRowNums: false 746 sortBy: [] 747 pluginVersion: 12.1.0-88106 748 targets: 749 - datasource: 750 type: prometheus 751 uid: ${datasource} 752 editorMode: code 753 exemplar: false 754 expr: >- 755 count by (container,version) 756 (pyroscope_build_info{cluster=~"$cluster",namespace=~"$namespace"}) or 757 count by (container,version) 758 (cloud_backend_gateway_build_info{cluster=~"$cluster",namespace=~"$namespace"}) 759 format: table 760 instant: true 761 legendFormat: __auto 762 range: false 763 refId: A 764 - datasource: 765 type: prometheus 766 uid: ${datasource} 767 hide: false 768 refId: B 769 title: Version 770 transformations: 771 - id: organize 772 options: 773 excludeByName: 774 Time: true 775 indexByName: {} 776 renameByName: 777 Value: Pods 778 type: table 779 - datasource: 780 type: prometheus 781 uid: $datasource 782 fieldConfig: 783 defaults: 784 color: 785 mode: palette-classic 786 custom: 787 axisBorderShow: false 788 axisCenteredZero: false 789 axisColorMode: text 790 axisLabel: '' 791 axisPlacement: auto 792 barAlignment: 0 793 barWidthFactor: 0.6 794 drawStyle: line 795 fillOpacity: 100 796 gradientMode: none 797 hideFrom: 798 legend: false 799 tooltip: false 800 viz: false 801 insertNulls: false 802 lineInterpolation: linear 803 lineWidth: 1 804 pointSize: 5 805 scaleDistribution: 806 type: linear 807 showPoints: never 808 spanNulls: false 809 stacking: 810 group: A 811 mode: normal 812 thresholdsStyle: 813 mode: 'off' 814 mappings: [] 815 thresholds: 816 mode: absolute 817 steps: 818 - color: green 819 - color: red 820 value: 80 821 unit: reqps 822 overrides: 823 - matcher: 824 id: byName 825 options: '5xx ' 826 properties: 827 - id: color 828 value: 829 fixedColor: red 830 mode: fixed 831 - matcher: 832 id: byName 833 options: '2xx ' 834 properties: 835 - id: color 836 value: 837 fixedColor: green 838 mode: fixed 839 seriesBy: min 840 - matcher: 841 id: byName 842 options: '4xx ' 843 properties: 844 - id: color 845 value: 846 fixedColor: yellow 847 mode: fixed 848 gridPos: 849 h: 5 850 w: 6 851 x: 5 852 'y': 12 853 id: 6 854 options: 855 dataLinks: [] 856 legend: 857 calcs: [] 858 displayMode: list 859 placement: bottom 860 showLegend: true 861 tooltip: 862 hideZeros: false 863 mode: multi 864 sort: none 865 pluginVersion: 12.1.0-88106 866 targets: 867 - datasource: 868 uid: $datasource 869 editorMode: code 870 expr: |- 871 sum by (status) ( 872 label_replace( 873 label_replace( 874 rate(pyroscope_request_duration_seconds_count{cluster=~"$cluster", {{ include "pyroscope-monitoring.dashboards-ingest-selector" . }}, namespace=~"$namespace", route=~".*pyroscope_render.*|.*pyroscope_label.*|.*querierservice.*"}[$__rate_interval]), 875 "status", "${1}xx", "status_code", "([0-9]).."), 876 "status", "${1}", "status_code", "([a-z]+)") 877 ) 878 legendFormat: '{{"{{status}}"}} ' 879 range: true 880 refId: A 881 title: Queries/Second 882 type: timeseries 883 - datasource: 884 type: prometheus 885 uid: ${datasource} 886 fieldConfig: 887 defaults: 888 color: 889 mode: palette-classic 890 custom: 891 axisBorderShow: false 892 axisCenteredZero: false 893 axisColorMode: text 894 axisLabel: '' 895 axisPlacement: auto 896 barAlignment: 0 897 barWidthFactor: 0.6 898 drawStyle: line 899 fillOpacity: 0 900 gradientMode: none 901 hideFrom: 902 legend: false 903 tooltip: false 904 viz: false 905 insertNulls: false 906 lineInterpolation: linear 907 lineWidth: 1 908 pointSize: 5 909 scaleDistribution: 910 type: linear 911 showPoints: auto 912 spanNulls: false 913 stacking: 914 group: A 915 mode: none 916 thresholdsStyle: 917 mode: 'off' 918 mappings: [] 919 thresholds: 920 mode: absolute 921 steps: 922 - color: green 923 - color: red 924 value: 80 925 unit: s 926 overrides: [] 927 gridPos: 928 h: 5 929 w: 7 930 x: 11 931 'y': 12 932 id: 26 933 options: 934 legend: 935 calcs: [] 936 displayMode: table 937 placement: right 938 showLegend: true 939 tooltip: 940 hideZeros: false 941 mode: single 942 sort: none 943 pluginVersion: 12.1.0-88106 944 targets: 945 - datasource: 946 type: prometheus 947 uid: ${datasource} 948 editorMode: code 949 expr: >- 950 histogram_quantile(0.99, sum by (route) ( 951 rate(pyroscope_request_duration_seconds{{ "{" }}{{ include "pyroscope-monitoring.dashboards-ingest-selector" . }}, namespace=~"$namespace", 952 cluster=~"$cluster",route=~".*pyroscope_render.*|.*pyroscope_label.*|.*querierservice.*"}[$__rate_interval]))) 953 legendFormat: .99 {{"{{route}}"}} 954 range: true 955 refId: A 956 - datasource: 957 type: prometheus 958 uid: ${datasource} 959 editorMode: code 960 expr: >- 961 histogram_quantile(0.75, sum by (route) ( 962 rate(pyroscope_request_duration_seconds{{ "{" }}{{ include "pyroscope-monitoring.dashboards-ingest-selector" . }}, namespace=~"$namespace", 963 cluster=~"$cluster",route=~".*pyroscope_render.*|.*pyroscope_label.*|.*querierservice.*"}[$__rate_interval]))) 964 hide: false 965 legendFormat: .75 {{"{{route}}"}} 966 range: true 967 refId: B 968 - datasource: 969 type: prometheus 970 uid: ${datasource} 971 editorMode: code 972 expr: >- 973 histogram_quantile(0.5, sum by (route) ( 974 rate(pyroscope_request_duration_seconds{{ "{" }}{{ include "pyroscope-monitoring.dashboards-ingest-selector" . }}, namespace=~"$namespace", 975 cluster=~"$cluster",route=~".*pyroscope_render.*|.*pyroscope_label.*|.*querierservice.*"}[$__rate_interval]))) 976 hide: false 977 legendFormat: .5 {{"{{route}}"}} 978 range: true 979 refId: C 980 title: Query Latencies 981 type: timeseries 982 - datasource: 983 type: loki 984 uid: ${loki_datasource} 985 fieldConfig: 986 defaults: 987 color: 988 fixedColor: red 989 mode: fixed 990 custom: 991 axisBorderShow: false 992 axisCenteredZero: false 993 axisColorMode: text 994 axisLabel: '' 995 axisPlacement: auto 996 barAlignment: 0 997 barWidthFactor: 0.6 998 drawStyle: bars 999 fillOpacity: 23 1000 gradientMode: none 1001 hideFrom: 1002 legend: false 1003 tooltip: false 1004 viz: false 1005 insertNulls: false 1006 lineInterpolation: linear 1007 lineWidth: 1 1008 pointSize: 5 1009 scaleDistribution: 1010 type: linear 1011 showPoints: auto 1012 spanNulls: false 1013 stacking: 1014 group: A 1015 mode: normal 1016 thresholdsStyle: 1017 mode: 'off' 1018 mappings: [] 1019 thresholds: 1020 mode: absolute 1021 steps: 1022 - color: green 1023 - color: red 1024 value: 80 1025 overrides: [] 1026 gridPos: 1027 h: 3 1028 w: 24 1029 x: 0 1030 'y': 17 1031 id: 39 1032 options: 1033 legend: 1034 calcs: [] 1035 displayMode: table 1036 placement: right 1037 showLegend: true 1038 tooltip: 1039 hideZeros: false 1040 mode: single 1041 sort: none 1042 pluginVersion: 12.1.0-88106 1043 targets: 1044 - datasource: 1045 type: loki 1046 uid: ${loki_datasource} 1047 editorMode: code 1048 expr: >- 1049 sum by (container)( 1050 count_over_time({namespace=~"$namespace",cluster=~"$cluster"} != 1051 "stream context finished" != "fast-joining node failed" != "CAS 1052 attempt failed" != "memberlist TCPTransport" |= " panic " or "panic:" 1053 or " err=" or "level=error" [$__interval])) 1054 queryType: range 1055 refId: A 1056 title: Error & Panic rate 1057 type: timeseries 1058 - datasource: 1059 type: loki 1060 uid: ${loki_datasource} 1061 fieldConfig: 1062 defaults: {} 1063 overrides: [] 1064 gridPos: 1065 h: 10 1066 w: 24 1067 x: 0 1068 'y': 20 1069 id: 37 1070 options: 1071 dedupStrategy: numbers 1072 enableInfiniteScrolling: false 1073 enableLogDetails: true 1074 prettifyLogMessage: false 1075 showCommonLabels: false 1076 showLabels: false 1077 showTime: false 1078 sortOrder: Descending 1079 wrapLogMessage: true 1080 pluginVersion: 12.1.0-88106 1081 targets: 1082 - datasource: 1083 type: loki 1084 uid: ${loki_datasource} 1085 editorMode: code 1086 expr: >- 1087 {namespace=~"$namespace",cluster=~"$cluster"} != "stream context 1088 finished" != "fast-joining node failed" != "CAS attempt failed" != 1089 "memberlist TCPTransport" |= " panic " or "panic:" or " err=" or 1090 "level=error" 1091 queryType: range 1092 refId: A 1093 title: Errors and Panics 1094 type: logs 1095 - datasource: 1096 type: loki 1097 uid: ${loki_datasource} 1098 fieldConfig: 1099 defaults: 1100 color: 1101 mode: thresholds 1102 custom: 1103 align: auto 1104 cellOptions: 1105 type: auto 1106 filterable: true 1107 inspect: false 1108 mappings: [] 1109 thresholds: 1110 mode: absolute 1111 steps: 1112 - color: green 1113 - color: red 1114 value: 80 1115 overrides: 1116 - matcher: 1117 id: byName 1118 options: ts 1119 properties: 1120 - id: custom.width 1121 value: 271 1122 - matcher: 1123 id: byName 1124 options: path 1125 properties: 1126 - id: custom.width 1127 value: 388 1128 - matcher: 1129 id: byName 1130 options: status 1131 properties: 1132 - id: custom.cellOptions 1133 value: 1134 applyToRow: false 1135 mode: gradient 1136 type: color-background 1137 wrapText: false 1138 - id: mappings 1139 value: 1140 - options: 1141 from: 200 1142 result: 1143 color: light-green 1144 index: 0 1145 to: 399 1146 type: range 1147 - options: 1148 from: 400 1149 result: 1150 color: light-yellow 1151 index: 1 1152 to: 499 1153 type: range 1154 - options: 1155 from: 500 1156 result: 1157 color: light-red 1158 index: 2 1159 to: 599 1160 type: range 1161 - id: custom.width 1162 value: 96 1163 - matcher: 1164 id: byName 1165 options: traceID 1166 properties: 1167 - id: links 1168 value: 1169 - targetBlank: true 1170 title: Show trace 1171 url: >- 1172 /explore?schemaVersion=1&panes=%7B%22xbp%22:%7B%22datasource%22:%22grafanacloud-traces%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22datasource%22:%7B%22type%22:%22tempo%22,%22uid%22:%22grafanacloud-traces%22%7D,%22queryType%22:%22traceql%22,%22limit%22:20,%22tableType%22:%22traces%22,%22metricsQueryType%22:%22range%22,%22query%22:%22${__value.text}%5Cn%22%7D%5D,%22range%22:%7B%22from%22:%22${__from}%22,%22to%22:%22${__to}%22%7D%7D%7D&orgId=1 1173 gridPos: 1174 h: 10 1175 w: 24 1176 x: 0 1177 'y': 30 1178 id: 108 1179 options: 1180 cellHeight: sm 1181 footer: 1182 countRows: false 1183 enablePagination: true 1184 fields: '' 1185 reducer: 1186 - sum 1187 show: false 1188 showHeader: true 1189 sortBy: 1190 - desc: false 1191 displayName: status 1192 pluginVersion: 12.1.0-88106 1193 targets: 1194 - datasource: 1195 type: loki 1196 uid: ${loki_datasource} 1197 direction: backward 1198 editorMode: code 1199 expr: >- 1200 {namespace=~"$namespace", cluster=~"$cluster", container="cortex-gw"} 1201 |= `msg="request timings"` | logfmt 1202 queryType: range 1203 refId: A 1204 title: Query Activity 1205 transformations: 1206 - id: extractFields 1207 options: 1208 delimiter: ',' 1209 format: json 1210 replace: true 1211 source: labels 1212 - id: organize 1213 options: 1214 excludeByName: 1215 __adaptive_logs_sampled__: true 1216 caller: true 1217 cluster: true 1218 conn_send: false 1219 container: true 1220 detected_level: true 1221 downstream: false 1222 insight: true 1223 job: true 1224 level: true 1225 msg: true 1226 name: true 1227 namespace: true 1228 path: false 1229 pod_template_hash: true 1230 service_name: true 1231 stream: true 1232 total: false 1233 includeByName: {} 1234 indexByName: 1235 __adaptive_logs_sampled__: 22 1236 auth: 7 1237 caller: 8 1238 cluster: 9 1239 conn_send: 25 1240 container: 10 1241 dashboard_id: 24 1242 detected_level: 11 1243 downstream: 26 1244 grafana_username: 5 1245 insight: 12 1246 job: 13 1247 level: 14 1248 msg: 15 1249 name: 16 1250 namespace: 17 1251 panel_id: 27 1252 path: 1 1253 pod: 18 1254 pod_template_hash: 19 1255 query_hash: 23 1256 service_name: 20 1257 status: 2 1258 stream: 21 1259 total: 6 1260 traceID: 3 1261 ts: 0 1262 user: 4 1263 renameByName: 1264 auth: '' 1265 caller: '' 1266 path: '' 1267 total: '' 1268 user: tenant_id 1269 - id: convertFieldType 1270 options: 1271 conversions: [] 1272 fields: {} 1273 type: table 1274 - collapsed: true 1275 gridPos: 1276 h: 1 1277 w: 24 1278 x: 0 1279 'y': 40 1280 id: 52 1281 panels: 1282 - datasource: 1283 type: prometheus 1284 uid: ${datasource} 1285 fieldConfig: 1286 defaults: 1287 color: 1288 mode: palette-classic 1289 custom: 1290 axisBorderShow: false 1291 axisCenteredZero: false 1292 axisColorMode: text 1293 axisLabel: '' 1294 axisPlacement: auto 1295 barAlignment: 0 1296 drawStyle: line 1297 fillOpacity: 21 1298 gradientMode: none 1299 hideFrom: 1300 legend: false 1301 tooltip: false 1302 viz: false 1303 insertNulls: false 1304 lineInterpolation: linear 1305 lineStyle: 1306 fill: solid 1307 lineWidth: 1 1308 pointSize: 5 1309 scaleDistribution: 1310 type: linear 1311 showPoints: auto 1312 spanNulls: false 1313 stacking: 1314 group: A 1315 mode: normal 1316 thresholdsStyle: 1317 mode: dashed 1318 mappings: [] 1319 thresholds: 1320 mode: absolute 1321 steps: 1322 - color: green 1323 - color: yellow 1324 value: 80 1325 - color: red 1326 value: 90 1327 overrides: 1328 - matcher: 1329 id: byName 1330 options: request 1331 properties: 1332 - id: custom.stacking 1333 value: 1334 group: A 1335 mode: none 1336 - id: color 1337 value: 1338 fixedColor: yellow 1339 mode: fixed 1340 - id: custom.fillOpacity 1341 value: 0 1342 - id: custom.lineStyle 1343 value: 1344 dash: 1345 - 10 1346 - 10 1347 fill: dash 1348 - id: custom.lineWidth 1349 value: 2 1350 gridPos: 1351 h: 8 1352 w: 12 1353 x: 0 1354 'y': 2 1355 id: 53 1356 options: 1357 legend: 1358 calcs: [] 1359 displayMode: list 1360 placement: bottom 1361 showLegend: true 1362 tooltip: 1363 mode: multi 1364 sort: desc 1365 targets: 1366 - datasource: 1367 type: prometheus 1368 uid: ${datasource} 1369 editorMode: code 1370 expr: |- 1371 sum( 1372 node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~"$cluster", namespace=~"$namespace"} 1373 * on(namespace,pod) 1374 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*ingester", workload_type="statefulset"} 1375 ) by (pod) 1376 legendFormat: __auto 1377 range: true 1378 refId: by pod 1379 - datasource: 1380 type: prometheus 1381 uid: ${datasource} 1382 editorMode: code 1383 exemplar: false 1384 expr: |- 1385 sum( 1386 kube_pod_container_resource_requests{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="cpu"} 1387 * on(namespace,pod) 1388 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*ingester", workload_type="statefulset"} 1389 ) 1390 hide: false 1391 instant: false 1392 legendFormat: request 1393 range: true 1394 refId: request 1395 - datasource: 1396 type: prometheus 1397 uid: ${datasource} 1398 editorMode: code 1399 expr: |- 1400 sum( 1401 kube_pod_container_resource_limits{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="cpu"} 1402 * on(namespace,pod) 1403 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*ingester", workload_type="statefulset"} 1404 ) 1405 hide: false 1406 legendFormat: limit 1407 range: true 1408 refId: limit 1409 title: CPU Usage 1410 type: timeseries 1411 - datasource: 1412 type: prometheus 1413 uid: ${datasource} 1414 fieldConfig: 1415 defaults: 1416 color: 1417 mode: palette-classic 1418 custom: 1419 axisBorderShow: false 1420 axisCenteredZero: false 1421 axisColorMode: text 1422 axisLabel: '' 1423 axisPlacement: auto 1424 barAlignment: 0 1425 drawStyle: line 1426 fillOpacity: 21 1427 gradientMode: none 1428 hideFrom: 1429 legend: false 1430 tooltip: false 1431 viz: false 1432 insertNulls: false 1433 lineInterpolation: linear 1434 lineStyle: 1435 fill: solid 1436 lineWidth: 1 1437 pointSize: 5 1438 scaleDistribution: 1439 type: linear 1440 showPoints: auto 1441 spanNulls: false 1442 stacking: 1443 group: A 1444 mode: normal 1445 thresholdsStyle: 1446 mode: dashed 1447 mappings: [] 1448 thresholds: 1449 mode: absolute 1450 steps: 1451 - color: green 1452 - color: yellow 1453 value: 80 1454 - color: red 1455 value: 90 1456 unit: bytes 1457 overrides: 1458 - matcher: 1459 id: byName 1460 options: request 1461 properties: 1462 - id: custom.stacking 1463 value: 1464 group: A 1465 mode: none 1466 - id: color 1467 value: 1468 fixedColor: yellow 1469 mode: fixed 1470 - id: custom.fillOpacity 1471 value: 0 1472 - id: custom.lineStyle 1473 value: 1474 dash: 1475 - 10 1476 - 10 1477 fill: dash 1478 - id: custom.lineWidth 1479 value: 2 1480 - matcher: 1481 id: byName 1482 options: limit 1483 properties: 1484 - id: color 1485 value: 1486 fixedColor: red 1487 mode: fixed 1488 - id: custom.stacking 1489 value: 1490 group: A 1491 mode: none 1492 - id: custom.fillOpacity 1493 value: 0 1494 - id: custom.lineStyle 1495 value: 1496 dash: 1497 - 10 1498 - 10 1499 fill: dash 1500 - id: custom.lineWidth 1501 value: 2 1502 gridPos: 1503 h: 8 1504 w: 12 1505 x: 12 1506 'y': 2 1507 id: 54 1508 options: 1509 legend: 1510 calcs: [] 1511 displayMode: list 1512 placement: bottom 1513 showLegend: true 1514 tooltip: 1515 mode: multi 1516 sort: desc 1517 targets: 1518 - datasource: 1519 type: prometheus 1520 uid: ${datasource} 1521 editorMode: code 1522 expr: | 1523 sum( 1524 container_memory_working_set_bytes{cluster=~"$cluster", namespace=~"$namespace", container!="", image!=""} 1525 * on(namespace,pod) 1526 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*ingester", workload_type="statefulset"} 1527 ) by (pod) 1528 legendFormat: __auto 1529 range: true 1530 refId: by pod 1531 - datasource: 1532 type: prometheus 1533 uid: ${datasource} 1534 editorMode: code 1535 exemplar: false 1536 expr: |- 1537 sum( 1538 kube_pod_container_resource_requests{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="memory"} 1539 * on(namespace,pod) 1540 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*ingester", workload_type="statefulset"} 1541 ) 1542 hide: false 1543 instant: false 1544 legendFormat: request 1545 range: true 1546 refId: request 1547 - datasource: 1548 type: prometheus 1549 uid: ${datasource} 1550 editorMode: code 1551 expr: |- 1552 sum( 1553 kube_pod_container_resource_limits{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="memory"} 1554 * on(namespace,pod) 1555 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*ingester", workload_type="statefulset"} 1556 ) 1557 hide: false 1558 legendFormat: limit 1559 range: true 1560 refId: limit 1561 title: Memory Usage 1562 type: timeseries 1563 - datasource: 1564 type: prometheus 1565 uid: ${datasource} 1566 fieldConfig: 1567 defaults: 1568 color: 1569 mode: palette-classic 1570 custom: 1571 axisBorderShow: false 1572 axisCenteredZero: false 1573 axisColorMode: text 1574 axisLabel: '' 1575 axisPlacement: auto 1576 barAlignment: 0 1577 drawStyle: line 1578 fillOpacity: 13 1579 gradientMode: none 1580 hideFrom: 1581 legend: false 1582 tooltip: false 1583 viz: false 1584 insertNulls: false 1585 lineInterpolation: linear 1586 lineWidth: 1 1587 pointSize: 5 1588 scaleDistribution: 1589 type: linear 1590 showPoints: auto 1591 spanNulls: false 1592 stacking: 1593 group: A 1594 mode: normal 1595 thresholdsStyle: 1596 mode: 'off' 1597 mappings: [] 1598 thresholds: 1599 mode: absolute 1600 steps: 1601 - color: green 1602 - color: red 1603 value: 80 1604 unit: Bps 1605 overrides: [] 1606 gridPos: 1607 h: 8 1608 w: 12 1609 x: 0 1610 'y': 10 1611 id: 73 1612 options: 1613 legend: 1614 calcs: [] 1615 displayMode: list 1616 placement: bottom 1617 showLegend: false 1618 tooltip: 1619 mode: multi 1620 sort: none 1621 targets: 1622 - datasource: 1623 type: prometheus 1624 uid: ${datasource} 1625 editorMode: code 1626 expr: >- 1627 (sum(irate(container_network_receive_bytes_total{{ "{" }}{{ .Values.dashboards.cadvisorSelector }}, 1628 cluster=~"$cluster", namespace=~"$namespace"}[$__rate_interval]) 1629 1630 * on (namespace,pod) 1631 1632 group_left(workload,workload_type) 1633 namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", 1634 namespace=~"$namespace", workload=~".*ingester.*", 1635 workload_type="statefulset"}) by (pod)) 1636 legendFormat: __auto 1637 range: true 1638 refId: A 1639 title: Receive Bandwidth 1640 type: timeseries 1641 - datasource: 1642 type: prometheus 1643 uid: ${datasource} 1644 fieldConfig: 1645 defaults: 1646 color: 1647 mode: palette-classic 1648 custom: 1649 axisBorderShow: false 1650 axisCenteredZero: false 1651 axisColorMode: text 1652 axisLabel: '' 1653 axisPlacement: auto 1654 barAlignment: 0 1655 drawStyle: line 1656 fillOpacity: 13 1657 gradientMode: none 1658 hideFrom: 1659 legend: false 1660 tooltip: false 1661 viz: false 1662 insertNulls: false 1663 lineInterpolation: linear 1664 lineWidth: 1 1665 pointSize: 5 1666 scaleDistribution: 1667 type: linear 1668 showPoints: auto 1669 spanNulls: false 1670 stacking: 1671 group: A 1672 mode: normal 1673 thresholdsStyle: 1674 mode: 'off' 1675 mappings: [] 1676 thresholds: 1677 mode: absolute 1678 steps: 1679 - color: green 1680 - color: red 1681 value: 80 1682 unit: Bps 1683 overrides: [] 1684 gridPos: 1685 h: 8 1686 w: 12 1687 x: 12 1688 'y': 10 1689 id: 74 1690 options: 1691 legend: 1692 calcs: [] 1693 displayMode: list 1694 placement: bottom 1695 showLegend: false 1696 tooltip: 1697 mode: multi 1698 sort: none 1699 targets: 1700 - datasource: 1701 type: prometheus 1702 uid: ${datasource} 1703 editorMode: code 1704 expr: >- 1705 (sum(irate(container_network_transmit_bytes_total{{ "{" }}{{ .Values.dashboards.cadvisorSelector }}, 1706 cluster=~"$cluster", namespace=~"$namespace"}[$__rate_interval]) 1707 1708 * on (namespace,pod) 1709 1710 group_left(workload,workload_type) 1711 namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", 1712 namespace=~"$namespace", workload=~".*ingester.*", 1713 workload_type="statefulset"}) by (pod)) 1714 legendFormat: __auto 1715 range: true 1716 refId: A 1717 title: Transmit Bandwidth 1718 type: timeseries 1719 - datasource: 1720 type: prometheus 1721 uid: ${datasource} 1722 fieldConfig: 1723 defaults: 1724 color: 1725 mode: palette-classic 1726 custom: 1727 axisBorderShow: false 1728 axisCenteredZero: false 1729 axisColorMode: text 1730 axisLabel: '' 1731 axisPlacement: auto 1732 barAlignment: 0 1733 drawStyle: line 1734 fillOpacity: 0 1735 gradientMode: none 1736 hideFrom: 1737 legend: false 1738 tooltip: false 1739 viz: false 1740 insertNulls: false 1741 lineInterpolation: linear 1742 lineWidth: 1 1743 pointSize: 5 1744 scaleDistribution: 1745 type: linear 1746 showPoints: auto 1747 spanNulls: false 1748 stacking: 1749 group: A 1750 mode: none 1751 thresholdsStyle: 1752 mode: 'off' 1753 mappings: [] 1754 thresholds: 1755 mode: absolute 1756 steps: 1757 - color: green 1758 - color: red 1759 value: 80 1760 unit: s 1761 overrides: [] 1762 gridPos: 1763 h: 5 1764 w: 12 1765 x: 0 1766 'y': 18 1767 id: 96 1768 options: 1769 legend: 1770 calcs: [] 1771 displayMode: table 1772 placement: right 1773 showLegend: true 1774 tooltip: 1775 mode: single 1776 sort: none 1777 targets: 1778 - datasource: 1779 type: prometheus 1780 uid: ${datasource} 1781 editorMode: code 1782 expr: >- 1783 histogram_quantile(0.99, sum by (route) ( 1784 rate(pyroscope_request_duration_seconds{job=~"$namespace/ingester", 1785 cluster=~"$cluster",route=~".*push.*"}[$__rate_interval]))) 1786 legendFormat: .99 {{"{{route}}"}} 1787 range: true 1788 refId: A 1789 - datasource: 1790 type: prometheus 1791 uid: ${datasource} 1792 editorMode: code 1793 expr: >- 1794 histogram_quantile(0.75, sum by (route) ( 1795 rate(pyroscope_request_duration_seconds{job=~"$namespace/ingester", 1796 cluster=~"$cluster",route=~".*push.*"}[$__rate_interval]))) 1797 hide: false 1798 legendFormat: .75 {{"{{route}}"}} 1799 range: true 1800 refId: B 1801 - datasource: 1802 type: prometheus 1803 uid: ${datasource} 1804 editorMode: code 1805 expr: >- 1806 histogram_quantile(0.5, sum by (route) ( 1807 rate(pyroscope_request_duration_seconds{job=~"$namespace/ingester", 1808 cluster=~"$cluster",route=~".*push.*"}[$__rate_interval]))) 1809 hide: false 1810 legendFormat: .5 {{"{{route}}"}} 1811 range: true 1812 refId: C 1813 title: Push Latencies 1814 type: timeseries 1815 - datasource: 1816 type: prometheus 1817 uid: $datasource 1818 fieldConfig: 1819 defaults: 1820 color: 1821 mode: palette-classic 1822 custom: 1823 axisBorderShow: false 1824 axisCenteredZero: false 1825 axisColorMode: text 1826 axisLabel: '' 1827 axisPlacement: auto 1828 barAlignment: 0 1829 drawStyle: line 1830 fillOpacity: 40 1831 gradientMode: none 1832 hideFrom: 1833 legend: false 1834 tooltip: false 1835 viz: false 1836 insertNulls: false 1837 lineInterpolation: linear 1838 lineWidth: 1 1839 pointSize: 5 1840 scaleDistribution: 1841 type: linear 1842 showPoints: never 1843 spanNulls: false 1844 stacking: 1845 group: A 1846 mode: normal 1847 thresholdsStyle: 1848 mode: 'off' 1849 mappings: [] 1850 thresholds: 1851 mode: absolute 1852 steps: 1853 - color: green 1854 - color: red 1855 value: 80 1856 unit: reqps 1857 overrides: 1858 - matcher: 1859 id: byName 1860 options: '5xx ' 1861 properties: 1862 - id: color 1863 value: 1864 fixedColor: red 1865 mode: fixed 1866 - matcher: 1867 id: byName 1868 options: '2xx ' 1869 properties: 1870 - id: color 1871 value: 1872 fixedColor: green 1873 mode: fixed 1874 seriesBy: min 1875 - matcher: 1876 id: byName 1877 options: '4xx ' 1878 properties: 1879 - id: color 1880 value: 1881 fixedColor: yellow 1882 mode: fixed 1883 gridPos: 1884 h: 5 1885 w: 12 1886 x: 12 1887 'y': 18 1888 id: 69 1889 options: 1890 dataLinks: [] 1891 legend: 1892 calcs: [] 1893 displayMode: list 1894 placement: bottom 1895 showLegend: true 1896 tooltip: 1897 mode: multi 1898 sort: none 1899 targets: 1900 - datasource: 1901 uid: $datasource 1902 editorMode: code 1903 expr: |- 1904 sum by (status) ( 1905 label_replace( 1906 label_replace( 1907 rate(pyroscope_request_duration_seconds_count{cluster=~"$cluster", job=~"$namespace/ingester", route=~".*push.*"}[$__rate_interval]), 1908 "status", "${1}xx", "status_code", "([0-9]).."), 1909 "status", "${1}", "status_code", "([a-z]+)") 1910 ) 1911 legendFormat: '{{"{{status}}"}} ' 1912 range: true 1913 refId: A 1914 title: Pushes/Second 1915 type: timeseries 1916 - datasource: 1917 type: loki 1918 uid: ${loki_datasource} 1919 fieldConfig: 1920 defaults: 1921 color: 1922 mode: palette-classic 1923 custom: 1924 axisBorderShow: false 1925 axisCenteredZero: false 1926 axisColorMode: text 1927 axisLabel: '' 1928 axisPlacement: auto 1929 barAlignment: 0 1930 drawStyle: line 1931 fillOpacity: 0 1932 gradientMode: none 1933 hideFrom: 1934 legend: false 1935 tooltip: false 1936 viz: false 1937 insertNulls: false 1938 lineInterpolation: linear 1939 lineWidth: 1 1940 pointSize: 5 1941 scaleDistribution: 1942 type: linear 1943 showPoints: auto 1944 spanNulls: false 1945 stacking: 1946 group: A 1947 mode: none 1948 thresholdsStyle: 1949 mode: 'off' 1950 mappings: [] 1951 thresholds: 1952 mode: absolute 1953 steps: 1954 - color: green 1955 - color: red 1956 value: 80 1957 overrides: [] 1958 gridPos: 1959 h: 8 1960 w: 12 1961 x: 0 1962 'y': 23 1963 id: 83 1964 options: 1965 legend: 1966 calcs: [] 1967 displayMode: list 1968 placement: bottom 1969 showLegend: true 1970 tooltip: 1971 mode: multi 1972 sort: none 1973 targets: 1974 - datasource: 1975 type: loki 1976 uid: ${loki_datasource} 1977 editorMode: code 1978 expr: >- 1979 sum by (tenant) 1980 (count_over_time({cluster=~"$cluster",namespace=~"$namespace"} |= 1981 "upload new block" | logfmt [$__interval])) 1982 queryType: range 1983 refId: A 1984 title: Block Uploaded Per Tenant 1985 type: timeseries 1986 - datasource: 1987 type: loki 1988 uid: ${loki_datasource} 1989 fieldConfig: 1990 defaults: 1991 color: 1992 mode: palette-classic 1993 custom: 1994 axisBorderShow: false 1995 axisCenteredZero: false 1996 axisColorMode: text 1997 axisLabel: '' 1998 axisPlacement: auto 1999 barAlignment: 0 2000 drawStyle: bars 2001 fillOpacity: 15 2002 gradientMode: none 2003 hideFrom: 2004 legend: false 2005 tooltip: false 2006 viz: false 2007 insertNulls: false 2008 lineInterpolation: linear 2009 lineWidth: 1 2010 pointSize: 5 2011 scaleDistribution: 2012 type: linear 2013 showPoints: auto 2014 spanNulls: false 2015 stacking: 2016 group: A 2017 mode: normal 2018 thresholdsStyle: 2019 mode: 'off' 2020 mappings: [] 2021 thresholds: 2022 mode: absolute 2023 steps: 2024 - color: green 2025 - color: red 2026 value: 80 2027 overrides: [] 2028 gridPos: 2029 h: 8 2030 w: 12 2031 x: 12 2032 'y': 23 2033 id: 84 2034 options: 2035 legend: 2036 calcs: [] 2037 displayMode: list 2038 placement: bottom 2039 showLegend: true 2040 tooltip: 2041 mode: multi 2042 sort: none 2043 targets: 2044 - datasource: 2045 type: loki 2046 uid: ${loki_datasource} 2047 editorMode: code 2048 expr: >- 2049 sum by (tenant) 2050 (count_over_time({cluster=~"$cluster",namespace=~"$namespace"} |= 2051 "cut row group segment" | logfmt [$__interval])) 2052 hide: true 2053 queryType: range 2054 refId: A 2055 - datasource: 2056 type: loki 2057 uid: ${loki_datasource} 2058 editorMode: code 2059 expr: >- 2060 sum by (tenant) 2061 (sum_over_time({cluster=~"$cluster",namespace=~"$namespace"} |= 2062 "cut row group segment" | logfmt | unwrap numProfiles 2063 [$__interval])) 2064 hide: false 2065 queryType: range 2066 refId: B 2067 title: Profiles Flushed to disk 2068 type: timeseries 2069 title: Ingesters 2070 type: row 2071 - collapsed: true 2072 gridPos: 2073 h: 1 2074 w: 24 2075 x: 0 2076 'y': 41 2077 id: 41 2078 panels: 2079 - datasource: 2080 type: prometheus 2081 uid: ${datasource} 2082 fieldConfig: 2083 defaults: 2084 color: 2085 mode: palette-classic 2086 custom: 2087 axisCenteredZero: false 2088 axisColorMode: text 2089 axisLabel: '' 2090 axisPlacement: auto 2091 barAlignment: 0 2092 drawStyle: line 2093 fillOpacity: 0 2094 gradientMode: none 2095 hideFrom: 2096 legend: false 2097 tooltip: false 2098 viz: false 2099 lineInterpolation: linear 2100 lineWidth: 1 2101 pointSize: 5 2102 scaleDistribution: 2103 type: linear 2104 showPoints: auto 2105 spanNulls: false 2106 stacking: 2107 group: A 2108 mode: none 2109 thresholdsStyle: 2110 mode: 'off' 2111 mappings: [] 2112 thresholds: 2113 mode: absolute 2114 steps: 2115 - color: green 2116 - color: red 2117 value: 80 2118 overrides: [] 2119 gridPos: 2120 h: 8 2121 w: 12 2122 x: 0 2123 'y': 50 2124 id: 43 2125 options: 2126 legend: 2127 calcs: [] 2128 displayMode: list 2129 placement: bottom 2130 showLegend: true 2131 tooltip: 2132 mode: single 2133 sort: none 2134 targets: 2135 - datasource: 2136 type: prometheus 2137 uid: ${datasource} 2138 editorMode: code 2139 expr: >- 2140 topk(10,sum by (tenant, reason) 2141 (rate(pyroscope_discarded_samples_total{cluster=~"$cluster",namespace=~"$namespace"}[1m]))) 2142 legendFormat: '{{"{{ tenant }} - {{ reason }}"}}' 2143 range: true 2144 refId: A 2145 title: Discarded Profiles 2146 type: timeseries 2147 - datasource: 2148 type: prometheus 2149 uid: ${datasource} 2150 fieldConfig: 2151 defaults: 2152 color: 2153 mode: thresholds 2154 custom: 2155 align: auto 2156 cellOptions: 2157 type: auto 2158 inspect: false 2159 mappings: [] 2160 thresholds: 2161 mode: absolute 2162 steps: 2163 - color: green 2164 - color: red 2165 value: 80 2166 overrides: [] 2167 gridPos: 2168 h: 8 2169 w: 12 2170 x: 12 2171 'y': 50 2172 id: 45 2173 options: 2174 cellHeight: sm 2175 footer: 2176 countRows: false 2177 fields: '' 2178 reducer: 2179 - sum 2180 show: false 2181 showHeader: true 2182 showRowNums: false 2183 pluginVersion: 9.5.0-53857pre 2184 targets: 2185 - datasource: 2186 type: prometheus 2187 uid: ${datasource} 2188 editorMode: code 2189 exemplar: false 2190 expr: >- 2191 topk(10, sum by (tenant, reason) 2192 (sum_over_time(increase(pyroscope_discarded_samples_total{cluster=~"$cluster",namespace=~"$namespace"}[1m])[$__range:1m]))) 2193 format: table 2194 instant: true 2195 legendFormat: '{{"{{ tenant }} - {{ reason }}"}}' 2196 range: false 2197 refId: A 2198 title: Discarded Profiles Per Interval 2199 transformations: 2200 - id: organize 2201 options: 2202 excludeByName: 2203 Time: true 2204 indexByName: 2205 Time: 0 2206 Value: 3 2207 reason: 2 2208 tenant: 1 2209 renameByName: 2210 Value: Profiles 2211 reason: Reason 2212 tenant: Tenant 2213 type: table 2214 title: Tenant Limits 2215 type: row 2216 - collapsed: true 2217 gridPos: 2218 h: 1 2219 w: 24 2220 x: 0 2221 'y': 42 2222 id: 47 2223 panels: 2224 - datasource: 2225 type: prometheus 2226 uid: ${datasource} 2227 fieldConfig: 2228 defaults: 2229 color: 2230 mode: palette-classic 2231 custom: 2232 axisBorderShow: false 2233 axisCenteredZero: false 2234 axisColorMode: text 2235 axisLabel: '' 2236 axisPlacement: auto 2237 barAlignment: 0 2238 drawStyle: line 2239 fillOpacity: 21 2240 gradientMode: none 2241 hideFrom: 2242 legend: false 2243 tooltip: false 2244 viz: false 2245 insertNulls: false 2246 lineInterpolation: linear 2247 lineStyle: 2248 fill: solid 2249 lineWidth: 1 2250 pointSize: 5 2251 scaleDistribution: 2252 type: linear 2253 showPoints: auto 2254 spanNulls: false 2255 stacking: 2256 group: A 2257 mode: normal 2258 thresholdsStyle: 2259 mode: dashed 2260 mappings: [] 2261 thresholds: 2262 mode: absolute 2263 steps: 2264 - color: green 2265 - color: yellow 2266 value: 80 2267 - color: red 2268 value: 90 2269 overrides: 2270 - matcher: 2271 id: byName 2272 options: request 2273 properties: 2274 - id: custom.stacking 2275 value: 2276 group: A 2277 mode: none 2278 - id: color 2279 value: 2280 fixedColor: yellow 2281 mode: fixed 2282 - id: custom.fillOpacity 2283 value: 0 2284 - id: custom.lineStyle 2285 value: 2286 dash: 2287 - 10 2288 - 10 2289 fill: dash 2290 - id: custom.lineWidth 2291 value: 2 2292 gridPos: 2293 h: 8 2294 w: 12 2295 x: 0 2296 'y': 4 2297 id: 49 2298 options: 2299 legend: 2300 calcs: [] 2301 displayMode: list 2302 placement: bottom 2303 showLegend: true 2304 tooltip: 2305 mode: multi 2306 sort: desc 2307 targets: 2308 - datasource: 2309 type: prometheus 2310 uid: ${datasource} 2311 editorMode: code 2312 expr: |- 2313 sum( 2314 node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~"$cluster", namespace=~"$namespace"} 2315 * on(namespace,pod) 2316 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*distributor", workload_type="deployment"} 2317 ) by (pod) 2318 legendFormat: __auto 2319 range: true 2320 refId: by pod 2321 - datasource: 2322 type: prometheus 2323 uid: ${datasource} 2324 editorMode: code 2325 exemplar: false 2326 expr: |- 2327 sum( 2328 kube_pod_container_resource_requests{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="cpu"} 2329 * on(namespace,pod) 2330 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*distributor", workload_type="deployment"} 2331 ) 2332 hide: false 2333 instant: false 2334 legendFormat: request 2335 range: true 2336 refId: request 2337 - datasource: 2338 type: prometheus 2339 uid: ${datasource} 2340 editorMode: code 2341 expr: |- 2342 sum( 2343 kube_pod_container_resource_limits{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="cpu"} 2344 * on(namespace,pod) 2345 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*distributor", workload_type="deployment"} 2346 ) 2347 hide: false 2348 legendFormat: limit 2349 range: true 2350 refId: limit 2351 title: CPU Usage 2352 type: timeseries 2353 - datasource: 2354 type: prometheus 2355 uid: ${datasource} 2356 fieldConfig: 2357 defaults: 2358 color: 2359 mode: palette-classic 2360 custom: 2361 axisBorderShow: false 2362 axisCenteredZero: false 2363 axisColorMode: text 2364 axisLabel: '' 2365 axisPlacement: auto 2366 barAlignment: 0 2367 drawStyle: line 2368 fillOpacity: 21 2369 gradientMode: none 2370 hideFrom: 2371 legend: false 2372 tooltip: false 2373 viz: false 2374 insertNulls: false 2375 lineInterpolation: linear 2376 lineStyle: 2377 fill: solid 2378 lineWidth: 1 2379 pointSize: 5 2380 scaleDistribution: 2381 type: linear 2382 showPoints: auto 2383 spanNulls: false 2384 stacking: 2385 group: A 2386 mode: normal 2387 thresholdsStyle: 2388 mode: dashed 2389 mappings: [] 2390 thresholds: 2391 mode: absolute 2392 steps: 2393 - color: green 2394 - color: yellow 2395 value: 80 2396 - color: red 2397 value: 90 2398 unit: bytes 2399 overrides: 2400 - matcher: 2401 id: byName 2402 options: request 2403 properties: 2404 - id: custom.stacking 2405 value: 2406 group: A 2407 mode: none 2408 - id: color 2409 value: 2410 fixedColor: yellow 2411 mode: fixed 2412 - id: custom.fillOpacity 2413 value: 0 2414 - id: custom.lineStyle 2415 value: 2416 dash: 2417 - 10 2418 - 10 2419 fill: dash 2420 - id: custom.lineWidth 2421 value: 2 2422 - matcher: 2423 id: byName 2424 options: limit 2425 properties: 2426 - id: color 2427 value: 2428 fixedColor: red 2429 mode: fixed 2430 - id: custom.stacking 2431 value: 2432 group: A 2433 mode: none 2434 - id: custom.fillOpacity 2435 value: 0 2436 - id: custom.lineStyle 2437 value: 2438 dash: 2439 - 10 2440 - 10 2441 fill: dash 2442 - id: custom.lineWidth 2443 value: 2 2444 gridPos: 2445 h: 8 2446 w: 12 2447 x: 12 2448 'y': 4 2449 id: 50 2450 options: 2451 legend: 2452 calcs: [] 2453 displayMode: list 2454 placement: bottom 2455 showLegend: true 2456 tooltip: 2457 mode: multi 2458 sort: desc 2459 targets: 2460 - datasource: 2461 type: prometheus 2462 uid: ${datasource} 2463 editorMode: code 2464 expr: | 2465 sum( 2466 container_memory_working_set_bytes{cluster=~"$cluster", namespace=~"$namespace", container!="", image!=""} 2467 * on(namespace,pod) 2468 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*distributor", workload_type="deployment"} 2469 ) by (pod) 2470 legendFormat: __auto 2471 range: true 2472 refId: by pod 2473 - datasource: 2474 type: prometheus 2475 uid: ${datasource} 2476 editorMode: code 2477 exemplar: false 2478 expr: |- 2479 sum( 2480 kube_pod_container_resource_requests{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="memory"} 2481 * on(namespace,pod) 2482 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*distributor", workload_type="deployment"} 2483 ) 2484 hide: false 2485 instant: false 2486 legendFormat: request 2487 range: true 2488 refId: request 2489 - datasource: 2490 type: prometheus 2491 uid: ${datasource} 2492 editorMode: code 2493 expr: |- 2494 sum( 2495 kube_pod_container_resource_limits{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="memory"} 2496 * on(namespace,pod) 2497 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*distributor", workload_type="deployment"} 2498 ) 2499 hide: false 2500 legendFormat: limit 2501 range: true 2502 refId: limit 2503 title: Memory Usage 2504 type: timeseries 2505 - datasource: 2506 type: prometheus 2507 uid: ${datasource} 2508 fieldConfig: 2509 defaults: 2510 color: 2511 mode: palette-classic 2512 custom: 2513 axisBorderShow: false 2514 axisCenteredZero: false 2515 axisColorMode: text 2516 axisLabel: '' 2517 axisPlacement: auto 2518 barAlignment: 0 2519 drawStyle: line 2520 fillOpacity: 13 2521 gradientMode: none 2522 hideFrom: 2523 legend: false 2524 tooltip: false 2525 viz: false 2526 insertNulls: false 2527 lineInterpolation: linear 2528 lineWidth: 1 2529 pointSize: 5 2530 scaleDistribution: 2531 type: linear 2532 showPoints: auto 2533 spanNulls: false 2534 stacking: 2535 group: A 2536 mode: normal 2537 thresholdsStyle: 2538 mode: 'off' 2539 mappings: [] 2540 thresholds: 2541 mode: absolute 2542 steps: 2543 - color: green 2544 - color: red 2545 value: 80 2546 unit: Bps 2547 overrides: [] 2548 gridPos: 2549 h: 8 2550 w: 12 2551 x: 0 2552 'y': 12 2553 id: 71 2554 options: 2555 legend: 2556 calcs: [] 2557 displayMode: list 2558 placement: bottom 2559 showLegend: false 2560 tooltip: 2561 mode: multi 2562 sort: none 2563 targets: 2564 - datasource: 2565 type: prometheus 2566 uid: ${datasource} 2567 editorMode: code 2568 expr: >- 2569 (sum(irate(container_network_receive_bytes_total{{ "{" }}{{ .Values.dashboards.cadvisorSelector }}, 2570 cluster=~"$cluster", namespace=~"$namespace"}[$__rate_interval]) 2571 2572 * on (namespace,pod) 2573 2574 group_left(workload,workload_type) 2575 namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", 2576 namespace=~"$namespace", workload=~".*distributor.*", 2577 workload_type="deployment"}) by (pod)) 2578 legendFormat: __auto 2579 range: true 2580 refId: A 2581 title: Receive Bandwidth 2582 type: timeseries 2583 - datasource: 2584 type: prometheus 2585 uid: ${datasource} 2586 fieldConfig: 2587 defaults: 2588 color: 2589 mode: palette-classic 2590 custom: 2591 axisBorderShow: false 2592 axisCenteredZero: false 2593 axisColorMode: text 2594 axisLabel: '' 2595 axisPlacement: auto 2596 barAlignment: 0 2597 drawStyle: line 2598 fillOpacity: 13 2599 gradientMode: none 2600 hideFrom: 2601 legend: false 2602 tooltip: false 2603 viz: false 2604 insertNulls: false 2605 lineInterpolation: linear 2606 lineWidth: 1 2607 pointSize: 5 2608 scaleDistribution: 2609 type: linear 2610 showPoints: auto 2611 spanNulls: false 2612 stacking: 2613 group: A 2614 mode: normal 2615 thresholdsStyle: 2616 mode: 'off' 2617 mappings: [] 2618 thresholds: 2619 mode: absolute 2620 steps: 2621 - color: green 2622 - color: red 2623 value: 80 2624 unit: Bps 2625 overrides: [] 2626 gridPos: 2627 h: 8 2628 w: 12 2629 x: 12 2630 'y': 12 2631 id: 72 2632 options: 2633 legend: 2634 calcs: [] 2635 displayMode: list 2636 placement: bottom 2637 showLegend: false 2638 tooltip: 2639 mode: multi 2640 sort: none 2641 targets: 2642 - datasource: 2643 type: prometheus 2644 uid: ${datasource} 2645 editorMode: code 2646 expr: >- 2647 (sum(irate(container_network_transmit_bytes_total{{ "{" }}{{ .Values.dashboards.cadvisorSelector }}, 2648 cluster=~"$cluster", namespace=~"$namespace"}[$__rate_interval]) 2649 2650 * on (namespace,pod) 2651 2652 group_left(workload,workload_type) 2653 namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", 2654 namespace=~"$namespace", workload=~".*distributor.*", 2655 workload_type="deployment"}) by (pod)) 2656 legendFormat: __auto 2657 range: true 2658 refId: A 2659 title: Transmit Bandwidth 2660 type: timeseries 2661 - datasource: 2662 type: prometheus 2663 uid: ${datasource} 2664 fieldConfig: 2665 defaults: 2666 color: 2667 mode: palette-classic 2668 custom: 2669 axisBorderShow: false 2670 axisCenteredZero: false 2671 axisColorMode: text 2672 axisLabel: '' 2673 axisPlacement: auto 2674 barAlignment: 0 2675 drawStyle: line 2676 fillOpacity: 16 2677 gradientMode: none 2678 hideFrom: 2679 legend: false 2680 tooltip: false 2681 viz: false 2682 insertNulls: false 2683 lineInterpolation: linear 2684 lineWidth: 1 2685 pointSize: 5 2686 scaleDistribution: 2687 type: linear 2688 showPoints: auto 2689 spanNulls: false 2690 stacking: 2691 group: A 2692 mode: normal 2693 thresholdsStyle: 2694 mode: 'off' 2695 mappings: [] 2696 thresholds: 2697 mode: absolute 2698 steps: 2699 - color: green 2700 - color: red 2701 value: 80 2702 overrides: [] 2703 gridPos: 2704 h: 8 2705 w: 12 2706 x: 0 2707 'y': 20 2708 id: 63 2709 options: 2710 legend: 2711 calcs: 2712 - lastNotNull 2713 displayMode: table 2714 placement: right 2715 showLegend: true 2716 sortBy: Last * 2717 sortDesc: true 2718 tooltip: 2719 mode: single 2720 sort: none 2721 targets: 2722 - datasource: 2723 type: prometheus 2724 uid: ${datasource} 2725 editorMode: code 2726 expr: >- 2727 sum by (type) 2728 (rate(pyroscope_distributor_received_compressed_bytes_count{cluster=~"$cluster", 2729 namespace=~"$namespace"}[$__rate_interval])) 2730 legendFormat: __auto 2731 range: true 2732 refId: A 2733 title: Profiles /s 2734 type: timeseries 2735 - datasource: 2736 type: prometheus 2737 uid: ${datasource} 2738 fieldConfig: 2739 defaults: 2740 color: 2741 mode: palette-classic 2742 custom: 2743 axisBorderShow: false 2744 axisCenteredZero: false 2745 axisColorMode: text 2746 axisLabel: '' 2747 axisPlacement: auto 2748 barAlignment: 0 2749 drawStyle: line 2750 fillOpacity: 16 2751 gradientMode: none 2752 hideFrom: 2753 legend: false 2754 tooltip: false 2755 viz: false 2756 insertNulls: false 2757 lineInterpolation: linear 2758 lineWidth: 1 2759 pointSize: 5 2760 scaleDistribution: 2761 type: linear 2762 showPoints: auto 2763 spanNulls: false 2764 stacking: 2765 group: A 2766 mode: normal 2767 thresholdsStyle: 2768 mode: 'off' 2769 mappings: [] 2770 thresholds: 2771 mode: absolute 2772 steps: 2773 - color: green 2774 - color: red 2775 value: 80 2776 overrides: [] 2777 gridPos: 2778 h: 8 2779 w: 12 2780 x: 12 2781 'y': 20 2782 id: 62 2783 options: 2784 legend: 2785 calcs: 2786 - lastNotNull 2787 displayMode: table 2788 placement: right 2789 showLegend: true 2790 sortBy: Last * 2791 sortDesc: true 2792 tooltip: 2793 mode: single 2794 sort: none 2795 targets: 2796 - datasource: 2797 type: prometheus 2798 uid: ${datasource} 2799 editorMode: code 2800 expr: >- 2801 sum by (type) 2802 (rate(pyroscope_distributor_received_samples_sum{cluster=~"$cluster", 2803 namespace=~"$namespace"}[$__rate_interval])) 2804 legendFormat: __auto 2805 range: true 2806 refId: A 2807 title: Stacktrace Sample /s 2808 type: timeseries 2809 - datasource: 2810 type: prometheus 2811 uid: ${datasource} 2812 fieldConfig: 2813 defaults: 2814 color: 2815 mode: palette-classic 2816 custom: 2817 axisBorderShow: false 2818 axisCenteredZero: false 2819 axisColorMode: text 2820 axisLabel: '' 2821 axisPlacement: auto 2822 barAlignment: 0 2823 drawStyle: line 2824 fillOpacity: 16 2825 gradientMode: none 2826 hideFrom: 2827 legend: false 2828 tooltip: false 2829 viz: false 2830 insertNulls: false 2831 lineInterpolation: linear 2832 lineWidth: 1 2833 pointSize: 5 2834 scaleDistribution: 2835 type: linear 2836 showPoints: auto 2837 spanNulls: false 2838 stacking: 2839 group: A 2840 mode: normal 2841 thresholdsStyle: 2842 mode: 'off' 2843 mappings: [] 2844 thresholds: 2845 mode: absolute 2846 steps: 2847 - color: green 2848 - color: red 2849 value: 80 2850 unit: decbytes 2851 overrides: [] 2852 gridPos: 2853 h: 8 2854 w: 8 2855 x: 0 2856 'y': 28 2857 id: 65 2858 options: 2859 legend: 2860 calcs: 2861 - lastNotNull 2862 displayMode: table 2863 placement: right 2864 showLegend: false 2865 sortBy: Last * 2866 sortDesc: true 2867 tooltip: 2868 mode: multi 2869 sort: none 2870 targets: 2871 - datasource: 2872 type: prometheus 2873 uid: ${datasource} 2874 editorMode: code 2875 expr: >- 2876 sum by (type) 2877 (rate(pyroscope_distributor_received_decompressed_bytes_sum{cluster=~"$cluster", 2878 namespace=~"$namespace"}[$__rate_interval])) 2879 legendFormat: __auto 2880 range: true 2881 refId: A 2882 title: Throughput Decompressed 2883 type: timeseries 2884 - datasource: 2885 type: prometheus 2886 uid: ${datasource} 2887 fieldConfig: 2888 defaults: 2889 color: 2890 mode: palette-classic 2891 custom: 2892 axisBorderShow: false 2893 axisCenteredZero: false 2894 axisColorMode: text 2895 axisLabel: '' 2896 axisPlacement: auto 2897 barAlignment: 0 2898 drawStyle: line 2899 fillOpacity: 16 2900 gradientMode: none 2901 hideFrom: 2902 legend: false 2903 tooltip: false 2904 viz: false 2905 insertNulls: false 2906 lineInterpolation: linear 2907 lineWidth: 1 2908 pointSize: 5 2909 scaleDistribution: 2910 type: linear 2911 showPoints: auto 2912 spanNulls: false 2913 stacking: 2914 group: A 2915 mode: normal 2916 thresholdsStyle: 2917 mode: 'off' 2918 mappings: [] 2919 thresholds: 2920 mode: absolute 2921 steps: 2922 - color: green 2923 - color: red 2924 value: 80 2925 unit: decbytes 2926 overrides: [] 2927 gridPos: 2928 h: 8 2929 w: 8 2930 x: 8 2931 'y': 28 2932 id: 66 2933 options: 2934 legend: 2935 calcs: 2936 - lastNotNull 2937 displayMode: table 2938 placement: right 2939 showLegend: false 2940 sortBy: Last * 2941 sortDesc: true 2942 tooltip: 2943 mode: multi 2944 sort: none 2945 targets: 2946 - datasource: 2947 type: prometheus 2948 uid: ${datasource} 2949 editorMode: code 2950 expr: >- 2951 sum by (type) 2952 (rate(pyroscope_distributor_received_samples_bytes_sum{cluster=~"$cluster", 2953 namespace=~"$namespace"}[$__rate_interval])) 2954 legendFormat: __auto 2955 range: true 2956 refId: A 2957 title: Throughput Samples Decompressed 2958 type: timeseries 2959 - datasource: 2960 type: prometheus 2961 uid: ${datasource} 2962 fieldConfig: 2963 defaults: 2964 color: 2965 mode: palette-classic 2966 custom: 2967 axisBorderShow: false 2968 axisCenteredZero: false 2969 axisColorMode: text 2970 axisLabel: '' 2971 axisPlacement: auto 2972 barAlignment: 0 2973 drawStyle: line 2974 fillOpacity: 16 2975 gradientMode: none 2976 hideFrom: 2977 legend: false 2978 tooltip: false 2979 viz: false 2980 insertNulls: false 2981 lineInterpolation: linear 2982 lineWidth: 1 2983 pointSize: 5 2984 scaleDistribution: 2985 type: linear 2986 showPoints: auto 2987 spanNulls: false 2988 stacking: 2989 group: A 2990 mode: normal 2991 thresholdsStyle: 2992 mode: 'off' 2993 mappings: [] 2994 thresholds: 2995 mode: absolute 2996 steps: 2997 - color: green 2998 - color: red 2999 value: 80 3000 unit: decbytes 3001 overrides: [] 3002 gridPos: 3003 h: 8 3004 w: 8 3005 x: 16 3006 'y': 28 3007 id: 67 3008 options: 3009 legend: 3010 calcs: 3011 - lastNotNull 3012 displayMode: table 3013 placement: right 3014 showLegend: false 3015 sortBy: Last * 3016 sortDesc: true 3017 tooltip: 3018 mode: multi 3019 sort: none 3020 targets: 3021 - datasource: 3022 type: prometheus 3023 uid: ${datasource} 3024 editorMode: code 3025 expr: >- 3026 sum by (type) 3027 (rate(pyroscope_distributor_received_symbols_bytes_sum{cluster=~"$cluster", 3028 namespace=~"$namespace"}[$__rate_interval])) 3029 legendFormat: __auto 3030 range: true 3031 refId: A 3032 title: Throughput Symbols Decompressed 3033 type: timeseries 3034 title: Distributors 3035 type: row 3036 - collapsed: true 3037 gridPos: 3038 h: 1 3039 w: 24 3040 x: 0 3041 'y': 43 3042 id: 91 3043 panels: 3044 - datasource: 3045 type: prometheus 3046 uid: ${datasource} 3047 fieldConfig: 3048 defaults: 3049 color: 3050 mode: palette-classic 3051 custom: 3052 axisBorderShow: false 3053 axisCenteredZero: false 3054 axisColorMode: text 3055 axisLabel: '' 3056 axisPlacement: auto 3057 barAlignment: 0 3058 drawStyle: line 3059 fillOpacity: 0 3060 gradientMode: none 3061 hideFrom: 3062 legend: false 3063 tooltip: false 3064 viz: false 3065 insertNulls: false 3066 lineInterpolation: linear 3067 lineWidth: 1 3068 pointSize: 5 3069 scaleDistribution: 3070 type: linear 3071 showPoints: auto 3072 spanNulls: false 3073 stacking: 3074 group: A 3075 mode: none 3076 thresholdsStyle: 3077 mode: 'off' 3078 mappings: [] 3079 thresholds: 3080 mode: absolute 3081 steps: 3082 - color: green 3083 - color: red 3084 value: 80 3085 unit: s 3086 overrides: [] 3087 gridPos: 3088 h: 5 3089 w: 12 3090 x: 0 3091 'y': 5 3092 id: 100 3093 options: 3094 legend: 3095 calcs: [] 3096 displayMode: table 3097 placement: right 3098 showLegend: true 3099 tooltip: 3100 mode: single 3101 sort: none 3102 targets: 3103 - datasource: 3104 type: prometheus 3105 uid: ${datasource} 3106 editorMode: code 3107 expr: >- 3108 histogram_quantile(0.99, sum by (route) ( 3109 rate(pyroscope_request_duration_seconds{job=~"$namespace/query-frontend", 3110 cluster=~"$cluster",route=~".*render.*|.*querier.*"}[$__rate_interval]))) 3111 legendFormat: .99 {{"{{route}}"}} 3112 range: true 3113 refId: A 3114 - datasource: 3115 type: prometheus 3116 uid: ${datasource} 3117 editorMode: code 3118 expr: >- 3119 histogram_quantile(0.75, sum by (route) ( 3120 rate(pyroscope_request_duration_seconds{job=~"$namespace/query-frontend", 3121 cluster=~"$cluster",route=~".*render.*|.*querier.*"}[$__rate_interval]))) 3122 hide: true 3123 legendFormat: .75 {{"{{route}}"}} 3124 range: true 3125 refId: B 3126 - datasource: 3127 type: prometheus 3128 uid: ${datasource} 3129 editorMode: code 3130 expr: >- 3131 histogram_quantile(0.5, sum by (route) ( 3132 rate(pyroscope_request_duration_seconds{job=~"$namespace/query-frontend", 3133 cluster=~"$cluster",route=~".*render.*|.*querier.*"}[$__rate_interval]))) 3134 hide: true 3135 legendFormat: .5 {{"{{route}}"}} 3136 range: true 3137 refId: C 3138 title: API Latencies 3139 type: timeseries 3140 - datasource: 3141 type: prometheus 3142 uid: $datasource 3143 fieldConfig: 3144 defaults: 3145 color: 3146 mode: palette-classic 3147 custom: 3148 axisBorderShow: false 3149 axisCenteredZero: false 3150 axisColorMode: text 3151 axisLabel: '' 3152 axisPlacement: auto 3153 barAlignment: 0 3154 drawStyle: line 3155 fillOpacity: 40 3156 gradientMode: none 3157 hideFrom: 3158 legend: false 3159 tooltip: false 3160 viz: false 3161 insertNulls: false 3162 lineInterpolation: linear 3163 lineWidth: 1 3164 pointSize: 5 3165 scaleDistribution: 3166 type: linear 3167 showPoints: never 3168 spanNulls: false 3169 stacking: 3170 group: A 3171 mode: normal 3172 thresholdsStyle: 3173 mode: 'off' 3174 mappings: [] 3175 thresholds: 3176 mode: absolute 3177 steps: 3178 - color: green 3179 - color: red 3180 value: 80 3181 unit: reqps 3182 overrides: 3183 - matcher: 3184 id: byRegexp 3185 options: /2../ 3186 properties: 3187 - id: color 3188 value: 3189 fixedColor: green 3190 mode: shades 3191 - matcher: 3192 id: byRegexp 3193 options: /5../ 3194 properties: 3195 - id: color 3196 value: 3197 fixedColor: red 3198 mode: shades 3199 - matcher: 3200 id: byRegexp 3201 options: /4../ 3202 properties: 3203 - id: color 3204 value: 3205 fixedColor: yellow 3206 mode: shades 3207 gridPos: 3208 h: 5 3209 w: 12 3210 x: 12 3211 'y': 5 3212 id: 101 3213 options: 3214 dataLinks: [] 3215 legend: 3216 calcs: [] 3217 displayMode: list 3218 placement: bottom 3219 showLegend: true 3220 tooltip: 3221 mode: multi 3222 sort: none 3223 targets: 3224 - datasource: 3225 uid: $datasource 3226 editorMode: code 3227 expr: |- 3228 sum by (status,route) ( 3229 label_replace( 3230 label_replace( 3231 rate(pyroscope_request_duration_seconds_count{cluster=~"$cluster", job=~"$namespace/query-frontend", route=~".*render.*|.*querier.*"}[$__rate_interval]), 3232 "status", "${1}xx", "status_code", "([0-9]).."), 3233 "status", "${1}", "status_code", "([a-z]+)") 3234 ) 3235 legendFormat: '{{"{{status}} {{route}}"}}' 3236 range: true 3237 refId: A 3238 title: Request Rates 3239 type: timeseries 3240 - datasource: 3241 type: prometheus 3242 uid: ${datasource} 3243 fieldConfig: 3244 defaults: 3245 color: 3246 mode: palette-classic 3247 custom: 3248 axisBorderShow: false 3249 axisCenteredZero: false 3250 axisColorMode: text 3251 axisLabel: '' 3252 axisPlacement: auto 3253 barAlignment: 0 3254 drawStyle: line 3255 fillOpacity: 21 3256 gradientMode: none 3257 hideFrom: 3258 legend: false 3259 tooltip: false 3260 viz: false 3261 insertNulls: false 3262 lineInterpolation: linear 3263 lineStyle: 3264 fill: solid 3265 lineWidth: 1 3266 pointSize: 5 3267 scaleDistribution: 3268 type: linear 3269 showPoints: auto 3270 spanNulls: false 3271 stacking: 3272 group: A 3273 mode: normal 3274 thresholdsStyle: 3275 mode: dashed 3276 mappings: [] 3277 thresholds: 3278 mode: absolute 3279 steps: 3280 - color: green 3281 - color: yellow 3282 value: 80 3283 - color: red 3284 value: 90 3285 overrides: 3286 - matcher: 3287 id: byName 3288 options: request 3289 properties: 3290 - id: custom.stacking 3291 value: 3292 group: A 3293 mode: none 3294 - id: color 3295 value: 3296 fixedColor: yellow 3297 mode: fixed 3298 - id: custom.fillOpacity 3299 value: 0 3300 - id: custom.lineStyle 3301 value: 3302 dash: 3303 - 10 3304 - 10 3305 fill: dash 3306 - id: custom.lineWidth 3307 value: 2 3308 - matcher: 3309 id: byName 3310 options: limit 3311 properties: 3312 - id: custom.fillOpacity 3313 value: 0 3314 - id: custom.stacking 3315 value: 3316 group: A 3317 mode: none 3318 - id: custom.lineStyle 3319 value: 3320 dash: 3321 - 10 3322 - 10 3323 fill: dash 3324 - id: custom.lineWidth 3325 value: 2 3326 - id: color 3327 value: 3328 fixedColor: red 3329 mode: fixed 3330 gridPos: 3331 h: 8 3332 w: 12 3333 x: 0 3334 'y': 10 3335 id: 92 3336 options: 3337 legend: 3338 calcs: [] 3339 displayMode: list 3340 placement: bottom 3341 showLegend: true 3342 tooltip: 3343 mode: multi 3344 sort: desc 3345 targets: 3346 - datasource: 3347 type: prometheus 3348 uid: ${datasource} 3349 editorMode: code 3350 expr: |- 3351 sum( 3352 node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~"$cluster", namespace=~"$namespace"} 3353 * on(namespace,pod) 3354 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*query-frontend", workload_type="deployment"} 3355 ) by (pod) 3356 legendFormat: __auto 3357 range: true 3358 refId: by pod 3359 - datasource: 3360 type: prometheus 3361 uid: ${datasource} 3362 editorMode: code 3363 exemplar: false 3364 expr: |- 3365 sum( 3366 kube_pod_container_resource_requests{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="cpu"} 3367 * on(namespace,pod) 3368 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*query-frontend", workload_type="deployment"} 3369 ) 3370 hide: false 3371 instant: false 3372 legendFormat: request 3373 range: true 3374 refId: request 3375 - datasource: 3376 type: prometheus 3377 uid: ${datasource} 3378 editorMode: code 3379 expr: |- 3380 sum( 3381 kube_pod_container_resource_limits{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="cpu"} 3382 * on(namespace,pod) 3383 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*query-frontend", workload_type="deployment"} 3384 ) 3385 hide: false 3386 legendFormat: limit 3387 range: true 3388 refId: limit 3389 title: CPU Usage 3390 type: timeseries 3391 - datasource: 3392 type: prometheus 3393 uid: ${datasource} 3394 fieldConfig: 3395 defaults: 3396 color: 3397 mode: palette-classic 3398 custom: 3399 axisBorderShow: false 3400 axisCenteredZero: false 3401 axisColorMode: text 3402 axisLabel: '' 3403 axisPlacement: auto 3404 barAlignment: 0 3405 drawStyle: line 3406 fillOpacity: 21 3407 gradientMode: none 3408 hideFrom: 3409 legend: false 3410 tooltip: false 3411 viz: false 3412 insertNulls: false 3413 lineInterpolation: linear 3414 lineStyle: 3415 fill: solid 3416 lineWidth: 1 3417 pointSize: 5 3418 scaleDistribution: 3419 type: linear 3420 showPoints: auto 3421 spanNulls: false 3422 stacking: 3423 group: A 3424 mode: normal 3425 thresholdsStyle: 3426 mode: dashed 3427 mappings: [] 3428 thresholds: 3429 mode: absolute 3430 steps: 3431 - color: green 3432 - color: yellow 3433 value: 80 3434 - color: red 3435 value: 90 3436 unit: bytes 3437 overrides: 3438 - matcher: 3439 id: byName 3440 options: request 3441 properties: 3442 - id: custom.stacking 3443 value: 3444 group: A 3445 mode: none 3446 - id: color 3447 value: 3448 fixedColor: yellow 3449 mode: fixed 3450 - id: custom.fillOpacity 3451 value: 0 3452 - id: custom.lineStyle 3453 value: 3454 dash: 3455 - 10 3456 - 10 3457 fill: dash 3458 - id: custom.lineWidth 3459 value: 2 3460 - matcher: 3461 id: byName 3462 options: limit 3463 properties: 3464 - id: color 3465 value: 3466 fixedColor: red 3467 mode: fixed 3468 - id: custom.stacking 3469 value: 3470 group: A 3471 mode: none 3472 - id: custom.fillOpacity 3473 value: 0 3474 - id: custom.lineStyle 3475 value: 3476 dash: 3477 - 10 3478 - 10 3479 fill: dash 3480 - id: custom.lineWidth 3481 value: 2 3482 gridPos: 3483 h: 8 3484 w: 12 3485 x: 12 3486 'y': 10 3487 id: 88 3488 options: 3489 legend: 3490 calcs: [] 3491 displayMode: list 3492 placement: bottom 3493 showLegend: true 3494 tooltip: 3495 mode: multi 3496 sort: desc 3497 targets: 3498 - datasource: 3499 type: prometheus 3500 uid: ${datasource} 3501 editorMode: code 3502 expr: | 3503 sum( 3504 container_memory_working_set_bytes{cluster=~"$cluster", namespace=~"$namespace", container!="", image!=""} 3505 * on(namespace,pod) 3506 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*query-frontend", workload_type="deployment"} 3507 ) by (pod) 3508 legendFormat: __auto 3509 range: true 3510 refId: by pod 3511 - datasource: 3512 type: prometheus 3513 uid: ${datasource} 3514 editorMode: code 3515 exemplar: false 3516 expr: |- 3517 sum( 3518 kube_pod_container_resource_requests{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="memory"} 3519 * on(namespace,pod) 3520 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*query-frontend", workload_type="deployment"} 3521 ) 3522 hide: false 3523 instant: false 3524 legendFormat: request 3525 range: true 3526 refId: request 3527 - datasource: 3528 type: prometheus 3529 uid: ${datasource} 3530 editorMode: code 3531 expr: |- 3532 sum( 3533 kube_pod_container_resource_limits{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="memory"} 3534 * on(namespace,pod) 3535 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*query-frontend", workload_type="deployment"} 3536 ) 3537 hide: false 3538 legendFormat: limit 3539 range: true 3540 refId: limit 3541 title: Memory Usage 3542 type: timeseries 3543 - datasource: 3544 type: prometheus 3545 uid: ${datasource} 3546 fieldConfig: 3547 defaults: 3548 color: 3549 mode: palette-classic 3550 custom: 3551 axisBorderShow: false 3552 axisCenteredZero: false 3553 axisColorMode: text 3554 axisLabel: '' 3555 axisPlacement: auto 3556 barAlignment: 0 3557 drawStyle: line 3558 fillOpacity: 13 3559 gradientMode: none 3560 hideFrom: 3561 legend: false 3562 tooltip: false 3563 viz: false 3564 insertNulls: false 3565 lineInterpolation: linear 3566 lineWidth: 1 3567 pointSize: 5 3568 scaleDistribution: 3569 type: linear 3570 showPoints: auto 3571 spanNulls: false 3572 stacking: 3573 group: A 3574 mode: normal 3575 thresholdsStyle: 3576 mode: 'off' 3577 mappings: [] 3578 thresholds: 3579 mode: absolute 3580 steps: 3581 - color: green 3582 - color: red 3583 value: 80 3584 unit: Bps 3585 overrides: [] 3586 gridPos: 3587 h: 8 3588 w: 12 3589 x: 0 3590 'y': 18 3591 id: 94 3592 options: 3593 legend: 3594 calcs: [] 3595 displayMode: list 3596 placement: bottom 3597 showLegend: false 3598 tooltip: 3599 mode: multi 3600 sort: none 3601 targets: 3602 - datasource: 3603 type: prometheus 3604 uid: ${datasource} 3605 editorMode: code 3606 expr: >- 3607 (sum(irate(container_network_receive_bytes_total{{ "{" }}{{ .Values.dashboards.cadvisorSelector }}, 3608 cluster=~"$cluster", namespace=~"$namespace"}[$__rate_interval]) 3609 3610 * on (namespace,pod) 3611 3612 group_left(workload,workload_type) 3613 namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", 3614 namespace=~"$namespace", workload=~".*query-frontend.*", 3615 workload_type="deployment"}) by (pod)) 3616 legendFormat: __auto 3617 range: true 3618 refId: A 3619 title: Receive Bandwidth 3620 type: timeseries 3621 - datasource: 3622 type: prometheus 3623 uid: ${datasource} 3624 fieldConfig: 3625 defaults: 3626 color: 3627 mode: palette-classic 3628 custom: 3629 axisBorderShow: false 3630 axisCenteredZero: false 3631 axisColorMode: text 3632 axisLabel: '' 3633 axisPlacement: auto 3634 barAlignment: 0 3635 drawStyle: line 3636 fillOpacity: 13 3637 gradientMode: none 3638 hideFrom: 3639 legend: false 3640 tooltip: false 3641 viz: false 3642 insertNulls: false 3643 lineInterpolation: linear 3644 lineWidth: 1 3645 pointSize: 5 3646 scaleDistribution: 3647 type: linear 3648 showPoints: auto 3649 spanNulls: false 3650 stacking: 3651 group: A 3652 mode: normal 3653 thresholdsStyle: 3654 mode: 'off' 3655 mappings: [] 3656 thresholds: 3657 mode: absolute 3658 steps: 3659 - color: green 3660 - color: red 3661 value: 80 3662 unit: Bps 3663 overrides: [] 3664 gridPos: 3665 h: 8 3666 w: 12 3667 x: 12 3668 'y': 18 3669 id: 90 3670 options: 3671 legend: 3672 calcs: [] 3673 displayMode: list 3674 placement: bottom 3675 showLegend: false 3676 tooltip: 3677 mode: multi 3678 sort: none 3679 targets: 3680 - datasource: 3681 type: prometheus 3682 uid: ${datasource} 3683 editorMode: code 3684 expr: >- 3685 (sum(irate(container_network_transmit_bytes_total{{ "{" }}{{ .Values.dashboards.cadvisorSelector }}, 3686 cluster=~"$cluster", namespace=~"$namespace"}[$__rate_interval]) 3687 3688 * on (namespace,pod) 3689 3690 group_left(workload,workload_type) 3691 namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", 3692 namespace=~"$namespace", workload=~".*query-frontend.*", 3693 workload_type="deployment"}) by (pod)) 3694 legendFormat: __auto 3695 range: true 3696 refId: A 3697 title: Transmit Bandwidth 3698 type: timeseries 3699 - datasource: 3700 type: loki 3701 uid: ${loki_datasource} 3702 fieldConfig: 3703 defaults: {} 3704 overrides: [] 3705 gridPos: 3706 h: 18 3707 w: 24 3708 x: 0 3709 'y': 26 3710 id: 60 3711 options: 3712 dedupStrategy: none 3713 enableLogDetails: true 3714 prettifyLogMessage: false 3715 showCommonLabels: false 3716 showLabels: false 3717 showTime: false 3718 sortOrder: Descending 3719 wrapLogMessage: true 3720 targets: 3721 - datasource: 3722 type: loki 3723 uid: ${loki_datasource} 3724 editorMode: code 3725 expr: >- 3726 {namespace=~"$namespace",cluster=~"$cluster",container="query-frontend"} 3727 |= "http.go" |~ "QuerierService|pyroscope|render" | logfmt | 3728 line_format "tenant={{"{{.orgID}} traceID={{.traceID}} {{.msg}}"}}" 3729 queryType: range 3730 refId: A 3731 title: Queries 3732 type: logs 3733 title: Query Frontend 3734 type: row 3735 - collapsed: true 3736 gridPos: 3737 h: 1 3738 w: 24 3739 x: 0 3740 'y': 44 3741 id: 56 3742 panels: 3743 - datasource: 3744 type: prometheus 3745 uid: ${datasource} 3746 fieldConfig: 3747 defaults: 3748 color: 3749 mode: palette-classic 3750 custom: 3751 axisBorderShow: false 3752 axisCenteredZero: false 3753 axisColorMode: text 3754 axisLabel: '' 3755 axisPlacement: auto 3756 barAlignment: 0 3757 drawStyle: line 3758 fillOpacity: 21 3759 gradientMode: none 3760 hideFrom: 3761 legend: false 3762 tooltip: false 3763 viz: false 3764 insertNulls: false 3765 lineInterpolation: linear 3766 lineStyle: 3767 fill: solid 3768 lineWidth: 1 3769 pointSize: 5 3770 scaleDistribution: 3771 type: linear 3772 showPoints: auto 3773 spanNulls: false 3774 stacking: 3775 group: A 3776 mode: normal 3777 thresholdsStyle: 3778 mode: dashed 3779 mappings: [] 3780 thresholds: 3781 mode: absolute 3782 steps: 3783 - color: green 3784 - color: yellow 3785 value: 80 3786 - color: red 3787 value: 90 3788 overrides: 3789 - matcher: 3790 id: byName 3791 options: request 3792 properties: 3793 - id: custom.stacking 3794 value: 3795 group: A 3796 mode: none 3797 - id: color 3798 value: 3799 fixedColor: yellow 3800 mode: fixed 3801 - id: custom.fillOpacity 3802 value: 0 3803 - id: custom.lineStyle 3804 value: 3805 dash: 3806 - 10 3807 - 10 3808 fill: dash 3809 - id: custom.lineWidth 3810 value: 2 3811 - matcher: 3812 id: byName 3813 options: limit 3814 properties: 3815 - id: custom.fillOpacity 3816 value: 0 3817 - id: custom.stacking 3818 value: 3819 group: A 3820 mode: none 3821 - id: custom.lineStyle 3822 value: 3823 dash: 3824 - 10 3825 - 10 3826 fill: dash 3827 - id: custom.lineWidth 3828 value: 2 3829 - id: color 3830 value: 3831 fixedColor: red 3832 mode: fixed 3833 gridPos: 3834 h: 8 3835 w: 12 3836 x: 0 3837 'y': 6 3838 id: 87 3839 options: 3840 legend: 3841 calcs: [] 3842 displayMode: list 3843 placement: bottom 3844 showLegend: true 3845 tooltip: 3846 mode: multi 3847 sort: desc 3848 targets: 3849 - datasource: 3850 type: prometheus 3851 uid: ${datasource} 3852 editorMode: code 3853 expr: |- 3854 sum( 3855 node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~"$cluster", namespace=~"$namespace"} 3856 * on(namespace,pod) 3857 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*querier", workload_type="deployment"} 3858 ) by (pod) 3859 legendFormat: __auto 3860 range: true 3861 refId: by pod 3862 - datasource: 3863 type: prometheus 3864 uid: ${datasource} 3865 editorMode: code 3866 exemplar: false 3867 expr: |- 3868 sum( 3869 kube_pod_container_resource_requests{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="cpu"} 3870 * on(namespace,pod) 3871 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*querier", workload_type="deployment"} 3872 ) 3873 hide: false 3874 instant: false 3875 legendFormat: request 3876 range: true 3877 refId: request 3878 - datasource: 3879 type: prometheus 3880 uid: ${datasource} 3881 editorMode: code 3882 expr: |- 3883 sum( 3884 kube_pod_container_resource_limits{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="cpu"} 3885 * on(namespace,pod) 3886 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*querier", workload_type="deployment"} 3887 ) 3888 hide: false 3889 legendFormat: limit 3890 range: true 3891 refId: limit 3892 title: CPU Usage 3893 type: timeseries 3894 - datasource: 3895 type: prometheus 3896 uid: ${datasource} 3897 fieldConfig: 3898 defaults: 3899 color: 3900 mode: palette-classic 3901 custom: 3902 axisBorderShow: false 3903 axisCenteredZero: false 3904 axisColorMode: text 3905 axisLabel: '' 3906 axisPlacement: auto 3907 barAlignment: 0 3908 drawStyle: line 3909 fillOpacity: 21 3910 gradientMode: none 3911 hideFrom: 3912 legend: false 3913 tooltip: false 3914 viz: false 3915 insertNulls: false 3916 lineInterpolation: linear 3917 lineStyle: 3918 fill: solid 3919 lineWidth: 1 3920 pointSize: 5 3921 scaleDistribution: 3922 type: linear 3923 showPoints: auto 3924 spanNulls: false 3925 stacking: 3926 group: A 3927 mode: normal 3928 thresholdsStyle: 3929 mode: dashed 3930 mappings: [] 3931 thresholds: 3932 mode: absolute 3933 steps: 3934 - color: green 3935 - color: yellow 3936 value: 80 3937 - color: red 3938 value: 90 3939 unit: bytes 3940 overrides: 3941 - matcher: 3942 id: byName 3943 options: request 3944 properties: 3945 - id: custom.stacking 3946 value: 3947 group: A 3948 mode: none 3949 - id: color 3950 value: 3951 fixedColor: yellow 3952 mode: fixed 3953 - id: custom.fillOpacity 3954 value: 0 3955 - id: custom.lineStyle 3956 value: 3957 dash: 3958 - 10 3959 - 10 3960 fill: dash 3961 - id: custom.lineWidth 3962 value: 2 3963 - matcher: 3964 id: byName 3965 options: limit 3966 properties: 3967 - id: color 3968 value: 3969 fixedColor: red 3970 mode: fixed 3971 - id: custom.stacking 3972 value: 3973 group: A 3974 mode: none 3975 - id: custom.fillOpacity 3976 value: 0 3977 - id: custom.lineStyle 3978 value: 3979 dash: 3980 - 10 3981 - 10 3982 fill: dash 3983 - id: custom.lineWidth 3984 value: 2 3985 gridPos: 3986 h: 8 3987 w: 12 3988 x: 12 3989 'y': 6 3990 id: 58 3991 options: 3992 legend: 3993 calcs: [] 3994 displayMode: list 3995 placement: bottom 3996 showLegend: true 3997 tooltip: 3998 mode: multi 3999 sort: desc 4000 targets: 4001 - datasource: 4002 type: prometheus 4003 uid: ${datasource} 4004 editorMode: code 4005 expr: | 4006 sum( 4007 container_memory_working_set_bytes{cluster=~"$cluster", namespace=~"$namespace", container!="", image!=""} 4008 * on(namespace,pod) 4009 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*querier", workload_type="deployment"} 4010 ) by (pod) 4011 legendFormat: __auto 4012 range: true 4013 refId: by pod 4014 - datasource: 4015 type: prometheus 4016 uid: ${datasource} 4017 editorMode: code 4018 exemplar: false 4019 expr: |- 4020 sum( 4021 kube_pod_container_resource_requests{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="memory"} 4022 * on(namespace,pod) 4023 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*querier", workload_type="deployment"} 4024 ) 4025 hide: false 4026 instant: false 4027 legendFormat: request 4028 range: true 4029 refId: request 4030 - datasource: 4031 type: prometheus 4032 uid: ${datasource} 4033 editorMode: code 4034 expr: |- 4035 sum( 4036 kube_pod_container_resource_limits{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="memory"} 4037 * on(namespace,pod) 4038 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*querier", workload_type="deployment"} 4039 ) 4040 hide: false 4041 legendFormat: limit 4042 range: true 4043 refId: limit 4044 title: Memory Usage 4045 type: timeseries 4046 - datasource: 4047 type: prometheus 4048 uid: ${datasource} 4049 fieldConfig: 4050 defaults: 4051 color: 4052 mode: palette-classic 4053 custom: 4054 axisBorderShow: false 4055 axisCenteredZero: false 4056 axisColorMode: text 4057 axisLabel: '' 4058 axisPlacement: auto 4059 barAlignment: 0 4060 drawStyle: line 4061 fillOpacity: 13 4062 gradientMode: none 4063 hideFrom: 4064 legend: false 4065 tooltip: false 4066 viz: false 4067 insertNulls: false 4068 lineInterpolation: linear 4069 lineWidth: 1 4070 pointSize: 5 4071 scaleDistribution: 4072 type: linear 4073 showPoints: auto 4074 spanNulls: false 4075 stacking: 4076 group: A 4077 mode: normal 4078 thresholdsStyle: 4079 mode: 'off' 4080 mappings: [] 4081 thresholds: 4082 mode: absolute 4083 steps: 4084 - color: green 4085 - color: red 4086 value: 80 4087 unit: Bps 4088 overrides: [] 4089 gridPos: 4090 h: 8 4091 w: 12 4092 x: 0 4093 'y': 14 4094 id: 75 4095 options: 4096 legend: 4097 calcs: [] 4098 displayMode: list 4099 placement: bottom 4100 showLegend: false 4101 tooltip: 4102 mode: multi 4103 sort: none 4104 targets: 4105 - datasource: 4106 type: prometheus 4107 uid: ${datasource} 4108 editorMode: code 4109 expr: >- 4110 (sum(irate(container_network_receive_bytes_total{{ "{" }}{{ .Values.dashboards.cadvisorSelector }}, 4111 cluster=~"$cluster", namespace=~"$namespace"}[$__rate_interval]) 4112 4113 * on (namespace,pod) 4114 4115 group_left(workload,workload_type) 4116 namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", 4117 namespace=~"$namespace", workload=~".*querier.*", 4118 workload_type="deployment"}) by (pod)) 4119 legendFormat: __auto 4120 range: true 4121 refId: A 4122 title: Receive Bandwidth 4123 type: timeseries 4124 - datasource: 4125 type: prometheus 4126 uid: ${datasource} 4127 fieldConfig: 4128 defaults: 4129 color: 4130 mode: palette-classic 4131 custom: 4132 axisBorderShow: false 4133 axisCenteredZero: false 4134 axisColorMode: text 4135 axisLabel: '' 4136 axisPlacement: auto 4137 barAlignment: 0 4138 drawStyle: line 4139 fillOpacity: 13 4140 gradientMode: none 4141 hideFrom: 4142 legend: false 4143 tooltip: false 4144 viz: false 4145 insertNulls: false 4146 lineInterpolation: linear 4147 lineWidth: 1 4148 pointSize: 5 4149 scaleDistribution: 4150 type: linear 4151 showPoints: auto 4152 spanNulls: false 4153 stacking: 4154 group: A 4155 mode: normal 4156 thresholdsStyle: 4157 mode: 'off' 4158 mappings: [] 4159 thresholds: 4160 mode: absolute 4161 steps: 4162 - color: green 4163 - color: red 4164 value: 80 4165 unit: Bps 4166 overrides: [] 4167 gridPos: 4168 h: 8 4169 w: 12 4170 x: 12 4171 'y': 14 4172 id: 76 4173 options: 4174 legend: 4175 calcs: [] 4176 displayMode: list 4177 placement: bottom 4178 showLegend: false 4179 tooltip: 4180 mode: multi 4181 sort: none 4182 targets: 4183 - datasource: 4184 type: prometheus 4185 uid: ${datasource} 4186 editorMode: code 4187 expr: >- 4188 (sum(irate(container_network_transmit_bytes_total{{ "{" }}{{ .Values.dashboards.cadvisorSelector }}, 4189 cluster=~"$cluster", namespace=~"$namespace"}[$__rate_interval]) 4190 4191 * on (namespace,pod) 4192 4193 group_left(workload,workload_type) 4194 namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", 4195 namespace=~"$namespace", workload=~".*querier.*", 4196 workload_type="deployment"}) by (pod)) 4197 legendFormat: __auto 4198 range: true 4199 refId: A 4200 title: Transmit Bandwidth 4201 type: timeseries 4202 title: Queriers 4203 type: row 4204 - collapsed: true 4205 gridPos: 4206 h: 1 4207 w: 24 4208 x: 0 4209 'y': 45 4210 id: 102 4211 panels: 4212 - datasource: 4213 type: prometheus 4214 uid: ${datasource} 4215 fieldConfig: 4216 defaults: 4217 color: 4218 mode: palette-classic 4219 custom: 4220 axisBorderShow: false 4221 axisCenteredZero: false 4222 axisColorMode: text 4223 axisLabel: '' 4224 axisPlacement: auto 4225 barAlignment: 0 4226 drawStyle: line 4227 fillOpacity: 21 4228 gradientMode: none 4229 hideFrom: 4230 legend: false 4231 tooltip: false 4232 viz: false 4233 insertNulls: false 4234 lineInterpolation: linear 4235 lineStyle: 4236 fill: solid 4237 lineWidth: 1 4238 pointSize: 5 4239 scaleDistribution: 4240 type: linear 4241 showPoints: auto 4242 spanNulls: false 4243 stacking: 4244 group: A 4245 mode: normal 4246 thresholdsStyle: 4247 mode: dashed 4248 mappings: [] 4249 thresholds: 4250 mode: absolute 4251 steps: 4252 - color: green 4253 - color: yellow 4254 value: 80 4255 - color: red 4256 value: 90 4257 overrides: 4258 - matcher: 4259 id: byName 4260 options: request 4261 properties: 4262 - id: custom.stacking 4263 value: 4264 group: A 4265 mode: none 4266 - id: color 4267 value: 4268 fixedColor: yellow 4269 mode: fixed 4270 - id: custom.fillOpacity 4271 value: 0 4272 - id: custom.lineStyle 4273 value: 4274 dash: 4275 - 10 4276 - 10 4277 fill: dash 4278 - id: custom.lineWidth 4279 value: 2 4280 - matcher: 4281 id: byName 4282 options: limit 4283 properties: 4284 - id: custom.fillOpacity 4285 value: 0 4286 - id: custom.stacking 4287 value: 4288 group: A 4289 mode: none 4290 - id: custom.lineStyle 4291 value: 4292 dash: 4293 - 10 4294 - 10 4295 fill: dash 4296 - id: custom.lineWidth 4297 value: 2 4298 - id: color 4299 value: 4300 fixedColor: red 4301 mode: fixed 4302 gridPos: 4303 h: 8 4304 w: 12 4305 x: 0 4306 'y': 7 4307 id: 103 4308 options: 4309 legend: 4310 calcs: [] 4311 displayMode: list 4312 placement: bottom 4313 showLegend: true 4314 tooltip: 4315 mode: multi 4316 sort: desc 4317 targets: 4318 - datasource: 4319 type: prometheus 4320 uid: ${datasource} 4321 editorMode: code 4322 expr: |- 4323 sum( 4324 node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~"$cluster", namespace=~"$namespace"} 4325 * on(namespace,pod) 4326 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*query-scheduler", workload_type="deployment"} 4327 ) by (pod) 4328 legendFormat: __auto 4329 range: true 4330 refId: by pod 4331 - datasource: 4332 type: prometheus 4333 uid: ${datasource} 4334 editorMode: code 4335 exemplar: false 4336 expr: |- 4337 sum( 4338 kube_pod_container_resource_requests{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="cpu"} 4339 * on(namespace,pod) 4340 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*query-scheduler", workload_type="deployment"} 4341 ) 4342 hide: false 4343 instant: false 4344 legendFormat: request 4345 range: true 4346 refId: request 4347 - datasource: 4348 type: prometheus 4349 uid: ${datasource} 4350 editorMode: code 4351 expr: |- 4352 sum( 4353 kube_pod_container_resource_limits{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="cpu"} 4354 * on(namespace,pod) 4355 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*query-scheduler", workload_type="deployment"} 4356 ) 4357 hide: false 4358 legendFormat: limit 4359 range: true 4360 refId: limit 4361 title: CPU Usage 4362 type: timeseries 4363 - datasource: 4364 type: prometheus 4365 uid: ${datasource} 4366 fieldConfig: 4367 defaults: 4368 color: 4369 mode: palette-classic 4370 custom: 4371 axisBorderShow: false 4372 axisCenteredZero: false 4373 axisColorMode: text 4374 axisLabel: '' 4375 axisPlacement: auto 4376 barAlignment: 0 4377 drawStyle: line 4378 fillOpacity: 21 4379 gradientMode: none 4380 hideFrom: 4381 legend: false 4382 tooltip: false 4383 viz: false 4384 insertNulls: false 4385 lineInterpolation: linear 4386 lineStyle: 4387 fill: solid 4388 lineWidth: 1 4389 pointSize: 5 4390 scaleDistribution: 4391 type: linear 4392 showPoints: auto 4393 spanNulls: false 4394 stacking: 4395 group: A 4396 mode: normal 4397 thresholdsStyle: 4398 mode: dashed 4399 mappings: [] 4400 thresholds: 4401 mode: absolute 4402 steps: 4403 - color: green 4404 - color: yellow 4405 value: 80 4406 - color: red 4407 value: 90 4408 unit: bytes 4409 overrides: 4410 - matcher: 4411 id: byName 4412 options: request 4413 properties: 4414 - id: custom.stacking 4415 value: 4416 group: A 4417 mode: none 4418 - id: color 4419 value: 4420 fixedColor: yellow 4421 mode: fixed 4422 - id: custom.fillOpacity 4423 value: 0 4424 - id: custom.lineStyle 4425 value: 4426 dash: 4427 - 10 4428 - 10 4429 fill: dash 4430 - id: custom.lineWidth 4431 value: 2 4432 - matcher: 4433 id: byName 4434 options: limit 4435 properties: 4436 - id: color 4437 value: 4438 fixedColor: red 4439 mode: fixed 4440 - id: custom.stacking 4441 value: 4442 group: A 4443 mode: none 4444 - id: custom.fillOpacity 4445 value: 0 4446 - id: custom.lineStyle 4447 value: 4448 dash: 4449 - 10 4450 - 10 4451 fill: dash 4452 - id: custom.lineWidth 4453 value: 2 4454 gridPos: 4455 h: 8 4456 w: 12 4457 x: 12 4458 'y': 7 4459 id: 93 4460 options: 4461 legend: 4462 calcs: [] 4463 displayMode: list 4464 placement: bottom 4465 showLegend: true 4466 tooltip: 4467 mode: multi 4468 sort: desc 4469 targets: 4470 - datasource: 4471 type: prometheus 4472 uid: ${datasource} 4473 editorMode: code 4474 expr: | 4475 sum( 4476 container_memory_working_set_bytes{cluster=~"$cluster", namespace=~"$namespace", container!="", image!=""} 4477 * on(namespace,pod) 4478 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*query-scheduler", workload_type="deployment"} 4479 ) by (pod) 4480 legendFormat: __auto 4481 range: true 4482 refId: by pod 4483 - datasource: 4484 type: prometheus 4485 uid: ${datasource} 4486 editorMode: code 4487 exemplar: false 4488 expr: |- 4489 sum( 4490 kube_pod_container_resource_requests{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="memory"} 4491 * on(namespace,pod) 4492 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*query-scheduler", workload_type="deployment"} 4493 ) 4494 hide: false 4495 instant: false 4496 legendFormat: request 4497 range: true 4498 refId: request 4499 - datasource: 4500 type: prometheus 4501 uid: ${datasource} 4502 editorMode: code 4503 expr: |- 4504 sum( 4505 kube_pod_container_resource_limits{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="memory"} 4506 * on(namespace,pod) 4507 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*query-scheduler", workload_type="deployment"} 4508 ) 4509 hide: false 4510 legendFormat: limit 4511 range: true 4512 refId: limit 4513 title: Memory Usage 4514 type: timeseries 4515 title: Query Scheduler 4516 type: row 4517 - collapsed: true 4518 gridPos: 4519 h: 1 4520 w: 24 4521 x: 0 4522 'y': 46 4523 id: 86 4524 panels: 4525 - datasource: 4526 type: prometheus 4527 uid: ${datasource} 4528 fieldConfig: 4529 defaults: 4530 color: 4531 mode: palette-classic 4532 custom: 4533 axisBorderShow: false 4534 axisCenteredZero: false 4535 axisColorMode: text 4536 axisLabel: '' 4537 axisPlacement: auto 4538 barAlignment: 0 4539 drawStyle: line 4540 fillOpacity: 0 4541 gradientMode: none 4542 hideFrom: 4543 legend: false 4544 tooltip: false 4545 viz: false 4546 insertNulls: false 4547 lineInterpolation: linear 4548 lineWidth: 1 4549 pointSize: 5 4550 scaleDistribution: 4551 type: linear 4552 showPoints: auto 4553 spanNulls: false 4554 stacking: 4555 group: A 4556 mode: none 4557 thresholdsStyle: 4558 mode: 'off' 4559 mappings: [] 4560 thresholds: 4561 mode: absolute 4562 steps: 4563 - color: green 4564 - color: red 4565 value: 80 4566 unit: s 4567 overrides: [] 4568 gridPos: 4569 h: 5 4570 w: 12 4571 x: 0 4572 'y': 8 4573 id: 98 4574 options: 4575 legend: 4576 calcs: [] 4577 displayMode: table 4578 placement: right 4579 showLegend: true 4580 tooltip: 4581 mode: single 4582 sort: none 4583 targets: 4584 - datasource: 4585 type: prometheus 4586 uid: ${datasource} 4587 editorMode: code 4588 expr: >- 4589 histogram_quantile(0.99, sum by (route) ( 4590 rate(pyroscope_request_duration_seconds{job=~"$namespace/store-gateway", 4591 cluster=~"$cluster",route=~".*store.*"}[$__rate_interval]))) 4592 legendFormat: .99 {{"{{route}}"}} 4593 range: true 4594 refId: A 4595 - datasource: 4596 type: prometheus 4597 uid: ${datasource} 4598 editorMode: code 4599 expr: >- 4600 histogram_quantile(0.75, sum by (route) ( 4601 rate(pyroscope_request_duration_seconds{job=~"$namespace/store-gateway", 4602 cluster=~"$cluster",route=~".*store.*"}[$__rate_interval]))) 4603 hide: true 4604 legendFormat: .75 {{"{{route}}"}} 4605 range: true 4606 refId: B 4607 - datasource: 4608 type: prometheus 4609 uid: ${datasource} 4610 editorMode: code 4611 expr: >- 4612 histogram_quantile(0.5, sum by (route) ( 4613 rate(pyroscope_request_duration_seconds{job=~"$namespace/store-gateway", 4614 cluster=~"$cluster",route=~".*store.*"}[$__rate_interval]))) 4615 hide: true 4616 legendFormat: .5 {{"{{route}}"}} 4617 range: true 4618 refId: C 4619 title: API Latencies 4620 type: timeseries 4621 - datasource: 4622 type: prometheus 4623 uid: $datasource 4624 fieldConfig: 4625 defaults: 4626 color: 4627 mode: palette-classic 4628 custom: 4629 axisBorderShow: false 4630 axisCenteredZero: false 4631 axisColorMode: text 4632 axisLabel: '' 4633 axisPlacement: auto 4634 barAlignment: 0 4635 drawStyle: line 4636 fillOpacity: 40 4637 gradientMode: none 4638 hideFrom: 4639 legend: false 4640 tooltip: false 4641 viz: false 4642 insertNulls: false 4643 lineInterpolation: linear 4644 lineWidth: 1 4645 pointSize: 5 4646 scaleDistribution: 4647 type: linear 4648 showPoints: never 4649 spanNulls: false 4650 stacking: 4651 group: A 4652 mode: normal 4653 thresholdsStyle: 4654 mode: 'off' 4655 mappings: [] 4656 thresholds: 4657 mode: absolute 4658 steps: 4659 - color: green 4660 - color: red 4661 value: 80 4662 unit: reqps 4663 overrides: 4664 - matcher: 4665 id: byRegexp 4666 options: /2../ 4667 properties: 4668 - id: color 4669 value: 4670 fixedColor: green 4671 mode: shades 4672 - matcher: 4673 id: byRegexp 4674 options: /5../ 4675 properties: 4676 - id: color 4677 value: 4678 fixedColor: red 4679 mode: shades 4680 - matcher: 4681 id: byRegexp 4682 options: /4../ 4683 properties: 4684 - id: color 4685 value: 4686 fixedColor: yellow 4687 mode: shades 4688 gridPos: 4689 h: 5 4690 w: 12 4691 x: 12 4692 'y': 8 4693 id: 99 4694 options: 4695 dataLinks: [] 4696 legend: 4697 calcs: [] 4698 displayMode: list 4699 placement: bottom 4700 showLegend: true 4701 tooltip: 4702 mode: multi 4703 sort: none 4704 targets: 4705 - datasource: 4706 uid: $datasource 4707 editorMode: code 4708 expr: |- 4709 sum by (status,route) ( 4710 label_replace( 4711 label_replace( 4712 rate(pyroscope_request_duration_seconds_count{cluster=~"$cluster", job=~"$namespace/store-gateway", route=~".*store.*"}[$__rate_interval]), 4713 "status", "${1}xx", "status_code", "([0-9]).."), 4714 "status", "${1}", "status_code", "([a-z]+)") 4715 ) 4716 legendFormat: '{{"{{status}} {{route}}"}}' 4717 range: true 4718 refId: A 4719 title: Request Rates 4720 type: timeseries 4721 - datasource: 4722 type: prometheus 4723 uid: ${datasource} 4724 fieldConfig: 4725 defaults: 4726 color: 4727 mode: palette-classic 4728 custom: 4729 axisBorderShow: false 4730 axisCenteredZero: false 4731 axisColorMode: text 4732 axisLabel: '' 4733 axisPlacement: auto 4734 barAlignment: 0 4735 drawStyle: line 4736 fillOpacity: 21 4737 gradientMode: none 4738 hideFrom: 4739 legend: false 4740 tooltip: false 4741 viz: false 4742 insertNulls: false 4743 lineInterpolation: linear 4744 lineStyle: 4745 fill: solid 4746 lineWidth: 1 4747 pointSize: 5 4748 scaleDistribution: 4749 type: linear 4750 showPoints: auto 4751 spanNulls: false 4752 stacking: 4753 group: A 4754 mode: normal 4755 thresholdsStyle: 4756 mode: dashed 4757 mappings: [] 4758 thresholds: 4759 mode: absolute 4760 steps: 4761 - color: green 4762 - color: yellow 4763 value: 80 4764 - color: red 4765 value: 90 4766 overrides: 4767 - matcher: 4768 id: byName 4769 options: request 4770 properties: 4771 - id: custom.stacking 4772 value: 4773 group: A 4774 mode: none 4775 - id: color 4776 value: 4777 fixedColor: yellow 4778 mode: fixed 4779 - id: custom.fillOpacity 4780 value: 0 4781 - id: custom.lineStyle 4782 value: 4783 dash: 4784 - 10 4785 - 10 4786 fill: dash 4787 - id: custom.lineWidth 4788 value: 2 4789 - matcher: 4790 id: byName 4791 options: limit 4792 properties: 4793 - id: custom.fillOpacity 4794 value: 0 4795 - id: custom.stacking 4796 value: 4797 group: A 4798 mode: none 4799 - id: custom.lineStyle 4800 value: 4801 dash: 4802 - 10 4803 - 10 4804 fill: dash 4805 - id: custom.lineWidth 4806 value: 2 4807 - id: color 4808 value: 4809 fixedColor: red 4810 mode: fixed 4811 gridPos: 4812 h: 8 4813 w: 12 4814 x: 0 4815 'y': 13 4816 id: 57 4817 options: 4818 legend: 4819 calcs: [] 4820 displayMode: list 4821 placement: bottom 4822 showLegend: true 4823 tooltip: 4824 mode: multi 4825 sort: desc 4826 targets: 4827 - datasource: 4828 type: prometheus 4829 uid: ${datasource} 4830 editorMode: code 4831 expr: |- 4832 sum( 4833 node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~"$cluster", namespace=~"$namespace"} 4834 * on(namespace,pod) 4835 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*store-gateway", workload_type="statefulset"} 4836 ) by (pod) 4837 legendFormat: __auto 4838 range: true 4839 refId: by pod 4840 - datasource: 4841 type: prometheus 4842 uid: ${datasource} 4843 editorMode: code 4844 exemplar: false 4845 expr: |- 4846 sum( 4847 kube_pod_container_resource_requests{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="cpu"} 4848 * on(namespace,pod) 4849 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*store-gateway", workload_type="statefulset"} 4850 ) 4851 hide: false 4852 instant: false 4853 legendFormat: request 4854 range: true 4855 refId: request 4856 - datasource: 4857 type: prometheus 4858 uid: ${datasource} 4859 editorMode: code 4860 expr: |- 4861 sum( 4862 kube_pod_container_resource_limits{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="cpu"} 4863 * on(namespace,pod) 4864 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*store-gateway", workload_type="statefulset"} 4865 ) 4866 hide: false 4867 legendFormat: limit 4868 range: true 4869 refId: limit 4870 title: CPU Usage 4871 type: timeseries 4872 - datasource: 4873 type: prometheus 4874 uid: ${datasource} 4875 fieldConfig: 4876 defaults: 4877 color: 4878 mode: palette-classic 4879 custom: 4880 axisBorderShow: false 4881 axisCenteredZero: false 4882 axisColorMode: text 4883 axisLabel: '' 4884 axisPlacement: auto 4885 barAlignment: 0 4886 drawStyle: line 4887 fillOpacity: 21 4888 gradientMode: none 4889 hideFrom: 4890 legend: false 4891 tooltip: false 4892 viz: false 4893 insertNulls: false 4894 lineInterpolation: linear 4895 lineStyle: 4896 fill: solid 4897 lineWidth: 1 4898 pointSize: 5 4899 scaleDistribution: 4900 type: linear 4901 showPoints: auto 4902 spanNulls: false 4903 stacking: 4904 group: A 4905 mode: normal 4906 thresholdsStyle: 4907 mode: dashed 4908 mappings: [] 4909 thresholds: 4910 mode: absolute 4911 steps: 4912 - color: green 4913 - color: yellow 4914 value: 80 4915 - color: red 4916 value: 90 4917 unit: bytes 4918 overrides: 4919 - matcher: 4920 id: byName 4921 options: request 4922 properties: 4923 - id: custom.stacking 4924 value: 4925 group: A 4926 mode: none 4927 - id: color 4928 value: 4929 fixedColor: yellow 4930 mode: fixed 4931 - id: custom.fillOpacity 4932 value: 0 4933 - id: custom.lineStyle 4934 value: 4935 dash: 4936 - 10 4937 - 10 4938 fill: dash 4939 - id: custom.lineWidth 4940 value: 2 4941 - matcher: 4942 id: byName 4943 options: limit 4944 properties: 4945 - id: color 4946 value: 4947 fixedColor: red 4948 mode: fixed 4949 - id: custom.stacking 4950 value: 4951 group: A 4952 mode: none 4953 - id: custom.fillOpacity 4954 value: 0 4955 - id: custom.lineStyle 4956 value: 4957 dash: 4958 - 10 4959 - 10 4960 fill: dash 4961 - id: custom.lineWidth 4962 value: 2 4963 gridPos: 4964 h: 8 4965 w: 12 4966 x: 12 4967 'y': 13 4968 id: 104 4969 options: 4970 legend: 4971 calcs: [] 4972 displayMode: list 4973 placement: bottom 4974 showLegend: true 4975 tooltip: 4976 mode: multi 4977 sort: desc 4978 targets: 4979 - datasource: 4980 type: prometheus 4981 uid: ${datasource} 4982 editorMode: code 4983 expr: | 4984 sum( 4985 container_memory_working_set_bytes{cluster=~"$cluster", namespace=~"$namespace", container!="", image!=""} 4986 * on(namespace,pod) 4987 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*store-gateway", workload_type="statefulset"} 4988 ) by (pod) 4989 legendFormat: __auto 4990 range: true 4991 refId: by pod 4992 - datasource: 4993 type: prometheus 4994 uid: ${datasource} 4995 editorMode: code 4996 exemplar: false 4997 expr: |- 4998 sum( 4999 kube_pod_container_resource_requests{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="memory"} 5000 * on(namespace,pod) 5001 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*store-gateway", workload_type="statefulset"} 5002 ) 5003 hide: false 5004 instant: false 5005 legendFormat: request 5006 range: true 5007 refId: request 5008 - datasource: 5009 type: prometheus 5010 uid: ${datasource} 5011 editorMode: code 5012 expr: |- 5013 sum( 5014 kube_pod_container_resource_limits{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, cluster=~"$cluster", namespace=~"$namespace", resource="memory"} 5015 * on(namespace,pod) 5016 group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", namespace=~"$namespace", workload=~".*store-gateway", workload_type="statefulset"} 5017 ) 5018 hide: false 5019 legendFormat: limit 5020 range: true 5021 refId: limit 5022 title: Memory Usage 5023 type: timeseries 5024 - datasource: 5025 type: prometheus 5026 uid: ${datasource} 5027 fieldConfig: 5028 defaults: 5029 color: 5030 mode: palette-classic 5031 custom: 5032 axisBorderShow: false 5033 axisCenteredZero: false 5034 axisColorMode: text 5035 axisLabel: '' 5036 axisPlacement: auto 5037 barAlignment: 0 5038 drawStyle: line 5039 fillOpacity: 13 5040 gradientMode: none 5041 hideFrom: 5042 legend: false 5043 tooltip: false 5044 viz: false 5045 insertNulls: false 5046 lineInterpolation: linear 5047 lineWidth: 1 5048 pointSize: 5 5049 scaleDistribution: 5050 type: linear 5051 showPoints: auto 5052 spanNulls: false 5053 stacking: 5054 group: A 5055 mode: normal 5056 thresholdsStyle: 5057 mode: 'off' 5058 mappings: [] 5059 thresholds: 5060 mode: absolute 5061 steps: 5062 - color: green 5063 - color: red 5064 value: 80 5065 unit: Bps 5066 overrides: [] 5067 gridPos: 5068 h: 8 5069 w: 12 5070 x: 0 5071 'y': 21 5072 id: 89 5073 options: 5074 legend: 5075 calcs: [] 5076 displayMode: list 5077 placement: bottom 5078 showLegend: false 5079 tooltip: 5080 mode: multi 5081 sort: none 5082 targets: 5083 - datasource: 5084 type: prometheus 5085 uid: ${datasource} 5086 editorMode: code 5087 expr: >- 5088 (sum(irate(container_network_receive_bytes_total{{ "{" }}{{ .Values.dashboards.cadvisorSelector }}, 5089 cluster=~"$cluster", namespace=~"$namespace"}[$__rate_interval]) 5090 5091 * on (namespace,pod) 5092 5093 group_left(workload,workload_type) 5094 namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", 5095 namespace=~"$namespace", workload=~".*store-gateway.*", 5096 workload_type="statefulset"}) by (pod)) 5097 legendFormat: __auto 5098 range: true 5099 refId: A 5100 title: Receive Bandwidth 5101 type: timeseries 5102 - datasource: 5103 type: prometheus 5104 uid: ${datasource} 5105 fieldConfig: 5106 defaults: 5107 color: 5108 mode: palette-classic 5109 custom: 5110 axisBorderShow: false 5111 axisCenteredZero: false 5112 axisColorMode: text 5113 axisLabel: '' 5114 axisPlacement: auto 5115 barAlignment: 0 5116 drawStyle: line 5117 fillOpacity: 13 5118 gradientMode: none 5119 hideFrom: 5120 legend: false 5121 tooltip: false 5122 viz: false 5123 insertNulls: false 5124 lineInterpolation: linear 5125 lineWidth: 1 5126 pointSize: 5 5127 scaleDistribution: 5128 type: linear 5129 showPoints: auto 5130 spanNulls: false 5131 stacking: 5132 group: A 5133 mode: normal 5134 thresholdsStyle: 5135 mode: 'off' 5136 mappings: [] 5137 thresholds: 5138 mode: absolute 5139 steps: 5140 - color: green 5141 - color: red 5142 value: 80 5143 unit: Bps 5144 overrides: [] 5145 gridPos: 5146 h: 8 5147 w: 12 5148 x: 12 5149 'y': 21 5150 id: 95 5151 options: 5152 legend: 5153 calcs: [] 5154 displayMode: list 5155 placement: bottom 5156 showLegend: false 5157 tooltip: 5158 mode: multi 5159 sort: none 5160 targets: 5161 - datasource: 5162 type: prometheus 5163 uid: ${datasource} 5164 editorMode: code 5165 expr: >- 5166 (sum(irate(container_network_transmit_bytes_total{{ "{" }}{{ .Values.dashboards.cadvisorSelector }}, 5167 cluster=~"$cluster", namespace=~"$namespace"}[$__rate_interval]) 5168 5169 * on (namespace,pod) 5170 5171 group_left(workload,workload_type) 5172 namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", 5173 namespace=~"$namespace", workload=~".*store-gateway.*", 5174 workload_type="statefulset"}) by (pod)) 5175 legendFormat: __auto 5176 range: true 5177 refId: A 5178 title: Transmit Bandwidth 5179 type: timeseries 5180 title: Store Gateway 5181 type: row 5182 - collapsed: true 5183 gridPos: 5184 h: 1 5185 w: 24 5186 x: 0 5187 'y': 47 5188 id: 78 5189 panels: 5190 - datasource: 5191 type: prometheus 5192 uid: ${datasource} 5193 fieldConfig: 5194 defaults: 5195 color: 5196 mode: palette-classic 5197 custom: 5198 axisBorderShow: false 5199 axisCenteredZero: false 5200 axisColorMode: text 5201 axisLabel: '' 5202 axisPlacement: auto 5203 barAlignment: 0 5204 drawStyle: line 5205 fillOpacity: 12 5206 gradientMode: none 5207 hideFrom: 5208 legend: false 5209 tooltip: false 5210 viz: false 5211 insertNulls: false 5212 lineInterpolation: linear 5213 lineWidth: 1 5214 pointSize: 5 5215 scaleDistribution: 5216 type: linear 5217 showPoints: auto 5218 spanNulls: false 5219 stacking: 5220 group: A 5221 mode: normal 5222 thresholdsStyle: 5223 mode: 'off' 5224 mappings: [] 5225 thresholds: 5226 mode: absolute 5227 steps: 5228 - color: green 5229 - color: red 5230 value: 80 5231 overrides: 5232 - __systemRef: hideSeriesFrom 5233 matcher: 5234 id: byNames 5235 options: 5236 mode: exclude 5237 names: 5238 - get 5239 prefix: 'All except:' 5240 readOnly: true 5241 properties: 5242 - id: custom.hideFrom 5243 value: 5244 legend: false 5245 tooltip: false 5246 viz: true 5247 gridPos: 5248 h: 8 5249 w: 12 5250 x: 0 5251 'y': 58 5252 id: 80 5253 options: 5254 legend: 5255 calcs: [] 5256 displayMode: list 5257 placement: bottom 5258 showLegend: true 5259 tooltip: 5260 mode: multi 5261 sort: none 5262 targets: 5263 - datasource: 5264 type: prometheus 5265 uid: ${datasource} 5266 editorMode: code 5267 expr: >- 5268 sum by(operation) 5269 (rate(objstore_bucket_operations_total{namespace=~"$namespace",cluster=~"$cluster"}[$__rate_interval])) 5270 legendFormat: __auto 5271 range: true 5272 refId: A 5273 title: Bucket Operation /s 5274 type: timeseries 5275 - datasource: 5276 type: prometheus 5277 uid: ${datasource} 5278 fieldConfig: 5279 defaults: 5280 color: 5281 mode: palette-classic 5282 custom: 5283 axisBorderShow: false 5284 axisCenteredZero: false 5285 axisColorMode: text 5286 axisLabel: '' 5287 axisPlacement: auto 5288 barAlignment: 0 5289 drawStyle: line 5290 fillOpacity: 12 5291 gradientMode: none 5292 hideFrom: 5293 legend: false 5294 tooltip: false 5295 viz: false 5296 insertNulls: false 5297 lineInterpolation: linear 5298 lineWidth: 1 5299 pointSize: 5 5300 scaleDistribution: 5301 type: linear 5302 showPoints: auto 5303 spanNulls: false 5304 stacking: 5305 group: A 5306 mode: none 5307 thresholdsStyle: 5308 mode: 'off' 5309 mappings: [] 5310 thresholds: 5311 mode: absolute 5312 steps: 5313 - color: green 5314 - color: red 5315 value: 80 5316 unit: s 5317 overrides: [] 5318 gridPos: 5319 h: 8 5320 w: 12 5321 x: 12 5322 'y': 58 5323 id: 81 5324 options: 5325 legend: 5326 calcs: [] 5327 displayMode: list 5328 placement: bottom 5329 showLegend: true 5330 tooltip: 5331 mode: multi 5332 sort: none 5333 targets: 5334 - datasource: 5335 type: prometheus 5336 uid: ${datasource} 5337 editorMode: code 5338 expr: >- 5339 histogram_quantile(0.95, 5340 sum(rate(objstore_bucket_operation_duration_seconds_bucket{namespace=~"$namespace",cluster=~"$cluster"}[$__rate_interval])) 5341 by (le,operation)) 5342 legendFormat: __auto 5343 range: true 5344 refId: A 5345 title: Latencies per Operation 5346 type: timeseries 5347 title: Object Storage 5348 type: row 5349 - collapsed: false 5350 gridPos: 5351 h: 1 5352 w: 24 5353 x: 0 5354 'y': 48 5355 id: 105 5356 panels: [] 5357 title: GitHub API 5358 type: row 5359 - datasource: 5360 type: prometheus 5361 uid: ${datasource} 5362 fieldConfig: 5363 defaults: 5364 color: 5365 mode: palette-classic 5366 custom: 5367 axisBorderShow: false 5368 axisCenteredZero: false 5369 axisColorMode: text 5370 axisLabel: '' 5371 axisPlacement: auto 5372 barAlignment: 0 5373 barWidthFactor: 0.6 5374 drawStyle: line 5375 fillOpacity: 0 5376 gradientMode: none 5377 hideFrom: 5378 legend: false 5379 tooltip: false 5380 viz: false 5381 insertNulls: false 5382 lineInterpolation: linear 5383 lineWidth: 1 5384 pointSize: 5 5385 scaleDistribution: 5386 type: linear 5387 showPoints: auto 5388 spanNulls: false 5389 stacking: 5390 group: A 5391 mode: none 5392 thresholdsStyle: 5393 mode: 'off' 5394 mappings: [] 5395 noValue: '0' 5396 thresholds: 5397 mode: absolute 5398 steps: 5399 - color: green 5400 - color: red 5401 value: 80 5402 unit: s 5403 overrides: [] 5404 gridPos: 5405 h: 8 5406 w: 12 5407 x: 0 5408 'y': 49 5409 id: 106 5410 options: 5411 legend: 5412 calcs: [] 5413 displayMode: list 5414 placement: bottom 5415 showLegend: true 5416 tooltip: 5417 hideZeros: false 5418 mode: single 5419 sort: none 5420 pluginVersion: 12.1.0-88106 5421 targets: 5422 - datasource: 5423 type: prometheus 5424 uid: ${datasource} 5425 disableTextWrap: false 5426 editorMode: code 5427 exemplar: false 5428 expr: >- 5429 histogram_quantile(0.99, sum by (le, route) 5430 (rate(pyroscope_vcs_github_request_duration_bucket{cluster=~"$cluster", 5431 namespace=~"$namespace", container="querier"} [$__rate_interval]))) 5432 fullMetaSearch: false 5433 hide: false 5434 includeNullMetadata: true 5435 instant: false 5436 interval: '' 5437 legendFormat: __auto 5438 range: true 5439 refId: B 5440 useBackend: false 5441 title: GitHub API (P99) 5442 type: timeseries 5443 - datasource: 5444 type: prometheus 5445 uid: ${datasource} 5446 fieldConfig: 5447 defaults: 5448 color: 5449 mode: palette-classic 5450 custom: 5451 axisBorderShow: false 5452 axisCenteredZero: false 5453 axisColorMode: text 5454 axisLabel: '' 5455 axisPlacement: auto 5456 barAlignment: 0 5457 barWidthFactor: 0.6 5458 drawStyle: bars 5459 fillOpacity: 100 5460 gradientMode: none 5461 hideFrom: 5462 legend: false 5463 tooltip: false 5464 viz: false 5465 insertNulls: false 5466 lineInterpolation: linear 5467 lineWidth: 1 5468 pointSize: 5 5469 scaleDistribution: 5470 type: linear 5471 showPoints: auto 5472 spanNulls: false 5473 stacking: 5474 group: A 5475 mode: normal 5476 thresholdsStyle: 5477 mode: 'off' 5478 mappings: [] 5479 noValue: '0' 5480 thresholds: 5481 mode: absolute 5482 steps: 5483 - color: green 5484 unit: short 5485 overrides: [] 5486 gridPos: 5487 h: 8 5488 w: 12 5489 x: 12 5490 'y': 49 5491 id: 107 5492 options: 5493 legend: 5494 calcs: 5495 - sum 5496 displayMode: table 5497 placement: bottom 5498 showLegend: true 5499 tooltip: 5500 hideZeros: false 5501 mode: single 5502 sort: none 5503 pluginVersion: 12.1.0-88106 5504 targets: 5505 - datasource: 5506 type: prometheus 5507 uid: ${datasource} 5508 disableTextWrap: false 5509 editorMode: code 5510 exemplar: false 5511 expr: >- 5512 sum by (le, route, status_code) 5513 (rate(pyroscope_vcs_github_request_duration_count{cluster=~"$cluster", 5514 namespace=~"$namespace", container="querier"} [$__rate_interval])) 5515 fullMetaSearch: false 5516 hide: false 5517 includeNullMetadata: true 5518 instant: false 5519 interval: '' 5520 legendFormat: __auto 5521 range: true 5522 refId: B 5523 useBackend: false 5524 title: GitHub API (P99) 5525 type: timeseries 5526 preload: false 5527 refresh: 30s 5528 schemaVersion: 41 5529 tags: 5530 - pyroscope 5531 templating: 5532 list: 5533 - current: 5534 text: ops-cortex 5535 value: '000000134' 5536 includeAll: false 5537 label: Data Source 5538 name: datasource 5539 options: [] 5540 query: prometheus 5541 refresh: 1 5542 regex: '' 5543 type: datasource 5544 - current: 5545 text: Loki-Ops 5546 value: c-R8UWvVk 5547 includeAll: false 5548 name: loki_datasource 5549 options: [] 5550 query: loki 5551 refresh: 1 5552 regex: '' 5553 type: datasource 5554 - current: 5555 text: All 5556 value: 5557 - $__all 5558 datasource: 5559 type: prometheus 5560 uid: $datasource 5561 definition: label_values(pyroscope_build_info,cluster) 5562 includeAll: true 5563 label: cluster 5564 multi: true 5565 name: cluster 5566 options: [] 5567 query: 5568 labelFilters: [] 5569 query: label_values(pyroscope_build_info,cluster) 5570 refId: PrometheusVariableQueryEditor-VariableQuery 5571 refresh: 1 5572 regex: '' 5573 sort: 1 5574 type: query 5575 - current: 5576 text: 5577 - {{.Values.dashboards.namespace | quote}} 5578 value: 5579 - {{.Values.dashboards.namespace | quote}} 5580 datasource: 5581 type: prometheus 5582 uid: $datasource 5583 definition: label_values(pyroscope_build_info,namespace) 5584 includeAll: true 5585 label: namespace 5586 multi: true 5587 name: namespace 5588 options: [] 5589 query: 5590 labelFilters: [] 5591 query: label_values(pyroscope_build_info,namespace) 5592 refId: PrometheusVariableQueryEditor-VariableQuery 5593 refresh: 1 5594 regex: {{.Values.dashboards.namespaceRegex | quote}} 5595 sort: 1 5596 type: query 5597 time: 5598 from: now-1h 5599 to: now 5600 timepicker: {} 5601 timezone: '' 5602 title: Pyroscope / Operational 5603 version: 14 5604 5605 {{- end }}