github.com/grafana/pyroscope@v1.18.0/operations/monitoring/helm/pyroscope-monitoring/templates/_dashboard_v2-read-path.yaml (about) 1 {{- define "pyroscope-monitoring.dashboards.v2-read-path" -}} 2 {{- $dashboard := "v2-read-path" -}} 3 annotations: 4 list: 5 - builtIn: 1 6 datasource: 7 type: grafana 8 uid: '-- Grafana --' 9 enable: true 10 hide: true 11 iconColor: rgba(0, 211, 255, 1) 12 name: Annotations & Alerts 13 type: dashboard 14 - datasource: 15 type: loki 16 uid: ${loki_datasource} 17 enable: false 18 expr: >- 19 {cluster="$cluster", container="kube-diff-logger"} | json | 20 namespace_extracted="$namespace" 21 hide: false 22 iconColor: yellow 23 instant: false 24 name: K8s Changes 25 tagKeys: type,name_extracted,verb 26 textFormat: '{{"{{notes}}"}}' 27 titleFormat: '' 28 - datasource: 29 type: loki 30 uid: ${loki_datasource} 31 enable: false 32 expr: >- 33 {namespace="$namespace",container="eventrouter"} != "Preemption is not 34 helpful" != "Failed deploy model" | logfmt | type="Warning" 35 hide: false 36 iconColor: orange 37 instant: false 38 name: K8s Warnings 39 tagKeys: type,object_kind,object_name 40 textFormat: '{{"{{message}}"}}' 41 titleFormat: '' 42 - datasource: 43 type: loki 44 uid: ${loki_datasource} 45 enable: true 46 expr: >- 47 {namespace="$namespace",container="eventrouter"} | logfmt | 48 type!="Normal" | type!="Warning" 49 hide: false 50 iconColor: red 51 instant: false 52 name: K8s Errors 53 tagKeys: type,object_kind,object_name 54 textFormat: '{{"{{message}}"}}' 55 - datasource: 56 type: loki 57 uid: ${loki_datasource} 58 enable: false 59 expr: '{namespace="$namespace",container="eventrouter"} | logfmt' 60 hide: false 61 iconColor: blue 62 instant: false 63 name: K8s All events 64 tagKeys: type,object_kind,object_name 65 textFormat: '{{"{{message}}"}}' 66 editable: true 67 fiscalYearStartMonth: 0 68 graphTooltip: 1 69 id: 12593 70 links: 71 {{ .Values.dashboards.links.global | toYaml | nindent 2}} 72 {{- with (dig $dashboard (list) .Values.dashboards.links.perDashboard) }} 73 {{ . | toYaml | nindent 2}} 74 {{ end }} 75 panels: 76 - collapsed: true 77 gridPos: 78 h: 1 79 w: 24 80 x: 0 81 'y': 0 82 id: 68 83 panels: 84 - datasource: 85 type: loki 86 uid: ${loki_datasource} 87 description: >- 88 SelectMergeProfile filtered out by default as the API is used by 89 Canary Exporter 90 fieldConfig: 91 defaults: 92 color: 93 mode: thresholds 94 custom: 95 align: auto 96 cellOptions: 97 type: auto 98 filterable: true 99 inspect: false 100 mappings: [] 101 thresholds: 102 mode: absolute 103 steps: 104 - color: green 105 value: 0 106 - color: red 107 value: 80 108 overrides: 109 - matcher: 110 id: byName 111 options: pod 112 properties: 113 - id: custom.width 114 value: 280 115 - matcher: 116 id: byName 117 options: orgID 118 properties: 119 - id: custom.width 120 value: 86 121 - matcher: 122 id: byName 123 options: uri 124 properties: 125 - id: custom.width 126 value: 312 127 - matcher: 128 id: byName 129 options: Time 130 properties: 131 - id: custom.width 132 value: 179 133 - matcher: 134 id: byName 135 options: Trace ID 136 properties: 137 - id: custom.width 138 value: 318 139 - matcher: 140 id: byName 141 options: Endpoint 142 properties: 143 - id: custom.width 144 value: 420 145 - matcher: 146 id: byName 147 options: Pod 148 properties: 149 - id: custom.width 150 value: 379 151 - matcher: 152 id: byName 153 options: route 154 properties: 155 - id: custom.width 156 value: 347 157 - matcher: 158 id: byName 159 options: Route 160 properties: 161 - id: custom.width 162 value: 363 163 gridPos: 164 h: 8 165 w: 24 166 x: 0 167 'y': 1 168 id: 69 169 options: 170 cellHeight: sm 171 footer: 172 countRows: false 173 fields: '' 174 reducer: 175 - sum 176 show: false 177 showHeader: true 178 sortBy: 179 - desc: true 180 displayName: Time 181 pluginVersion: 12.1.0-90017 182 targets: 183 - datasource: 184 type: loki 185 uid: ${loki_datasource} 186 direction: backward 187 editorMode: code 188 expr: >- 189 {namespace=~"$namespace",cluster=~"$cluster",container="query-frontend"} 190 191 |= "route=/querier.v1." 192 193 | logfmt | line_format "tenant={{"{{.orgID}} status={{.status}}"}} 194 duration={{"{{.duration}} traceID={{.traceID}} {{.uri}}"}}" 195 queryType: range 196 refId: A 197 title: All Queries 198 transformations: 199 - id: extractFields 200 options: 201 delimiter: ',' 202 format: json 203 keepTime: false 204 replace: false 205 source: labels 206 - id: organize 207 options: 208 excludeByName: 209 Drone build for Grafana: true 210 Drone build for Grafana Enterprise: true 211 Line: true 212 Okta Logs: true 213 Profile TraceID: true 214 TraceID (json): true 215 __adaptive_logs_sampled__: true 216 app_kubernetes_io_component: true 217 app_kubernetes_io_instance: true 218 app_kubernetes_io_name: true 219 caller: true 220 cluster: true 221 container: true 222 debug: true 223 detected_level: true 224 id: true 225 job: true 226 labelTypes: true 227 labels: true 228 level: true 229 method: true 230 msg: true 231 name: true 232 pod_template_hash: true 233 request_body_size: true 234 service_name: true 235 status: false 236 stream: true 237 test_run_id: true 238 traceID: true 239 traceID (label): true 240 traceID 1: true 241 trace_id (labels): true 242 ts: true 243 tsNs: true 244 includeByName: {} 245 indexByName: 246 Drone build for Grafana: 30 247 Drone build for Grafana Enterprise: 31 248 Line: 4 249 Time: 1 250 TraceID (json): 8 251 TraceID (log line): 29 252 app_kubernetes_io_component: 11 253 app_kubernetes_io_instance: 12 254 app_kubernetes_io_name: 13 255 caller: 14 256 cluster: 15 257 container: 16 258 detected_level: 17 259 duration: 18 260 id: 7 261 job: 19 262 labelTypes: 6 263 labels: 0 264 level: 20 265 msg: 21 266 name: 22 267 namespace: 24 268 pod: 23 269 pod_template_hash: 25 270 route: 3 271 service_name: 26 272 stream: 27 273 tenant: 2 274 test_run_id: 10 275 traceID: 9 276 traceID (label): 32 277 trace_id (labels): 33 278 ts: 28 279 tsNs: 5 280 renameByName: 281 Line: '' 282 Time: '' 283 TraceID: Trace ID 284 duration: Duration 285 labels: '' 286 namespace: Namespace 287 orgID: Tenant 288 pod: Pod 289 pod_template_hash: '' 290 route: Route 291 status: Status 292 tenant: Tenant 293 uri: Endpoint 294 transparent: true 295 type: table 296 - datasource: 297 type: tempo 298 uid: ${tempo_datasource} 299 fieldConfig: 300 defaults: 301 color: 302 mode: thresholds 303 custom: 304 align: auto 305 cellOptions: 306 type: auto 307 wrapText: false 308 filterable: true 309 inspect: false 310 mappings: [] 311 thresholds: 312 mode: absolute 313 steps: 314 - color: green 315 value: 0 316 - color: red 317 value: 80 318 overrides: 319 - matcher: 320 id: byName 321 options: Trace Name 322 properties: 323 - id: custom.width 324 value: 455 325 - matcher: 326 id: byName 327 options: Trace ID 328 properties: 329 - id: custom.width 330 value: 311 331 gridPos: 332 h: 10 333 w: 24 334 x: 0 335 'y': 9 336 id: 70 337 options: 338 cellHeight: sm 339 footer: 340 countRows: false 341 enablePagination: false 342 fields: '' 343 reducer: 344 - sum 345 show: false 346 frameIndex: 0 347 showHeader: true 348 sortBy: 349 - desc: true 350 displayName: Start time 351 pluginVersion: 12.1.0-90017 352 targets: 353 - datasource: 354 type: tempo 355 uid: ${tempo_datasource} 356 filters: 357 - id: d5be7410 358 operator: '=' 359 scope: resource 360 tag: k8s.namespace.name 361 value: 362 - $namespace 363 valueType: string 364 - id: service-name 365 operator: '=' 366 scope: resource 367 tag: service.name 368 value: 369 - pyroscope-query-frontend 370 - pyroscope-query-backend 371 valueType: string 372 - id: min-duration 373 operator: '>' 374 tag: duration 375 value: 100ms 376 valueType: duration 377 - id: span-name 378 operator: '!=' 379 scope: span 380 tag: name 381 value: 382 - HTTP GET - debug_pprof 383 valueType: string 384 limit: 25 385 metricsQueryType: instant 386 queryType: traceqlSearch 387 refId: A 388 spss: 5 389 tableType: traces 390 title: Traces 391 transparent: true 392 type: table 393 title: Query Log 394 type: row 395 - collapsed: false 396 gridPos: 397 h: 1 398 w: 24 399 x: 0 400 'y': 1 401 id: 57 402 panels: [] 403 title: Metadata Query Service (metastore) 404 type: row 405 - datasource: 406 type: prometheus 407 uid: ${datasource} 408 description: >- 409 Raft node roles observed by each node. The view is not meant to be 100% 410 consistent, and only serves monitoring purposes 411 fieldConfig: 412 defaults: 413 color: 414 mode: continuous-GrYlRd 415 custom: 416 axisPlacement: auto 417 fillOpacity: 15 418 hideFrom: 419 legend: false 420 tooltip: false 421 viz: false 422 insertNulls: false 423 lineWidth: 1 424 spanNulls: false 425 fieldMinMax: false 426 mappings: 427 - options: 428 from: 0 429 result: 430 color: green 431 index: 0 432 text: Follower 433 to: 99 434 type: range 435 - options: 436 from: 100 437 result: 438 color: dark-red 439 index: 1 440 text: Leader 441 to: 1000000 442 type: range 443 thresholds: 444 mode: absolute 445 steps: 446 - color: dark-blue 447 value: 0 448 - color: light-red 449 value: 100 450 unit: short 451 overrides: [] 452 gridPos: 453 h: 7 454 w: 12 455 x: 0 456 'y': 2 457 id: 54 458 options: 459 alignValue: center 460 legend: 461 displayMode: list 462 placement: bottom 463 showLegend: false 464 mergeValues: true 465 rowHeight: 0.8 466 showValue: never 467 tooltip: 468 hideZeros: false 469 mode: multi 470 sort: asc 471 pluginVersion: 12.1.0-90017 472 targets: 473 - datasource: 474 type: prometheus 475 uid: ${datasource} 476 editorMode: code 477 exemplar: false 478 expr: |- 479 sum by (pod) ( 480 pyroscope_metastore_raft_state{namespace="$namespace",state="Follower"} or 481 pyroscope_metastore_raft_state{namespace="$namespace",state="Leader"} * 100 482 ) 483 format: time_series 484 hide: false 485 instant: false 486 legendFormat: '{{"{{label_name}}"}}' 487 range: true 488 refId: D 489 title: Raft Nodes 490 transparent: true 491 type: state-timeline 492 - datasource: 493 type: prometheus 494 uid: ${datasource} 495 description: |- 496 The latency of QueryMetadata endpoint observed by metastore nodes. 497 498 The only user is the query-frontend service. 499 fieldConfig: 500 defaults: 501 color: 502 mode: palette-classic 503 custom: 504 axisBorderShow: false 505 axisCenteredZero: false 506 axisColorMode: text 507 axisLabel: '' 508 axisPlacement: auto 509 barAlignment: 0 510 barWidthFactor: 0.6 511 drawStyle: line 512 fillOpacity: 0 513 gradientMode: none 514 hideFrom: 515 legend: false 516 tooltip: false 517 viz: false 518 insertNulls: false 519 lineInterpolation: linear 520 lineWidth: 1 521 pointSize: 5 522 scaleDistribution: 523 type: linear 524 showPoints: auto 525 spanNulls: false 526 stacking: 527 group: A 528 mode: none 529 thresholdsStyle: 530 mode: 'off' 531 mappings: [] 532 thresholds: 533 mode: absolute 534 steps: 535 - color: green 536 value: 0 537 - color: red 538 value: 80 539 unit: s 540 overrides: 541 - matcher: 542 id: byFrameRefID 543 options: B 544 properties: 545 - id: custom.drawStyle 546 value: points 547 - id: color 548 value: 549 fixedColor: red 550 mode: fixed 551 - id: custom.pointSize 552 value: 11 553 gridPos: 554 h: 7 555 w: 6 556 x: 12 557 'y': 2 558 id: 56 559 options: 560 legend: 561 calcs: [] 562 displayMode: list 563 placement: bottom 564 showLegend: true 565 tooltip: 566 hideZeros: false 567 mode: single 568 sort: none 569 pluginVersion: 12.1.0-90017 570 targets: 571 - datasource: 572 type: prometheus 573 uid: ${datasource} 574 editorMode: code 575 expr: >- 576 histogram_quantile(0.99, 577 sum(rate(pyroscope_request_duration_seconds_bucket{ 578 namespace="$namespace", 579 method="gRPC", 580 route="/metastore.v1.MetadataQueryService/QueryMetadata", 581 }[$__rate_interval])) by (le)) 582 hide: false 583 instant: false 584 legendFormat: p99 585 range: true 586 refId: D 587 - datasource: 588 type: prometheus 589 uid: ${datasource} 590 editorMode: code 591 expr: >- 592 histogram_quantile(0.95, 593 sum(rate(pyroscope_request_duration_seconds_bucket{ 594 namespace="$namespace", 595 method="gRPC", 596 route="/metastore.v1.MetadataQueryService/QueryMetadata", 597 }[$__rate_interval])) by (le)) 598 hide: false 599 instant: false 600 legendFormat: p95 601 range: true 602 refId: A 603 title: QueryMetadata Latency 604 transparent: true 605 type: timeseries 606 - datasource: 607 type: prometheus 608 uid: ${datasource} 609 description: |- 610 The latency of QueryMetadataLabels endpoint observed by metastore nodes. 611 612 The only user is the query-frontend service. 613 fieldConfig: 614 defaults: 615 color: 616 mode: palette-classic 617 custom: 618 axisBorderShow: false 619 axisCenteredZero: false 620 axisColorMode: text 621 axisLabel: '' 622 axisPlacement: auto 623 barAlignment: 0 624 barWidthFactor: 0.6 625 drawStyle: line 626 fillOpacity: 0 627 gradientMode: none 628 hideFrom: 629 legend: false 630 tooltip: false 631 viz: false 632 insertNulls: false 633 lineInterpolation: linear 634 lineWidth: 1 635 pointSize: 5 636 scaleDistribution: 637 type: linear 638 showPoints: auto 639 spanNulls: false 640 stacking: 641 group: A 642 mode: none 643 thresholdsStyle: 644 mode: 'off' 645 mappings: [] 646 thresholds: 647 mode: absolute 648 steps: 649 - color: green 650 value: 0 651 - color: red 652 value: 80 653 unit: s 654 overrides: 655 - matcher: 656 id: byFrameRefID 657 options: B 658 properties: 659 - id: custom.drawStyle 660 value: points 661 - id: color 662 value: 663 fixedColor: red 664 mode: fixed 665 - id: custom.pointSize 666 value: 11 667 gridPos: 668 h: 7 669 w: 6 670 x: 18 671 'y': 2 672 id: 55 673 options: 674 legend: 675 calcs: [] 676 displayMode: list 677 placement: bottom 678 showLegend: false 679 tooltip: 680 hideZeros: false 681 mode: single 682 sort: none 683 pluginVersion: 12.1.0-90017 684 targets: 685 - datasource: 686 type: prometheus 687 uid: ${datasource} 688 editorMode: code 689 expr: >- 690 histogram_quantile(0.99, 691 sum(rate(pyroscope_request_duration_seconds_bucket{ 692 namespace="$namespace", 693 method="gRPC", 694 route="/metastore.v1.MetadataQueryService/QueryMetadataLabels", 695 }[$__rate_interval])) by (le)) 696 hide: false 697 instant: false 698 legendFormat: p99 699 range: true 700 refId: D 701 - datasource: 702 type: prometheus 703 uid: ${datasource} 704 editorMode: code 705 expr: >- 706 histogram_quantile(0.95, 707 sum(rate(pyroscope_request_duration_seconds_bucket{ 708 namespace="$namespace", 709 method="gRPC", 710 route="/metastore.v1.MetadataQueryService/QueryMetadataLabels", 711 }[$__rate_interval])) by (le)) 712 hide: false 713 instant: false 714 legendFormat: p95 715 range: true 716 refId: A 717 title: QueryMetadataLabels Latency 718 transparent: true 719 type: timeseries 720 - datasource: 721 type: prometheus 722 uid: ${datasource} 723 description: The size of BoltDB snapshot taken by FSM 724 fieldConfig: 725 defaults: 726 color: 727 mode: palette-classic 728 custom: 729 axisBorderShow: false 730 axisCenteredZero: false 731 axisColorMode: text 732 axisLabel: '' 733 axisPlacement: auto 734 barAlignment: 0 735 barWidthFactor: 0.6 736 drawStyle: line 737 fillOpacity: 0 738 gradientMode: none 739 hideFrom: 740 legend: false 741 tooltip: false 742 viz: false 743 insertNulls: false 744 lineInterpolation: linear 745 lineWidth: 1 746 pointSize: 5 747 scaleDistribution: 748 type: linear 749 showPoints: auto 750 spanNulls: false 751 stacking: 752 group: A 753 mode: none 754 thresholdsStyle: 755 mode: 'off' 756 mappings: [] 757 thresholds: 758 mode: absolute 759 steps: 760 - color: green 761 value: 0 762 - color: red 763 value: 80 764 unit: decbytes 765 overrides: 766 - matcher: 767 id: byFrameRefID 768 options: B 769 properties: 770 - id: custom.drawStyle 771 value: points 772 - id: color 773 value: 774 fixedColor: red 775 mode: fixed 776 - id: custom.pointSize 777 value: 11 778 gridPos: 779 h: 7 780 w: 12 781 x: 0 782 'y': 9 783 id: 60 784 options: 785 legend: 786 calcs: [] 787 displayMode: list 788 placement: bottom 789 showLegend: false 790 tooltip: 791 hideZeros: false 792 mode: single 793 sort: none 794 pluginVersion: 12.1.0-90017 795 targets: 796 - datasource: 797 type: prometheus 798 uid: ${datasource} 799 editorMode: code 800 expr: >- 801 pyroscope_metastore_boltdb_persist_snapshot_size_bytes{namespace="$namespace"} 802 hide: false 803 instant: false 804 legendFormat: '{{"{{pod}}"}}' 805 range: true 806 refId: D 807 title: Snapshot Size 808 transparent: true 809 type: timeseries 810 - datasource: 811 type: prometheus 812 uid: ${datasource} 813 fieldConfig: 814 defaults: 815 color: 816 mode: palette-classic 817 custom: 818 axisBorderShow: false 819 axisCenteredZero: false 820 axisColorMode: text 821 axisLabel: '' 822 axisPlacement: auto 823 barAlignment: 0 824 barWidthFactor: 0.6 825 drawStyle: line 826 fillOpacity: 0 827 gradientMode: none 828 hideFrom: 829 legend: false 830 tooltip: false 831 viz: false 832 insertNulls: false 833 lineInterpolation: linear 834 lineWidth: 1 835 pointSize: 5 836 scaleDistribution: 837 type: linear 838 showPoints: auto 839 spanNulls: false 840 stacking: 841 group: A 842 mode: none 843 thresholdsStyle: 844 mode: 'off' 845 mappings: [] 846 thresholds: 847 mode: absolute 848 steps: 849 - color: green 850 value: 0 851 - color: red 852 value: 80 853 unit: ops 854 overrides: 855 - matcher: 856 id: byName 857 options: error 858 properties: 859 - id: color 860 value: 861 fixedColor: red 862 mode: fixed 863 - matcher: 864 id: byName 865 options: success 866 properties: 867 - id: color 868 value: 869 fixedColor: green 870 mode: fixed 871 gridPos: 872 h: 7 873 w: 6 874 x: 12 875 'y': 9 876 id: 58 877 options: 878 legend: 879 calcs: [] 880 displayMode: list 881 placement: bottom 882 showLegend: false 883 tooltip: 884 hideZeros: false 885 mode: single 886 sort: none 887 pluginVersion: 12.1.0-90017 888 targets: 889 - datasource: 890 type: prometheus 891 uid: ${datasource} 892 editorMode: code 893 expr: |- 894 sum(irate(pyroscope_request_duration_seconds_count{ 895 namespace="$namespace", 896 method="gRPC", 897 route="/metastore.v1.MetadataQueryService/QueryMetadata", 898 }[$__rate_interval])) by (status_code) 899 hide: false 900 instant: false 901 legendFormat: __auto 902 range: true 903 refId: D 904 title: '' 905 transparent: true 906 type: timeseries 907 - datasource: 908 type: prometheus 909 uid: ${datasource} 910 fieldConfig: 911 defaults: 912 color: 913 mode: palette-classic 914 custom: 915 axisBorderShow: false 916 axisCenteredZero: false 917 axisColorMode: text 918 axisLabel: '' 919 axisPlacement: auto 920 barAlignment: 0 921 barWidthFactor: 0.6 922 drawStyle: line 923 fillOpacity: 0 924 gradientMode: none 925 hideFrom: 926 legend: false 927 tooltip: false 928 viz: false 929 insertNulls: false 930 lineInterpolation: linear 931 lineWidth: 1 932 pointSize: 5 933 scaleDistribution: 934 type: linear 935 showPoints: auto 936 spanNulls: false 937 stacking: 938 group: A 939 mode: none 940 thresholdsStyle: 941 mode: 'off' 942 mappings: [] 943 thresholds: 944 mode: absolute 945 steps: 946 - color: green 947 value: 0 948 - color: red 949 value: 80 950 unit: ops 951 overrides: 952 - matcher: 953 id: byName 954 options: error 955 properties: 956 - id: color 957 value: 958 fixedColor: red 959 mode: fixed 960 - matcher: 961 id: byName 962 options: success 963 properties: 964 - id: color 965 value: 966 fixedColor: green 967 mode: fixed 968 gridPos: 969 h: 7 970 w: 6 971 x: 18 972 'y': 9 973 id: 59 974 options: 975 legend: 976 calcs: [] 977 displayMode: list 978 placement: bottom 979 showLegend: false 980 tooltip: 981 hideZeros: false 982 mode: single 983 sort: none 984 pluginVersion: 12.1.0-90017 985 targets: 986 - datasource: 987 type: prometheus 988 uid: ${datasource} 989 editorMode: code 990 expr: |- 991 sum(irate(pyroscope_request_duration_seconds_count{ 992 namespace="$namespace", 993 method="gRPC", 994 route="/metastore.v1.MetadataQueryService/QueryMetadataLabels", 995 }[$__rate_interval])) by (status_code) 996 hide: false 997 instant: false 998 legendFormat: __auto 999 range: true 1000 refId: D 1001 title: '' 1002 transparent: true 1003 type: timeseries 1004 - collapsed: false 1005 gridPos: 1006 h: 1 1007 w: 24 1008 x: 0 1009 'y': 16 1010 id: 63 1011 panels: [] 1012 title: Query Frontend 1013 type: row 1014 - datasource: 1015 type: prometheus 1016 uid: $datasource 1017 description: Query rate observed by cortex-gw 1018 fieldConfig: 1019 defaults: 1020 color: 1021 mode: palette-classic 1022 custom: 1023 axisBorderShow: false 1024 axisCenteredZero: true 1025 axisColorMode: text 1026 axisLabel: '' 1027 axisPlacement: auto 1028 axisSoftMin: 0 1029 barAlignment: 0 1030 barWidthFactor: 0.6 1031 drawStyle: line 1032 fillOpacity: 0 1033 gradientMode: none 1034 hideFrom: 1035 legend: false 1036 tooltip: false 1037 viz: false 1038 insertNulls: false 1039 lineInterpolation: linear 1040 lineStyle: 1041 fill: solid 1042 lineWidth: 1 1043 pointSize: 1 1044 scaleDistribution: 1045 type: linear 1046 showPoints: never 1047 spanNulls: false 1048 stacking: 1049 group: A 1050 mode: none 1051 thresholdsStyle: 1052 mode: 'off' 1053 mappings: [] 1054 min: 0 1055 thresholds: 1056 mode: absolute 1057 steps: 1058 - color: green 1059 value: 0 1060 - color: red 1061 value: 80 1062 unit: reqps 1063 overrides: [] 1064 gridPos: 1065 h: 9 1066 w: 12 1067 x: 0 1068 'y': 17 1069 id: 67 1070 options: 1071 dataLinks: [] 1072 legend: 1073 calcs: 1074 - max 1075 - mean 1076 displayMode: table 1077 placement: right 1078 showLegend: true 1079 tooltip: 1080 hideZeros: true 1081 mode: multi 1082 sort: none 1083 pluginVersion: 12.1.0-90017 1084 targets: 1085 - datasource: 1086 uid: $datasource 1087 editorMode: code 1088 expr: >- 1089 sum by (status_code) 1090 (rate(pyroscope_request_duration_seconds_count{cluster=~"$cluster", 1091 {{ include "pyroscope-monitoring.dashboards-ingest-selector" . }}, namespace=~"$namespace", 1092 route=~".*pyroscope_render.*|.*pyroscope_label.*|.*querierservice.*"}[5m])) 1093 legendFormat: '{{"{{status_code}}"}} ' 1094 range: true 1095 refId: A 1096 title: Query rate 1097 transparent: true 1098 type: timeseries 1099 - datasource: 1100 type: prometheus 1101 uid: ${datasource} 1102 description: Query duration observed by cortex-gw 1103 fieldConfig: 1104 defaults: 1105 color: 1106 mode: palette-classic 1107 custom: 1108 axisBorderShow: false 1109 axisCenteredZero: false 1110 axisColorMode: text 1111 axisLabel: '' 1112 axisPlacement: auto 1113 barAlignment: 0 1114 barWidthFactor: 0.6 1115 drawStyle: line 1116 fillOpacity: 0 1117 gradientMode: none 1118 hideFrom: 1119 legend: false 1120 tooltip: false 1121 viz: false 1122 insertNulls: false 1123 lineInterpolation: linear 1124 lineWidth: 1 1125 pointSize: 5 1126 scaleDistribution: 1127 type: linear 1128 showPoints: auto 1129 spanNulls: false 1130 stacking: 1131 group: A 1132 mode: none 1133 thresholdsStyle: 1134 mode: 'off' 1135 mappings: [] 1136 thresholds: 1137 mode: absolute 1138 steps: 1139 - color: green 1140 value: 0 1141 - color: red 1142 value: 80 1143 unit: s 1144 overrides: 1145 - matcher: 1146 id: byFrameRefID 1147 options: B 1148 properties: 1149 - id: custom.drawStyle 1150 value: points 1151 - id: color 1152 value: 1153 fixedColor: red 1154 mode: fixed 1155 - id: custom.pointSize 1156 value: 11 1157 gridPos: 1158 h: 9 1159 w: 12 1160 x: 12 1161 'y': 17 1162 id: 66 1163 options: 1164 legend: 1165 calcs: 1166 - max 1167 displayMode: table 1168 placement: right 1169 showLegend: true 1170 sortBy: Name 1171 sortDesc: true 1172 tooltip: 1173 hideZeros: false 1174 mode: single 1175 sort: none 1176 pluginVersion: 12.1.0-90017 1177 targets: 1178 - datasource: 1179 type: prometheus 1180 uid: ${datasource} 1181 editorMode: code 1182 expr: >- 1183 histogram_quantile(0.99, sum by (le,route) ( 1184 rate(pyroscope_request_duration_seconds_bucket{{ "{" }}{{ include "pyroscope-monitoring.dashboards-ingest-selector" . }}, namespace=~"$namespace", 1185 cluster=~"$cluster",route=~".*pyroscope_render.*|.*pyroscope_label.*|.*querierservice.*"}[$__rate_interval]))) 1186 hide: false 1187 instant: false 1188 legendFormat: __auto 1189 range: true 1190 refId: D 1191 title: Query Duartion 1192 transparent: true 1193 type: timeseries 1194 - collapsed: false 1195 gridPos: 1196 h: 1 1197 w: 24 1198 x: 0 1199 'y': 26 1200 id: 61 1201 panels: [] 1202 title: Query Backend 1203 type: row 1204 - datasource: 1205 type: prometheus 1206 uid: ${datasource} 1207 description: > 1208 Number of query-backend invocations. 1209 1210 1211 Errors are expected: usually those indicate that the query is throttled 1212 with the concurrency limiter. 1213 1214 1215 If errors are presents in the query-frontend/gateway, the query is failed. 1216 fieldConfig: 1217 defaults: 1218 color: 1219 mode: palette-classic 1220 custom: 1221 axisBorderShow: false 1222 axisCenteredZero: false 1223 axisColorMode: text 1224 axisLabel: '' 1225 axisPlacement: auto 1226 barAlignment: 0 1227 barWidthFactor: 0.6 1228 drawStyle: line 1229 fillOpacity: 0 1230 gradientMode: none 1231 hideFrom: 1232 legend: false 1233 tooltip: false 1234 viz: false 1235 insertNulls: false 1236 lineInterpolation: linear 1237 lineWidth: 1 1238 pointSize: 5 1239 scaleDistribution: 1240 type: linear 1241 showPoints: auto 1242 spanNulls: false 1243 stacking: 1244 group: A 1245 mode: none 1246 thresholdsStyle: 1247 mode: 'off' 1248 mappings: [] 1249 thresholds: 1250 mode: absolute 1251 steps: 1252 - color: green 1253 value: 0 1254 - color: red 1255 value: 80 1256 unit: ops 1257 overrides: 1258 - matcher: 1259 id: byName 1260 options: error 1261 properties: 1262 - id: color 1263 value: 1264 fixedColor: red 1265 mode: fixed 1266 - matcher: 1267 id: byName 1268 options: success 1269 properties: 1270 - id: color 1271 value: 1272 fixedColor: green 1273 mode: fixed 1274 gridPos: 1275 h: 8 1276 w: 6 1277 x: 0 1278 'y': 27 1279 id: 64 1280 options: 1281 legend: 1282 calcs: [] 1283 displayMode: list 1284 placement: bottom 1285 showLegend: false 1286 tooltip: 1287 hideZeros: false 1288 mode: single 1289 sort: none 1290 pluginVersion: 12.1.0-90017 1291 targets: 1292 - datasource: 1293 type: prometheus 1294 uid: ${datasource} 1295 editorMode: code 1296 expr: |- 1297 sum(irate(pyroscope_request_duration_seconds_count{ 1298 namespace="$namespace", 1299 method="gRPC", 1300 route="/query.v1.QueryBackendService/Invoke", 1301 }[$__rate_interval])) by (status_code) 1302 hide: false 1303 instant: false 1304 legendFormat: __auto 1305 range: true 1306 refId: D 1307 title: Invocations 1308 transparent: true 1309 type: timeseries 1310 - datasource: 1311 type: prometheus 1312 uid: ${datasource} 1313 description: QueryBackend.Invoke latency observed by query-backend instances 1314 fieldConfig: 1315 defaults: 1316 color: 1317 mode: palette-classic 1318 custom: 1319 axisBorderShow: false 1320 axisCenteredZero: false 1321 axisColorMode: text 1322 axisLabel: '' 1323 axisPlacement: auto 1324 barAlignment: 0 1325 barWidthFactor: 0.6 1326 drawStyle: line 1327 fillOpacity: 0 1328 gradientMode: none 1329 hideFrom: 1330 legend: false 1331 tooltip: false 1332 viz: false 1333 insertNulls: false 1334 lineInterpolation: linear 1335 lineWidth: 1 1336 pointSize: 5 1337 scaleDistribution: 1338 type: linear 1339 showPoints: auto 1340 spanNulls: false 1341 stacking: 1342 group: A 1343 mode: none 1344 thresholdsStyle: 1345 mode: 'off' 1346 mappings: [] 1347 thresholds: 1348 mode: absolute 1349 steps: 1350 - color: green 1351 value: 0 1352 - color: red 1353 value: 80 1354 unit: s 1355 overrides: 1356 - matcher: 1357 id: byFrameRefID 1358 options: B 1359 properties: 1360 - id: custom.drawStyle 1361 value: points 1362 - id: color 1363 value: 1364 fixedColor: red 1365 mode: fixed 1366 - id: custom.pointSize 1367 value: 11 1368 gridPos: 1369 h: 8 1370 w: 6 1371 x: 6 1372 'y': 27 1373 id: 62 1374 options: 1375 legend: 1376 calcs: [] 1377 displayMode: list 1378 placement: bottom 1379 showLegend: false 1380 tooltip: 1381 hideZeros: false 1382 mode: single 1383 sort: none 1384 pluginVersion: 12.1.0-90017 1385 targets: 1386 - datasource: 1387 type: prometheus 1388 uid: ${datasource} 1389 editorMode: code 1390 expr: >- 1391 histogram_quantile(0.99, 1392 sum(rate(pyroscope_request_duration_seconds_bucket{ 1393 namespace="$namespace", 1394 method="gRPC", 1395 route="/query.v1.QueryBackendService/Invoke", 1396 }[$__rate_interval])) by (le)) 1397 hide: false 1398 instant: false 1399 legendFormat: p99 1400 range: true 1401 refId: D 1402 - datasource: 1403 type: prometheus 1404 uid: ${datasource} 1405 editorMode: code 1406 expr: >- 1407 histogram_quantile(0.95, 1408 sum(rate(pyroscope_request_duration_seconds_bucket{ 1409 namespace="$namespace", 1410 method="gRPC", 1411 route="/query.v1.QueryBackendService/Invoke", 1412 }[$__rate_interval])) by (le)) 1413 hide: false 1414 instant: false 1415 legendFormat: p95 1416 range: true 1417 refId: A 1418 title: Invocation Duration 1419 transparent: true 1420 type: timeseries 1421 - datasource: 1422 type: prometheus 1423 uid: ${datasource} 1424 description: QueryBackend.Invokeduration observed by query-backend instances 1425 fieldConfig: 1426 defaults: 1427 custom: 1428 hideFrom: 1429 legend: false 1430 tooltip: false 1431 viz: false 1432 scaleDistribution: 1433 type: linear 1434 overrides: 1435 - matcher: 1436 id: byFrameRefID 1437 options: B 1438 properties: [] 1439 gridPos: 1440 h: 8 1441 w: 6 1442 x: 12 1443 'y': 27 1444 id: 71 1445 options: 1446 calculate: false 1447 cellGap: 0 1448 cellValues: {} 1449 color: 1450 exponent: 0.5 1451 fill: dark-orange 1452 mode: scheme 1453 reverse: false 1454 scale: exponential 1455 scheme: Rainbow 1456 steps: 64 1457 exemplars: 1458 color: rgba(255,0,255,0.7) 1459 filterValues: 1460 le: 1.e-9 1461 legend: 1462 show: true 1463 rowsFrame: 1464 layout: auto 1465 tooltip: 1466 mode: single 1467 showColorScale: false 1468 yHistogram: false 1469 yAxis: 1470 axisPlacement: left 1471 reverse: false 1472 unit: s 1473 pluginVersion: 12.1.0-90017 1474 targets: 1475 - datasource: 1476 type: prometheus 1477 uid: ${datasource} 1478 editorMode: code 1479 expr: |- 1480 sum (rate(pyroscope_request_duration_seconds{ 1481 namespace="$namespace", 1482 method="gRPC", 1483 route="/query.v1.QueryBackendService/Invoke", 1484 status_code="success" 1485 }[$__rate_interval])) 1486 hide: false 1487 instant: false 1488 legendFormat: p99 1489 range: true 1490 refId: D 1491 title: Heatmap 1492 transparent: true 1493 type: heatmap 1494 - datasource: 1495 type: prometheus 1496 uid: ${datasource} 1497 description: '' 1498 fieldConfig: 1499 defaults: 1500 color: 1501 mode: palette-classic 1502 custom: 1503 axisBorderShow: false 1504 axisCenteredZero: false 1505 axisColorMode: text 1506 axisLabel: '' 1507 axisPlacement: auto 1508 barAlignment: 0 1509 barWidthFactor: 0.6 1510 drawStyle: line 1511 fillOpacity: 0 1512 gradientMode: none 1513 hideFrom: 1514 legend: false 1515 tooltip: false 1516 viz: false 1517 insertNulls: false 1518 lineInterpolation: linear 1519 lineWidth: 1 1520 pointSize: 5 1521 scaleDistribution: 1522 type: linear 1523 showPoints: auto 1524 spanNulls: false 1525 stacking: 1526 group: A 1527 mode: none 1528 thresholdsStyle: 1529 mode: 'off' 1530 mappings: [] 1531 thresholds: 1532 mode: absolute 1533 steps: 1534 - color: green 1535 value: 0 1536 - color: red 1537 value: 80 1538 unit: s 1539 overrides: 1540 - matcher: 1541 id: byFrameRefID 1542 options: B 1543 properties: 1544 - id: custom.drawStyle 1545 value: points 1546 - id: color 1547 value: 1548 fixedColor: red 1549 mode: fixed 1550 - id: custom.pointSize 1551 value: 11 1552 gridPos: 1553 h: 8 1554 w: 6 1555 x: 18 1556 'y': 27 1557 id: 65 1558 options: 1559 legend: 1560 calcs: [] 1561 displayMode: list 1562 placement: bottom 1563 showLegend: false 1564 tooltip: 1565 hideZeros: false 1566 mode: single 1567 sort: none 1568 pluginVersion: 12.1.0-90017 1569 targets: 1570 - datasource: 1571 type: prometheus 1572 uid: ${datasource} 1573 editorMode: code 1574 expr: >- 1575 histogram_quantile(0.99, 1576 sum(rate(pyroscope_request_duration_seconds_bucket{ 1577 namespace="$namespace", 1578 method="gRPC", 1579 route="/query.v1.QueryBackendService/Invoke", 1580 }[$__rate_interval])) by (le,pod)) 1581 hide: false 1582 instant: false 1583 legendFormat: '{{"{{pod}}"}}' 1584 range: true 1585 refId: D 1586 title: Breakdown 1587 transparent: true 1588 type: timeseries 1589 - collapsed: false 1590 gridPos: 1591 h: 1 1592 w: 24 1593 x: 0 1594 'y': 35 1595 id: 32 1596 panels: [] 1597 title: Resource Usage 1598 type: row 1599 - datasource: 1600 type: datasource 1601 uid: '-- Mixed --' 1602 fieldConfig: 1603 defaults: 1604 color: 1605 mode: palette-classic 1606 custom: 1607 axisBorderShow: false 1608 axisCenteredZero: false 1609 axisColorMode: text 1610 axisLabel: '' 1611 axisPlacement: auto 1612 axisSoftMin: 0 1613 barAlignment: 1 1614 barWidthFactor: 1 1615 drawStyle: line 1616 fillOpacity: 0 1617 gradientMode: none 1618 hideFrom: 1619 legend: false 1620 tooltip: false 1621 viz: false 1622 insertNulls: false 1623 lineInterpolation: linear 1624 lineStyle: 1625 fill: solid 1626 lineWidth: 1 1627 pointSize: 5 1628 scaleDistribution: 1629 type: linear 1630 showPoints: auto 1631 spanNulls: false 1632 stacking: 1633 group: A 1634 mode: none 1635 thresholdsStyle: 1636 mode: 'off' 1637 fieldMinMax: false 1638 mappings: [] 1639 min: 0 1640 thresholds: 1641 mode: absolute 1642 steps: 1643 - color: green 1644 value: 0 1645 overrides: 1646 - matcher: 1647 id: byFrameRefID 1648 options: A 1649 properties: 1650 - id: custom.axisColorMode 1651 value: text 1652 - id: color 1653 value: 1654 fixedColor: orange 1655 mode: fixed 1656 - id: custom.lineWidth 1657 value: 1 1658 - id: custom.lineStyle 1659 value: 1660 dash: 1661 - 10 1662 - 10 1663 fill: dash 1664 - id: custom.stacking 1665 value: 1666 group: A 1667 mode: none 1668 gridPos: 1669 h: 8 1670 w: 6 1671 x: 0 1672 'y': 36 1673 id: 29 1674 options: 1675 legend: 1676 calcs: 1677 - last 1678 - max 1679 displayMode: table 1680 placement: bottom 1681 showLegend: false 1682 sortBy: Max 1683 sortDesc: true 1684 tooltip: 1685 hideZeros: false 1686 mode: multi 1687 sort: desc 1688 pluginVersion: 12.1.0-90017 1689 targets: 1690 - datasource: 1691 type: prometheus 1692 uid: ${datasource} 1693 editorMode: code 1694 expr: >- 1695 sum by (pod) 1696 (irate(container_cpu_usage_seconds_total{namespace="$namespace",container="query-backend"}[$__rate_interval])) 1697 legendFormat: __auto 1698 range: true 1699 refId: by pod 1700 - datasource: 1701 type: prometheus 1702 uid: ${datasource} 1703 editorMode: code 1704 expr: |- 1705 max( 1706 kube_pod_container_resource_requests{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, namespace="$namespace", resource="cpu"} 1707 * on(namespace,pod) 1708 group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{namespace="$namespace", workload=~".*-query-backend"} 1709 ) 1710 hide: false 1711 legendFormat: Request 1712 range: true 1713 refId: A 1714 - datasource: 1715 type: prometheus 1716 uid: ${datasource} 1717 editorMode: code 1718 expr: |- 1719 max( 1720 kube_pod_container_resource_limits{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, namespace="$namespace", resource="cpu"} 1721 * on(namespace,pod) 1722 group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{namespace="$namespace", workload=~".*-query-backend"} 1723 ) 1724 hide: false 1725 legendFormat: Limit 1726 range: true 1727 refId: B 1728 title: CPU Usage 1729 transparent: true 1730 type: timeseries 1731 - datasource: 1732 type: datasource 1733 uid: '-- Mixed --' 1734 fieldConfig: 1735 defaults: 1736 color: 1737 mode: palette-classic 1738 custom: 1739 axisBorderShow: false 1740 axisCenteredZero: false 1741 axisColorMode: text 1742 axisLabel: '' 1743 axisPlacement: auto 1744 axisSoftMin: 0 1745 barAlignment: 1 1746 barWidthFactor: 1 1747 drawStyle: line 1748 fillOpacity: 0 1749 gradientMode: none 1750 hideFrom: 1751 legend: false 1752 tooltip: false 1753 viz: false 1754 insertNulls: false 1755 lineInterpolation: linear 1756 lineStyle: 1757 fill: solid 1758 lineWidth: 1 1759 pointSize: 5 1760 scaleDistribution: 1761 type: linear 1762 showPoints: auto 1763 spanNulls: false 1764 stacking: 1765 group: A 1766 mode: normal 1767 thresholdsStyle: 1768 mode: 'off' 1769 fieldMinMax: false 1770 mappings: [] 1771 min: 0 1772 thresholds: 1773 mode: absolute 1774 steps: 1775 - color: green 1776 value: 0 1777 overrides: 1778 - matcher: 1779 id: byFrameRefID 1780 options: A 1781 properties: 1782 - id: custom.axisColorMode 1783 value: text 1784 - id: color 1785 value: 1786 fixedColor: red 1787 mode: fixed 1788 - id: custom.stacking 1789 value: 1790 group: A 1791 mode: none 1792 - id: custom.lineWidth 1793 value: 2 1794 - matcher: 1795 id: byName 1796 options: Request 1797 properties: 1798 - id: custom.stacking 1799 value: 1800 group: A 1801 mode: none 1802 - id: color 1803 value: 1804 fixedColor: yellow 1805 mode: fixed 1806 gridPos: 1807 h: 8 1808 w: 6 1809 x: 6 1810 'y': 36 1811 id: 72 1812 options: 1813 legend: 1814 calcs: 1815 - last 1816 - max 1817 displayMode: table 1818 placement: right 1819 showLegend: false 1820 sortBy: Max 1821 sortDesc: true 1822 tooltip: 1823 hideZeros: false 1824 mode: multi 1825 sort: desc 1826 pluginVersion: 12.1.0-90017 1827 targets: 1828 - datasource: 1829 type: prometheus 1830 uid: ${datasource} 1831 editorMode: code 1832 expr: >- 1833 sum by (pod) 1834 (irate(container_cpu_usage_seconds_total{namespace="$namespace",container="query-backend"}[$__rate_interval])) 1835 legendFormat: __auto 1836 range: true 1837 refId: by pod 1838 - datasource: 1839 type: prometheus 1840 uid: ${datasource} 1841 editorMode: code 1842 expr: >- 1843 sum 1844 (irate(container_cpu_usage_seconds_total{namespace="$namespace",container="query-backend"}[$__rate_interval])) 1845 hide: false 1846 legendFormat: Total 1847 range: true 1848 refId: A 1849 - datasource: 1850 type: prometheus 1851 uid: ${datasource} 1852 editorMode: code 1853 expr: >- 1854 sum(kube_pod_container_resource_requests{namespace="$namespace", 1855 resource="cpu",container="query-backend"}) 1856 hide: false 1857 legendFormat: Request 1858 range: true 1859 refId: B 1860 title: CPU Usage (stack) 1861 transparent: true 1862 type: timeseries 1863 - datasource: 1864 type: datasource 1865 uid: '-- Mixed --' 1866 fieldConfig: 1867 defaults: 1868 color: 1869 mode: palette-classic 1870 custom: 1871 axisBorderShow: false 1872 axisCenteredZero: false 1873 axisColorMode: text 1874 axisLabel: '' 1875 axisPlacement: auto 1876 axisSoftMin: 0 1877 barAlignment: 0 1878 barWidthFactor: 0.6 1879 drawStyle: line 1880 fillOpacity: 0 1881 gradientMode: none 1882 hideFrom: 1883 legend: false 1884 tooltip: false 1885 viz: false 1886 insertNulls: false 1887 lineInterpolation: linear 1888 lineStyle: 1889 fill: solid 1890 lineWidth: 1 1891 pointSize: 5 1892 scaleDistribution: 1893 type: linear 1894 showPoints: auto 1895 spanNulls: false 1896 stacking: 1897 group: A 1898 mode: none 1899 thresholdsStyle: 1900 mode: dashed 1901 mappings: [] 1902 thresholds: 1903 mode: absolute 1904 steps: 1905 - color: green 1906 value: 0 1907 unit: bytes 1908 overrides: 1909 - matcher: 1910 id: byName 1911 options: Request 1912 properties: 1913 - id: color 1914 value: 1915 fixedColor: yellow 1916 mode: fixed 1917 - matcher: 1918 id: byName 1919 options: Limit 1920 properties: 1921 - id: color 1922 value: 1923 fixedColor: red 1924 mode: fixed 1925 gridPos: 1926 h: 8 1927 w: 12 1928 x: 12 1929 'y': 36 1930 id: 30 1931 options: 1932 legend: 1933 calcs: 1934 - last 1935 - max 1936 displayMode: table 1937 placement: bottom 1938 showLegend: false 1939 sortBy: Max 1940 sortDesc: true 1941 tooltip: 1942 hideZeros: false 1943 mode: multi 1944 sort: desc 1945 pluginVersion: 12.1.0-90017 1946 targets: 1947 - datasource: 1948 type: prometheus 1949 uid: ${datasource} 1950 editorMode: code 1951 expr: >- 1952 sum(container_memory_working_set_bytes{namespace="$namespace", 1953 container="query-backend", image!=""}) by (pod) 1954 hide: false 1955 legendFormat: __auto 1956 range: true 1957 refId: A 1958 - datasource: 1959 type: prometheus 1960 uid: ${datasource} 1961 editorMode: code 1962 expr: >- 1963 max(kube_pod_container_resource_requests{namespace="$namespace", 1964 resource="memory",container="query-backend"}) 1965 hide: false 1966 legendFormat: Request 1967 range: true 1968 refId: B 1969 - datasource: 1970 type: prometheus 1971 uid: ${datasource} 1972 editorMode: code 1973 expr: >- 1974 max (kube_pod_container_resource_limits{namespace="$namespace", 1975 resource="memory",container="query-backend"}) 1976 hide: false 1977 legendFormat: Limit 1978 range: true 1979 refId: C 1980 title: Memory Usage 1981 transparent: true 1982 type: timeseries 1983 - datasource: 1984 type: datasource 1985 uid: '-- Mixed --' 1986 fieldConfig: 1987 defaults: 1988 color: 1989 mode: palette-classic 1990 custom: 1991 axisBorderShow: false 1992 axisCenteredZero: false 1993 axisColorMode: text 1994 axisLabel: '' 1995 axisPlacement: auto 1996 barAlignment: 0 1997 barWidthFactor: 0.6 1998 drawStyle: line 1999 fillOpacity: 0 2000 gradientMode: none 2001 hideFrom: 2002 legend: false 2003 tooltip: false 2004 viz: false 2005 insertNulls: false 2006 lineInterpolation: linear 2007 lineWidth: 1 2008 pointSize: 5 2009 scaleDistribution: 2010 type: linear 2011 showPoints: auto 2012 spanNulls: false 2013 stacking: 2014 group: A 2015 mode: none 2016 thresholdsStyle: 2017 mode: 'off' 2018 mappings: [] 2019 min: 0 2020 thresholds: 2021 mode: absolute 2022 steps: 2023 - color: green 2024 value: 0 2025 unit: ops 2026 overrides: [] 2027 gridPos: 2028 h: 8 2029 w: 12 2030 x: 0 2031 'y': 44 2032 id: 31 2033 options: 2034 legend: 2035 calcs: 2036 - mean 2037 - max 2038 - last 2039 displayMode: table 2040 placement: bottom 2041 showLegend: false 2042 tooltip: 2043 hideZeros: false 2044 mode: multi 2045 sort: none 2046 pluginVersion: 12.1.0-90017 2047 targets: 2048 - datasource: 2049 type: prometheus 2050 uid: ${datasource} 2051 editorMode: code 2052 expr: >- 2053 sum by(operation) 2054 (irate(objstore_bucket_operations_total{namespace="$namespace",container="query-backend"}[$__rate_interval])) 2055 legendFormat: __auto 2056 range: true 2057 refId: A 2058 title: Storage I/O 2059 transparent: true 2060 type: timeseries 2061 - datasource: 2062 type: datasource 2063 uid: '-- Mixed --' 2064 fieldConfig: 2065 defaults: 2066 color: 2067 mode: palette-classic 2068 custom: 2069 axisBorderShow: false 2070 axisCenteredZero: false 2071 axisColorMode: text 2072 axisLabel: '' 2073 axisPlacement: auto 2074 barAlignment: 1 2075 barWidthFactor: 1 2076 drawStyle: line 2077 fillOpacity: 10 2078 gradientMode: none 2079 hideFrom: 2080 legend: false 2081 tooltip: false 2082 viz: false 2083 insertNulls: false 2084 lineInterpolation: linear 2085 lineStyle: 2086 fill: solid 2087 lineWidth: 1 2088 pointSize: 5 2089 scaleDistribution: 2090 type: linear 2091 showPoints: auto 2092 spanNulls: false 2093 stacking: 2094 group: A 2095 mode: normal 2096 thresholdsStyle: 2097 mode: 'off' 2098 fieldMinMax: false 2099 mappings: [] 2100 thresholds: 2101 mode: absolute 2102 steps: 2103 - color: green 2104 value: 0 2105 unit: Bps 2106 overrides: [] 2107 gridPos: 2108 h: 8 2109 w: 12 2110 x: 12 2111 'y': 44 2112 id: 53 2113 options: 2114 legend: 2115 calcs: 2116 - last 2117 - max 2118 displayMode: table 2119 placement: bottom 2120 showLegend: false 2121 sortBy: Max 2122 sortDesc: true 2123 tooltip: 2124 hideZeros: false 2125 mode: multi 2126 sort: desc 2127 pluginVersion: 12.1.0-90017 2128 targets: 2129 - datasource: 2130 type: prometheus 2131 uid: ${datasource} 2132 editorMode: code 2133 expr: >- 2134 (sum(irate(container_network_transmit_bytes_total{{ "{" }}{{ .Values.dashboards.cadvisorSelector }}, 2135 cluster=~"$cluster", namespace=~"$namespace"}[$__rate_interval]) * on 2136 (namespace,pod) 2137 2138 group_left(workload,workload_type) 2139 namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", 2140 namespace=~"$namespace", workload=~".*query-backend.*"}) by 2141 (namespace,pod)) * -1 2142 hide: false 2143 legendFormat: tx {{"{{pod}}"}} 2144 range: true 2145 refId: A 2146 - datasource: 2147 type: prometheus 2148 uid: ${datasource} 2149 editorMode: code 2150 expr: >- 2151 (sum(irate(container_network_receive_bytes_total{{ "{" }}{{ .Values.dashboards.cadvisorSelector }}, 2152 cluster=~"$cluster", namespace=~"$namespace"}[$__rate_interval]) * on 2153 (namespace,pod) 2154 2155 group_left(workload,workload_type) 2156 namespace_workload_pod:kube_pod_owner:relabel{cluster=~"$cluster", 2157 namespace=~"$namespace", workload=~".*query-backend.*"}) by 2158 (namespace,pod)) 2159 hide: false 2160 legendFormat: rx {{"{{pod}}"}} 2161 range: true 2162 refId: B 2163 title: Network 2164 transparent: true 2165 type: timeseries 2166 - collapsed: true 2167 gridPos: 2168 h: 1 2169 w: 24 2170 x: 0 2171 'y': 52 2172 id: 43 2173 panels: 2174 - datasource: 2175 type: datasource 2176 uid: '-- Mixed --' 2177 description: Number of OS threads created 2178 fieldConfig: 2179 defaults: 2180 color: 2181 mode: palette-classic 2182 custom: 2183 axisBorderShow: false 2184 axisCenteredZero: false 2185 axisColorMode: text 2186 axisLabel: '' 2187 axisPlacement: auto 2188 axisSoftMin: 0 2189 barAlignment: 0 2190 barWidthFactor: 0.6 2191 drawStyle: line 2192 fillOpacity: 0 2193 gradientMode: none 2194 hideFrom: 2195 legend: false 2196 tooltip: false 2197 viz: false 2198 insertNulls: false 2199 lineInterpolation: linear 2200 lineStyle: 2201 fill: solid 2202 lineWidth: 1 2203 pointSize: 5 2204 scaleDistribution: 2205 type: linear 2206 showPoints: auto 2207 spanNulls: false 2208 stacking: 2209 group: A 2210 mode: none 2211 thresholdsStyle: 2212 mode: dashed 2213 mappings: [] 2214 thresholds: 2215 mode: absolute 2216 steps: 2217 - color: green 2218 unit: short 2219 overrides: 2220 - matcher: 2221 id: byName 2222 options: Limit 2223 properties: 2224 - id: custom.lineStyle 2225 value: 2226 dash: 2227 - 10 2228 - 10 2229 fill: dash 2230 gridPos: 2231 h: 8 2232 w: 6 2233 x: 0 2234 'y': 155 2235 id: 48 2236 options: 2237 legend: 2238 calcs: 2239 - last 2240 - max 2241 displayMode: table 2242 placement: bottom 2243 showLegend: false 2244 sortBy: Max 2245 sortDesc: true 2246 tooltip: 2247 hideZeros: false 2248 mode: multi 2249 sort: desc 2250 pluginVersion: 12.0.0-85113 2251 targets: 2252 - datasource: 2253 type: prometheus 2254 uid: ${datasource} 2255 editorMode: code 2256 expr: go_threads{namespace="$namespace",container="query-backend"} 2257 hide: false 2258 legendFormat: '{{"{{pod}}"}}' 2259 range: true 2260 refId: D 2261 title: Go Threads 2262 transparent: true 2263 type: timeseries 2264 - datasource: 2265 type: datasource 2266 uid: '-- Mixed --' 2267 fieldConfig: 2268 defaults: 2269 color: 2270 mode: palette-classic 2271 custom: 2272 axisBorderShow: false 2273 axisCenteredZero: false 2274 axisColorMode: text 2275 axisLabel: '' 2276 axisPlacement: auto 2277 axisSoftMin: 0 2278 barAlignment: 0 2279 barWidthFactor: 0.6 2280 drawStyle: line 2281 fillOpacity: 0 2282 gradientMode: none 2283 hideFrom: 2284 legend: false 2285 tooltip: false 2286 viz: false 2287 insertNulls: false 2288 lineInterpolation: linear 2289 lineStyle: 2290 fill: solid 2291 lineWidth: 1 2292 pointSize: 5 2293 scaleDistribution: 2294 type: linear 2295 showPoints: auto 2296 spanNulls: false 2297 stacking: 2298 group: A 2299 mode: none 2300 thresholdsStyle: 2301 mode: dashed 2302 mappings: [] 2303 thresholds: 2304 mode: absolute 2305 steps: 2306 - color: green 2307 unit: short 2308 overrides: 2309 - matcher: 2310 id: byName 2311 options: Limit 2312 properties: 2313 - id: custom.lineStyle 2314 value: 2315 dash: 2316 - 10 2317 - 10 2318 fill: dash 2319 gridPos: 2320 h: 8 2321 w: 6 2322 x: 6 2323 'y': 155 2324 id: 46 2325 options: 2326 legend: 2327 calcs: 2328 - last 2329 - max 2330 displayMode: table 2331 placement: bottom 2332 showLegend: false 2333 sortBy: Max 2334 sortDesc: true 2335 tooltip: 2336 hideZeros: false 2337 mode: multi 2338 sort: desc 2339 pluginVersion: 12.0.0-85113 2340 targets: 2341 - datasource: 2342 type: prometheus 2343 uid: ${datasource} 2344 editorMode: code 2345 expr: >- 2346 sum by (pod) 2347 (go_goroutines{namespace="$namespace",container="query-backend"}) 2348 hide: false 2349 legendFormat: __auto 2350 range: true 2351 refId: D 2352 title: Goroutines 2353 transparent: true 2354 type: timeseries 2355 - datasource: 2356 type: datasource 2357 uid: '-- Mixed --' 2358 fieldConfig: 2359 defaults: 2360 color: 2361 mode: palette-classic 2362 custom: 2363 axisBorderShow: false 2364 axisCenteredZero: false 2365 axisColorMode: text 2366 axisLabel: '' 2367 axisPlacement: auto 2368 axisSoftMin: 0 2369 barAlignment: 0 2370 barWidthFactor: 0.6 2371 drawStyle: line 2372 fillOpacity: 0 2373 gradientMode: none 2374 hideFrom: 2375 legend: false 2376 tooltip: false 2377 viz: false 2378 insertNulls: false 2379 lineInterpolation: linear 2380 lineStyle: 2381 fill: solid 2382 lineWidth: 1 2383 pointSize: 5 2384 scaleDistribution: 2385 type: linear 2386 showPoints: auto 2387 spanNulls: false 2388 stacking: 2389 group: A 2390 mode: none 2391 thresholdsStyle: 2392 mode: dashed 2393 mappings: [] 2394 thresholds: 2395 mode: absolute 2396 steps: 2397 - color: green 2398 unit: bytes 2399 overrides: 2400 - matcher: 2401 id: byName 2402 options: Limit 2403 properties: 2404 - id: custom.lineStyle 2405 value: 2406 dash: 2407 - 10 2408 - 10 2409 fill: dash 2410 gridPos: 2411 h: 8 2412 w: 6 2413 x: 12 2414 'y': 155 2415 id: 44 2416 options: 2417 legend: 2418 calcs: 2419 - last 2420 - max 2421 displayMode: table 2422 placement: bottom 2423 showLegend: false 2424 sortBy: Max 2425 sortDesc: true 2426 tooltip: 2427 hideZeros: false 2428 mode: multi 2429 sort: desc 2430 pluginVersion: 12.0.0-85113 2431 targets: 2432 - datasource: 2433 type: prometheus 2434 uid: ${datasource} 2435 editorMode: code 2436 expr: >- 2437 sum by (pod) 2438 (go_memstats_heap_inuse_bytes{namespace="$namespace",container="query-backend"}) 2439 hide: false 2440 legendFormat: __auto 2441 range: true 2442 refId: D 2443 title: Heap In-Use 2444 transparent: true 2445 type: timeseries 2446 - datasource: 2447 type: datasource 2448 uid: '-- Mixed --' 2449 fieldConfig: 2450 defaults: 2451 color: 2452 mode: palette-classic 2453 custom: 2454 axisBorderShow: false 2455 axisCenteredZero: false 2456 axisColorMode: text 2457 axisLabel: '' 2458 axisPlacement: auto 2459 axisSoftMin: 0 2460 barAlignment: 1 2461 barWidthFactor: 1 2462 drawStyle: line 2463 fillOpacity: 0 2464 gradientMode: none 2465 hideFrom: 2466 legend: false 2467 tooltip: false 2468 viz: false 2469 insertNulls: false 2470 lineInterpolation: linear 2471 lineStyle: 2472 fill: solid 2473 lineWidth: 1 2474 pointSize: 5 2475 scaleDistribution: 2476 type: linear 2477 showPoints: auto 2478 spanNulls: false 2479 stacking: 2480 group: A 2481 mode: none 2482 thresholdsStyle: 2483 mode: 'off' 2484 fieldMinMax: false 2485 mappings: [] 2486 min: 0 2487 thresholds: 2488 mode: absolute 2489 steps: 2490 - color: green 2491 overrides: 2492 - matcher: 2493 id: byName 2494 options: Limit 2495 properties: 2496 - id: custom.lineStyle 2497 value: 2498 dash: 2499 - 10 2500 - 10 2501 fill: dash 2502 gridPos: 2503 h: 8 2504 w: 6 2505 x: 18 2506 'y': 155 2507 id: 45 2508 options: 2509 legend: 2510 calcs: 2511 - last 2512 - max 2513 displayMode: table 2514 placement: bottom 2515 showLegend: false 2516 sortBy: Max 2517 sortDesc: true 2518 tooltip: 2519 hideZeros: false 2520 mode: multi 2521 sort: desc 2522 pluginVersion: 12.0.0-85113 2523 targets: 2524 - datasource: 2525 type: prometheus 2526 uid: ${datasource} 2527 editorMode: code 2528 expr: >- 2529 sum by (pod) 2530 (container_file_descriptors{namespace="$namespace",container="query-backend"}) 2531 hide: false 2532 legendFormat: __auto 2533 range: true 2534 refId: A 2535 title: File Descriptors 2536 transparent: true 2537 type: timeseries 2538 title: Runtime 2539 type: row 2540 - collapsed: true 2541 gridPos: 2542 h: 1 2543 w: 24 2544 x: 0 2545 'y': 53 2546 id: 42 2547 panels: 2548 - datasource: 2549 type: loki 2550 uid: OP27Xzxnk 2551 fieldConfig: 2552 defaults: {} 2553 overrides: [] 2554 gridPos: 2555 h: 8 2556 w: 24 2557 x: 0 2558 'y': 156 2559 id: 41 2560 options: 2561 dedupStrategy: none 2562 enableInfiniteScrolling: false 2563 enableLogDetails: true 2564 prettifyLogMessage: false 2565 showCommonLabels: false 2566 showLabels: false 2567 showTime: false 2568 sortOrder: Descending 2569 wrapLogMessage: false 2570 pluginVersion: 12.0.0-85113 2571 targets: 2572 - datasource: 2573 type: loki 2574 uid: OP27Xzxnk 2575 direction: backward 2576 editorMode: code 2577 expr: >- 2578 {cluster="$cluster", namespace="$namespace", 2579 container="query-backend"} |= "level=err" or "level=warn" or 2580 "panic" 2581 queryType: range 2582 refId: A 2583 title: Worker log 2584 transparent: true 2585 type: logs 2586 title: Errors and warnings 2587 type: row 2588 preload: false 2589 schemaVersion: 41 2590 tags: 2591 - pyroscope 2592 - pyroscope-v2 2593 templating: 2594 list: 2595 - allowCustomValue: true 2596 current: 2597 text: ops-cortex 2598 value: '000000134' 2599 label: metrics 2600 name: datasource 2601 options: [] 2602 query: prometheus 2603 refresh: 1 2604 regex: '' 2605 type: datasource 2606 - allowCustomValue: false 2607 current: 2608 text: Grafana Logging 2609 value: '000000193' 2610 description: '' 2611 label: logs 2612 name: loki_datasource 2613 options: [] 2614 query: loki 2615 refresh: 1 2616 regex: '' 2617 type: datasource 2618 - allowCustomValue: false 2619 current: 2620 text: Tempo Ops (tempo-ops-01) 2621 value: fds8vtxx3ao74b 2622 description: '' 2623 label: traces 2624 name: tempo_datasource 2625 options: [] 2626 query: tempo 2627 refresh: 1 2628 regex: '' 2629 type: datasource 2630 - allowCustomValue: false 2631 current: 2632 text: {{.Values.dashboards.cluster | quote}} 2633 value: {{.Values.dashboards.cluster | quote}} 2634 datasource: 2635 type: prometheus 2636 uid: ${datasource} 2637 definition: label_values(pyroscope_build_info{namespace="$namespace"},cluster) 2638 label: cluster 2639 name: cluster 2640 options: [] 2641 query: 2642 qryType: 1 2643 query: label_values(pyroscope_build_info{namespace="$namespace"},cluster) 2644 refId: PrometheusVariableQueryEditor-VariableQuery 2645 refresh: 1 2646 regex: '' 2647 sort: 1 2648 type: query 2649 - allowCustomValue: false 2650 current: 2651 text: {{.Values.dashboards.namespace | quote}} 2652 value: {{.Values.dashboards.namespace | quote}} 2653 datasource: 2654 type: prometheus 2655 uid: ${datasource} 2656 definition: label_values(pyroscope_build_info,namespace) 2657 label: namespace 2658 name: namespace 2659 options: [] 2660 query: 2661 qryType: 1 2662 query: label_values(pyroscope_build_info,namespace) 2663 refId: PrometheusVariableQueryEditor-VariableQuery 2664 refresh: 1 2665 regex: {{.Values.dashboards.namespaceRegex | quote}} 2666 type: query 2667 time: 2668 from: now-6h 2669 to: now 2670 timepicker: {} 2671 timezone: utc 2672 title: Pyroscope v2 / Read Path 2673 2674 {{- end }}