github.com/grafana/pyroscope@v1.18.0/operations/monitoring/helm/pyroscope-monitoring/templates/_dashboard_v2-write-path.yaml (about) 1 {{- define "pyroscope-monitoring.dashboards.v2-write-path" -}} 2 {{- $dashboard := "v2-write-path" -}} 3 annotations: 4 list: 5 - builtIn: 1 6 datasource: 7 type: grafana 8 uid: '-- Grafana --' 9 enable: true 10 hide: true 11 iconColor: rgba(0, 211, 255, 1) 12 name: Annotations & Alerts 13 type: dashboard 14 - datasource: 15 type: loki 16 uid: ${loki_datasource} 17 enable: true 18 expr: >- 19 {cluster="$cluster", container="kube-diff-logger"} | json | 20 namespace_extracted="$namespace" 21 hide: false 22 iconColor: yellow 23 instant: false 24 name: K8s Changes 25 tagKeys: type,name_extracted,verb 26 textFormat: '{{"{{notes}}"}}' 27 titleFormat: '' 28 - datasource: 29 type: loki 30 uid: ${loki_datasource} 31 enable: false 32 expr: >- 33 {namespace="$namespace",container="eventrouter"} != "Preemption is not 34 helpful" != "Failed deploy model" | logfmt | type="Warning" 35 hide: false 36 iconColor: orange 37 instant: false 38 name: K8s Warnings 39 tagKeys: type,object_kind,object_name 40 textFormat: '{{"{{message}}"}}' 41 titleFormat: '' 42 - datasource: 43 type: loki 44 uid: ${loki_datasource} 45 enable: true 46 expr: >- 47 {namespace="$namespace",container="eventrouter"} | logfmt | 48 type!="Normal" | type!="Warning" 49 hide: false 50 iconColor: red 51 instant: false 52 name: K8s Errors 53 tagKeys: type,object_kind,object_name 54 textFormat: '{{"{{message}}"}}' 55 - datasource: 56 type: loki 57 uid: ${loki_datasource} 58 enable: false 59 expr: '{namespace="$namespace",container="eventrouter"} | logfmt' 60 hide: false 61 iconColor: blue 62 instant: false 63 name: K8s All Events 64 tagKeys: type,object_kind,object_name 65 textFormat: '{{"{{message}}"}}' 66 editable: true 67 fiscalYearStartMonth: 0 68 graphTooltip: 1 69 id: 0 70 links: 71 {{ .Values.dashboards.links.global | toYaml | nindent 2}} 72 {{- with (dig $dashboard (list) .Values.dashboards.links.perDashboard) }} 73 {{ . | toYaml | nindent 2}} 74 {{ end }} 75 panels: 76 - collapsed: false 77 gridPos: 78 h: 1 79 w: 24 80 x: 0 81 'y': 0 82 id: 85 83 panels: [] 84 title: '' 85 type: row 86 - datasource: 87 type: prometheus 88 uid: ${datasource} 89 fieldConfig: 90 defaults: 91 color: 92 mode: thresholds 93 custom: 94 align: left 95 cellOptions: 96 type: auto 97 filterable: true 98 inspect: false 99 mappings: [] 100 thresholds: 101 mode: absolute 102 steps: 103 - color: green 104 value: 0 105 - color: red 106 value: 80 107 overrides: 108 - matcher: 109 id: byName 110 options: Value 111 properties: 112 - id: unit 113 value: decbytes 114 - matcher: 115 id: byName 116 options: tenant 117 properties: 118 - id: custom.width 119 value: 78 120 - matcher: 121 id: byName 122 options: slug 123 properties: 124 - id: custom.width 125 value: 132 126 - matcher: 127 id: byName 128 options: org_name 129 properties: 130 - id: custom.width 131 value: 142 132 - matcher: 133 id: byName 134 options: Value 135 properties: 136 - id: custom.cellOptions 137 value: 138 mode: lcd 139 type: gauge 140 valueDisplayMode: text 141 - id: color 142 value: 143 fixedColor: green 144 mode: fixed 145 - matcher: 146 id: byName 147 options: Slug 148 properties: 149 - id: custom.width 150 value: 89 151 - matcher: 152 id: byName 153 options: Name 154 properties: 155 - id: custom.width 156 value: 143 157 - matcher: 158 id: byName 159 options: ID 160 properties: 161 - id: custom.width 162 value: 104 163 gridPos: 164 h: 8 165 w: 8 166 x: 0 167 'y': 1 168 id: 68 169 options: 170 cellHeight: sm 171 footer: 172 countRows: true 173 enablePagination: false 174 fields: '' 175 reducer: 176 - count 177 show: true 178 showHeader: true 179 sortBy: 180 - desc: true 181 displayName: Value 182 pluginVersion: 12.1.0-90139 183 targets: 184 - datasource: 185 type: prometheus 186 uid: ${datasource} 187 editorMode: code 188 exemplar: false 189 expr: | 190 {{ .Values.dashboards.tenantQuery | nindent 10 }} 191 format: table 192 instant: true 193 legendFormat: __auto 194 range: false 195 refId: A 196 title: Tenants 197 transformations: 198 - id: organize 199 options: 200 excludeByName: 201 Time: true 202 Value: false 203 cluster: true 204 environment: true 205 slug: false 206 includeByName: {} 207 indexByName: 208 Time: 0 209 Value: 4 210 environment: 5 211 org_name: 3 212 slug: 2 213 tenant: 1 214 renameByName: 215 Time: '' 216 Value: '' 217 environment: Env 218 org_name: Name 219 slug: Slug 220 tenant: ID 221 type: table 222 - datasource: 223 type: prometheus 224 uid: ${datasource} 225 fieldConfig: 226 defaults: 227 color: 228 mode: palette-classic 229 custom: 230 axisBorderShow: false 231 axisCenteredZero: false 232 axisColorMode: text 233 axisLabel: '' 234 axisPlacement: auto 235 axisSoftMin: 0 236 barAlignment: 0 237 barWidthFactor: 0.6 238 drawStyle: line 239 fillOpacity: 10 240 gradientMode: none 241 hideFrom: 242 legend: false 243 tooltip: false 244 viz: false 245 insertNulls: false 246 lineInterpolation: linear 247 lineWidth: 1 248 pointSize: 5 249 scaleDistribution: 250 type: linear 251 showPoints: auto 252 spanNulls: false 253 stacking: 254 group: A 255 mode: none 256 thresholdsStyle: 257 mode: 'off' 258 mappings: [] 259 thresholds: 260 mode: absolute 261 steps: 262 - color: green 263 value: 0 264 - color: red 265 value: 80 266 unit: binBps 267 overrides: [] 268 gridPos: 269 h: 8 270 w: 4 271 x: 8 272 'y': 1 273 id: 66 274 options: 275 legend: 276 calcs: 277 - max 278 displayMode: table 279 placement: right 280 showLegend: false 281 sortBy: Max 282 sortDesc: true 283 tooltip: 284 hideZeros: false 285 mode: single 286 sort: none 287 pluginVersion: 12.1.0-90139 288 targets: 289 - datasource: 290 type: prometheus 291 uid: ${datasource} 292 editorMode: code 293 expr: >- 294 sum by (tenant) 295 (rate(pyroscope_distributor_received_decompressed_bytes_sum{cluster=~"$cluster",namespace=~"$namespace"}[$__rate_interval])) 296 hide: false 297 legendFormat: __auto 298 range: true 299 refId: A 300 title: MBs per Tenant (Decompressed) 301 type: timeseries 302 - datasource: 303 type: prometheus 304 uid: ${datasource} 305 fieldConfig: 306 defaults: 307 color: 308 mode: palette-classic 309 custom: 310 axisBorderShow: false 311 axisCenteredZero: false 312 axisColorMode: text 313 axisLabel: '' 314 axisPlacement: auto 315 axisSoftMin: 0 316 barAlignment: 0 317 barWidthFactor: 0.6 318 drawStyle: line 319 fillOpacity: 4 320 gradientMode: none 321 hideFrom: 322 legend: false 323 tooltip: false 324 viz: false 325 insertNulls: false 326 lineInterpolation: linear 327 lineWidth: 1 328 pointSize: 5 329 scaleDistribution: 330 type: linear 331 showPoints: auto 332 spanNulls: false 333 stacking: 334 group: A 335 mode: none 336 thresholdsStyle: 337 mode: 'off' 338 mappings: [] 339 thresholds: 340 mode: absolute 341 steps: 342 - color: green 343 value: 0 344 - color: red 345 value: 80 346 unit: cps 347 overrides: [] 348 gridPos: 349 h: 8 350 w: 4 351 x: 12 352 'y': 1 353 id: 67 354 options: 355 legend: 356 calcs: 357 - mean 358 - max 359 displayMode: table 360 placement: right 361 showLegend: false 362 sortBy: Mean 363 sortDesc: true 364 tooltip: 365 hideZeros: false 366 mode: single 367 sort: none 368 pluginVersion: 12.1.0-90139 369 targets: 370 - datasource: 371 type: prometheus 372 uid: ${datasource} 373 editorMode: code 374 expr: >- 375 sum by (tenant) 376 (rate(pyroscope_distributor_received_decompressed_bytes_count{cluster=~"$cluster",namespace=~"$namespace"}[$__rate_interval])) 377 legendFormat: __auto 378 range: true 379 refId: A 380 title: Profiles/s per Tenant 381 type: timeseries 382 - datasource: 383 type: prometheus 384 uid: ${datasource} 385 fieldConfig: 386 defaults: 387 color: 388 mode: palette-classic 389 custom: 390 hideFrom: 391 legend: false 392 tooltip: false 393 viz: false 394 mappings: [] 395 overrides: [] 396 gridPos: 397 h: 8 398 w: 4 399 x: 16 400 'y': 1 401 id: 82 402 options: 403 legend: 404 displayMode: table 405 placement: bottom 406 showLegend: true 407 values: 408 - value 409 pieType: pie 410 reduceOptions: 411 calcs: 412 - lastNotNull 413 fields: '' 414 values: false 415 tooltip: 416 hideZeros: false 417 mode: single 418 sort: none 419 pluginVersion: 12.1.0-90139 420 targets: 421 - datasource: 422 type: prometheus 423 uid: ${datasource} 424 editorMode: code 425 exemplar: false 426 expr: >- 427 sum by (state) (pyroscope_ring_members{cluster=~"$cluster", 428 namespace=~"$namespace", name="distributor"}) / count by 429 (state)(pyroscope_ring_members{cluster=~"$cluster", 430 namespace=~"$namespace", name="distributor"}) 431 format: time_series 432 instant: true 433 legendFormat: __auto 434 range: false 435 refId: A 436 title: Distributor Ring 437 type: piechart 438 - datasource: 439 type: prometheus 440 uid: ${datasource} 441 description: 'The ring is used by distributor to discover segment-writer instances. ' 442 fieldConfig: 443 defaults: 444 color: 445 mode: palette-classic 446 custom: 447 hideFrom: 448 legend: false 449 tooltip: false 450 viz: false 451 mappings: [] 452 overrides: [] 453 gridPos: 454 h: 8 455 w: 4 456 x: 20 457 'y': 1 458 id: 81 459 options: 460 legend: 461 displayMode: table 462 placement: bottom 463 showLegend: true 464 values: 465 - value 466 pieType: pie 467 reduceOptions: 468 calcs: 469 - lastNotNull 470 fields: '' 471 values: false 472 tooltip: 473 hideZeros: false 474 mode: single 475 sort: none 476 pluginVersion: 12.1.0-90139 477 targets: 478 - datasource: 479 type: prometheus 480 uid: ${datasource} 481 editorMode: code 482 exemplar: false 483 expr: >- 484 sum by (state) (pyroscope_ring_members{cluster=~"$cluster", 485 namespace=~"$namespace", name="segment-writer"}) / count by 486 (state)(pyroscope_ring_members{cluster=~"$cluster", 487 namespace=~"$namespace", name="segment-writer"}) 488 format: time_series 489 instant: true 490 legendFormat: __auto 491 range: false 492 refId: A 493 title: Segment Writer Ring 494 type: piechart 495 {{- if .Values.dashboards.cloudBackendGateway }} 496 - collapsed: false 497 gridPos: 498 h: 1 499 w: 24 500 x: 0 501 'y': 9 502 id: 43 503 panels: [] 504 title: Cortex Gateway 505 type: row 506 - datasource: 507 type: datasource 508 uid: '-- Mixed --' 509 description: >- 510 The latency observed by cortex-gw, indicating the overall write path 511 latency. 512 513 514 Only successful requests are counted. 515 516 * the purple line (1s) is our internal SLO 517 * the red line (2s) is the threshold when we start burning SLO budget 518 fieldConfig: 519 defaults: 520 color: 521 fixedColor: dark-red 522 mode: palette-classic 523 custom: 524 axisBorderShow: true 525 axisCenteredZero: true 526 axisColorMode: text 527 axisGridShow: true 528 axisLabel: '' 529 axisPlacement: auto 530 axisSoftMin: 0 531 barAlignment: 0 532 barWidthFactor: 0.6 533 drawStyle: line 534 fillOpacity: 0 535 gradientMode: none 536 hideFrom: 537 legend: false 538 tooltip: false 539 viz: false 540 insertNulls: false 541 lineInterpolation: linear 542 lineStyle: 543 fill: solid 544 lineWidth: 1 545 pointSize: 1 546 scaleDistribution: 547 log: 10 548 type: log 549 showPoints: never 550 spanNulls: false 551 stacking: 552 group: A 553 mode: none 554 thresholdsStyle: 555 mode: dashed 556 fieldMinMax: false 557 mappings: [] 558 min: 0 559 thresholds: 560 mode: absolute 561 steps: 562 - color: green 563 value: 0 564 - color: purple 565 value: 1 566 - color: red 567 value: 2 568 unit: s 569 overrides: 570 - matcher: 571 id: byName 572 options: p999 573 properties: 574 - id: color 575 value: 576 fixedColor: green 577 mode: fixed 578 - matcher: 579 id: byName 580 options: p999 envoy-upstream 581 properties: 582 - id: color 583 value: 584 fixedColor: red 585 mode: fixed 586 - matcher: 587 id: byName 588 options: envoy-upstream p999 589 properties: 590 - id: color 591 value: 592 fixedColor: orange 593 mode: fixed 594 - __systemRef: hideSeriesFrom 595 matcher: 596 id: byNames 597 options: 598 mode: exclude 599 names: 600 - p99 601 - p95 602 - p90 603 - avg 604 - p50 605 prefix: 'All except:' 606 readOnly: true 607 properties: 608 - id: custom.hideFrom 609 value: 610 legend: false 611 tooltip: false 612 viz: true 613 gridPos: 614 h: 8 615 w: 8 616 x: 0 617 'y': 10 618 id: 40 619 options: 620 legend: 621 calcs: 622 - mean 623 displayMode: list 624 placement: bottom 625 showLegend: true 626 tooltip: 627 hideZeros: false 628 mode: multi 629 sort: none 630 pluginVersion: 12.1.0-90139 631 targets: 632 - datasource: 633 type: prometheus 634 uid: ${datasource} 635 editorMode: code 636 expr: | 637 histogram_quantile(0.999, 638 sum by(cluster, namespace,le) (rate(envoy_cluster_upstream_rq_time_bucket{ 639 namespace="$namespace", 640 container="envoy", 641 }[$__rate_interval])) 642 ) / 1000 643 hide: false 644 legendFormat: envoy-upstream p999 645 range: true 646 refId: F 647 - datasource: 648 type: prometheus 649 uid: ${datasource} 650 editorMode: code 651 expr: | 652 histogram_quantile(0.999, 653 sum by(cluster, namespace,pod,le) (rate(envoy_cluster_external_upstream_rq_time_bucket{ 654 namespace="$namespace", 655 container="envoy", 656 }[$__rate_interval])) 657 ) / 1000 658 hide: true 659 legendFormat: p999 envoy-upstream-external 660 range: true 661 refId: D 662 - datasource: 663 type: prometheus 664 uid: ${datasource} 665 editorMode: code 666 expr: >- 667 histogram_quantile(0.999, sum by (le,status_code) 668 (rate(pyroscope_request_duration_seconds_bucket{ 669 {{ include "pyroscope-monitoring.dashboards-ingest-selector" . }}, namespace=~"$namespace", 670 route=~".*pusher.*|.*ingest.*", 671 status_code="200", 672 }[$__rate_interval]))) 673 hide: false 674 legendFormat: p999 675 range: true 676 refId: E 677 - datasource: 678 type: prometheus 679 uid: ${datasource} 680 editorMode: code 681 expr: >- 682 histogram_quantile(0.99, sum by (le,status_code) 683 (rate(pyroscope_request_duration_seconds_bucket{ 684 {{ include "pyroscope-monitoring.dashboards-ingest-selector" . }}, namespace=~"$namespace", 685 route=~".*pusher.*|.*ingest.*", 686 status_code="200", 687 }[$__rate_interval]))) 688 hide: false 689 legendFormat: p99 690 range: true 691 refId: A 692 - datasource: 693 type: prometheus 694 uid: ${datasource} 695 editorMode: code 696 expr: >- 697 histogram_quantile(0.95, sum by (le,status_code) 698 (rate(pyroscope_request_duration_seconds_bucket{ 699 {{ include "pyroscope-monitoring.dashboards-ingest-selector" . }}, namespace=~"$namespace", 700 route=~".*pusher.*|.*ingest.*", 701 status_code="200", 702 }[$__rate_interval]))) 703 hide: false 704 legendFormat: p95 705 range: true 706 refId: B 707 - datasource: 708 type: prometheus 709 uid: ${datasource} 710 editorMode: code 711 expr: >- 712 histogram_quantile(0.50, sum by (le,status_code) 713 (rate(pyroscope_request_duration_seconds_bucket{ 714 {{ include "pyroscope-monitoring.dashboards-ingest-selector" . }}, namespace=~"$namespace", 715 route=~".*pusher.*|.*ingest.*", 716 status_code="200", 717 }[$__rate_interval]))) 718 hide: false 719 legendFormat: p50 720 range: true 721 refId: C 722 - datasource: 723 type: prometheus 724 uid: ${datasource} 725 editorMode: code 726 expr: |- 727 sum (rate(pyroscope_request_duration_seconds_sum{ 728 {{ include "pyroscope-monitoring.dashboards-ingest-selector" . }}, namespace=~"$namespace", 729 route=~".*pusher.*|.*ingest.*", 730 status_code="200", 731 }[$__rate_interval])) 732 / 733 sum (rate(pyroscope_request_duration_seconds_count{ 734 {{ include "pyroscope-monitoring.dashboards-ingest-selector" . }}, namespace=~"$namespace", 735 route=~".*pusher.*|.*ingest.*", 736 status_code="200", 737 }[$__rate_interval])) 738 hide: false 739 legendFormat: avg 740 range: true 741 refId: G 742 title: Latency 743 transparent: true 744 type: timeseries 745 - datasource: 746 type: datasource 747 uid: '-- Mixed --' 748 fieldConfig: 749 defaults: 750 custom: 751 hideFrom: 752 legend: false 753 tooltip: false 754 viz: false 755 scaleDistribution: 756 type: linear 757 overrides: [] 758 gridPos: 759 h: 8 760 w: 8 761 x: 8 762 'y': 10 763 id: 53 764 options: 765 calculate: false 766 calculation: 767 xBuckets: 768 mode: count 769 value: '' 770 yBuckets: 771 mode: count 772 scale: 773 type: linear 774 cellGap: 0 775 cellValues: 776 unit: requests 777 color: 778 exponent: 0.2 779 fill: super-light-red 780 mode: scheme 781 reverse: true 782 scale: exponential 783 scheme: Turbo 784 steps: 128 785 exemplars: 786 color: orange 787 filterValues: 788 le: 1.e-9 789 legend: 790 show: false 791 rowsFrame: 792 layout: auto 793 tooltip: 794 mode: single 795 showColorScale: true 796 yHistogram: true 797 yAxis: 798 axisPlacement: left 799 reverse: false 800 unit: s 801 pluginVersion: 12.1.0-90139 802 targets: 803 - datasource: 804 type: prometheus 805 uid: ${datasource} 806 editorMode: code 807 expr: |- 808 sum(rate(pyroscope_request_duration_seconds{ 809 namespace="$namespace", 810 route=~".*push.*|.*ingest.*", 811 container="cortex-gw", 812 }[$__rate_interval])) 813 hide: false 814 legendFormat: __auto 815 range: true 816 refId: B 817 title: Latency Heatmap 818 transparent: true 819 type: heatmap 820 - datasource: 821 type: prometheus 822 uid: ${datasource} 823 fieldConfig: 824 defaults: 825 color: 826 mode: palette-classic 827 custom: 828 axisBorderShow: false 829 axisCenteredZero: false 830 axisColorMode: text 831 axisLabel: '' 832 axisPlacement: auto 833 barAlignment: 0 834 barWidthFactor: 0.6 835 drawStyle: line 836 fillOpacity: 0 837 gradientMode: none 838 hideFrom: 839 legend: false 840 tooltip: false 841 viz: false 842 insertNulls: false 843 lineInterpolation: linear 844 lineWidth: 1 845 pointSize: 5 846 scaleDistribution: 847 log: 2 848 type: log 849 showPoints: auto 850 spanNulls: false 851 stacking: 852 group: A 853 mode: none 854 thresholdsStyle: 855 mode: 'off' 856 mappings: [] 857 thresholds: 858 mode: absolute 859 steps: 860 - color: green 861 value: 0 862 - color: red 863 value: 80 864 unit: reqps 865 overrides: [] 866 gridPos: 867 h: 8 868 w: 8 869 x: 16 870 'y': 10 871 id: 41 872 options: 873 legend: 874 calcs: [] 875 displayMode: list 876 placement: bottom 877 showLegend: true 878 tooltip: 879 hideZeros: false 880 mode: single 881 sort: none 882 pluginVersion: 12.1.0-90139 883 targets: 884 - datasource: 885 type: prometheus 886 uid: ${datasource} 887 editorMode: code 888 expr: |- 889 sum by (status_code) (rate( 890 pyroscope_request_duration_seconds_count{ 891 namespace="$namespace", 892 container="cortex-gw", 893 route=~".*push.*|.*ingest.*", 894 }[1m]) 895 ) 896 hide: false 897 legendFormat: __auto 898 range: true 899 refId: A 900 title: Rate 901 transparent: true 902 type: timeseries 903 - collapsed: true 904 gridPos: 905 h: 1 906 w: 24 907 x: 0 908 'y': 18 909 id: 100 910 panels: 911 - datasource: 912 type: prometheus 913 uid: ${datasource} 914 description: >- 915 During load balancing, Envoy will generally only consider available 916 (healthy or degraded) hosts in an upstream cluster. However, if the 917 percentage of available hosts in the cluster becomes too low, Envoy 918 will disregard health status and balance either amongst all hosts or 919 no hosts. This is known as the panic threshold. The default panic 920 threshold is 50%. 921 fieldConfig: 922 defaults: 923 color: 924 mode: palette-classic 925 custom: 926 axisBorderShow: false 927 axisCenteredZero: false 928 axisColorMode: text 929 axisLabel: '' 930 axisPlacement: auto 931 barAlignment: 0 932 barWidthFactor: 0.6 933 drawStyle: line 934 fillOpacity: 0 935 gradientMode: none 936 hideFrom: 937 legend: false 938 tooltip: false 939 viz: false 940 insertNulls: false 941 lineInterpolation: linear 942 lineWidth: 1 943 pointSize: 5 944 scaleDistribution: 945 type: linear 946 showPoints: auto 947 spanNulls: false 948 stacking: 949 group: A 950 mode: none 951 thresholdsStyle: 952 mode: 'off' 953 mappings: [] 954 thresholds: 955 mode: absolute 956 steps: 957 - color: green 958 value: 0 959 - color: red 960 value: 80 961 overrides: [] 962 gridPos: 963 h: 7 964 w: 8 965 x: 0 966 'y': 19 967 id: 102 968 options: 969 legend: 970 calcs: [] 971 displayMode: table 972 placement: right 973 showLegend: false 974 tooltip: 975 hideZeros: false 976 mode: single 977 sort: none 978 pluginVersion: 12.1.0-90058 979 targets: 980 - datasource: 981 type: prometheus 982 uid: ${datasource} 983 editorMode: code 984 expr: >- 985 sum by (pod) 986 (rate(envoy_cluster_lb_healthy_panic{namespace="$namespace",container="envoy"}[$__rate_interval])) 987 hide: false 988 legendFormat: __auto 989 range: true 990 refId: A 991 title: Envoy Panics 992 transparent: true 993 type: timeseries 994 - datasource: 995 type: prometheus 996 uid: ${datasource} 997 fieldConfig: 998 defaults: 999 color: 1000 mode: palette-classic 1001 custom: 1002 axisBorderShow: false 1003 axisCenteredZero: false 1004 axisColorMode: text 1005 axisLabel: '' 1006 axisPlacement: auto 1007 barAlignment: 0 1008 barWidthFactor: 0.6 1009 drawStyle: line 1010 fillOpacity: 0 1011 gradientMode: none 1012 hideFrom: 1013 legend: false 1014 tooltip: false 1015 viz: false 1016 insertNulls: false 1017 lineInterpolation: linear 1018 lineWidth: 1 1019 pointSize: 5 1020 scaleDistribution: 1021 type: linear 1022 showPoints: auto 1023 spanNulls: false 1024 stacking: 1025 group: A 1026 mode: none 1027 thresholdsStyle: 1028 mode: 'off' 1029 mappings: [] 1030 thresholds: 1031 mode: absolute 1032 steps: 1033 - color: green 1034 value: 0 1035 - color: red 1036 value: 80 1037 overrides: [] 1038 gridPos: 1039 h: 7 1040 w: 8 1041 x: 8 1042 'y': 19 1043 id: 99 1044 options: 1045 legend: 1046 calcs: [] 1047 displayMode: table 1048 placement: right 1049 showLegend: false 1050 tooltip: 1051 hideZeros: false 1052 mode: single 1053 sort: none 1054 pluginVersion: 12.1.0-90058 1055 targets: 1056 - datasource: 1057 type: prometheus 1058 uid: ${datasource} 1059 editorMode: code 1060 expr: >- 1061 sum by (direction,pod) 1062 (conntrack_pod_connections{namespace="$namespace",pod=~"cortex-gw-.*"}) 1063 hide: false 1064 legendFormat: '{{"{{pod}} {{direction}}"}}' 1065 range: true 1066 refId: A 1067 title: Connections per pod 1068 transparent: true 1069 type: timeseries 1070 - datasource: 1071 type: prometheus 1072 uid: ${datasource} 1073 description: >- 1074 See 1075 https://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats 1076 fieldConfig: 1077 defaults: 1078 color: 1079 mode: palette-classic 1080 custom: 1081 axisBorderShow: false 1082 axisCenteredZero: false 1083 axisColorMode: text 1084 axisLabel: '' 1085 axisPlacement: auto 1086 barAlignment: 0 1087 barWidthFactor: 0.6 1088 drawStyle: line 1089 fillOpacity: 0 1090 gradientMode: none 1091 hideFrom: 1092 legend: false 1093 tooltip: false 1094 viz: false 1095 insertNulls: false 1096 lineInterpolation: linear 1097 lineWidth: 1 1098 pointSize: 5 1099 scaleDistribution: 1100 type: linear 1101 showPoints: auto 1102 spanNulls: false 1103 stacking: 1104 group: A 1105 mode: none 1106 thresholdsStyle: 1107 mode: 'off' 1108 mappings: [] 1109 thresholds: 1110 mode: absolute 1111 steps: 1112 - color: green 1113 value: 0 1114 - color: red 1115 value: 80 1116 overrides: [] 1117 gridPos: 1118 h: 7 1119 w: 8 1120 x: 16 1121 'y': 19 1122 id: 105 1123 options: 1124 legend: 1125 calcs: [] 1126 displayMode: list 1127 placement: bottom 1128 showLegend: true 1129 tooltip: 1130 hideZeros: false 1131 mode: single 1132 sort: none 1133 pluginVersion: 12.1.0-90058 1134 targets: 1135 - datasource: 1136 type: prometheus 1137 uid: ${datasource} 1138 editorMode: code 1139 expr: >- 1140 sum by () 1141 (rate(envoy_cluster_upstream_rq_pending_overflow{namespace="$namespace",container="envoy"}[$__rate_interval])) 1142 hide: false 1143 instant: false 1144 legendFormat: pending overflow 1145 range: true 1146 refId: B 1147 - datasource: 1148 type: prometheus 1149 uid: ${datasource} 1150 editorMode: code 1151 expr: >- 1152 sum by () 1153 (rate(envoy_cluster_upstream_rq_pending_failure_eject{namespace="$namespace",container="envoy"}[$__rate_interval])) 1154 hide: false 1155 instant: false 1156 legendFormat: pending failure eject 1157 range: true 1158 refId: C 1159 - datasource: 1160 type: prometheus 1161 uid: ${datasource} 1162 editorMode: code 1163 expr: >- 1164 sum by () 1165 (rate(envoy_cluster_upstream_rq_timeout{namespace="$namespace",container="envoy"}[$__rate_interval])) 1166 hide: false 1167 instant: false 1168 legendFormat: timeout 1169 range: true 1170 refId: A 1171 - datasource: 1172 type: prometheus 1173 uid: ${datasource} 1174 editorMode: code 1175 expr: >- 1176 sum by () 1177 (rate(envoy_cluster_upstream_rq_cancelled{namespace="$namespace",container="envoy"}[$__rate_interval])) 1178 hide: false 1179 instant: false 1180 legendFormat: cancelled 1181 range: true 1182 refId: D 1183 - datasource: 1184 type: prometheus 1185 uid: ${datasource} 1186 editorMode: code 1187 expr: >- 1188 sum by () 1189 (rate(envoy_cluster_upstream_rq_max_duration_reached{namespace="$namespace",container="envoy"}[$__rate_interval])) 1190 hide: false 1191 instant: false 1192 legendFormat: max_duration_reached 1193 range: true 1194 refId: E 1195 title: Envoy RQ Errors 1196 transparent: true 1197 type: timeseries 1198 - datasource: 1199 type: prometheus 1200 uid: ${datasource} 1201 description: >- 1202 See 1203 https://www.envoyproxy.io/docs/envoy/latest/configuration/upstream/cluster_manager/cluster_stats 1204 fieldConfig: 1205 defaults: 1206 color: 1207 mode: palette-classic 1208 custom: 1209 axisBorderShow: false 1210 axisCenteredZero: false 1211 axisColorMode: text 1212 axisLabel: '' 1213 axisPlacement: auto 1214 barAlignment: 0 1215 barWidthFactor: 0.6 1216 drawStyle: line 1217 fillOpacity: 0 1218 gradientMode: none 1219 hideFrom: 1220 legend: false 1221 tooltip: false 1222 viz: false 1223 insertNulls: false 1224 lineInterpolation: linear 1225 lineWidth: 1 1226 pointSize: 5 1227 scaleDistribution: 1228 type: linear 1229 showPoints: auto 1230 spanNulls: false 1231 stacking: 1232 group: A 1233 mode: none 1234 thresholdsStyle: 1235 mode: 'off' 1236 mappings: [] 1237 thresholds: 1238 mode: absolute 1239 steps: 1240 - color: green 1241 value: 0 1242 - color: red 1243 value: 80 1244 overrides: [] 1245 gridPos: 1246 h: 7 1247 w: 8 1248 x: 0 1249 'y': 26 1250 id: 106 1251 options: 1252 legend: 1253 calcs: [] 1254 displayMode: list 1255 placement: bottom 1256 showLegend: false 1257 tooltip: 1258 hideZeros: false 1259 mode: single 1260 sort: none 1261 pluginVersion: 12.1.0-90058 1262 targets: 1263 - datasource: 1264 type: prometheus 1265 uid: ${datasource} 1266 editorMode: code 1267 expr: >- 1268 sum by (pod) 1269 (envoy_cluster_membership_healthy{namespace="$namespace",container="envoy",envoy_cluster_name="phlare_write_h2c"}) 1270 hide: true 1271 instant: false 1272 legendFormat: healthy {{"{{pod}}"}} 1273 range: true 1274 refId: A 1275 - datasource: 1276 type: prometheus 1277 uid: ${datasource} 1278 editorMode: code 1279 expr: >- 1280 sum by (pod) 1281 (envoy_cluster_membership_total{namespace="$namespace",container="envoy",envoy_cluster_name="phlare_write_h2c"}) 1282 hide: false 1283 instant: false 1284 legendFormat: total {{"{{pod}}"}} 1285 range: true 1286 refId: B 1287 title: Envoy Membership 1288 transparent: true 1289 type: timeseries 1290 - datasource: 1291 type: prometheus 1292 uid: ${datasource} 1293 fieldConfig: 1294 defaults: 1295 color: 1296 mode: palette-classic 1297 custom: 1298 axisBorderShow: false 1299 axisCenteredZero: false 1300 axisColorMode: text 1301 axisLabel: '' 1302 axisPlacement: auto 1303 barAlignment: 0 1304 barWidthFactor: 0.6 1305 drawStyle: line 1306 fillOpacity: 0 1307 gradientMode: none 1308 hideFrom: 1309 legend: false 1310 tooltip: false 1311 viz: false 1312 insertNulls: false 1313 lineInterpolation: linear 1314 lineWidth: 1 1315 pointSize: 5 1316 scaleDistribution: 1317 type: linear 1318 showPoints: auto 1319 spanNulls: false 1320 stacking: 1321 group: A 1322 mode: none 1323 thresholdsStyle: 1324 mode: 'off' 1325 mappings: [] 1326 thresholds: 1327 mode: absolute 1328 steps: 1329 - color: green 1330 value: 0 1331 - color: red 1332 value: 80 1333 overrides: [] 1334 gridPos: 1335 h: 7 1336 w: 5 1337 x: 8 1338 'y': 26 1339 id: 103 1340 options: 1341 legend: 1342 calcs: [] 1343 displayMode: list 1344 placement: bottom 1345 showLegend: true 1346 tooltip: 1347 hideZeros: false 1348 mode: single 1349 sort: none 1350 pluginVersion: 12.1.0-90058 1351 targets: 1352 - datasource: 1353 type: prometheus 1354 uid: ${datasource} 1355 editorMode: code 1356 expr: >- 1357 sum by () 1358 (rate(envoy_cluster_upstream_cx_connect_fail{namespace="$namespace",container="envoy"}[$__rate_interval])) 1359 hide: false 1360 instant: false 1361 legendFormat: '{{"{{pod}}"}} fail' 1362 range: true 1363 refId: B 1364 - datasource: 1365 type: prometheus 1366 uid: ${datasource} 1367 editorMode: code 1368 expr: >- 1369 sum by () 1370 (rate(envoy_cluster_upstream_cx_connect_timeout{namespace="$namespace",container="envoy"}[$__rate_interval])) 1371 hide: false 1372 instant: false 1373 legendFormat: '{{"{{pod}}"}} timeout' 1374 range: true 1375 refId: C 1376 - datasource: 1377 type: prometheus 1378 uid: ${datasource} 1379 editorMode: code 1380 expr: >- 1381 sum by () 1382 (rate(envoy_cluster_upstream_cx_connect_attempts_exceeded{namespace="$namespace",container="envoy"}[$__rate_interval])) 1383 hide: false 1384 instant: false 1385 legendFormat: attempts exceeded 1386 range: true 1387 refId: A 1388 title: Envoy CX Connect 1389 transparent: true 1390 type: timeseries 1391 - datasource: 1392 type: prometheus 1393 uid: ${datasource} 1394 description: |- 1395 How and when upstream connections are destroyed. 1396 1397 Only connections with 1+ in-flight requests are accounted for. 1398 fieldConfig: 1399 defaults: 1400 color: 1401 mode: palette-classic 1402 custom: 1403 axisBorderShow: false 1404 axisCenteredZero: false 1405 axisColorMode: text 1406 axisLabel: '' 1407 axisPlacement: auto 1408 barAlignment: 0 1409 barWidthFactor: 0.6 1410 drawStyle: line 1411 fillOpacity: 0 1412 gradientMode: none 1413 hideFrom: 1414 legend: false 1415 tooltip: false 1416 viz: false 1417 insertNulls: false 1418 lineInterpolation: linear 1419 lineWidth: 1 1420 pointSize: 5 1421 scaleDistribution: 1422 type: linear 1423 showPoints: auto 1424 spanNulls: false 1425 stacking: 1426 group: A 1427 mode: none 1428 thresholdsStyle: 1429 mode: 'off' 1430 mappings: [] 1431 thresholds: 1432 mode: absolute 1433 steps: 1434 - color: green 1435 value: 0 1436 - color: red 1437 value: 80 1438 overrides: [] 1439 gridPos: 1440 h: 7 1441 w: 5 1442 x: 13 1443 'y': 26 1444 id: 101 1445 options: 1446 legend: 1447 calcs: [] 1448 displayMode: list 1449 placement: bottom 1450 showLegend: true 1451 tooltip: 1452 hideZeros: false 1453 mode: single 1454 sort: none 1455 pluginVersion: 12.1.0-90058 1456 targets: 1457 - datasource: 1458 type: prometheus 1459 uid: ${datasource} 1460 editorMode: code 1461 expr: >- 1462 sum by () 1463 (rate(envoy_cluster_upstream_cx_destroy_local_with_active_rq{namespace="$namespace",container="envoy"}[$__rate_interval])) 1464 hide: false 1465 instant: false 1466 legendFormat: local destroy 1467 range: true 1468 refId: B 1469 - datasource: 1470 type: prometheus 1471 uid: ${datasource} 1472 editorMode: code 1473 expr: >- 1474 sum by () 1475 (rate(envoy_cluster_upstream_cx_destroy_remote_with_active_rq{namespace="$namespace",container="envoy"}[$__rate_interval])) 1476 hide: false 1477 instant: false 1478 legendFormat: remote destroy 1479 range: true 1480 refId: C 1481 title: Envoy CX Destroy 1482 transparent: true 1483 type: timeseries 1484 - datasource: 1485 type: prometheus 1486 uid: ${datasource} 1487 description: Connection errors 1488 fieldConfig: 1489 defaults: 1490 color: 1491 mode: palette-classic 1492 custom: 1493 axisBorderShow: false 1494 axisCenteredZero: false 1495 axisColorMode: text 1496 axisLabel: '' 1497 axisPlacement: auto 1498 barAlignment: 0 1499 barWidthFactor: 0.6 1500 drawStyle: line 1501 fillOpacity: 0 1502 gradientMode: none 1503 hideFrom: 1504 legend: false 1505 tooltip: false 1506 viz: false 1507 insertNulls: false 1508 lineInterpolation: linear 1509 lineWidth: 1 1510 pointSize: 5 1511 scaleDistribution: 1512 type: linear 1513 showPoints: auto 1514 spanNulls: false 1515 stacking: 1516 group: A 1517 mode: none 1518 thresholdsStyle: 1519 mode: 'off' 1520 mappings: [] 1521 thresholds: 1522 mode: absolute 1523 steps: 1524 - color: green 1525 value: 0 1526 - color: red 1527 value: 80 1528 overrides: [] 1529 gridPos: 1530 h: 7 1531 w: 6 1532 x: 18 1533 'y': 26 1534 id: 104 1535 options: 1536 legend: 1537 calcs: [] 1538 displayMode: list 1539 placement: bottom 1540 showLegend: true 1541 tooltip: 1542 hideZeros: false 1543 mode: single 1544 sort: none 1545 pluginVersion: 12.1.0-90058 1546 targets: 1547 - datasource: 1548 type: prometheus 1549 uid: ${datasource} 1550 editorMode: code 1551 expr: >- 1552 sum by () 1553 (rate(envoy_cluster_upstream_cx_protocol_error{namespace="$namespace",container="envoy"}[$__rate_interval])) 1554 hide: false 1555 instant: false 1556 legendFormat: protocol error 1557 range: true 1558 refId: B 1559 - datasource: 1560 type: prometheus 1561 uid: ${datasource} 1562 editorMode: code 1563 expr: >- 1564 sum by () 1565 (rate(envoy_cluster_upstream_cx_idle_timeout{namespace="$namespace",container="envoy"}[$__rate_interval])) 1566 hide: false 1567 instant: false 1568 legendFormat: idle timeout 1569 range: true 1570 refId: A 1571 - datasource: 1572 type: prometheus 1573 uid: ${datasource} 1574 editorMode: code 1575 expr: >- 1576 sum by () 1577 (rate(envoy_cluster_upstream_cx_overflow{namespace="$namespace",container="envoy"}[$__rate_interval])) 1578 hide: false 1579 instant: false 1580 legendFormat: cb overflow 1581 range: true 1582 refId: C 1583 - datasource: 1584 type: prometheus 1585 uid: ${datasource} 1586 editorMode: code 1587 expr: >- 1588 sum by () 1589 (rate(envoy_cluster_upstream_cx_pool_overflow{namespace="$namespace",container="envoy"}[$__rate_interval])) 1590 hide: false 1591 instant: false 1592 legendFormat: pool overflow 1593 range: true 1594 refId: E 1595 title: Envoy CX Errors 1596 transparent: true 1597 type: timeseries 1598 title: Cortex Gateway Networking 1599 type: row 1600 {{- end }} 1601 - collapsed: true 1602 gridPos: 1603 h: 1 1604 w: 24 1605 x: 0 1606 'y': 19 1607 id: 90 1608 panels: 1609 - datasource: 1610 type: datasource 1611 uid: '-- Mixed --' 1612 description: >- 1613 The latency observed by distributor, indicating the overall write path 1614 latency. 1615 1616 1617 Only successful requests are counted. 1618 fieldConfig: 1619 defaults: 1620 color: 1621 fixedColor: dark-red 1622 mode: palette-classic 1623 custom: 1624 axisBorderShow: true 1625 axisCenteredZero: true 1626 axisColorMode: text 1627 axisLabel: '' 1628 axisPlacement: auto 1629 barAlignment: 0 1630 barWidthFactor: 0.6 1631 drawStyle: line 1632 fillOpacity: 0 1633 gradientMode: none 1634 hideFrom: 1635 legend: false 1636 tooltip: false 1637 viz: false 1638 insertNulls: false 1639 lineInterpolation: linear 1640 lineStyle: 1641 fill: solid 1642 lineWidth: 1 1643 pointSize: 1 1644 scaleDistribution: 1645 type: linear 1646 showPoints: never 1647 spanNulls: false 1648 stacking: 1649 group: A 1650 mode: none 1651 thresholdsStyle: 1652 mode: 'off' 1653 fieldMinMax: false 1654 mappings: [] 1655 min: 0 1656 thresholds: 1657 mode: absolute 1658 steps: 1659 - color: green 1660 value: 0 1661 unit: s 1662 overrides: 1663 - matcher: 1664 id: byName 1665 options: p99 overall 1666 properties: 1667 - id: custom.lineWidth 1668 value: 2 1669 - id: color 1670 value: 1671 fixedColor: green 1672 mode: fixed 1673 gridPos: 1674 h: 8 1675 w: 8 1676 x: 0 1677 'y': 20 1678 id: 94 1679 options: 1680 legend: 1681 calcs: 1682 - mean 1683 displayMode: table 1684 placement: right 1685 showLegend: false 1686 tooltip: 1687 hideZeros: false 1688 mode: multi 1689 sort: desc 1690 pluginVersion: 12.1.0-90058 1691 targets: 1692 - datasource: 1693 type: prometheus 1694 uid: ${datasource} 1695 editorMode: code 1696 expr: >- 1697 histogram_quantile(0.99, sum by (le,pod) 1698 (rate(pyroscope_request_duration_seconds_bucket{ 1699 namespace="$namespace", 1700 container="distributor", 1701 route=~".*pusher.*|.*ingest.*", 1702 status_code="200", 1703 }[$__rate_interval]))) 1704 hide: false 1705 legendFormat: __auto 1706 range: true 1707 refId: E 1708 - datasource: 1709 type: prometheus 1710 uid: ${datasource} 1711 editorMode: code 1712 expr: >- 1713 histogram_quantile(0.99, sum by (le) 1714 (rate(pyroscope_request_duration_seconds_bucket{ 1715 namespace="$namespace", 1716 container="distributor", 1717 route=~".*pusher.*|.*ingest.*", 1718 status_code="200", 1719 }[$__rate_interval]))) 1720 hide: true 1721 legendFormat: p99 overall 1722 range: true 1723 refId: A 1724 title: Latency 1725 transparent: true 1726 type: timeseries 1727 - datasource: 1728 type: datasource 1729 uid: '-- Mixed --' 1730 fieldConfig: 1731 defaults: 1732 custom: 1733 hideFrom: 1734 legend: false 1735 tooltip: false 1736 viz: false 1737 scaleDistribution: 1738 type: linear 1739 overrides: [] 1740 gridPos: 1741 h: 8 1742 w: 8 1743 x: 8 1744 'y': 20 1745 id: 93 1746 options: 1747 calculate: false 1748 calculation: 1749 xBuckets: 1750 mode: count 1751 value: '' 1752 yBuckets: 1753 mode: count 1754 scale: 1755 type: linear 1756 cellGap: 0 1757 cellValues: 1758 unit: requests 1759 color: 1760 exponent: 0.2 1761 fill: super-light-red 1762 mode: scheme 1763 reverse: true 1764 scale: exponential 1765 scheme: Turbo 1766 steps: 128 1767 exemplars: 1768 color: orange 1769 filterValues: 1770 le: 1.e-9 1771 legend: 1772 show: false 1773 rowsFrame: 1774 layout: auto 1775 tooltip: 1776 mode: single 1777 showColorScale: true 1778 yHistogram: true 1779 yAxis: 1780 axisPlacement: left 1781 reverse: false 1782 unit: s 1783 pluginVersion: 12.1.0-90058 1784 targets: 1785 - datasource: 1786 type: prometheus 1787 uid: ${datasource} 1788 editorMode: code 1789 expr: |- 1790 sum(rate(pyroscope_request_duration_seconds{ 1791 namespace="$namespace", 1792 container="distributor", 1793 route=~".*push.*|.*ingest.*", 1794 }[$__rate_interval])) 1795 hide: false 1796 legendFormat: __auto 1797 range: true 1798 refId: B 1799 title: Latency Heatmap 1800 transparent: true 1801 type: heatmap 1802 - datasource: 1803 type: prometheus 1804 uid: ${datasource} 1805 fieldConfig: 1806 defaults: 1807 color: 1808 mode: palette-classic 1809 custom: 1810 axisBorderShow: false 1811 axisCenteredZero: false 1812 axisColorMode: text 1813 axisLabel: '' 1814 axisPlacement: auto 1815 axisSoftMin: 0 1816 barAlignment: 0 1817 barWidthFactor: 0.6 1818 drawStyle: line 1819 fillOpacity: 0 1820 gradientMode: none 1821 hideFrom: 1822 legend: false 1823 tooltip: false 1824 viz: false 1825 insertNulls: false 1826 lineInterpolation: linear 1827 lineWidth: 1 1828 pointSize: 5 1829 scaleDistribution: 1830 log: 2 1831 type: log 1832 showPoints: never 1833 spanNulls: true 1834 stacking: 1835 group: A 1836 mode: none 1837 thresholdsStyle: 1838 mode: 'off' 1839 fieldMinMax: false 1840 mappings: [] 1841 thresholds: 1842 mode: absolute 1843 steps: 1844 - color: green 1845 value: 0 1846 - color: red 1847 value: 80 1848 unit: reqps 1849 overrides: [] 1850 gridPos: 1851 h: 8 1852 w: 8 1853 x: 16 1854 'y': 20 1855 id: 42 1856 options: 1857 legend: 1858 calcs: [] 1859 displayMode: list 1860 placement: bottom 1861 showLegend: true 1862 sortBy: Name 1863 sortDesc: false 1864 tooltip: 1865 hideZeros: false 1866 mode: multi 1867 sort: none 1868 pluginVersion: 12.1.0-90058 1869 targets: 1870 - datasource: 1871 type: prometheus 1872 uid: ${datasource} 1873 editorMode: code 1874 expr: |- 1875 sum by (status_code) (rate( 1876 pyroscope_request_duration_seconds_count{ 1877 namespace="$namespace", 1878 container="distributor", 1879 route=~".*push.*|.*ingest.*", 1880 }[1m]) 1881 ) 1882 instant: false 1883 legendFormat: __auto 1884 range: true 1885 refId: A 1886 title: Rate 1887 transparent: true 1888 type: timeseries 1889 - datasource: 1890 type: prometheus 1891 uid: ${datasource} 1892 description: >- 1893 The latency observed by distributors (write path router), broken down 1894 by instance. 1895 1896 1897 Latencies are expected to be distributed evenly across instances. An 1898 outlier here likely indicates a neighborhood issue. 1899 1900 1901 Only successful requests are counted. 1902 fieldConfig: 1903 defaults: 1904 color: 1905 mode: palette-classic 1906 custom: 1907 axisBorderShow: false 1908 axisCenteredZero: false 1909 axisColorMode: text 1910 axisLabel: '' 1911 axisPlacement: auto 1912 axisSoftMin: 0 1913 barAlignment: 0 1914 barWidthFactor: 0.6 1915 drawStyle: line 1916 fillOpacity: 0 1917 gradientMode: none 1918 hideFrom: 1919 legend: false 1920 tooltip: false 1921 viz: false 1922 insertNulls: false 1923 lineInterpolation: linear 1924 lineWidth: 1 1925 pointSize: 5 1926 scaleDistribution: 1927 log: 2 1928 type: log 1929 showPoints: auto 1930 spanNulls: false 1931 stacking: 1932 group: A 1933 mode: none 1934 thresholdsStyle: 1935 mode: 'off' 1936 fieldMinMax: false 1937 mappings: [] 1938 thresholds: 1939 mode: absolute 1940 steps: 1941 - color: green 1942 value: 0 1943 - color: red 1944 value: 80 1945 unit: s 1946 overrides: [] 1947 gridPos: 1948 h: 8 1949 w: 8 1950 x: 0 1951 'y': 28 1952 id: 72 1953 options: 1954 legend: 1955 calcs: [] 1956 displayMode: list 1957 placement: bottom 1958 showLegend: false 1959 tooltip: 1960 hideZeros: false 1961 mode: single 1962 sort: none 1963 pluginVersion: 12.1.0-90058 1964 targets: 1965 - datasource: 1966 type: prometheus 1967 uid: ${datasource} 1968 editorMode: code 1969 expr: >- 1970 histogram_quantile(0.99, 1971 sum(rate(pyroscope_write_path_downstream_request_duration_seconds_bucket{ 1972 namespace="$namespace", 1973 status="200", 1974 primary="1", 1975 }[$__rate_interval])) by (le,route,pod)) 1976 hide: false 1977 instant: false 1978 legendFormat: p99 primary {{"{{pod}} => {{route}}"}} 1979 range: true 1980 refId: A 1981 - datasource: 1982 type: prometheus 1983 uid: ${datasource} 1984 editorMode: code 1985 expr: >- 1986 histogram_quantile(0.99, 1987 sum(rate(pyroscope_write_path_downstream_request_duration_seconds_bucket{ 1988 namespace="$namespace", 1989 status="200", 1990 primary!="1", 1991 }[$__rate_interval])) by (le,route,pod)) 1992 hide: false 1993 instant: false 1994 legendFormat: p99 secondary {{"{{pod}} => {{route}}"}} 1995 range: true 1996 refId: B 1997 title: Downstream Latency 1998 transparent: true 1999 type: timeseries 2000 - datasource: 2001 type: datasource 2002 uid: '-- Mixed --' 2003 description: The latency observed by the segment-writer client 2004 fieldConfig: 2005 defaults: 2006 custom: 2007 hideFrom: 2008 legend: false 2009 tooltip: false 2010 viz: false 2011 scaleDistribution: 2012 type: linear 2013 overrides: [] 2014 gridPos: 2015 h: 8 2016 w: 8 2017 x: 8 2018 'y': 28 2019 id: 56 2020 options: 2021 calculate: false 2022 calculation: 2023 xBuckets: 2024 mode: count 2025 value: '' 2026 yBuckets: 2027 mode: count 2028 scale: 2029 type: linear 2030 cellGap: 0 2031 cellValues: 2032 unit: requests 2033 color: 2034 exponent: 0.2 2035 fill: super-light-red 2036 mode: scheme 2037 reverse: true 2038 scale: exponential 2039 scheme: Turbo 2040 steps: 128 2041 exemplars: 2042 color: orange 2043 filterValues: 2044 le: 1.e-9 2045 legend: 2046 show: false 2047 rowsFrame: 2048 layout: auto 2049 tooltip: 2050 mode: single 2051 showColorScale: true 2052 yHistogram: true 2053 yAxis: 2054 axisPlacement: left 2055 reverse: false 2056 unit: s 2057 pluginVersion: 12.1.0-90058 2058 targets: 2059 - datasource: 2060 type: prometheus 2061 uid: ${datasource} 2062 editorMode: code 2063 expr: |- 2064 sum(rate(pyroscope_write_path_downstream_request_duration_seconds{ 2065 namespace="$namespace", 2066 route="segment-writer", 2067 }[$__rate_interval])) 2068 hide: false 2069 legendFormat: __auto 2070 range: true 2071 refId: B 2072 title: Latency Heatmap 2073 transparent: true 2074 type: heatmap 2075 - datasource: 2076 type: prometheus 2077 uid: ${datasource} 2078 fieldConfig: 2079 defaults: 2080 color: 2081 mode: palette-classic 2082 custom: 2083 axisBorderShow: false 2084 axisCenteredZero: false 2085 axisColorMode: text 2086 axisLabel: '' 2087 axisPlacement: auto 2088 axisSoftMin: 0 2089 barAlignment: 0 2090 barWidthFactor: 0.6 2091 drawStyle: line 2092 fillOpacity: 0 2093 gradientMode: none 2094 hideFrom: 2095 legend: false 2096 tooltip: false 2097 viz: false 2098 insertNulls: false 2099 lineInterpolation: linear 2100 lineWidth: 1 2101 pointSize: 5 2102 scaleDistribution: 2103 log: 2 2104 type: log 2105 showPoints: never 2106 spanNulls: true 2107 stacking: 2108 group: A 2109 mode: none 2110 thresholdsStyle: 2111 mode: 'off' 2112 fieldMinMax: false 2113 mappings: [] 2114 thresholds: 2115 mode: absolute 2116 steps: 2117 - color: green 2118 value: 0 2119 - color: red 2120 value: 80 2121 unit: reqps 2122 overrides: [] 2123 gridPos: 2124 h: 8 2125 w: 8 2126 x: 16 2127 'y': 28 2128 id: 95 2129 options: 2130 legend: 2131 calcs: 2132 - last 2133 displayMode: table 2134 placement: right 2135 showLegend: false 2136 sortBy: Name 2137 sortDesc: false 2138 tooltip: 2139 hideZeros: false 2140 mode: multi 2141 sort: none 2142 pluginVersion: 12.1.0-90058 2143 targets: 2144 - datasource: 2145 type: prometheus 2146 uid: ${datasource} 2147 editorMode: code 2148 expr: >- 2149 sum(irate(pyroscope_write_path_downstream_request_duration_seconds_count{ 2150 namespace="$namespace", 2151 primary="1", 2152 }[$__rate_interval])) by (status,route) 2153 instant: false 2154 legendFormat: primary {{"{{route}} {{status}}"}} 2155 range: true 2156 refId: A 2157 - datasource: 2158 type: prometheus 2159 uid: ${datasource} 2160 editorMode: code 2161 expr: >- 2162 sum(irate(pyroscope_write_path_downstream_request_duration_seconds_count{ 2163 namespace="$namespace", 2164 primary="0", 2165 }[$__rate_interval])) by (status,route) 2166 hide: false 2167 instant: false 2168 legendFormat: secondary {{"{{route}} {{status}}"}} 2169 range: true 2170 refId: C 2171 title: Downstream Rate 2172 transparent: true 2173 type: timeseries 2174 - datasource: 2175 type: datasource 2176 uid: '-- Mixed --' 2177 fieldConfig: 2178 defaults: 2179 color: 2180 mode: palette-classic 2181 custom: 2182 axisBorderShow: false 2183 axisCenteredZero: false 2184 axisColorMode: text 2185 axisLabel: '' 2186 axisPlacement: auto 2187 axisSoftMin: 0 2188 barAlignment: 1 2189 barWidthFactor: 1 2190 drawStyle: line 2191 fillOpacity: 0 2192 gradientMode: none 2193 hideFrom: 2194 legend: false 2195 tooltip: false 2196 viz: false 2197 insertNulls: false 2198 lineInterpolation: linear 2199 lineStyle: 2200 fill: solid 2201 lineWidth: 1 2202 pointSize: 5 2203 scaleDistribution: 2204 type: linear 2205 showPoints: auto 2206 spanNulls: false 2207 stacking: 2208 group: A 2209 mode: none 2210 thresholdsStyle: 2211 mode: 'off' 2212 fieldMinMax: false 2213 mappings: [] 2214 min: 0 2215 thresholds: 2216 mode: absolute 2217 steps: 2218 - color: green 2219 value: 0 2220 unit: bytes 2221 overrides: 2222 - matcher: 2223 id: byName 2224 options: Limit 2225 properties: 2226 - id: custom.lineStyle 2227 value: 2228 dash: 2229 - 10 2230 - 10 2231 fill: dash 2232 - matcher: 2233 id: byFrameRefID 2234 options: A 2235 properties: 2236 - id: custom.axisColorMode 2237 value: text 2238 - id: color 2239 value: 2240 fixedColor: semi-dark-red 2241 mode: fixed 2242 - id: custom.lineWidth 2243 value: 2 2244 gridPos: 2245 h: 8 2246 w: 8 2247 x: 0 2248 'y': 36 2249 id: 97 2250 options: 2251 legend: 2252 calcs: 2253 - last 2254 - max 2255 displayMode: table 2256 placement: bottom 2257 showLegend: false 2258 sortBy: Max 2259 sortDesc: true 2260 tooltip: 2261 hideZeros: false 2262 mode: multi 2263 sort: desc 2264 pluginVersion: 12.1.0-90058 2265 targets: 2266 - datasource: 2267 type: prometheus 2268 uid: ${datasource} 2269 editorMode: code 2270 expr: >- 2271 sum by (pod) 2272 (container_memory_working_set_bytes{namespace="$namespace",container="distributor"}) 2273 legendFormat: __auto 2274 range: true 2275 refId: by pod 2276 - datasource: 2277 type: prometheus 2278 uid: ${datasource} 2279 editorMode: code 2280 expr: >- 2281 max(kube_pod_container_resource_requests{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, 2282 namespace="$namespace", resource="memory", 2283 container="distributor"}) 2284 hide: false 2285 legendFormat: Request 2286 range: true 2287 refId: A 2288 title: Memory Usage 2289 transparent: true 2290 type: timeseries 2291 - datasource: 2292 type: datasource 2293 uid: '-- Mixed --' 2294 fieldConfig: 2295 defaults: 2296 color: 2297 mode: palette-classic 2298 custom: 2299 axisBorderShow: false 2300 axisCenteredZero: false 2301 axisColorMode: text 2302 axisLabel: '' 2303 axisPlacement: auto 2304 axisSoftMin: 0 2305 barAlignment: 1 2306 barWidthFactor: 1 2307 drawStyle: line 2308 fillOpacity: 0 2309 gradientMode: none 2310 hideFrom: 2311 legend: false 2312 tooltip: false 2313 viz: false 2314 insertNulls: false 2315 lineInterpolation: linear 2316 lineStyle: 2317 fill: solid 2318 lineWidth: 1 2319 pointSize: 5 2320 scaleDistribution: 2321 type: linear 2322 showPoints: auto 2323 spanNulls: false 2324 stacking: 2325 group: A 2326 mode: none 2327 thresholdsStyle: 2328 mode: 'off' 2329 fieldMinMax: false 2330 mappings: [] 2331 min: 0 2332 thresholds: 2333 mode: absolute 2334 steps: 2335 - color: green 2336 value: 0 2337 overrides: 2338 - matcher: 2339 id: byName 2340 options: Limit 2341 properties: 2342 - id: custom.lineStyle 2343 value: 2344 dash: 2345 - 10 2346 - 10 2347 fill: dash 2348 - matcher: 2349 id: byFrameRefID 2350 options: A 2351 properties: 2352 - id: custom.axisColorMode 2353 value: text 2354 - id: color 2355 value: 2356 fixedColor: semi-dark-red 2357 mode: fixed 2358 - id: custom.lineWidth 2359 value: 2 2360 gridPos: 2361 h: 8 2362 w: 8 2363 x: 8 2364 'y': 36 2365 id: 62 2366 options: 2367 legend: 2368 calcs: 2369 - last 2370 - max 2371 displayMode: table 2372 placement: bottom 2373 showLegend: false 2374 sortBy: Max 2375 sortDesc: true 2376 tooltip: 2377 hideZeros: false 2378 mode: multi 2379 sort: desc 2380 pluginVersion: 12.1.0-90058 2381 targets: 2382 - datasource: 2383 type: prometheus 2384 uid: ${datasource} 2385 editorMode: code 2386 expr: >- 2387 sum by (pod) 2388 (node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace="$namespace",container="distributor"}) 2389 legendFormat: __auto 2390 range: true 2391 refId: by pod 2392 - datasource: 2393 type: prometheus 2394 uid: ${datasource} 2395 editorMode: code 2396 expr: |- 2397 max( 2398 kube_pod_container_resource_requests{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, namespace="$namespace", resource="cpu"} 2399 * on(namespace,pod) 2400 group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{namespace="$namespace", workload=~".*-distributor"} 2401 ) 2402 hide: false 2403 legendFormat: Request 2404 range: true 2405 refId: A 2406 - datasource: 2407 type: prometheus 2408 uid: ${datasource} 2409 editorMode: code 2410 expr: |- 2411 max( 2412 kube_pod_container_resource_limits{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, namespace="$namespace", resource="cpu"} 2413 * on(namespace,pod) 2414 group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{namespace="$namespace", workload=~".*-distributor"} 2415 ) 2416 hide: false 2417 legendFormat: Limit 2418 range: true 2419 refId: B 2420 title: CPU Usage 2421 transparent: true 2422 type: timeseries 2423 - datasource: 2424 type: prometheus 2425 uid: ${datasource} 2426 description: >- 2427 The chart showd distribution of requests from the upstream 2428 (cortex-gw/envoy) 2429 fieldConfig: 2430 defaults: 2431 color: 2432 mode: palette-classic 2433 custom: 2434 axisBorderShow: false 2435 axisCenteredZero: false 2436 axisColorMode: text 2437 axisLabel: '' 2438 axisPlacement: auto 2439 axisSoftMin: 0 2440 barAlignment: 0 2441 barWidthFactor: 0.6 2442 drawStyle: line 2443 fillOpacity: 0 2444 gradientMode: none 2445 hideFrom: 2446 legend: false 2447 tooltip: false 2448 viz: false 2449 insertNulls: false 2450 lineInterpolation: linear 2451 lineWidth: 1 2452 pointSize: 5 2453 scaleDistribution: 2454 log: 2 2455 type: log 2456 showPoints: never 2457 spanNulls: true 2458 stacking: 2459 group: A 2460 mode: none 2461 thresholdsStyle: 2462 mode: 'off' 2463 fieldMinMax: false 2464 mappings: [] 2465 thresholds: 2466 mode: absolute 2467 steps: 2468 - color: green 2469 value: 0 2470 - color: red 2471 value: 80 2472 unit: reqps 2473 overrides: [] 2474 gridPos: 2475 h: 8 2476 w: 8 2477 x: 16 2478 'y': 36 2479 id: 98 2480 options: 2481 legend: 2482 calcs: 2483 - last 2484 displayMode: table 2485 placement: right 2486 showLegend: false 2487 sortBy: Name 2488 sortDesc: false 2489 tooltip: 2490 hideZeros: false 2491 mode: single 2492 sort: asc 2493 pluginVersion: 12.1.0-90058 2494 targets: 2495 - datasource: 2496 type: prometheus 2497 uid: ${datasource} 2498 editorMode: code 2499 expr: |- 2500 sum by (pod) (rate( 2501 pyroscope_request_duration_seconds_count{ 2502 namespace="$namespace", 2503 container="distributor", 2504 route=~".*push.*|.*ingest.*", 2505 }[1m]) 2506 ) 2507 instant: false 2508 legendFormat: __auto 2509 range: true 2510 refId: A 2511 title: Upstream Distribution 2512 transparent: true 2513 type: timeseries 2514 title: Distributor 2515 type: row 2516 - collapsed: false 2517 gridPos: 2518 h: 1 2519 w: 24 2520 x: 0 2521 'y': 20 2522 id: 47 2523 panels: [] 2524 title: Segment Writer 2525 type: row 2526 - datasource: 2527 type: prometheus 2528 uid: ${datasource} 2529 description: The latency of the Push endpoint observed by segment-writer. 2530 fieldConfig: 2531 defaults: 2532 color: 2533 mode: palette-classic 2534 custom: 2535 axisBorderShow: false 2536 axisCenteredZero: false 2537 axisColorMode: text 2538 axisLabel: '' 2539 axisPlacement: auto 2540 barAlignment: 0 2541 barWidthFactor: 0.6 2542 drawStyle: line 2543 fillOpacity: 0 2544 gradientMode: none 2545 hideFrom: 2546 legend: false 2547 tooltip: false 2548 viz: false 2549 insertNulls: false 2550 lineInterpolation: linear 2551 lineWidth: 1 2552 pointSize: 5 2553 scaleDistribution: 2554 type: linear 2555 showPoints: auto 2556 spanNulls: false 2557 stacking: 2558 group: A 2559 mode: none 2560 thresholdsStyle: 2561 mode: 'off' 2562 mappings: [] 2563 thresholds: 2564 mode: absolute 2565 steps: 2566 - color: green 2567 value: 0 2568 - color: red 2569 value: 80 2570 unit: s 2571 overrides: 2572 - matcher: 2573 id: byFrameRefID 2574 options: Overall p99 2575 properties: 2576 - id: custom.lineWidth 2577 value: 2 2578 - id: color 2579 value: 2580 fixedColor: green 2581 mode: fixed 2582 - matcher: 2583 id: byFrameRefID 2584 properties: [] 2585 gridPos: 2586 h: 8 2587 w: 8 2588 x: 0 2589 'y': 21 2590 id: 79 2591 options: 2592 legend: 2593 calcs: [] 2594 displayMode: list 2595 placement: bottom 2596 showLegend: false 2597 tooltip: 2598 hideZeros: false 2599 mode: multi 2600 sort: desc 2601 pluginVersion: 12.1.0-90139 2602 targets: 2603 - datasource: 2604 type: prometheus 2605 uid: ${datasource} 2606 editorMode: code 2607 expr: |- 2608 histogram_quantile(0.99, sum(rate(pyroscope_request_duration_seconds{ 2609 namespace="$namespace", 2610 method="gRPC", 2611 # status_code="success", 2612 route="/segmentwriter.v1.SegmentWriterService/Push", 2613 }[$__rate_interval])) by (le,pod)) 2614 hide: false 2615 instant: false 2616 legendFormat: p99 {{"{{pod}}"}} 2617 range: true 2618 refId: B 2619 - datasource: 2620 type: prometheus 2621 uid: ${datasource} 2622 editorMode: code 2623 expr: |- 2624 histogram_quantile(0.99, sum(rate(pyroscope_request_duration_seconds{ 2625 namespace="$namespace", 2626 method="gRPC", 2627 # status_code="success", 2628 route="/segmentwriter.v1.SegmentWriterService/Push", 2629 }[$__rate_interval])) by (le)) 2630 hide: false 2631 instant: false 2632 legendFormat: p99 overall 2633 range: true 2634 refId: Overall p99 2635 - datasource: 2636 type: prometheus 2637 uid: ${datasource} 2638 editorMode: code 2639 expr: |- 2640 histogram_quantile(0.90, sum(rate(pyroscope_request_duration_seconds{ 2641 namespace="$namespace", 2642 method="gRPC", 2643 # status_code="success", 2644 route="/segmentwriter.v1.SegmentWriterService/Push", 2645 }[$__rate_interval])) by (le)) 2646 hide: false 2647 instant: false 2648 legendFormat: p90 overall 2649 range: true 2650 refId: Overall p90 2651 - datasource: 2652 type: prometheus 2653 uid: ${datasource} 2654 editorMode: code 2655 expr: >- 2656 histogram_quantile(0.50, 2657 sum(rate(pyroscope_request_duration_seconds_bucket{ 2658 namespace="$namespace", 2659 method="gRPC", 2660 # status_code="success", 2661 route="/segmentwriter.v1.SegmentWriterService/Push", 2662 }[$__rate_interval])) by (le)) 2663 hide: false 2664 instant: false 2665 legendFormat: p50 overall 2666 range: true 2667 refId: Overall p50 2668 title: Push Latency 2669 transparent: true 2670 type: timeseries 2671 - datasource: 2672 type: datasource 2673 uid: '-- Mixed --' 2674 description: The latency of the Push endpoint observed by segment-writer 2675 fieldConfig: 2676 defaults: 2677 custom: 2678 hideFrom: 2679 legend: false 2680 tooltip: false 2681 viz: false 2682 scaleDistribution: 2683 type: linear 2684 overrides: [] 2685 gridPos: 2686 h: 8 2687 w: 8 2688 x: 8 2689 'y': 21 2690 id: 76 2691 options: 2692 calculate: false 2693 calculation: 2694 xBuckets: 2695 mode: count 2696 value: '' 2697 yBuckets: 2698 mode: count 2699 scale: 2700 type: linear 2701 cellGap: 0 2702 cellValues: 2703 unit: requests 2704 color: 2705 exponent: 0.2 2706 fill: super-light-red 2707 mode: scheme 2708 reverse: true 2709 scale: exponential 2710 scheme: Turbo 2711 steps: 128 2712 exemplars: 2713 color: orange 2714 filterValues: 2715 le: 1.e-9 2716 legend: 2717 show: false 2718 rowsFrame: 2719 layout: auto 2720 tooltip: 2721 mode: single 2722 showColorScale: true 2723 yHistogram: true 2724 yAxis: 2725 axisPlacement: left 2726 reverse: false 2727 unit: s 2728 pluginVersion: 12.1.0-90139 2729 targets: 2730 - datasource: 2731 type: prometheus 2732 uid: ${datasource} 2733 editorMode: code 2734 expr: |- 2735 sum(rate(pyroscope_request_duration_seconds{ 2736 namespace="$namespace", 2737 container="segment-writer", 2738 method="gRPC", 2739 # status_code="success", 2740 route="/segmentwriter.v1.SegmentWriterService/Push", 2741 }[$__rate_interval])) 2742 hide: false 2743 legendFormat: __auto 2744 range: true 2745 refId: B 2746 title: Latency Heatmap 2747 transparent: true 2748 type: heatmap 2749 - datasource: 2750 type: prometheus 2751 uid: ${datasource} 2752 fieldConfig: 2753 defaults: 2754 color: 2755 mode: palette-classic 2756 custom: 2757 axisBorderShow: false 2758 axisCenteredZero: false 2759 axisColorMode: text 2760 axisLabel: '' 2761 axisPlacement: auto 2762 axisSoftMin: 0 2763 barAlignment: 0 2764 barWidthFactor: 0.6 2765 drawStyle: line 2766 fillOpacity: 0 2767 gradientMode: none 2768 hideFrom: 2769 legend: false 2770 tooltip: false 2771 viz: false 2772 insertNulls: false 2773 lineInterpolation: linear 2774 lineWidth: 1 2775 pointSize: 5 2776 scaleDistribution: 2777 log: 2 2778 type: log 2779 showPoints: auto 2780 spanNulls: false 2781 stacking: 2782 group: A 2783 mode: none 2784 thresholdsStyle: 2785 mode: 'off' 2786 fieldMinMax: false 2787 mappings: [] 2788 thresholds: 2789 mode: absolute 2790 steps: 2791 - color: green 2792 value: 0 2793 - color: red 2794 value: 80 2795 unit: reqps 2796 overrides: [] 2797 gridPos: 2798 h: 8 2799 w: 8 2800 x: 16 2801 'y': 21 2802 id: 52 2803 options: 2804 legend: 2805 calcs: [] 2806 displayMode: list 2807 placement: bottom 2808 showLegend: true 2809 tooltip: 2810 hideZeros: false 2811 mode: single 2812 sort: none 2813 pluginVersion: 12.1.0-90139 2814 targets: 2815 - datasource: 2816 type: prometheus 2817 uid: ${datasource} 2818 editorMode: code 2819 expr: |- 2820 sum(irate(pyroscope_request_duration_seconds_count{ 2821 namespace="$namespace", 2822 method="gRPC", 2823 route="/segmentwriter.v1.SegmentWriterService/Push", 2824 }[$__rate_interval])) by (status_code) 2825 instant: false 2826 legendFormat: __auto 2827 range: true 2828 refId: A 2829 title: Push Rate 2830 transparent: true 2831 type: timeseries 2832 - datasource: 2833 type: prometheus 2834 uid: ${datasource} 2835 description: >- 2836 The latency segment-writer observes when uploading segments to object 2837 store, including retries. 2838 fieldConfig: 2839 defaults: 2840 color: 2841 mode: palette-classic 2842 custom: 2843 axisBorderShow: false 2844 axisCenteredZero: false 2845 axisColorMode: text 2846 axisLabel: '' 2847 axisPlacement: auto 2848 barAlignment: 0 2849 barWidthFactor: 0.6 2850 drawStyle: line 2851 fillOpacity: 0 2852 gradientMode: none 2853 hideFrom: 2854 legend: false 2855 tooltip: false 2856 viz: false 2857 insertNulls: false 2858 lineInterpolation: linear 2859 lineWidth: 1 2860 pointSize: 5 2861 scaleDistribution: 2862 type: linear 2863 showPoints: auto 2864 spanNulls: false 2865 stacking: 2866 group: A 2867 mode: none 2868 thresholdsStyle: 2869 mode: 'off' 2870 mappings: [] 2871 thresholds: 2872 mode: absolute 2873 steps: 2874 - color: green 2875 value: 0 2876 - color: red 2877 value: 80 2878 unit: s 2879 overrides: 2880 - matcher: 2881 id: byFrameRefID 2882 options: Overall p99 2883 properties: 2884 - id: custom.lineWidth 2885 value: 2 2886 - id: color 2887 value: 2888 fixedColor: green 2889 mode: fixed 2890 - matcher: 2891 id: byFrameRefID 2892 properties: [] 2893 gridPos: 2894 h: 8 2895 w: 8 2896 x: 0 2897 'y': 29 2898 id: 57 2899 options: 2900 legend: 2901 calcs: [] 2902 displayMode: list 2903 placement: bottom 2904 showLegend: false 2905 tooltip: 2906 hideZeros: false 2907 mode: multi 2908 sort: desc 2909 pluginVersion: 12.1.0-90139 2910 targets: 2911 - datasource: 2912 type: prometheus 2913 uid: ${datasource} 2914 editorMode: code 2915 expr: >- 2916 histogram_quantile(0.99, 2917 sum(rate(pyroscope_segment_writer_upload_duration_seconds_bucket{namespace="$namespace",container="segment-writer"}[$__rate_interval])) 2918 by (le,pod)) 2919 hide: false 2920 instant: false 2921 legendFormat: p99 {{"{{pod}}"}} 2922 range: true 2923 refId: B 2924 - datasource: 2925 type: prometheus 2926 uid: ${datasource} 2927 editorMode: code 2928 expr: >- 2929 histogram_quantile(0.99, 2930 sum(rate(pyroscope_segment_writer_upload_duration_seconds_bucket{namespace="$namespace",container="segment-writer"}[$__rate_interval])) 2931 by (le)) 2932 hide: false 2933 instant: false 2934 legendFormat: p99 overall 2935 range: true 2936 refId: Overall p99 2937 - datasource: 2938 type: prometheus 2939 uid: ${datasource} 2940 editorMode: code 2941 expr: >- 2942 histogram_quantile(0.90, 2943 sum(rate(pyroscope_segment_writer_upload_duration_seconds_bucket{namespace="$namespace",container="segment-writer"}[$__rate_interval])) 2944 by (le)) 2945 hide: false 2946 instant: false 2947 legendFormat: p90 overall 2948 range: true 2949 refId: Overall p90 2950 - datasource: 2951 type: prometheus 2952 uid: ${datasource} 2953 editorMode: code 2954 expr: >- 2955 histogram_quantile(0.50, 2956 sum(rate(pyroscope_segment_writer_upload_duration_seconds_bucket{namespace="$namespace",container="segment-writer"}[$__rate_interval])) 2957 by (le)) 2958 hide: false 2959 instant: false 2960 legendFormat: p50 overall 2961 range: true 2962 refId: Overall p50 2963 title: Object Store Upload Latency 2964 transparent: true 2965 type: timeseries 2966 - datasource: 2967 type: datasource 2968 uid: '-- Mixed --' 2969 description: The latency of upload calls observed by the segment-writer 2970 fieldConfig: 2971 defaults: 2972 custom: 2973 hideFrom: 2974 legend: false 2975 tooltip: false 2976 viz: false 2977 scaleDistribution: 2978 type: linear 2979 overrides: [] 2980 gridPos: 2981 h: 8 2982 w: 8 2983 x: 8 2984 'y': 29 2985 id: 80 2986 options: 2987 calculate: false 2988 calculation: 2989 xBuckets: 2990 mode: count 2991 value: '' 2992 yBuckets: 2993 mode: count 2994 scale: 2995 type: linear 2996 cellGap: 0 2997 cellValues: 2998 unit: requests 2999 color: 3000 exponent: 0.2 3001 fill: super-light-red 3002 mode: scheme 3003 reverse: true 3004 scale: exponential 3005 scheme: Turbo 3006 steps: 128 3007 exemplars: 3008 color: orange 3009 filterValues: 3010 le: 1.e-9 3011 legend: 3012 show: false 3013 rowsFrame: 3014 layout: auto 3015 tooltip: 3016 mode: single 3017 showColorScale: true 3018 yHistogram: true 3019 yAxis: 3020 axisPlacement: left 3021 reverse: false 3022 unit: s 3023 pluginVersion: 12.1.0-90139 3024 targets: 3025 - datasource: 3026 type: prometheus 3027 uid: ${datasource} 3028 editorMode: code 3029 expr: >- 3030 sum(rate(pyroscope_segment_writer_upload_duration_seconds{namespace="$namespace",container="segment-writer"}[$__rate_interval])) 3031 hide: false 3032 legendFormat: __auto 3033 range: true 3034 refId: B 3035 title: Latency Heatmap 3036 transparent: true 3037 type: heatmap 3038 - datasource: 3039 type: prometheus 3040 uid: ${datasource} 3041 description: New objects created by segment-writer 3042 fieldConfig: 3043 defaults: 3044 color: 3045 mode: palette-classic 3046 custom: 3047 axisBorderShow: false 3048 axisCenteredZero: false 3049 axisColorMode: text 3050 axisLabel: '' 3051 axisPlacement: auto 3052 axisSoftMin: 0 3053 barAlignment: 0 3054 barWidthFactor: 0.6 3055 drawStyle: line 3056 fillOpacity: 0 3057 gradientMode: none 3058 hideFrom: 3059 legend: false 3060 tooltip: false 3061 viz: false 3062 insertNulls: false 3063 lineInterpolation: linear 3064 lineWidth: 1 3065 pointSize: 5 3066 scaleDistribution: 3067 type: linear 3068 showPoints: auto 3069 spanNulls: false 3070 stacking: 3071 group: A 3072 mode: none 3073 thresholdsStyle: 3074 mode: 'off' 3075 mappings: [] 3076 thresholds: 3077 mode: absolute 3078 steps: 3079 - color: green 3080 value: 0 3081 - color: red 3082 value: 80 3083 unit: ops 3084 overrides: 3085 - matcher: 3086 id: byFrameRefID 3087 options: B 3088 properties: 3089 - id: custom.axisPlacement 3090 value: right 3091 - id: custom.axisLabel 3092 value: Hedged requests 3093 gridPos: 3094 h: 8 3095 w: 8 3096 x: 16 3097 'y': 29 3098 id: 59 3099 options: 3100 legend: 3101 calcs: [] 3102 displayMode: list 3103 placement: bottom 3104 showLegend: true 3105 tooltip: 3106 hideZeros: false 3107 mode: single 3108 sort: none 3109 pluginVersion: 12.1.0-90139 3110 targets: 3111 - datasource: 3112 type: prometheus 3113 uid: ${datasource} 3114 editorMode: code 3115 expr: >- 3116 sum by (status) 3117 (rate(pyroscope_segment_writer_upload_duration_seconds_count{namespace="$namespace"}[$__rate_interval])) 3118 hide: false 3119 instant: false 3120 legendFormat: __auto 3121 range: true 3122 refId: A 3123 - datasource: 3124 type: prometheus 3125 uid: ${datasource} 3126 editorMode: code 3127 expr: >- 3128 sum by (status) 3129 (rate(pyroscope_segment_writer_hedged_upload_duration_seconds_count{namespace="$namespace"}[$__rate_interval])) 3130 hide: false 3131 instant: false 3132 legendFormat: '{{"{{status}}"}} (hedged)' 3133 range: true 3134 refId: B 3135 title: Object Store Upload Rate 3136 transparent: true 3137 type: timeseries 3138 - datasource: 3139 type: prometheus 3140 uid: ${datasource} 3141 description: >- 3142 The latency segment-writer observes when adding metadata entries to 3143 metastore, including retries. 3144 fieldConfig: 3145 defaults: 3146 color: 3147 mode: palette-classic 3148 custom: 3149 axisBorderShow: false 3150 axisCenteredZero: false 3151 axisColorMode: text 3152 axisLabel: '' 3153 axisPlacement: auto 3154 barAlignment: 0 3155 barWidthFactor: 0.6 3156 drawStyle: line 3157 fillOpacity: 0 3158 gradientMode: none 3159 hideFrom: 3160 legend: false 3161 tooltip: false 3162 viz: false 3163 insertNulls: false 3164 lineInterpolation: linear 3165 lineWidth: 1 3166 pointSize: 5 3167 scaleDistribution: 3168 type: linear 3169 showPoints: auto 3170 spanNulls: false 3171 stacking: 3172 group: A 3173 mode: none 3174 thresholdsStyle: 3175 mode: 'off' 3176 mappings: [] 3177 thresholds: 3178 mode: absolute 3179 steps: 3180 - color: green 3181 value: 0 3182 - color: red 3183 value: 80 3184 unit: s 3185 overrides: 3186 - matcher: 3187 id: byFrameRefID 3188 options: Overall p99 3189 properties: 3190 - id: custom.lineWidth 3191 value: 2 3192 - id: color 3193 value: 3194 fixedColor: green 3195 mode: fixed 3196 - matcher: 3197 id: byFrameRefID 3198 properties: [] 3199 gridPos: 3200 h: 8 3201 w: 8 3202 x: 0 3203 'y': 37 3204 id: 77 3205 options: 3206 legend: 3207 calcs: [] 3208 displayMode: list 3209 placement: bottom 3210 showLegend: false 3211 tooltip: 3212 hideZeros: false 3213 mode: multi 3214 sort: desc 3215 pluginVersion: 12.1.0-90139 3216 targets: 3217 - datasource: 3218 type: prometheus 3219 uid: ${datasource} 3220 editorMode: code 3221 expr: >- 3222 histogram_quantile(0.99, 3223 sum(rate(pyroscope_segment_writer_store_metadata_duration_seconds_bucket{namespace="$namespace",container="segment-writer"}[$__rate_interval])) 3224 by (le,pod)) 3225 hide: false 3226 instant: false 3227 legendFormat: p99 {{"{{pod}}"}} 3228 range: true 3229 refId: B 3230 - datasource: 3231 type: prometheus 3232 uid: ${datasource} 3233 editorMode: code 3234 expr: >- 3235 histogram_quantile(0.99, 3236 sum(rate(pyroscope_segment_writer_store_metadata_duration_seconds_bucket{namespace="$namespace",container="segment-writer"}[$__rate_interval])) 3237 by (le)) 3238 hide: false 3239 instant: false 3240 legendFormat: p99 overall 3241 range: true 3242 refId: Overall p99 3243 - datasource: 3244 type: prometheus 3245 uid: ${datasource} 3246 editorMode: code 3247 expr: >- 3248 histogram_quantile(0.90, 3249 sum(rate(pyroscope_segment_writer_store_metadata_duration_seconds_bucket{namespace="$namespace",container="segment-writer"}[$__rate_interval])) 3250 by (le)) 3251 hide: false 3252 instant: false 3253 legendFormat: p90 overall 3254 range: true 3255 refId: Overall p90 3256 - datasource: 3257 type: prometheus 3258 uid: ${datasource} 3259 editorMode: code 3260 expr: >- 3261 histogram_quantile(0.50, 3262 sum(rate(pyroscope_segment_writer_store_metadata_duration_seconds_bucket{namespace="$namespace",container="segment-writer"}[$__rate_interval])) 3263 by (le)) 3264 hide: false 3265 instant: false 3266 legendFormat: p50 overall 3267 range: true 3268 refId: Overall p50 3269 title: Store Metadata Latency 3270 transparent: true 3271 type: timeseries 3272 - datasource: 3273 type: datasource 3274 uid: '-- Mixed --' 3275 description: >- 3276 The latency segment-writer observes when adding metadata entries, 3277 including retries 3278 fieldConfig: 3279 defaults: 3280 custom: 3281 hideFrom: 3282 legend: false 3283 tooltip: false 3284 viz: false 3285 scaleDistribution: 3286 type: linear 3287 overrides: [] 3288 gridPos: 3289 h: 8 3290 w: 8 3291 x: 8 3292 'y': 37 3293 id: 78 3294 options: 3295 calculate: false 3296 calculation: 3297 xBuckets: 3298 mode: count 3299 value: '' 3300 yBuckets: 3301 mode: count 3302 scale: 3303 type: linear 3304 cellGap: 0 3305 cellValues: 3306 unit: requests 3307 color: 3308 exponent: 0.2 3309 fill: super-light-red 3310 mode: scheme 3311 reverse: true 3312 scale: exponential 3313 scheme: Turbo 3314 steps: 128 3315 exemplars: 3316 color: orange 3317 filterValues: 3318 le: 1.e-9 3319 legend: 3320 show: false 3321 rowsFrame: 3322 layout: auto 3323 tooltip: 3324 mode: single 3325 showColorScale: true 3326 yHistogram: true 3327 yAxis: 3328 axisPlacement: left 3329 reverse: false 3330 unit: s 3331 pluginVersion: 12.1.0-90139 3332 targets: 3333 - datasource: 3334 type: prometheus 3335 uid: ${datasource} 3336 editorMode: code 3337 expr: >- 3338 sum(rate(pyroscope_segment_writer_store_metadata_duration_seconds{namespace="$namespace",container="segment-writer"}[$__rate_interval])) 3339 hide: false 3340 legendFormat: __auto 3341 range: true 3342 refId: B 3343 title: Latency Heatmap 3344 transparent: true 3345 type: heatmap 3346 - datasource: 3347 type: prometheus 3348 uid: ${datasource} 3349 description: New metadata entries created by segment-writers 3350 fieldConfig: 3351 defaults: 3352 color: 3353 mode: palette-classic 3354 custom: 3355 axisBorderShow: false 3356 axisCenteredZero: false 3357 axisColorMode: text 3358 axisLabel: '' 3359 axisPlacement: auto 3360 axisSoftMin: 0 3361 barAlignment: 0 3362 barWidthFactor: 0.6 3363 drawStyle: line 3364 fillOpacity: 0 3365 gradientMode: none 3366 hideFrom: 3367 legend: false 3368 tooltip: false 3369 viz: false 3370 insertNulls: false 3371 lineInterpolation: linear 3372 lineWidth: 1 3373 pointSize: 5 3374 scaleDistribution: 3375 type: linear 3376 showPoints: auto 3377 spanNulls: false 3378 stacking: 3379 group: A 3380 mode: none 3381 thresholdsStyle: 3382 mode: 'off' 3383 mappings: [] 3384 thresholds: 3385 mode: absolute 3386 steps: 3387 - color: green 3388 value: 0 3389 - color: red 3390 value: 80 3391 unit: ops 3392 overrides: 3393 - matcher: 3394 id: byFrameRefID 3395 options: B 3396 properties: 3397 - id: custom.drawStyle 3398 value: points 3399 - id: color 3400 value: 3401 fixedColor: red 3402 mode: fixed 3403 - id: custom.pointSize 3404 value: 11 3405 gridPos: 3406 h: 8 3407 w: 8 3408 x: 16 3409 'y': 37 3410 id: 73 3411 options: 3412 legend: 3413 calcs: [] 3414 displayMode: list 3415 placement: bottom 3416 showLegend: true 3417 tooltip: 3418 hideZeros: false 3419 mode: single 3420 sort: none 3421 pluginVersion: 12.1.0-90139 3422 targets: 3423 - datasource: 3424 type: prometheus 3425 uid: ${datasource} 3426 editorMode: code 3427 expr: >- 3428 sum by (status) 3429 (rate(pyroscope_segment_writer_store_metadata_duration_seconds_count{namespace="$namespace"}[$__rate_interval])) 3430 hide: false 3431 instant: false 3432 legendFormat: __auto 3433 range: true 3434 refId: A 3435 - datasource: 3436 type: prometheus 3437 uid: ${datasource} 3438 editorMode: code 3439 expr: >- 3440 sum by (status) 3441 (rate(pyroscope_segment_writer_store_metadata_dql{namespace="$namespace"}[$__rate_interval])) 3442 hide: false 3443 instant: false 3444 legendFormat: '{{"{{status}}"}} (DLQ)' 3445 range: true 3446 refId: B 3447 - datasource: 3448 type: prometheus 3449 uid: ${datasource} 3450 editorMode: code 3451 expr: >- 3452 sum(rate(pyroscope_segment_writer_store_metadata_dlq{namespace="$namespace"}[$__rate_interval])) 3453 hide: false 3454 instant: false 3455 legendFormat: dlq 3456 range: true 3457 refId: C 3458 title: Store Metadata Rate 3459 transparent: true 3460 type: timeseries 3461 - datasource: 3462 default: false 3463 type: prometheus 3464 uid: ${datasource} 3465 fieldConfig: 3466 defaults: 3467 color: 3468 mode: palette-classic 3469 custom: 3470 axisBorderShow: false 3471 axisCenteredZero: false 3472 axisColorMode: text 3473 axisLabel: '' 3474 axisPlacement: auto 3475 axisSoftMin: 0 3476 barAlignment: 0 3477 barWidthFactor: 0.6 3478 drawStyle: line 3479 fillOpacity: 0 3480 gradientMode: none 3481 hideFrom: 3482 legend: false 3483 tooltip: false 3484 viz: false 3485 insertNulls: false 3486 lineInterpolation: linear 3487 lineWidth: 1 3488 pointSize: 5 3489 scaleDistribution: 3490 type: linear 3491 showPoints: auto 3492 spanNulls: false 3493 stacking: 3494 group: A 3495 mode: none 3496 thresholdsStyle: 3497 mode: 'off' 3498 mappings: [] 3499 thresholds: 3500 mode: absolute 3501 steps: 3502 - color: green 3503 value: 0 3504 - color: red 3505 value: 80 3506 unit: bytes 3507 overrides: [] 3508 gridPos: 3509 h: 8 3510 w: 8 3511 x: 0 3512 'y': 45 3513 id: 21 3514 options: 3515 legend: 3516 calcs: 3517 - mean 3518 displayMode: table 3519 placement: right 3520 showLegend: false 3521 tooltip: 3522 hideZeros: false 3523 mode: multi 3524 sort: desc 3525 pluginVersion: 12.1.0-90139 3526 targets: 3527 - datasource: 3528 type: prometheus 3529 uid: ${datasource} 3530 editorMode: code 3531 expr: >- 3532 sum by (pod) 3533 (irate(pyroscope_segment_writer_segment_ingest_bytes_sum{namespace="$namespace", 3534 container="segment-writer"}[10m])) / 3535 3536 sum by (pod) 3537 (rate(container_cpu_usage_seconds_total{namespace="$namespace", 3538 container="segment-writer"}[10m])) 3539 legendFormat: __auto 3540 range: true 3541 refId: A 3542 title: Processing Rate (MB/sec/core) 3543 transparent: true 3544 type: timeseries 3545 - datasource: 3546 type: datasource 3547 uid: '-- Mixed --' 3548 fieldConfig: 3549 defaults: 3550 color: 3551 mode: palette-classic 3552 custom: 3553 axisBorderShow: false 3554 axisCenteredZero: false 3555 axisColorMode: text 3556 axisLabel: '' 3557 axisPlacement: auto 3558 axisSoftMin: 0 3559 barAlignment: 1 3560 barWidthFactor: 1 3561 drawStyle: line 3562 fillOpacity: 0 3563 gradientMode: none 3564 hideFrom: 3565 legend: false 3566 tooltip: false 3567 viz: false 3568 insertNulls: false 3569 lineInterpolation: linear 3570 lineStyle: 3571 fill: solid 3572 lineWidth: 1 3573 pointSize: 5 3574 scaleDistribution: 3575 type: linear 3576 showPoints: auto 3577 spanNulls: false 3578 stacking: 3579 group: A 3580 mode: none 3581 thresholdsStyle: 3582 mode: 'off' 3583 fieldMinMax: false 3584 mappings: [] 3585 min: 0 3586 thresholds: 3587 mode: absolute 3588 steps: 3589 - color: green 3590 value: 0 3591 overrides: 3592 - matcher: 3593 id: byName 3594 options: Limit 3595 properties: 3596 - id: custom.lineStyle 3597 value: 3598 dash: 3599 - 10 3600 - 10 3601 fill: dash 3602 - matcher: 3603 id: byFrameRefID 3604 options: A 3605 properties: 3606 - id: custom.axisColorMode 3607 value: text 3608 - id: color 3609 value: 3610 fixedColor: semi-dark-red 3611 mode: fixed 3612 - id: custom.lineWidth 3613 value: 2 3614 gridPos: 3615 h: 8 3616 w: 8 3617 x: 8 3618 'y': 45 3619 id: 63 3620 options: 3621 legend: 3622 calcs: 3623 - last 3624 - max 3625 displayMode: table 3626 placement: bottom 3627 showLegend: false 3628 sortBy: Max 3629 sortDesc: true 3630 tooltip: 3631 hideZeros: false 3632 mode: multi 3633 sort: desc 3634 pluginVersion: 12.1.0-90139 3635 targets: 3636 - datasource: 3637 type: prometheus 3638 uid: ${datasource} 3639 editorMode: code 3640 expr: >- 3641 sum by (pod) 3642 (irate(container_cpu_usage_seconds_total{namespace="$namespace",container="segment-writer"}[$__rate_interval])) 3643 legendFormat: __auto 3644 range: true 3645 refId: by pod 3646 - datasource: 3647 type: prometheus 3648 uid: ${datasource} 3649 editorMode: code 3650 expr: |- 3651 max( 3652 kube_pod_container_resource_requests{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, namespace="$namespace", resource="cpu"} 3653 * on(namespace,pod) 3654 group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{namespace="$namespace", workload=~".*-segment-writer"} 3655 ) 3656 hide: false 3657 legendFormat: Request 3658 range: true 3659 refId: A 3660 - datasource: 3661 type: prometheus 3662 uid: ${datasource} 3663 editorMode: code 3664 expr: |- 3665 max( 3666 kube_pod_container_resource_limits{{ "{" }}{{ .Values.dashboards.kubeStateMetricsSelector }}, namespace="$namespace", resource="cpu"} 3667 * on(namespace,pod) 3668 group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{namespace="$namespace", workload=~".*-segment-writer"} 3669 ) 3670 hide: false 3671 legendFormat: Limit 3672 range: true 3673 refId: B 3674 title: CPU Usage 3675 transparent: true 3676 type: timeseries 3677 - datasource: 3678 default: false 3679 type: prometheus 3680 uid: ${datasource} 3681 fieldConfig: 3682 defaults: 3683 color: 3684 mode: palette-classic 3685 custom: 3686 axisBorderShow: false 3687 axisCenteredZero: false 3688 axisColorMode: text 3689 axisLabel: '' 3690 axisPlacement: auto 3691 axisSoftMin: 0 3692 barAlignment: 0 3693 barWidthFactor: 0.6 3694 drawStyle: line 3695 fillOpacity: 7 3696 gradientMode: none 3697 hideFrom: 3698 legend: false 3699 tooltip: false 3700 viz: false 3701 insertNulls: false 3702 lineInterpolation: linear 3703 lineWidth: 1 3704 pointSize: 5 3705 scaleDistribution: 3706 type: linear 3707 showPoints: auto 3708 spanNulls: false 3709 stacking: 3710 group: A 3711 mode: normal 3712 thresholdsStyle: 3713 mode: 'off' 3714 mappings: [] 3715 thresholds: 3716 mode: absolute 3717 steps: 3718 - color: green 3719 value: 0 3720 - color: red 3721 value: 80 3722 unit: short 3723 overrides: 3724 - matcher: 3725 id: byName 3726 options: Total 3727 properties: 3728 - id: custom.axisPlacement 3729 value: right 3730 - id: custom.lineWidth 3731 value: 2 3732 - id: color 3733 value: 3734 fixedColor: red 3735 mode: fixed 3736 gridPos: 3737 h: 8 3738 w: 8 3739 x: 16 3740 'y': 45 3741 id: 27 3742 options: 3743 legend: 3744 calcs: 3745 - mean 3746 displayMode: table 3747 placement: right 3748 showLegend: false 3749 sortBy: Mean 3750 sortDesc: true 3751 tooltip: 3752 hideZeros: false 3753 mode: single 3754 sort: none 3755 pluginVersion: 12.1.0-90139 3756 targets: 3757 - datasource: 3758 type: prometheus 3759 uid: ${datasource} 3760 editorMode: code 3761 expr: >- 3762 sum(irate(container_cpu_usage_seconds_total{namespace="$namespace",container="segment-writer"}[$__rate_interval])) 3763 by (pod) 3764 legendFormat: __auto 3765 range: true 3766 refId: A 3767 - datasource: 3768 type: prometheus 3769 uid: ${datasource} 3770 editorMode: code 3771 expr: >- 3772 sum(irate(container_cpu_usage_seconds_total{namespace="$namespace",container="segment-writer"}[$__rate_interval])) 3773 hide: false 3774 legendFormat: Total 3775 range: true 3776 refId: B 3777 title: CPU Usage (stack) 3778 transparent: true 3779 type: timeseries 3780 - collapsed: true 3781 gridPos: 3782 h: 1 3783 w: 24 3784 x: 0 3785 'y': 53 3786 id: 51 3787 panels: 3788 - datasource: 3789 type: prometheus 3790 uid: ${datasource} 3791 description: The latency of raft commands observed by the metastore leader 3792 fieldConfig: 3793 defaults: 3794 color: 3795 mode: continuous-GrYlRd 3796 custom: 3797 axisPlacement: auto 3798 fillOpacity: 15 3799 hideFrom: 3800 legend: false 3801 tooltip: false 3802 viz: false 3803 insertNulls: false 3804 lineWidth: 1 3805 spanNulls: false 3806 fieldMinMax: false 3807 mappings: 3808 - options: 3809 from: 0 3810 result: 3811 color: green 3812 index: 0 3813 text: Follower 3814 to: 99 3815 type: range 3816 - options: 3817 from: 100 3818 result: 3819 color: dark-red 3820 index: 1 3821 text: Leader 3822 to: 1000000 3823 type: range 3824 thresholds: 3825 mode: absolute 3826 steps: 3827 - color: dark-blue 3828 value: 0 3829 - color: light-red 3830 value: 100 3831 unit: short 3832 overrides: [] 3833 gridPos: 3834 h: 5 3835 w: 24 3836 x: 0 3837 'y': 92 3838 id: 65 3839 options: 3840 alignValue: center 3841 legend: 3842 displayMode: list 3843 placement: bottom 3844 showLegend: false 3845 mergeValues: true 3846 rowHeight: 0.8 3847 showValue: never 3848 tooltip: 3849 hideZeros: false 3850 mode: multi 3851 sort: asc 3852 pluginVersion: 12.1.0-90058 3853 targets: 3854 - datasource: 3855 type: prometheus 3856 uid: ${datasource} 3857 editorMode: code 3858 exemplar: false 3859 expr: |- 3860 sum by (pod) ( 3861 pyroscope_metastore_raft_state{namespace="$namespace",state="Follower"} or 3862 pyroscope_metastore_raft_state{namespace="$namespace",state="Leader"} * 100 3863 ) 3864 format: time_series 3865 hide: false 3866 instant: false 3867 legendFormat: '{{"{{label_name}}"}}' 3868 range: true 3869 refId: D 3870 title: Raft Nodes 3871 transparent: true 3872 type: state-timeline 3873 - datasource: 3874 type: prometheus 3875 uid: ${datasource} 3876 description: The latency of AddBlock endpoint observed by metastore nodes 3877 fieldConfig: 3878 defaults: 3879 color: 3880 mode: palette-classic 3881 custom: 3882 axisBorderShow: false 3883 axisCenteredZero: false 3884 axisColorMode: text 3885 axisLabel: '' 3886 axisPlacement: auto 3887 barAlignment: 0 3888 barWidthFactor: 0.6 3889 drawStyle: line 3890 fillOpacity: 0 3891 gradientMode: none 3892 hideFrom: 3893 legend: false 3894 tooltip: false 3895 viz: false 3896 insertNulls: false 3897 lineInterpolation: linear 3898 lineWidth: 1 3899 pointSize: 5 3900 scaleDistribution: 3901 type: linear 3902 showPoints: auto 3903 spanNulls: false 3904 stacking: 3905 group: A 3906 mode: none 3907 thresholdsStyle: 3908 mode: 'off' 3909 mappings: [] 3910 thresholds: 3911 mode: absolute 3912 steps: 3913 - color: green 3914 value: 0 3915 - color: red 3916 value: 80 3917 unit: s 3918 overrides: 3919 - matcher: 3920 id: byFrameRefID 3921 options: B 3922 properties: 3923 - id: custom.drawStyle 3924 value: points 3925 - id: color 3926 value: 3927 fixedColor: red 3928 mode: fixed 3929 - id: custom.pointSize 3930 value: 11 3931 - id: custom.axisPlacement 3932 value: right 3933 gridPos: 3934 h: 8 3935 w: 12 3936 x: 0 3937 'y': 97 3938 id: 49 3939 options: 3940 legend: 3941 calcs: [] 3942 displayMode: list 3943 placement: bottom 3944 showLegend: true 3945 tooltip: 3946 hideZeros: false 3947 mode: single 3948 sort: none 3949 pluginVersion: 12.1.0-90058 3950 targets: 3951 - datasource: 3952 type: prometheus 3953 uid: ${datasource} 3954 editorMode: code 3955 expr: |- 3956 ( 3957 histogram_quantile(0.99,(sum by (le, pod) ( 3958 irate(pyroscope_metastore_boltdb_persist_snapshot_duration_seconds_bucket{namespace="$namespace"}[$__rate_interval]))) 3959 ) * on(pod) group_left() (pyroscope_metastore_raft_state{namespace="$namespace", state="Leader"} >= 1) 3960 ) 3961 hide: false 3962 instant: false 3963 legendFormat: Metastore leader snapshot ({{"{{pod}}"}}) 3964 range: true 3965 refId: B 3966 - datasource: 3967 type: prometheus 3968 uid: ${datasource} 3969 editorMode: code 3970 expr: >- 3971 histogram_quantile(0.99, 3972 sum(rate(pyroscope_request_duration_seconds_bucket{ 3973 namespace="$namespace", 3974 method="gRPC", 3975 route="/metastore.v1.IndexService/AddBlock", 3976 }[$__rate_interval])) by (le)) 3977 hide: false 3978 instant: false 3979 legendFormat: p99 3980 range: true 3981 refId: D 3982 - datasource: 3983 type: prometheus 3984 uid: ${datasource} 3985 editorMode: code 3986 expr: >- 3987 histogram_quantile(0.95, 3988 sum(rate(pyroscope_request_duration_seconds_bucket{ 3989 namespace="$namespace", 3990 method="gRPC", 3991 route="/metastore.v1.IndexService/AddBlock", 3992 }[$__rate_interval])) by (le)) 3993 hide: false 3994 instant: false 3995 legendFormat: p95 3996 range: true 3997 refId: A 3998 title: Latency 3999 transparent: true 4000 type: timeseries 4001 - datasource: 4002 type: prometheus 4003 uid: ${datasource} 4004 fieldConfig: 4005 defaults: 4006 color: 4007 mode: palette-classic 4008 custom: 4009 axisBorderShow: false 4010 axisCenteredZero: false 4011 axisColorMode: text 4012 axisLabel: '' 4013 axisPlacement: auto 4014 barAlignment: 0 4015 barWidthFactor: 0.6 4016 drawStyle: line 4017 fillOpacity: 0 4018 gradientMode: none 4019 hideFrom: 4020 legend: false 4021 tooltip: false 4022 viz: false 4023 insertNulls: false 4024 lineInterpolation: linear 4025 lineWidth: 1 4026 pointSize: 5 4027 scaleDistribution: 4028 type: linear 4029 showPoints: auto 4030 spanNulls: false 4031 stacking: 4032 group: A 4033 mode: none 4034 thresholdsStyle: 4035 mode: 'off' 4036 mappings: [] 4037 min: 0 4038 thresholds: 4039 mode: absolute 4040 steps: 4041 - color: green 4042 value: 0 4043 - color: red 4044 value: 80 4045 unit: ops 4046 overrides: 4047 - matcher: 4048 id: byName 4049 options: error 4050 properties: 4051 - id: color 4052 value: 4053 fixedColor: red 4054 mode: fixed 4055 - matcher: 4056 id: byName 4057 options: success 4058 properties: 4059 - id: color 4060 value: 4061 fixedColor: green 4062 mode: fixed 4063 gridPos: 4064 h: 8 4065 w: 12 4066 x: 12 4067 'y': 97 4068 id: 50 4069 options: 4070 legend: 4071 calcs: [] 4072 displayMode: list 4073 placement: bottom 4074 showLegend: true 4075 tooltip: 4076 hideZeros: false 4077 mode: single 4078 sort: none 4079 pluginVersion: 12.1.0-90058 4080 targets: 4081 - datasource: 4082 type: prometheus 4083 uid: ${datasource} 4084 editorMode: code 4085 expr: |- 4086 sum(rate(pyroscope_request_duration_seconds_count{ 4087 namespace="$namespace", 4088 method="gRPC", 4089 container="metastore", 4090 route="/metastore.v1.IndexService/AddBlock", 4091 }[$__rate_interval])) by (status_code) 4092 hide: false 4093 instant: false 4094 legendFormat: '{{"{{status_code}}"}}' 4095 range: true 4096 refId: D 4097 title: '' 4098 transparent: true 4099 type: timeseries 4100 title: Metastore 4101 type: row 4102 - collapsed: true 4103 gridPos: 4104 h: 1 4105 w: 24 4106 x: 0 4107 'y': 54 4108 id: 22 4109 panels: 4110 - datasource: 4111 type: prometheus 4112 uid: ${datasource} 4113 fieldConfig: 4114 defaults: 4115 color: 4116 mode: continuous-BlYlRd 4117 custom: 4118 axisBorderShow: false 4119 axisCenteredZero: false 4120 axisColorMode: text 4121 axisLabel: '' 4122 axisPlacement: auto 4123 axisSoftMin: 0 4124 barAlignment: 0 4125 barWidthFactor: 0.6 4126 drawStyle: line 4127 fillOpacity: 3 4128 gradientMode: none 4129 hideFrom: 4130 legend: false 4131 tooltip: false 4132 viz: false 4133 insertNulls: false 4134 lineInterpolation: linear 4135 lineWidth: 1 4136 pointSize: 5 4137 scaleDistribution: 4138 type: linear 4139 showPoints: auto 4140 spanNulls: false 4141 stacking: 4142 group: A 4143 mode: none 4144 thresholdsStyle: 4145 mode: 'off' 4146 mappings: [] 4147 thresholds: 4148 mode: absolute 4149 steps: 4150 - color: green 4151 value: 0 4152 - color: red 4153 value: 80 4154 overrides: [] 4155 gridPos: 4156 h: 9 4157 w: 17 4158 x: 0 4159 'y': 55 4160 id: 44 4161 options: 4162 legend: 4163 calcs: 4164 - lastNotNull 4165 - mean 4166 displayMode: table 4167 placement: right 4168 showLegend: true 4169 sortBy: Last * 4170 sortDesc: true 4171 tooltip: 4172 hideZeros: false 4173 mode: single 4174 sort: none 4175 pluginVersion: 12.1.0-90058 4176 targets: 4177 - datasource: 4178 type: prometheus 4179 uid: ${datasource} 4180 editorMode: code 4181 expr: |- 4182 sum by (dataset, tenant) ( 4183 pyroscope_adaptive_placement_dataset_shard_limit{namespace="$namespace"} 4184 ) > 1 4185 legendFormat: '{{"{{tenant}}/{{dataset}}"}}' 4186 range: true 4187 refId: A 4188 title: Dataset Shards 4189 transparent: true 4190 type: timeseries 4191 - datasource: 4192 type: prometheus 4193 uid: ${datasource} 4194 description: Top-10 4195 fieldConfig: 4196 defaults: 4197 color: 4198 mode: continuous-BlYlRd 4199 custom: 4200 axisBorderShow: false 4201 axisCenteredZero: false 4202 axisColorMode: text 4203 axisLabel: '' 4204 axisPlacement: auto 4205 axisSoftMin: 0 4206 barAlignment: 0 4207 barWidthFactor: 0.6 4208 drawStyle: line 4209 fillOpacity: 3 4210 gradientMode: none 4211 hideFrom: 4212 legend: false 4213 tooltip: false 4214 viz: false 4215 insertNulls: false 4216 lineInterpolation: linear 4217 lineWidth: 1 4218 pointSize: 5 4219 scaleDistribution: 4220 type: linear 4221 showPoints: auto 4222 spanNulls: false 4223 stacking: 4224 group: A 4225 mode: none 4226 thresholdsStyle: 4227 mode: 'off' 4228 mappings: [] 4229 thresholds: 4230 mode: absolute 4231 steps: 4232 - color: green 4233 value: 0 4234 - color: red 4235 value: 80 4236 overrides: [] 4237 gridPos: 4238 h: 9 4239 w: 7 4240 x: 17 4241 'y': 55 4242 id: 107 4243 options: 4244 legend: 4245 calcs: 4246 - lastNotNull 4247 displayMode: table 4248 placement: right 4249 showLegend: true 4250 sortBy: Last * 4251 sortDesc: true 4252 tooltip: 4253 hideZeros: false 4254 mode: single 4255 sort: none 4256 pluginVersion: 12.1.0-90058 4257 targets: 4258 - datasource: 4259 type: prometheus 4260 uid: ${datasource} 4261 editorMode: code 4262 expr: |- 4263 topk(10, count by (tenant) (sum by (dataset, tenant) ( 4264 pyroscope_adaptive_placement_dataset_shard_limit{namespace="$namespace"} 4265 ))) 4266 legendFormat: '{{"{{tenant}}"}}' 4267 range: true 4268 refId: A 4269 title: Datasets per tenant 4270 transparent: true 4271 type: timeseries 4272 - datasource: 4273 default: false 4274 type: prometheus 4275 uid: ${datasource} 4276 fieldConfig: 4277 defaults: 4278 custom: 4279 hideFrom: 4280 legend: false 4281 tooltip: false 4282 viz: false 4283 scaleDistribution: 4284 type: linear 4285 overrides: [] 4286 gridPos: 4287 h: 11 4288 w: 24 4289 x: 0 4290 'y': 64 4291 id: 23 4292 options: 4293 calculate: false 4294 cellGap: 0 4295 cellValues: 4296 unit: Bps 4297 color: 4298 exponent: 0.5 4299 fill: red 4300 mode: scheme 4301 reverse: false 4302 scale: exponential 4303 scheme: Rainbow 4304 steps: 64 4305 exemplars: 4306 color: rgba(255,0,255,0.7) 4307 filterValues: 4308 le: 1.e-9 4309 legend: 4310 show: true 4311 rowsFrame: 4312 layout: auto 4313 value: Shard 4314 tooltip: 4315 mode: single 4316 showColorScale: true 4317 yHistogram: true 4318 yAxis: 4319 axisLabel: Shard 4320 axisPlacement: right 4321 reverse: true 4322 unit: none 4323 pluginVersion: 12.1.0-90058 4324 targets: 4325 - datasource: 4326 type: prometheus 4327 uid: ${datasource} 4328 editorMode: code 4329 expr: >- 4330 sum by (shard) 4331 (rate(pyroscope_segment_writer_client_sent_bytes_sum{namespace="$namespace"}[$__rate_interval])) 4332 hide: false 4333 instant: false 4334 legendFormat: __auto 4335 range: true 4336 refId: B 4337 title: Distributed Bytes (per shard) 4338 transparent: true 4339 type: heatmap 4340 - datasource: 4341 default: false 4342 type: prometheus 4343 uid: ${datasource} 4344 fieldConfig: 4345 defaults: 4346 color: 4347 mode: palette-classic 4348 custom: 4349 axisBorderShow: false 4350 axisCenteredZero: false 4351 axisColorMode: text 4352 axisLabel: '' 4353 axisPlacement: auto 4354 axisSoftMin: 0 4355 barAlignment: 0 4356 barWidthFactor: 0.6 4357 drawStyle: line 4358 fillOpacity: 0 4359 gradientMode: none 4360 hideFrom: 4361 legend: false 4362 tooltip: false 4363 viz: false 4364 insertNulls: false 4365 lineInterpolation: linear 4366 lineWidth: 1 4367 pointSize: 5 4368 scaleDistribution: 4369 type: linear 4370 showPoints: auto 4371 spanNulls: false 4372 stacking: 4373 group: A 4374 mode: none 4375 thresholdsStyle: 4376 mode: 'off' 4377 mappings: [] 4378 thresholds: 4379 mode: absolute 4380 steps: 4381 - color: green 4382 value: 0 4383 - color: red 4384 value: 80 4385 unit: bytes 4386 overrides: [] 4387 gridPos: 4388 h: 9 4389 w: 12 4390 x: 0 4391 'y': 75 4392 id: 64 4393 options: 4394 legend: 4395 calcs: 4396 - mean 4397 displayMode: table 4398 placement: right 4399 showLegend: true 4400 sortBy: Name 4401 sortDesc: false 4402 tooltip: 4403 hideZeros: false 4404 mode: single 4405 sort: none 4406 pluginVersion: 12.1.0-90058 4407 targets: 4408 - datasource: 4409 type: prometheus 4410 uid: ${datasource} 4411 editorMode: code 4412 expr: >- 4413 sum by (shard) 4414 (rate(pyroscope_segment_writer_client_sent_bytes_sum{namespace="$namespace"}[$__rate_interval])) 4415 hide: false 4416 instant: false 4417 legendFormat: __auto 4418 range: true 4419 refId: B 4420 title: Distributed Bytes (per shard) 4421 transparent: true 4422 type: timeseries 4423 - datasource: 4424 type: prometheus 4425 uid: ${datasource} 4426 fieldConfig: 4427 defaults: 4428 color: 4429 mode: palette-classic 4430 custom: 4431 axisBorderShow: false 4432 axisCenteredZero: false 4433 axisColorMode: text 4434 axisLabel: '' 4435 axisPlacement: auto 4436 axisSoftMin: 0 4437 barAlignment: 0 4438 barWidthFactor: 0.6 4439 drawStyle: line 4440 fillOpacity: 7 4441 gradientMode: none 4442 hideFrom: 4443 legend: false 4444 tooltip: false 4445 viz: false 4446 insertNulls: false 4447 lineInterpolation: linear 4448 lineWidth: 1 4449 pointSize: 5 4450 scaleDistribution: 4451 type: linear 4452 showPoints: auto 4453 spanNulls: false 4454 stacking: 4455 group: A 4456 mode: normal 4457 thresholdsStyle: 4458 mode: 'off' 4459 mappings: [] 4460 thresholds: 4461 mode: absolute 4462 steps: 4463 - color: green 4464 value: 0 4465 - color: red 4466 value: 80 4467 unit: bytes 4468 overrides: [] 4469 gridPos: 4470 h: 9 4471 w: 12 4472 x: 12 4473 'y': 75 4474 id: 26 4475 options: 4476 legend: 4477 calcs: 4478 - mean 4479 displayMode: table 4480 placement: right 4481 showLegend: true 4482 sortBy: Name 4483 sortDesc: false 4484 tooltip: 4485 hideZeros: false 4486 mode: single 4487 sort: none 4488 pluginVersion: 12.1.0-90058 4489 targets: 4490 - datasource: 4491 type: prometheus 4492 uid: ${datasource} 4493 editorMode: code 4494 expr: >- 4495 sum by (shard) 4496 (rate(pyroscope_segment_writer_client_sent_bytes_sum{namespace="$namespace"}[$__rate_interval])) 4497 hide: false 4498 instant: false 4499 legendFormat: __auto 4500 range: true 4501 refId: B 4502 title: Distributed Bytes (per shard, stack) 4503 transparent: true 4504 type: timeseries 4505 - datasource: 4506 type: prometheus 4507 uid: ${datasource} 4508 fieldConfig: 4509 defaults: 4510 color: 4511 mode: palette-classic 4512 custom: 4513 axisBorderShow: false 4514 axisCenteredZero: false 4515 axisColorMode: text 4516 axisLabel: '' 4517 axisPlacement: auto 4518 axisSoftMin: 0 4519 barAlignment: 0 4520 barWidthFactor: 0.6 4521 drawStyle: line 4522 fillOpacity: 0 4523 gradientMode: none 4524 hideFrom: 4525 legend: false 4526 tooltip: false 4527 viz: false 4528 insertNulls: false 4529 lineInterpolation: linear 4530 lineWidth: 1 4531 pointSize: 5 4532 scaleDistribution: 4533 type: linear 4534 showPoints: auto 4535 spanNulls: false 4536 stacking: 4537 group: A 4538 mode: none 4539 thresholdsStyle: 4540 mode: 'off' 4541 mappings: [] 4542 thresholds: 4543 mode: absolute 4544 steps: 4545 - color: green 4546 value: 0 4547 - color: red 4548 value: 80 4549 unit: bytes 4550 overrides: [] 4551 gridPos: 4552 h: 9 4553 w: 12 4554 x: 0 4555 'y': 84 4556 id: 24 4557 options: 4558 legend: 4559 calcs: 4560 - mean 4561 displayMode: table 4562 placement: right 4563 showLegend: true 4564 sortBy: Name 4565 sortDesc: false 4566 tooltip: 4567 hideZeros: false 4568 mode: single 4569 sort: none 4570 pluginVersion: 12.1.0-90058 4571 targets: 4572 - datasource: 4573 type: prometheus 4574 uid: ${datasource} 4575 editorMode: code 4576 expr: >- 4577 sum by (pod) 4578 (rate(pyroscope_segment_writer_segment_ingest_bytes_sum{namespace="$namespace"}[$__rate_interval])) 4579 legendFormat: __auto 4580 range: true 4581 refId: A 4582 title: Ingested Bytes (per node) 4583 transparent: true 4584 type: timeseries 4585 - datasource: 4586 type: prometheus 4587 uid: ${datasource} 4588 fieldConfig: 4589 defaults: 4590 color: 4591 mode: palette-classic 4592 custom: 4593 axisBorderShow: false 4594 axisCenteredZero: false 4595 axisColorMode: text 4596 axisLabel: '' 4597 axisPlacement: auto 4598 axisSoftMin: 0 4599 barAlignment: 0 4600 barWidthFactor: 0.6 4601 drawStyle: line 4602 fillOpacity: 7 4603 gradientMode: none 4604 hideFrom: 4605 legend: false 4606 tooltip: false 4607 viz: false 4608 insertNulls: false 4609 lineInterpolation: linear 4610 lineWidth: 1 4611 pointSize: 5 4612 scaleDistribution: 4613 type: linear 4614 showPoints: auto 4615 spanNulls: false 4616 stacking: 4617 group: A 4618 mode: normal 4619 thresholdsStyle: 4620 mode: 'off' 4621 mappings: [] 4622 thresholds: 4623 mode: absolute 4624 steps: 4625 - color: green 4626 value: 0 4627 - color: red 4628 value: 80 4629 unit: bytes 4630 overrides: [] 4631 gridPos: 4632 h: 9 4633 w: 12 4634 x: 12 4635 'y': 84 4636 id: 34 4637 options: 4638 legend: 4639 calcs: 4640 - mean 4641 displayMode: table 4642 placement: right 4643 showLegend: true 4644 sortBy: Name 4645 sortDesc: false 4646 tooltip: 4647 hideZeros: false 4648 mode: single 4649 sort: none 4650 pluginVersion: 12.1.0-90058 4651 targets: 4652 - datasource: 4653 type: prometheus 4654 uid: ${datasource} 4655 editorMode: code 4656 expr: >- 4657 sum by (pod) 4658 (rate(pyroscope_segment_writer_segment_ingest_bytes_sum{namespace="$namespace"}[$__rate_interval])) 4659 legendFormat: __auto 4660 range: true 4661 refId: A 4662 - datasource: 4663 type: prometheus 4664 uid: ${datasource} 4665 editorMode: code 4666 expr: '' 4667 hide: false 4668 instant: false 4669 legendFormat: __auto 4670 range: true 4671 refId: B 4672 title: Ingested Bytes (per node, stack) 4673 transparent: true 4674 type: timeseries 4675 - datasource: 4676 type: prometheus 4677 uid: ${datasource} 4678 fieldConfig: 4679 defaults: 4680 color: 4681 mode: palette-classic 4682 custom: 4683 axisBorderShow: false 4684 axisCenteredZero: false 4685 axisColorMode: text 4686 axisLabel: '' 4687 axisPlacement: auto 4688 axisSoftMin: 0 4689 barAlignment: 0 4690 barWidthFactor: 0.6 4691 drawStyle: line 4692 fillOpacity: 0 4693 gradientMode: none 4694 hideFrom: 4695 legend: false 4696 tooltip: false 4697 viz: false 4698 insertNulls: false 4699 lineInterpolation: linear 4700 lineWidth: 1 4701 pointSize: 5 4702 scaleDistribution: 4703 type: linear 4704 showPoints: auto 4705 spanNulls: false 4706 stacking: 4707 group: A 4708 mode: none 4709 thresholdsStyle: 4710 mode: 'off' 4711 mappings: [] 4712 thresholds: 4713 mode: absolute 4714 steps: 4715 - color: green 4716 value: 0 4717 - color: red 4718 value: 80 4719 unit: short 4720 overrides: [] 4721 gridPos: 4722 h: 9 4723 w: 12 4724 x: 0 4725 'y': 93 4726 id: 35 4727 options: 4728 legend: 4729 calcs: 4730 - mean 4731 displayMode: table 4732 placement: right 4733 showLegend: true 4734 sortBy: Name 4735 sortDesc: false 4736 tooltip: 4737 hideZeros: false 4738 mode: multi 4739 sort: desc 4740 pluginVersion: 12.1.0-90058 4741 targets: 4742 - datasource: 4743 type: prometheus 4744 uid: ${datasource} 4745 editorMode: code 4746 expr: >- 4747 sum by (pod) 4748 (rate(pyroscope_segment_writer_segment_ingest_bytes_count{namespace="$namespace"}[$__rate_interval])) 4749 legendFormat: __auto 4750 range: true 4751 refId: A 4752 title: Ingested Profiles (per node) 4753 transparent: true 4754 type: timeseries 4755 - datasource: 4756 type: prometheus 4757 uid: ${datasource} 4758 fieldConfig: 4759 defaults: 4760 color: 4761 mode: palette-classic 4762 custom: 4763 axisBorderShow: false 4764 axisCenteredZero: false 4765 axisColorMode: text 4766 axisLabel: '' 4767 axisPlacement: auto 4768 axisSoftMin: 0 4769 barAlignment: 0 4770 barWidthFactor: 0.6 4771 drawStyle: line 4772 fillOpacity: 7 4773 gradientMode: none 4774 hideFrom: 4775 legend: false 4776 tooltip: false 4777 viz: false 4778 insertNulls: false 4779 lineInterpolation: linear 4780 lineWidth: 1 4781 pointSize: 5 4782 scaleDistribution: 4783 type: linear 4784 showPoints: auto 4785 spanNulls: false 4786 stacking: 4787 group: A 4788 mode: normal 4789 thresholdsStyle: 4790 mode: 'off' 4791 mappings: [] 4792 thresholds: 4793 mode: absolute 4794 steps: 4795 - color: green 4796 value: 0 4797 - color: red 4798 value: 80 4799 unit: short 4800 overrides: [] 4801 gridPos: 4802 h: 9 4803 w: 12 4804 x: 12 4805 'y': 93 4806 id: 36 4807 options: 4808 legend: 4809 calcs: 4810 - mean 4811 displayMode: table 4812 placement: right 4813 showLegend: true 4814 sortBy: Name 4815 sortDesc: false 4816 tooltip: 4817 hideZeros: false 4818 mode: multi 4819 sort: desc 4820 pluginVersion: 12.1.0-90058 4821 targets: 4822 - datasource: 4823 type: prometheus 4824 uid: ${datasource} 4825 editorMode: code 4826 expr: >- 4827 sum by (pod) 4828 (rate(pyroscope_segment_writer_segment_ingest_bytes_count{namespace="$namespace"}[$__rate_interval])) 4829 legendFormat: __auto 4830 range: true 4831 refId: A 4832 title: Ingested Profiles (per node, stack) 4833 transparent: true 4834 type: timeseries 4835 title: Data Distribution 4836 type: row 4837 - collapsed: true 4838 gridPos: 4839 h: 1 4840 w: 24 4841 x: 0 4842 'y': 55 4843 id: 86 4844 panels: 4845 - datasource: 4846 type: prometheus 4847 uid: ${datasource} 4848 fieldConfig: 4849 defaults: 4850 color: 4851 mode: thresholds 4852 fieldMinMax: false 4853 mappings: [] 4854 thresholds: 4855 mode: absolute 4856 steps: 4857 - color: green 4858 value: 0 4859 overrides: [] 4860 gridPos: 4861 h: 13 4862 w: 12 4863 x: 0 4864 'y': 154 4865 id: 87 4866 options: 4867 groupByField: container 4868 labelFields: 4869 - container 4870 - instance_type 4871 textField: instance_type 4872 tiling: treemapSquarify 4873 pluginVersion: 2.1.1 4874 targets: 4875 - datasource: 4876 type: prometheus 4877 uid: ${datasource} 4878 editorMode: code 4879 exemplar: false 4880 expr: |- 4881 count by (instance_type, container) ( 4882 rate(container_cpu_usage_seconds_total{ 4883 cluster="$cluster", 4884 namespace="$namespace", 4885 container=~"segment-writer|distributor|cortex-gw|metastore|compaction-worker", 4886 }[$__rate_interval]) + on (instance) group_left(instance_type) (0 * label_replace( 4887 karpenter_nodes_allocatable{cluster="$cluster", resource_type="cpu"}, "instance", "$1", "node_name", "(.*)") 4888 ) 4889 ) 4890 format: table 4891 hide: false 4892 instant: true 4893 legendFormat: '{{"{{instance_type}}"}}' 4894 range: false 4895 refId: A 4896 title: AWS | Intances By Machine Type 4897 transparent: true 4898 type: marcusolsson-treemap-panel 4899 - datasource: 4900 type: prometheus 4901 uid: ${datasource} 4902 fieldConfig: 4903 defaults: 4904 color: 4905 mode: palette-classic 4906 custom: 4907 axisBorderShow: false 4908 axisCenteredZero: false 4909 axisColorMode: text 4910 axisLabel: '' 4911 axisPlacement: auto 4912 axisSoftMin: 0 4913 barAlignment: 0 4914 barWidthFactor: 0.6 4915 drawStyle: line 4916 fillOpacity: 0 4917 gradientMode: none 4918 hideFrom: 4919 legend: false 4920 tooltip: false 4921 viz: false 4922 insertNulls: false 4923 lineInterpolation: linear 4924 lineWidth: 1 4925 pointSize: 5 4926 scaleDistribution: 4927 type: linear 4928 showPoints: auto 4929 spanNulls: false 4930 stacking: 4931 group: A 4932 mode: none 4933 thresholdsStyle: 4934 mode: 'off' 4935 mappings: [] 4936 thresholds: 4937 mode: absolute 4938 steps: 4939 - color: green 4940 value: 0 4941 - color: red 4942 value: 80 4943 overrides: [] 4944 gridPos: 4945 h: 13 4946 w: 12 4947 x: 12 4948 'y': 154 4949 id: 84 4950 options: 4951 legend: 4952 calcs: 4953 - mean 4954 - max 4955 displayMode: table 4956 placement: right 4957 showLegend: true 4958 tooltip: 4959 hideZeros: false 4960 mode: single 4961 sort: none 4962 pluginVersion: 12.1.0-90058 4963 targets: 4964 - datasource: 4965 type: prometheus 4966 uid: ${datasource} 4967 editorMode: code 4968 expr: |- 4969 avg by (instance_type, container) ( 4970 rate(container_cpu_usage_seconds_total{ 4971 cluster="$cluster", 4972 namespace="$namespace", 4973 container=~"segment-writer|distributor|cortex-gw", 4974 }[$__rate_interval]) + on (instance) group_left(instance_type) (0 * label_replace( 4975 karpenter_nodes_allocatable{cluster="$cluster", resource_type="cpu"}, "instance", "$1", "node_name", "(.*)") 4976 ) 4977 ) 4978 hide: false 4979 legendFormat: '{{"{{container}}/{{instance_type}}"}}' 4980 range: true 4981 refId: A 4982 title: AWS | Average CPU Usage By Type 4983 transparent: true 4984 type: timeseries 4985 title: Resource Allocation 4986 type: row 4987 preload: false 4988 schemaVersion: 41 4989 tags: 4990 - pyroscope 4991 - pyroscope-v2 4992 templating: 4993 list: 4994 - current: 4995 text: ops-cortex 4996 value: '000000134' 4997 includeAll: false 4998 name: datasource 4999 options: [] 5000 query: prometheus 5001 refresh: 1 5002 regex: '' 5003 type: datasource 5004 - allowCustomValue: false 5005 current: 5006 text: Grafana Logging 5007 value: '000000193' 5008 name: loki_datasource 5009 options: [] 5010 query: loki 5011 refresh: 1 5012 regex: '' 5013 type: datasource 5014 - allowCustomValue: false 5015 current: 5016 text: prod-eu-west-2 5017 value: prod-eu-west-2 5018 datasource: 5019 type: prometheus 5020 uid: ${datasource} 5021 definition: label_values(pyroscope_build_info{namespace="$namespace"},cluster) 5022 label: cluster 5023 name: cluster 5024 options: [] 5025 query: 5026 qryType: 1 5027 query: label_values(pyroscope_build_info{namespace="$namespace"},cluster) 5028 refId: PrometheusVariableQueryEditor-VariableQuery 5029 refresh: 1 5030 regex: '' 5031 sort: 1 5032 type: query 5033 - allowCustomValue: false 5034 current: 5035 text: {{.Values.dashboards.namespace | quote}} 5036 value: {{.Values.dashboards.namespace | quote}} 5037 datasource: 5038 type: prometheus 5039 uid: ${datasource} 5040 definition: label_values(pyroscope_build_info,namespace) 5041 label: namespace 5042 name: namespace 5043 options: [] 5044 query: 5045 qryType: 1 5046 query: label_values(pyroscope_build_info,namespace) 5047 refId: PrometheusVariableQueryEditor-VariableQuery 5048 refresh: 1 5049 regex: {{.Values.dashboards.namespaceRegex | quote}} 5050 type: query 5051 - baseFilters: [] 5052 datasource: 5053 type: prometheus 5054 uid: ${datasource} 5055 filters: [] 5056 name: Filters 5057 type: adhoc 5058 time: 5059 from: now-6h 5060 to: now 5061 timepicker: {} 5062 timezone: utc 5063 title: Pyroscope v2 / Write Path 5064 5065 {{- end }}