github.com/grafana/pyroscope@v1.18.0/operations/monitoring/dashboards/operational.json (about) 1 { 2 "annotations": { 3 "list": [ 4 { 5 "builtIn": 1, 6 "datasource": { 7 "type": "datasource", 8 "uid": "grafana" 9 }, 10 "enable": true, 11 "hide": true, 12 "iconColor": "rgba(0, 211, 255, 1)", 13 "name": "Annotations \u0026 Alerts", 14 "target": { 15 "limit": 100, 16 "matchAny": false, 17 "tags": [], 18 "type": "dashboard" 19 }, 20 "type": "dashboard" 21 }, 22 { 23 "datasource": { 24 "uid": "$loki_datasource" 25 }, 26 "enable": true, 27 "expr": "{cluster=~\"$cluster\", container=\"kube-diff-logger\"} | json | namespace_extracted=~\"$namespace\" | notes!~\".*Replicas.*\"", 28 "hide": false, 29 "iconColor": "rgba(255, 96, 96, 1)", 30 "name": "K8s Changes", 31 "showIn": 0, 32 "target": {} 33 }, 34 { 35 "datasource": { 36 "uid": "$loki_datasource" 37 }, 38 "enable": true, 39 "expr": "{cluster=~\"$cluster\", container=\"kube-diff-logger\"} | json | namespace_extracted=~\"$namespace\" | notes=~\".*Replicas.*\"", 40 "hide": false, 41 "iconColor": "rgba(255, 96, 96, 1)", 42 "name": "K8s Changes(Replicas)", 43 "showIn": 0, 44 "target": {} 45 }, 46 { 47 "datasource": { 48 "type": "loki", 49 "uid": "${loki_datasource}" 50 }, 51 "enable": false, 52 "expr": "{namespace=~\"$namespace\",cluster=~\"$cluster\"} |= \"head successfully written to block\" | logfmt", 53 "hide": false, 54 "iconColor": "yellow", 55 "instant": false, 56 "name": "Flush Events", 57 "tagKeys": "block_path,tenant", 58 "textFormat": "head successfully written to block", 59 "titleFormat": "Head Flush" 60 } 61 ] 62 }, 63 "description": "Bird eyes view of Pyroscope clusters", 64 "editable": true, 65 "fiscalYearStartMonth": 0, 66 "graphTooltip": 0, 67 "id": 9933, 68 "links": [ 69 { 70 "asDropdown": true, 71 "icon": "external link", 72 "includeVars": true, 73 "keepTime": true, 74 "tags": [ 75 "pyroscope" 76 ], 77 "targetBlank": false, 78 "title": "Pyroscope Dashboards", 79 "type": "dashboards" 80 } 81 ], 82 "panels": [ 83 { 84 "collapsed": false, 85 "gridPos": { 86 "h": 1, 87 "w": 24, 88 "x": 0, 89 "y": 0 90 }, 91 "id": 4, 92 "panels": [], 93 "title": "Global", 94 "type": "row" 95 }, 96 { 97 "datasource": { 98 "type": "prometheus", 99 "uid": "${datasource}" 100 }, 101 "fieldConfig": { 102 "defaults": { 103 "color": { 104 "mode": "palette-classic" 105 }, 106 "custom": { 107 "axisBorderShow": false, 108 "axisCenteredZero": false, 109 "axisColorMode": "text", 110 "axisLabel": "", 111 "axisPlacement": "auto", 112 "barAlignment": 0, 113 "barWidthFactor": 0.6, 114 "drawStyle": "line", 115 "fillOpacity": 10, 116 "gradientMode": "none", 117 "hideFrom": { 118 "legend": false, 119 "tooltip": false, 120 "viz": false 121 }, 122 "insertNulls": false, 123 "lineInterpolation": "linear", 124 "lineWidth": 1, 125 "pointSize": 5, 126 "scaleDistribution": { 127 "type": "linear" 128 }, 129 "showPoints": "auto", 130 "spanNulls": false, 131 "stacking": { 132 "group": "A", 133 "mode": "normal" 134 }, 135 "thresholdsStyle": { 136 "mode": "off" 137 } 138 }, 139 "mappings": [], 140 "thresholds": { 141 "mode": "absolute", 142 "steps": [ 143 { 144 "color": "green" 145 }, 146 { 147 "color": "red", 148 "value": 80 149 } 150 ] 151 }, 152 "unit": "binBps" 153 }, 154 "overrides": [] 155 }, 156 "gridPos": { 157 "h": 6, 158 "w": 7, 159 "x": 0, 160 "y": 1 161 }, 162 "id": 29, 163 "options": { 164 "legend": { 165 "calcs": [], 166 "displayMode": "table", 167 "placement": "right", 168 "showLegend": true 169 }, 170 "tooltip": { 171 "hideZeros": false, 172 "mode": "single", 173 "sort": "none" 174 } 175 }, 176 "pluginVersion": "12.1.0-88106", 177 "targets": [ 178 { 179 "datasource": { 180 "type": "prometheus", 181 "uid": "${datasource}" 182 }, 183 "editorMode": "code", 184 "expr": "sum by (tenant) (rate(pyroscope_distributor_received_decompressed_bytes_sum{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval]))", 185 "hide": false, 186 "legendFormat": "__auto", 187 "range": true, 188 "refId": "A" 189 } 190 ], 191 "title": "MBs per Tenant (Decompressed)", 192 "type": "timeseries" 193 }, 194 { 195 "datasource": { 196 "type": "prometheus", 197 "uid": "${datasource}" 198 }, 199 "fieldConfig": { 200 "defaults": { 201 "color": { 202 "mode": "palette-classic" 203 }, 204 "custom": { 205 "axisBorderShow": false, 206 "axisCenteredZero": false, 207 "axisColorMode": "text", 208 "axisLabel": "", 209 "axisPlacement": "auto", 210 "barAlignment": 0, 211 "barWidthFactor": 0.6, 212 "drawStyle": "line", 213 "fillOpacity": 10, 214 "gradientMode": "none", 215 "hideFrom": { 216 "legend": false, 217 "tooltip": false, 218 "viz": false 219 }, 220 "insertNulls": false, 221 "lineInterpolation": "linear", 222 "lineWidth": 1, 223 "pointSize": 5, 224 "scaleDistribution": { 225 "type": "linear" 226 }, 227 "showPoints": "auto", 228 "spanNulls": false, 229 "stacking": { 230 "group": "A", 231 "mode": "normal" 232 }, 233 "thresholdsStyle": { 234 "mode": "off" 235 } 236 }, 237 "mappings": [], 238 "thresholds": { 239 "mode": "absolute", 240 "steps": [ 241 { 242 "color": "green" 243 }, 244 { 245 "color": "red", 246 "value": 80 247 } 248 ] 249 }, 250 "unit": "cps" 251 }, 252 "overrides": [] 253 }, 254 "gridPos": { 255 "h": 6, 256 "w": 6, 257 "x": 7, 258 "y": 1 259 }, 260 "id": 32, 261 "options": { 262 "legend": { 263 "calcs": [], 264 "displayMode": "table", 265 "placement": "right", 266 "showLegend": true 267 }, 268 "tooltip": { 269 "hideZeros": false, 270 "mode": "single", 271 "sort": "none" 272 } 273 }, 274 "pluginVersion": "12.1.0-88106", 275 "targets": [ 276 { 277 "datasource": { 278 "type": "prometheus", 279 "uid": "${datasource}" 280 }, 281 "editorMode": "code", 282 "expr": "sum by (tenant) (rate(pyroscope_distributor_received_decompressed_bytes_count{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval]))", 283 "legendFormat": "__auto", 284 "range": true, 285 "refId": "A" 286 } 287 ], 288 "title": "Profiles/s per Tenant", 289 "type": "timeseries" 290 }, 291 { 292 "datasource": { 293 "type": "prometheus", 294 "uid": "${datasource}" 295 }, 296 "fieldConfig": { 297 "defaults": { 298 "color": { 299 "mode": "palette-classic" 300 }, 301 "custom": { 302 "axisBorderShow": false, 303 "axisCenteredZero": false, 304 "axisColorMode": "text", 305 "axisLabel": "", 306 "axisPlacement": "auto", 307 "barAlignment": 0, 308 "barWidthFactor": 0.6, 309 "drawStyle": "line", 310 "fillOpacity": 10, 311 "gradientMode": "none", 312 "hideFrom": { 313 "legend": false, 314 "tooltip": false, 315 "viz": false 316 }, 317 "insertNulls": false, 318 "lineInterpolation": "linear", 319 "lineWidth": 1, 320 "pointSize": 5, 321 "scaleDistribution": { 322 "type": "linear" 323 }, 324 "showPoints": "never", 325 "spanNulls": false, 326 "stacking": { 327 "group": "A", 328 "mode": "none" 329 }, 330 "thresholdsStyle": { 331 "mode": "off" 332 } 333 }, 334 "mappings": [], 335 "thresholds": { 336 "mode": "absolute", 337 "steps": [ 338 { 339 "color": "green" 340 }, 341 { 342 "color": "red", 343 "value": 80 344 } 345 ] 346 }, 347 "unit": "short" 348 }, 349 "overrides": [] 350 }, 351 "gridPos": { 352 "h": 6, 353 "w": 5, 354 "x": 13, 355 "y": 1 356 }, 357 "id": 24, 358 "options": { 359 "dataLinks": [], 360 "legend": { 361 "calcs": [], 362 "displayMode": "list", 363 "placement": "bottom", 364 "showLegend": false 365 }, 366 "tooltip": { 367 "hideZeros": false, 368 "mode": "multi", 369 "sort": "desc" 370 } 371 }, 372 "pluginVersion": "12.1.0-88106", 373 "targets": [ 374 { 375 "datasource": { 376 "type": "prometheus", 377 "uid": "${datasource}" 378 }, 379 "expr": "increase(kube_pod_container_status_restarts_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[10m]) \u003e 0", 380 "hide": false, 381 "interval": "", 382 "legendFormat": "{{container}}-{{pod}}", 383 "refId": "B" 384 } 385 ], 386 "title": "Container Restarts", 387 "type": "timeseries" 388 }, 389 { 390 "datasource": { 391 "type": "prometheus", 392 "uid": "${datasource}" 393 }, 394 "fieldConfig": { 395 "defaults": { 396 "color": { 397 "mode": "palette-classic" 398 }, 399 "custom": { 400 "hideFrom": { 401 "legend": false, 402 "tooltip": false, 403 "viz": false 404 } 405 }, 406 "mappings": [] 407 }, 408 "overrides": [] 409 }, 410 "gridPos": { 411 "h": 6, 412 "w": 3, 413 "x": 18, 414 "y": 1 415 }, 416 "id": 34, 417 "options": { 418 "legend": { 419 "displayMode": "list", 420 "placement": "bottom", 421 "showLegend": true 422 }, 423 "pieType": "pie", 424 "reduceOptions": { 425 "calcs": [ 426 "lastNotNull" 427 ], 428 "fields": "", 429 "values": false 430 }, 431 "tooltip": { 432 "hideZeros": false, 433 "mode": "single", 434 "sort": "none" 435 } 436 }, 437 "pluginVersion": "12.1.0-88106", 438 "targets": [ 439 { 440 "datasource": { 441 "type": "prometheus", 442 "uid": "${datasource}" 443 }, 444 "editorMode": "code", 445 "exemplar": false, 446 "expr": "sum by (state) (pyroscope_ring_members{cluster=~\"$cluster\", namespace=~\"$namespace\", name=\"ingester\"}) / count by (state)(pyroscope_ring_members{cluster=~\"$cluster\", namespace=~\"$namespace\", name=\"ingester\"})", 447 "format": "time_series", 448 "instant": true, 449 "legendFormat": "__auto", 450 "range": false, 451 "refId": "A" 452 } 453 ], 454 "title": "Ingester Ring Status", 455 "type": "piechart" 456 }, 457 { 458 "datasource": { 459 "type": "prometheus", 460 "uid": "${datasource}" 461 }, 462 "description": "Distributor Ring is used for rate limiting", 463 "fieldConfig": { 464 "defaults": { 465 "color": { 466 "mode": "palette-classic" 467 }, 468 "custom": { 469 "hideFrom": { 470 "legend": false, 471 "tooltip": false, 472 "viz": false 473 } 474 }, 475 "mappings": [] 476 }, 477 "overrides": [] 478 }, 479 "gridPos": { 480 "h": 6, 481 "w": 3, 482 "x": 21, 483 "y": 1 484 }, 485 "id": 35, 486 "options": { 487 "legend": { 488 "displayMode": "list", 489 "placement": "bottom", 490 "showLegend": true 491 }, 492 "pieType": "pie", 493 "reduceOptions": { 494 "calcs": [ 495 "lastNotNull" 496 ], 497 "fields": "", 498 "values": false 499 }, 500 "tooltip": { 501 "hideZeros": false, 502 "mode": "single", 503 "sort": "none" 504 } 505 }, 506 "pluginVersion": "12.1.0-88106", 507 "targets": [ 508 { 509 "datasource": { 510 "type": "prometheus", 511 "uid": "${datasource}" 512 }, 513 "editorMode": "code", 514 "exemplar": false, 515 "expr": "sum by (state) (pyroscope_ring_members{cluster=~\"$cluster\", namespace=~\"$namespace\", name=\"distributor\"}) / count by (state)(pyroscope_ring_members{cluster=~\"$cluster\", namespace=~\"$namespace\", name=\"distributor\"})", 516 "format": "time_series", 517 "instant": true, 518 "legendFormat": "__auto", 519 "range": false, 520 "refId": "A" 521 } 522 ], 523 "title": "Distributor Ring Status", 524 "type": "piechart" 525 }, 526 { 527 "datasource": { 528 "type": "prometheus", 529 "uid": "${datasource}" 530 }, 531 "fieldConfig": { 532 "defaults": { 533 "color": { 534 "mode": "thresholds" 535 }, 536 "custom": { 537 "align": "left", 538 "cellOptions": { 539 "type": "auto" 540 }, 541 "filterable": true, 542 "inspect": false 543 }, 544 "mappings": [], 545 "thresholds": { 546 "mode": "absolute", 547 "steps": [ 548 { 549 "color": "green" 550 }, 551 { 552 "color": "red", 553 "value": 80 554 } 555 ] 556 } 557 }, 558 "overrides": [ 559 { 560 "matcher": { 561 "id": "byName", 562 "options": "Value" 563 }, 564 "properties": [ 565 { 566 "id": "unit", 567 "value": "decbytes" 568 } 569 ] 570 }, 571 { 572 "matcher": { 573 "id": "byName", 574 "options": "tenant" 575 }, 576 "properties": [ 577 { 578 "id": "custom.width", 579 "value": 78 580 } 581 ] 582 }, 583 { 584 "matcher": { 585 "id": "byName", 586 "options": "slug" 587 }, 588 "properties": [ 589 { 590 "id": "custom.width", 591 "value": 132 592 } 593 ] 594 }, 595 { 596 "matcher": { 597 "id": "byName", 598 "options": "org_name" 599 }, 600 "properties": [ 601 { 602 "id": "custom.width", 603 "value": 142 604 } 605 ] 606 } 607 ] 608 }, 609 "gridPos": { 610 "h": 10, 611 "w": 5, 612 "x": 0, 613 "y": 7 614 }, 615 "id": 85, 616 "options": { 617 "cellHeight": "sm", 618 "footer": { 619 "countRows": false, 620 "enablePagination": true, 621 "fields": "", 622 "reducer": [ 623 "count" 624 ], 625 "show": false 626 }, 627 "showHeader": true, 628 "sortBy": [ 629 { 630 "desc": true, 631 "displayName": "Value" 632 } 633 ] 634 }, 635 "pluginVersion": "12.1.0-88106", 636 "targets": [ 637 { 638 "datasource": { 639 "type": "prometheus", 640 "uid": "${datasource}" 641 }, 642 "editorMode": "code", 643 "exemplar": false, 644 "expr": "\nsum by (tenant, slug, org_name, environment) (\n rate(pyroscope_distributor_received_decompressed_bytes_sum{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])\n)\n", 645 "format": "table", 646 "instant": true, 647 "legendFormat": "__auto", 648 "range": false, 649 "refId": "A" 650 } 651 ], 652 "title": "Tenant id / name mapping", 653 "transformations": [ 654 { 655 "id": "organize", 656 "options": { 657 "excludeByName": { 658 "Time": true, 659 "Value": false, 660 "cluster": true 661 }, 662 "includeByName": {}, 663 "indexByName": { 664 "Time": 0, 665 "Value": 5, 666 "cluster": 1, 667 "org_name": 4, 668 "slug": 3, 669 "tenant": 2 670 }, 671 "renameByName": {} 672 } 673 } 674 ], 675 "type": "table" 676 }, 677 { 678 "datasource": { 679 "type": "prometheus", 680 "uid": "$datasource" 681 }, 682 "fieldConfig": { 683 "defaults": { 684 "color": { 685 "mode": "palette-classic" 686 }, 687 "custom": { 688 "axisBorderShow": false, 689 "axisCenteredZero": false, 690 "axisColorMode": "text", 691 "axisLabel": "", 692 "axisPlacement": "auto", 693 "barAlignment": 0, 694 "barWidthFactor": 0.6, 695 "drawStyle": "line", 696 "fillOpacity": 40, 697 "gradientMode": "none", 698 "hideFrom": { 699 "legend": false, 700 "tooltip": false, 701 "viz": false 702 }, 703 "insertNulls": false, 704 "lineInterpolation": "linear", 705 "lineWidth": 1, 706 "pointSize": 5, 707 "scaleDistribution": { 708 "type": "linear" 709 }, 710 "showPoints": "never", 711 "spanNulls": false, 712 "stacking": { 713 "group": "A", 714 "mode": "normal" 715 }, 716 "thresholdsStyle": { 717 "mode": "off" 718 } 719 }, 720 "mappings": [], 721 "thresholds": { 722 "mode": "absolute", 723 "steps": [ 724 { 725 "color": "green" 726 }, 727 { 728 "color": "red", 729 "value": 80 730 } 731 ] 732 }, 733 "unit": "reqps" 734 }, 735 "overrides": [ 736 { 737 "matcher": { 738 "id": "byName", 739 "options": "5xx " 740 }, 741 "properties": [ 742 { 743 "id": "color", 744 "value": { 745 "fixedColor": "red", 746 "mode": "fixed" 747 } 748 } 749 ] 750 }, 751 { 752 "matcher": { 753 "id": "byName", 754 "options": "2xx " 755 }, 756 "properties": [ 757 { 758 "id": "color", 759 "value": { 760 "fixedColor": "green", 761 "mode": "fixed", 762 "seriesBy": "min" 763 } 764 } 765 ] 766 }, 767 { 768 "matcher": { 769 "id": "byName", 770 "options": "4xx " 771 }, 772 "properties": [ 773 { 774 "id": "color", 775 "value": { 776 "fixedColor": "yellow", 777 "mode": "fixed" 778 } 779 } 780 ] 781 } 782 ] 783 }, 784 "gridPos": { 785 "h": 5, 786 "w": 6, 787 "x": 5, 788 "y": 7 789 }, 790 "id": 7, 791 "options": { 792 "dataLinks": [], 793 "legend": { 794 "calcs": [], 795 "displayMode": "list", 796 "placement": "bottom", 797 "showLegend": true 798 }, 799 "tooltip": { 800 "hideZeros": false, 801 "mode": "multi", 802 "sort": "none" 803 } 804 }, 805 "pluginVersion": "12.1.0-88106", 806 "targets": [ 807 { 808 "datasource": { 809 "uid": "$datasource" 810 }, 811 "editorMode": "code", 812 "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(pyroscope_request_duration_seconds_count{cluster=~\"$cluster\", container=~\"pyroscope|distributor|query-frontend\", namespace=~\"$namespace\", route=~\".*ingest.*|.*pusher.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)", 813 "legendFormat": "{{status}} ", 814 "range": true, 815 "refId": "A" 816 } 817 ], 818 "title": "Pushes/Second", 819 "type": "timeseries" 820 }, 821 { 822 "datasource": { 823 "type": "prometheus", 824 "uid": "${datasource}" 825 }, 826 "fieldConfig": { 827 "defaults": { 828 "color": { 829 "mode": "palette-classic" 830 }, 831 "custom": { 832 "axisBorderShow": false, 833 "axisCenteredZero": false, 834 "axisColorMode": "text", 835 "axisLabel": "", 836 "axisPlacement": "auto", 837 "barAlignment": 0, 838 "barWidthFactor": 0.6, 839 "drawStyle": "line", 840 "fillOpacity": 0, 841 "gradientMode": "none", 842 "hideFrom": { 843 "legend": false, 844 "tooltip": false, 845 "viz": false 846 }, 847 "insertNulls": false, 848 "lineInterpolation": "linear", 849 "lineWidth": 1, 850 "pointSize": 5, 851 "scaleDistribution": { 852 "type": "linear" 853 }, 854 "showPoints": "auto", 855 "spanNulls": false, 856 "stacking": { 857 "group": "A", 858 "mode": "none" 859 }, 860 "thresholdsStyle": { 861 "mode": "off" 862 } 863 }, 864 "mappings": [], 865 "thresholds": { 866 "mode": "absolute", 867 "steps": [ 868 { 869 "color": "green" 870 }, 871 { 872 "color": "red", 873 "value": 80 874 } 875 ] 876 }, 877 "unit": "s" 878 }, 879 "overrides": [] 880 }, 881 "gridPos": { 882 "h": 5, 883 "w": 7, 884 "x": 11, 885 "y": 7 886 }, 887 "id": 27, 888 "options": { 889 "legend": { 890 "calcs": [], 891 "displayMode": "table", 892 "placement": "right", 893 "showLegend": true 894 }, 895 "tooltip": { 896 "hideZeros": false, 897 "mode": "single", 898 "sort": "none" 899 } 900 }, 901 "pluginVersion": "12.1.0-88106", 902 "targets": [ 903 { 904 "datasource": { 905 "type": "prometheus", 906 "uid": "${datasource}" 907 }, 908 "editorMode": "code", 909 "expr": "histogram_quantile(0.99, sum by (route) ( rate(pyroscope_request_duration_seconds{container=~\"pyroscope|distributor|query-frontend\", namespace=~\"$namespace\", cluster=~\"$cluster\",route=~\".*ingest.*|.*pusher.*\"}[$__rate_interval])))", 910 "legendFormat": ".99 {{route}}", 911 "range": true, 912 "refId": "A" 913 }, 914 { 915 "datasource": { 916 "type": "prometheus", 917 "uid": "${datasource}" 918 }, 919 "editorMode": "code", 920 "expr": "histogram_quantile(0.75, sum by (route) ( rate(pyroscope_request_duration_seconds{container=~\"pyroscope|distributor|query-frontend\", namespace=~\"$namespace\", cluster=~\"$cluster\",route=~\".*ingest.*|.*pusher.*\"}[$__rate_interval])))", 921 "hide": false, 922 "legendFormat": ".75 {{route}}", 923 "range": true, 924 "refId": "B" 925 }, 926 { 927 "datasource": { 928 "type": "prometheus", 929 "uid": "${datasource}" 930 }, 931 "editorMode": "code", 932 "expr": "histogram_quantile(0.5, sum by (route) ( rate(pyroscope_request_duration_seconds{container=~\"pyroscope|distributor|query-frontend\", namespace=~\"$namespace\", cluster=~\"$cluster\",route=~\".*ingest.*|.*pusher.*\"}[$__rate_interval])))", 933 "hide": false, 934 "legendFormat": ".5 {{route}}", 935 "range": true, 936 "refId": "C" 937 } 938 ], 939 "title": "Push Latencies", 940 "type": "timeseries" 941 }, 942 { 943 "datasource": { 944 "type": "prometheus", 945 "uid": "${datasource}" 946 }, 947 "fieldConfig": { 948 "defaults": { 949 "color": { 950 "mode": "thresholds" 951 }, 952 "custom": { 953 "align": "center", 954 "cellOptions": { 955 "type": "auto" 956 }, 957 "filterable": true, 958 "inspect": false 959 }, 960 "mappings": [], 961 "thresholds": { 962 "mode": "absolute", 963 "steps": [ 964 { 965 "color": "green" 966 }, 967 { 968 "color": "red", 969 "value": 80 970 } 971 ] 972 } 973 }, 974 "overrides": [] 975 }, 976 "gridPos": { 977 "h": 10, 978 "w": 6, 979 "x": 18, 980 "y": 7 981 }, 982 "id": 31, 983 "options": { 984 "cellHeight": "sm", 985 "footer": { 986 "countRows": false, 987 "enablePagination": false, 988 "fields": "", 989 "reducer": [ 990 "sum" 991 ], 992 "show": true 993 }, 994 "showHeader": true, 995 "showRowNums": false, 996 "sortBy": [] 997 }, 998 "pluginVersion": "12.1.0-88106", 999 "targets": [ 1000 { 1001 "datasource": { 1002 "type": "prometheus", 1003 "uid": "${datasource}" 1004 }, 1005 "editorMode": "code", 1006 "exemplar": false, 1007 "expr": "count by (container,version) (pyroscope_build_info{cluster=~\"$cluster\",namespace=~\"$namespace\"}) or count by (container,version) (cloud_backend_gateway_build_info{cluster=~\"$cluster\",namespace=~\"$namespace\"})", 1008 "format": "table", 1009 "instant": true, 1010 "legendFormat": "__auto", 1011 "range": false, 1012 "refId": "A" 1013 }, 1014 { 1015 "datasource": { 1016 "type": "prometheus", 1017 "uid": "${datasource}" 1018 }, 1019 "hide": false, 1020 "refId": "B" 1021 } 1022 ], 1023 "title": "Version", 1024 "transformations": [ 1025 { 1026 "id": "organize", 1027 "options": { 1028 "excludeByName": { 1029 "Time": true 1030 }, 1031 "indexByName": {}, 1032 "renameByName": { 1033 "Value": "Pods" 1034 } 1035 } 1036 } 1037 ], 1038 "type": "table" 1039 }, 1040 { 1041 "datasource": { 1042 "type": "prometheus", 1043 "uid": "$datasource" 1044 }, 1045 "fieldConfig": { 1046 "defaults": { 1047 "color": { 1048 "mode": "palette-classic" 1049 }, 1050 "custom": { 1051 "axisBorderShow": false, 1052 "axisCenteredZero": false, 1053 "axisColorMode": "text", 1054 "axisLabel": "", 1055 "axisPlacement": "auto", 1056 "barAlignment": 0, 1057 "barWidthFactor": 0.6, 1058 "drawStyle": "line", 1059 "fillOpacity": 100, 1060 "gradientMode": "none", 1061 "hideFrom": { 1062 "legend": false, 1063 "tooltip": false, 1064 "viz": false 1065 }, 1066 "insertNulls": false, 1067 "lineInterpolation": "linear", 1068 "lineWidth": 1, 1069 "pointSize": 5, 1070 "scaleDistribution": { 1071 "type": "linear" 1072 }, 1073 "showPoints": "never", 1074 "spanNulls": false, 1075 "stacking": { 1076 "group": "A", 1077 "mode": "normal" 1078 }, 1079 "thresholdsStyle": { 1080 "mode": "off" 1081 } 1082 }, 1083 "mappings": [], 1084 "thresholds": { 1085 "mode": "absolute", 1086 "steps": [ 1087 { 1088 "color": "green" 1089 }, 1090 { 1091 "color": "red", 1092 "value": 80 1093 } 1094 ] 1095 }, 1096 "unit": "reqps" 1097 }, 1098 "overrides": [ 1099 { 1100 "matcher": { 1101 "id": "byName", 1102 "options": "5xx " 1103 }, 1104 "properties": [ 1105 { 1106 "id": "color", 1107 "value": { 1108 "fixedColor": "red", 1109 "mode": "fixed" 1110 } 1111 } 1112 ] 1113 }, 1114 { 1115 "matcher": { 1116 "id": "byName", 1117 "options": "2xx " 1118 }, 1119 "properties": [ 1120 { 1121 "id": "color", 1122 "value": { 1123 "fixedColor": "green", 1124 "mode": "fixed", 1125 "seriesBy": "min" 1126 } 1127 } 1128 ] 1129 }, 1130 { 1131 "matcher": { 1132 "id": "byName", 1133 "options": "4xx " 1134 }, 1135 "properties": [ 1136 { 1137 "id": "color", 1138 "value": { 1139 "fixedColor": "yellow", 1140 "mode": "fixed" 1141 } 1142 } 1143 ] 1144 } 1145 ] 1146 }, 1147 "gridPos": { 1148 "h": 5, 1149 "w": 6, 1150 "x": 5, 1151 "y": 12 1152 }, 1153 "id": 6, 1154 "options": { 1155 "dataLinks": [], 1156 "legend": { 1157 "calcs": [], 1158 "displayMode": "list", 1159 "placement": "bottom", 1160 "showLegend": true 1161 }, 1162 "tooltip": { 1163 "hideZeros": false, 1164 "mode": "multi", 1165 "sort": "none" 1166 } 1167 }, 1168 "pluginVersion": "12.1.0-88106", 1169 "targets": [ 1170 { 1171 "datasource": { 1172 "uid": "$datasource" 1173 }, 1174 "editorMode": "code", 1175 "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(pyroscope_request_duration_seconds_count{cluster=~\"$cluster\", container=~\"pyroscope|distributor|query-frontend\", namespace=~\"$namespace\", route=~\".*pyroscope_render.*|.*pyroscope_label.*|.*querierservice.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)", 1176 "legendFormat": "{{status}} ", 1177 "range": true, 1178 "refId": "A" 1179 } 1180 ], 1181 "title": "Queries/Second", 1182 "type": "timeseries" 1183 }, 1184 { 1185 "datasource": { 1186 "type": "prometheus", 1187 "uid": "${datasource}" 1188 }, 1189 "fieldConfig": { 1190 "defaults": { 1191 "color": { 1192 "mode": "palette-classic" 1193 }, 1194 "custom": { 1195 "axisBorderShow": false, 1196 "axisCenteredZero": false, 1197 "axisColorMode": "text", 1198 "axisLabel": "", 1199 "axisPlacement": "auto", 1200 "barAlignment": 0, 1201 "barWidthFactor": 0.6, 1202 "drawStyle": "line", 1203 "fillOpacity": 0, 1204 "gradientMode": "none", 1205 "hideFrom": { 1206 "legend": false, 1207 "tooltip": false, 1208 "viz": false 1209 }, 1210 "insertNulls": false, 1211 "lineInterpolation": "linear", 1212 "lineWidth": 1, 1213 "pointSize": 5, 1214 "scaleDistribution": { 1215 "type": "linear" 1216 }, 1217 "showPoints": "auto", 1218 "spanNulls": false, 1219 "stacking": { 1220 "group": "A", 1221 "mode": "none" 1222 }, 1223 "thresholdsStyle": { 1224 "mode": "off" 1225 } 1226 }, 1227 "mappings": [], 1228 "thresholds": { 1229 "mode": "absolute", 1230 "steps": [ 1231 { 1232 "color": "green" 1233 }, 1234 { 1235 "color": "red", 1236 "value": 80 1237 } 1238 ] 1239 }, 1240 "unit": "s" 1241 }, 1242 "overrides": [] 1243 }, 1244 "gridPos": { 1245 "h": 5, 1246 "w": 7, 1247 "x": 11, 1248 "y": 12 1249 }, 1250 "id": 26, 1251 "options": { 1252 "legend": { 1253 "calcs": [], 1254 "displayMode": "table", 1255 "placement": "right", 1256 "showLegend": true 1257 }, 1258 "tooltip": { 1259 "hideZeros": false, 1260 "mode": "single", 1261 "sort": "none" 1262 } 1263 }, 1264 "pluginVersion": "12.1.0-88106", 1265 "targets": [ 1266 { 1267 "datasource": { 1268 "type": "prometheus", 1269 "uid": "${datasource}" 1270 }, 1271 "editorMode": "code", 1272 "expr": "histogram_quantile(0.99, sum by (route) ( rate(pyroscope_request_duration_seconds{container=~\"pyroscope|distributor|query-frontend\", namespace=~\"$namespace\", cluster=~\"$cluster\",route=~\".*pyroscope_render.*|.*pyroscope_label.*|.*querierservice.*\"}[$__rate_interval])))", 1273 "legendFormat": ".99 {{route}}", 1274 "range": true, 1275 "refId": "A" 1276 }, 1277 { 1278 "datasource": { 1279 "type": "prometheus", 1280 "uid": "${datasource}" 1281 }, 1282 "editorMode": "code", 1283 "expr": "histogram_quantile(0.75, sum by (route) ( rate(pyroscope_request_duration_seconds{container=~\"pyroscope|distributor|query-frontend\", namespace=~\"$namespace\", cluster=~\"$cluster\",route=~\".*pyroscope_render.*|.*pyroscope_label.*|.*querierservice.*\"}[$__rate_interval])))", 1284 "hide": false, 1285 "legendFormat": ".75 {{route}}", 1286 "range": true, 1287 "refId": "B" 1288 }, 1289 { 1290 "datasource": { 1291 "type": "prometheus", 1292 "uid": "${datasource}" 1293 }, 1294 "editorMode": "code", 1295 "expr": "histogram_quantile(0.5, sum by (route) ( rate(pyroscope_request_duration_seconds{container=~\"pyroscope|distributor|query-frontend\", namespace=~\"$namespace\", cluster=~\"$cluster\",route=~\".*pyroscope_render.*|.*pyroscope_label.*|.*querierservice.*\"}[$__rate_interval])))", 1296 "hide": false, 1297 "legendFormat": ".5 {{route}}", 1298 "range": true, 1299 "refId": "C" 1300 } 1301 ], 1302 "title": "Query Latencies", 1303 "type": "timeseries" 1304 }, 1305 { 1306 "datasource": { 1307 "type": "loki", 1308 "uid": "${loki_datasource}" 1309 }, 1310 "fieldConfig": { 1311 "defaults": { 1312 "color": { 1313 "fixedColor": "red", 1314 "mode": "fixed" 1315 }, 1316 "custom": { 1317 "axisBorderShow": false, 1318 "axisCenteredZero": false, 1319 "axisColorMode": "text", 1320 "axisLabel": "", 1321 "axisPlacement": "auto", 1322 "barAlignment": 0, 1323 "barWidthFactor": 0.6, 1324 "drawStyle": "bars", 1325 "fillOpacity": 23, 1326 "gradientMode": "none", 1327 "hideFrom": { 1328 "legend": false, 1329 "tooltip": false, 1330 "viz": false 1331 }, 1332 "insertNulls": false, 1333 "lineInterpolation": "linear", 1334 "lineWidth": 1, 1335 "pointSize": 5, 1336 "scaleDistribution": { 1337 "type": "linear" 1338 }, 1339 "showPoints": "auto", 1340 "spanNulls": false, 1341 "stacking": { 1342 "group": "A", 1343 "mode": "normal" 1344 }, 1345 "thresholdsStyle": { 1346 "mode": "off" 1347 } 1348 }, 1349 "mappings": [], 1350 "thresholds": { 1351 "mode": "absolute", 1352 "steps": [ 1353 { 1354 "color": "green" 1355 }, 1356 { 1357 "color": "red", 1358 "value": 80 1359 } 1360 ] 1361 } 1362 }, 1363 "overrides": [] 1364 }, 1365 "gridPos": { 1366 "h": 3, 1367 "w": 24, 1368 "x": 0, 1369 "y": 17 1370 }, 1371 "id": 39, 1372 "options": { 1373 "legend": { 1374 "calcs": [], 1375 "displayMode": "table", 1376 "placement": "right", 1377 "showLegend": true 1378 }, 1379 "tooltip": { 1380 "hideZeros": false, 1381 "mode": "single", 1382 "sort": "none" 1383 } 1384 }, 1385 "pluginVersion": "12.1.0-88106", 1386 "targets": [ 1387 { 1388 "datasource": { 1389 "type": "loki", 1390 "uid": "${loki_datasource}" 1391 }, 1392 "editorMode": "code", 1393 "expr": "sum by (container)( count_over_time({namespace=~\"$namespace\",cluster=~\"$cluster\"} != \"stream context finished\" != \"fast-joining node failed\" != \"CAS attempt failed\" != \"memberlist TCPTransport\" |= \" panic \" or \"panic:\" or \" err=\" or \"level=error\" [$__interval]))", 1394 "queryType": "range", 1395 "refId": "A" 1396 } 1397 ], 1398 "title": "Error \u0026 Panic rate", 1399 "type": "timeseries" 1400 }, 1401 { 1402 "datasource": { 1403 "type": "loki", 1404 "uid": "${loki_datasource}" 1405 }, 1406 "fieldConfig": { 1407 "defaults": {}, 1408 "overrides": [] 1409 }, 1410 "gridPos": { 1411 "h": 10, 1412 "w": 24, 1413 "x": 0, 1414 "y": 20 1415 }, 1416 "id": 37, 1417 "options": { 1418 "dedupStrategy": "numbers", 1419 "enableInfiniteScrolling": false, 1420 "enableLogDetails": true, 1421 "prettifyLogMessage": false, 1422 "showCommonLabels": false, 1423 "showLabels": false, 1424 "showTime": false, 1425 "sortOrder": "Descending", 1426 "wrapLogMessage": true 1427 }, 1428 "pluginVersion": "12.1.0-88106", 1429 "targets": [ 1430 { 1431 "datasource": { 1432 "type": "loki", 1433 "uid": "${loki_datasource}" 1434 }, 1435 "editorMode": "code", 1436 "expr": "{namespace=~\"$namespace\",cluster=~\"$cluster\"} != \"stream context finished\" != \"fast-joining node failed\" != \"CAS attempt failed\" != \"memberlist TCPTransport\" |= \" panic \" or \"panic:\" or \" err=\" or \"level=error\"", 1437 "queryType": "range", 1438 "refId": "A" 1439 } 1440 ], 1441 "title": "Errors and Panics", 1442 "type": "logs" 1443 }, 1444 { 1445 "datasource": { 1446 "type": "loki", 1447 "uid": "${loki_datasource}" 1448 }, 1449 "fieldConfig": { 1450 "defaults": { 1451 "color": { 1452 "mode": "thresholds" 1453 }, 1454 "custom": { 1455 "align": "auto", 1456 "cellOptions": { 1457 "type": "auto" 1458 }, 1459 "filterable": true, 1460 "inspect": false 1461 }, 1462 "mappings": [], 1463 "thresholds": { 1464 "mode": "absolute", 1465 "steps": [ 1466 { 1467 "color": "green" 1468 }, 1469 { 1470 "color": "red", 1471 "value": 80 1472 } 1473 ] 1474 } 1475 }, 1476 "overrides": [ 1477 { 1478 "matcher": { 1479 "id": "byName", 1480 "options": "ts" 1481 }, 1482 "properties": [ 1483 { 1484 "id": "custom.width", 1485 "value": 271 1486 } 1487 ] 1488 }, 1489 { 1490 "matcher": { 1491 "id": "byName", 1492 "options": "path" 1493 }, 1494 "properties": [ 1495 { 1496 "id": "custom.width", 1497 "value": 388 1498 } 1499 ] 1500 }, 1501 { 1502 "matcher": { 1503 "id": "byName", 1504 "options": "status" 1505 }, 1506 "properties": [ 1507 { 1508 "id": "custom.cellOptions", 1509 "value": { 1510 "applyToRow": false, 1511 "mode": "gradient", 1512 "type": "color-background", 1513 "wrapText": false 1514 } 1515 }, 1516 { 1517 "id": "mappings", 1518 "value": [ 1519 { 1520 "options": { 1521 "from": 200, 1522 "result": { 1523 "color": "light-green", 1524 "index": 0 1525 }, 1526 "to": 399 1527 }, 1528 "type": "range" 1529 }, 1530 { 1531 "options": { 1532 "from": 400, 1533 "result": { 1534 "color": "light-yellow", 1535 "index": 1 1536 }, 1537 "to": 499 1538 }, 1539 "type": "range" 1540 }, 1541 { 1542 "options": { 1543 "from": 500, 1544 "result": { 1545 "color": "light-red", 1546 "index": 2 1547 }, 1548 "to": 599 1549 }, 1550 "type": "range" 1551 } 1552 ] 1553 }, 1554 { 1555 "id": "custom.width", 1556 "value": 96 1557 } 1558 ] 1559 }, 1560 { 1561 "matcher": { 1562 "id": "byName", 1563 "options": "traceID" 1564 }, 1565 "properties": [ 1566 { 1567 "id": "links", 1568 "value": [ 1569 { 1570 "targetBlank": true, 1571 "title": "Show trace", 1572 "url": "/explore?schemaVersion=1\u0026panes=%7B%22xbp%22:%7B%22datasource%22:%22grafanacloud-traces%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22datasource%22:%7B%22type%22:%22tempo%22,%22uid%22:%22grafanacloud-traces%22%7D,%22queryType%22:%22traceql%22,%22limit%22:20,%22tableType%22:%22traces%22,%22metricsQueryType%22:%22range%22,%22query%22:%22${__value.text}%5Cn%22%7D%5D,%22range%22:%7B%22from%22:%22${__from}%22,%22to%22:%22${__to}%22%7D%7D%7D\u0026orgId=1" 1573 } 1574 ] 1575 } 1576 ] 1577 } 1578 ] 1579 }, 1580 "gridPos": { 1581 "h": 10, 1582 "w": 24, 1583 "x": 0, 1584 "y": 30 1585 }, 1586 "id": 108, 1587 "options": { 1588 "cellHeight": "sm", 1589 "footer": { 1590 "countRows": false, 1591 "enablePagination": true, 1592 "fields": "", 1593 "reducer": [ 1594 "sum" 1595 ], 1596 "show": false 1597 }, 1598 "showHeader": true, 1599 "sortBy": [ 1600 { 1601 "desc": false, 1602 "displayName": "status" 1603 } 1604 ] 1605 }, 1606 "pluginVersion": "12.1.0-88106", 1607 "targets": [ 1608 { 1609 "datasource": { 1610 "type": "loki", 1611 "uid": "${loki_datasource}" 1612 }, 1613 "direction": "backward", 1614 "editorMode": "code", 1615 "expr": "{namespace=~\"$namespace\", cluster=~\"$cluster\", container=\"cortex-gw\"} |= `msg=\"request timings\"` | logfmt", 1616 "queryType": "range", 1617 "refId": "A" 1618 } 1619 ], 1620 "title": "Query Activity", 1621 "transformations": [ 1622 { 1623 "id": "extractFields", 1624 "options": { 1625 "delimiter": ",", 1626 "format": "json", 1627 "replace": true, 1628 "source": "labels" 1629 } 1630 }, 1631 { 1632 "id": "organize", 1633 "options": { 1634 "excludeByName": { 1635 "__adaptive_logs_sampled__": true, 1636 "caller": true, 1637 "cluster": true, 1638 "conn_send": false, 1639 "container": true, 1640 "detected_level": true, 1641 "downstream": false, 1642 "insight": true, 1643 "job": true, 1644 "level": true, 1645 "msg": true, 1646 "name": true, 1647 "namespace": true, 1648 "path": false, 1649 "pod_template_hash": true, 1650 "service_name": true, 1651 "stream": true, 1652 "total": false 1653 }, 1654 "includeByName": {}, 1655 "indexByName": { 1656 "__adaptive_logs_sampled__": 22, 1657 "auth": 7, 1658 "caller": 8, 1659 "cluster": 9, 1660 "conn_send": 25, 1661 "container": 10, 1662 "dashboard_id": 24, 1663 "detected_level": 11, 1664 "downstream": 26, 1665 "grafana_username": 5, 1666 "insight": 12, 1667 "job": 13, 1668 "level": 14, 1669 "msg": 15, 1670 "name": 16, 1671 "namespace": 17, 1672 "panel_id": 27, 1673 "path": 1, 1674 "pod": 18, 1675 "pod_template_hash": 19, 1676 "query_hash": 23, 1677 "service_name": 20, 1678 "status": 2, 1679 "stream": 21, 1680 "total": 6, 1681 "traceID": 3, 1682 "ts": 0, 1683 "user": 4 1684 }, 1685 "renameByName": { 1686 "auth": "", 1687 "caller": "", 1688 "path": "", 1689 "total": "", 1690 "user": "tenant_id" 1691 } 1692 } 1693 }, 1694 { 1695 "id": "convertFieldType", 1696 "options": { 1697 "conversions": [], 1698 "fields": {} 1699 } 1700 } 1701 ], 1702 "type": "table" 1703 }, 1704 { 1705 "collapsed": true, 1706 "gridPos": { 1707 "h": 1, 1708 "w": 24, 1709 "x": 0, 1710 "y": 40 1711 }, 1712 "id": 52, 1713 "panels": [ 1714 { 1715 "datasource": { 1716 "type": "prometheus", 1717 "uid": "${datasource}" 1718 }, 1719 "fieldConfig": { 1720 "defaults": { 1721 "color": { 1722 "mode": "palette-classic" 1723 }, 1724 "custom": { 1725 "axisBorderShow": false, 1726 "axisCenteredZero": false, 1727 "axisColorMode": "text", 1728 "axisLabel": "", 1729 "axisPlacement": "auto", 1730 "barAlignment": 0, 1731 "drawStyle": "line", 1732 "fillOpacity": 21, 1733 "gradientMode": "none", 1734 "hideFrom": { 1735 "legend": false, 1736 "tooltip": false, 1737 "viz": false 1738 }, 1739 "insertNulls": false, 1740 "lineInterpolation": "linear", 1741 "lineStyle": { 1742 "fill": "solid" 1743 }, 1744 "lineWidth": 1, 1745 "pointSize": 5, 1746 "scaleDistribution": { 1747 "type": "linear" 1748 }, 1749 "showPoints": "auto", 1750 "spanNulls": false, 1751 "stacking": { 1752 "group": "A", 1753 "mode": "normal" 1754 }, 1755 "thresholdsStyle": { 1756 "mode": "dashed" 1757 } 1758 }, 1759 "mappings": [], 1760 "thresholds": { 1761 "mode": "absolute", 1762 "steps": [ 1763 { 1764 "color": "green" 1765 }, 1766 { 1767 "color": "yellow", 1768 "value": 80 1769 }, 1770 { 1771 "color": "red", 1772 "value": 90 1773 } 1774 ] 1775 } 1776 }, 1777 "overrides": [ 1778 { 1779 "matcher": { 1780 "id": "byName", 1781 "options": "request" 1782 }, 1783 "properties": [ 1784 { 1785 "id": "custom.stacking", 1786 "value": { 1787 "group": "A", 1788 "mode": "none" 1789 } 1790 }, 1791 { 1792 "id": "color", 1793 "value": { 1794 "fixedColor": "yellow", 1795 "mode": "fixed" 1796 } 1797 }, 1798 { 1799 "id": "custom.fillOpacity", 1800 "value": 0 1801 }, 1802 { 1803 "id": "custom.lineStyle", 1804 "value": { 1805 "dash": [ 1806 10, 1807 10 1808 ], 1809 "fill": "dash" 1810 } 1811 }, 1812 { 1813 "id": "custom.lineWidth", 1814 "value": 2 1815 } 1816 ] 1817 } 1818 ] 1819 }, 1820 "gridPos": { 1821 "h": 8, 1822 "w": 12, 1823 "x": 0, 1824 "y": 2 1825 }, 1826 "id": 53, 1827 "options": { 1828 "legend": { 1829 "calcs": [], 1830 "displayMode": "list", 1831 "placement": "bottom", 1832 "showLegend": true 1833 }, 1834 "tooltip": { 1835 "mode": "multi", 1836 "sort": "desc" 1837 } 1838 }, 1839 "targets": [ 1840 { 1841 "datasource": { 1842 "type": "prometheus", 1843 "uid": "${datasource}" 1844 }, 1845 "editorMode": "code", 1846 "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*ingester\", workload_type=\"statefulset\"}\n) by (pod)", 1847 "legendFormat": "__auto", 1848 "range": true, 1849 "refId": "by pod" 1850 }, 1851 { 1852 "datasource": { 1853 "type": "prometheus", 1854 "uid": "${datasource}" 1855 }, 1856 "editorMode": "code", 1857 "exemplar": false, 1858 "expr": "sum(\n kube_pod_container_resource_requests{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*ingester\", workload_type=\"statefulset\"}\n)", 1859 "hide": false, 1860 "instant": false, 1861 "legendFormat": "request", 1862 "range": true, 1863 "refId": "request" 1864 }, 1865 { 1866 "datasource": { 1867 "type": "prometheus", 1868 "uid": "${datasource}" 1869 }, 1870 "editorMode": "code", 1871 "expr": "sum(\n kube_pod_container_resource_limits{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*ingester\", workload_type=\"statefulset\"}\n)", 1872 "hide": false, 1873 "legendFormat": "limit", 1874 "range": true, 1875 "refId": "limit" 1876 } 1877 ], 1878 "title": "CPU Usage", 1879 "type": "timeseries" 1880 }, 1881 { 1882 "datasource": { 1883 "type": "prometheus", 1884 "uid": "${datasource}" 1885 }, 1886 "fieldConfig": { 1887 "defaults": { 1888 "color": { 1889 "mode": "palette-classic" 1890 }, 1891 "custom": { 1892 "axisBorderShow": false, 1893 "axisCenteredZero": false, 1894 "axisColorMode": "text", 1895 "axisLabel": "", 1896 "axisPlacement": "auto", 1897 "barAlignment": 0, 1898 "drawStyle": "line", 1899 "fillOpacity": 21, 1900 "gradientMode": "none", 1901 "hideFrom": { 1902 "legend": false, 1903 "tooltip": false, 1904 "viz": false 1905 }, 1906 "insertNulls": false, 1907 "lineInterpolation": "linear", 1908 "lineStyle": { 1909 "fill": "solid" 1910 }, 1911 "lineWidth": 1, 1912 "pointSize": 5, 1913 "scaleDistribution": { 1914 "type": "linear" 1915 }, 1916 "showPoints": "auto", 1917 "spanNulls": false, 1918 "stacking": { 1919 "group": "A", 1920 "mode": "normal" 1921 }, 1922 "thresholdsStyle": { 1923 "mode": "dashed" 1924 } 1925 }, 1926 "mappings": [], 1927 "thresholds": { 1928 "mode": "absolute", 1929 "steps": [ 1930 { 1931 "color": "green" 1932 }, 1933 { 1934 "color": "yellow", 1935 "value": 80 1936 }, 1937 { 1938 "color": "red", 1939 "value": 90 1940 } 1941 ] 1942 }, 1943 "unit": "bytes" 1944 }, 1945 "overrides": [ 1946 { 1947 "matcher": { 1948 "id": "byName", 1949 "options": "request" 1950 }, 1951 "properties": [ 1952 { 1953 "id": "custom.stacking", 1954 "value": { 1955 "group": "A", 1956 "mode": "none" 1957 } 1958 }, 1959 { 1960 "id": "color", 1961 "value": { 1962 "fixedColor": "yellow", 1963 "mode": "fixed" 1964 } 1965 }, 1966 { 1967 "id": "custom.fillOpacity", 1968 "value": 0 1969 }, 1970 { 1971 "id": "custom.lineStyle", 1972 "value": { 1973 "dash": [ 1974 10, 1975 10 1976 ], 1977 "fill": "dash" 1978 } 1979 }, 1980 { 1981 "id": "custom.lineWidth", 1982 "value": 2 1983 } 1984 ] 1985 }, 1986 { 1987 "matcher": { 1988 "id": "byName", 1989 "options": "limit" 1990 }, 1991 "properties": [ 1992 { 1993 "id": "color", 1994 "value": { 1995 "fixedColor": "red", 1996 "mode": "fixed" 1997 } 1998 }, 1999 { 2000 "id": "custom.stacking", 2001 "value": { 2002 "group": "A", 2003 "mode": "none" 2004 } 2005 }, 2006 { 2007 "id": "custom.fillOpacity", 2008 "value": 0 2009 }, 2010 { 2011 "id": "custom.lineStyle", 2012 "value": { 2013 "dash": [ 2014 10, 2015 10 2016 ], 2017 "fill": "dash" 2018 } 2019 }, 2020 { 2021 "id": "custom.lineWidth", 2022 "value": 2 2023 } 2024 ] 2025 } 2026 ] 2027 }, 2028 "gridPos": { 2029 "h": 8, 2030 "w": 12, 2031 "x": 12, 2032 "y": 2 2033 }, 2034 "id": 54, 2035 "options": { 2036 "legend": { 2037 "calcs": [], 2038 "displayMode": "list", 2039 "placement": "bottom", 2040 "showLegend": true 2041 }, 2042 "tooltip": { 2043 "mode": "multi", 2044 "sort": "desc" 2045 } 2046 }, 2047 "targets": [ 2048 { 2049 "datasource": { 2050 "type": "prometheus", 2051 "uid": "${datasource}" 2052 }, 2053 "editorMode": "code", 2054 "expr": "sum(\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*ingester\", workload_type=\"statefulset\"}\n) by (pod)\n", 2055 "legendFormat": "__auto", 2056 "range": true, 2057 "refId": "by pod" 2058 }, 2059 { 2060 "datasource": { 2061 "type": "prometheus", 2062 "uid": "${datasource}" 2063 }, 2064 "editorMode": "code", 2065 "exemplar": false, 2066 "expr": "sum(\n kube_pod_container_resource_requests{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*ingester\", workload_type=\"statefulset\"}\n)", 2067 "hide": false, 2068 "instant": false, 2069 "legendFormat": "request", 2070 "range": true, 2071 "refId": "request" 2072 }, 2073 { 2074 "datasource": { 2075 "type": "prometheus", 2076 "uid": "${datasource}" 2077 }, 2078 "editorMode": "code", 2079 "expr": "sum(\n kube_pod_container_resource_limits{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*ingester\", workload_type=\"statefulset\"}\n)", 2080 "hide": false, 2081 "legendFormat": "limit", 2082 "range": true, 2083 "refId": "limit" 2084 } 2085 ], 2086 "title": "Memory Usage", 2087 "type": "timeseries" 2088 }, 2089 { 2090 "datasource": { 2091 "type": "prometheus", 2092 "uid": "${datasource}" 2093 }, 2094 "fieldConfig": { 2095 "defaults": { 2096 "color": { 2097 "mode": "palette-classic" 2098 }, 2099 "custom": { 2100 "axisBorderShow": false, 2101 "axisCenteredZero": false, 2102 "axisColorMode": "text", 2103 "axisLabel": "", 2104 "axisPlacement": "auto", 2105 "barAlignment": 0, 2106 "drawStyle": "line", 2107 "fillOpacity": 13, 2108 "gradientMode": "none", 2109 "hideFrom": { 2110 "legend": false, 2111 "tooltip": false, 2112 "viz": false 2113 }, 2114 "insertNulls": false, 2115 "lineInterpolation": "linear", 2116 "lineWidth": 1, 2117 "pointSize": 5, 2118 "scaleDistribution": { 2119 "type": "linear" 2120 }, 2121 "showPoints": "auto", 2122 "spanNulls": false, 2123 "stacking": { 2124 "group": "A", 2125 "mode": "normal" 2126 }, 2127 "thresholdsStyle": { 2128 "mode": "off" 2129 } 2130 }, 2131 "mappings": [], 2132 "thresholds": { 2133 "mode": "absolute", 2134 "steps": [ 2135 { 2136 "color": "green" 2137 }, 2138 { 2139 "color": "red", 2140 "value": 80 2141 } 2142 ] 2143 }, 2144 "unit": "Bps" 2145 }, 2146 "overrides": [] 2147 }, 2148 "gridPos": { 2149 "h": 8, 2150 "w": 12, 2151 "x": 0, 2152 "y": 10 2153 }, 2154 "id": 73, 2155 "options": { 2156 "legend": { 2157 "calcs": [], 2158 "displayMode": "list", 2159 "placement": "bottom", 2160 "showLegend": false 2161 }, 2162 "tooltip": { 2163 "mode": "multi", 2164 "sort": "none" 2165 } 2166 }, 2167 "targets": [ 2168 { 2169 "datasource": { 2170 "type": "prometheus", 2171 "uid": "${datasource}" 2172 }, 2173 "editorMode": "code", 2174 "expr": "(sum(irate(container_network_receive_bytes_total{job=~\"(.*/)?cadvisor\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*ingester.*\", workload_type=\"statefulset\"}) by (pod))", 2175 "legendFormat": "__auto", 2176 "range": true, 2177 "refId": "A" 2178 } 2179 ], 2180 "title": "Receive Bandwidth", 2181 "type": "timeseries" 2182 }, 2183 { 2184 "datasource": { 2185 "type": "prometheus", 2186 "uid": "${datasource}" 2187 }, 2188 "fieldConfig": { 2189 "defaults": { 2190 "color": { 2191 "mode": "palette-classic" 2192 }, 2193 "custom": { 2194 "axisBorderShow": false, 2195 "axisCenteredZero": false, 2196 "axisColorMode": "text", 2197 "axisLabel": "", 2198 "axisPlacement": "auto", 2199 "barAlignment": 0, 2200 "drawStyle": "line", 2201 "fillOpacity": 13, 2202 "gradientMode": "none", 2203 "hideFrom": { 2204 "legend": false, 2205 "tooltip": false, 2206 "viz": false 2207 }, 2208 "insertNulls": false, 2209 "lineInterpolation": "linear", 2210 "lineWidth": 1, 2211 "pointSize": 5, 2212 "scaleDistribution": { 2213 "type": "linear" 2214 }, 2215 "showPoints": "auto", 2216 "spanNulls": false, 2217 "stacking": { 2218 "group": "A", 2219 "mode": "normal" 2220 }, 2221 "thresholdsStyle": { 2222 "mode": "off" 2223 } 2224 }, 2225 "mappings": [], 2226 "thresholds": { 2227 "mode": "absolute", 2228 "steps": [ 2229 { 2230 "color": "green" 2231 }, 2232 { 2233 "color": "red", 2234 "value": 80 2235 } 2236 ] 2237 }, 2238 "unit": "Bps" 2239 }, 2240 "overrides": [] 2241 }, 2242 "gridPos": { 2243 "h": 8, 2244 "w": 12, 2245 "x": 12, 2246 "y": 10 2247 }, 2248 "id": 74, 2249 "options": { 2250 "legend": { 2251 "calcs": [], 2252 "displayMode": "list", 2253 "placement": "bottom", 2254 "showLegend": false 2255 }, 2256 "tooltip": { 2257 "mode": "multi", 2258 "sort": "none" 2259 } 2260 }, 2261 "targets": [ 2262 { 2263 "datasource": { 2264 "type": "prometheus", 2265 "uid": "${datasource}" 2266 }, 2267 "editorMode": "code", 2268 "expr": "(sum(irate(container_network_transmit_bytes_total{job=~\"(.*/)?cadvisor\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*ingester.*\", workload_type=\"statefulset\"}) by (pod))", 2269 "legendFormat": "__auto", 2270 "range": true, 2271 "refId": "A" 2272 } 2273 ], 2274 "title": "Transmit Bandwidth", 2275 "type": "timeseries" 2276 }, 2277 { 2278 "datasource": { 2279 "type": "prometheus", 2280 "uid": "${datasource}" 2281 }, 2282 "fieldConfig": { 2283 "defaults": { 2284 "color": { 2285 "mode": "palette-classic" 2286 }, 2287 "custom": { 2288 "axisBorderShow": false, 2289 "axisCenteredZero": false, 2290 "axisColorMode": "text", 2291 "axisLabel": "", 2292 "axisPlacement": "auto", 2293 "barAlignment": 0, 2294 "drawStyle": "line", 2295 "fillOpacity": 0, 2296 "gradientMode": "none", 2297 "hideFrom": { 2298 "legend": false, 2299 "tooltip": false, 2300 "viz": false 2301 }, 2302 "insertNulls": false, 2303 "lineInterpolation": "linear", 2304 "lineWidth": 1, 2305 "pointSize": 5, 2306 "scaleDistribution": { 2307 "type": "linear" 2308 }, 2309 "showPoints": "auto", 2310 "spanNulls": false, 2311 "stacking": { 2312 "group": "A", 2313 "mode": "none" 2314 }, 2315 "thresholdsStyle": { 2316 "mode": "off" 2317 } 2318 }, 2319 "mappings": [], 2320 "thresholds": { 2321 "mode": "absolute", 2322 "steps": [ 2323 { 2324 "color": "green" 2325 }, 2326 { 2327 "color": "red", 2328 "value": 80 2329 } 2330 ] 2331 }, 2332 "unit": "s" 2333 }, 2334 "overrides": [] 2335 }, 2336 "gridPos": { 2337 "h": 5, 2338 "w": 12, 2339 "x": 0, 2340 "y": 18 2341 }, 2342 "id": 96, 2343 "options": { 2344 "legend": { 2345 "calcs": [], 2346 "displayMode": "table", 2347 "placement": "right", 2348 "showLegend": true 2349 }, 2350 "tooltip": { 2351 "mode": "single", 2352 "sort": "none" 2353 } 2354 }, 2355 "targets": [ 2356 { 2357 "datasource": { 2358 "type": "prometheus", 2359 "uid": "${datasource}" 2360 }, 2361 "editorMode": "code", 2362 "expr": "histogram_quantile(0.99, sum by (route) ( rate(pyroscope_request_duration_seconds{job=~\"$namespace/ingester\", cluster=~\"$cluster\",route=~\".*push.*\"}[$__rate_interval])))", 2363 "legendFormat": ".99 {{route}}", 2364 "range": true, 2365 "refId": "A" 2366 }, 2367 { 2368 "datasource": { 2369 "type": "prometheus", 2370 "uid": "${datasource}" 2371 }, 2372 "editorMode": "code", 2373 "expr": "histogram_quantile(0.75, sum by (route) ( rate(pyroscope_request_duration_seconds{job=~\"$namespace/ingester\", cluster=~\"$cluster\",route=~\".*push.*\"}[$__rate_interval])))", 2374 "hide": false, 2375 "legendFormat": ".75 {{route}}", 2376 "range": true, 2377 "refId": "B" 2378 }, 2379 { 2380 "datasource": { 2381 "type": "prometheus", 2382 "uid": "${datasource}" 2383 }, 2384 "editorMode": "code", 2385 "expr": "histogram_quantile(0.5, sum by (route) ( rate(pyroscope_request_duration_seconds{job=~\"$namespace/ingester\", cluster=~\"$cluster\",route=~\".*push.*\"}[$__rate_interval])))", 2386 "hide": false, 2387 "legendFormat": ".5 {{route}}", 2388 "range": true, 2389 "refId": "C" 2390 } 2391 ], 2392 "title": "Push Latencies", 2393 "type": "timeseries" 2394 }, 2395 { 2396 "datasource": { 2397 "type": "prometheus", 2398 "uid": "$datasource" 2399 }, 2400 "fieldConfig": { 2401 "defaults": { 2402 "color": { 2403 "mode": "palette-classic" 2404 }, 2405 "custom": { 2406 "axisBorderShow": false, 2407 "axisCenteredZero": false, 2408 "axisColorMode": "text", 2409 "axisLabel": "", 2410 "axisPlacement": "auto", 2411 "barAlignment": 0, 2412 "drawStyle": "line", 2413 "fillOpacity": 40, 2414 "gradientMode": "none", 2415 "hideFrom": { 2416 "legend": false, 2417 "tooltip": false, 2418 "viz": false 2419 }, 2420 "insertNulls": false, 2421 "lineInterpolation": "linear", 2422 "lineWidth": 1, 2423 "pointSize": 5, 2424 "scaleDistribution": { 2425 "type": "linear" 2426 }, 2427 "showPoints": "never", 2428 "spanNulls": false, 2429 "stacking": { 2430 "group": "A", 2431 "mode": "normal" 2432 }, 2433 "thresholdsStyle": { 2434 "mode": "off" 2435 } 2436 }, 2437 "mappings": [], 2438 "thresholds": { 2439 "mode": "absolute", 2440 "steps": [ 2441 { 2442 "color": "green" 2443 }, 2444 { 2445 "color": "red", 2446 "value": 80 2447 } 2448 ] 2449 }, 2450 "unit": "reqps" 2451 }, 2452 "overrides": [ 2453 { 2454 "matcher": { 2455 "id": "byName", 2456 "options": "5xx " 2457 }, 2458 "properties": [ 2459 { 2460 "id": "color", 2461 "value": { 2462 "fixedColor": "red", 2463 "mode": "fixed" 2464 } 2465 } 2466 ] 2467 }, 2468 { 2469 "matcher": { 2470 "id": "byName", 2471 "options": "2xx " 2472 }, 2473 "properties": [ 2474 { 2475 "id": "color", 2476 "value": { 2477 "fixedColor": "green", 2478 "mode": "fixed", 2479 "seriesBy": "min" 2480 } 2481 } 2482 ] 2483 }, 2484 { 2485 "matcher": { 2486 "id": "byName", 2487 "options": "4xx " 2488 }, 2489 "properties": [ 2490 { 2491 "id": "color", 2492 "value": { 2493 "fixedColor": "yellow", 2494 "mode": "fixed" 2495 } 2496 } 2497 ] 2498 } 2499 ] 2500 }, 2501 "gridPos": { 2502 "h": 5, 2503 "w": 12, 2504 "x": 12, 2505 "y": 18 2506 }, 2507 "id": 69, 2508 "options": { 2509 "dataLinks": [], 2510 "legend": { 2511 "calcs": [], 2512 "displayMode": "list", 2513 "placement": "bottom", 2514 "showLegend": true 2515 }, 2516 "tooltip": { 2517 "mode": "multi", 2518 "sort": "none" 2519 } 2520 }, 2521 "targets": [ 2522 { 2523 "datasource": { 2524 "uid": "$datasource" 2525 }, 2526 "editorMode": "code", 2527 "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(pyroscope_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"$namespace/ingester\", route=~\".*push.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)", 2528 "legendFormat": "{{status}} ", 2529 "range": true, 2530 "refId": "A" 2531 } 2532 ], 2533 "title": "Pushes/Second", 2534 "type": "timeseries" 2535 }, 2536 { 2537 "datasource": { 2538 "type": "loki", 2539 "uid": "${loki_datasource}" 2540 }, 2541 "fieldConfig": { 2542 "defaults": { 2543 "color": { 2544 "mode": "palette-classic" 2545 }, 2546 "custom": { 2547 "axisBorderShow": false, 2548 "axisCenteredZero": false, 2549 "axisColorMode": "text", 2550 "axisLabel": "", 2551 "axisPlacement": "auto", 2552 "barAlignment": 0, 2553 "drawStyle": "line", 2554 "fillOpacity": 0, 2555 "gradientMode": "none", 2556 "hideFrom": { 2557 "legend": false, 2558 "tooltip": false, 2559 "viz": false 2560 }, 2561 "insertNulls": false, 2562 "lineInterpolation": "linear", 2563 "lineWidth": 1, 2564 "pointSize": 5, 2565 "scaleDistribution": { 2566 "type": "linear" 2567 }, 2568 "showPoints": "auto", 2569 "spanNulls": false, 2570 "stacking": { 2571 "group": "A", 2572 "mode": "none" 2573 }, 2574 "thresholdsStyle": { 2575 "mode": "off" 2576 } 2577 }, 2578 "mappings": [], 2579 "thresholds": { 2580 "mode": "absolute", 2581 "steps": [ 2582 { 2583 "color": "green" 2584 }, 2585 { 2586 "color": "red", 2587 "value": 80 2588 } 2589 ] 2590 } 2591 }, 2592 "overrides": [] 2593 }, 2594 "gridPos": { 2595 "h": 8, 2596 "w": 12, 2597 "x": 0, 2598 "y": 23 2599 }, 2600 "id": 83, 2601 "options": { 2602 "legend": { 2603 "calcs": [], 2604 "displayMode": "list", 2605 "placement": "bottom", 2606 "showLegend": true 2607 }, 2608 "tooltip": { 2609 "mode": "multi", 2610 "sort": "none" 2611 } 2612 }, 2613 "targets": [ 2614 { 2615 "datasource": { 2616 "type": "loki", 2617 "uid": "${loki_datasource}" 2618 }, 2619 "editorMode": "code", 2620 "expr": "sum by (tenant) (count_over_time({cluster=~\"$cluster\",namespace=~\"$namespace\"} |= \"upload new block\" | logfmt [$__interval]))", 2621 "queryType": "range", 2622 "refId": "A" 2623 } 2624 ], 2625 "title": "Block Uploaded Per Tenant", 2626 "type": "timeseries" 2627 }, 2628 { 2629 "datasource": { 2630 "type": "loki", 2631 "uid": "${loki_datasource}" 2632 }, 2633 "fieldConfig": { 2634 "defaults": { 2635 "color": { 2636 "mode": "palette-classic" 2637 }, 2638 "custom": { 2639 "axisBorderShow": false, 2640 "axisCenteredZero": false, 2641 "axisColorMode": "text", 2642 "axisLabel": "", 2643 "axisPlacement": "auto", 2644 "barAlignment": 0, 2645 "drawStyle": "bars", 2646 "fillOpacity": 15, 2647 "gradientMode": "none", 2648 "hideFrom": { 2649 "legend": false, 2650 "tooltip": false, 2651 "viz": false 2652 }, 2653 "insertNulls": false, 2654 "lineInterpolation": "linear", 2655 "lineWidth": 1, 2656 "pointSize": 5, 2657 "scaleDistribution": { 2658 "type": "linear" 2659 }, 2660 "showPoints": "auto", 2661 "spanNulls": false, 2662 "stacking": { 2663 "group": "A", 2664 "mode": "normal" 2665 }, 2666 "thresholdsStyle": { 2667 "mode": "off" 2668 } 2669 }, 2670 "mappings": [], 2671 "thresholds": { 2672 "mode": "absolute", 2673 "steps": [ 2674 { 2675 "color": "green" 2676 }, 2677 { 2678 "color": "red", 2679 "value": 80 2680 } 2681 ] 2682 } 2683 }, 2684 "overrides": [] 2685 }, 2686 "gridPos": { 2687 "h": 8, 2688 "w": 12, 2689 "x": 12, 2690 "y": 23 2691 }, 2692 "id": 84, 2693 "options": { 2694 "legend": { 2695 "calcs": [], 2696 "displayMode": "list", 2697 "placement": "bottom", 2698 "showLegend": true 2699 }, 2700 "tooltip": { 2701 "mode": "multi", 2702 "sort": "none" 2703 } 2704 }, 2705 "targets": [ 2706 { 2707 "datasource": { 2708 "type": "loki", 2709 "uid": "${loki_datasource}" 2710 }, 2711 "editorMode": "code", 2712 "expr": "sum by (tenant) (count_over_time({cluster=~\"$cluster\",namespace=~\"$namespace\"} |= \"cut row group segment\" | logfmt [$__interval]))", 2713 "hide": true, 2714 "queryType": "range", 2715 "refId": "A" 2716 }, 2717 { 2718 "datasource": { 2719 "type": "loki", 2720 "uid": "${loki_datasource}" 2721 }, 2722 "editorMode": "code", 2723 "expr": "sum by (tenant) (sum_over_time({cluster=~\"$cluster\",namespace=~\"$namespace\"} |= \"cut row group segment\" | logfmt | unwrap numProfiles [$__interval]))", 2724 "hide": false, 2725 "queryType": "range", 2726 "refId": "B" 2727 } 2728 ], 2729 "title": "Profiles Flushed to disk", 2730 "type": "timeseries" 2731 } 2732 ], 2733 "title": "Ingesters", 2734 "type": "row" 2735 }, 2736 { 2737 "collapsed": true, 2738 "gridPos": { 2739 "h": 1, 2740 "w": 24, 2741 "x": 0, 2742 "y": 41 2743 }, 2744 "id": 41, 2745 "panels": [ 2746 { 2747 "datasource": { 2748 "type": "prometheus", 2749 "uid": "${datasource}" 2750 }, 2751 "fieldConfig": { 2752 "defaults": { 2753 "color": { 2754 "mode": "palette-classic" 2755 }, 2756 "custom": { 2757 "axisCenteredZero": false, 2758 "axisColorMode": "text", 2759 "axisLabel": "", 2760 "axisPlacement": "auto", 2761 "barAlignment": 0, 2762 "drawStyle": "line", 2763 "fillOpacity": 0, 2764 "gradientMode": "none", 2765 "hideFrom": { 2766 "legend": false, 2767 "tooltip": false, 2768 "viz": false 2769 }, 2770 "lineInterpolation": "linear", 2771 "lineWidth": 1, 2772 "pointSize": 5, 2773 "scaleDistribution": { 2774 "type": "linear" 2775 }, 2776 "showPoints": "auto", 2777 "spanNulls": false, 2778 "stacking": { 2779 "group": "A", 2780 "mode": "none" 2781 }, 2782 "thresholdsStyle": { 2783 "mode": "off" 2784 } 2785 }, 2786 "mappings": [], 2787 "thresholds": { 2788 "mode": "absolute", 2789 "steps": [ 2790 { 2791 "color": "green" 2792 }, 2793 { 2794 "color": "red", 2795 "value": 80 2796 } 2797 ] 2798 } 2799 }, 2800 "overrides": [] 2801 }, 2802 "gridPos": { 2803 "h": 8, 2804 "w": 12, 2805 "x": 0, 2806 "y": 50 2807 }, 2808 "id": 43, 2809 "options": { 2810 "legend": { 2811 "calcs": [], 2812 "displayMode": "list", 2813 "placement": "bottom", 2814 "showLegend": true 2815 }, 2816 "tooltip": { 2817 "mode": "single", 2818 "sort": "none" 2819 } 2820 }, 2821 "targets": [ 2822 { 2823 "datasource": { 2824 "type": "prometheus", 2825 "uid": "${datasource}" 2826 }, 2827 "editorMode": "code", 2828 "expr": "topk(10,sum by (tenant, reason) (rate(pyroscope_discarded_samples_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[1m])))", 2829 "legendFormat": "{{ tenant }} - {{ reason }}", 2830 "range": true, 2831 "refId": "A" 2832 } 2833 ], 2834 "title": "Discarded Profiles", 2835 "type": "timeseries" 2836 }, 2837 { 2838 "datasource": { 2839 "type": "prometheus", 2840 "uid": "${datasource}" 2841 }, 2842 "fieldConfig": { 2843 "defaults": { 2844 "color": { 2845 "mode": "thresholds" 2846 }, 2847 "custom": { 2848 "align": "auto", 2849 "cellOptions": { 2850 "type": "auto" 2851 }, 2852 "inspect": false 2853 }, 2854 "mappings": [], 2855 "thresholds": { 2856 "mode": "absolute", 2857 "steps": [ 2858 { 2859 "color": "green" 2860 }, 2861 { 2862 "color": "red", 2863 "value": 80 2864 } 2865 ] 2866 } 2867 }, 2868 "overrides": [] 2869 }, 2870 "gridPos": { 2871 "h": 8, 2872 "w": 12, 2873 "x": 12, 2874 "y": 50 2875 }, 2876 "id": 45, 2877 "options": { 2878 "cellHeight": "sm", 2879 "footer": { 2880 "countRows": false, 2881 "fields": "", 2882 "reducer": [ 2883 "sum" 2884 ], 2885 "show": false 2886 }, 2887 "showHeader": true, 2888 "showRowNums": false 2889 }, 2890 "pluginVersion": "9.5.0-53857pre", 2891 "targets": [ 2892 { 2893 "datasource": { 2894 "type": "prometheus", 2895 "uid": "${datasource}" 2896 }, 2897 "editorMode": "code", 2898 "exemplar": false, 2899 "expr": "topk(10, sum by (tenant, reason) (sum_over_time(increase(pyroscope_discarded_samples_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[1m])[$__range:1m])))", 2900 "format": "table", 2901 "instant": true, 2902 "legendFormat": "{{ tenant }} - {{ reason }}", 2903 "range": false, 2904 "refId": "A" 2905 } 2906 ], 2907 "title": "Discarded Profiles Per Interval", 2908 "transformations": [ 2909 { 2910 "id": "organize", 2911 "options": { 2912 "excludeByName": { 2913 "Time": true 2914 }, 2915 "indexByName": { 2916 "Time": 0, 2917 "Value": 3, 2918 "reason": 2, 2919 "tenant": 1 2920 }, 2921 "renameByName": { 2922 "Value": "Profiles", 2923 "reason": "Reason", 2924 "tenant": "Tenant" 2925 } 2926 } 2927 } 2928 ], 2929 "type": "table" 2930 } 2931 ], 2932 "title": "Tenant Limits", 2933 "type": "row" 2934 }, 2935 { 2936 "collapsed": true, 2937 "gridPos": { 2938 "h": 1, 2939 "w": 24, 2940 "x": 0, 2941 "y": 42 2942 }, 2943 "id": 47, 2944 "panels": [ 2945 { 2946 "datasource": { 2947 "type": "prometheus", 2948 "uid": "${datasource}" 2949 }, 2950 "fieldConfig": { 2951 "defaults": { 2952 "color": { 2953 "mode": "palette-classic" 2954 }, 2955 "custom": { 2956 "axisBorderShow": false, 2957 "axisCenteredZero": false, 2958 "axisColorMode": "text", 2959 "axisLabel": "", 2960 "axisPlacement": "auto", 2961 "barAlignment": 0, 2962 "drawStyle": "line", 2963 "fillOpacity": 21, 2964 "gradientMode": "none", 2965 "hideFrom": { 2966 "legend": false, 2967 "tooltip": false, 2968 "viz": false 2969 }, 2970 "insertNulls": false, 2971 "lineInterpolation": "linear", 2972 "lineStyle": { 2973 "fill": "solid" 2974 }, 2975 "lineWidth": 1, 2976 "pointSize": 5, 2977 "scaleDistribution": { 2978 "type": "linear" 2979 }, 2980 "showPoints": "auto", 2981 "spanNulls": false, 2982 "stacking": { 2983 "group": "A", 2984 "mode": "normal" 2985 }, 2986 "thresholdsStyle": { 2987 "mode": "dashed" 2988 } 2989 }, 2990 "mappings": [], 2991 "thresholds": { 2992 "mode": "absolute", 2993 "steps": [ 2994 { 2995 "color": "green" 2996 }, 2997 { 2998 "color": "yellow", 2999 "value": 80 3000 }, 3001 { 3002 "color": "red", 3003 "value": 90 3004 } 3005 ] 3006 } 3007 }, 3008 "overrides": [ 3009 { 3010 "matcher": { 3011 "id": "byName", 3012 "options": "request" 3013 }, 3014 "properties": [ 3015 { 3016 "id": "custom.stacking", 3017 "value": { 3018 "group": "A", 3019 "mode": "none" 3020 } 3021 }, 3022 { 3023 "id": "color", 3024 "value": { 3025 "fixedColor": "yellow", 3026 "mode": "fixed" 3027 } 3028 }, 3029 { 3030 "id": "custom.fillOpacity", 3031 "value": 0 3032 }, 3033 { 3034 "id": "custom.lineStyle", 3035 "value": { 3036 "dash": [ 3037 10, 3038 10 3039 ], 3040 "fill": "dash" 3041 } 3042 }, 3043 { 3044 "id": "custom.lineWidth", 3045 "value": 2 3046 } 3047 ] 3048 } 3049 ] 3050 }, 3051 "gridPos": { 3052 "h": 8, 3053 "w": 12, 3054 "x": 0, 3055 "y": 4 3056 }, 3057 "id": 49, 3058 "options": { 3059 "legend": { 3060 "calcs": [], 3061 "displayMode": "list", 3062 "placement": "bottom", 3063 "showLegend": true 3064 }, 3065 "tooltip": { 3066 "mode": "multi", 3067 "sort": "desc" 3068 } 3069 }, 3070 "targets": [ 3071 { 3072 "datasource": { 3073 "type": "prometheus", 3074 "uid": "${datasource}" 3075 }, 3076 "editorMode": "code", 3077 "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*distributor\", workload_type=\"deployment\"}\n) by (pod)", 3078 "legendFormat": "__auto", 3079 "range": true, 3080 "refId": "by pod" 3081 }, 3082 { 3083 "datasource": { 3084 "type": "prometheus", 3085 "uid": "${datasource}" 3086 }, 3087 "editorMode": "code", 3088 "exemplar": false, 3089 "expr": "sum(\n kube_pod_container_resource_requests{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*distributor\", workload_type=\"deployment\"}\n)", 3090 "hide": false, 3091 "instant": false, 3092 "legendFormat": "request", 3093 "range": true, 3094 "refId": "request" 3095 }, 3096 { 3097 "datasource": { 3098 "type": "prometheus", 3099 "uid": "${datasource}" 3100 }, 3101 "editorMode": "code", 3102 "expr": "sum(\n kube_pod_container_resource_limits{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*distributor\", workload_type=\"deployment\"}\n)", 3103 "hide": false, 3104 "legendFormat": "limit", 3105 "range": true, 3106 "refId": "limit" 3107 } 3108 ], 3109 "title": "CPU Usage", 3110 "type": "timeseries" 3111 }, 3112 { 3113 "datasource": { 3114 "type": "prometheus", 3115 "uid": "${datasource}" 3116 }, 3117 "fieldConfig": { 3118 "defaults": { 3119 "color": { 3120 "mode": "palette-classic" 3121 }, 3122 "custom": { 3123 "axisBorderShow": false, 3124 "axisCenteredZero": false, 3125 "axisColorMode": "text", 3126 "axisLabel": "", 3127 "axisPlacement": "auto", 3128 "barAlignment": 0, 3129 "drawStyle": "line", 3130 "fillOpacity": 21, 3131 "gradientMode": "none", 3132 "hideFrom": { 3133 "legend": false, 3134 "tooltip": false, 3135 "viz": false 3136 }, 3137 "insertNulls": false, 3138 "lineInterpolation": "linear", 3139 "lineStyle": { 3140 "fill": "solid" 3141 }, 3142 "lineWidth": 1, 3143 "pointSize": 5, 3144 "scaleDistribution": { 3145 "type": "linear" 3146 }, 3147 "showPoints": "auto", 3148 "spanNulls": false, 3149 "stacking": { 3150 "group": "A", 3151 "mode": "normal" 3152 }, 3153 "thresholdsStyle": { 3154 "mode": "dashed" 3155 } 3156 }, 3157 "mappings": [], 3158 "thresholds": { 3159 "mode": "absolute", 3160 "steps": [ 3161 { 3162 "color": "green" 3163 }, 3164 { 3165 "color": "yellow", 3166 "value": 80 3167 }, 3168 { 3169 "color": "red", 3170 "value": 90 3171 } 3172 ] 3173 }, 3174 "unit": "bytes" 3175 }, 3176 "overrides": [ 3177 { 3178 "matcher": { 3179 "id": "byName", 3180 "options": "request" 3181 }, 3182 "properties": [ 3183 { 3184 "id": "custom.stacking", 3185 "value": { 3186 "group": "A", 3187 "mode": "none" 3188 } 3189 }, 3190 { 3191 "id": "color", 3192 "value": { 3193 "fixedColor": "yellow", 3194 "mode": "fixed" 3195 } 3196 }, 3197 { 3198 "id": "custom.fillOpacity", 3199 "value": 0 3200 }, 3201 { 3202 "id": "custom.lineStyle", 3203 "value": { 3204 "dash": [ 3205 10, 3206 10 3207 ], 3208 "fill": "dash" 3209 } 3210 }, 3211 { 3212 "id": "custom.lineWidth", 3213 "value": 2 3214 } 3215 ] 3216 }, 3217 { 3218 "matcher": { 3219 "id": "byName", 3220 "options": "limit" 3221 }, 3222 "properties": [ 3223 { 3224 "id": "color", 3225 "value": { 3226 "fixedColor": "red", 3227 "mode": "fixed" 3228 } 3229 }, 3230 { 3231 "id": "custom.stacking", 3232 "value": { 3233 "group": "A", 3234 "mode": "none" 3235 } 3236 }, 3237 { 3238 "id": "custom.fillOpacity", 3239 "value": 0 3240 }, 3241 { 3242 "id": "custom.lineStyle", 3243 "value": { 3244 "dash": [ 3245 10, 3246 10 3247 ], 3248 "fill": "dash" 3249 } 3250 }, 3251 { 3252 "id": "custom.lineWidth", 3253 "value": 2 3254 } 3255 ] 3256 } 3257 ] 3258 }, 3259 "gridPos": { 3260 "h": 8, 3261 "w": 12, 3262 "x": 12, 3263 "y": 4 3264 }, 3265 "id": 50, 3266 "options": { 3267 "legend": { 3268 "calcs": [], 3269 "displayMode": "list", 3270 "placement": "bottom", 3271 "showLegend": true 3272 }, 3273 "tooltip": { 3274 "mode": "multi", 3275 "sort": "desc" 3276 } 3277 }, 3278 "targets": [ 3279 { 3280 "datasource": { 3281 "type": "prometheus", 3282 "uid": "${datasource}" 3283 }, 3284 "editorMode": "code", 3285 "expr": "sum(\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*distributor\", workload_type=\"deployment\"}\n) by (pod)\n", 3286 "legendFormat": "__auto", 3287 "range": true, 3288 "refId": "by pod" 3289 }, 3290 { 3291 "datasource": { 3292 "type": "prometheus", 3293 "uid": "${datasource}" 3294 }, 3295 "editorMode": "code", 3296 "exemplar": false, 3297 "expr": "sum(\n kube_pod_container_resource_requests{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*distributor\", workload_type=\"deployment\"}\n)", 3298 "hide": false, 3299 "instant": false, 3300 "legendFormat": "request", 3301 "range": true, 3302 "refId": "request" 3303 }, 3304 { 3305 "datasource": { 3306 "type": "prometheus", 3307 "uid": "${datasource}" 3308 }, 3309 "editorMode": "code", 3310 "expr": "sum(\n kube_pod_container_resource_limits{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*distributor\", workload_type=\"deployment\"}\n)", 3311 "hide": false, 3312 "legendFormat": "limit", 3313 "range": true, 3314 "refId": "limit" 3315 } 3316 ], 3317 "title": "Memory Usage", 3318 "type": "timeseries" 3319 }, 3320 { 3321 "datasource": { 3322 "type": "prometheus", 3323 "uid": "${datasource}" 3324 }, 3325 "fieldConfig": { 3326 "defaults": { 3327 "color": { 3328 "mode": "palette-classic" 3329 }, 3330 "custom": { 3331 "axisBorderShow": false, 3332 "axisCenteredZero": false, 3333 "axisColorMode": "text", 3334 "axisLabel": "", 3335 "axisPlacement": "auto", 3336 "barAlignment": 0, 3337 "drawStyle": "line", 3338 "fillOpacity": 13, 3339 "gradientMode": "none", 3340 "hideFrom": { 3341 "legend": false, 3342 "tooltip": false, 3343 "viz": false 3344 }, 3345 "insertNulls": false, 3346 "lineInterpolation": "linear", 3347 "lineWidth": 1, 3348 "pointSize": 5, 3349 "scaleDistribution": { 3350 "type": "linear" 3351 }, 3352 "showPoints": "auto", 3353 "spanNulls": false, 3354 "stacking": { 3355 "group": "A", 3356 "mode": "normal" 3357 }, 3358 "thresholdsStyle": { 3359 "mode": "off" 3360 } 3361 }, 3362 "mappings": [], 3363 "thresholds": { 3364 "mode": "absolute", 3365 "steps": [ 3366 { 3367 "color": "green" 3368 }, 3369 { 3370 "color": "red", 3371 "value": 80 3372 } 3373 ] 3374 }, 3375 "unit": "Bps" 3376 }, 3377 "overrides": [] 3378 }, 3379 "gridPos": { 3380 "h": 8, 3381 "w": 12, 3382 "x": 0, 3383 "y": 12 3384 }, 3385 "id": 71, 3386 "options": { 3387 "legend": { 3388 "calcs": [], 3389 "displayMode": "list", 3390 "placement": "bottom", 3391 "showLegend": false 3392 }, 3393 "tooltip": { 3394 "mode": "multi", 3395 "sort": "none" 3396 } 3397 }, 3398 "targets": [ 3399 { 3400 "datasource": { 3401 "type": "prometheus", 3402 "uid": "${datasource}" 3403 }, 3404 "editorMode": "code", 3405 "expr": "(sum(irate(container_network_receive_bytes_total{job=~\"(.*/)?cadvisor\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*distributor.*\", workload_type=\"deployment\"}) by (pod))", 3406 "legendFormat": "__auto", 3407 "range": true, 3408 "refId": "A" 3409 } 3410 ], 3411 "title": "Receive Bandwidth", 3412 "type": "timeseries" 3413 }, 3414 { 3415 "datasource": { 3416 "type": "prometheus", 3417 "uid": "${datasource}" 3418 }, 3419 "fieldConfig": { 3420 "defaults": { 3421 "color": { 3422 "mode": "palette-classic" 3423 }, 3424 "custom": { 3425 "axisBorderShow": false, 3426 "axisCenteredZero": false, 3427 "axisColorMode": "text", 3428 "axisLabel": "", 3429 "axisPlacement": "auto", 3430 "barAlignment": 0, 3431 "drawStyle": "line", 3432 "fillOpacity": 13, 3433 "gradientMode": "none", 3434 "hideFrom": { 3435 "legend": false, 3436 "tooltip": false, 3437 "viz": false 3438 }, 3439 "insertNulls": false, 3440 "lineInterpolation": "linear", 3441 "lineWidth": 1, 3442 "pointSize": 5, 3443 "scaleDistribution": { 3444 "type": "linear" 3445 }, 3446 "showPoints": "auto", 3447 "spanNulls": false, 3448 "stacking": { 3449 "group": "A", 3450 "mode": "normal" 3451 }, 3452 "thresholdsStyle": { 3453 "mode": "off" 3454 } 3455 }, 3456 "mappings": [], 3457 "thresholds": { 3458 "mode": "absolute", 3459 "steps": [ 3460 { 3461 "color": "green" 3462 }, 3463 { 3464 "color": "red", 3465 "value": 80 3466 } 3467 ] 3468 }, 3469 "unit": "Bps" 3470 }, 3471 "overrides": [] 3472 }, 3473 "gridPos": { 3474 "h": 8, 3475 "w": 12, 3476 "x": 12, 3477 "y": 12 3478 }, 3479 "id": 72, 3480 "options": { 3481 "legend": { 3482 "calcs": [], 3483 "displayMode": "list", 3484 "placement": "bottom", 3485 "showLegend": false 3486 }, 3487 "tooltip": { 3488 "mode": "multi", 3489 "sort": "none" 3490 } 3491 }, 3492 "targets": [ 3493 { 3494 "datasource": { 3495 "type": "prometheus", 3496 "uid": "${datasource}" 3497 }, 3498 "editorMode": "code", 3499 "expr": "(sum(irate(container_network_transmit_bytes_total{job=~\"(.*/)?cadvisor\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*distributor.*\", workload_type=\"deployment\"}) by (pod))", 3500 "legendFormat": "__auto", 3501 "range": true, 3502 "refId": "A" 3503 } 3504 ], 3505 "title": "Transmit Bandwidth", 3506 "type": "timeseries" 3507 }, 3508 { 3509 "datasource": { 3510 "type": "prometheus", 3511 "uid": "${datasource}" 3512 }, 3513 "fieldConfig": { 3514 "defaults": { 3515 "color": { 3516 "mode": "palette-classic" 3517 }, 3518 "custom": { 3519 "axisBorderShow": false, 3520 "axisCenteredZero": false, 3521 "axisColorMode": "text", 3522 "axisLabel": "", 3523 "axisPlacement": "auto", 3524 "barAlignment": 0, 3525 "drawStyle": "line", 3526 "fillOpacity": 16, 3527 "gradientMode": "none", 3528 "hideFrom": { 3529 "legend": false, 3530 "tooltip": false, 3531 "viz": false 3532 }, 3533 "insertNulls": false, 3534 "lineInterpolation": "linear", 3535 "lineWidth": 1, 3536 "pointSize": 5, 3537 "scaleDistribution": { 3538 "type": "linear" 3539 }, 3540 "showPoints": "auto", 3541 "spanNulls": false, 3542 "stacking": { 3543 "group": "A", 3544 "mode": "normal" 3545 }, 3546 "thresholdsStyle": { 3547 "mode": "off" 3548 } 3549 }, 3550 "mappings": [], 3551 "thresholds": { 3552 "mode": "absolute", 3553 "steps": [ 3554 { 3555 "color": "green" 3556 }, 3557 { 3558 "color": "red", 3559 "value": 80 3560 } 3561 ] 3562 } 3563 }, 3564 "overrides": [] 3565 }, 3566 "gridPos": { 3567 "h": 8, 3568 "w": 12, 3569 "x": 0, 3570 "y": 20 3571 }, 3572 "id": 63, 3573 "options": { 3574 "legend": { 3575 "calcs": [ 3576 "lastNotNull" 3577 ], 3578 "displayMode": "table", 3579 "placement": "right", 3580 "showLegend": true, 3581 "sortBy": "Last *", 3582 "sortDesc": true 3583 }, 3584 "tooltip": { 3585 "mode": "single", 3586 "sort": "none" 3587 } 3588 }, 3589 "targets": [ 3590 { 3591 "datasource": { 3592 "type": "prometheus", 3593 "uid": "${datasource}" 3594 }, 3595 "editorMode": "code", 3596 "expr": "sum by (type) (rate(pyroscope_distributor_received_compressed_bytes_count{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", 3597 "legendFormat": "__auto", 3598 "range": true, 3599 "refId": "A" 3600 } 3601 ], 3602 "title": "Profiles /s", 3603 "type": "timeseries" 3604 }, 3605 { 3606 "datasource": { 3607 "type": "prometheus", 3608 "uid": "${datasource}" 3609 }, 3610 "fieldConfig": { 3611 "defaults": { 3612 "color": { 3613 "mode": "palette-classic" 3614 }, 3615 "custom": { 3616 "axisBorderShow": false, 3617 "axisCenteredZero": false, 3618 "axisColorMode": "text", 3619 "axisLabel": "", 3620 "axisPlacement": "auto", 3621 "barAlignment": 0, 3622 "drawStyle": "line", 3623 "fillOpacity": 16, 3624 "gradientMode": "none", 3625 "hideFrom": { 3626 "legend": false, 3627 "tooltip": false, 3628 "viz": false 3629 }, 3630 "insertNulls": false, 3631 "lineInterpolation": "linear", 3632 "lineWidth": 1, 3633 "pointSize": 5, 3634 "scaleDistribution": { 3635 "type": "linear" 3636 }, 3637 "showPoints": "auto", 3638 "spanNulls": false, 3639 "stacking": { 3640 "group": "A", 3641 "mode": "normal" 3642 }, 3643 "thresholdsStyle": { 3644 "mode": "off" 3645 } 3646 }, 3647 "mappings": [], 3648 "thresholds": { 3649 "mode": "absolute", 3650 "steps": [ 3651 { 3652 "color": "green" 3653 }, 3654 { 3655 "color": "red", 3656 "value": 80 3657 } 3658 ] 3659 } 3660 }, 3661 "overrides": [] 3662 }, 3663 "gridPos": { 3664 "h": 8, 3665 "w": 12, 3666 "x": 12, 3667 "y": 20 3668 }, 3669 "id": 62, 3670 "options": { 3671 "legend": { 3672 "calcs": [ 3673 "lastNotNull" 3674 ], 3675 "displayMode": "table", 3676 "placement": "right", 3677 "showLegend": true, 3678 "sortBy": "Last *", 3679 "sortDesc": true 3680 }, 3681 "tooltip": { 3682 "mode": "single", 3683 "sort": "none" 3684 } 3685 }, 3686 "targets": [ 3687 { 3688 "datasource": { 3689 "type": "prometheus", 3690 "uid": "${datasource}" 3691 }, 3692 "editorMode": "code", 3693 "expr": "sum by (type) (rate(pyroscope_distributor_received_samples_sum{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", 3694 "legendFormat": "__auto", 3695 "range": true, 3696 "refId": "A" 3697 } 3698 ], 3699 "title": "Stacktrace Sample /s", 3700 "type": "timeseries" 3701 }, 3702 { 3703 "datasource": { 3704 "type": "prometheus", 3705 "uid": "${datasource}" 3706 }, 3707 "fieldConfig": { 3708 "defaults": { 3709 "color": { 3710 "mode": "palette-classic" 3711 }, 3712 "custom": { 3713 "axisBorderShow": false, 3714 "axisCenteredZero": false, 3715 "axisColorMode": "text", 3716 "axisLabel": "", 3717 "axisPlacement": "auto", 3718 "barAlignment": 0, 3719 "drawStyle": "line", 3720 "fillOpacity": 16, 3721 "gradientMode": "none", 3722 "hideFrom": { 3723 "legend": false, 3724 "tooltip": false, 3725 "viz": false 3726 }, 3727 "insertNulls": false, 3728 "lineInterpolation": "linear", 3729 "lineWidth": 1, 3730 "pointSize": 5, 3731 "scaleDistribution": { 3732 "type": "linear" 3733 }, 3734 "showPoints": "auto", 3735 "spanNulls": false, 3736 "stacking": { 3737 "group": "A", 3738 "mode": "normal" 3739 }, 3740 "thresholdsStyle": { 3741 "mode": "off" 3742 } 3743 }, 3744 "mappings": [], 3745 "thresholds": { 3746 "mode": "absolute", 3747 "steps": [ 3748 { 3749 "color": "green" 3750 }, 3751 { 3752 "color": "red", 3753 "value": 80 3754 } 3755 ] 3756 }, 3757 "unit": "decbytes" 3758 }, 3759 "overrides": [] 3760 }, 3761 "gridPos": { 3762 "h": 8, 3763 "w": 8, 3764 "x": 0, 3765 "y": 28 3766 }, 3767 "id": 65, 3768 "options": { 3769 "legend": { 3770 "calcs": [ 3771 "lastNotNull" 3772 ], 3773 "displayMode": "table", 3774 "placement": "right", 3775 "showLegend": false, 3776 "sortBy": "Last *", 3777 "sortDesc": true 3778 }, 3779 "tooltip": { 3780 "mode": "multi", 3781 "sort": "none" 3782 } 3783 }, 3784 "targets": [ 3785 { 3786 "datasource": { 3787 "type": "prometheus", 3788 "uid": "${datasource}" 3789 }, 3790 "editorMode": "code", 3791 "expr": "sum by (type) (rate(pyroscope_distributor_received_decompressed_bytes_sum{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", 3792 "legendFormat": "__auto", 3793 "range": true, 3794 "refId": "A" 3795 } 3796 ], 3797 "title": "Throughput Decompressed", 3798 "type": "timeseries" 3799 }, 3800 { 3801 "datasource": { 3802 "type": "prometheus", 3803 "uid": "${datasource}" 3804 }, 3805 "fieldConfig": { 3806 "defaults": { 3807 "color": { 3808 "mode": "palette-classic" 3809 }, 3810 "custom": { 3811 "axisBorderShow": false, 3812 "axisCenteredZero": false, 3813 "axisColorMode": "text", 3814 "axisLabel": "", 3815 "axisPlacement": "auto", 3816 "barAlignment": 0, 3817 "drawStyle": "line", 3818 "fillOpacity": 16, 3819 "gradientMode": "none", 3820 "hideFrom": { 3821 "legend": false, 3822 "tooltip": false, 3823 "viz": false 3824 }, 3825 "insertNulls": false, 3826 "lineInterpolation": "linear", 3827 "lineWidth": 1, 3828 "pointSize": 5, 3829 "scaleDistribution": { 3830 "type": "linear" 3831 }, 3832 "showPoints": "auto", 3833 "spanNulls": false, 3834 "stacking": { 3835 "group": "A", 3836 "mode": "normal" 3837 }, 3838 "thresholdsStyle": { 3839 "mode": "off" 3840 } 3841 }, 3842 "mappings": [], 3843 "thresholds": { 3844 "mode": "absolute", 3845 "steps": [ 3846 { 3847 "color": "green" 3848 }, 3849 { 3850 "color": "red", 3851 "value": 80 3852 } 3853 ] 3854 }, 3855 "unit": "decbytes" 3856 }, 3857 "overrides": [] 3858 }, 3859 "gridPos": { 3860 "h": 8, 3861 "w": 8, 3862 "x": 8, 3863 "y": 28 3864 }, 3865 "id": 66, 3866 "options": { 3867 "legend": { 3868 "calcs": [ 3869 "lastNotNull" 3870 ], 3871 "displayMode": "table", 3872 "placement": "right", 3873 "showLegend": false, 3874 "sortBy": "Last *", 3875 "sortDesc": true 3876 }, 3877 "tooltip": { 3878 "mode": "multi", 3879 "sort": "none" 3880 } 3881 }, 3882 "targets": [ 3883 { 3884 "datasource": { 3885 "type": "prometheus", 3886 "uid": "${datasource}" 3887 }, 3888 "editorMode": "code", 3889 "expr": "sum by (type) (rate(pyroscope_distributor_received_samples_bytes_sum{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", 3890 "legendFormat": "__auto", 3891 "range": true, 3892 "refId": "A" 3893 } 3894 ], 3895 "title": "Throughput Samples Decompressed", 3896 "type": "timeseries" 3897 }, 3898 { 3899 "datasource": { 3900 "type": "prometheus", 3901 "uid": "${datasource}" 3902 }, 3903 "fieldConfig": { 3904 "defaults": { 3905 "color": { 3906 "mode": "palette-classic" 3907 }, 3908 "custom": { 3909 "axisBorderShow": false, 3910 "axisCenteredZero": false, 3911 "axisColorMode": "text", 3912 "axisLabel": "", 3913 "axisPlacement": "auto", 3914 "barAlignment": 0, 3915 "drawStyle": "line", 3916 "fillOpacity": 16, 3917 "gradientMode": "none", 3918 "hideFrom": { 3919 "legend": false, 3920 "tooltip": false, 3921 "viz": false 3922 }, 3923 "insertNulls": false, 3924 "lineInterpolation": "linear", 3925 "lineWidth": 1, 3926 "pointSize": 5, 3927 "scaleDistribution": { 3928 "type": "linear" 3929 }, 3930 "showPoints": "auto", 3931 "spanNulls": false, 3932 "stacking": { 3933 "group": "A", 3934 "mode": "normal" 3935 }, 3936 "thresholdsStyle": { 3937 "mode": "off" 3938 } 3939 }, 3940 "mappings": [], 3941 "thresholds": { 3942 "mode": "absolute", 3943 "steps": [ 3944 { 3945 "color": "green" 3946 }, 3947 { 3948 "color": "red", 3949 "value": 80 3950 } 3951 ] 3952 }, 3953 "unit": "decbytes" 3954 }, 3955 "overrides": [] 3956 }, 3957 "gridPos": { 3958 "h": 8, 3959 "w": 8, 3960 "x": 16, 3961 "y": 28 3962 }, 3963 "id": 67, 3964 "options": { 3965 "legend": { 3966 "calcs": [ 3967 "lastNotNull" 3968 ], 3969 "displayMode": "table", 3970 "placement": "right", 3971 "showLegend": false, 3972 "sortBy": "Last *", 3973 "sortDesc": true 3974 }, 3975 "tooltip": { 3976 "mode": "multi", 3977 "sort": "none" 3978 } 3979 }, 3980 "targets": [ 3981 { 3982 "datasource": { 3983 "type": "prometheus", 3984 "uid": "${datasource}" 3985 }, 3986 "editorMode": "code", 3987 "expr": "sum by (type) (rate(pyroscope_distributor_received_symbols_bytes_sum{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", 3988 "legendFormat": "__auto", 3989 "range": true, 3990 "refId": "A" 3991 } 3992 ], 3993 "title": "Throughput Symbols Decompressed", 3994 "type": "timeseries" 3995 } 3996 ], 3997 "title": "Distributors", 3998 "type": "row" 3999 }, 4000 { 4001 "collapsed": true, 4002 "gridPos": { 4003 "h": 1, 4004 "w": 24, 4005 "x": 0, 4006 "y": 43 4007 }, 4008 "id": 91, 4009 "panels": [ 4010 { 4011 "datasource": { 4012 "type": "prometheus", 4013 "uid": "${datasource}" 4014 }, 4015 "fieldConfig": { 4016 "defaults": { 4017 "color": { 4018 "mode": "palette-classic" 4019 }, 4020 "custom": { 4021 "axisBorderShow": false, 4022 "axisCenteredZero": false, 4023 "axisColorMode": "text", 4024 "axisLabel": "", 4025 "axisPlacement": "auto", 4026 "barAlignment": 0, 4027 "drawStyle": "line", 4028 "fillOpacity": 0, 4029 "gradientMode": "none", 4030 "hideFrom": { 4031 "legend": false, 4032 "tooltip": false, 4033 "viz": false 4034 }, 4035 "insertNulls": false, 4036 "lineInterpolation": "linear", 4037 "lineWidth": 1, 4038 "pointSize": 5, 4039 "scaleDistribution": { 4040 "type": "linear" 4041 }, 4042 "showPoints": "auto", 4043 "spanNulls": false, 4044 "stacking": { 4045 "group": "A", 4046 "mode": "none" 4047 }, 4048 "thresholdsStyle": { 4049 "mode": "off" 4050 } 4051 }, 4052 "mappings": [], 4053 "thresholds": { 4054 "mode": "absolute", 4055 "steps": [ 4056 { 4057 "color": "green" 4058 }, 4059 { 4060 "color": "red", 4061 "value": 80 4062 } 4063 ] 4064 }, 4065 "unit": "s" 4066 }, 4067 "overrides": [] 4068 }, 4069 "gridPos": { 4070 "h": 5, 4071 "w": 12, 4072 "x": 0, 4073 "y": 5 4074 }, 4075 "id": 100, 4076 "options": { 4077 "legend": { 4078 "calcs": [], 4079 "displayMode": "table", 4080 "placement": "right", 4081 "showLegend": true 4082 }, 4083 "tooltip": { 4084 "mode": "single", 4085 "sort": "none" 4086 } 4087 }, 4088 "targets": [ 4089 { 4090 "datasource": { 4091 "type": "prometheus", 4092 "uid": "${datasource}" 4093 }, 4094 "editorMode": "code", 4095 "expr": "histogram_quantile(0.99, sum by (route) ( rate(pyroscope_request_duration_seconds{job=~\"$namespace/query-frontend\", cluster=~\"$cluster\",route=~\".*render.*|.*querier.*\"}[$__rate_interval])))", 4096 "legendFormat": ".99 {{route}}", 4097 "range": true, 4098 "refId": "A" 4099 }, 4100 { 4101 "datasource": { 4102 "type": "prometheus", 4103 "uid": "${datasource}" 4104 }, 4105 "editorMode": "code", 4106 "expr": "histogram_quantile(0.75, sum by (route) ( rate(pyroscope_request_duration_seconds{job=~\"$namespace/query-frontend\", cluster=~\"$cluster\",route=~\".*render.*|.*querier.*\"}[$__rate_interval])))", 4107 "hide": true, 4108 "legendFormat": ".75 {{route}}", 4109 "range": true, 4110 "refId": "B" 4111 }, 4112 { 4113 "datasource": { 4114 "type": "prometheus", 4115 "uid": "${datasource}" 4116 }, 4117 "editorMode": "code", 4118 "expr": "histogram_quantile(0.5, sum by (route) ( rate(pyroscope_request_duration_seconds{job=~\"$namespace/query-frontend\", cluster=~\"$cluster\",route=~\".*render.*|.*querier.*\"}[$__rate_interval])))", 4119 "hide": true, 4120 "legendFormat": ".5 {{route}}", 4121 "range": true, 4122 "refId": "C" 4123 } 4124 ], 4125 "title": "API Latencies", 4126 "type": "timeseries" 4127 }, 4128 { 4129 "datasource": { 4130 "type": "prometheus", 4131 "uid": "$datasource" 4132 }, 4133 "fieldConfig": { 4134 "defaults": { 4135 "color": { 4136 "mode": "palette-classic" 4137 }, 4138 "custom": { 4139 "axisBorderShow": false, 4140 "axisCenteredZero": false, 4141 "axisColorMode": "text", 4142 "axisLabel": "", 4143 "axisPlacement": "auto", 4144 "barAlignment": 0, 4145 "drawStyle": "line", 4146 "fillOpacity": 40, 4147 "gradientMode": "none", 4148 "hideFrom": { 4149 "legend": false, 4150 "tooltip": false, 4151 "viz": false 4152 }, 4153 "insertNulls": false, 4154 "lineInterpolation": "linear", 4155 "lineWidth": 1, 4156 "pointSize": 5, 4157 "scaleDistribution": { 4158 "type": "linear" 4159 }, 4160 "showPoints": "never", 4161 "spanNulls": false, 4162 "stacking": { 4163 "group": "A", 4164 "mode": "normal" 4165 }, 4166 "thresholdsStyle": { 4167 "mode": "off" 4168 } 4169 }, 4170 "mappings": [], 4171 "thresholds": { 4172 "mode": "absolute", 4173 "steps": [ 4174 { 4175 "color": "green" 4176 }, 4177 { 4178 "color": "red", 4179 "value": 80 4180 } 4181 ] 4182 }, 4183 "unit": "reqps" 4184 }, 4185 "overrides": [ 4186 { 4187 "matcher": { 4188 "id": "byRegexp", 4189 "options": "/2../" 4190 }, 4191 "properties": [ 4192 { 4193 "id": "color", 4194 "value": { 4195 "fixedColor": "green", 4196 "mode": "shades" 4197 } 4198 } 4199 ] 4200 }, 4201 { 4202 "matcher": { 4203 "id": "byRegexp", 4204 "options": "/5../" 4205 }, 4206 "properties": [ 4207 { 4208 "id": "color", 4209 "value": { 4210 "fixedColor": "red", 4211 "mode": "shades" 4212 } 4213 } 4214 ] 4215 }, 4216 { 4217 "matcher": { 4218 "id": "byRegexp", 4219 "options": "/4../" 4220 }, 4221 "properties": [ 4222 { 4223 "id": "color", 4224 "value": { 4225 "fixedColor": "yellow", 4226 "mode": "shades" 4227 } 4228 } 4229 ] 4230 } 4231 ] 4232 }, 4233 "gridPos": { 4234 "h": 5, 4235 "w": 12, 4236 "x": 12, 4237 "y": 5 4238 }, 4239 "id": 101, 4240 "options": { 4241 "dataLinks": [], 4242 "legend": { 4243 "calcs": [], 4244 "displayMode": "list", 4245 "placement": "bottom", 4246 "showLegend": true 4247 }, 4248 "tooltip": { 4249 "mode": "multi", 4250 "sort": "none" 4251 } 4252 }, 4253 "targets": [ 4254 { 4255 "datasource": { 4256 "uid": "$datasource" 4257 }, 4258 "editorMode": "code", 4259 "expr": "sum by (status,route) (\nlabel_replace(\n label_replace(\n rate(pyroscope_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"$namespace/query-frontend\", route=~\".*render.*|.*querier.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)", 4260 "legendFormat": "{{status}} {{route}}", 4261 "range": true, 4262 "refId": "A" 4263 } 4264 ], 4265 "title": "Request Rates", 4266 "type": "timeseries" 4267 }, 4268 { 4269 "datasource": { 4270 "type": "prometheus", 4271 "uid": "${datasource}" 4272 }, 4273 "fieldConfig": { 4274 "defaults": { 4275 "color": { 4276 "mode": "palette-classic" 4277 }, 4278 "custom": { 4279 "axisBorderShow": false, 4280 "axisCenteredZero": false, 4281 "axisColorMode": "text", 4282 "axisLabel": "", 4283 "axisPlacement": "auto", 4284 "barAlignment": 0, 4285 "drawStyle": "line", 4286 "fillOpacity": 21, 4287 "gradientMode": "none", 4288 "hideFrom": { 4289 "legend": false, 4290 "tooltip": false, 4291 "viz": false 4292 }, 4293 "insertNulls": false, 4294 "lineInterpolation": "linear", 4295 "lineStyle": { 4296 "fill": "solid" 4297 }, 4298 "lineWidth": 1, 4299 "pointSize": 5, 4300 "scaleDistribution": { 4301 "type": "linear" 4302 }, 4303 "showPoints": "auto", 4304 "spanNulls": false, 4305 "stacking": { 4306 "group": "A", 4307 "mode": "normal" 4308 }, 4309 "thresholdsStyle": { 4310 "mode": "dashed" 4311 } 4312 }, 4313 "mappings": [], 4314 "thresholds": { 4315 "mode": "absolute", 4316 "steps": [ 4317 { 4318 "color": "green" 4319 }, 4320 { 4321 "color": "yellow", 4322 "value": 80 4323 }, 4324 { 4325 "color": "red", 4326 "value": 90 4327 } 4328 ] 4329 } 4330 }, 4331 "overrides": [ 4332 { 4333 "matcher": { 4334 "id": "byName", 4335 "options": "request" 4336 }, 4337 "properties": [ 4338 { 4339 "id": "custom.stacking", 4340 "value": { 4341 "group": "A", 4342 "mode": "none" 4343 } 4344 }, 4345 { 4346 "id": "color", 4347 "value": { 4348 "fixedColor": "yellow", 4349 "mode": "fixed" 4350 } 4351 }, 4352 { 4353 "id": "custom.fillOpacity", 4354 "value": 0 4355 }, 4356 { 4357 "id": "custom.lineStyle", 4358 "value": { 4359 "dash": [ 4360 10, 4361 10 4362 ], 4363 "fill": "dash" 4364 } 4365 }, 4366 { 4367 "id": "custom.lineWidth", 4368 "value": 2 4369 } 4370 ] 4371 }, 4372 { 4373 "matcher": { 4374 "id": "byName", 4375 "options": "limit" 4376 }, 4377 "properties": [ 4378 { 4379 "id": "custom.fillOpacity", 4380 "value": 0 4381 }, 4382 { 4383 "id": "custom.stacking", 4384 "value": { 4385 "group": "A", 4386 "mode": "none" 4387 } 4388 }, 4389 { 4390 "id": "custom.lineStyle", 4391 "value": { 4392 "dash": [ 4393 10, 4394 10 4395 ], 4396 "fill": "dash" 4397 } 4398 }, 4399 { 4400 "id": "custom.lineWidth", 4401 "value": 2 4402 }, 4403 { 4404 "id": "color", 4405 "value": { 4406 "fixedColor": "red", 4407 "mode": "fixed" 4408 } 4409 } 4410 ] 4411 } 4412 ] 4413 }, 4414 "gridPos": { 4415 "h": 8, 4416 "w": 12, 4417 "x": 0, 4418 "y": 10 4419 }, 4420 "id": 92, 4421 "options": { 4422 "legend": { 4423 "calcs": [], 4424 "displayMode": "list", 4425 "placement": "bottom", 4426 "showLegend": true 4427 }, 4428 "tooltip": { 4429 "mode": "multi", 4430 "sort": "desc" 4431 } 4432 }, 4433 "targets": [ 4434 { 4435 "datasource": { 4436 "type": "prometheus", 4437 "uid": "${datasource}" 4438 }, 4439 "editorMode": "code", 4440 "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*query-frontend\", workload_type=\"deployment\"}\n) by (pod)", 4441 "legendFormat": "__auto", 4442 "range": true, 4443 "refId": "by pod" 4444 }, 4445 { 4446 "datasource": { 4447 "type": "prometheus", 4448 "uid": "${datasource}" 4449 }, 4450 "editorMode": "code", 4451 "exemplar": false, 4452 "expr": "sum(\n kube_pod_container_resource_requests{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*query-frontend\", workload_type=\"deployment\"}\n)", 4453 "hide": false, 4454 "instant": false, 4455 "legendFormat": "request", 4456 "range": true, 4457 "refId": "request" 4458 }, 4459 { 4460 "datasource": { 4461 "type": "prometheus", 4462 "uid": "${datasource}" 4463 }, 4464 "editorMode": "code", 4465 "expr": "sum(\n kube_pod_container_resource_limits{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*query-frontend\", workload_type=\"deployment\"}\n)", 4466 "hide": false, 4467 "legendFormat": "limit", 4468 "range": true, 4469 "refId": "limit" 4470 } 4471 ], 4472 "title": "CPU Usage", 4473 "type": "timeseries" 4474 }, 4475 { 4476 "datasource": { 4477 "type": "prometheus", 4478 "uid": "${datasource}" 4479 }, 4480 "fieldConfig": { 4481 "defaults": { 4482 "color": { 4483 "mode": "palette-classic" 4484 }, 4485 "custom": { 4486 "axisBorderShow": false, 4487 "axisCenteredZero": false, 4488 "axisColorMode": "text", 4489 "axisLabel": "", 4490 "axisPlacement": "auto", 4491 "barAlignment": 0, 4492 "drawStyle": "line", 4493 "fillOpacity": 21, 4494 "gradientMode": "none", 4495 "hideFrom": { 4496 "legend": false, 4497 "tooltip": false, 4498 "viz": false 4499 }, 4500 "insertNulls": false, 4501 "lineInterpolation": "linear", 4502 "lineStyle": { 4503 "fill": "solid" 4504 }, 4505 "lineWidth": 1, 4506 "pointSize": 5, 4507 "scaleDistribution": { 4508 "type": "linear" 4509 }, 4510 "showPoints": "auto", 4511 "spanNulls": false, 4512 "stacking": { 4513 "group": "A", 4514 "mode": "normal" 4515 }, 4516 "thresholdsStyle": { 4517 "mode": "dashed" 4518 } 4519 }, 4520 "mappings": [], 4521 "thresholds": { 4522 "mode": "absolute", 4523 "steps": [ 4524 { 4525 "color": "green" 4526 }, 4527 { 4528 "color": "yellow", 4529 "value": 80 4530 }, 4531 { 4532 "color": "red", 4533 "value": 90 4534 } 4535 ] 4536 }, 4537 "unit": "bytes" 4538 }, 4539 "overrides": [ 4540 { 4541 "matcher": { 4542 "id": "byName", 4543 "options": "request" 4544 }, 4545 "properties": [ 4546 { 4547 "id": "custom.stacking", 4548 "value": { 4549 "group": "A", 4550 "mode": "none" 4551 } 4552 }, 4553 { 4554 "id": "color", 4555 "value": { 4556 "fixedColor": "yellow", 4557 "mode": "fixed" 4558 } 4559 }, 4560 { 4561 "id": "custom.fillOpacity", 4562 "value": 0 4563 }, 4564 { 4565 "id": "custom.lineStyle", 4566 "value": { 4567 "dash": [ 4568 10, 4569 10 4570 ], 4571 "fill": "dash" 4572 } 4573 }, 4574 { 4575 "id": "custom.lineWidth", 4576 "value": 2 4577 } 4578 ] 4579 }, 4580 { 4581 "matcher": { 4582 "id": "byName", 4583 "options": "limit" 4584 }, 4585 "properties": [ 4586 { 4587 "id": "color", 4588 "value": { 4589 "fixedColor": "red", 4590 "mode": "fixed" 4591 } 4592 }, 4593 { 4594 "id": "custom.stacking", 4595 "value": { 4596 "group": "A", 4597 "mode": "none" 4598 } 4599 }, 4600 { 4601 "id": "custom.fillOpacity", 4602 "value": 0 4603 }, 4604 { 4605 "id": "custom.lineStyle", 4606 "value": { 4607 "dash": [ 4608 10, 4609 10 4610 ], 4611 "fill": "dash" 4612 } 4613 }, 4614 { 4615 "id": "custom.lineWidth", 4616 "value": 2 4617 } 4618 ] 4619 } 4620 ] 4621 }, 4622 "gridPos": { 4623 "h": 8, 4624 "w": 12, 4625 "x": 12, 4626 "y": 10 4627 }, 4628 "id": 88, 4629 "options": { 4630 "legend": { 4631 "calcs": [], 4632 "displayMode": "list", 4633 "placement": "bottom", 4634 "showLegend": true 4635 }, 4636 "tooltip": { 4637 "mode": "multi", 4638 "sort": "desc" 4639 } 4640 }, 4641 "targets": [ 4642 { 4643 "datasource": { 4644 "type": "prometheus", 4645 "uid": "${datasource}" 4646 }, 4647 "editorMode": "code", 4648 "expr": "sum(\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*query-frontend\", workload_type=\"deployment\"}\n) by (pod)\n", 4649 "legendFormat": "__auto", 4650 "range": true, 4651 "refId": "by pod" 4652 }, 4653 { 4654 "datasource": { 4655 "type": "prometheus", 4656 "uid": "${datasource}" 4657 }, 4658 "editorMode": "code", 4659 "exemplar": false, 4660 "expr": "sum(\n kube_pod_container_resource_requests{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*query-frontend\", workload_type=\"deployment\"}\n)", 4661 "hide": false, 4662 "instant": false, 4663 "legendFormat": "request", 4664 "range": true, 4665 "refId": "request" 4666 }, 4667 { 4668 "datasource": { 4669 "type": "prometheus", 4670 "uid": "${datasource}" 4671 }, 4672 "editorMode": "code", 4673 "expr": "sum(\n kube_pod_container_resource_limits{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*query-frontend\", workload_type=\"deployment\"}\n)", 4674 "hide": false, 4675 "legendFormat": "limit", 4676 "range": true, 4677 "refId": "limit" 4678 } 4679 ], 4680 "title": "Memory Usage", 4681 "type": "timeseries" 4682 }, 4683 { 4684 "datasource": { 4685 "type": "prometheus", 4686 "uid": "${datasource}" 4687 }, 4688 "fieldConfig": { 4689 "defaults": { 4690 "color": { 4691 "mode": "palette-classic" 4692 }, 4693 "custom": { 4694 "axisBorderShow": false, 4695 "axisCenteredZero": false, 4696 "axisColorMode": "text", 4697 "axisLabel": "", 4698 "axisPlacement": "auto", 4699 "barAlignment": 0, 4700 "drawStyle": "line", 4701 "fillOpacity": 13, 4702 "gradientMode": "none", 4703 "hideFrom": { 4704 "legend": false, 4705 "tooltip": false, 4706 "viz": false 4707 }, 4708 "insertNulls": false, 4709 "lineInterpolation": "linear", 4710 "lineWidth": 1, 4711 "pointSize": 5, 4712 "scaleDistribution": { 4713 "type": "linear" 4714 }, 4715 "showPoints": "auto", 4716 "spanNulls": false, 4717 "stacking": { 4718 "group": "A", 4719 "mode": "normal" 4720 }, 4721 "thresholdsStyle": { 4722 "mode": "off" 4723 } 4724 }, 4725 "mappings": [], 4726 "thresholds": { 4727 "mode": "absolute", 4728 "steps": [ 4729 { 4730 "color": "green" 4731 }, 4732 { 4733 "color": "red", 4734 "value": 80 4735 } 4736 ] 4737 }, 4738 "unit": "Bps" 4739 }, 4740 "overrides": [] 4741 }, 4742 "gridPos": { 4743 "h": 8, 4744 "w": 12, 4745 "x": 0, 4746 "y": 18 4747 }, 4748 "id": 94, 4749 "options": { 4750 "legend": { 4751 "calcs": [], 4752 "displayMode": "list", 4753 "placement": "bottom", 4754 "showLegend": false 4755 }, 4756 "tooltip": { 4757 "mode": "multi", 4758 "sort": "none" 4759 } 4760 }, 4761 "targets": [ 4762 { 4763 "datasource": { 4764 "type": "prometheus", 4765 "uid": "${datasource}" 4766 }, 4767 "editorMode": "code", 4768 "expr": "(sum(irate(container_network_receive_bytes_total{job=~\"(.*/)?cadvisor\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*query-frontend.*\", workload_type=\"deployment\"}) by (pod))", 4769 "legendFormat": "__auto", 4770 "range": true, 4771 "refId": "A" 4772 } 4773 ], 4774 "title": "Receive Bandwidth", 4775 "type": "timeseries" 4776 }, 4777 { 4778 "datasource": { 4779 "type": "prometheus", 4780 "uid": "${datasource}" 4781 }, 4782 "fieldConfig": { 4783 "defaults": { 4784 "color": { 4785 "mode": "palette-classic" 4786 }, 4787 "custom": { 4788 "axisBorderShow": false, 4789 "axisCenteredZero": false, 4790 "axisColorMode": "text", 4791 "axisLabel": "", 4792 "axisPlacement": "auto", 4793 "barAlignment": 0, 4794 "drawStyle": "line", 4795 "fillOpacity": 13, 4796 "gradientMode": "none", 4797 "hideFrom": { 4798 "legend": false, 4799 "tooltip": false, 4800 "viz": false 4801 }, 4802 "insertNulls": false, 4803 "lineInterpolation": "linear", 4804 "lineWidth": 1, 4805 "pointSize": 5, 4806 "scaleDistribution": { 4807 "type": "linear" 4808 }, 4809 "showPoints": "auto", 4810 "spanNulls": false, 4811 "stacking": { 4812 "group": "A", 4813 "mode": "normal" 4814 }, 4815 "thresholdsStyle": { 4816 "mode": "off" 4817 } 4818 }, 4819 "mappings": [], 4820 "thresholds": { 4821 "mode": "absolute", 4822 "steps": [ 4823 { 4824 "color": "green" 4825 }, 4826 { 4827 "color": "red", 4828 "value": 80 4829 } 4830 ] 4831 }, 4832 "unit": "Bps" 4833 }, 4834 "overrides": [] 4835 }, 4836 "gridPos": { 4837 "h": 8, 4838 "w": 12, 4839 "x": 12, 4840 "y": 18 4841 }, 4842 "id": 90, 4843 "options": { 4844 "legend": { 4845 "calcs": [], 4846 "displayMode": "list", 4847 "placement": "bottom", 4848 "showLegend": false 4849 }, 4850 "tooltip": { 4851 "mode": "multi", 4852 "sort": "none" 4853 } 4854 }, 4855 "targets": [ 4856 { 4857 "datasource": { 4858 "type": "prometheus", 4859 "uid": "${datasource}" 4860 }, 4861 "editorMode": "code", 4862 "expr": "(sum(irate(container_network_transmit_bytes_total{job=~\"(.*/)?cadvisor\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*query-frontend.*\", workload_type=\"deployment\"}) by (pod))", 4863 "legendFormat": "__auto", 4864 "range": true, 4865 "refId": "A" 4866 } 4867 ], 4868 "title": "Transmit Bandwidth", 4869 "type": "timeseries" 4870 }, 4871 { 4872 "datasource": { 4873 "type": "loki", 4874 "uid": "${loki_datasource}" 4875 }, 4876 "fieldConfig": { 4877 "defaults": {}, 4878 "overrides": [] 4879 }, 4880 "gridPos": { 4881 "h": 18, 4882 "w": 24, 4883 "x": 0, 4884 "y": 26 4885 }, 4886 "id": 60, 4887 "options": { 4888 "dedupStrategy": "none", 4889 "enableLogDetails": true, 4890 "prettifyLogMessage": false, 4891 "showCommonLabels": false, 4892 "showLabels": false, 4893 "showTime": false, 4894 "sortOrder": "Descending", 4895 "wrapLogMessage": true 4896 }, 4897 "targets": [ 4898 { 4899 "datasource": { 4900 "type": "loki", 4901 "uid": "${loki_datasource}" 4902 }, 4903 "editorMode": "code", 4904 "expr": "{namespace=~\"$namespace\",cluster=~\"$cluster\",container=\"query-frontend\"} |= \"http.go\" |~ \"QuerierService|pyroscope|render\" | logfmt | line_format \"tenant={{.orgID}} traceID={{.traceID}} {{.msg}}\"", 4905 "queryType": "range", 4906 "refId": "A" 4907 } 4908 ], 4909 "title": "Queries", 4910 "type": "logs" 4911 } 4912 ], 4913 "title": "Query Frontend", 4914 "type": "row" 4915 }, 4916 { 4917 "collapsed": true, 4918 "gridPos": { 4919 "h": 1, 4920 "w": 24, 4921 "x": 0, 4922 "y": 44 4923 }, 4924 "id": 56, 4925 "panels": [ 4926 { 4927 "datasource": { 4928 "type": "prometheus", 4929 "uid": "${datasource}" 4930 }, 4931 "fieldConfig": { 4932 "defaults": { 4933 "color": { 4934 "mode": "palette-classic" 4935 }, 4936 "custom": { 4937 "axisBorderShow": false, 4938 "axisCenteredZero": false, 4939 "axisColorMode": "text", 4940 "axisLabel": "", 4941 "axisPlacement": "auto", 4942 "barAlignment": 0, 4943 "drawStyle": "line", 4944 "fillOpacity": 21, 4945 "gradientMode": "none", 4946 "hideFrom": { 4947 "legend": false, 4948 "tooltip": false, 4949 "viz": false 4950 }, 4951 "insertNulls": false, 4952 "lineInterpolation": "linear", 4953 "lineStyle": { 4954 "fill": "solid" 4955 }, 4956 "lineWidth": 1, 4957 "pointSize": 5, 4958 "scaleDistribution": { 4959 "type": "linear" 4960 }, 4961 "showPoints": "auto", 4962 "spanNulls": false, 4963 "stacking": { 4964 "group": "A", 4965 "mode": "normal" 4966 }, 4967 "thresholdsStyle": { 4968 "mode": "dashed" 4969 } 4970 }, 4971 "mappings": [], 4972 "thresholds": { 4973 "mode": "absolute", 4974 "steps": [ 4975 { 4976 "color": "green" 4977 }, 4978 { 4979 "color": "yellow", 4980 "value": 80 4981 }, 4982 { 4983 "color": "red", 4984 "value": 90 4985 } 4986 ] 4987 } 4988 }, 4989 "overrides": [ 4990 { 4991 "matcher": { 4992 "id": "byName", 4993 "options": "request" 4994 }, 4995 "properties": [ 4996 { 4997 "id": "custom.stacking", 4998 "value": { 4999 "group": "A", 5000 "mode": "none" 5001 } 5002 }, 5003 { 5004 "id": "color", 5005 "value": { 5006 "fixedColor": "yellow", 5007 "mode": "fixed" 5008 } 5009 }, 5010 { 5011 "id": "custom.fillOpacity", 5012 "value": 0 5013 }, 5014 { 5015 "id": "custom.lineStyle", 5016 "value": { 5017 "dash": [ 5018 10, 5019 10 5020 ], 5021 "fill": "dash" 5022 } 5023 }, 5024 { 5025 "id": "custom.lineWidth", 5026 "value": 2 5027 } 5028 ] 5029 }, 5030 { 5031 "matcher": { 5032 "id": "byName", 5033 "options": "limit" 5034 }, 5035 "properties": [ 5036 { 5037 "id": "custom.fillOpacity", 5038 "value": 0 5039 }, 5040 { 5041 "id": "custom.stacking", 5042 "value": { 5043 "group": "A", 5044 "mode": "none" 5045 } 5046 }, 5047 { 5048 "id": "custom.lineStyle", 5049 "value": { 5050 "dash": [ 5051 10, 5052 10 5053 ], 5054 "fill": "dash" 5055 } 5056 }, 5057 { 5058 "id": "custom.lineWidth", 5059 "value": 2 5060 }, 5061 { 5062 "id": "color", 5063 "value": { 5064 "fixedColor": "red", 5065 "mode": "fixed" 5066 } 5067 } 5068 ] 5069 } 5070 ] 5071 }, 5072 "gridPos": { 5073 "h": 8, 5074 "w": 12, 5075 "x": 0, 5076 "y": 6 5077 }, 5078 "id": 87, 5079 "options": { 5080 "legend": { 5081 "calcs": [], 5082 "displayMode": "list", 5083 "placement": "bottom", 5084 "showLegend": true 5085 }, 5086 "tooltip": { 5087 "mode": "multi", 5088 "sort": "desc" 5089 } 5090 }, 5091 "targets": [ 5092 { 5093 "datasource": { 5094 "type": "prometheus", 5095 "uid": "${datasource}" 5096 }, 5097 "editorMode": "code", 5098 "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*querier\", workload_type=\"deployment\"}\n) by (pod)", 5099 "legendFormat": "__auto", 5100 "range": true, 5101 "refId": "by pod" 5102 }, 5103 { 5104 "datasource": { 5105 "type": "prometheus", 5106 "uid": "${datasource}" 5107 }, 5108 "editorMode": "code", 5109 "exemplar": false, 5110 "expr": "sum(\n kube_pod_container_resource_requests{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*querier\", workload_type=\"deployment\"}\n)", 5111 "hide": false, 5112 "instant": false, 5113 "legendFormat": "request", 5114 "range": true, 5115 "refId": "request" 5116 }, 5117 { 5118 "datasource": { 5119 "type": "prometheus", 5120 "uid": "${datasource}" 5121 }, 5122 "editorMode": "code", 5123 "expr": "sum(\n kube_pod_container_resource_limits{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*querier\", workload_type=\"deployment\"}\n)", 5124 "hide": false, 5125 "legendFormat": "limit", 5126 "range": true, 5127 "refId": "limit" 5128 } 5129 ], 5130 "title": "CPU Usage", 5131 "type": "timeseries" 5132 }, 5133 { 5134 "datasource": { 5135 "type": "prometheus", 5136 "uid": "${datasource}" 5137 }, 5138 "fieldConfig": { 5139 "defaults": { 5140 "color": { 5141 "mode": "palette-classic" 5142 }, 5143 "custom": { 5144 "axisBorderShow": false, 5145 "axisCenteredZero": false, 5146 "axisColorMode": "text", 5147 "axisLabel": "", 5148 "axisPlacement": "auto", 5149 "barAlignment": 0, 5150 "drawStyle": "line", 5151 "fillOpacity": 21, 5152 "gradientMode": "none", 5153 "hideFrom": { 5154 "legend": false, 5155 "tooltip": false, 5156 "viz": false 5157 }, 5158 "insertNulls": false, 5159 "lineInterpolation": "linear", 5160 "lineStyle": { 5161 "fill": "solid" 5162 }, 5163 "lineWidth": 1, 5164 "pointSize": 5, 5165 "scaleDistribution": { 5166 "type": "linear" 5167 }, 5168 "showPoints": "auto", 5169 "spanNulls": false, 5170 "stacking": { 5171 "group": "A", 5172 "mode": "normal" 5173 }, 5174 "thresholdsStyle": { 5175 "mode": "dashed" 5176 } 5177 }, 5178 "mappings": [], 5179 "thresholds": { 5180 "mode": "absolute", 5181 "steps": [ 5182 { 5183 "color": "green" 5184 }, 5185 { 5186 "color": "yellow", 5187 "value": 80 5188 }, 5189 { 5190 "color": "red", 5191 "value": 90 5192 } 5193 ] 5194 }, 5195 "unit": "bytes" 5196 }, 5197 "overrides": [ 5198 { 5199 "matcher": { 5200 "id": "byName", 5201 "options": "request" 5202 }, 5203 "properties": [ 5204 { 5205 "id": "custom.stacking", 5206 "value": { 5207 "group": "A", 5208 "mode": "none" 5209 } 5210 }, 5211 { 5212 "id": "color", 5213 "value": { 5214 "fixedColor": "yellow", 5215 "mode": "fixed" 5216 } 5217 }, 5218 { 5219 "id": "custom.fillOpacity", 5220 "value": 0 5221 }, 5222 { 5223 "id": "custom.lineStyle", 5224 "value": { 5225 "dash": [ 5226 10, 5227 10 5228 ], 5229 "fill": "dash" 5230 } 5231 }, 5232 { 5233 "id": "custom.lineWidth", 5234 "value": 2 5235 } 5236 ] 5237 }, 5238 { 5239 "matcher": { 5240 "id": "byName", 5241 "options": "limit" 5242 }, 5243 "properties": [ 5244 { 5245 "id": "color", 5246 "value": { 5247 "fixedColor": "red", 5248 "mode": "fixed" 5249 } 5250 }, 5251 { 5252 "id": "custom.stacking", 5253 "value": { 5254 "group": "A", 5255 "mode": "none" 5256 } 5257 }, 5258 { 5259 "id": "custom.fillOpacity", 5260 "value": 0 5261 }, 5262 { 5263 "id": "custom.lineStyle", 5264 "value": { 5265 "dash": [ 5266 10, 5267 10 5268 ], 5269 "fill": "dash" 5270 } 5271 }, 5272 { 5273 "id": "custom.lineWidth", 5274 "value": 2 5275 } 5276 ] 5277 } 5278 ] 5279 }, 5280 "gridPos": { 5281 "h": 8, 5282 "w": 12, 5283 "x": 12, 5284 "y": 6 5285 }, 5286 "id": 58, 5287 "options": { 5288 "legend": { 5289 "calcs": [], 5290 "displayMode": "list", 5291 "placement": "bottom", 5292 "showLegend": true 5293 }, 5294 "tooltip": { 5295 "mode": "multi", 5296 "sort": "desc" 5297 } 5298 }, 5299 "targets": [ 5300 { 5301 "datasource": { 5302 "type": "prometheus", 5303 "uid": "${datasource}" 5304 }, 5305 "editorMode": "code", 5306 "expr": "sum(\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*querier\", workload_type=\"deployment\"}\n) by (pod)\n", 5307 "legendFormat": "__auto", 5308 "range": true, 5309 "refId": "by pod" 5310 }, 5311 { 5312 "datasource": { 5313 "type": "prometheus", 5314 "uid": "${datasource}" 5315 }, 5316 "editorMode": "code", 5317 "exemplar": false, 5318 "expr": "sum(\n kube_pod_container_resource_requests{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*querier\", workload_type=\"deployment\"}\n)", 5319 "hide": false, 5320 "instant": false, 5321 "legendFormat": "request", 5322 "range": true, 5323 "refId": "request" 5324 }, 5325 { 5326 "datasource": { 5327 "type": "prometheus", 5328 "uid": "${datasource}" 5329 }, 5330 "editorMode": "code", 5331 "expr": "sum(\n kube_pod_container_resource_limits{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*querier\", workload_type=\"deployment\"}\n)", 5332 "hide": false, 5333 "legendFormat": "limit", 5334 "range": true, 5335 "refId": "limit" 5336 } 5337 ], 5338 "title": "Memory Usage", 5339 "type": "timeseries" 5340 }, 5341 { 5342 "datasource": { 5343 "type": "prometheus", 5344 "uid": "${datasource}" 5345 }, 5346 "fieldConfig": { 5347 "defaults": { 5348 "color": { 5349 "mode": "palette-classic" 5350 }, 5351 "custom": { 5352 "axisBorderShow": false, 5353 "axisCenteredZero": false, 5354 "axisColorMode": "text", 5355 "axisLabel": "", 5356 "axisPlacement": "auto", 5357 "barAlignment": 0, 5358 "drawStyle": "line", 5359 "fillOpacity": 13, 5360 "gradientMode": "none", 5361 "hideFrom": { 5362 "legend": false, 5363 "tooltip": false, 5364 "viz": false 5365 }, 5366 "insertNulls": false, 5367 "lineInterpolation": "linear", 5368 "lineWidth": 1, 5369 "pointSize": 5, 5370 "scaleDistribution": { 5371 "type": "linear" 5372 }, 5373 "showPoints": "auto", 5374 "spanNulls": false, 5375 "stacking": { 5376 "group": "A", 5377 "mode": "normal" 5378 }, 5379 "thresholdsStyle": { 5380 "mode": "off" 5381 } 5382 }, 5383 "mappings": [], 5384 "thresholds": { 5385 "mode": "absolute", 5386 "steps": [ 5387 { 5388 "color": "green" 5389 }, 5390 { 5391 "color": "red", 5392 "value": 80 5393 } 5394 ] 5395 }, 5396 "unit": "Bps" 5397 }, 5398 "overrides": [] 5399 }, 5400 "gridPos": { 5401 "h": 8, 5402 "w": 12, 5403 "x": 0, 5404 "y": 14 5405 }, 5406 "id": 75, 5407 "options": { 5408 "legend": { 5409 "calcs": [], 5410 "displayMode": "list", 5411 "placement": "bottom", 5412 "showLegend": false 5413 }, 5414 "tooltip": { 5415 "mode": "multi", 5416 "sort": "none" 5417 } 5418 }, 5419 "targets": [ 5420 { 5421 "datasource": { 5422 "type": "prometheus", 5423 "uid": "${datasource}" 5424 }, 5425 "editorMode": "code", 5426 "expr": "(sum(irate(container_network_receive_bytes_total{job=~\"(.*/)?cadvisor\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*querier.*\", workload_type=\"deployment\"}) by (pod))", 5427 "legendFormat": "__auto", 5428 "range": true, 5429 "refId": "A" 5430 } 5431 ], 5432 "title": "Receive Bandwidth", 5433 "type": "timeseries" 5434 }, 5435 { 5436 "datasource": { 5437 "type": "prometheus", 5438 "uid": "${datasource}" 5439 }, 5440 "fieldConfig": { 5441 "defaults": { 5442 "color": { 5443 "mode": "palette-classic" 5444 }, 5445 "custom": { 5446 "axisBorderShow": false, 5447 "axisCenteredZero": false, 5448 "axisColorMode": "text", 5449 "axisLabel": "", 5450 "axisPlacement": "auto", 5451 "barAlignment": 0, 5452 "drawStyle": "line", 5453 "fillOpacity": 13, 5454 "gradientMode": "none", 5455 "hideFrom": { 5456 "legend": false, 5457 "tooltip": false, 5458 "viz": false 5459 }, 5460 "insertNulls": false, 5461 "lineInterpolation": "linear", 5462 "lineWidth": 1, 5463 "pointSize": 5, 5464 "scaleDistribution": { 5465 "type": "linear" 5466 }, 5467 "showPoints": "auto", 5468 "spanNulls": false, 5469 "stacking": { 5470 "group": "A", 5471 "mode": "normal" 5472 }, 5473 "thresholdsStyle": { 5474 "mode": "off" 5475 } 5476 }, 5477 "mappings": [], 5478 "thresholds": { 5479 "mode": "absolute", 5480 "steps": [ 5481 { 5482 "color": "green" 5483 }, 5484 { 5485 "color": "red", 5486 "value": 80 5487 } 5488 ] 5489 }, 5490 "unit": "Bps" 5491 }, 5492 "overrides": [] 5493 }, 5494 "gridPos": { 5495 "h": 8, 5496 "w": 12, 5497 "x": 12, 5498 "y": 14 5499 }, 5500 "id": 76, 5501 "options": { 5502 "legend": { 5503 "calcs": [], 5504 "displayMode": "list", 5505 "placement": "bottom", 5506 "showLegend": false 5507 }, 5508 "tooltip": { 5509 "mode": "multi", 5510 "sort": "none" 5511 } 5512 }, 5513 "targets": [ 5514 { 5515 "datasource": { 5516 "type": "prometheus", 5517 "uid": "${datasource}" 5518 }, 5519 "editorMode": "code", 5520 "expr": "(sum(irate(container_network_transmit_bytes_total{job=~\"(.*/)?cadvisor\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*querier.*\", workload_type=\"deployment\"}) by (pod))", 5521 "legendFormat": "__auto", 5522 "range": true, 5523 "refId": "A" 5524 } 5525 ], 5526 "title": "Transmit Bandwidth", 5527 "type": "timeseries" 5528 } 5529 ], 5530 "title": "Queriers", 5531 "type": "row" 5532 }, 5533 { 5534 "collapsed": true, 5535 "gridPos": { 5536 "h": 1, 5537 "w": 24, 5538 "x": 0, 5539 "y": 45 5540 }, 5541 "id": 102, 5542 "panels": [ 5543 { 5544 "datasource": { 5545 "type": "prometheus", 5546 "uid": "${datasource}" 5547 }, 5548 "fieldConfig": { 5549 "defaults": { 5550 "color": { 5551 "mode": "palette-classic" 5552 }, 5553 "custom": { 5554 "axisBorderShow": false, 5555 "axisCenteredZero": false, 5556 "axisColorMode": "text", 5557 "axisLabel": "", 5558 "axisPlacement": "auto", 5559 "barAlignment": 0, 5560 "drawStyle": "line", 5561 "fillOpacity": 21, 5562 "gradientMode": "none", 5563 "hideFrom": { 5564 "legend": false, 5565 "tooltip": false, 5566 "viz": false 5567 }, 5568 "insertNulls": false, 5569 "lineInterpolation": "linear", 5570 "lineStyle": { 5571 "fill": "solid" 5572 }, 5573 "lineWidth": 1, 5574 "pointSize": 5, 5575 "scaleDistribution": { 5576 "type": "linear" 5577 }, 5578 "showPoints": "auto", 5579 "spanNulls": false, 5580 "stacking": { 5581 "group": "A", 5582 "mode": "normal" 5583 }, 5584 "thresholdsStyle": { 5585 "mode": "dashed" 5586 } 5587 }, 5588 "mappings": [], 5589 "thresholds": { 5590 "mode": "absolute", 5591 "steps": [ 5592 { 5593 "color": "green" 5594 }, 5595 { 5596 "color": "yellow", 5597 "value": 80 5598 }, 5599 { 5600 "color": "red", 5601 "value": 90 5602 } 5603 ] 5604 } 5605 }, 5606 "overrides": [ 5607 { 5608 "matcher": { 5609 "id": "byName", 5610 "options": "request" 5611 }, 5612 "properties": [ 5613 { 5614 "id": "custom.stacking", 5615 "value": { 5616 "group": "A", 5617 "mode": "none" 5618 } 5619 }, 5620 { 5621 "id": "color", 5622 "value": { 5623 "fixedColor": "yellow", 5624 "mode": "fixed" 5625 } 5626 }, 5627 { 5628 "id": "custom.fillOpacity", 5629 "value": 0 5630 }, 5631 { 5632 "id": "custom.lineStyle", 5633 "value": { 5634 "dash": [ 5635 10, 5636 10 5637 ], 5638 "fill": "dash" 5639 } 5640 }, 5641 { 5642 "id": "custom.lineWidth", 5643 "value": 2 5644 } 5645 ] 5646 }, 5647 { 5648 "matcher": { 5649 "id": "byName", 5650 "options": "limit" 5651 }, 5652 "properties": [ 5653 { 5654 "id": "custom.fillOpacity", 5655 "value": 0 5656 }, 5657 { 5658 "id": "custom.stacking", 5659 "value": { 5660 "group": "A", 5661 "mode": "none" 5662 } 5663 }, 5664 { 5665 "id": "custom.lineStyle", 5666 "value": { 5667 "dash": [ 5668 10, 5669 10 5670 ], 5671 "fill": "dash" 5672 } 5673 }, 5674 { 5675 "id": "custom.lineWidth", 5676 "value": 2 5677 }, 5678 { 5679 "id": "color", 5680 "value": { 5681 "fixedColor": "red", 5682 "mode": "fixed" 5683 } 5684 } 5685 ] 5686 } 5687 ] 5688 }, 5689 "gridPos": { 5690 "h": 8, 5691 "w": 12, 5692 "x": 0, 5693 "y": 7 5694 }, 5695 "id": 103, 5696 "options": { 5697 "legend": { 5698 "calcs": [], 5699 "displayMode": "list", 5700 "placement": "bottom", 5701 "showLegend": true 5702 }, 5703 "tooltip": { 5704 "mode": "multi", 5705 "sort": "desc" 5706 } 5707 }, 5708 "targets": [ 5709 { 5710 "datasource": { 5711 "type": "prometheus", 5712 "uid": "${datasource}" 5713 }, 5714 "editorMode": "code", 5715 "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*query-scheduler\", workload_type=\"deployment\"}\n) by (pod)", 5716 "legendFormat": "__auto", 5717 "range": true, 5718 "refId": "by pod" 5719 }, 5720 { 5721 "datasource": { 5722 "type": "prometheus", 5723 "uid": "${datasource}" 5724 }, 5725 "editorMode": "code", 5726 "exemplar": false, 5727 "expr": "sum(\n kube_pod_container_resource_requests{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*query-scheduler\", workload_type=\"deployment\"}\n)", 5728 "hide": false, 5729 "instant": false, 5730 "legendFormat": "request", 5731 "range": true, 5732 "refId": "request" 5733 }, 5734 { 5735 "datasource": { 5736 "type": "prometheus", 5737 "uid": "${datasource}" 5738 }, 5739 "editorMode": "code", 5740 "expr": "sum(\n kube_pod_container_resource_limits{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*query-scheduler\", workload_type=\"deployment\"}\n)", 5741 "hide": false, 5742 "legendFormat": "limit", 5743 "range": true, 5744 "refId": "limit" 5745 } 5746 ], 5747 "title": "CPU Usage", 5748 "type": "timeseries" 5749 }, 5750 { 5751 "datasource": { 5752 "type": "prometheus", 5753 "uid": "${datasource}" 5754 }, 5755 "fieldConfig": { 5756 "defaults": { 5757 "color": { 5758 "mode": "palette-classic" 5759 }, 5760 "custom": { 5761 "axisBorderShow": false, 5762 "axisCenteredZero": false, 5763 "axisColorMode": "text", 5764 "axisLabel": "", 5765 "axisPlacement": "auto", 5766 "barAlignment": 0, 5767 "drawStyle": "line", 5768 "fillOpacity": 21, 5769 "gradientMode": "none", 5770 "hideFrom": { 5771 "legend": false, 5772 "tooltip": false, 5773 "viz": false 5774 }, 5775 "insertNulls": false, 5776 "lineInterpolation": "linear", 5777 "lineStyle": { 5778 "fill": "solid" 5779 }, 5780 "lineWidth": 1, 5781 "pointSize": 5, 5782 "scaleDistribution": { 5783 "type": "linear" 5784 }, 5785 "showPoints": "auto", 5786 "spanNulls": false, 5787 "stacking": { 5788 "group": "A", 5789 "mode": "normal" 5790 }, 5791 "thresholdsStyle": { 5792 "mode": "dashed" 5793 } 5794 }, 5795 "mappings": [], 5796 "thresholds": { 5797 "mode": "absolute", 5798 "steps": [ 5799 { 5800 "color": "green" 5801 }, 5802 { 5803 "color": "yellow", 5804 "value": 80 5805 }, 5806 { 5807 "color": "red", 5808 "value": 90 5809 } 5810 ] 5811 }, 5812 "unit": "bytes" 5813 }, 5814 "overrides": [ 5815 { 5816 "matcher": { 5817 "id": "byName", 5818 "options": "request" 5819 }, 5820 "properties": [ 5821 { 5822 "id": "custom.stacking", 5823 "value": { 5824 "group": "A", 5825 "mode": "none" 5826 } 5827 }, 5828 { 5829 "id": "color", 5830 "value": { 5831 "fixedColor": "yellow", 5832 "mode": "fixed" 5833 } 5834 }, 5835 { 5836 "id": "custom.fillOpacity", 5837 "value": 0 5838 }, 5839 { 5840 "id": "custom.lineStyle", 5841 "value": { 5842 "dash": [ 5843 10, 5844 10 5845 ], 5846 "fill": "dash" 5847 } 5848 }, 5849 { 5850 "id": "custom.lineWidth", 5851 "value": 2 5852 } 5853 ] 5854 }, 5855 { 5856 "matcher": { 5857 "id": "byName", 5858 "options": "limit" 5859 }, 5860 "properties": [ 5861 { 5862 "id": "color", 5863 "value": { 5864 "fixedColor": "red", 5865 "mode": "fixed" 5866 } 5867 }, 5868 { 5869 "id": "custom.stacking", 5870 "value": { 5871 "group": "A", 5872 "mode": "none" 5873 } 5874 }, 5875 { 5876 "id": "custom.fillOpacity", 5877 "value": 0 5878 }, 5879 { 5880 "id": "custom.lineStyle", 5881 "value": { 5882 "dash": [ 5883 10, 5884 10 5885 ], 5886 "fill": "dash" 5887 } 5888 }, 5889 { 5890 "id": "custom.lineWidth", 5891 "value": 2 5892 } 5893 ] 5894 } 5895 ] 5896 }, 5897 "gridPos": { 5898 "h": 8, 5899 "w": 12, 5900 "x": 12, 5901 "y": 7 5902 }, 5903 "id": 93, 5904 "options": { 5905 "legend": { 5906 "calcs": [], 5907 "displayMode": "list", 5908 "placement": "bottom", 5909 "showLegend": true 5910 }, 5911 "tooltip": { 5912 "mode": "multi", 5913 "sort": "desc" 5914 } 5915 }, 5916 "targets": [ 5917 { 5918 "datasource": { 5919 "type": "prometheus", 5920 "uid": "${datasource}" 5921 }, 5922 "editorMode": "code", 5923 "expr": "sum(\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*query-scheduler\", workload_type=\"deployment\"}\n) by (pod)\n", 5924 "legendFormat": "__auto", 5925 "range": true, 5926 "refId": "by pod" 5927 }, 5928 { 5929 "datasource": { 5930 "type": "prometheus", 5931 "uid": "${datasource}" 5932 }, 5933 "editorMode": "code", 5934 "exemplar": false, 5935 "expr": "sum(\n kube_pod_container_resource_requests{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*query-scheduler\", workload_type=\"deployment\"}\n)", 5936 "hide": false, 5937 "instant": false, 5938 "legendFormat": "request", 5939 "range": true, 5940 "refId": "request" 5941 }, 5942 { 5943 "datasource": { 5944 "type": "prometheus", 5945 "uid": "${datasource}" 5946 }, 5947 "editorMode": "code", 5948 "expr": "sum(\n kube_pod_container_resource_limits{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*query-scheduler\", workload_type=\"deployment\"}\n)", 5949 "hide": false, 5950 "legendFormat": "limit", 5951 "range": true, 5952 "refId": "limit" 5953 } 5954 ], 5955 "title": "Memory Usage", 5956 "type": "timeseries" 5957 } 5958 ], 5959 "title": "Query Scheduler", 5960 "type": "row" 5961 }, 5962 { 5963 "collapsed": true, 5964 "gridPos": { 5965 "h": 1, 5966 "w": 24, 5967 "x": 0, 5968 "y": 46 5969 }, 5970 "id": 86, 5971 "panels": [ 5972 { 5973 "datasource": { 5974 "type": "prometheus", 5975 "uid": "${datasource}" 5976 }, 5977 "fieldConfig": { 5978 "defaults": { 5979 "color": { 5980 "mode": "palette-classic" 5981 }, 5982 "custom": { 5983 "axisBorderShow": false, 5984 "axisCenteredZero": false, 5985 "axisColorMode": "text", 5986 "axisLabel": "", 5987 "axisPlacement": "auto", 5988 "barAlignment": 0, 5989 "drawStyle": "line", 5990 "fillOpacity": 0, 5991 "gradientMode": "none", 5992 "hideFrom": { 5993 "legend": false, 5994 "tooltip": false, 5995 "viz": false 5996 }, 5997 "insertNulls": false, 5998 "lineInterpolation": "linear", 5999 "lineWidth": 1, 6000 "pointSize": 5, 6001 "scaleDistribution": { 6002 "type": "linear" 6003 }, 6004 "showPoints": "auto", 6005 "spanNulls": false, 6006 "stacking": { 6007 "group": "A", 6008 "mode": "none" 6009 }, 6010 "thresholdsStyle": { 6011 "mode": "off" 6012 } 6013 }, 6014 "mappings": [], 6015 "thresholds": { 6016 "mode": "absolute", 6017 "steps": [ 6018 { 6019 "color": "green" 6020 }, 6021 { 6022 "color": "red", 6023 "value": 80 6024 } 6025 ] 6026 }, 6027 "unit": "s" 6028 }, 6029 "overrides": [] 6030 }, 6031 "gridPos": { 6032 "h": 5, 6033 "w": 12, 6034 "x": 0, 6035 "y": 8 6036 }, 6037 "id": 98, 6038 "options": { 6039 "legend": { 6040 "calcs": [], 6041 "displayMode": "table", 6042 "placement": "right", 6043 "showLegend": true 6044 }, 6045 "tooltip": { 6046 "mode": "single", 6047 "sort": "none" 6048 } 6049 }, 6050 "targets": [ 6051 { 6052 "datasource": { 6053 "type": "prometheus", 6054 "uid": "${datasource}" 6055 }, 6056 "editorMode": "code", 6057 "expr": "histogram_quantile(0.99, sum by (route) ( rate(pyroscope_request_duration_seconds{job=~\"$namespace/store-gateway\", cluster=~\"$cluster\",route=~\".*store.*\"}[$__rate_interval])))", 6058 "legendFormat": ".99 {{route}}", 6059 "range": true, 6060 "refId": "A" 6061 }, 6062 { 6063 "datasource": { 6064 "type": "prometheus", 6065 "uid": "${datasource}" 6066 }, 6067 "editorMode": "code", 6068 "expr": "histogram_quantile(0.75, sum by (route) ( rate(pyroscope_request_duration_seconds{job=~\"$namespace/store-gateway\", cluster=~\"$cluster\",route=~\".*store.*\"}[$__rate_interval])))", 6069 "hide": true, 6070 "legendFormat": ".75 {{route}}", 6071 "range": true, 6072 "refId": "B" 6073 }, 6074 { 6075 "datasource": { 6076 "type": "prometheus", 6077 "uid": "${datasource}" 6078 }, 6079 "editorMode": "code", 6080 "expr": "histogram_quantile(0.5, sum by (route) ( rate(pyroscope_request_duration_seconds{job=~\"$namespace/store-gateway\", cluster=~\"$cluster\",route=~\".*store.*\"}[$__rate_interval])))", 6081 "hide": true, 6082 "legendFormat": ".5 {{route}}", 6083 "range": true, 6084 "refId": "C" 6085 } 6086 ], 6087 "title": "API Latencies", 6088 "type": "timeseries" 6089 }, 6090 { 6091 "datasource": { 6092 "type": "prometheus", 6093 "uid": "$datasource" 6094 }, 6095 "fieldConfig": { 6096 "defaults": { 6097 "color": { 6098 "mode": "palette-classic" 6099 }, 6100 "custom": { 6101 "axisBorderShow": false, 6102 "axisCenteredZero": false, 6103 "axisColorMode": "text", 6104 "axisLabel": "", 6105 "axisPlacement": "auto", 6106 "barAlignment": 0, 6107 "drawStyle": "line", 6108 "fillOpacity": 40, 6109 "gradientMode": "none", 6110 "hideFrom": { 6111 "legend": false, 6112 "tooltip": false, 6113 "viz": false 6114 }, 6115 "insertNulls": false, 6116 "lineInterpolation": "linear", 6117 "lineWidth": 1, 6118 "pointSize": 5, 6119 "scaleDistribution": { 6120 "type": "linear" 6121 }, 6122 "showPoints": "never", 6123 "spanNulls": false, 6124 "stacking": { 6125 "group": "A", 6126 "mode": "normal" 6127 }, 6128 "thresholdsStyle": { 6129 "mode": "off" 6130 } 6131 }, 6132 "mappings": [], 6133 "thresholds": { 6134 "mode": "absolute", 6135 "steps": [ 6136 { 6137 "color": "green" 6138 }, 6139 { 6140 "color": "red", 6141 "value": 80 6142 } 6143 ] 6144 }, 6145 "unit": "reqps" 6146 }, 6147 "overrides": [ 6148 { 6149 "matcher": { 6150 "id": "byRegexp", 6151 "options": "/2../" 6152 }, 6153 "properties": [ 6154 { 6155 "id": "color", 6156 "value": { 6157 "fixedColor": "green", 6158 "mode": "shades" 6159 } 6160 } 6161 ] 6162 }, 6163 { 6164 "matcher": { 6165 "id": "byRegexp", 6166 "options": "/5../" 6167 }, 6168 "properties": [ 6169 { 6170 "id": "color", 6171 "value": { 6172 "fixedColor": "red", 6173 "mode": "shades" 6174 } 6175 } 6176 ] 6177 }, 6178 { 6179 "matcher": { 6180 "id": "byRegexp", 6181 "options": "/4../" 6182 }, 6183 "properties": [ 6184 { 6185 "id": "color", 6186 "value": { 6187 "fixedColor": "yellow", 6188 "mode": "shades" 6189 } 6190 } 6191 ] 6192 } 6193 ] 6194 }, 6195 "gridPos": { 6196 "h": 5, 6197 "w": 12, 6198 "x": 12, 6199 "y": 8 6200 }, 6201 "id": 99, 6202 "options": { 6203 "dataLinks": [], 6204 "legend": { 6205 "calcs": [], 6206 "displayMode": "list", 6207 "placement": "bottom", 6208 "showLegend": true 6209 }, 6210 "tooltip": { 6211 "mode": "multi", 6212 "sort": "none" 6213 } 6214 }, 6215 "targets": [ 6216 { 6217 "datasource": { 6218 "uid": "$datasource" 6219 }, 6220 "editorMode": "code", 6221 "expr": "sum by (status,route) (\nlabel_replace(\n label_replace(\n rate(pyroscope_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"$namespace/store-gateway\", route=~\".*store.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)", 6222 "legendFormat": "{{status}} {{route}}", 6223 "range": true, 6224 "refId": "A" 6225 } 6226 ], 6227 "title": "Request Rates", 6228 "type": "timeseries" 6229 }, 6230 { 6231 "datasource": { 6232 "type": "prometheus", 6233 "uid": "${datasource}" 6234 }, 6235 "fieldConfig": { 6236 "defaults": { 6237 "color": { 6238 "mode": "palette-classic" 6239 }, 6240 "custom": { 6241 "axisBorderShow": false, 6242 "axisCenteredZero": false, 6243 "axisColorMode": "text", 6244 "axisLabel": "", 6245 "axisPlacement": "auto", 6246 "barAlignment": 0, 6247 "drawStyle": "line", 6248 "fillOpacity": 21, 6249 "gradientMode": "none", 6250 "hideFrom": { 6251 "legend": false, 6252 "tooltip": false, 6253 "viz": false 6254 }, 6255 "insertNulls": false, 6256 "lineInterpolation": "linear", 6257 "lineStyle": { 6258 "fill": "solid" 6259 }, 6260 "lineWidth": 1, 6261 "pointSize": 5, 6262 "scaleDistribution": { 6263 "type": "linear" 6264 }, 6265 "showPoints": "auto", 6266 "spanNulls": false, 6267 "stacking": { 6268 "group": "A", 6269 "mode": "normal" 6270 }, 6271 "thresholdsStyle": { 6272 "mode": "dashed" 6273 } 6274 }, 6275 "mappings": [], 6276 "thresholds": { 6277 "mode": "absolute", 6278 "steps": [ 6279 { 6280 "color": "green" 6281 }, 6282 { 6283 "color": "yellow", 6284 "value": 80 6285 }, 6286 { 6287 "color": "red", 6288 "value": 90 6289 } 6290 ] 6291 } 6292 }, 6293 "overrides": [ 6294 { 6295 "matcher": { 6296 "id": "byName", 6297 "options": "request" 6298 }, 6299 "properties": [ 6300 { 6301 "id": "custom.stacking", 6302 "value": { 6303 "group": "A", 6304 "mode": "none" 6305 } 6306 }, 6307 { 6308 "id": "color", 6309 "value": { 6310 "fixedColor": "yellow", 6311 "mode": "fixed" 6312 } 6313 }, 6314 { 6315 "id": "custom.fillOpacity", 6316 "value": 0 6317 }, 6318 { 6319 "id": "custom.lineStyle", 6320 "value": { 6321 "dash": [ 6322 10, 6323 10 6324 ], 6325 "fill": "dash" 6326 } 6327 }, 6328 { 6329 "id": "custom.lineWidth", 6330 "value": 2 6331 } 6332 ] 6333 }, 6334 { 6335 "matcher": { 6336 "id": "byName", 6337 "options": "limit" 6338 }, 6339 "properties": [ 6340 { 6341 "id": "custom.fillOpacity", 6342 "value": 0 6343 }, 6344 { 6345 "id": "custom.stacking", 6346 "value": { 6347 "group": "A", 6348 "mode": "none" 6349 } 6350 }, 6351 { 6352 "id": "custom.lineStyle", 6353 "value": { 6354 "dash": [ 6355 10, 6356 10 6357 ], 6358 "fill": "dash" 6359 } 6360 }, 6361 { 6362 "id": "custom.lineWidth", 6363 "value": 2 6364 }, 6365 { 6366 "id": "color", 6367 "value": { 6368 "fixedColor": "red", 6369 "mode": "fixed" 6370 } 6371 } 6372 ] 6373 } 6374 ] 6375 }, 6376 "gridPos": { 6377 "h": 8, 6378 "w": 12, 6379 "x": 0, 6380 "y": 13 6381 }, 6382 "id": 57, 6383 "options": { 6384 "legend": { 6385 "calcs": [], 6386 "displayMode": "list", 6387 "placement": "bottom", 6388 "showLegend": true 6389 }, 6390 "tooltip": { 6391 "mode": "multi", 6392 "sort": "desc" 6393 } 6394 }, 6395 "targets": [ 6396 { 6397 "datasource": { 6398 "type": "prometheus", 6399 "uid": "${datasource}" 6400 }, 6401 "editorMode": "code", 6402 "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*store-gateway\", workload_type=\"statefulset\"}\n) by (pod)", 6403 "legendFormat": "__auto", 6404 "range": true, 6405 "refId": "by pod" 6406 }, 6407 { 6408 "datasource": { 6409 "type": "prometheus", 6410 "uid": "${datasource}" 6411 }, 6412 "editorMode": "code", 6413 "exemplar": false, 6414 "expr": "sum(\n kube_pod_container_resource_requests{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*store-gateway\", workload_type=\"statefulset\"}\n)", 6415 "hide": false, 6416 "instant": false, 6417 "legendFormat": "request", 6418 "range": true, 6419 "refId": "request" 6420 }, 6421 { 6422 "datasource": { 6423 "type": "prometheus", 6424 "uid": "${datasource}" 6425 }, 6426 "editorMode": "code", 6427 "expr": "sum(\n kube_pod_container_resource_limits{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*store-gateway\", workload_type=\"statefulset\"}\n)", 6428 "hide": false, 6429 "legendFormat": "limit", 6430 "range": true, 6431 "refId": "limit" 6432 } 6433 ], 6434 "title": "CPU Usage", 6435 "type": "timeseries" 6436 }, 6437 { 6438 "datasource": { 6439 "type": "prometheus", 6440 "uid": "${datasource}" 6441 }, 6442 "fieldConfig": { 6443 "defaults": { 6444 "color": { 6445 "mode": "palette-classic" 6446 }, 6447 "custom": { 6448 "axisBorderShow": false, 6449 "axisCenteredZero": false, 6450 "axisColorMode": "text", 6451 "axisLabel": "", 6452 "axisPlacement": "auto", 6453 "barAlignment": 0, 6454 "drawStyle": "line", 6455 "fillOpacity": 21, 6456 "gradientMode": "none", 6457 "hideFrom": { 6458 "legend": false, 6459 "tooltip": false, 6460 "viz": false 6461 }, 6462 "insertNulls": false, 6463 "lineInterpolation": "linear", 6464 "lineStyle": { 6465 "fill": "solid" 6466 }, 6467 "lineWidth": 1, 6468 "pointSize": 5, 6469 "scaleDistribution": { 6470 "type": "linear" 6471 }, 6472 "showPoints": "auto", 6473 "spanNulls": false, 6474 "stacking": { 6475 "group": "A", 6476 "mode": "normal" 6477 }, 6478 "thresholdsStyle": { 6479 "mode": "dashed" 6480 } 6481 }, 6482 "mappings": [], 6483 "thresholds": { 6484 "mode": "absolute", 6485 "steps": [ 6486 { 6487 "color": "green" 6488 }, 6489 { 6490 "color": "yellow", 6491 "value": 80 6492 }, 6493 { 6494 "color": "red", 6495 "value": 90 6496 } 6497 ] 6498 }, 6499 "unit": "bytes" 6500 }, 6501 "overrides": [ 6502 { 6503 "matcher": { 6504 "id": "byName", 6505 "options": "request" 6506 }, 6507 "properties": [ 6508 { 6509 "id": "custom.stacking", 6510 "value": { 6511 "group": "A", 6512 "mode": "none" 6513 } 6514 }, 6515 { 6516 "id": "color", 6517 "value": { 6518 "fixedColor": "yellow", 6519 "mode": "fixed" 6520 } 6521 }, 6522 { 6523 "id": "custom.fillOpacity", 6524 "value": 0 6525 }, 6526 { 6527 "id": "custom.lineStyle", 6528 "value": { 6529 "dash": [ 6530 10, 6531 10 6532 ], 6533 "fill": "dash" 6534 } 6535 }, 6536 { 6537 "id": "custom.lineWidth", 6538 "value": 2 6539 } 6540 ] 6541 }, 6542 { 6543 "matcher": { 6544 "id": "byName", 6545 "options": "limit" 6546 }, 6547 "properties": [ 6548 { 6549 "id": "color", 6550 "value": { 6551 "fixedColor": "red", 6552 "mode": "fixed" 6553 } 6554 }, 6555 { 6556 "id": "custom.stacking", 6557 "value": { 6558 "group": "A", 6559 "mode": "none" 6560 } 6561 }, 6562 { 6563 "id": "custom.fillOpacity", 6564 "value": 0 6565 }, 6566 { 6567 "id": "custom.lineStyle", 6568 "value": { 6569 "dash": [ 6570 10, 6571 10 6572 ], 6573 "fill": "dash" 6574 } 6575 }, 6576 { 6577 "id": "custom.lineWidth", 6578 "value": 2 6579 } 6580 ] 6581 } 6582 ] 6583 }, 6584 "gridPos": { 6585 "h": 8, 6586 "w": 12, 6587 "x": 12, 6588 "y": 13 6589 }, 6590 "id": 104, 6591 "options": { 6592 "legend": { 6593 "calcs": [], 6594 "displayMode": "list", 6595 "placement": "bottom", 6596 "showLegend": true 6597 }, 6598 "tooltip": { 6599 "mode": "multi", 6600 "sort": "desc" 6601 } 6602 }, 6603 "targets": [ 6604 { 6605 "datasource": { 6606 "type": "prometheus", 6607 "uid": "${datasource}" 6608 }, 6609 "editorMode": "code", 6610 "expr": "sum(\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*store-gateway\", workload_type=\"statefulset\"}\n) by (pod)\n", 6611 "legendFormat": "__auto", 6612 "range": true, 6613 "refId": "by pod" 6614 }, 6615 { 6616 "datasource": { 6617 "type": "prometheus", 6618 "uid": "${datasource}" 6619 }, 6620 "editorMode": "code", 6621 "exemplar": false, 6622 "expr": "sum(\n kube_pod_container_resource_requests{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*store-gateway\", workload_type=\"statefulset\"}\n)", 6623 "hide": false, 6624 "instant": false, 6625 "legendFormat": "request", 6626 "range": true, 6627 "refId": "request" 6628 }, 6629 { 6630 "datasource": { 6631 "type": "prometheus", 6632 "uid": "${datasource}" 6633 }, 6634 "editorMode": "code", 6635 "expr": "sum(\n kube_pod_container_resource_limits{job=~\"(.*/)?kube-state-metrics\", cluster=~\"$cluster\", namespace=~\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*store-gateway\", workload_type=\"statefulset\"}\n)", 6636 "hide": false, 6637 "legendFormat": "limit", 6638 "range": true, 6639 "refId": "limit" 6640 } 6641 ], 6642 "title": "Memory Usage", 6643 "type": "timeseries" 6644 }, 6645 { 6646 "datasource": { 6647 "type": "prometheus", 6648 "uid": "${datasource}" 6649 }, 6650 "fieldConfig": { 6651 "defaults": { 6652 "color": { 6653 "mode": "palette-classic" 6654 }, 6655 "custom": { 6656 "axisBorderShow": false, 6657 "axisCenteredZero": false, 6658 "axisColorMode": "text", 6659 "axisLabel": "", 6660 "axisPlacement": "auto", 6661 "barAlignment": 0, 6662 "drawStyle": "line", 6663 "fillOpacity": 13, 6664 "gradientMode": "none", 6665 "hideFrom": { 6666 "legend": false, 6667 "tooltip": false, 6668 "viz": false 6669 }, 6670 "insertNulls": false, 6671 "lineInterpolation": "linear", 6672 "lineWidth": 1, 6673 "pointSize": 5, 6674 "scaleDistribution": { 6675 "type": "linear" 6676 }, 6677 "showPoints": "auto", 6678 "spanNulls": false, 6679 "stacking": { 6680 "group": "A", 6681 "mode": "normal" 6682 }, 6683 "thresholdsStyle": { 6684 "mode": "off" 6685 } 6686 }, 6687 "mappings": [], 6688 "thresholds": { 6689 "mode": "absolute", 6690 "steps": [ 6691 { 6692 "color": "green" 6693 }, 6694 { 6695 "color": "red", 6696 "value": 80 6697 } 6698 ] 6699 }, 6700 "unit": "Bps" 6701 }, 6702 "overrides": [] 6703 }, 6704 "gridPos": { 6705 "h": 8, 6706 "w": 12, 6707 "x": 0, 6708 "y": 21 6709 }, 6710 "id": 89, 6711 "options": { 6712 "legend": { 6713 "calcs": [], 6714 "displayMode": "list", 6715 "placement": "bottom", 6716 "showLegend": false 6717 }, 6718 "tooltip": { 6719 "mode": "multi", 6720 "sort": "none" 6721 } 6722 }, 6723 "targets": [ 6724 { 6725 "datasource": { 6726 "type": "prometheus", 6727 "uid": "${datasource}" 6728 }, 6729 "editorMode": "code", 6730 "expr": "(sum(irate(container_network_receive_bytes_total{job=~\"(.*/)?cadvisor\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*store-gateway.*\", workload_type=\"statefulset\"}) by (pod))", 6731 "legendFormat": "__auto", 6732 "range": true, 6733 "refId": "A" 6734 } 6735 ], 6736 "title": "Receive Bandwidth", 6737 "type": "timeseries" 6738 }, 6739 { 6740 "datasource": { 6741 "type": "prometheus", 6742 "uid": "${datasource}" 6743 }, 6744 "fieldConfig": { 6745 "defaults": { 6746 "color": { 6747 "mode": "palette-classic" 6748 }, 6749 "custom": { 6750 "axisBorderShow": false, 6751 "axisCenteredZero": false, 6752 "axisColorMode": "text", 6753 "axisLabel": "", 6754 "axisPlacement": "auto", 6755 "barAlignment": 0, 6756 "drawStyle": "line", 6757 "fillOpacity": 13, 6758 "gradientMode": "none", 6759 "hideFrom": { 6760 "legend": false, 6761 "tooltip": false, 6762 "viz": false 6763 }, 6764 "insertNulls": false, 6765 "lineInterpolation": "linear", 6766 "lineWidth": 1, 6767 "pointSize": 5, 6768 "scaleDistribution": { 6769 "type": "linear" 6770 }, 6771 "showPoints": "auto", 6772 "spanNulls": false, 6773 "stacking": { 6774 "group": "A", 6775 "mode": "normal" 6776 }, 6777 "thresholdsStyle": { 6778 "mode": "off" 6779 } 6780 }, 6781 "mappings": [], 6782 "thresholds": { 6783 "mode": "absolute", 6784 "steps": [ 6785 { 6786 "color": "green" 6787 }, 6788 { 6789 "color": "red", 6790 "value": 80 6791 } 6792 ] 6793 }, 6794 "unit": "Bps" 6795 }, 6796 "overrides": [] 6797 }, 6798 "gridPos": { 6799 "h": 8, 6800 "w": 12, 6801 "x": 12, 6802 "y": 21 6803 }, 6804 "id": 95, 6805 "options": { 6806 "legend": { 6807 "calcs": [], 6808 "displayMode": "list", 6809 "placement": "bottom", 6810 "showLegend": false 6811 }, 6812 "tooltip": { 6813 "mode": "multi", 6814 "sort": "none" 6815 } 6816 }, 6817 "targets": [ 6818 { 6819 "datasource": { 6820 "type": "prometheus", 6821 "uid": "${datasource}" 6822 }, 6823 "editorMode": "code", 6824 "expr": "(sum(irate(container_network_transmit_bytes_total{job=~\"(.*/)?cadvisor\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*store-gateway.*\", workload_type=\"statefulset\"}) by (pod))", 6825 "legendFormat": "__auto", 6826 "range": true, 6827 "refId": "A" 6828 } 6829 ], 6830 "title": "Transmit Bandwidth", 6831 "type": "timeseries" 6832 } 6833 ], 6834 "title": "Store Gateway", 6835 "type": "row" 6836 }, 6837 { 6838 "collapsed": true, 6839 "gridPos": { 6840 "h": 1, 6841 "w": 24, 6842 "x": 0, 6843 "y": 47 6844 }, 6845 "id": 78, 6846 "panels": [ 6847 { 6848 "datasource": { 6849 "type": "prometheus", 6850 "uid": "${datasource}" 6851 }, 6852 "fieldConfig": { 6853 "defaults": { 6854 "color": { 6855 "mode": "palette-classic" 6856 }, 6857 "custom": { 6858 "axisBorderShow": false, 6859 "axisCenteredZero": false, 6860 "axisColorMode": "text", 6861 "axisLabel": "", 6862 "axisPlacement": "auto", 6863 "barAlignment": 0, 6864 "drawStyle": "line", 6865 "fillOpacity": 12, 6866 "gradientMode": "none", 6867 "hideFrom": { 6868 "legend": false, 6869 "tooltip": false, 6870 "viz": false 6871 }, 6872 "insertNulls": false, 6873 "lineInterpolation": "linear", 6874 "lineWidth": 1, 6875 "pointSize": 5, 6876 "scaleDistribution": { 6877 "type": "linear" 6878 }, 6879 "showPoints": "auto", 6880 "spanNulls": false, 6881 "stacking": { 6882 "group": "A", 6883 "mode": "normal" 6884 }, 6885 "thresholdsStyle": { 6886 "mode": "off" 6887 } 6888 }, 6889 "mappings": [], 6890 "thresholds": { 6891 "mode": "absolute", 6892 "steps": [ 6893 { 6894 "color": "green" 6895 }, 6896 { 6897 "color": "red", 6898 "value": 80 6899 } 6900 ] 6901 } 6902 }, 6903 "overrides": [ 6904 { 6905 "__systemRef": "hideSeriesFrom", 6906 "matcher": { 6907 "id": "byNames", 6908 "options": { 6909 "mode": "exclude", 6910 "names": [ 6911 "get" 6912 ], 6913 "prefix": "All except:", 6914 "readOnly": true 6915 } 6916 }, 6917 "properties": [ 6918 { 6919 "id": "custom.hideFrom", 6920 "value": { 6921 "legend": false, 6922 "tooltip": false, 6923 "viz": true 6924 } 6925 } 6926 ] 6927 } 6928 ] 6929 }, 6930 "gridPos": { 6931 "h": 8, 6932 "w": 12, 6933 "x": 0, 6934 "y": 58 6935 }, 6936 "id": 80, 6937 "options": { 6938 "legend": { 6939 "calcs": [], 6940 "displayMode": "list", 6941 "placement": "bottom", 6942 "showLegend": true 6943 }, 6944 "tooltip": { 6945 "mode": "multi", 6946 "sort": "none" 6947 } 6948 }, 6949 "targets": [ 6950 { 6951 "datasource": { 6952 "type": "prometheus", 6953 "uid": "${datasource}" 6954 }, 6955 "editorMode": "code", 6956 "expr": "sum by(operation) (rate(objstore_bucket_operations_total{namespace=~\"$namespace\",cluster=~\"$cluster\"}[$__rate_interval]))", 6957 "legendFormat": "__auto", 6958 "range": true, 6959 "refId": "A" 6960 } 6961 ], 6962 "title": "Bucket Operation /s", 6963 "type": "timeseries" 6964 }, 6965 { 6966 "datasource": { 6967 "type": "prometheus", 6968 "uid": "${datasource}" 6969 }, 6970 "fieldConfig": { 6971 "defaults": { 6972 "color": { 6973 "mode": "palette-classic" 6974 }, 6975 "custom": { 6976 "axisBorderShow": false, 6977 "axisCenteredZero": false, 6978 "axisColorMode": "text", 6979 "axisLabel": "", 6980 "axisPlacement": "auto", 6981 "barAlignment": 0, 6982 "drawStyle": "line", 6983 "fillOpacity": 12, 6984 "gradientMode": "none", 6985 "hideFrom": { 6986 "legend": false, 6987 "tooltip": false, 6988 "viz": false 6989 }, 6990 "insertNulls": false, 6991 "lineInterpolation": "linear", 6992 "lineWidth": 1, 6993 "pointSize": 5, 6994 "scaleDistribution": { 6995 "type": "linear" 6996 }, 6997 "showPoints": "auto", 6998 "spanNulls": false, 6999 "stacking": { 7000 "group": "A", 7001 "mode": "none" 7002 }, 7003 "thresholdsStyle": { 7004 "mode": "off" 7005 } 7006 }, 7007 "mappings": [], 7008 "thresholds": { 7009 "mode": "absolute", 7010 "steps": [ 7011 { 7012 "color": "green" 7013 }, 7014 { 7015 "color": "red", 7016 "value": 80 7017 } 7018 ] 7019 }, 7020 "unit": "s" 7021 }, 7022 "overrides": [] 7023 }, 7024 "gridPos": { 7025 "h": 8, 7026 "w": 12, 7027 "x": 12, 7028 "y": 58 7029 }, 7030 "id": 81, 7031 "options": { 7032 "legend": { 7033 "calcs": [], 7034 "displayMode": "list", 7035 "placement": "bottom", 7036 "showLegend": true 7037 }, 7038 "tooltip": { 7039 "mode": "multi", 7040 "sort": "none" 7041 } 7042 }, 7043 "targets": [ 7044 { 7045 "datasource": { 7046 "type": "prometheus", 7047 "uid": "${datasource}" 7048 }, 7049 "editorMode": "code", 7050 "expr": "histogram_quantile(0.95, sum(rate(objstore_bucket_operation_duration_seconds_bucket{namespace=~\"$namespace\",cluster=~\"$cluster\"}[$__rate_interval])) by (le,operation))", 7051 "legendFormat": "__auto", 7052 "range": true, 7053 "refId": "A" 7054 } 7055 ], 7056 "title": "Latencies per Operation", 7057 "type": "timeseries" 7058 } 7059 ], 7060 "title": "Object Storage", 7061 "type": "row" 7062 }, 7063 { 7064 "collapsed": false, 7065 "gridPos": { 7066 "h": 1, 7067 "w": 24, 7068 "x": 0, 7069 "y": 48 7070 }, 7071 "id": 105, 7072 "panels": [], 7073 "title": "GitHub API", 7074 "type": "row" 7075 }, 7076 { 7077 "datasource": { 7078 "type": "prometheus", 7079 "uid": "${datasource}" 7080 }, 7081 "fieldConfig": { 7082 "defaults": { 7083 "color": { 7084 "mode": "palette-classic" 7085 }, 7086 "custom": { 7087 "axisBorderShow": false, 7088 "axisCenteredZero": false, 7089 "axisColorMode": "text", 7090 "axisLabel": "", 7091 "axisPlacement": "auto", 7092 "barAlignment": 0, 7093 "barWidthFactor": 0.6, 7094 "drawStyle": "line", 7095 "fillOpacity": 0, 7096 "gradientMode": "none", 7097 "hideFrom": { 7098 "legend": false, 7099 "tooltip": false, 7100 "viz": false 7101 }, 7102 "insertNulls": false, 7103 "lineInterpolation": "linear", 7104 "lineWidth": 1, 7105 "pointSize": 5, 7106 "scaleDistribution": { 7107 "type": "linear" 7108 }, 7109 "showPoints": "auto", 7110 "spanNulls": false, 7111 "stacking": { 7112 "group": "A", 7113 "mode": "none" 7114 }, 7115 "thresholdsStyle": { 7116 "mode": "off" 7117 } 7118 }, 7119 "mappings": [], 7120 "noValue": "0", 7121 "thresholds": { 7122 "mode": "absolute", 7123 "steps": [ 7124 { 7125 "color": "green" 7126 }, 7127 { 7128 "color": "red", 7129 "value": 80 7130 } 7131 ] 7132 }, 7133 "unit": "s" 7134 }, 7135 "overrides": [] 7136 }, 7137 "gridPos": { 7138 "h": 8, 7139 "w": 12, 7140 "x": 0, 7141 "y": 49 7142 }, 7143 "id": 106, 7144 "options": { 7145 "legend": { 7146 "calcs": [], 7147 "displayMode": "list", 7148 "placement": "bottom", 7149 "showLegend": true 7150 }, 7151 "tooltip": { 7152 "hideZeros": false, 7153 "mode": "single", 7154 "sort": "none" 7155 } 7156 }, 7157 "pluginVersion": "12.1.0-88106", 7158 "targets": [ 7159 { 7160 "datasource": { 7161 "type": "prometheus", 7162 "uid": "${datasource}" 7163 }, 7164 "disableTextWrap": false, 7165 "editorMode": "code", 7166 "exemplar": false, 7167 "expr": "histogram_quantile(0.99, sum by (le, route) (rate(pyroscope_vcs_github_request_duration_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"querier\"} [$__rate_interval])))", 7168 "fullMetaSearch": false, 7169 "hide": false, 7170 "includeNullMetadata": true, 7171 "instant": false, 7172 "interval": "", 7173 "legendFormat": "__auto", 7174 "range": true, 7175 "refId": "B", 7176 "useBackend": false 7177 } 7178 ], 7179 "title": "GitHub API (P99)", 7180 "type": "timeseries" 7181 }, 7182 { 7183 "datasource": { 7184 "type": "prometheus", 7185 "uid": "${datasource}" 7186 }, 7187 "fieldConfig": { 7188 "defaults": { 7189 "color": { 7190 "mode": "palette-classic" 7191 }, 7192 "custom": { 7193 "axisBorderShow": false, 7194 "axisCenteredZero": false, 7195 "axisColorMode": "text", 7196 "axisLabel": "", 7197 "axisPlacement": "auto", 7198 "barAlignment": 0, 7199 "barWidthFactor": 0.6, 7200 "drawStyle": "bars", 7201 "fillOpacity": 100, 7202 "gradientMode": "none", 7203 "hideFrom": { 7204 "legend": false, 7205 "tooltip": false, 7206 "viz": false 7207 }, 7208 "insertNulls": false, 7209 "lineInterpolation": "linear", 7210 "lineWidth": 1, 7211 "pointSize": 5, 7212 "scaleDistribution": { 7213 "type": "linear" 7214 }, 7215 "showPoints": "auto", 7216 "spanNulls": false, 7217 "stacking": { 7218 "group": "A", 7219 "mode": "normal" 7220 }, 7221 "thresholdsStyle": { 7222 "mode": "off" 7223 } 7224 }, 7225 "mappings": [], 7226 "noValue": "0", 7227 "thresholds": { 7228 "mode": "absolute", 7229 "steps": [ 7230 { 7231 "color": "green" 7232 } 7233 ] 7234 }, 7235 "unit": "short" 7236 }, 7237 "overrides": [] 7238 }, 7239 "gridPos": { 7240 "h": 8, 7241 "w": 12, 7242 "x": 12, 7243 "y": 49 7244 }, 7245 "id": 107, 7246 "options": { 7247 "legend": { 7248 "calcs": [ 7249 "sum" 7250 ], 7251 "displayMode": "table", 7252 "placement": "bottom", 7253 "showLegend": true 7254 }, 7255 "tooltip": { 7256 "hideZeros": false, 7257 "mode": "single", 7258 "sort": "none" 7259 } 7260 }, 7261 "pluginVersion": "12.1.0-88106", 7262 "targets": [ 7263 { 7264 "datasource": { 7265 "type": "prometheus", 7266 "uid": "${datasource}" 7267 }, 7268 "disableTextWrap": false, 7269 "editorMode": "code", 7270 "exemplar": false, 7271 "expr": "sum by (le, route, status_code) (rate(pyroscope_vcs_github_request_duration_count{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"querier\"} [$__rate_interval]))", 7272 "fullMetaSearch": false, 7273 "hide": false, 7274 "includeNullMetadata": true, 7275 "instant": false, 7276 "interval": "", 7277 "legendFormat": "__auto", 7278 "range": true, 7279 "refId": "B", 7280 "useBackend": false 7281 } 7282 ], 7283 "title": "GitHub API (P99)", 7284 "type": "timeseries" 7285 } 7286 ], 7287 "preload": false, 7288 "refresh": "30s", 7289 "schemaVersion": 41, 7290 "tags": [ 7291 "pyroscope" 7292 ], 7293 "templating": { 7294 "list": [ 7295 { 7296 "current": { 7297 "text": "ops-cortex", 7298 "value": "000000134" 7299 }, 7300 "includeAll": false, 7301 "label": "Data Source", 7302 "name": "datasource", 7303 "options": [], 7304 "query": "prometheus", 7305 "refresh": 1, 7306 "regex": "", 7307 "type": "datasource" 7308 }, 7309 { 7310 "current": { 7311 "text": "Loki-Ops", 7312 "value": "c-R8UWvVk" 7313 }, 7314 "includeAll": false, 7315 "name": "loki_datasource", 7316 "options": [], 7317 "query": "loki", 7318 "refresh": 1, 7319 "regex": "", 7320 "type": "datasource" 7321 }, 7322 { 7323 "current": { 7324 "text": "All", 7325 "value": [ 7326 "$__all" 7327 ] 7328 }, 7329 "datasource": { 7330 "type": "prometheus", 7331 "uid": "$datasource" 7332 }, 7333 "definition": "label_values(pyroscope_build_info,cluster)", 7334 "includeAll": true, 7335 "label": "cluster", 7336 "multi": true, 7337 "name": "cluster", 7338 "options": [], 7339 "query": { 7340 "labelFilters": [], 7341 "query": "label_values(pyroscope_build_info,cluster)", 7342 "refId": "PrometheusVariableQueryEditor-VariableQuery" 7343 }, 7344 "refresh": 1, 7345 "regex": "", 7346 "sort": 1, 7347 "type": "query" 7348 }, 7349 { 7350 "current": { 7351 "text": [ 7352 "default" 7353 ], 7354 "value": [ 7355 "default" 7356 ] 7357 }, 7358 "datasource": { 7359 "type": "prometheus", 7360 "uid": "$datasource" 7361 }, 7362 "definition": "label_values(pyroscope_build_info,namespace)", 7363 "includeAll": true, 7364 "label": "namespace", 7365 "multi": true, 7366 "name": "namespace", 7367 "options": [], 7368 "query": { 7369 "labelFilters": [], 7370 "query": "label_values(pyroscope_build_info,namespace)", 7371 "refId": "PrometheusVariableQueryEditor-VariableQuery" 7372 }, 7373 "refresh": 1, 7374 "regex": ".*", 7375 "sort": 1, 7376 "type": "query" 7377 } 7378 ] 7379 }, 7380 "time": { 7381 "from": "now-1h", 7382 "to": "now" 7383 }, 7384 "timepicker": {}, 7385 "timezone": "", 7386 "title": "Pyroscope / Operational", 7387 "version": 14 7388 }