github.com/grafana/pyroscope@v1.18.0/operations/monitoring/dashboards/v2-write-path.json (about) 1 { 2 "annotations": { 3 "list": [ 4 { 5 "builtIn": 1, 6 "datasource": { 7 "type": "grafana", 8 "uid": "-- Grafana --" 9 }, 10 "enable": true, 11 "hide": true, 12 "iconColor": "rgba(0, 211, 255, 1)", 13 "name": "Annotations \u0026 Alerts", 14 "type": "dashboard" 15 }, 16 { 17 "datasource": { 18 "type": "loki", 19 "uid": "${loki_datasource}" 20 }, 21 "enable": true, 22 "expr": "{cluster=\"$cluster\", container=\"kube-diff-logger\"} | json | namespace_extracted=\"$namespace\"", 23 "hide": false, 24 "iconColor": "yellow", 25 "instant": false, 26 "name": "K8s Changes", 27 "tagKeys": "type,name_extracted,verb", 28 "textFormat": "{{notes}}", 29 "titleFormat": "" 30 }, 31 { 32 "datasource": { 33 "type": "loki", 34 "uid": "${loki_datasource}" 35 }, 36 "enable": false, 37 "expr": "{namespace=\"$namespace\",container=\"eventrouter\"} != \"Preemption is not helpful\" != \"Failed deploy model\" | logfmt | type=\"Warning\"", 38 "hide": false, 39 "iconColor": "orange", 40 "instant": false, 41 "name": "K8s Warnings", 42 "tagKeys": "type,object_kind,object_name", 43 "textFormat": "{{message}}", 44 "titleFormat": "" 45 }, 46 { 47 "datasource": { 48 "type": "loki", 49 "uid": "${loki_datasource}" 50 }, 51 "enable": true, 52 "expr": "{namespace=\"$namespace\",container=\"eventrouter\"} | logfmt | type!=\"Normal\" | type!=\"Warning\"", 53 "hide": false, 54 "iconColor": "red", 55 "instant": false, 56 "name": "K8s Errors", 57 "tagKeys": "type,object_kind,object_name", 58 "textFormat": "{{message}}" 59 }, 60 { 61 "datasource": { 62 "type": "loki", 63 "uid": "${loki_datasource}" 64 }, 65 "enable": false, 66 "expr": "{namespace=\"$namespace\",container=\"eventrouter\"} | logfmt", 67 "hide": false, 68 "iconColor": "blue", 69 "instant": false, 70 "name": "K8s All Events", 71 "tagKeys": "type,object_kind,object_name", 72 "textFormat": "{{message}}" 73 } 74 ] 75 }, 76 "editable": true, 77 "fiscalYearStartMonth": 0, 78 "graphTooltip": 1, 79 "id": 0, 80 "links": [ 81 { 82 "asDropdown": true, 83 "icon": "external link", 84 "includeVars": true, 85 "keepTime": true, 86 "tags": [ 87 "pyroscope" 88 ], 89 "targetBlank": false, 90 "title": "Pyroscope Dashboards", 91 "type": "dashboards" 92 } 93 ], 94 "panels": [ 95 { 96 "collapsed": false, 97 "gridPos": { 98 "h": 1, 99 "w": 24, 100 "x": 0, 101 "y": 0 102 }, 103 "id": 85, 104 "panels": [], 105 "title": "", 106 "type": "row" 107 }, 108 { 109 "datasource": { 110 "type": "prometheus", 111 "uid": "${datasource}" 112 }, 113 "fieldConfig": { 114 "defaults": { 115 "color": { 116 "mode": "thresholds" 117 }, 118 "custom": { 119 "align": "left", 120 "cellOptions": { 121 "type": "auto" 122 }, 123 "filterable": true, 124 "inspect": false 125 }, 126 "mappings": [], 127 "thresholds": { 128 "mode": "absolute", 129 "steps": [ 130 { 131 "color": "green", 132 "value": 0 133 }, 134 { 135 "color": "red", 136 "value": 80 137 } 138 ] 139 } 140 }, 141 "overrides": [ 142 { 143 "matcher": { 144 "id": "byName", 145 "options": "Value" 146 }, 147 "properties": [ 148 { 149 "id": "unit", 150 "value": "decbytes" 151 } 152 ] 153 }, 154 { 155 "matcher": { 156 "id": "byName", 157 "options": "tenant" 158 }, 159 "properties": [ 160 { 161 "id": "custom.width", 162 "value": 78 163 } 164 ] 165 }, 166 { 167 "matcher": { 168 "id": "byName", 169 "options": "slug" 170 }, 171 "properties": [ 172 { 173 "id": "custom.width", 174 "value": 132 175 } 176 ] 177 }, 178 { 179 "matcher": { 180 "id": "byName", 181 "options": "org_name" 182 }, 183 "properties": [ 184 { 185 "id": "custom.width", 186 "value": 142 187 } 188 ] 189 }, 190 { 191 "matcher": { 192 "id": "byName", 193 "options": "Value" 194 }, 195 "properties": [ 196 { 197 "id": "custom.cellOptions", 198 "value": { 199 "mode": "lcd", 200 "type": "gauge", 201 "valueDisplayMode": "text" 202 } 203 }, 204 { 205 "id": "color", 206 "value": { 207 "fixedColor": "green", 208 "mode": "fixed" 209 } 210 } 211 ] 212 }, 213 { 214 "matcher": { 215 "id": "byName", 216 "options": "Slug" 217 }, 218 "properties": [ 219 { 220 "id": "custom.width", 221 "value": 89 222 } 223 ] 224 }, 225 { 226 "matcher": { 227 "id": "byName", 228 "options": "Name" 229 }, 230 "properties": [ 231 { 232 "id": "custom.width", 233 "value": 143 234 } 235 ] 236 }, 237 { 238 "matcher": { 239 "id": "byName", 240 "options": "ID" 241 }, 242 "properties": [ 243 { 244 "id": "custom.width", 245 "value": 104 246 } 247 ] 248 } 249 ] 250 }, 251 "gridPos": { 252 "h": 8, 253 "w": 8, 254 "x": 0, 255 "y": 1 256 }, 257 "id": 68, 258 "options": { 259 "cellHeight": "sm", 260 "footer": { 261 "countRows": true, 262 "enablePagination": false, 263 "fields": "", 264 "reducer": [ 265 "count" 266 ], 267 "show": true 268 }, 269 "showHeader": true, 270 "sortBy": [ 271 { 272 "desc": true, 273 "displayName": "Value" 274 } 275 ] 276 }, 277 "pluginVersion": "12.1.0-90139", 278 "targets": [ 279 { 280 "datasource": { 281 "type": "prometheus", 282 "uid": "${datasource}" 283 }, 284 "editorMode": "code", 285 "exemplar": false, 286 "expr": "\nsum by (tenant, slug, org_name, environment) (\n rate(pyroscope_distributor_received_decompressed_bytes_sum{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])\n)\n", 287 "format": "table", 288 "instant": true, 289 "legendFormat": "__auto", 290 "range": false, 291 "refId": "A" 292 } 293 ], 294 "title": "Tenants", 295 "transformations": [ 296 { 297 "id": "organize", 298 "options": { 299 "excludeByName": { 300 "Time": true, 301 "Value": false, 302 "cluster": true, 303 "environment": true, 304 "slug": false 305 }, 306 "includeByName": {}, 307 "indexByName": { 308 "Time": 0, 309 "Value": 4, 310 "environment": 5, 311 "org_name": 3, 312 "slug": 2, 313 "tenant": 1 314 }, 315 "renameByName": { 316 "Time": "", 317 "Value": "", 318 "environment": "Env", 319 "org_name": "Name", 320 "slug": "Slug", 321 "tenant": "ID" 322 } 323 } 324 } 325 ], 326 "type": "table" 327 }, 328 { 329 "datasource": { 330 "type": "prometheus", 331 "uid": "${datasource}" 332 }, 333 "fieldConfig": { 334 "defaults": { 335 "color": { 336 "mode": "palette-classic" 337 }, 338 "custom": { 339 "axisBorderShow": false, 340 "axisCenteredZero": false, 341 "axisColorMode": "text", 342 "axisLabel": "", 343 "axisPlacement": "auto", 344 "axisSoftMin": 0, 345 "barAlignment": 0, 346 "barWidthFactor": 0.6, 347 "drawStyle": "line", 348 "fillOpacity": 10, 349 "gradientMode": "none", 350 "hideFrom": { 351 "legend": false, 352 "tooltip": false, 353 "viz": false 354 }, 355 "insertNulls": false, 356 "lineInterpolation": "linear", 357 "lineWidth": 1, 358 "pointSize": 5, 359 "scaleDistribution": { 360 "type": "linear" 361 }, 362 "showPoints": "auto", 363 "spanNulls": false, 364 "stacking": { 365 "group": "A", 366 "mode": "none" 367 }, 368 "thresholdsStyle": { 369 "mode": "off" 370 } 371 }, 372 "mappings": [], 373 "thresholds": { 374 "mode": "absolute", 375 "steps": [ 376 { 377 "color": "green", 378 "value": 0 379 }, 380 { 381 "color": "red", 382 "value": 80 383 } 384 ] 385 }, 386 "unit": "binBps" 387 }, 388 "overrides": [] 389 }, 390 "gridPos": { 391 "h": 8, 392 "w": 4, 393 "x": 8, 394 "y": 1 395 }, 396 "id": 66, 397 "options": { 398 "legend": { 399 "calcs": [ 400 "max" 401 ], 402 "displayMode": "table", 403 "placement": "right", 404 "showLegend": false, 405 "sortBy": "Max", 406 "sortDesc": true 407 }, 408 "tooltip": { 409 "hideZeros": false, 410 "mode": "single", 411 "sort": "none" 412 } 413 }, 414 "pluginVersion": "12.1.0-90139", 415 "targets": [ 416 { 417 "datasource": { 418 "type": "prometheus", 419 "uid": "${datasource}" 420 }, 421 "editorMode": "code", 422 "expr": "sum by (tenant) (rate(pyroscope_distributor_received_decompressed_bytes_sum{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval]))", 423 "hide": false, 424 "legendFormat": "__auto", 425 "range": true, 426 "refId": "A" 427 } 428 ], 429 "title": "MBs per Tenant (Decompressed)", 430 "type": "timeseries" 431 }, 432 { 433 "datasource": { 434 "type": "prometheus", 435 "uid": "${datasource}" 436 }, 437 "fieldConfig": { 438 "defaults": { 439 "color": { 440 "mode": "palette-classic" 441 }, 442 "custom": { 443 "axisBorderShow": false, 444 "axisCenteredZero": false, 445 "axisColorMode": "text", 446 "axisLabel": "", 447 "axisPlacement": "auto", 448 "axisSoftMin": 0, 449 "barAlignment": 0, 450 "barWidthFactor": 0.6, 451 "drawStyle": "line", 452 "fillOpacity": 4, 453 "gradientMode": "none", 454 "hideFrom": { 455 "legend": false, 456 "tooltip": false, 457 "viz": false 458 }, 459 "insertNulls": false, 460 "lineInterpolation": "linear", 461 "lineWidth": 1, 462 "pointSize": 5, 463 "scaleDistribution": { 464 "type": "linear" 465 }, 466 "showPoints": "auto", 467 "spanNulls": false, 468 "stacking": { 469 "group": "A", 470 "mode": "none" 471 }, 472 "thresholdsStyle": { 473 "mode": "off" 474 } 475 }, 476 "mappings": [], 477 "thresholds": { 478 "mode": "absolute", 479 "steps": [ 480 { 481 "color": "green", 482 "value": 0 483 }, 484 { 485 "color": "red", 486 "value": 80 487 } 488 ] 489 }, 490 "unit": "cps" 491 }, 492 "overrides": [] 493 }, 494 "gridPos": { 495 "h": 8, 496 "w": 4, 497 "x": 12, 498 "y": 1 499 }, 500 "id": 67, 501 "options": { 502 "legend": { 503 "calcs": [ 504 "mean", 505 "max" 506 ], 507 "displayMode": "table", 508 "placement": "right", 509 "showLegend": false, 510 "sortBy": "Mean", 511 "sortDesc": true 512 }, 513 "tooltip": { 514 "hideZeros": false, 515 "mode": "single", 516 "sort": "none" 517 } 518 }, 519 "pluginVersion": "12.1.0-90139", 520 "targets": [ 521 { 522 "datasource": { 523 "type": "prometheus", 524 "uid": "${datasource}" 525 }, 526 "editorMode": "code", 527 "expr": "sum by (tenant) (rate(pyroscope_distributor_received_decompressed_bytes_count{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval]))", 528 "legendFormat": "__auto", 529 "range": true, 530 "refId": "A" 531 } 532 ], 533 "title": "Profiles/s per Tenant", 534 "type": "timeseries" 535 }, 536 { 537 "datasource": { 538 "type": "prometheus", 539 "uid": "${datasource}" 540 }, 541 "fieldConfig": { 542 "defaults": { 543 "color": { 544 "mode": "palette-classic" 545 }, 546 "custom": { 547 "hideFrom": { 548 "legend": false, 549 "tooltip": false, 550 "viz": false 551 } 552 }, 553 "mappings": [] 554 }, 555 "overrides": [] 556 }, 557 "gridPos": { 558 "h": 8, 559 "w": 4, 560 "x": 16, 561 "y": 1 562 }, 563 "id": 82, 564 "options": { 565 "legend": { 566 "displayMode": "table", 567 "placement": "bottom", 568 "showLegend": true, 569 "values": [ 570 "value" 571 ] 572 }, 573 "pieType": "pie", 574 "reduceOptions": { 575 "calcs": [ 576 "lastNotNull" 577 ], 578 "fields": "", 579 "values": false 580 }, 581 "tooltip": { 582 "hideZeros": false, 583 "mode": "single", 584 "sort": "none" 585 } 586 }, 587 "pluginVersion": "12.1.0-90139", 588 "targets": [ 589 { 590 "datasource": { 591 "type": "prometheus", 592 "uid": "${datasource}" 593 }, 594 "editorMode": "code", 595 "exemplar": false, 596 "expr": "sum by (state) (pyroscope_ring_members{cluster=~\"$cluster\", namespace=~\"$namespace\", name=\"distributor\"}) / count by (state)(pyroscope_ring_members{cluster=~\"$cluster\", namespace=~\"$namespace\", name=\"distributor\"})", 597 "format": "time_series", 598 "instant": true, 599 "legendFormat": "__auto", 600 "range": false, 601 "refId": "A" 602 } 603 ], 604 "title": "Distributor Ring", 605 "type": "piechart" 606 }, 607 { 608 "datasource": { 609 "type": "prometheus", 610 "uid": "${datasource}" 611 }, 612 "description": "The ring is used by distributor to discover segment-writer instances. ", 613 "fieldConfig": { 614 "defaults": { 615 "color": { 616 "mode": "palette-classic" 617 }, 618 "custom": { 619 "hideFrom": { 620 "legend": false, 621 "tooltip": false, 622 "viz": false 623 } 624 }, 625 "mappings": [] 626 }, 627 "overrides": [] 628 }, 629 "gridPos": { 630 "h": 8, 631 "w": 4, 632 "x": 20, 633 "y": 1 634 }, 635 "id": 81, 636 "options": { 637 "legend": { 638 "displayMode": "table", 639 "placement": "bottom", 640 "showLegend": true, 641 "values": [ 642 "value" 643 ] 644 }, 645 "pieType": "pie", 646 "reduceOptions": { 647 "calcs": [ 648 "lastNotNull" 649 ], 650 "fields": "", 651 "values": false 652 }, 653 "tooltip": { 654 "hideZeros": false, 655 "mode": "single", 656 "sort": "none" 657 } 658 }, 659 "pluginVersion": "12.1.0-90139", 660 "targets": [ 661 { 662 "datasource": { 663 "type": "prometheus", 664 "uid": "${datasource}" 665 }, 666 "editorMode": "code", 667 "exemplar": false, 668 "expr": "sum by (state) (pyroscope_ring_members{cluster=~\"$cluster\", namespace=~\"$namespace\", name=\"segment-writer\"}) / count by (state)(pyroscope_ring_members{cluster=~\"$cluster\", namespace=~\"$namespace\", name=\"segment-writer\"})", 669 "format": "time_series", 670 "instant": true, 671 "legendFormat": "__auto", 672 "range": false, 673 "refId": "A" 674 } 675 ], 676 "title": "Segment Writer Ring", 677 "type": "piechart" 678 }, 679 { 680 "collapsed": true, 681 "gridPos": { 682 "h": 1, 683 "w": 24, 684 "x": 0, 685 "y": 19 686 }, 687 "id": 90, 688 "panels": [ 689 { 690 "datasource": { 691 "type": "datasource", 692 "uid": "-- Mixed --" 693 }, 694 "description": "The latency observed by distributor, indicating the overall write path latency.\n\nOnly successful requests are counted.", 695 "fieldConfig": { 696 "defaults": { 697 "color": { 698 "fixedColor": "dark-red", 699 "mode": "palette-classic" 700 }, 701 "custom": { 702 "axisBorderShow": true, 703 "axisCenteredZero": true, 704 "axisColorMode": "text", 705 "axisLabel": "", 706 "axisPlacement": "auto", 707 "barAlignment": 0, 708 "barWidthFactor": 0.6, 709 "drawStyle": "line", 710 "fillOpacity": 0, 711 "gradientMode": "none", 712 "hideFrom": { 713 "legend": false, 714 "tooltip": false, 715 "viz": false 716 }, 717 "insertNulls": false, 718 "lineInterpolation": "linear", 719 "lineStyle": { 720 "fill": "solid" 721 }, 722 "lineWidth": 1, 723 "pointSize": 1, 724 "scaleDistribution": { 725 "type": "linear" 726 }, 727 "showPoints": "never", 728 "spanNulls": false, 729 "stacking": { 730 "group": "A", 731 "mode": "none" 732 }, 733 "thresholdsStyle": { 734 "mode": "off" 735 } 736 }, 737 "fieldMinMax": false, 738 "mappings": [], 739 "min": 0, 740 "thresholds": { 741 "mode": "absolute", 742 "steps": [ 743 { 744 "color": "green", 745 "value": 0 746 } 747 ] 748 }, 749 "unit": "s" 750 }, 751 "overrides": [ 752 { 753 "matcher": { 754 "id": "byName", 755 "options": "p99 overall" 756 }, 757 "properties": [ 758 { 759 "id": "custom.lineWidth", 760 "value": 2 761 }, 762 { 763 "id": "color", 764 "value": { 765 "fixedColor": "green", 766 "mode": "fixed" 767 } 768 } 769 ] 770 } 771 ] 772 }, 773 "gridPos": { 774 "h": 8, 775 "w": 8, 776 "x": 0, 777 "y": 20 778 }, 779 "id": 94, 780 "options": { 781 "legend": { 782 "calcs": [ 783 "mean" 784 ], 785 "displayMode": "table", 786 "placement": "right", 787 "showLegend": false 788 }, 789 "tooltip": { 790 "hideZeros": false, 791 "mode": "multi", 792 "sort": "desc" 793 } 794 }, 795 "pluginVersion": "12.1.0-90058", 796 "targets": [ 797 { 798 "datasource": { 799 "type": "prometheus", 800 "uid": "${datasource}" 801 }, 802 "editorMode": "code", 803 "expr": "histogram_quantile(0.99, sum by (le,pod) (rate(pyroscope_request_duration_seconds_bucket{\n namespace=\"$namespace\",\n container=\"distributor\",\n route=~\".*pusher.*|.*ingest.*\",\n status_code=\"200\",\n}[$__rate_interval])))", 804 "hide": false, 805 "legendFormat": "__auto", 806 "range": true, 807 "refId": "E" 808 }, 809 { 810 "datasource": { 811 "type": "prometheus", 812 "uid": "${datasource}" 813 }, 814 "editorMode": "code", 815 "expr": "histogram_quantile(0.99, sum by (le) (rate(pyroscope_request_duration_seconds_bucket{\n namespace=\"$namespace\",\n container=\"distributor\",\n route=~\".*pusher.*|.*ingest.*\",\n status_code=\"200\",\n}[$__rate_interval])))", 816 "hide": true, 817 "legendFormat": "p99 overall", 818 "range": true, 819 "refId": "A" 820 } 821 ], 822 "title": "Latency", 823 "transparent": true, 824 "type": "timeseries" 825 }, 826 { 827 "datasource": { 828 "type": "datasource", 829 "uid": "-- Mixed --" 830 }, 831 "fieldConfig": { 832 "defaults": { 833 "custom": { 834 "hideFrom": { 835 "legend": false, 836 "tooltip": false, 837 "viz": false 838 }, 839 "scaleDistribution": { 840 "type": "linear" 841 } 842 } 843 }, 844 "overrides": [] 845 }, 846 "gridPos": { 847 "h": 8, 848 "w": 8, 849 "x": 8, 850 "y": 20 851 }, 852 "id": 93, 853 "options": { 854 "calculate": false, 855 "calculation": { 856 "xBuckets": { 857 "mode": "count", 858 "value": "" 859 }, 860 "yBuckets": { 861 "mode": "count", 862 "scale": { 863 "type": "linear" 864 } 865 } 866 }, 867 "cellGap": 0, 868 "cellValues": { 869 "unit": "requests" 870 }, 871 "color": { 872 "exponent": 0.2, 873 "fill": "super-light-red", 874 "mode": "scheme", 875 "reverse": true, 876 "scale": "exponential", 877 "scheme": "Turbo", 878 "steps": 128 879 }, 880 "exemplars": { 881 "color": "orange" 882 }, 883 "filterValues": { 884 "le": 1e-9 885 }, 886 "legend": { 887 "show": false 888 }, 889 "rowsFrame": { 890 "layout": "auto" 891 }, 892 "tooltip": { 893 "mode": "single", 894 "showColorScale": true, 895 "yHistogram": true 896 }, 897 "yAxis": { 898 "axisPlacement": "left", 899 "reverse": false, 900 "unit": "s" 901 } 902 }, 903 "pluginVersion": "12.1.0-90058", 904 "targets": [ 905 { 906 "datasource": { 907 "type": "prometheus", 908 "uid": "${datasource}" 909 }, 910 "editorMode": "code", 911 "expr": "sum(rate(pyroscope_request_duration_seconds{\n namespace=\"$namespace\",\n container=\"distributor\",\n route=~\".*push.*|.*ingest.*\",\n}[$__rate_interval]))", 912 "hide": false, 913 "legendFormat": "__auto", 914 "range": true, 915 "refId": "B" 916 } 917 ], 918 "title": "Latency Heatmap", 919 "transparent": true, 920 "type": "heatmap" 921 }, 922 { 923 "datasource": { 924 "type": "prometheus", 925 "uid": "${datasource}" 926 }, 927 "fieldConfig": { 928 "defaults": { 929 "color": { 930 "mode": "palette-classic" 931 }, 932 "custom": { 933 "axisBorderShow": false, 934 "axisCenteredZero": false, 935 "axisColorMode": "text", 936 "axisLabel": "", 937 "axisPlacement": "auto", 938 "axisSoftMin": 0, 939 "barAlignment": 0, 940 "barWidthFactor": 0.6, 941 "drawStyle": "line", 942 "fillOpacity": 0, 943 "gradientMode": "none", 944 "hideFrom": { 945 "legend": false, 946 "tooltip": false, 947 "viz": false 948 }, 949 "insertNulls": false, 950 "lineInterpolation": "linear", 951 "lineWidth": 1, 952 "pointSize": 5, 953 "scaleDistribution": { 954 "log": 2, 955 "type": "log" 956 }, 957 "showPoints": "never", 958 "spanNulls": true, 959 "stacking": { 960 "group": "A", 961 "mode": "none" 962 }, 963 "thresholdsStyle": { 964 "mode": "off" 965 } 966 }, 967 "fieldMinMax": false, 968 "mappings": [], 969 "thresholds": { 970 "mode": "absolute", 971 "steps": [ 972 { 973 "color": "green", 974 "value": 0 975 }, 976 { 977 "color": "red", 978 "value": 80 979 } 980 ] 981 }, 982 "unit": "reqps" 983 }, 984 "overrides": [] 985 }, 986 "gridPos": { 987 "h": 8, 988 "w": 8, 989 "x": 16, 990 "y": 20 991 }, 992 "id": 42, 993 "options": { 994 "legend": { 995 "calcs": [], 996 "displayMode": "list", 997 "placement": "bottom", 998 "showLegend": true, 999 "sortBy": "Name", 1000 "sortDesc": false 1001 }, 1002 "tooltip": { 1003 "hideZeros": false, 1004 "mode": "multi", 1005 "sort": "none" 1006 } 1007 }, 1008 "pluginVersion": "12.1.0-90058", 1009 "targets": [ 1010 { 1011 "datasource": { 1012 "type": "prometheus", 1013 "uid": "${datasource}" 1014 }, 1015 "editorMode": "code", 1016 "expr": "sum by (status_code) (rate(\n pyroscope_request_duration_seconds_count{\n namespace=\"$namespace\",\n container=\"distributor\",\n route=~\".*push.*|.*ingest.*\",\n }[1m])\n)", 1017 "instant": false, 1018 "legendFormat": "__auto", 1019 "range": true, 1020 "refId": "A" 1021 } 1022 ], 1023 "title": "Rate", 1024 "transparent": true, 1025 "type": "timeseries" 1026 }, 1027 { 1028 "datasource": { 1029 "type": "prometheus", 1030 "uid": "${datasource}" 1031 }, 1032 "description": "The latency observed by distributors (write path router), broken down by instance.\n\nLatencies are expected to be distributed evenly across instances. An outlier here likely indicates a neighborhood issue.\n\nOnly successful requests are counted.", 1033 "fieldConfig": { 1034 "defaults": { 1035 "color": { 1036 "mode": "palette-classic" 1037 }, 1038 "custom": { 1039 "axisBorderShow": false, 1040 "axisCenteredZero": false, 1041 "axisColorMode": "text", 1042 "axisLabel": "", 1043 "axisPlacement": "auto", 1044 "axisSoftMin": 0, 1045 "barAlignment": 0, 1046 "barWidthFactor": 0.6, 1047 "drawStyle": "line", 1048 "fillOpacity": 0, 1049 "gradientMode": "none", 1050 "hideFrom": { 1051 "legend": false, 1052 "tooltip": false, 1053 "viz": false 1054 }, 1055 "insertNulls": false, 1056 "lineInterpolation": "linear", 1057 "lineWidth": 1, 1058 "pointSize": 5, 1059 "scaleDistribution": { 1060 "log": 2, 1061 "type": "log" 1062 }, 1063 "showPoints": "auto", 1064 "spanNulls": false, 1065 "stacking": { 1066 "group": "A", 1067 "mode": "none" 1068 }, 1069 "thresholdsStyle": { 1070 "mode": "off" 1071 } 1072 }, 1073 "fieldMinMax": false, 1074 "mappings": [], 1075 "thresholds": { 1076 "mode": "absolute", 1077 "steps": [ 1078 { 1079 "color": "green", 1080 "value": 0 1081 }, 1082 { 1083 "color": "red", 1084 "value": 80 1085 } 1086 ] 1087 }, 1088 "unit": "s" 1089 }, 1090 "overrides": [] 1091 }, 1092 "gridPos": { 1093 "h": 8, 1094 "w": 8, 1095 "x": 0, 1096 "y": 28 1097 }, 1098 "id": 72, 1099 "options": { 1100 "legend": { 1101 "calcs": [], 1102 "displayMode": "list", 1103 "placement": "bottom", 1104 "showLegend": false 1105 }, 1106 "tooltip": { 1107 "hideZeros": false, 1108 "mode": "single", 1109 "sort": "none" 1110 } 1111 }, 1112 "pluginVersion": "12.1.0-90058", 1113 "targets": [ 1114 { 1115 "datasource": { 1116 "type": "prometheus", 1117 "uid": "${datasource}" 1118 }, 1119 "editorMode": "code", 1120 "expr": "histogram_quantile(0.99, sum(rate(pyroscope_write_path_downstream_request_duration_seconds_bucket{\n namespace=\"$namespace\",\n status=\"200\",\n primary=\"1\",\n}[$__rate_interval])) by (le,route,pod))", 1121 "hide": false, 1122 "instant": false, 1123 "legendFormat": "p99 primary {{pod}} =\u003e {{route}}", 1124 "range": true, 1125 "refId": "A" 1126 }, 1127 { 1128 "datasource": { 1129 "type": "prometheus", 1130 "uid": "${datasource}" 1131 }, 1132 "editorMode": "code", 1133 "expr": "histogram_quantile(0.99, sum(rate(pyroscope_write_path_downstream_request_duration_seconds_bucket{\n namespace=\"$namespace\",\n status=\"200\",\n primary!=\"1\",\n}[$__rate_interval])) by (le,route,pod))", 1134 "hide": false, 1135 "instant": false, 1136 "legendFormat": "p99 secondary {{pod}} =\u003e {{route}}", 1137 "range": true, 1138 "refId": "B" 1139 } 1140 ], 1141 "title": "Downstream Latency", 1142 "transparent": true, 1143 "type": "timeseries" 1144 }, 1145 { 1146 "datasource": { 1147 "type": "datasource", 1148 "uid": "-- Mixed --" 1149 }, 1150 "description": "The latency observed by the segment-writer client", 1151 "fieldConfig": { 1152 "defaults": { 1153 "custom": { 1154 "hideFrom": { 1155 "legend": false, 1156 "tooltip": false, 1157 "viz": false 1158 }, 1159 "scaleDistribution": { 1160 "type": "linear" 1161 } 1162 } 1163 }, 1164 "overrides": [] 1165 }, 1166 "gridPos": { 1167 "h": 8, 1168 "w": 8, 1169 "x": 8, 1170 "y": 28 1171 }, 1172 "id": 56, 1173 "options": { 1174 "calculate": false, 1175 "calculation": { 1176 "xBuckets": { 1177 "mode": "count", 1178 "value": "" 1179 }, 1180 "yBuckets": { 1181 "mode": "count", 1182 "scale": { 1183 "type": "linear" 1184 } 1185 } 1186 }, 1187 "cellGap": 0, 1188 "cellValues": { 1189 "unit": "requests" 1190 }, 1191 "color": { 1192 "exponent": 0.2, 1193 "fill": "super-light-red", 1194 "mode": "scheme", 1195 "reverse": true, 1196 "scale": "exponential", 1197 "scheme": "Turbo", 1198 "steps": 128 1199 }, 1200 "exemplars": { 1201 "color": "orange" 1202 }, 1203 "filterValues": { 1204 "le": 1e-9 1205 }, 1206 "legend": { 1207 "show": false 1208 }, 1209 "rowsFrame": { 1210 "layout": "auto" 1211 }, 1212 "tooltip": { 1213 "mode": "single", 1214 "showColorScale": true, 1215 "yHistogram": true 1216 }, 1217 "yAxis": { 1218 "axisPlacement": "left", 1219 "reverse": false, 1220 "unit": "s" 1221 } 1222 }, 1223 "pluginVersion": "12.1.0-90058", 1224 "targets": [ 1225 { 1226 "datasource": { 1227 "type": "prometheus", 1228 "uid": "${datasource}" 1229 }, 1230 "editorMode": "code", 1231 "expr": "sum(rate(pyroscope_write_path_downstream_request_duration_seconds{\n namespace=\"$namespace\",\n route=\"segment-writer\",\n}[$__rate_interval]))", 1232 "hide": false, 1233 "legendFormat": "__auto", 1234 "range": true, 1235 "refId": "B" 1236 } 1237 ], 1238 "title": "Latency Heatmap", 1239 "transparent": true, 1240 "type": "heatmap" 1241 }, 1242 { 1243 "datasource": { 1244 "type": "prometheus", 1245 "uid": "${datasource}" 1246 }, 1247 "fieldConfig": { 1248 "defaults": { 1249 "color": { 1250 "mode": "palette-classic" 1251 }, 1252 "custom": { 1253 "axisBorderShow": false, 1254 "axisCenteredZero": false, 1255 "axisColorMode": "text", 1256 "axisLabel": "", 1257 "axisPlacement": "auto", 1258 "axisSoftMin": 0, 1259 "barAlignment": 0, 1260 "barWidthFactor": 0.6, 1261 "drawStyle": "line", 1262 "fillOpacity": 0, 1263 "gradientMode": "none", 1264 "hideFrom": { 1265 "legend": false, 1266 "tooltip": false, 1267 "viz": false 1268 }, 1269 "insertNulls": false, 1270 "lineInterpolation": "linear", 1271 "lineWidth": 1, 1272 "pointSize": 5, 1273 "scaleDistribution": { 1274 "log": 2, 1275 "type": "log" 1276 }, 1277 "showPoints": "never", 1278 "spanNulls": true, 1279 "stacking": { 1280 "group": "A", 1281 "mode": "none" 1282 }, 1283 "thresholdsStyle": { 1284 "mode": "off" 1285 } 1286 }, 1287 "fieldMinMax": false, 1288 "mappings": [], 1289 "thresholds": { 1290 "mode": "absolute", 1291 "steps": [ 1292 { 1293 "color": "green", 1294 "value": 0 1295 }, 1296 { 1297 "color": "red", 1298 "value": 80 1299 } 1300 ] 1301 }, 1302 "unit": "reqps" 1303 }, 1304 "overrides": [] 1305 }, 1306 "gridPos": { 1307 "h": 8, 1308 "w": 8, 1309 "x": 16, 1310 "y": 28 1311 }, 1312 "id": 95, 1313 "options": { 1314 "legend": { 1315 "calcs": [ 1316 "last" 1317 ], 1318 "displayMode": "table", 1319 "placement": "right", 1320 "showLegend": false, 1321 "sortBy": "Name", 1322 "sortDesc": false 1323 }, 1324 "tooltip": { 1325 "hideZeros": false, 1326 "mode": "multi", 1327 "sort": "none" 1328 } 1329 }, 1330 "pluginVersion": "12.1.0-90058", 1331 "targets": [ 1332 { 1333 "datasource": { 1334 "type": "prometheus", 1335 "uid": "${datasource}" 1336 }, 1337 "editorMode": "code", 1338 "expr": "sum(irate(pyroscope_write_path_downstream_request_duration_seconds_count{\n namespace=\"$namespace\",\n primary=\"1\",\n}[$__rate_interval])) by (status,route)", 1339 "instant": false, 1340 "legendFormat": "primary {{route}} {{status}}", 1341 "range": true, 1342 "refId": "A" 1343 }, 1344 { 1345 "datasource": { 1346 "type": "prometheus", 1347 "uid": "${datasource}" 1348 }, 1349 "editorMode": "code", 1350 "expr": "sum(irate(pyroscope_write_path_downstream_request_duration_seconds_count{\n namespace=\"$namespace\",\n primary=\"0\",\n}[$__rate_interval])) by (status,route)", 1351 "hide": false, 1352 "instant": false, 1353 "legendFormat": "secondary {{route}} {{status}}", 1354 "range": true, 1355 "refId": "C" 1356 } 1357 ], 1358 "title": "Downstream Rate", 1359 "transparent": true, 1360 "type": "timeseries" 1361 }, 1362 { 1363 "datasource": { 1364 "type": "datasource", 1365 "uid": "-- Mixed --" 1366 }, 1367 "fieldConfig": { 1368 "defaults": { 1369 "color": { 1370 "mode": "palette-classic" 1371 }, 1372 "custom": { 1373 "axisBorderShow": false, 1374 "axisCenteredZero": false, 1375 "axisColorMode": "text", 1376 "axisLabel": "", 1377 "axisPlacement": "auto", 1378 "axisSoftMin": 0, 1379 "barAlignment": 1, 1380 "barWidthFactor": 1, 1381 "drawStyle": "line", 1382 "fillOpacity": 0, 1383 "gradientMode": "none", 1384 "hideFrom": { 1385 "legend": false, 1386 "tooltip": false, 1387 "viz": false 1388 }, 1389 "insertNulls": false, 1390 "lineInterpolation": "linear", 1391 "lineStyle": { 1392 "fill": "solid" 1393 }, 1394 "lineWidth": 1, 1395 "pointSize": 5, 1396 "scaleDistribution": { 1397 "type": "linear" 1398 }, 1399 "showPoints": "auto", 1400 "spanNulls": false, 1401 "stacking": { 1402 "group": "A", 1403 "mode": "none" 1404 }, 1405 "thresholdsStyle": { 1406 "mode": "off" 1407 } 1408 }, 1409 "fieldMinMax": false, 1410 "mappings": [], 1411 "min": 0, 1412 "thresholds": { 1413 "mode": "absolute", 1414 "steps": [ 1415 { 1416 "color": "green", 1417 "value": 0 1418 } 1419 ] 1420 }, 1421 "unit": "bytes" 1422 }, 1423 "overrides": [ 1424 { 1425 "matcher": { 1426 "id": "byName", 1427 "options": "Limit" 1428 }, 1429 "properties": [ 1430 { 1431 "id": "custom.lineStyle", 1432 "value": { 1433 "dash": [ 1434 10, 1435 10 1436 ], 1437 "fill": "dash" 1438 } 1439 } 1440 ] 1441 }, 1442 { 1443 "matcher": { 1444 "id": "byFrameRefID", 1445 "options": "A" 1446 }, 1447 "properties": [ 1448 { 1449 "id": "custom.axisColorMode", 1450 "value": "text" 1451 }, 1452 { 1453 "id": "color", 1454 "value": { 1455 "fixedColor": "semi-dark-red", 1456 "mode": "fixed" 1457 } 1458 }, 1459 { 1460 "id": "custom.lineWidth", 1461 "value": 2 1462 } 1463 ] 1464 } 1465 ] 1466 }, 1467 "gridPos": { 1468 "h": 8, 1469 "w": 8, 1470 "x": 0, 1471 "y": 36 1472 }, 1473 "id": 97, 1474 "options": { 1475 "legend": { 1476 "calcs": [ 1477 "last", 1478 "max" 1479 ], 1480 "displayMode": "table", 1481 "placement": "bottom", 1482 "showLegend": false, 1483 "sortBy": "Max", 1484 "sortDesc": true 1485 }, 1486 "tooltip": { 1487 "hideZeros": false, 1488 "mode": "multi", 1489 "sort": "desc" 1490 } 1491 }, 1492 "pluginVersion": "12.1.0-90058", 1493 "targets": [ 1494 { 1495 "datasource": { 1496 "type": "prometheus", 1497 "uid": "${datasource}" 1498 }, 1499 "editorMode": "code", 1500 "expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"$namespace\",container=\"distributor\"})", 1501 "legendFormat": "__auto", 1502 "range": true, 1503 "refId": "by pod" 1504 }, 1505 { 1506 "datasource": { 1507 "type": "prometheus", 1508 "uid": "${datasource}" 1509 }, 1510 "editorMode": "code", 1511 "expr": "max(kube_pod_container_resource_requests{job=~\"(.*/)?kube-state-metrics\", namespace=\"$namespace\", resource=\"memory\", container=\"distributor\"})", 1512 "hide": false, 1513 "legendFormat": "Request", 1514 "range": true, 1515 "refId": "A" 1516 } 1517 ], 1518 "title": "Memory Usage", 1519 "transparent": true, 1520 "type": "timeseries" 1521 }, 1522 { 1523 "datasource": { 1524 "type": "datasource", 1525 "uid": "-- Mixed --" 1526 }, 1527 "fieldConfig": { 1528 "defaults": { 1529 "color": { 1530 "mode": "palette-classic" 1531 }, 1532 "custom": { 1533 "axisBorderShow": false, 1534 "axisCenteredZero": false, 1535 "axisColorMode": "text", 1536 "axisLabel": "", 1537 "axisPlacement": "auto", 1538 "axisSoftMin": 0, 1539 "barAlignment": 1, 1540 "barWidthFactor": 1, 1541 "drawStyle": "line", 1542 "fillOpacity": 0, 1543 "gradientMode": "none", 1544 "hideFrom": { 1545 "legend": false, 1546 "tooltip": false, 1547 "viz": false 1548 }, 1549 "insertNulls": false, 1550 "lineInterpolation": "linear", 1551 "lineStyle": { 1552 "fill": "solid" 1553 }, 1554 "lineWidth": 1, 1555 "pointSize": 5, 1556 "scaleDistribution": { 1557 "type": "linear" 1558 }, 1559 "showPoints": "auto", 1560 "spanNulls": false, 1561 "stacking": { 1562 "group": "A", 1563 "mode": "none" 1564 }, 1565 "thresholdsStyle": { 1566 "mode": "off" 1567 } 1568 }, 1569 "fieldMinMax": false, 1570 "mappings": [], 1571 "min": 0, 1572 "thresholds": { 1573 "mode": "absolute", 1574 "steps": [ 1575 { 1576 "color": "green", 1577 "value": 0 1578 } 1579 ] 1580 } 1581 }, 1582 "overrides": [ 1583 { 1584 "matcher": { 1585 "id": "byName", 1586 "options": "Limit" 1587 }, 1588 "properties": [ 1589 { 1590 "id": "custom.lineStyle", 1591 "value": { 1592 "dash": [ 1593 10, 1594 10 1595 ], 1596 "fill": "dash" 1597 } 1598 } 1599 ] 1600 }, 1601 { 1602 "matcher": { 1603 "id": "byFrameRefID", 1604 "options": "A" 1605 }, 1606 "properties": [ 1607 { 1608 "id": "custom.axisColorMode", 1609 "value": "text" 1610 }, 1611 { 1612 "id": "color", 1613 "value": { 1614 "fixedColor": "semi-dark-red", 1615 "mode": "fixed" 1616 } 1617 }, 1618 { 1619 "id": "custom.lineWidth", 1620 "value": 2 1621 } 1622 ] 1623 } 1624 ] 1625 }, 1626 "gridPos": { 1627 "h": 8, 1628 "w": 8, 1629 "x": 8, 1630 "y": 36 1631 }, 1632 "id": 62, 1633 "options": { 1634 "legend": { 1635 "calcs": [ 1636 "last", 1637 "max" 1638 ], 1639 "displayMode": "table", 1640 "placement": "bottom", 1641 "showLegend": false, 1642 "sortBy": "Max", 1643 "sortDesc": true 1644 }, 1645 "tooltip": { 1646 "hideZeros": false, 1647 "mode": "multi", 1648 "sort": "desc" 1649 } 1650 }, 1651 "pluginVersion": "12.1.0-90058", 1652 "targets": [ 1653 { 1654 "datasource": { 1655 "type": "prometheus", 1656 "uid": "${datasource}" 1657 }, 1658 "editorMode": "code", 1659 "expr": "sum by (pod) (node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\",container=\"distributor\"})", 1660 "legendFormat": "__auto", 1661 "range": true, 1662 "refId": "by pod" 1663 }, 1664 { 1665 "datasource": { 1666 "type": "prometheus", 1667 "uid": "${datasource}" 1668 }, 1669 "editorMode": "code", 1670 "expr": "max(\n kube_pod_container_resource_requests{job=~\"(.*/)?kube-state-metrics\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".*-distributor\"}\n)", 1671 "hide": false, 1672 "legendFormat": "Request", 1673 "range": true, 1674 "refId": "A" 1675 }, 1676 { 1677 "datasource": { 1678 "type": "prometheus", 1679 "uid": "${datasource}" 1680 }, 1681 "editorMode": "code", 1682 "expr": "max(\n kube_pod_container_resource_limits{job=~\"(.*/)?kube-state-metrics\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".*-distributor\"}\n)", 1683 "hide": false, 1684 "legendFormat": "Limit", 1685 "range": true, 1686 "refId": "B" 1687 } 1688 ], 1689 "title": "CPU Usage", 1690 "transparent": true, 1691 "type": "timeseries" 1692 }, 1693 { 1694 "datasource": { 1695 "type": "prometheus", 1696 "uid": "${datasource}" 1697 }, 1698 "description": "The chart showd distribution of requests from the upstream (cortex-gw/envoy)", 1699 "fieldConfig": { 1700 "defaults": { 1701 "color": { 1702 "mode": "palette-classic" 1703 }, 1704 "custom": { 1705 "axisBorderShow": false, 1706 "axisCenteredZero": false, 1707 "axisColorMode": "text", 1708 "axisLabel": "", 1709 "axisPlacement": "auto", 1710 "axisSoftMin": 0, 1711 "barAlignment": 0, 1712 "barWidthFactor": 0.6, 1713 "drawStyle": "line", 1714 "fillOpacity": 0, 1715 "gradientMode": "none", 1716 "hideFrom": { 1717 "legend": false, 1718 "tooltip": false, 1719 "viz": false 1720 }, 1721 "insertNulls": false, 1722 "lineInterpolation": "linear", 1723 "lineWidth": 1, 1724 "pointSize": 5, 1725 "scaleDistribution": { 1726 "log": 2, 1727 "type": "log" 1728 }, 1729 "showPoints": "never", 1730 "spanNulls": true, 1731 "stacking": { 1732 "group": "A", 1733 "mode": "none" 1734 }, 1735 "thresholdsStyle": { 1736 "mode": "off" 1737 } 1738 }, 1739 "fieldMinMax": false, 1740 "mappings": [], 1741 "thresholds": { 1742 "mode": "absolute", 1743 "steps": [ 1744 { 1745 "color": "green", 1746 "value": 0 1747 }, 1748 { 1749 "color": "red", 1750 "value": 80 1751 } 1752 ] 1753 }, 1754 "unit": "reqps" 1755 }, 1756 "overrides": [] 1757 }, 1758 "gridPos": { 1759 "h": 8, 1760 "w": 8, 1761 "x": 16, 1762 "y": 36 1763 }, 1764 "id": 98, 1765 "options": { 1766 "legend": { 1767 "calcs": [ 1768 "last" 1769 ], 1770 "displayMode": "table", 1771 "placement": "right", 1772 "showLegend": false, 1773 "sortBy": "Name", 1774 "sortDesc": false 1775 }, 1776 "tooltip": { 1777 "hideZeros": false, 1778 "mode": "single", 1779 "sort": "asc" 1780 } 1781 }, 1782 "pluginVersion": "12.1.0-90058", 1783 "targets": [ 1784 { 1785 "datasource": { 1786 "type": "prometheus", 1787 "uid": "${datasource}" 1788 }, 1789 "editorMode": "code", 1790 "expr": "sum by (pod) (rate(\n pyroscope_request_duration_seconds_count{\n namespace=\"$namespace\",\n container=\"distributor\",\n route=~\".*push.*|.*ingest.*\",\n }[1m])\n)", 1791 "instant": false, 1792 "legendFormat": "__auto", 1793 "range": true, 1794 "refId": "A" 1795 } 1796 ], 1797 "title": "Upstream Distribution", 1798 "transparent": true, 1799 "type": "timeseries" 1800 } 1801 ], 1802 "title": "Distributor", 1803 "type": "row" 1804 }, 1805 { 1806 "collapsed": false, 1807 "gridPos": { 1808 "h": 1, 1809 "w": 24, 1810 "x": 0, 1811 "y": 20 1812 }, 1813 "id": 47, 1814 "panels": [], 1815 "title": "Segment Writer", 1816 "type": "row" 1817 }, 1818 { 1819 "datasource": { 1820 "type": "prometheus", 1821 "uid": "${datasource}" 1822 }, 1823 "description": "The latency of the Push endpoint observed by segment-writer.", 1824 "fieldConfig": { 1825 "defaults": { 1826 "color": { 1827 "mode": "palette-classic" 1828 }, 1829 "custom": { 1830 "axisBorderShow": false, 1831 "axisCenteredZero": false, 1832 "axisColorMode": "text", 1833 "axisLabel": "", 1834 "axisPlacement": "auto", 1835 "barAlignment": 0, 1836 "barWidthFactor": 0.6, 1837 "drawStyle": "line", 1838 "fillOpacity": 0, 1839 "gradientMode": "none", 1840 "hideFrom": { 1841 "legend": false, 1842 "tooltip": false, 1843 "viz": false 1844 }, 1845 "insertNulls": false, 1846 "lineInterpolation": "linear", 1847 "lineWidth": 1, 1848 "pointSize": 5, 1849 "scaleDistribution": { 1850 "type": "linear" 1851 }, 1852 "showPoints": "auto", 1853 "spanNulls": false, 1854 "stacking": { 1855 "group": "A", 1856 "mode": "none" 1857 }, 1858 "thresholdsStyle": { 1859 "mode": "off" 1860 } 1861 }, 1862 "mappings": [], 1863 "thresholds": { 1864 "mode": "absolute", 1865 "steps": [ 1866 { 1867 "color": "green", 1868 "value": 0 1869 }, 1870 { 1871 "color": "red", 1872 "value": 80 1873 } 1874 ] 1875 }, 1876 "unit": "s" 1877 }, 1878 "overrides": [ 1879 { 1880 "matcher": { 1881 "id": "byFrameRefID", 1882 "options": "Overall p99" 1883 }, 1884 "properties": [ 1885 { 1886 "id": "custom.lineWidth", 1887 "value": 2 1888 }, 1889 { 1890 "id": "color", 1891 "value": { 1892 "fixedColor": "green", 1893 "mode": "fixed" 1894 } 1895 } 1896 ] 1897 }, 1898 { 1899 "matcher": { 1900 "id": "byFrameRefID" 1901 }, 1902 "properties": [] 1903 } 1904 ] 1905 }, 1906 "gridPos": { 1907 "h": 8, 1908 "w": 8, 1909 "x": 0, 1910 "y": 21 1911 }, 1912 "id": 79, 1913 "options": { 1914 "legend": { 1915 "calcs": [], 1916 "displayMode": "list", 1917 "placement": "bottom", 1918 "showLegend": false 1919 }, 1920 "tooltip": { 1921 "hideZeros": false, 1922 "mode": "multi", 1923 "sort": "desc" 1924 } 1925 }, 1926 "pluginVersion": "12.1.0-90139", 1927 "targets": [ 1928 { 1929 "datasource": { 1930 "type": "prometheus", 1931 "uid": "${datasource}" 1932 }, 1933 "editorMode": "code", 1934 "expr": "histogram_quantile(0.99, sum(rate(pyroscope_request_duration_seconds{\n namespace=\"$namespace\",\n method=\"gRPC\",\n # status_code=\"success\",\n route=\"/segmentwriter.v1.SegmentWriterService/Push\",\n}[$__rate_interval])) by (le,pod))", 1935 "hide": false, 1936 "instant": false, 1937 "legendFormat": "p99 {{pod}}", 1938 "range": true, 1939 "refId": "B" 1940 }, 1941 { 1942 "datasource": { 1943 "type": "prometheus", 1944 "uid": "${datasource}" 1945 }, 1946 "editorMode": "code", 1947 "expr": "histogram_quantile(0.99, sum(rate(pyroscope_request_duration_seconds{\n namespace=\"$namespace\",\n method=\"gRPC\",\n # status_code=\"success\",\n route=\"/segmentwriter.v1.SegmentWriterService/Push\",\n}[$__rate_interval])) by (le))", 1948 "hide": false, 1949 "instant": false, 1950 "legendFormat": "p99 overall", 1951 "range": true, 1952 "refId": "Overall p99" 1953 }, 1954 { 1955 "datasource": { 1956 "type": "prometheus", 1957 "uid": "${datasource}" 1958 }, 1959 "editorMode": "code", 1960 "expr": "histogram_quantile(0.90, sum(rate(pyroscope_request_duration_seconds{\n namespace=\"$namespace\",\n method=\"gRPC\",\n # status_code=\"success\",\n route=\"/segmentwriter.v1.SegmentWriterService/Push\",\n}[$__rate_interval])) by (le))", 1961 "hide": false, 1962 "instant": false, 1963 "legendFormat": "p90 overall", 1964 "range": true, 1965 "refId": "Overall p90" 1966 }, 1967 { 1968 "datasource": { 1969 "type": "prometheus", 1970 "uid": "${datasource}" 1971 }, 1972 "editorMode": "code", 1973 "expr": "histogram_quantile(0.50, sum(rate(pyroscope_request_duration_seconds_bucket{\n namespace=\"$namespace\",\n method=\"gRPC\",\n # status_code=\"success\",\n route=\"/segmentwriter.v1.SegmentWriterService/Push\",\n}[$__rate_interval])) by (le))", 1974 "hide": false, 1975 "instant": false, 1976 "legendFormat": "p50 overall", 1977 "range": true, 1978 "refId": "Overall p50" 1979 } 1980 ], 1981 "title": "Push Latency", 1982 "transparent": true, 1983 "type": "timeseries" 1984 }, 1985 { 1986 "datasource": { 1987 "type": "datasource", 1988 "uid": "-- Mixed --" 1989 }, 1990 "description": "The latency of the Push endpoint observed by segment-writer", 1991 "fieldConfig": { 1992 "defaults": { 1993 "custom": { 1994 "hideFrom": { 1995 "legend": false, 1996 "tooltip": false, 1997 "viz": false 1998 }, 1999 "scaleDistribution": { 2000 "type": "linear" 2001 } 2002 } 2003 }, 2004 "overrides": [] 2005 }, 2006 "gridPos": { 2007 "h": 8, 2008 "w": 8, 2009 "x": 8, 2010 "y": 21 2011 }, 2012 "id": 76, 2013 "options": { 2014 "calculate": false, 2015 "calculation": { 2016 "xBuckets": { 2017 "mode": "count", 2018 "value": "" 2019 }, 2020 "yBuckets": { 2021 "mode": "count", 2022 "scale": { 2023 "type": "linear" 2024 } 2025 } 2026 }, 2027 "cellGap": 0, 2028 "cellValues": { 2029 "unit": "requests" 2030 }, 2031 "color": { 2032 "exponent": 0.2, 2033 "fill": "super-light-red", 2034 "mode": "scheme", 2035 "reverse": true, 2036 "scale": "exponential", 2037 "scheme": "Turbo", 2038 "steps": 128 2039 }, 2040 "exemplars": { 2041 "color": "orange" 2042 }, 2043 "filterValues": { 2044 "le": 1e-9 2045 }, 2046 "legend": { 2047 "show": false 2048 }, 2049 "rowsFrame": { 2050 "layout": "auto" 2051 }, 2052 "tooltip": { 2053 "mode": "single", 2054 "showColorScale": true, 2055 "yHistogram": true 2056 }, 2057 "yAxis": { 2058 "axisPlacement": "left", 2059 "reverse": false, 2060 "unit": "s" 2061 } 2062 }, 2063 "pluginVersion": "12.1.0-90139", 2064 "targets": [ 2065 { 2066 "datasource": { 2067 "type": "prometheus", 2068 "uid": "${datasource}" 2069 }, 2070 "editorMode": "code", 2071 "expr": "sum(rate(pyroscope_request_duration_seconds{\n namespace=\"$namespace\",\n container=\"segment-writer\",\n method=\"gRPC\",\n # status_code=\"success\",\n route=\"/segmentwriter.v1.SegmentWriterService/Push\",\n}[$__rate_interval]))", 2072 "hide": false, 2073 "legendFormat": "__auto", 2074 "range": true, 2075 "refId": "B" 2076 } 2077 ], 2078 "title": "Latency Heatmap", 2079 "transparent": true, 2080 "type": "heatmap" 2081 }, 2082 { 2083 "datasource": { 2084 "type": "prometheus", 2085 "uid": "${datasource}" 2086 }, 2087 "fieldConfig": { 2088 "defaults": { 2089 "color": { 2090 "mode": "palette-classic" 2091 }, 2092 "custom": { 2093 "axisBorderShow": false, 2094 "axisCenteredZero": false, 2095 "axisColorMode": "text", 2096 "axisLabel": "", 2097 "axisPlacement": "auto", 2098 "axisSoftMin": 0, 2099 "barAlignment": 0, 2100 "barWidthFactor": 0.6, 2101 "drawStyle": "line", 2102 "fillOpacity": 0, 2103 "gradientMode": "none", 2104 "hideFrom": { 2105 "legend": false, 2106 "tooltip": false, 2107 "viz": false 2108 }, 2109 "insertNulls": false, 2110 "lineInterpolation": "linear", 2111 "lineWidth": 1, 2112 "pointSize": 5, 2113 "scaleDistribution": { 2114 "log": 2, 2115 "type": "log" 2116 }, 2117 "showPoints": "auto", 2118 "spanNulls": false, 2119 "stacking": { 2120 "group": "A", 2121 "mode": "none" 2122 }, 2123 "thresholdsStyle": { 2124 "mode": "off" 2125 } 2126 }, 2127 "fieldMinMax": false, 2128 "mappings": [], 2129 "thresholds": { 2130 "mode": "absolute", 2131 "steps": [ 2132 { 2133 "color": "green", 2134 "value": 0 2135 }, 2136 { 2137 "color": "red", 2138 "value": 80 2139 } 2140 ] 2141 }, 2142 "unit": "reqps" 2143 }, 2144 "overrides": [] 2145 }, 2146 "gridPos": { 2147 "h": 8, 2148 "w": 8, 2149 "x": 16, 2150 "y": 21 2151 }, 2152 "id": 52, 2153 "options": { 2154 "legend": { 2155 "calcs": [], 2156 "displayMode": "list", 2157 "placement": "bottom", 2158 "showLegend": true 2159 }, 2160 "tooltip": { 2161 "hideZeros": false, 2162 "mode": "single", 2163 "sort": "none" 2164 } 2165 }, 2166 "pluginVersion": "12.1.0-90139", 2167 "targets": [ 2168 { 2169 "datasource": { 2170 "type": "prometheus", 2171 "uid": "${datasource}" 2172 }, 2173 "editorMode": "code", 2174 "expr": "sum(irate(pyroscope_request_duration_seconds_count{\n namespace=\"$namespace\",\n method=\"gRPC\",\n route=\"/segmentwriter.v1.SegmentWriterService/Push\",\n}[$__rate_interval])) by (status_code)", 2175 "instant": false, 2176 "legendFormat": "__auto", 2177 "range": true, 2178 "refId": "A" 2179 } 2180 ], 2181 "title": "Push Rate", 2182 "transparent": true, 2183 "type": "timeseries" 2184 }, 2185 { 2186 "datasource": { 2187 "type": "prometheus", 2188 "uid": "${datasource}" 2189 }, 2190 "description": "The latency segment-writer observes when uploading segments to object store, including retries.", 2191 "fieldConfig": { 2192 "defaults": { 2193 "color": { 2194 "mode": "palette-classic" 2195 }, 2196 "custom": { 2197 "axisBorderShow": false, 2198 "axisCenteredZero": false, 2199 "axisColorMode": "text", 2200 "axisLabel": "", 2201 "axisPlacement": "auto", 2202 "barAlignment": 0, 2203 "barWidthFactor": 0.6, 2204 "drawStyle": "line", 2205 "fillOpacity": 0, 2206 "gradientMode": "none", 2207 "hideFrom": { 2208 "legend": false, 2209 "tooltip": false, 2210 "viz": false 2211 }, 2212 "insertNulls": false, 2213 "lineInterpolation": "linear", 2214 "lineWidth": 1, 2215 "pointSize": 5, 2216 "scaleDistribution": { 2217 "type": "linear" 2218 }, 2219 "showPoints": "auto", 2220 "spanNulls": false, 2221 "stacking": { 2222 "group": "A", 2223 "mode": "none" 2224 }, 2225 "thresholdsStyle": { 2226 "mode": "off" 2227 } 2228 }, 2229 "mappings": [], 2230 "thresholds": { 2231 "mode": "absolute", 2232 "steps": [ 2233 { 2234 "color": "green", 2235 "value": 0 2236 }, 2237 { 2238 "color": "red", 2239 "value": 80 2240 } 2241 ] 2242 }, 2243 "unit": "s" 2244 }, 2245 "overrides": [ 2246 { 2247 "matcher": { 2248 "id": "byFrameRefID", 2249 "options": "Overall p99" 2250 }, 2251 "properties": [ 2252 { 2253 "id": "custom.lineWidth", 2254 "value": 2 2255 }, 2256 { 2257 "id": "color", 2258 "value": { 2259 "fixedColor": "green", 2260 "mode": "fixed" 2261 } 2262 } 2263 ] 2264 }, 2265 { 2266 "matcher": { 2267 "id": "byFrameRefID" 2268 }, 2269 "properties": [] 2270 } 2271 ] 2272 }, 2273 "gridPos": { 2274 "h": 8, 2275 "w": 8, 2276 "x": 0, 2277 "y": 29 2278 }, 2279 "id": 57, 2280 "options": { 2281 "legend": { 2282 "calcs": [], 2283 "displayMode": "list", 2284 "placement": "bottom", 2285 "showLegend": false 2286 }, 2287 "tooltip": { 2288 "hideZeros": false, 2289 "mode": "multi", 2290 "sort": "desc" 2291 } 2292 }, 2293 "pluginVersion": "12.1.0-90139", 2294 "targets": [ 2295 { 2296 "datasource": { 2297 "type": "prometheus", 2298 "uid": "${datasource}" 2299 }, 2300 "editorMode": "code", 2301 "expr": "histogram_quantile(0.99, sum(rate(pyroscope_segment_writer_upload_duration_seconds_bucket{namespace=\"$namespace\",container=\"segment-writer\"}[$__rate_interval])) by (le,pod))", 2302 "hide": false, 2303 "instant": false, 2304 "legendFormat": "p99 {{pod}}", 2305 "range": true, 2306 "refId": "B" 2307 }, 2308 { 2309 "datasource": { 2310 "type": "prometheus", 2311 "uid": "${datasource}" 2312 }, 2313 "editorMode": "code", 2314 "expr": "histogram_quantile(0.99, sum(rate(pyroscope_segment_writer_upload_duration_seconds_bucket{namespace=\"$namespace\",container=\"segment-writer\"}[$__rate_interval])) by (le))", 2315 "hide": false, 2316 "instant": false, 2317 "legendFormat": "p99 overall", 2318 "range": true, 2319 "refId": "Overall p99" 2320 }, 2321 { 2322 "datasource": { 2323 "type": "prometheus", 2324 "uid": "${datasource}" 2325 }, 2326 "editorMode": "code", 2327 "expr": "histogram_quantile(0.90, sum(rate(pyroscope_segment_writer_upload_duration_seconds_bucket{namespace=\"$namespace\",container=\"segment-writer\"}[$__rate_interval])) by (le))", 2328 "hide": false, 2329 "instant": false, 2330 "legendFormat": "p90 overall", 2331 "range": true, 2332 "refId": "Overall p90" 2333 }, 2334 { 2335 "datasource": { 2336 "type": "prometheus", 2337 "uid": "${datasource}" 2338 }, 2339 "editorMode": "code", 2340 "expr": "histogram_quantile(0.50, sum(rate(pyroscope_segment_writer_upload_duration_seconds_bucket{namespace=\"$namespace\",container=\"segment-writer\"}[$__rate_interval])) by (le))", 2341 "hide": false, 2342 "instant": false, 2343 "legendFormat": "p50 overall", 2344 "range": true, 2345 "refId": "Overall p50" 2346 } 2347 ], 2348 "title": "Object Store Upload Latency", 2349 "transparent": true, 2350 "type": "timeseries" 2351 }, 2352 { 2353 "datasource": { 2354 "type": "datasource", 2355 "uid": "-- Mixed --" 2356 }, 2357 "description": "The latency of upload calls observed by the segment-writer", 2358 "fieldConfig": { 2359 "defaults": { 2360 "custom": { 2361 "hideFrom": { 2362 "legend": false, 2363 "tooltip": false, 2364 "viz": false 2365 }, 2366 "scaleDistribution": { 2367 "type": "linear" 2368 } 2369 } 2370 }, 2371 "overrides": [] 2372 }, 2373 "gridPos": { 2374 "h": 8, 2375 "w": 8, 2376 "x": 8, 2377 "y": 29 2378 }, 2379 "id": 80, 2380 "options": { 2381 "calculate": false, 2382 "calculation": { 2383 "xBuckets": { 2384 "mode": "count", 2385 "value": "" 2386 }, 2387 "yBuckets": { 2388 "mode": "count", 2389 "scale": { 2390 "type": "linear" 2391 } 2392 } 2393 }, 2394 "cellGap": 0, 2395 "cellValues": { 2396 "unit": "requests" 2397 }, 2398 "color": { 2399 "exponent": 0.2, 2400 "fill": "super-light-red", 2401 "mode": "scheme", 2402 "reverse": true, 2403 "scale": "exponential", 2404 "scheme": "Turbo", 2405 "steps": 128 2406 }, 2407 "exemplars": { 2408 "color": "orange" 2409 }, 2410 "filterValues": { 2411 "le": 1e-9 2412 }, 2413 "legend": { 2414 "show": false 2415 }, 2416 "rowsFrame": { 2417 "layout": "auto" 2418 }, 2419 "tooltip": { 2420 "mode": "single", 2421 "showColorScale": true, 2422 "yHistogram": true 2423 }, 2424 "yAxis": { 2425 "axisPlacement": "left", 2426 "reverse": false, 2427 "unit": "s" 2428 } 2429 }, 2430 "pluginVersion": "12.1.0-90139", 2431 "targets": [ 2432 { 2433 "datasource": { 2434 "type": "prometheus", 2435 "uid": "${datasource}" 2436 }, 2437 "editorMode": "code", 2438 "expr": "sum(rate(pyroscope_segment_writer_upload_duration_seconds{namespace=\"$namespace\",container=\"segment-writer\"}[$__rate_interval]))", 2439 "hide": false, 2440 "legendFormat": "__auto", 2441 "range": true, 2442 "refId": "B" 2443 } 2444 ], 2445 "title": "Latency Heatmap", 2446 "transparent": true, 2447 "type": "heatmap" 2448 }, 2449 { 2450 "datasource": { 2451 "type": "prometheus", 2452 "uid": "${datasource}" 2453 }, 2454 "description": "New objects created by segment-writer", 2455 "fieldConfig": { 2456 "defaults": { 2457 "color": { 2458 "mode": "palette-classic" 2459 }, 2460 "custom": { 2461 "axisBorderShow": false, 2462 "axisCenteredZero": false, 2463 "axisColorMode": "text", 2464 "axisLabel": "", 2465 "axisPlacement": "auto", 2466 "axisSoftMin": 0, 2467 "barAlignment": 0, 2468 "barWidthFactor": 0.6, 2469 "drawStyle": "line", 2470 "fillOpacity": 0, 2471 "gradientMode": "none", 2472 "hideFrom": { 2473 "legend": false, 2474 "tooltip": false, 2475 "viz": false 2476 }, 2477 "insertNulls": false, 2478 "lineInterpolation": "linear", 2479 "lineWidth": 1, 2480 "pointSize": 5, 2481 "scaleDistribution": { 2482 "type": "linear" 2483 }, 2484 "showPoints": "auto", 2485 "spanNulls": false, 2486 "stacking": { 2487 "group": "A", 2488 "mode": "none" 2489 }, 2490 "thresholdsStyle": { 2491 "mode": "off" 2492 } 2493 }, 2494 "mappings": [], 2495 "thresholds": { 2496 "mode": "absolute", 2497 "steps": [ 2498 { 2499 "color": "green", 2500 "value": 0 2501 }, 2502 { 2503 "color": "red", 2504 "value": 80 2505 } 2506 ] 2507 }, 2508 "unit": "ops" 2509 }, 2510 "overrides": [ 2511 { 2512 "matcher": { 2513 "id": "byFrameRefID", 2514 "options": "B" 2515 }, 2516 "properties": [ 2517 { 2518 "id": "custom.axisPlacement", 2519 "value": "right" 2520 }, 2521 { 2522 "id": "custom.axisLabel", 2523 "value": "Hedged requests" 2524 } 2525 ] 2526 } 2527 ] 2528 }, 2529 "gridPos": { 2530 "h": 8, 2531 "w": 8, 2532 "x": 16, 2533 "y": 29 2534 }, 2535 "id": 59, 2536 "options": { 2537 "legend": { 2538 "calcs": [], 2539 "displayMode": "list", 2540 "placement": "bottom", 2541 "showLegend": true 2542 }, 2543 "tooltip": { 2544 "hideZeros": false, 2545 "mode": "single", 2546 "sort": "none" 2547 } 2548 }, 2549 "pluginVersion": "12.1.0-90139", 2550 "targets": [ 2551 { 2552 "datasource": { 2553 "type": "prometheus", 2554 "uid": "${datasource}" 2555 }, 2556 "editorMode": "code", 2557 "expr": "sum by (status) (rate(pyroscope_segment_writer_upload_duration_seconds_count{namespace=\"$namespace\"}[$__rate_interval]))", 2558 "hide": false, 2559 "instant": false, 2560 "legendFormat": "__auto", 2561 "range": true, 2562 "refId": "A" 2563 }, 2564 { 2565 "datasource": { 2566 "type": "prometheus", 2567 "uid": "${datasource}" 2568 }, 2569 "editorMode": "code", 2570 "expr": "sum by (status) (rate(pyroscope_segment_writer_hedged_upload_duration_seconds_count{namespace=\"$namespace\"}[$__rate_interval]))", 2571 "hide": false, 2572 "instant": false, 2573 "legendFormat": "{{status}} (hedged)", 2574 "range": true, 2575 "refId": "B" 2576 } 2577 ], 2578 "title": "Object Store Upload Rate", 2579 "transparent": true, 2580 "type": "timeseries" 2581 }, 2582 { 2583 "datasource": { 2584 "type": "prometheus", 2585 "uid": "${datasource}" 2586 }, 2587 "description": "The latency segment-writer observes when adding metadata entries to metastore, including retries.", 2588 "fieldConfig": { 2589 "defaults": { 2590 "color": { 2591 "mode": "palette-classic" 2592 }, 2593 "custom": { 2594 "axisBorderShow": false, 2595 "axisCenteredZero": false, 2596 "axisColorMode": "text", 2597 "axisLabel": "", 2598 "axisPlacement": "auto", 2599 "barAlignment": 0, 2600 "barWidthFactor": 0.6, 2601 "drawStyle": "line", 2602 "fillOpacity": 0, 2603 "gradientMode": "none", 2604 "hideFrom": { 2605 "legend": false, 2606 "tooltip": false, 2607 "viz": false 2608 }, 2609 "insertNulls": false, 2610 "lineInterpolation": "linear", 2611 "lineWidth": 1, 2612 "pointSize": 5, 2613 "scaleDistribution": { 2614 "type": "linear" 2615 }, 2616 "showPoints": "auto", 2617 "spanNulls": false, 2618 "stacking": { 2619 "group": "A", 2620 "mode": "none" 2621 }, 2622 "thresholdsStyle": { 2623 "mode": "off" 2624 } 2625 }, 2626 "mappings": [], 2627 "thresholds": { 2628 "mode": "absolute", 2629 "steps": [ 2630 { 2631 "color": "green", 2632 "value": 0 2633 }, 2634 { 2635 "color": "red", 2636 "value": 80 2637 } 2638 ] 2639 }, 2640 "unit": "s" 2641 }, 2642 "overrides": [ 2643 { 2644 "matcher": { 2645 "id": "byFrameRefID", 2646 "options": "Overall p99" 2647 }, 2648 "properties": [ 2649 { 2650 "id": "custom.lineWidth", 2651 "value": 2 2652 }, 2653 { 2654 "id": "color", 2655 "value": { 2656 "fixedColor": "green", 2657 "mode": "fixed" 2658 } 2659 } 2660 ] 2661 }, 2662 { 2663 "matcher": { 2664 "id": "byFrameRefID" 2665 }, 2666 "properties": [] 2667 } 2668 ] 2669 }, 2670 "gridPos": { 2671 "h": 8, 2672 "w": 8, 2673 "x": 0, 2674 "y": 37 2675 }, 2676 "id": 77, 2677 "options": { 2678 "legend": { 2679 "calcs": [], 2680 "displayMode": "list", 2681 "placement": "bottom", 2682 "showLegend": false 2683 }, 2684 "tooltip": { 2685 "hideZeros": false, 2686 "mode": "multi", 2687 "sort": "desc" 2688 } 2689 }, 2690 "pluginVersion": "12.1.0-90139", 2691 "targets": [ 2692 { 2693 "datasource": { 2694 "type": "prometheus", 2695 "uid": "${datasource}" 2696 }, 2697 "editorMode": "code", 2698 "expr": "histogram_quantile(0.99, sum(rate(pyroscope_segment_writer_store_metadata_duration_seconds_bucket{namespace=\"$namespace\",container=\"segment-writer\"}[$__rate_interval])) by (le,pod))", 2699 "hide": false, 2700 "instant": false, 2701 "legendFormat": "p99 {{pod}}", 2702 "range": true, 2703 "refId": "B" 2704 }, 2705 { 2706 "datasource": { 2707 "type": "prometheus", 2708 "uid": "${datasource}" 2709 }, 2710 "editorMode": "code", 2711 "expr": "histogram_quantile(0.99, sum(rate(pyroscope_segment_writer_store_metadata_duration_seconds_bucket{namespace=\"$namespace\",container=\"segment-writer\"}[$__rate_interval])) by (le))", 2712 "hide": false, 2713 "instant": false, 2714 "legendFormat": "p99 overall", 2715 "range": true, 2716 "refId": "Overall p99" 2717 }, 2718 { 2719 "datasource": { 2720 "type": "prometheus", 2721 "uid": "${datasource}" 2722 }, 2723 "editorMode": "code", 2724 "expr": "histogram_quantile(0.90, sum(rate(pyroscope_segment_writer_store_metadata_duration_seconds_bucket{namespace=\"$namespace\",container=\"segment-writer\"}[$__rate_interval])) by (le))", 2725 "hide": false, 2726 "instant": false, 2727 "legendFormat": "p90 overall", 2728 "range": true, 2729 "refId": "Overall p90" 2730 }, 2731 { 2732 "datasource": { 2733 "type": "prometheus", 2734 "uid": "${datasource}" 2735 }, 2736 "editorMode": "code", 2737 "expr": "histogram_quantile(0.50, sum(rate(pyroscope_segment_writer_store_metadata_duration_seconds_bucket{namespace=\"$namespace\",container=\"segment-writer\"}[$__rate_interval])) by (le))", 2738 "hide": false, 2739 "instant": false, 2740 "legendFormat": "p50 overall", 2741 "range": true, 2742 "refId": "Overall p50" 2743 } 2744 ], 2745 "title": "Store Metadata Latency", 2746 "transparent": true, 2747 "type": "timeseries" 2748 }, 2749 { 2750 "datasource": { 2751 "type": "datasource", 2752 "uid": "-- Mixed --" 2753 }, 2754 "description": "The latency segment-writer observes when adding metadata entries, including retries", 2755 "fieldConfig": { 2756 "defaults": { 2757 "custom": { 2758 "hideFrom": { 2759 "legend": false, 2760 "tooltip": false, 2761 "viz": false 2762 }, 2763 "scaleDistribution": { 2764 "type": "linear" 2765 } 2766 } 2767 }, 2768 "overrides": [] 2769 }, 2770 "gridPos": { 2771 "h": 8, 2772 "w": 8, 2773 "x": 8, 2774 "y": 37 2775 }, 2776 "id": 78, 2777 "options": { 2778 "calculate": false, 2779 "calculation": { 2780 "xBuckets": { 2781 "mode": "count", 2782 "value": "" 2783 }, 2784 "yBuckets": { 2785 "mode": "count", 2786 "scale": { 2787 "type": "linear" 2788 } 2789 } 2790 }, 2791 "cellGap": 0, 2792 "cellValues": { 2793 "unit": "requests" 2794 }, 2795 "color": { 2796 "exponent": 0.2, 2797 "fill": "super-light-red", 2798 "mode": "scheme", 2799 "reverse": true, 2800 "scale": "exponential", 2801 "scheme": "Turbo", 2802 "steps": 128 2803 }, 2804 "exemplars": { 2805 "color": "orange" 2806 }, 2807 "filterValues": { 2808 "le": 1e-9 2809 }, 2810 "legend": { 2811 "show": false 2812 }, 2813 "rowsFrame": { 2814 "layout": "auto" 2815 }, 2816 "tooltip": { 2817 "mode": "single", 2818 "showColorScale": true, 2819 "yHistogram": true 2820 }, 2821 "yAxis": { 2822 "axisPlacement": "left", 2823 "reverse": false, 2824 "unit": "s" 2825 } 2826 }, 2827 "pluginVersion": "12.1.0-90139", 2828 "targets": [ 2829 { 2830 "datasource": { 2831 "type": "prometheus", 2832 "uid": "${datasource}" 2833 }, 2834 "editorMode": "code", 2835 "expr": "sum(rate(pyroscope_segment_writer_store_metadata_duration_seconds{namespace=\"$namespace\",container=\"segment-writer\"}[$__rate_interval]))", 2836 "hide": false, 2837 "legendFormat": "__auto", 2838 "range": true, 2839 "refId": "B" 2840 } 2841 ], 2842 "title": "Latency Heatmap", 2843 "transparent": true, 2844 "type": "heatmap" 2845 }, 2846 { 2847 "datasource": { 2848 "type": "prometheus", 2849 "uid": "${datasource}" 2850 }, 2851 "description": "New metadata entries created by segment-writers", 2852 "fieldConfig": { 2853 "defaults": { 2854 "color": { 2855 "mode": "palette-classic" 2856 }, 2857 "custom": { 2858 "axisBorderShow": false, 2859 "axisCenteredZero": false, 2860 "axisColorMode": "text", 2861 "axisLabel": "", 2862 "axisPlacement": "auto", 2863 "axisSoftMin": 0, 2864 "barAlignment": 0, 2865 "barWidthFactor": 0.6, 2866 "drawStyle": "line", 2867 "fillOpacity": 0, 2868 "gradientMode": "none", 2869 "hideFrom": { 2870 "legend": false, 2871 "tooltip": false, 2872 "viz": false 2873 }, 2874 "insertNulls": false, 2875 "lineInterpolation": "linear", 2876 "lineWidth": 1, 2877 "pointSize": 5, 2878 "scaleDistribution": { 2879 "type": "linear" 2880 }, 2881 "showPoints": "auto", 2882 "spanNulls": false, 2883 "stacking": { 2884 "group": "A", 2885 "mode": "none" 2886 }, 2887 "thresholdsStyle": { 2888 "mode": "off" 2889 } 2890 }, 2891 "mappings": [], 2892 "thresholds": { 2893 "mode": "absolute", 2894 "steps": [ 2895 { 2896 "color": "green", 2897 "value": 0 2898 }, 2899 { 2900 "color": "red", 2901 "value": 80 2902 } 2903 ] 2904 }, 2905 "unit": "ops" 2906 }, 2907 "overrides": [ 2908 { 2909 "matcher": { 2910 "id": "byFrameRefID", 2911 "options": "B" 2912 }, 2913 "properties": [ 2914 { 2915 "id": "custom.drawStyle", 2916 "value": "points" 2917 }, 2918 { 2919 "id": "color", 2920 "value": { 2921 "fixedColor": "red", 2922 "mode": "fixed" 2923 } 2924 }, 2925 { 2926 "id": "custom.pointSize", 2927 "value": 11 2928 } 2929 ] 2930 } 2931 ] 2932 }, 2933 "gridPos": { 2934 "h": 8, 2935 "w": 8, 2936 "x": 16, 2937 "y": 37 2938 }, 2939 "id": 73, 2940 "options": { 2941 "legend": { 2942 "calcs": [], 2943 "displayMode": "list", 2944 "placement": "bottom", 2945 "showLegend": true 2946 }, 2947 "tooltip": { 2948 "hideZeros": false, 2949 "mode": "single", 2950 "sort": "none" 2951 } 2952 }, 2953 "pluginVersion": "12.1.0-90139", 2954 "targets": [ 2955 { 2956 "datasource": { 2957 "type": "prometheus", 2958 "uid": "${datasource}" 2959 }, 2960 "editorMode": "code", 2961 "expr": "sum by (status) (rate(pyroscope_segment_writer_store_metadata_duration_seconds_count{namespace=\"$namespace\"}[$__rate_interval]))", 2962 "hide": false, 2963 "instant": false, 2964 "legendFormat": "__auto", 2965 "range": true, 2966 "refId": "A" 2967 }, 2968 { 2969 "datasource": { 2970 "type": "prometheus", 2971 "uid": "${datasource}" 2972 }, 2973 "editorMode": "code", 2974 "expr": "sum by (status) (rate(pyroscope_segment_writer_store_metadata_dql{namespace=\"$namespace\"}[$__rate_interval]))", 2975 "hide": false, 2976 "instant": false, 2977 "legendFormat": "{{status}} (DLQ)", 2978 "range": true, 2979 "refId": "B" 2980 }, 2981 { 2982 "datasource": { 2983 "type": "prometheus", 2984 "uid": "${datasource}" 2985 }, 2986 "editorMode": "code", 2987 "expr": "sum(rate(pyroscope_segment_writer_store_metadata_dlq{namespace=\"$namespace\"}[$__rate_interval]))", 2988 "hide": false, 2989 "instant": false, 2990 "legendFormat": "dlq", 2991 "range": true, 2992 "refId": "C" 2993 } 2994 ], 2995 "title": "Store Metadata Rate", 2996 "transparent": true, 2997 "type": "timeseries" 2998 }, 2999 { 3000 "datasource": { 3001 "default": false, 3002 "type": "prometheus", 3003 "uid": "${datasource}" 3004 }, 3005 "fieldConfig": { 3006 "defaults": { 3007 "color": { 3008 "mode": "palette-classic" 3009 }, 3010 "custom": { 3011 "axisBorderShow": false, 3012 "axisCenteredZero": false, 3013 "axisColorMode": "text", 3014 "axisLabel": "", 3015 "axisPlacement": "auto", 3016 "axisSoftMin": 0, 3017 "barAlignment": 0, 3018 "barWidthFactor": 0.6, 3019 "drawStyle": "line", 3020 "fillOpacity": 0, 3021 "gradientMode": "none", 3022 "hideFrom": { 3023 "legend": false, 3024 "tooltip": false, 3025 "viz": false 3026 }, 3027 "insertNulls": false, 3028 "lineInterpolation": "linear", 3029 "lineWidth": 1, 3030 "pointSize": 5, 3031 "scaleDistribution": { 3032 "type": "linear" 3033 }, 3034 "showPoints": "auto", 3035 "spanNulls": false, 3036 "stacking": { 3037 "group": "A", 3038 "mode": "none" 3039 }, 3040 "thresholdsStyle": { 3041 "mode": "off" 3042 } 3043 }, 3044 "mappings": [], 3045 "thresholds": { 3046 "mode": "absolute", 3047 "steps": [ 3048 { 3049 "color": "green", 3050 "value": 0 3051 }, 3052 { 3053 "color": "red", 3054 "value": 80 3055 } 3056 ] 3057 }, 3058 "unit": "bytes" 3059 }, 3060 "overrides": [] 3061 }, 3062 "gridPos": { 3063 "h": 8, 3064 "w": 8, 3065 "x": 0, 3066 "y": 45 3067 }, 3068 "id": 21, 3069 "options": { 3070 "legend": { 3071 "calcs": [ 3072 "mean" 3073 ], 3074 "displayMode": "table", 3075 "placement": "right", 3076 "showLegend": false 3077 }, 3078 "tooltip": { 3079 "hideZeros": false, 3080 "mode": "multi", 3081 "sort": "desc" 3082 } 3083 }, 3084 "pluginVersion": "12.1.0-90139", 3085 "targets": [ 3086 { 3087 "datasource": { 3088 "type": "prometheus", 3089 "uid": "${datasource}" 3090 }, 3091 "editorMode": "code", 3092 "expr": "sum by (pod) (irate(pyroscope_segment_writer_segment_ingest_bytes_sum{namespace=\"$namespace\", container=\"segment-writer\"}[10m])) /\nsum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", container=\"segment-writer\"}[10m]))", 3093 "legendFormat": "__auto", 3094 "range": true, 3095 "refId": "A" 3096 } 3097 ], 3098 "title": "Processing Rate (MB/sec/core)", 3099 "transparent": true, 3100 "type": "timeseries" 3101 }, 3102 { 3103 "datasource": { 3104 "type": "datasource", 3105 "uid": "-- Mixed --" 3106 }, 3107 "fieldConfig": { 3108 "defaults": { 3109 "color": { 3110 "mode": "palette-classic" 3111 }, 3112 "custom": { 3113 "axisBorderShow": false, 3114 "axisCenteredZero": false, 3115 "axisColorMode": "text", 3116 "axisLabel": "", 3117 "axisPlacement": "auto", 3118 "axisSoftMin": 0, 3119 "barAlignment": 1, 3120 "barWidthFactor": 1, 3121 "drawStyle": "line", 3122 "fillOpacity": 0, 3123 "gradientMode": "none", 3124 "hideFrom": { 3125 "legend": false, 3126 "tooltip": false, 3127 "viz": false 3128 }, 3129 "insertNulls": false, 3130 "lineInterpolation": "linear", 3131 "lineStyle": { 3132 "fill": "solid" 3133 }, 3134 "lineWidth": 1, 3135 "pointSize": 5, 3136 "scaleDistribution": { 3137 "type": "linear" 3138 }, 3139 "showPoints": "auto", 3140 "spanNulls": false, 3141 "stacking": { 3142 "group": "A", 3143 "mode": "none" 3144 }, 3145 "thresholdsStyle": { 3146 "mode": "off" 3147 } 3148 }, 3149 "fieldMinMax": false, 3150 "mappings": [], 3151 "min": 0, 3152 "thresholds": { 3153 "mode": "absolute", 3154 "steps": [ 3155 { 3156 "color": "green", 3157 "value": 0 3158 } 3159 ] 3160 } 3161 }, 3162 "overrides": [ 3163 { 3164 "matcher": { 3165 "id": "byName", 3166 "options": "Limit" 3167 }, 3168 "properties": [ 3169 { 3170 "id": "custom.lineStyle", 3171 "value": { 3172 "dash": [ 3173 10, 3174 10 3175 ], 3176 "fill": "dash" 3177 } 3178 } 3179 ] 3180 }, 3181 { 3182 "matcher": { 3183 "id": "byFrameRefID", 3184 "options": "A" 3185 }, 3186 "properties": [ 3187 { 3188 "id": "custom.axisColorMode", 3189 "value": "text" 3190 }, 3191 { 3192 "id": "color", 3193 "value": { 3194 "fixedColor": "semi-dark-red", 3195 "mode": "fixed" 3196 } 3197 }, 3198 { 3199 "id": "custom.lineWidth", 3200 "value": 2 3201 } 3202 ] 3203 } 3204 ] 3205 }, 3206 "gridPos": { 3207 "h": 8, 3208 "w": 8, 3209 "x": 8, 3210 "y": 45 3211 }, 3212 "id": 63, 3213 "options": { 3214 "legend": { 3215 "calcs": [ 3216 "last", 3217 "max" 3218 ], 3219 "displayMode": "table", 3220 "placement": "bottom", 3221 "showLegend": false, 3222 "sortBy": "Max", 3223 "sortDesc": true 3224 }, 3225 "tooltip": { 3226 "hideZeros": false, 3227 "mode": "multi", 3228 "sort": "desc" 3229 } 3230 }, 3231 "pluginVersion": "12.1.0-90139", 3232 "targets": [ 3233 { 3234 "datasource": { 3235 "type": "prometheus", 3236 "uid": "${datasource}" 3237 }, 3238 "editorMode": "code", 3239 "expr": "sum by (pod) (irate(container_cpu_usage_seconds_total{namespace=\"$namespace\",container=\"segment-writer\"}[$__rate_interval]))", 3240 "legendFormat": "__auto", 3241 "range": true, 3242 "refId": "by pod" 3243 }, 3244 { 3245 "datasource": { 3246 "type": "prometheus", 3247 "uid": "${datasource}" 3248 }, 3249 "editorMode": "code", 3250 "expr": "max(\n kube_pod_container_resource_requests{job=~\"(.*/)?kube-state-metrics\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".*-segment-writer\"}\n)", 3251 "hide": false, 3252 "legendFormat": "Request", 3253 "range": true, 3254 "refId": "A" 3255 }, 3256 { 3257 "datasource": { 3258 "type": "prometheus", 3259 "uid": "${datasource}" 3260 }, 3261 "editorMode": "code", 3262 "expr": "max(\n kube_pod_container_resource_limits{job=~\"(.*/)?kube-state-metrics\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".*-segment-writer\"}\n)", 3263 "hide": false, 3264 "legendFormat": "Limit", 3265 "range": true, 3266 "refId": "B" 3267 } 3268 ], 3269 "title": "CPU Usage", 3270 "transparent": true, 3271 "type": "timeseries" 3272 }, 3273 { 3274 "datasource": { 3275 "default": false, 3276 "type": "prometheus", 3277 "uid": "${datasource}" 3278 }, 3279 "fieldConfig": { 3280 "defaults": { 3281 "color": { 3282 "mode": "palette-classic" 3283 }, 3284 "custom": { 3285 "axisBorderShow": false, 3286 "axisCenteredZero": false, 3287 "axisColorMode": "text", 3288 "axisLabel": "", 3289 "axisPlacement": "auto", 3290 "axisSoftMin": 0, 3291 "barAlignment": 0, 3292 "barWidthFactor": 0.6, 3293 "drawStyle": "line", 3294 "fillOpacity": 7, 3295 "gradientMode": "none", 3296 "hideFrom": { 3297 "legend": false, 3298 "tooltip": false, 3299 "viz": false 3300 }, 3301 "insertNulls": false, 3302 "lineInterpolation": "linear", 3303 "lineWidth": 1, 3304 "pointSize": 5, 3305 "scaleDistribution": { 3306 "type": "linear" 3307 }, 3308 "showPoints": "auto", 3309 "spanNulls": false, 3310 "stacking": { 3311 "group": "A", 3312 "mode": "normal" 3313 }, 3314 "thresholdsStyle": { 3315 "mode": "off" 3316 } 3317 }, 3318 "mappings": [], 3319 "thresholds": { 3320 "mode": "absolute", 3321 "steps": [ 3322 { 3323 "color": "green", 3324 "value": 0 3325 }, 3326 { 3327 "color": "red", 3328 "value": 80 3329 } 3330 ] 3331 }, 3332 "unit": "short" 3333 }, 3334 "overrides": [ 3335 { 3336 "matcher": { 3337 "id": "byName", 3338 "options": "Total" 3339 }, 3340 "properties": [ 3341 { 3342 "id": "custom.axisPlacement", 3343 "value": "right" 3344 }, 3345 { 3346 "id": "custom.lineWidth", 3347 "value": 2 3348 }, 3349 { 3350 "id": "color", 3351 "value": { 3352 "fixedColor": "red", 3353 "mode": "fixed" 3354 } 3355 } 3356 ] 3357 } 3358 ] 3359 }, 3360 "gridPos": { 3361 "h": 8, 3362 "w": 8, 3363 "x": 16, 3364 "y": 45 3365 }, 3366 "id": 27, 3367 "options": { 3368 "legend": { 3369 "calcs": [ 3370 "mean" 3371 ], 3372 "displayMode": "table", 3373 "placement": "right", 3374 "showLegend": false, 3375 "sortBy": "Mean", 3376 "sortDesc": true 3377 }, 3378 "tooltip": { 3379 "hideZeros": false, 3380 "mode": "single", 3381 "sort": "none" 3382 } 3383 }, 3384 "pluginVersion": "12.1.0-90139", 3385 "targets": [ 3386 { 3387 "datasource": { 3388 "type": "prometheus", 3389 "uid": "${datasource}" 3390 }, 3391 "editorMode": "code", 3392 "expr": "sum(irate(container_cpu_usage_seconds_total{namespace=\"$namespace\",container=\"segment-writer\"}[$__rate_interval])) by (pod)", 3393 "legendFormat": "__auto", 3394 "range": true, 3395 "refId": "A" 3396 }, 3397 { 3398 "datasource": { 3399 "type": "prometheus", 3400 "uid": "${datasource}" 3401 }, 3402 "editorMode": "code", 3403 "expr": "sum(irate(container_cpu_usage_seconds_total{namespace=\"$namespace\",container=\"segment-writer\"}[$__rate_interval]))", 3404 "hide": false, 3405 "legendFormat": "Total", 3406 "range": true, 3407 "refId": "B" 3408 } 3409 ], 3410 "title": "CPU Usage (stack)", 3411 "transparent": true, 3412 "type": "timeseries" 3413 }, 3414 { 3415 "collapsed": true, 3416 "gridPos": { 3417 "h": 1, 3418 "w": 24, 3419 "x": 0, 3420 "y": 53 3421 }, 3422 "id": 51, 3423 "panels": [ 3424 { 3425 "datasource": { 3426 "type": "prometheus", 3427 "uid": "${datasource}" 3428 }, 3429 "description": "The latency of raft commands observed by the metastore leader", 3430 "fieldConfig": { 3431 "defaults": { 3432 "color": { 3433 "mode": "continuous-GrYlRd" 3434 }, 3435 "custom": { 3436 "axisPlacement": "auto", 3437 "fillOpacity": 15, 3438 "hideFrom": { 3439 "legend": false, 3440 "tooltip": false, 3441 "viz": false 3442 }, 3443 "insertNulls": false, 3444 "lineWidth": 1, 3445 "spanNulls": false 3446 }, 3447 "fieldMinMax": false, 3448 "mappings": [ 3449 { 3450 "options": { 3451 "from": 0, 3452 "result": { 3453 "color": "green", 3454 "index": 0, 3455 "text": "Follower" 3456 }, 3457 "to": 99 3458 }, 3459 "type": "range" 3460 }, 3461 { 3462 "options": { 3463 "from": 100, 3464 "result": { 3465 "color": "dark-red", 3466 "index": 1, 3467 "text": "Leader" 3468 }, 3469 "to": 1000000 3470 }, 3471 "type": "range" 3472 } 3473 ], 3474 "thresholds": { 3475 "mode": "absolute", 3476 "steps": [ 3477 { 3478 "color": "dark-blue", 3479 "value": 0 3480 }, 3481 { 3482 "color": "light-red", 3483 "value": 100 3484 } 3485 ] 3486 }, 3487 "unit": "short" 3488 }, 3489 "overrides": [] 3490 }, 3491 "gridPos": { 3492 "h": 5, 3493 "w": 24, 3494 "x": 0, 3495 "y": 92 3496 }, 3497 "id": 65, 3498 "options": { 3499 "alignValue": "center", 3500 "legend": { 3501 "displayMode": "list", 3502 "placement": "bottom", 3503 "showLegend": false 3504 }, 3505 "mergeValues": true, 3506 "rowHeight": 0.8, 3507 "showValue": "never", 3508 "tooltip": { 3509 "hideZeros": false, 3510 "mode": "multi", 3511 "sort": "asc" 3512 } 3513 }, 3514 "pluginVersion": "12.1.0-90058", 3515 "targets": [ 3516 { 3517 "datasource": { 3518 "type": "prometheus", 3519 "uid": "${datasource}" 3520 }, 3521 "editorMode": "code", 3522 "exemplar": false, 3523 "expr": "sum by (pod) (\n pyroscope_metastore_raft_state{namespace=\"$namespace\",state=\"Follower\"} or\n pyroscope_metastore_raft_state{namespace=\"$namespace\",state=\"Leader\"} * 100\n)", 3524 "format": "time_series", 3525 "hide": false, 3526 "instant": false, 3527 "legendFormat": "{{label_name}}", 3528 "range": true, 3529 "refId": "D" 3530 } 3531 ], 3532 "title": "Raft Nodes", 3533 "transparent": true, 3534 "type": "state-timeline" 3535 }, 3536 { 3537 "datasource": { 3538 "type": "prometheus", 3539 "uid": "${datasource}" 3540 }, 3541 "description": "The latency of AddBlock endpoint observed by metastore nodes", 3542 "fieldConfig": { 3543 "defaults": { 3544 "color": { 3545 "mode": "palette-classic" 3546 }, 3547 "custom": { 3548 "axisBorderShow": false, 3549 "axisCenteredZero": false, 3550 "axisColorMode": "text", 3551 "axisLabel": "", 3552 "axisPlacement": "auto", 3553 "barAlignment": 0, 3554 "barWidthFactor": 0.6, 3555 "drawStyle": "line", 3556 "fillOpacity": 0, 3557 "gradientMode": "none", 3558 "hideFrom": { 3559 "legend": false, 3560 "tooltip": false, 3561 "viz": false 3562 }, 3563 "insertNulls": false, 3564 "lineInterpolation": "linear", 3565 "lineWidth": 1, 3566 "pointSize": 5, 3567 "scaleDistribution": { 3568 "type": "linear" 3569 }, 3570 "showPoints": "auto", 3571 "spanNulls": false, 3572 "stacking": { 3573 "group": "A", 3574 "mode": "none" 3575 }, 3576 "thresholdsStyle": { 3577 "mode": "off" 3578 } 3579 }, 3580 "mappings": [], 3581 "thresholds": { 3582 "mode": "absolute", 3583 "steps": [ 3584 { 3585 "color": "green", 3586 "value": 0 3587 }, 3588 { 3589 "color": "red", 3590 "value": 80 3591 } 3592 ] 3593 }, 3594 "unit": "s" 3595 }, 3596 "overrides": [ 3597 { 3598 "matcher": { 3599 "id": "byFrameRefID", 3600 "options": "B" 3601 }, 3602 "properties": [ 3603 { 3604 "id": "custom.drawStyle", 3605 "value": "points" 3606 }, 3607 { 3608 "id": "color", 3609 "value": { 3610 "fixedColor": "red", 3611 "mode": "fixed" 3612 } 3613 }, 3614 { 3615 "id": "custom.pointSize", 3616 "value": 11 3617 }, 3618 { 3619 "id": "custom.axisPlacement", 3620 "value": "right" 3621 } 3622 ] 3623 } 3624 ] 3625 }, 3626 "gridPos": { 3627 "h": 8, 3628 "w": 12, 3629 "x": 0, 3630 "y": 97 3631 }, 3632 "id": 49, 3633 "options": { 3634 "legend": { 3635 "calcs": [], 3636 "displayMode": "list", 3637 "placement": "bottom", 3638 "showLegend": true 3639 }, 3640 "tooltip": { 3641 "hideZeros": false, 3642 "mode": "single", 3643 "sort": "none" 3644 } 3645 }, 3646 "pluginVersion": "12.1.0-90058", 3647 "targets": [ 3648 { 3649 "datasource": { 3650 "type": "prometheus", 3651 "uid": "${datasource}" 3652 }, 3653 "editorMode": "code", 3654 "expr": "(\n histogram_quantile(0.99,(sum by (le, pod) (\n irate(pyroscope_metastore_boltdb_persist_snapshot_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])))\n ) * on(pod) group_left() (pyroscope_metastore_raft_state{namespace=\"$namespace\", state=\"Leader\"} \u003e= 1) \n)", 3655 "hide": false, 3656 "instant": false, 3657 "legendFormat": "Metastore leader snapshot ({{pod}})", 3658 "range": true, 3659 "refId": "B" 3660 }, 3661 { 3662 "datasource": { 3663 "type": "prometheus", 3664 "uid": "${datasource}" 3665 }, 3666 "editorMode": "code", 3667 "expr": "histogram_quantile(0.99, sum(rate(pyroscope_request_duration_seconds_bucket{\n namespace=\"$namespace\",\n method=\"gRPC\",\n route=\"/metastore.v1.IndexService/AddBlock\",\n}[$__rate_interval])) by (le))", 3668 "hide": false, 3669 "instant": false, 3670 "legendFormat": "p99", 3671 "range": true, 3672 "refId": "D" 3673 }, 3674 { 3675 "datasource": { 3676 "type": "prometheus", 3677 "uid": "${datasource}" 3678 }, 3679 "editorMode": "code", 3680 "expr": "histogram_quantile(0.95, sum(rate(pyroscope_request_duration_seconds_bucket{\n namespace=\"$namespace\",\n method=\"gRPC\",\n route=\"/metastore.v1.IndexService/AddBlock\",\n}[$__rate_interval])) by (le))", 3681 "hide": false, 3682 "instant": false, 3683 "legendFormat": "p95", 3684 "range": true, 3685 "refId": "A" 3686 } 3687 ], 3688 "title": "Latency", 3689 "transparent": true, 3690 "type": "timeseries" 3691 }, 3692 { 3693 "datasource": { 3694 "type": "prometheus", 3695 "uid": "${datasource}" 3696 }, 3697 "fieldConfig": { 3698 "defaults": { 3699 "color": { 3700 "mode": "palette-classic" 3701 }, 3702 "custom": { 3703 "axisBorderShow": false, 3704 "axisCenteredZero": false, 3705 "axisColorMode": "text", 3706 "axisLabel": "", 3707 "axisPlacement": "auto", 3708 "barAlignment": 0, 3709 "barWidthFactor": 0.6, 3710 "drawStyle": "line", 3711 "fillOpacity": 0, 3712 "gradientMode": "none", 3713 "hideFrom": { 3714 "legend": false, 3715 "tooltip": false, 3716 "viz": false 3717 }, 3718 "insertNulls": false, 3719 "lineInterpolation": "linear", 3720 "lineWidth": 1, 3721 "pointSize": 5, 3722 "scaleDistribution": { 3723 "type": "linear" 3724 }, 3725 "showPoints": "auto", 3726 "spanNulls": false, 3727 "stacking": { 3728 "group": "A", 3729 "mode": "none" 3730 }, 3731 "thresholdsStyle": { 3732 "mode": "off" 3733 } 3734 }, 3735 "mappings": [], 3736 "min": 0, 3737 "thresholds": { 3738 "mode": "absolute", 3739 "steps": [ 3740 { 3741 "color": "green", 3742 "value": 0 3743 }, 3744 { 3745 "color": "red", 3746 "value": 80 3747 } 3748 ] 3749 }, 3750 "unit": "ops" 3751 }, 3752 "overrides": [ 3753 { 3754 "matcher": { 3755 "id": "byName", 3756 "options": "error" 3757 }, 3758 "properties": [ 3759 { 3760 "id": "color", 3761 "value": { 3762 "fixedColor": "red", 3763 "mode": "fixed" 3764 } 3765 } 3766 ] 3767 }, 3768 { 3769 "matcher": { 3770 "id": "byName", 3771 "options": "success" 3772 }, 3773 "properties": [ 3774 { 3775 "id": "color", 3776 "value": { 3777 "fixedColor": "green", 3778 "mode": "fixed" 3779 } 3780 } 3781 ] 3782 } 3783 ] 3784 }, 3785 "gridPos": { 3786 "h": 8, 3787 "w": 12, 3788 "x": 12, 3789 "y": 97 3790 }, 3791 "id": 50, 3792 "options": { 3793 "legend": { 3794 "calcs": [], 3795 "displayMode": "list", 3796 "placement": "bottom", 3797 "showLegend": true 3798 }, 3799 "tooltip": { 3800 "hideZeros": false, 3801 "mode": "single", 3802 "sort": "none" 3803 } 3804 }, 3805 "pluginVersion": "12.1.0-90058", 3806 "targets": [ 3807 { 3808 "datasource": { 3809 "type": "prometheus", 3810 "uid": "${datasource}" 3811 }, 3812 "editorMode": "code", 3813 "expr": "sum(rate(pyroscope_request_duration_seconds_count{\n namespace=\"$namespace\",\n method=\"gRPC\",\n container=\"metastore\",\n route=\"/metastore.v1.IndexService/AddBlock\",\n}[$__rate_interval])) by (status_code)", 3814 "hide": false, 3815 "instant": false, 3816 "legendFormat": "{{status_code}}", 3817 "range": true, 3818 "refId": "D" 3819 } 3820 ], 3821 "title": "", 3822 "transparent": true, 3823 "type": "timeseries" 3824 } 3825 ], 3826 "title": "Metastore", 3827 "type": "row" 3828 }, 3829 { 3830 "collapsed": true, 3831 "gridPos": { 3832 "h": 1, 3833 "w": 24, 3834 "x": 0, 3835 "y": 54 3836 }, 3837 "id": 22, 3838 "panels": [ 3839 { 3840 "datasource": { 3841 "type": "prometheus", 3842 "uid": "${datasource}" 3843 }, 3844 "fieldConfig": { 3845 "defaults": { 3846 "color": { 3847 "mode": "continuous-BlYlRd" 3848 }, 3849 "custom": { 3850 "axisBorderShow": false, 3851 "axisCenteredZero": false, 3852 "axisColorMode": "text", 3853 "axisLabel": "", 3854 "axisPlacement": "auto", 3855 "axisSoftMin": 0, 3856 "barAlignment": 0, 3857 "barWidthFactor": 0.6, 3858 "drawStyle": "line", 3859 "fillOpacity": 3, 3860 "gradientMode": "none", 3861 "hideFrom": { 3862 "legend": false, 3863 "tooltip": false, 3864 "viz": false 3865 }, 3866 "insertNulls": false, 3867 "lineInterpolation": "linear", 3868 "lineWidth": 1, 3869 "pointSize": 5, 3870 "scaleDistribution": { 3871 "type": "linear" 3872 }, 3873 "showPoints": "auto", 3874 "spanNulls": false, 3875 "stacking": { 3876 "group": "A", 3877 "mode": "none" 3878 }, 3879 "thresholdsStyle": { 3880 "mode": "off" 3881 } 3882 }, 3883 "mappings": [], 3884 "thresholds": { 3885 "mode": "absolute", 3886 "steps": [ 3887 { 3888 "color": "green", 3889 "value": 0 3890 }, 3891 { 3892 "color": "red", 3893 "value": 80 3894 } 3895 ] 3896 } 3897 }, 3898 "overrides": [] 3899 }, 3900 "gridPos": { 3901 "h": 9, 3902 "w": 17, 3903 "x": 0, 3904 "y": 55 3905 }, 3906 "id": 44, 3907 "options": { 3908 "legend": { 3909 "calcs": [ 3910 "lastNotNull", 3911 "mean" 3912 ], 3913 "displayMode": "table", 3914 "placement": "right", 3915 "showLegend": true, 3916 "sortBy": "Last *", 3917 "sortDesc": true 3918 }, 3919 "tooltip": { 3920 "hideZeros": false, 3921 "mode": "single", 3922 "sort": "none" 3923 } 3924 }, 3925 "pluginVersion": "12.1.0-90058", 3926 "targets": [ 3927 { 3928 "datasource": { 3929 "type": "prometheus", 3930 "uid": "${datasource}" 3931 }, 3932 "editorMode": "code", 3933 "expr": "sum by (dataset, tenant) (\n pyroscope_adaptive_placement_dataset_shard_limit{namespace=\"$namespace\"}\n) \u003e 1", 3934 "legendFormat": "{{tenant}}/{{dataset}}", 3935 "range": true, 3936 "refId": "A" 3937 } 3938 ], 3939 "title": "Dataset Shards", 3940 "transparent": true, 3941 "type": "timeseries" 3942 }, 3943 { 3944 "datasource": { 3945 "type": "prometheus", 3946 "uid": "${datasource}" 3947 }, 3948 "description": "Top-10", 3949 "fieldConfig": { 3950 "defaults": { 3951 "color": { 3952 "mode": "continuous-BlYlRd" 3953 }, 3954 "custom": { 3955 "axisBorderShow": false, 3956 "axisCenteredZero": false, 3957 "axisColorMode": "text", 3958 "axisLabel": "", 3959 "axisPlacement": "auto", 3960 "axisSoftMin": 0, 3961 "barAlignment": 0, 3962 "barWidthFactor": 0.6, 3963 "drawStyle": "line", 3964 "fillOpacity": 3, 3965 "gradientMode": "none", 3966 "hideFrom": { 3967 "legend": false, 3968 "tooltip": false, 3969 "viz": false 3970 }, 3971 "insertNulls": false, 3972 "lineInterpolation": "linear", 3973 "lineWidth": 1, 3974 "pointSize": 5, 3975 "scaleDistribution": { 3976 "type": "linear" 3977 }, 3978 "showPoints": "auto", 3979 "spanNulls": false, 3980 "stacking": { 3981 "group": "A", 3982 "mode": "none" 3983 }, 3984 "thresholdsStyle": { 3985 "mode": "off" 3986 } 3987 }, 3988 "mappings": [], 3989 "thresholds": { 3990 "mode": "absolute", 3991 "steps": [ 3992 { 3993 "color": "green", 3994 "value": 0 3995 }, 3996 { 3997 "color": "red", 3998 "value": 80 3999 } 4000 ] 4001 } 4002 }, 4003 "overrides": [] 4004 }, 4005 "gridPos": { 4006 "h": 9, 4007 "w": 7, 4008 "x": 17, 4009 "y": 55 4010 }, 4011 "id": 107, 4012 "options": { 4013 "legend": { 4014 "calcs": [ 4015 "lastNotNull" 4016 ], 4017 "displayMode": "table", 4018 "placement": "right", 4019 "showLegend": true, 4020 "sortBy": "Last *", 4021 "sortDesc": true 4022 }, 4023 "tooltip": { 4024 "hideZeros": false, 4025 "mode": "single", 4026 "sort": "none" 4027 } 4028 }, 4029 "pluginVersion": "12.1.0-90058", 4030 "targets": [ 4031 { 4032 "datasource": { 4033 "type": "prometheus", 4034 "uid": "${datasource}" 4035 }, 4036 "editorMode": "code", 4037 "expr": "topk(10, count by (tenant) (sum by (dataset, tenant) (\n pyroscope_adaptive_placement_dataset_shard_limit{namespace=\"$namespace\"}\n)))", 4038 "legendFormat": "{{tenant}}", 4039 "range": true, 4040 "refId": "A" 4041 } 4042 ], 4043 "title": "Datasets per tenant", 4044 "transparent": true, 4045 "type": "timeseries" 4046 }, 4047 { 4048 "datasource": { 4049 "default": false, 4050 "type": "prometheus", 4051 "uid": "${datasource}" 4052 }, 4053 "fieldConfig": { 4054 "defaults": { 4055 "custom": { 4056 "hideFrom": { 4057 "legend": false, 4058 "tooltip": false, 4059 "viz": false 4060 }, 4061 "scaleDistribution": { 4062 "type": "linear" 4063 } 4064 } 4065 }, 4066 "overrides": [] 4067 }, 4068 "gridPos": { 4069 "h": 11, 4070 "w": 24, 4071 "x": 0, 4072 "y": 64 4073 }, 4074 "id": 23, 4075 "options": { 4076 "calculate": false, 4077 "cellGap": 0, 4078 "cellValues": { 4079 "unit": "Bps" 4080 }, 4081 "color": { 4082 "exponent": 0.5, 4083 "fill": "red", 4084 "mode": "scheme", 4085 "reverse": false, 4086 "scale": "exponential", 4087 "scheme": "Rainbow", 4088 "steps": 64 4089 }, 4090 "exemplars": { 4091 "color": "rgba(255,0,255,0.7)" 4092 }, 4093 "filterValues": { 4094 "le": 1e-9 4095 }, 4096 "legend": { 4097 "show": true 4098 }, 4099 "rowsFrame": { 4100 "layout": "auto", 4101 "value": "Shard" 4102 }, 4103 "tooltip": { 4104 "mode": "single", 4105 "showColorScale": true, 4106 "yHistogram": true 4107 }, 4108 "yAxis": { 4109 "axisLabel": "Shard", 4110 "axisPlacement": "right", 4111 "reverse": true, 4112 "unit": "none" 4113 } 4114 }, 4115 "pluginVersion": "12.1.0-90058", 4116 "targets": [ 4117 { 4118 "datasource": { 4119 "type": "prometheus", 4120 "uid": "${datasource}" 4121 }, 4122 "editorMode": "code", 4123 "expr": "sum by (shard) (rate(pyroscope_segment_writer_client_sent_bytes_sum{namespace=\"$namespace\"}[$__rate_interval]))", 4124 "hide": false, 4125 "instant": false, 4126 "legendFormat": "__auto", 4127 "range": true, 4128 "refId": "B" 4129 } 4130 ], 4131 "title": "Distributed Bytes (per shard)", 4132 "transparent": true, 4133 "type": "heatmap" 4134 }, 4135 { 4136 "datasource": { 4137 "default": false, 4138 "type": "prometheus", 4139 "uid": "${datasource}" 4140 }, 4141 "fieldConfig": { 4142 "defaults": { 4143 "color": { 4144 "mode": "palette-classic" 4145 }, 4146 "custom": { 4147 "axisBorderShow": false, 4148 "axisCenteredZero": false, 4149 "axisColorMode": "text", 4150 "axisLabel": "", 4151 "axisPlacement": "auto", 4152 "axisSoftMin": 0, 4153 "barAlignment": 0, 4154 "barWidthFactor": 0.6, 4155 "drawStyle": "line", 4156 "fillOpacity": 0, 4157 "gradientMode": "none", 4158 "hideFrom": { 4159 "legend": false, 4160 "tooltip": false, 4161 "viz": false 4162 }, 4163 "insertNulls": false, 4164 "lineInterpolation": "linear", 4165 "lineWidth": 1, 4166 "pointSize": 5, 4167 "scaleDistribution": { 4168 "type": "linear" 4169 }, 4170 "showPoints": "auto", 4171 "spanNulls": false, 4172 "stacking": { 4173 "group": "A", 4174 "mode": "none" 4175 }, 4176 "thresholdsStyle": { 4177 "mode": "off" 4178 } 4179 }, 4180 "mappings": [], 4181 "thresholds": { 4182 "mode": "absolute", 4183 "steps": [ 4184 { 4185 "color": "green", 4186 "value": 0 4187 }, 4188 { 4189 "color": "red", 4190 "value": 80 4191 } 4192 ] 4193 }, 4194 "unit": "bytes" 4195 }, 4196 "overrides": [] 4197 }, 4198 "gridPos": { 4199 "h": 9, 4200 "w": 12, 4201 "x": 0, 4202 "y": 75 4203 }, 4204 "id": 64, 4205 "options": { 4206 "legend": { 4207 "calcs": [ 4208 "mean" 4209 ], 4210 "displayMode": "table", 4211 "placement": "right", 4212 "showLegend": true, 4213 "sortBy": "Name", 4214 "sortDesc": false 4215 }, 4216 "tooltip": { 4217 "hideZeros": false, 4218 "mode": "single", 4219 "sort": "none" 4220 } 4221 }, 4222 "pluginVersion": "12.1.0-90058", 4223 "targets": [ 4224 { 4225 "datasource": { 4226 "type": "prometheus", 4227 "uid": "${datasource}" 4228 }, 4229 "editorMode": "code", 4230 "expr": "sum by (shard) (rate(pyroscope_segment_writer_client_sent_bytes_sum{namespace=\"$namespace\"}[$__rate_interval]))", 4231 "hide": false, 4232 "instant": false, 4233 "legendFormat": "__auto", 4234 "range": true, 4235 "refId": "B" 4236 } 4237 ], 4238 "title": "Distributed Bytes (per shard)", 4239 "transparent": true, 4240 "type": "timeseries" 4241 }, 4242 { 4243 "datasource": { 4244 "type": "prometheus", 4245 "uid": "${datasource}" 4246 }, 4247 "fieldConfig": { 4248 "defaults": { 4249 "color": { 4250 "mode": "palette-classic" 4251 }, 4252 "custom": { 4253 "axisBorderShow": false, 4254 "axisCenteredZero": false, 4255 "axisColorMode": "text", 4256 "axisLabel": "", 4257 "axisPlacement": "auto", 4258 "axisSoftMin": 0, 4259 "barAlignment": 0, 4260 "barWidthFactor": 0.6, 4261 "drawStyle": "line", 4262 "fillOpacity": 7, 4263 "gradientMode": "none", 4264 "hideFrom": { 4265 "legend": false, 4266 "tooltip": false, 4267 "viz": false 4268 }, 4269 "insertNulls": false, 4270 "lineInterpolation": "linear", 4271 "lineWidth": 1, 4272 "pointSize": 5, 4273 "scaleDistribution": { 4274 "type": "linear" 4275 }, 4276 "showPoints": "auto", 4277 "spanNulls": false, 4278 "stacking": { 4279 "group": "A", 4280 "mode": "normal" 4281 }, 4282 "thresholdsStyle": { 4283 "mode": "off" 4284 } 4285 }, 4286 "mappings": [], 4287 "thresholds": { 4288 "mode": "absolute", 4289 "steps": [ 4290 { 4291 "color": "green", 4292 "value": 0 4293 }, 4294 { 4295 "color": "red", 4296 "value": 80 4297 } 4298 ] 4299 }, 4300 "unit": "bytes" 4301 }, 4302 "overrides": [] 4303 }, 4304 "gridPos": { 4305 "h": 9, 4306 "w": 12, 4307 "x": 12, 4308 "y": 75 4309 }, 4310 "id": 26, 4311 "options": { 4312 "legend": { 4313 "calcs": [ 4314 "mean" 4315 ], 4316 "displayMode": "table", 4317 "placement": "right", 4318 "showLegend": true, 4319 "sortBy": "Name", 4320 "sortDesc": false 4321 }, 4322 "tooltip": { 4323 "hideZeros": false, 4324 "mode": "single", 4325 "sort": "none" 4326 } 4327 }, 4328 "pluginVersion": "12.1.0-90058", 4329 "targets": [ 4330 { 4331 "datasource": { 4332 "type": "prometheus", 4333 "uid": "${datasource}" 4334 }, 4335 "editorMode": "code", 4336 "expr": "sum by (shard) (rate(pyroscope_segment_writer_client_sent_bytes_sum{namespace=\"$namespace\"}[$__rate_interval]))", 4337 "hide": false, 4338 "instant": false, 4339 "legendFormat": "__auto", 4340 "range": true, 4341 "refId": "B" 4342 } 4343 ], 4344 "title": "Distributed Bytes (per shard, stack)", 4345 "transparent": true, 4346 "type": "timeseries" 4347 }, 4348 { 4349 "datasource": { 4350 "type": "prometheus", 4351 "uid": "${datasource}" 4352 }, 4353 "fieldConfig": { 4354 "defaults": { 4355 "color": { 4356 "mode": "palette-classic" 4357 }, 4358 "custom": { 4359 "axisBorderShow": false, 4360 "axisCenteredZero": false, 4361 "axisColorMode": "text", 4362 "axisLabel": "", 4363 "axisPlacement": "auto", 4364 "axisSoftMin": 0, 4365 "barAlignment": 0, 4366 "barWidthFactor": 0.6, 4367 "drawStyle": "line", 4368 "fillOpacity": 0, 4369 "gradientMode": "none", 4370 "hideFrom": { 4371 "legend": false, 4372 "tooltip": false, 4373 "viz": false 4374 }, 4375 "insertNulls": false, 4376 "lineInterpolation": "linear", 4377 "lineWidth": 1, 4378 "pointSize": 5, 4379 "scaleDistribution": { 4380 "type": "linear" 4381 }, 4382 "showPoints": "auto", 4383 "spanNulls": false, 4384 "stacking": { 4385 "group": "A", 4386 "mode": "none" 4387 }, 4388 "thresholdsStyle": { 4389 "mode": "off" 4390 } 4391 }, 4392 "mappings": [], 4393 "thresholds": { 4394 "mode": "absolute", 4395 "steps": [ 4396 { 4397 "color": "green", 4398 "value": 0 4399 }, 4400 { 4401 "color": "red", 4402 "value": 80 4403 } 4404 ] 4405 }, 4406 "unit": "bytes" 4407 }, 4408 "overrides": [] 4409 }, 4410 "gridPos": { 4411 "h": 9, 4412 "w": 12, 4413 "x": 0, 4414 "y": 84 4415 }, 4416 "id": 24, 4417 "options": { 4418 "legend": { 4419 "calcs": [ 4420 "mean" 4421 ], 4422 "displayMode": "table", 4423 "placement": "right", 4424 "showLegend": true, 4425 "sortBy": "Name", 4426 "sortDesc": false 4427 }, 4428 "tooltip": { 4429 "hideZeros": false, 4430 "mode": "single", 4431 "sort": "none" 4432 } 4433 }, 4434 "pluginVersion": "12.1.0-90058", 4435 "targets": [ 4436 { 4437 "datasource": { 4438 "type": "prometheus", 4439 "uid": "${datasource}" 4440 }, 4441 "editorMode": "code", 4442 "expr": "sum by (pod) (rate(pyroscope_segment_writer_segment_ingest_bytes_sum{namespace=\"$namespace\"}[$__rate_interval]))", 4443 "legendFormat": "__auto", 4444 "range": true, 4445 "refId": "A" 4446 } 4447 ], 4448 "title": "Ingested Bytes (per node)", 4449 "transparent": true, 4450 "type": "timeseries" 4451 }, 4452 { 4453 "datasource": { 4454 "type": "prometheus", 4455 "uid": "${datasource}" 4456 }, 4457 "fieldConfig": { 4458 "defaults": { 4459 "color": { 4460 "mode": "palette-classic" 4461 }, 4462 "custom": { 4463 "axisBorderShow": false, 4464 "axisCenteredZero": false, 4465 "axisColorMode": "text", 4466 "axisLabel": "", 4467 "axisPlacement": "auto", 4468 "axisSoftMin": 0, 4469 "barAlignment": 0, 4470 "barWidthFactor": 0.6, 4471 "drawStyle": "line", 4472 "fillOpacity": 7, 4473 "gradientMode": "none", 4474 "hideFrom": { 4475 "legend": false, 4476 "tooltip": false, 4477 "viz": false 4478 }, 4479 "insertNulls": false, 4480 "lineInterpolation": "linear", 4481 "lineWidth": 1, 4482 "pointSize": 5, 4483 "scaleDistribution": { 4484 "type": "linear" 4485 }, 4486 "showPoints": "auto", 4487 "spanNulls": false, 4488 "stacking": { 4489 "group": "A", 4490 "mode": "normal" 4491 }, 4492 "thresholdsStyle": { 4493 "mode": "off" 4494 } 4495 }, 4496 "mappings": [], 4497 "thresholds": { 4498 "mode": "absolute", 4499 "steps": [ 4500 { 4501 "color": "green", 4502 "value": 0 4503 }, 4504 { 4505 "color": "red", 4506 "value": 80 4507 } 4508 ] 4509 }, 4510 "unit": "bytes" 4511 }, 4512 "overrides": [] 4513 }, 4514 "gridPos": { 4515 "h": 9, 4516 "w": 12, 4517 "x": 12, 4518 "y": 84 4519 }, 4520 "id": 34, 4521 "options": { 4522 "legend": { 4523 "calcs": [ 4524 "mean" 4525 ], 4526 "displayMode": "table", 4527 "placement": "right", 4528 "showLegend": true, 4529 "sortBy": "Name", 4530 "sortDesc": false 4531 }, 4532 "tooltip": { 4533 "hideZeros": false, 4534 "mode": "single", 4535 "sort": "none" 4536 } 4537 }, 4538 "pluginVersion": "12.1.0-90058", 4539 "targets": [ 4540 { 4541 "datasource": { 4542 "type": "prometheus", 4543 "uid": "${datasource}" 4544 }, 4545 "editorMode": "code", 4546 "expr": "sum by (pod) (rate(pyroscope_segment_writer_segment_ingest_bytes_sum{namespace=\"$namespace\"}[$__rate_interval]))", 4547 "legendFormat": "__auto", 4548 "range": true, 4549 "refId": "A" 4550 }, 4551 { 4552 "datasource": { 4553 "type": "prometheus", 4554 "uid": "${datasource}" 4555 }, 4556 "editorMode": "code", 4557 "expr": "", 4558 "hide": false, 4559 "instant": false, 4560 "legendFormat": "__auto", 4561 "range": true, 4562 "refId": "B" 4563 } 4564 ], 4565 "title": "Ingested Bytes (per node, stack)", 4566 "transparent": true, 4567 "type": "timeseries" 4568 }, 4569 { 4570 "datasource": { 4571 "type": "prometheus", 4572 "uid": "${datasource}" 4573 }, 4574 "fieldConfig": { 4575 "defaults": { 4576 "color": { 4577 "mode": "palette-classic" 4578 }, 4579 "custom": { 4580 "axisBorderShow": false, 4581 "axisCenteredZero": false, 4582 "axisColorMode": "text", 4583 "axisLabel": "", 4584 "axisPlacement": "auto", 4585 "axisSoftMin": 0, 4586 "barAlignment": 0, 4587 "barWidthFactor": 0.6, 4588 "drawStyle": "line", 4589 "fillOpacity": 0, 4590 "gradientMode": "none", 4591 "hideFrom": { 4592 "legend": false, 4593 "tooltip": false, 4594 "viz": false 4595 }, 4596 "insertNulls": false, 4597 "lineInterpolation": "linear", 4598 "lineWidth": 1, 4599 "pointSize": 5, 4600 "scaleDistribution": { 4601 "type": "linear" 4602 }, 4603 "showPoints": "auto", 4604 "spanNulls": false, 4605 "stacking": { 4606 "group": "A", 4607 "mode": "none" 4608 }, 4609 "thresholdsStyle": { 4610 "mode": "off" 4611 } 4612 }, 4613 "mappings": [], 4614 "thresholds": { 4615 "mode": "absolute", 4616 "steps": [ 4617 { 4618 "color": "green", 4619 "value": 0 4620 }, 4621 { 4622 "color": "red", 4623 "value": 80 4624 } 4625 ] 4626 }, 4627 "unit": "short" 4628 }, 4629 "overrides": [] 4630 }, 4631 "gridPos": { 4632 "h": 9, 4633 "w": 12, 4634 "x": 0, 4635 "y": 93 4636 }, 4637 "id": 35, 4638 "options": { 4639 "legend": { 4640 "calcs": [ 4641 "mean" 4642 ], 4643 "displayMode": "table", 4644 "placement": "right", 4645 "showLegend": true, 4646 "sortBy": "Name", 4647 "sortDesc": false 4648 }, 4649 "tooltip": { 4650 "hideZeros": false, 4651 "mode": "multi", 4652 "sort": "desc" 4653 } 4654 }, 4655 "pluginVersion": "12.1.0-90058", 4656 "targets": [ 4657 { 4658 "datasource": { 4659 "type": "prometheus", 4660 "uid": "${datasource}" 4661 }, 4662 "editorMode": "code", 4663 "expr": "sum by (pod) (rate(pyroscope_segment_writer_segment_ingest_bytes_count{namespace=\"$namespace\"}[$__rate_interval]))", 4664 "legendFormat": "__auto", 4665 "range": true, 4666 "refId": "A" 4667 } 4668 ], 4669 "title": "Ingested Profiles (per node)", 4670 "transparent": true, 4671 "type": "timeseries" 4672 }, 4673 { 4674 "datasource": { 4675 "type": "prometheus", 4676 "uid": "${datasource}" 4677 }, 4678 "fieldConfig": { 4679 "defaults": { 4680 "color": { 4681 "mode": "palette-classic" 4682 }, 4683 "custom": { 4684 "axisBorderShow": false, 4685 "axisCenteredZero": false, 4686 "axisColorMode": "text", 4687 "axisLabel": "", 4688 "axisPlacement": "auto", 4689 "axisSoftMin": 0, 4690 "barAlignment": 0, 4691 "barWidthFactor": 0.6, 4692 "drawStyle": "line", 4693 "fillOpacity": 7, 4694 "gradientMode": "none", 4695 "hideFrom": { 4696 "legend": false, 4697 "tooltip": false, 4698 "viz": false 4699 }, 4700 "insertNulls": false, 4701 "lineInterpolation": "linear", 4702 "lineWidth": 1, 4703 "pointSize": 5, 4704 "scaleDistribution": { 4705 "type": "linear" 4706 }, 4707 "showPoints": "auto", 4708 "spanNulls": false, 4709 "stacking": { 4710 "group": "A", 4711 "mode": "normal" 4712 }, 4713 "thresholdsStyle": { 4714 "mode": "off" 4715 } 4716 }, 4717 "mappings": [], 4718 "thresholds": { 4719 "mode": "absolute", 4720 "steps": [ 4721 { 4722 "color": "green", 4723 "value": 0 4724 }, 4725 { 4726 "color": "red", 4727 "value": 80 4728 } 4729 ] 4730 }, 4731 "unit": "short" 4732 }, 4733 "overrides": [] 4734 }, 4735 "gridPos": { 4736 "h": 9, 4737 "w": 12, 4738 "x": 12, 4739 "y": 93 4740 }, 4741 "id": 36, 4742 "options": { 4743 "legend": { 4744 "calcs": [ 4745 "mean" 4746 ], 4747 "displayMode": "table", 4748 "placement": "right", 4749 "showLegend": true, 4750 "sortBy": "Name", 4751 "sortDesc": false 4752 }, 4753 "tooltip": { 4754 "hideZeros": false, 4755 "mode": "multi", 4756 "sort": "desc" 4757 } 4758 }, 4759 "pluginVersion": "12.1.0-90058", 4760 "targets": [ 4761 { 4762 "datasource": { 4763 "type": "prometheus", 4764 "uid": "${datasource}" 4765 }, 4766 "editorMode": "code", 4767 "expr": "sum by (pod) (rate(pyroscope_segment_writer_segment_ingest_bytes_count{namespace=\"$namespace\"}[$__rate_interval]))", 4768 "legendFormat": "__auto", 4769 "range": true, 4770 "refId": "A" 4771 } 4772 ], 4773 "title": "Ingested Profiles (per node, stack)", 4774 "transparent": true, 4775 "type": "timeseries" 4776 } 4777 ], 4778 "title": "Data Distribution", 4779 "type": "row" 4780 }, 4781 { 4782 "collapsed": true, 4783 "gridPos": { 4784 "h": 1, 4785 "w": 24, 4786 "x": 0, 4787 "y": 55 4788 }, 4789 "id": 86, 4790 "panels": [ 4791 { 4792 "datasource": { 4793 "type": "prometheus", 4794 "uid": "${datasource}" 4795 }, 4796 "fieldConfig": { 4797 "defaults": { 4798 "color": { 4799 "mode": "thresholds" 4800 }, 4801 "fieldMinMax": false, 4802 "mappings": [], 4803 "thresholds": { 4804 "mode": "absolute", 4805 "steps": [ 4806 { 4807 "color": "green", 4808 "value": 0 4809 } 4810 ] 4811 } 4812 }, 4813 "overrides": [] 4814 }, 4815 "gridPos": { 4816 "h": 13, 4817 "w": 12, 4818 "x": 0, 4819 "y": 154 4820 }, 4821 "id": 87, 4822 "options": { 4823 "groupByField": "container", 4824 "labelFields": [ 4825 "container", 4826 "instance_type" 4827 ], 4828 "textField": "instance_type", 4829 "tiling": "treemapSquarify" 4830 }, 4831 "pluginVersion": "2.1.1", 4832 "targets": [ 4833 { 4834 "datasource": { 4835 "type": "prometheus", 4836 "uid": "${datasource}" 4837 }, 4838 "editorMode": "code", 4839 "exemplar": false, 4840 "expr": "count by (instance_type, container) (\n rate(container_cpu_usage_seconds_total{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n container=~\"segment-writer|distributor|cortex-gw|metastore|compaction-worker\",\n }[$__rate_interval]) + on (instance) group_left(instance_type) (0 * label_replace(\n karpenter_nodes_allocatable{cluster=\"$cluster\", resource_type=\"cpu\"}, \"instance\", \"$1\", \"node_name\", \"(.*)\")\n )\n)", 4841 "format": "table", 4842 "hide": false, 4843 "instant": true, 4844 "legendFormat": "{{instance_type}}", 4845 "range": false, 4846 "refId": "A" 4847 } 4848 ], 4849 "title": "AWS | Intances By Machine Type", 4850 "transparent": true, 4851 "type": "marcusolsson-treemap-panel" 4852 }, 4853 { 4854 "datasource": { 4855 "type": "prometheus", 4856 "uid": "${datasource}" 4857 }, 4858 "fieldConfig": { 4859 "defaults": { 4860 "color": { 4861 "mode": "palette-classic" 4862 }, 4863 "custom": { 4864 "axisBorderShow": false, 4865 "axisCenteredZero": false, 4866 "axisColorMode": "text", 4867 "axisLabel": "", 4868 "axisPlacement": "auto", 4869 "axisSoftMin": 0, 4870 "barAlignment": 0, 4871 "barWidthFactor": 0.6, 4872 "drawStyle": "line", 4873 "fillOpacity": 0, 4874 "gradientMode": "none", 4875 "hideFrom": { 4876 "legend": false, 4877 "tooltip": false, 4878 "viz": false 4879 }, 4880 "insertNulls": false, 4881 "lineInterpolation": "linear", 4882 "lineWidth": 1, 4883 "pointSize": 5, 4884 "scaleDistribution": { 4885 "type": "linear" 4886 }, 4887 "showPoints": "auto", 4888 "spanNulls": false, 4889 "stacking": { 4890 "group": "A", 4891 "mode": "none" 4892 }, 4893 "thresholdsStyle": { 4894 "mode": "off" 4895 } 4896 }, 4897 "mappings": [], 4898 "thresholds": { 4899 "mode": "absolute", 4900 "steps": [ 4901 { 4902 "color": "green", 4903 "value": 0 4904 }, 4905 { 4906 "color": "red", 4907 "value": 80 4908 } 4909 ] 4910 } 4911 }, 4912 "overrides": [] 4913 }, 4914 "gridPos": { 4915 "h": 13, 4916 "w": 12, 4917 "x": 12, 4918 "y": 154 4919 }, 4920 "id": 84, 4921 "options": { 4922 "legend": { 4923 "calcs": [ 4924 "mean", 4925 "max" 4926 ], 4927 "displayMode": "table", 4928 "placement": "right", 4929 "showLegend": true 4930 }, 4931 "tooltip": { 4932 "hideZeros": false, 4933 "mode": "single", 4934 "sort": "none" 4935 } 4936 }, 4937 "pluginVersion": "12.1.0-90058", 4938 "targets": [ 4939 { 4940 "datasource": { 4941 "type": "prometheus", 4942 "uid": "${datasource}" 4943 }, 4944 "editorMode": "code", 4945 "expr": "avg by (instance_type, container) (\n rate(container_cpu_usage_seconds_total{\n cluster=\"$cluster\",\n namespace=\"$namespace\",\n container=~\"segment-writer|distributor|cortex-gw\",\n }[$__rate_interval]) + on (instance) group_left(instance_type) (0 * label_replace(\n karpenter_nodes_allocatable{cluster=\"$cluster\", resource_type=\"cpu\"}, \"instance\", \"$1\", \"node_name\", \"(.*)\")\n )\n)", 4946 "hide": false, 4947 "legendFormat": "{{container}}/{{instance_type}}", 4948 "range": true, 4949 "refId": "A" 4950 } 4951 ], 4952 "title": "AWS | Average CPU Usage By Type", 4953 "transparent": true, 4954 "type": "timeseries" 4955 } 4956 ], 4957 "title": "Resource Allocation", 4958 "type": "row" 4959 } 4960 ], 4961 "preload": false, 4962 "schemaVersion": 41, 4963 "tags": [ 4964 "pyroscope", 4965 "pyroscope-v2" 4966 ], 4967 "templating": { 4968 "list": [ 4969 { 4970 "current": { 4971 "text": "ops-cortex", 4972 "value": "000000134" 4973 }, 4974 "includeAll": false, 4975 "name": "datasource", 4976 "options": [], 4977 "query": "prometheus", 4978 "refresh": 1, 4979 "regex": "", 4980 "type": "datasource" 4981 }, 4982 { 4983 "allowCustomValue": false, 4984 "current": { 4985 "text": "Grafana Logging", 4986 "value": "000000193" 4987 }, 4988 "name": "loki_datasource", 4989 "options": [], 4990 "query": "loki", 4991 "refresh": 1, 4992 "regex": "", 4993 "type": "datasource" 4994 }, 4995 { 4996 "allowCustomValue": false, 4997 "current": { 4998 "text": "prod-eu-west-2", 4999 "value": "prod-eu-west-2" 5000 }, 5001 "datasource": { 5002 "type": "prometheus", 5003 "uid": "${datasource}" 5004 }, 5005 "definition": "label_values(pyroscope_build_info{namespace=\"$namespace\"},cluster)", 5006 "label": "cluster", 5007 "name": "cluster", 5008 "options": [], 5009 "query": { 5010 "qryType": 1, 5011 "query": "label_values(pyroscope_build_info{namespace=\"$namespace\"},cluster)", 5012 "refId": "PrometheusVariableQueryEditor-VariableQuery" 5013 }, 5014 "refresh": 1, 5015 "regex": "", 5016 "sort": 1, 5017 "type": "query" 5018 }, 5019 { 5020 "allowCustomValue": false, 5021 "current": { 5022 "text": "default", 5023 "value": "default" 5024 }, 5025 "datasource": { 5026 "type": "prometheus", 5027 "uid": "${datasource}" 5028 }, 5029 "definition": "label_values(pyroscope_build_info,namespace)", 5030 "label": "namespace", 5031 "name": "namespace", 5032 "options": [], 5033 "query": { 5034 "qryType": 1, 5035 "query": "label_values(pyroscope_build_info,namespace)", 5036 "refId": "PrometheusVariableQueryEditor-VariableQuery" 5037 }, 5038 "refresh": 1, 5039 "regex": ".*", 5040 "type": "query" 5041 }, 5042 { 5043 "baseFilters": [], 5044 "datasource": { 5045 "type": "prometheus", 5046 "uid": "${datasource}" 5047 }, 5048 "filters": [], 5049 "name": "Filters", 5050 "type": "adhoc" 5051 } 5052 ] 5053 }, 5054 "time": { 5055 "from": "now-6h", 5056 "to": "now" 5057 }, 5058 "timepicker": {}, 5059 "timezone": "utc", 5060 "title": "Pyroscope v2 / Write Path" 5061 }