github.com/1aal/kubeblocks@v0.0.0-20231107070852-e1c03e598921/deploy/helm/dashboards/loki.json (about) 1 { 2 "annotations": { 3 "list": [ 4 { 5 "$$hashKey": "object:7", 6 "builtIn": 1, 7 "datasource": { 8 "type": "datasource", 9 "uid": "grafana" 10 }, 11 "enable": true, 12 "hide": true, 13 "iconColor": "rgba(0, 211, 255, 1)", 14 "name": "Annotations & Alerts", 15 "target": { 16 "limit": 100, 17 "matchAny": false, 18 "tags": [], 19 "type": "dashboard" 20 }, 21 "type": "dashboard" 22 } 23 ] 24 }, 25 "description": "Loki metrics via 2.0", 26 "editable": true, 27 "fiscalYearStartMonth": 0, 28 "gnetId": 13407, 29 "graphTooltip": 0, 30 "id": 16, 31 "links": [], 32 "liveNow": false, 33 "panels": [ 34 { 35 "datasource": { 36 "type": "prometheus", 37 "uid": "prometheus" 38 }, 39 "description": "", 40 "gridPos": { 41 "h": 2, 42 "w": 24, 43 "x": 0, 44 "y": 0 45 }, 46 "id": 21, 47 "interval": "", 48 "options": { 49 "code": { 50 "language": "plaintext", 51 "showLineNumbers": false, 52 "showMiniMap": false 53 }, 54 "content": "<div class=\"left dashboard-header\">\n <img src=\"https://marketplace-assets.digitalocean.com/logos/loki-logo.svg\" style=\"height:29px;\"/>\n <span>Loki Instance Metrics</span>\n</div>", 55 "mode": "html" 56 }, 57 "pluginVersion": "9.2.4", 58 "targets": [ 59 { 60 "datasource": { 61 "type": "prometheus", 62 "uid": "prometheus" 63 }, 64 "expr": "loki_build_info", 65 "format": "table", 66 "instant": false, 67 "interval": "", 68 "legendFormat": "", 69 "refId": "A" 70 } 71 ], 72 "transparent": true, 73 "type": "text" 74 }, 75 { 76 "datasource": { 77 "type": "prometheus", 78 "uid": "prometheus" 79 }, 80 "fieldConfig": { 81 "defaults": { 82 "decimals": 4, 83 "links": [], 84 "mappings": [], 85 "thresholds": { 86 "mode": "absolute", 87 "steps": [ 88 { 89 "color": "green", 90 "value": null 91 } 92 ] 93 }, 94 "unit": "short" 95 }, 96 "overrides": [] 97 }, 98 "gridPos": { 99 "h": 3, 100 "w": 3, 101 "x": 0, 102 "y": 2 103 }, 104 "id": 75, 105 "interval": "", 106 "options": { 107 "colorMode": "value", 108 "graphMode": "none", 109 "justifyMode": "center", 110 "orientation": "auto", 111 "reduceOptions": { 112 "calcs": [ 113 "last" 114 ], 115 "fields": "/^instance$/", 116 "values": false 117 }, 118 "textMode": "auto" 119 }, 120 "pluginVersion": "9.2.4", 121 "targets": [ 122 { 123 "datasource": { 124 "type": "prometheus", 125 "uid": "prometheus" 126 }, 127 "expr": "loki_build_info", 128 "format": "table", 129 "instant": false, 130 "interval": "", 131 "legendFormat": "", 132 "refId": "A" 133 } 134 ], 135 "title": "Loki Instance", 136 "type": "stat" 137 }, 138 { 139 "datasource": { 140 "type": "prometheus", 141 "uid": "prometheus" 142 }, 143 "fieldConfig": { 144 "defaults": { 145 "decimals": 4, 146 "links": [], 147 "mappings": [], 148 "thresholds": { 149 "mode": "absolute", 150 "steps": [ 151 { 152 "color": "green", 153 "value": null 154 } 155 ] 156 }, 157 "unit": "short" 158 }, 159 "overrides": [] 160 }, 161 "gridPos": { 162 "h": 3, 163 "w": 3, 164 "x": 3, 165 "y": 2 166 }, 167 "id": 59, 168 "interval": "", 169 "options": { 170 "colorMode": "value", 171 "graphMode": "none", 172 "justifyMode": "center", 173 "orientation": "auto", 174 "reduceOptions": { 175 "calcs": [ 176 "last" 177 ], 178 "fields": "/^version$/", 179 "values": false 180 }, 181 "textMode": "auto" 182 }, 183 "pluginVersion": "9.2.4", 184 "targets": [ 185 { 186 "datasource": { 187 "type": "prometheus", 188 "uid": "prometheus" 189 }, 190 "expr": "loki_build_info", 191 "format": "table", 192 "instant": false, 193 "interval": "", 194 "legendFormat": "", 195 "refId": "A" 196 } 197 ], 198 "title": "Loki Version", 199 "type": "stat" 200 }, 201 { 202 "datasource": { 203 "type": "prometheus", 204 "uid": "prometheus" 205 }, 206 "description": "", 207 "fieldConfig": { 208 "defaults": { 209 "decimals": 4, 210 "links": [], 211 "mappings": [], 212 "thresholds": { 213 "mode": "absolute", 214 "steps": [ 215 { 216 "color": "green", 217 "value": null 218 } 219 ] 220 }, 221 "unit": "short" 222 }, 223 "overrides": [] 224 }, 225 "gridPos": { 226 "h": 3, 227 "w": 3, 228 "x": 6, 229 "y": 2 230 }, 231 "id": 76, 232 "interval": "", 233 "options": { 234 "colorMode": "value", 235 "graphMode": "none", 236 "justifyMode": "center", 237 "orientation": "auto", 238 "reduceOptions": { 239 "calcs": [ 240 "last" 241 ], 242 "fields": "/^goos$/", 243 "values": false 244 }, 245 "textMode": "auto" 246 }, 247 "pluginVersion": "9.2.4", 248 "targets": [ 249 { 250 "datasource": { 251 "type": "prometheus", 252 "uid": "prometheus" 253 }, 254 "expr": "loki_build_info", 255 "format": "table", 256 "instant": false, 257 "interval": "", 258 "legendFormat": "", 259 "refId": "A" 260 } 261 ], 262 "title": "Loki OS", 263 "type": "stat" 264 }, 265 { 266 "datasource": { 267 "type": "prometheus", 268 "uid": "prometheus" 269 }, 270 "fieldConfig": { 271 "defaults": { 272 "decimals": 4, 273 "links": [], 274 "mappings": [], 275 "thresholds": { 276 "mode": "absolute", 277 "steps": [ 278 { 279 "color": "green", 280 "value": null 281 } 282 ] 283 }, 284 "unit": "short" 285 }, 286 "overrides": [] 287 }, 288 "gridPos": { 289 "h": 3, 290 "w": 3, 291 "x": 9, 292 "y": 2 293 }, 294 "id": 77, 295 "interval": "", 296 "options": { 297 "colorMode": "value", 298 "graphMode": "none", 299 "justifyMode": "center", 300 "orientation": "auto", 301 "reduceOptions": { 302 "calcs": [ 303 "last" 304 ], 305 "fields": "/^goarch$/", 306 "values": false 307 }, 308 "textMode": "auto" 309 }, 310 "pluginVersion": "9.2.4", 311 "targets": [ 312 { 313 "datasource": { 314 "type": "prometheus", 315 "uid": "prometheus" 316 }, 317 "expr": "loki_build_info", 318 "format": "table", 319 "instant": false, 320 "interval": "", 321 "legendFormat": "", 322 "refId": "A" 323 } 324 ], 325 "title": "Loki Arch", 326 "type": "stat" 327 }, 328 { 329 "datasource": { 330 "type": "prometheus", 331 "uid": "prometheus" 332 }, 333 "fieldConfig": { 334 "defaults": { 335 "decimals": 0, 336 "links": [], 337 "mappings": [], 338 "thresholds": { 339 "mode": "absolute", 340 "steps": [ 341 { 342 "color": "green", 343 "value": null 344 } 345 ] 346 }, 347 "unit": "short" 348 }, 349 "overrides": [] 350 }, 351 "gridPos": { 352 "h": 3, 353 "w": 3, 354 "x": 12, 355 "y": 2 356 }, 357 "id": 10, 358 "interval": "", 359 "options": { 360 "colorMode": "value", 361 "graphMode": "none", 362 "justifyMode": "center", 363 "orientation": "auto", 364 "reduceOptions": { 365 "calcs": [ 366 "last" 367 ], 368 "fields": "", 369 "values": false 370 }, 371 "textMode": "auto" 372 }, 373 "pluginVersion": "9.2.4", 374 "targets": [ 375 { 376 "datasource": { 377 "type": "prometheus", 378 "uid": "prometheus" 379 }, 380 "editorMode": "code", 381 "expr": "sum(loki_log_messages_total)", 382 "instant": false, 383 "interval": "", 384 "legendFormat": "", 385 "refId": "A" 386 } 387 ], 388 "title": "Message Total", 389 "type": "stat" 390 }, 391 { 392 "datasource": { 393 "type": "prometheus", 394 "uid": "prometheus" 395 }, 396 "fieldConfig": { 397 "defaults": { 398 "decimals": 2, 399 "links": [], 400 "mappings": [], 401 "thresholds": { 402 "mode": "absolute", 403 "steps": [ 404 { 405 "color": "green", 406 "value": null 407 } 408 ] 409 }, 410 "unit": "short" 411 }, 412 "overrides": [] 413 }, 414 "gridPos": { 415 "h": 3, 416 "w": 3, 417 "x": 15, 418 "y": 2 419 }, 420 "id": 23, 421 "interval": "", 422 "options": { 423 "colorMode": "value", 424 "graphMode": "area", 425 "justifyMode": "auto", 426 "orientation": "auto", 427 "reduceOptions": { 428 "calcs": [ 429 "mean" 430 ], 431 "fields": "", 432 "values": false 433 }, 434 "textMode": "auto" 435 }, 436 "pluginVersion": "9.2.4", 437 "targets": [ 438 { 439 "datasource": { 440 "type": "prometheus", 441 "uid": "prometheus" 442 }, 443 "expr": "sum(loki_store_series_total)", 444 "instant": false, 445 "interval": "", 446 "legendFormat": "", 447 "refId": "A" 448 } 449 ], 450 "title": "Store Series Total", 451 "type": "stat" 452 }, 453 { 454 "datasource": { 455 "type": "prometheus", 456 "uid": "prometheus" 457 }, 458 "fieldConfig": { 459 "defaults": { 460 "decimals": 2, 461 "links": [], 462 "mappings": [], 463 "thresholds": { 464 "mode": "absolute", 465 "steps": [ 466 { 467 "color": "green", 468 "value": null 469 } 470 ] 471 }, 472 "unit": "bytes" 473 }, 474 "overrides": [] 475 }, 476 "gridPos": { 477 "h": 3, 478 "w": 3, 479 "x": 18, 480 "y": 2 481 }, 482 "id": 41, 483 "interval": "", 484 "options": { 485 "colorMode": "value", 486 "graphMode": "none", 487 "justifyMode": "auto", 488 "orientation": "auto", 489 "reduceOptions": { 490 "calcs": [ 491 "mean" 492 ], 493 "fields": "", 494 "values": false 495 }, 496 "textMode": "auto" 497 }, 498 "pluginVersion": "9.2.4", 499 "targets": [ 500 { 501 "datasource": { 502 "type": "prometheus", 503 "uid": "prometheus" 504 }, 505 "expr": "sum(loki_ingester_chunk_stored_bytes_total)", 506 "instant": true, 507 "interval": "", 508 "legendFormat": "", 509 "refId": "A" 510 } 511 ], 512 "title": "Store Chunks Total", 513 "type": "stat" 514 }, 515 { 516 "datasource": { 517 "type": "prometheus", 518 "uid": "prometheus" 519 }, 520 "fieldConfig": { 521 "defaults": { 522 "links": [], 523 "mappings": [], 524 "thresholds": { 525 "mode": "absolute", 526 "steps": [ 527 { 528 "color": "green", 529 "value": null 530 } 531 ] 532 }, 533 "unit": "none" 534 }, 535 "overrides": [] 536 }, 537 "gridPos": { 538 "h": 3, 539 "w": 3, 540 "x": 21, 541 "y": 2 542 }, 543 "id": 74, 544 "interval": "", 545 "options": { 546 "colorMode": "value", 547 "graphMode": "none", 548 "justifyMode": "auto", 549 "orientation": "auto", 550 "reduceOptions": { 551 "calcs": [ 552 "mean" 553 ], 554 "fields": "", 555 "values": false 556 }, 557 "textMode": "auto" 558 }, 559 "pluginVersion": "9.2.4", 560 "targets": [ 561 { 562 "datasource": { 563 "type": "prometheus", 564 "uid": "prometheus" 565 }, 566 "editorMode": "code", 567 "expr": "sum(loki_boltdb_shipper_compact_tables_operation_total)", 568 "instant": true, 569 "interval": "", 570 "legendFormat": "", 571 "refId": "A" 572 } 573 ], 574 "title": "Compact Tables Operation", 575 "type": "stat" 576 }, 577 { 578 "aliasColors": {}, 579 "bars": false, 580 "dashLength": 10, 581 "dashes": false, 582 "datasource": { 583 "type": "prometheus", 584 "uid": "prometheus" 585 }, 586 "fieldConfig": { 587 "defaults": { 588 "links": [] 589 }, 590 "overrides": [] 591 }, 592 "fill": 5, 593 "fillGradient": 0, 594 "gridPos": { 595 "h": 6, 596 "w": 12, 597 "x": 0, 598 "y": 5 599 }, 600 "hiddenSeries": false, 601 "id": 8, 602 "interval": "", 603 "legend": { 604 "avg": false, 605 "current": true, 606 "max": false, 607 "min": false, 608 "show": true, 609 "total": false, 610 "values": true 611 }, 612 "lines": true, 613 "linewidth": 1, 614 "nullPointMode": "null", 615 "options": { 616 "alertThreshold": true 617 }, 618 "percentage": false, 619 "pluginVersion": "9.2.4", 620 "pointradius": 2, 621 "points": false, 622 "renderer": "flot", 623 "seriesOverrides": [], 624 "spaceLength": 10, 625 "stack": true, 626 "steppedLine": false, 627 "targets": [ 628 { 629 "datasource": { 630 "type": "prometheus", 631 "uid": "prometheus" 632 }, 633 "editorMode": "code", 634 "expr": "sum(irate(loki_log_messages_total[1m])) by (level)", 635 "interval": "", 636 "legendFormat": "{{ level }}", 637 "range": true, 638 "refId": "A" 639 } 640 ], 641 "thresholds": [], 642 "timeRegions": [], 643 "title": "Messages Input", 644 "tooltip": { 645 "shared": true, 646 "sort": 0, 647 "value_type": "individual" 648 }, 649 "type": "graph", 650 "xaxis": { 651 "mode": "time", 652 "show": true, 653 "values": [] 654 }, 655 "yaxes": [ 656 { 657 "$$hashKey": "object:155", 658 "format": "short", 659 "logBase": 1, 660 "show": true 661 }, 662 { 663 "$$hashKey": "object:156", 664 "format": "short", 665 "logBase": 1, 666 "show": true 667 } 668 ], 669 "yaxis": { 670 "align": false 671 } 672 }, 673 { 674 "aliasColors": {}, 675 "bars": false, 676 "dashLength": 10, 677 "dashes": false, 678 "datasource": { 679 "type": "prometheus", 680 "uid": "prometheus" 681 }, 682 "fieldConfig": { 683 "defaults": { 684 "links": [] 685 }, 686 "overrides": [] 687 }, 688 "fill": 0, 689 "fillGradient": 0, 690 "gridPos": { 691 "h": 6, 692 "w": 12, 693 "x": 12, 694 "y": 5 695 }, 696 "hiddenSeries": false, 697 "id": 12, 698 "interval": "", 699 "legend": { 700 "alignAsTable": true, 701 "avg": false, 702 "current": true, 703 "max": false, 704 "min": false, 705 "rightSide": true, 706 "show": true, 707 "sort": "current", 708 "sortDesc": true, 709 "total": false, 710 "values": true 711 }, 712 "lines": true, 713 "linewidth": 1, 714 "nullPointMode": "null as zero", 715 "options": { 716 "alertThreshold": true 717 }, 718 "percentage": false, 719 "pluginVersion": "9.2.4", 720 "pointradius": 2, 721 "points": false, 722 "renderer": "flot", 723 "seriesOverrides": [], 724 "spaceLength": 10, 725 "stack": false, 726 "steppedLine": false, 727 "targets": [ 728 { 729 "datasource": { 730 "type": "prometheus", 731 "uid": "prometheus" 732 }, 733 "expr": "histogram_quantile(0.95, sum(rate(loki_logql_querystats_latency_seconds_bucket[5m])) by (le,type))", 734 "interval": "", 735 "legendFormat": "{{ type }}", 736 "refId": "A" 737 } 738 ], 739 "thresholds": [], 740 "timeRegions": [], 741 "title": "LogQL Latency", 742 "tooltip": { 743 "shared": true, 744 "sort": 0, 745 "value_type": "individual" 746 }, 747 "type": "graph", 748 "xaxis": { 749 "mode": "time", 750 "show": true, 751 "values": [] 752 }, 753 "yaxes": [ 754 { 755 "$$hashKey": "object:155", 756 "format": "s", 757 "logBase": 1, 758 "show": true 759 }, 760 { 761 "$$hashKey": "object:156", 762 "format": "short", 763 "logBase": 1, 764 "show": true 765 } 766 ], 767 "yaxis": { 768 "align": false 769 } 770 }, 771 { 772 "aliasColors": {}, 773 "bars": false, 774 "dashLength": 10, 775 "dashes": false, 776 "datasource": { 777 "type": "prometheus", 778 "uid": "prometheus" 779 }, 780 "fieldConfig": { 781 "defaults": { 782 "links": [] 783 }, 784 "overrides": [] 785 }, 786 "fill": 0, 787 "fillGradient": 0, 788 "gridPos": { 789 "h": 6, 790 "w": 12, 791 "x": 0, 792 "y": 11 793 }, 794 "hiddenSeries": false, 795 "id": 11, 796 "interval": "", 797 "legend": { 798 "alignAsTable": true, 799 "avg": false, 800 "current": true, 801 "max": false, 802 "min": false, 803 "rightSide": true, 804 "show": true, 805 "sort": "current", 806 "sortDesc": true, 807 "total": false, 808 "values": true 809 }, 810 "lines": true, 811 "linewidth": 1, 812 "nullPointMode": "null as zero", 813 "options": { 814 "alertThreshold": true 815 }, 816 "percentage": false, 817 "pluginVersion": "9.2.4", 818 "pointradius": 2, 819 "points": false, 820 "renderer": "flot", 821 "seriesOverrides": [], 822 "spaceLength": 10, 823 "stack": false, 824 "steppedLine": false, 825 "targets": [ 826 { 827 "datasource": { 828 "type": "prometheus", 829 "uid": "prometheus" 830 }, 831 "editorMode": "code", 832 "expr": "histogram_quantile(0.95, sum(rate(loki_request_duration_seconds_bucket[5m])) by (le,route))", 833 "interval": "", 834 "legendFormat": "{{ route }}", 835 "range": true, 836 "refId": "A" 837 } 838 ], 839 "thresholds": [], 840 "timeRegions": [], 841 "title": "API Request Durations", 842 "tooltip": { 843 "shared": true, 844 "sort": 0, 845 "value_type": "individual" 846 }, 847 "type": "graph", 848 "xaxis": { 849 "mode": "time", 850 "show": true, 851 "values": [] 852 }, 853 "yaxes": [ 854 { 855 "$$hashKey": "object:155", 856 "format": "s", 857 "logBase": 1, 858 "show": true 859 }, 860 { 861 "$$hashKey": "object:156", 862 "format": "short", 863 "logBase": 1, 864 "show": true 865 } 866 ], 867 "yaxis": { 868 "align": false 869 } 870 }, 871 { 872 "aliasColors": {}, 873 "bars": false, 874 "dashLength": 10, 875 "dashes": false, 876 "datasource": { 877 "type": "prometheus", 878 "uid": "prometheus" 879 }, 880 "fieldConfig": { 881 "defaults": { 882 "links": [] 883 }, 884 "overrides": [] 885 }, 886 "fill": 0, 887 "fillGradient": 0, 888 "gridPos": { 889 "h": 6, 890 "w": 12, 891 "x": 12, 892 "y": 11 893 }, 894 "hiddenSeries": false, 895 "id": 13, 896 "interval": "", 897 "legend": { 898 "alignAsTable": false, 899 "avg": false, 900 "current": true, 901 "max": false, 902 "min": false, 903 "rightSide": false, 904 "show": true, 905 "sort": "current", 906 "sortDesc": true, 907 "total": false, 908 "values": true 909 }, 910 "lines": true, 911 "linewidth": 1, 912 "nullPointMode": "null as zero", 913 "options": { 914 "alertThreshold": true 915 }, 916 "percentage": false, 917 "pluginVersion": "9.2.4", 918 "pointradius": 2, 919 "points": false, 920 "renderer": "flot", 921 "seriesOverrides": [], 922 "spaceLength": 10, 923 "stack": false, 924 "steppedLine": false, 925 "targets": [ 926 { 927 "datasource": { 928 "type": "prometheus", 929 "uid": "prometheus" 930 }, 931 "editorMode": "code", 932 "expr": "histogram_quantile(0.95, sum(rate(loki_logql_querystats_bytes_processed_per_seconds_bucket[5m])) by (le,type))", 933 "interval": "", 934 "legendFormat": "{{ type }}", 935 "range": true, 936 "refId": "A" 937 } 938 ], 939 "thresholds": [], 940 "timeRegions": [], 941 "title": "LogQL Processed bytes", 942 "tooltip": { 943 "shared": true, 944 "sort": 0, 945 "value_type": "individual" 946 }, 947 "type": "graph", 948 "xaxis": { 949 "mode": "time", 950 "show": true, 951 "values": [] 952 }, 953 "yaxes": [ 954 { 955 "$$hashKey": "object:155", 956 "format": "Bps", 957 "logBase": 1, 958 "show": true 959 }, 960 { 961 "$$hashKey": "object:156", 962 "format": "short", 963 "logBase": 1, 964 "show": true 965 } 966 ], 967 "yaxis": { 968 "align": false 969 } 970 }, 971 { 972 "collapsed": false, 973 "datasource": { 974 "type": "prometheus", 975 "uid": "prometheus" 976 }, 977 "gridPos": { 978 "h": 1, 979 "w": 24, 980 "x": 0, 981 "y": 17 982 }, 983 "id": 34, 984 "panels": [], 985 "targets": [ 986 { 987 "datasource": { 988 "type": "prometheus", 989 "uid": "prometheus" 990 }, 991 "refId": "A" 992 } 993 ], 994 "title": "Cache ", 995 "type": "row" 996 }, 997 { 998 "aliasColors": {}, 999 "bars": false, 1000 "dashLength": 10, 1001 "dashes": false, 1002 "datasource": { 1003 "type": "prometheus", 1004 "uid": "prometheus" 1005 }, 1006 "fieldConfig": { 1007 "defaults": { 1008 "links": [] 1009 }, 1010 "overrides": [] 1011 }, 1012 "fill": 0, 1013 "fillGradient": 0, 1014 "gridPos": { 1015 "h": 6, 1016 "w": 12, 1017 "x": 0, 1018 "y": 18 1019 }, 1020 "hiddenSeries": false, 1021 "id": 45, 1022 "interval": "", 1023 "legend": { 1024 "alignAsTable": true, 1025 "avg": false, 1026 "current": true, 1027 "max": false, 1028 "min": false, 1029 "rightSide": true, 1030 "show": true, 1031 "sort": "current", 1032 "sortDesc": true, 1033 "total": false, 1034 "values": true 1035 }, 1036 "lines": true, 1037 "linewidth": 1, 1038 "nullPointMode": "null as zero", 1039 "options": { 1040 "alertThreshold": true 1041 }, 1042 "percentage": false, 1043 "pluginVersion": "9.2.4", 1044 "pointradius": 2, 1045 "points": false, 1046 "renderer": "flot", 1047 "seriesOverrides": [], 1048 "spaceLength": 10, 1049 "stack": false, 1050 "steppedLine": false, 1051 "targets": [ 1052 { 1053 "datasource": { 1054 "type": "prometheus", 1055 "uid": "prometheus" 1056 }, 1057 "editorMode": "code", 1058 "expr": "histogram_quantile(0.95, sum(rate(loki_cache_request_duration_seconds_bucket[5m])) by (le,name,method))", 1059 "interval": "", 1060 "legendFormat": "{{method}}", 1061 "range": true, 1062 "refId": "A" 1063 } 1064 ], 1065 "thresholds": [], 1066 "timeRegions": [], 1067 "title": "Cache Request Duration", 1068 "tooltip": { 1069 "shared": true, 1070 "sort": 0, 1071 "value_type": "individual" 1072 }, 1073 "type": "graph", 1074 "xaxis": { 1075 "mode": "time", 1076 "show": true, 1077 "values": [] 1078 }, 1079 "yaxes": [ 1080 { 1081 "$$hashKey": "object:155", 1082 "format": "s", 1083 "logBase": 1, 1084 "show": true 1085 }, 1086 { 1087 "$$hashKey": "object:156", 1088 "format": "short", 1089 "logBase": 1, 1090 "show": true 1091 } 1092 ], 1093 "yaxis": { 1094 "align": false 1095 } 1096 }, 1097 { 1098 "aliasColors": {}, 1099 "bars": false, 1100 "dashLength": 10, 1101 "dashes": false, 1102 "datasource": { 1103 "type": "prometheus", 1104 "uid": "prometheus" 1105 }, 1106 "fieldConfig": { 1107 "defaults": { 1108 "links": [] 1109 }, 1110 "overrides": [] 1111 }, 1112 "fill": 0, 1113 "fillGradient": 0, 1114 "gridPos": { 1115 "h": 6, 1116 "w": 12, 1117 "x": 12, 1118 "y": 18 1119 }, 1120 "hiddenSeries": false, 1121 "id": 29, 1122 "interval": "", 1123 "legend": { 1124 "alignAsTable": true, 1125 "avg": false, 1126 "current": true, 1127 "max": false, 1128 "min": false, 1129 "rightSide": true, 1130 "show": true, 1131 "sort": "current", 1132 "sortDesc": true, 1133 "total": false, 1134 "values": true 1135 }, 1136 "lines": true, 1137 "linewidth": 1, 1138 "nullPointMode": "null as zero", 1139 "options": { 1140 "alertThreshold": true 1141 }, 1142 "percentage": false, 1143 "pluginVersion": "9.2.4", 1144 "pointradius": 2, 1145 "points": false, 1146 "renderer": "flot", 1147 "seriesOverrides": [], 1148 "spaceLength": 10, 1149 "stack": false, 1150 "steppedLine": false, 1151 "targets": [ 1152 { 1153 "datasource": { 1154 "type": "prometheus", 1155 "uid": "prometheus" 1156 }, 1157 "editorMode": "code", 1158 "expr": "histogram_quantile(0.95, sum(rate(loki_cache_value_size_bytes_bucket[5m])) by (le,name,method))", 1159 "interval": "", 1160 "legendFormat": "{{name}} / {{method}}", 1161 "range": true, 1162 "refId": "A" 1163 } 1164 ], 1165 "thresholds": [], 1166 "timeRegions": [], 1167 "title": "Cache Value Size bytes", 1168 "tooltip": { 1169 "shared": true, 1170 "sort": 0, 1171 "value_type": "individual" 1172 }, 1173 "type": "graph", 1174 "xaxis": { 1175 "mode": "time", 1176 "show": true, 1177 "values": [] 1178 }, 1179 "yaxes": [ 1180 { 1181 "$$hashKey": "object:155", 1182 "format": "decbytes", 1183 "logBase": 1, 1184 "show": true 1185 }, 1186 { 1187 "$$hashKey": "object:156", 1188 "format": "short", 1189 "logBase": 1, 1190 "show": true 1191 } 1192 ], 1193 "yaxis": { 1194 "align": false 1195 } 1196 }, 1197 { 1198 "aliasColors": {}, 1199 "bars": false, 1200 "dashLength": 10, 1201 "dashes": false, 1202 "datasource": { 1203 "type": "prometheus", 1204 "uid": "prometheus" 1205 }, 1206 "fill": 1, 1207 "fillGradient": 0, 1208 "gridPos": { 1209 "h": 6, 1210 "w": 10, 1211 "x": 0, 1212 "y": 24 1213 }, 1214 "hiddenSeries": false, 1215 "id": 27, 1216 "legend": { 1217 "avg": false, 1218 "current": true, 1219 "max": false, 1220 "min": false, 1221 "show": true, 1222 "total": false, 1223 "values": true 1224 }, 1225 "lines": true, 1226 "linewidth": 1, 1227 "nullPointMode": "null", 1228 "options": { 1229 "alertThreshold": true 1230 }, 1231 "percentage": false, 1232 "pluginVersion": "9.2.4", 1233 "pointradius": 2, 1234 "points": false, 1235 "renderer": "flot", 1236 "seriesOverrides": [], 1237 "spaceLength": 10, 1238 "stack": false, 1239 "steppedLine": false, 1240 "targets": [ 1241 { 1242 "datasource": { 1243 "type": "prometheus", 1244 "uid": "prometheus" 1245 }, 1246 "editorMode": "code", 1247 "expr": "rate(loki_cache_hits[5m])", 1248 "interval": "", 1249 "legendFormat": "{{name}}", 1250 "range": true, 1251 "refId": "A" 1252 } 1253 ], 1254 "thresholds": [], 1255 "timeRegions": [], 1256 "title": "Hits Keys", 1257 "tooltip": { 1258 "shared": true, 1259 "sort": 0, 1260 "value_type": "individual" 1261 }, 1262 "type": "graph", 1263 "xaxis": { 1264 "mode": "time", 1265 "show": true, 1266 "values": [] 1267 }, 1268 "yaxes": [ 1269 { 1270 "$$hashKey": "object:1092", 1271 "format": "short", 1272 "logBase": 1, 1273 "show": true 1274 }, 1275 { 1276 "$$hashKey": "object:1093", 1277 "format": "short", 1278 "logBase": 1, 1279 "show": true 1280 } 1281 ], 1282 "yaxis": { 1283 "align": false 1284 } 1285 }, 1286 { 1287 "aliasColors": {}, 1288 "bars": false, 1289 "dashLength": 10, 1290 "dashes": false, 1291 "datasource": { 1292 "type": "prometheus", 1293 "uid": "prometheus" 1294 }, 1295 "fill": 1, 1296 "fillGradient": 0, 1297 "gridPos": { 1298 "h": 6, 1299 "w": 7, 1300 "x": 10, 1301 "y": 24 1302 }, 1303 "hiddenSeries": false, 1304 "id": 30, 1305 "legend": { 1306 "avg": false, 1307 "current": true, 1308 "max": false, 1309 "min": false, 1310 "show": true, 1311 "total": false, 1312 "values": true 1313 }, 1314 "lines": true, 1315 "linewidth": 1, 1316 "nullPointMode": "null", 1317 "options": { 1318 "alertThreshold": true 1319 }, 1320 "percentage": false, 1321 "pluginVersion": "9.2.4", 1322 "pointradius": 2, 1323 "points": false, 1324 "renderer": "flot", 1325 "seriesOverrides": [], 1326 "spaceLength": 10, 1327 "stack": false, 1328 "steppedLine": false, 1329 "targets": [ 1330 { 1331 "datasource": { 1332 "type": "prometheus", 1333 "uid": "prometheus" 1334 }, 1335 "editorMode": "code", 1336 "expr": "loki_cache_corrupt_chunks_total", 1337 "interval": "", 1338 "legendFormat": "{{name}}", 1339 "range": true, 1340 "refId": "A" 1341 } 1342 ], 1343 "thresholds": [], 1344 "timeRegions": [], 1345 "title": "Corrupt Chunks Total", 1346 "tooltip": { 1347 "shared": true, 1348 "sort": 0, 1349 "value_type": "individual" 1350 }, 1351 "type": "graph", 1352 "xaxis": { 1353 "mode": "time", 1354 "show": true, 1355 "values": [] 1356 }, 1357 "yaxes": [ 1358 { 1359 "$$hashKey": "object:1092", 1360 "format": "short", 1361 "logBase": 1, 1362 "min": "0", 1363 "show": true 1364 }, 1365 { 1366 "$$hashKey": "object:1093", 1367 "format": "short", 1368 "logBase": 1, 1369 "show": true 1370 } 1371 ], 1372 "yaxis": { 1373 "align": false 1374 } 1375 }, 1376 { 1377 "aliasColors": {}, 1378 "bars": false, 1379 "dashLength": 10, 1380 "dashes": false, 1381 "datasource": { 1382 "type": "prometheus", 1383 "uid": "prometheus" 1384 }, 1385 "fill": 1, 1386 "fillGradient": 0, 1387 "gridPos": { 1388 "h": 6, 1389 "w": 7, 1390 "x": 17, 1391 "y": 24 1392 }, 1393 "hiddenSeries": false, 1394 "id": 26, 1395 "legend": { 1396 "avg": false, 1397 "current": true, 1398 "max": false, 1399 "min": false, 1400 "show": true, 1401 "total": false, 1402 "values": true 1403 }, 1404 "lines": true, 1405 "linewidth": 1, 1406 "nullPointMode": "null", 1407 "options": { 1408 "alertThreshold": true 1409 }, 1410 "percentage": false, 1411 "pluginVersion": "9.2.4", 1412 "pointradius": 2, 1413 "points": false, 1414 "renderer": "flot", 1415 "seriesOverrides": [], 1416 "spaceLength": 10, 1417 "stack": false, 1418 "steppedLine": false, 1419 "targets": [ 1420 { 1421 "datasource": { 1422 "type": "prometheus", 1423 "uid": "prometheus" 1424 }, 1425 "editorMode": "code", 1426 "expr": "loki_cache_fetched_keys", 1427 "interval": "", 1428 "legendFormat": "{{name}}", 1429 "range": true, 1430 "refId": "A" 1431 } 1432 ], 1433 "thresholds": [], 1434 "timeRegions": [], 1435 "title": "Fetched Keys", 1436 "tooltip": { 1437 "shared": true, 1438 "sort": 0, 1439 "value_type": "individual" 1440 }, 1441 "type": "graph", 1442 "xaxis": { 1443 "mode": "time", 1444 "show": true, 1445 "values": [] 1446 }, 1447 "yaxes": [ 1448 { 1449 "$$hashKey": "object:1092", 1450 "format": "short", 1451 "logBase": 1, 1452 "show": true 1453 }, 1454 { 1455 "$$hashKey": "object:1093", 1456 "format": "short", 1457 "logBase": 1, 1458 "show": true 1459 } 1460 ], 1461 "yaxis": { 1462 "align": false 1463 } 1464 }, 1465 { 1466 "collapsed": false, 1467 "datasource": { 1468 "type": "prometheus", 1469 "uid": "prometheus" 1470 }, 1471 "gridPos": { 1472 "h": 1, 1473 "w": 24, 1474 "x": 0, 1475 "y": 30 1476 }, 1477 "id": 19, 1478 "panels": [], 1479 "targets": [ 1480 { 1481 "datasource": { 1482 "type": "prometheus", 1483 "uid": "prometheus" 1484 }, 1485 "refId": "A" 1486 } 1487 ], 1488 "title": "Distributor", 1489 "type": "row" 1490 }, 1491 { 1492 "aliasColors": {}, 1493 "bars": false, 1494 "dashLength": 10, 1495 "dashes": false, 1496 "datasource": { 1497 "type": "prometheus", 1498 "uid": "prometheus" 1499 }, 1500 "fill": 1, 1501 "fillGradient": 0, 1502 "gridPos": { 1503 "h": 6, 1504 "w": 6, 1505 "x": 0, 1506 "y": 31 1507 }, 1508 "hiddenSeries": false, 1509 "id": 20, 1510 "legend": { 1511 "avg": false, 1512 "current": false, 1513 "max": false, 1514 "min": false, 1515 "show": true, 1516 "total": false, 1517 "values": false 1518 }, 1519 "lines": true, 1520 "linewidth": 1, 1521 "nullPointMode": "null", 1522 "options": { 1523 "alertThreshold": true 1524 }, 1525 "percentage": false, 1526 "pluginVersion": "9.2.4", 1527 "pointradius": 2, 1528 "points": false, 1529 "renderer": "flot", 1530 "seriesOverrides": [], 1531 "spaceLength": 10, 1532 "stack": false, 1533 "steppedLine": false, 1534 "targets": [ 1535 { 1536 "datasource": { 1537 "type": "prometheus", 1538 "uid": "prometheus" 1539 }, 1540 "expr": "sum by (tenant) (rate(loki_distributor_lines_received_total[5m]))", 1541 "interval": "", 1542 "legendFormat": "distributor / {{ tenant}}", 1543 "refId": "A" 1544 } 1545 ], 1546 "thresholds": [], 1547 "timeRegions": [], 1548 "title": "Received Lines / sec", 1549 "tooltip": { 1550 "shared": true, 1551 "sort": 0, 1552 "value_type": "individual" 1553 }, 1554 "type": "graph", 1555 "xaxis": { 1556 "mode": "time", 1557 "show": true, 1558 "values": [] 1559 }, 1560 "yaxes": [ 1561 { 1562 "$$hashKey": "object:1092", 1563 "format": "short", 1564 "logBase": 1, 1565 "show": true 1566 }, 1567 { 1568 "$$hashKey": "object:1093", 1569 "format": "short", 1570 "logBase": 1, 1571 "show": true 1572 } 1573 ], 1574 "yaxis": { 1575 "align": false 1576 } 1577 }, 1578 { 1579 "aliasColors": {}, 1580 "bars": false, 1581 "dashLength": 10, 1582 "dashes": false, 1583 "datasource": { 1584 "type": "prometheus", 1585 "uid": "prometheus" 1586 }, 1587 "fill": 1, 1588 "fillGradient": 0, 1589 "gridPos": { 1590 "h": 6, 1591 "w": 6, 1592 "x": 6, 1593 "y": 31 1594 }, 1595 "hiddenSeries": false, 1596 "id": 15, 1597 "legend": { 1598 "avg": false, 1599 "current": false, 1600 "max": false, 1601 "min": false, 1602 "show": true, 1603 "total": false, 1604 "values": false 1605 }, 1606 "lines": true, 1607 "linewidth": 1, 1608 "nullPointMode": "null", 1609 "options": { 1610 "alertThreshold": true 1611 }, 1612 "percentage": false, 1613 "pluginVersion": "9.2.4", 1614 "pointradius": 2, 1615 "points": false, 1616 "renderer": "flot", 1617 "seriesOverrides": [], 1618 "spaceLength": 10, 1619 "stack": false, 1620 "steppedLine": false, 1621 "targets": [ 1622 { 1623 "datasource": { 1624 "type": "prometheus", 1625 "uid": "prometheus" 1626 }, 1627 "expr": "sum(rate(loki_distributor_bytes_received_total[5m])) by (tenant)", 1628 "interval": "", 1629 "legendFormat": "distributor / {{tenant}}", 1630 "refId": "A" 1631 } 1632 ], 1633 "thresholds": [], 1634 "timeRegions": [], 1635 "title": "Distributor Received bytes / sec", 1636 "tooltip": { 1637 "shared": true, 1638 "sort": 0, 1639 "value_type": "individual" 1640 }, 1641 "type": "graph", 1642 "xaxis": { 1643 "mode": "time", 1644 "show": true, 1645 "values": [] 1646 }, 1647 "yaxes": [ 1648 { 1649 "$$hashKey": "object:837", 1650 "format": "Bps", 1651 "logBase": 1, 1652 "show": true 1653 }, 1654 { 1655 "$$hashKey": "object:838", 1656 "format": "short", 1657 "logBase": 1, 1658 "show": true 1659 } 1660 ], 1661 "yaxis": { 1662 "align": false 1663 } 1664 }, 1665 { 1666 "aliasColors": {}, 1667 "bars": false, 1668 "dashLength": 10, 1669 "dashes": false, 1670 "datasource": { 1671 "type": "prometheus", 1672 "uid": "prometheus" 1673 }, 1674 "fill": 1, 1675 "fillGradient": 0, 1676 "gridPos": { 1677 "h": 6, 1678 "w": 6, 1679 "x": 12, 1680 "y": 31 1681 }, 1682 "hiddenSeries": false, 1683 "id": 17, 1684 "legend": { 1685 "avg": false, 1686 "current": true, 1687 "max": false, 1688 "min": false, 1689 "show": true, 1690 "total": false, 1691 "values": true 1692 }, 1693 "lines": true, 1694 "linewidth": 1, 1695 "nullPointMode": "null", 1696 "options": { 1697 "alertThreshold": true 1698 }, 1699 "percentage": false, 1700 "pluginVersion": "9.2.4", 1701 "pointradius": 2, 1702 "points": false, 1703 "renderer": "flot", 1704 "seriesOverrides": [], 1705 "spaceLength": 10, 1706 "stack": false, 1707 "steppedLine": false, 1708 "targets": [ 1709 { 1710 "datasource": { 1711 "type": "prometheus", 1712 "uid": "prometheus" 1713 }, 1714 "expr": "rate(loki_distributor_ingester_appends_total[5m])", 1715 "interval": "", 1716 "legendFormat": "successed", 1717 "refId": "A" 1718 }, 1719 { 1720 "datasource": { 1721 "type": "prometheus", 1722 "uid": "prometheus" 1723 }, 1724 "expr": "loki_distributor_ingester_append_failures_total", 1725 "interval": "", 1726 "legendFormat": "failed", 1727 "refId": "B" 1728 } 1729 ], 1730 "thresholds": [], 1731 "timeRegions": [], 1732 "title": "batch appends sent to ingesters / sec", 1733 "tooltip": { 1734 "shared": true, 1735 "sort": 0, 1736 "value_type": "individual" 1737 }, 1738 "type": "graph", 1739 "xaxis": { 1740 "mode": "time", 1741 "show": true, 1742 "values": [] 1743 }, 1744 "yaxes": [ 1745 { 1746 "$$hashKey": "object:1092", 1747 "format": "short", 1748 "logBase": 1, 1749 "show": true 1750 }, 1751 { 1752 "$$hashKey": "object:1093", 1753 "format": "short", 1754 "logBase": 1, 1755 "show": true 1756 } 1757 ], 1758 "yaxis": { 1759 "align": false 1760 } 1761 }, 1762 { 1763 "aliasColors": {}, 1764 "bars": false, 1765 "dashLength": 10, 1766 "dashes": false, 1767 "datasource": { 1768 "type": "prometheus", 1769 "uid": "prometheus" 1770 }, 1771 "fieldConfig": { 1772 "defaults": { 1773 "links": [], 1774 "unit": "short" 1775 }, 1776 "overrides": [] 1777 }, 1778 "fill": 1, 1779 "fillGradient": 0, 1780 "gridPos": { 1781 "h": 6, 1782 "w": 6, 1783 "x": 18, 1784 "y": 31 1785 }, 1786 "hiddenSeries": false, 1787 "id": 22, 1788 "interval": "", 1789 "legend": { 1790 "avg": false, 1791 "current": true, 1792 "max": false, 1793 "min": false, 1794 "show": true, 1795 "total": false, 1796 "values": true 1797 }, 1798 "lines": true, 1799 "linewidth": 1, 1800 "nullPointMode": "null", 1801 "options": { 1802 "alertThreshold": true 1803 }, 1804 "percentage": false, 1805 "pluginVersion": "9.2.4", 1806 "pointradius": 2, 1807 "points": false, 1808 "renderer": "flot", 1809 "seriesOverrides": [], 1810 "spaceLength": 10, 1811 "stack": false, 1812 "steppedLine": false, 1813 "targets": [ 1814 { 1815 "datasource": { 1816 "type": "prometheus", 1817 "uid": "prometheus" 1818 }, 1819 "expr": "sum by (status) (rate(loki_store_series_total[5m]))", 1820 "instant": false, 1821 "interval": "", 1822 "legendFormat": "{{status}}", 1823 "refId": "A" 1824 } 1825 ], 1826 "thresholds": [], 1827 "timeRegions": [], 1828 "title": "Store Series / sec", 1829 "tooltip": { 1830 "shared": true, 1831 "sort": 0, 1832 "value_type": "individual" 1833 }, 1834 "type": "graph", 1835 "xaxis": { 1836 "mode": "time", 1837 "show": true, 1838 "values": [] 1839 }, 1840 "yaxes": [ 1841 { 1842 "$$hashKey": "object:88", 1843 "format": "short", 1844 "logBase": 1, 1845 "show": true 1846 }, 1847 { 1848 "$$hashKey": "object:89", 1849 "format": "short", 1850 "logBase": 1, 1851 "show": true 1852 } 1853 ], 1854 "yaxis": { 1855 "align": false 1856 } 1857 }, 1858 { 1859 "collapsed": false, 1860 "datasource": { 1861 "type": "prometheus", 1862 "uid": "prometheus" 1863 }, 1864 "gridPos": { 1865 "h": 1, 1866 "w": 24, 1867 "x": 0, 1868 "y": 37 1869 }, 1870 "id": 66, 1871 "panels": [], 1872 "targets": [ 1873 { 1874 "datasource": { 1875 "type": "prometheus", 1876 "uid": "prometheus" 1877 }, 1878 "refId": "A" 1879 } 1880 ], 1881 "title": "Ingster", 1882 "type": "row" 1883 }, 1884 { 1885 "aliasColors": {}, 1886 "bars": false, 1887 "dashLength": 10, 1888 "dashes": false, 1889 "datasource": { 1890 "type": "prometheus", 1891 "uid": "prometheus" 1892 }, 1893 "fieldConfig": { 1894 "defaults": { 1895 "links": [] 1896 }, 1897 "overrides": [] 1898 }, 1899 "fill": 0, 1900 "fillGradient": 0, 1901 "gridPos": { 1902 "h": 6, 1903 "w": 8, 1904 "x": 0, 1905 "y": 38 1906 }, 1907 "hiddenSeries": false, 1908 "id": 71, 1909 "interval": "", 1910 "legend": { 1911 "alignAsTable": false, 1912 "avg": false, 1913 "current": true, 1914 "max": false, 1915 "min": false, 1916 "rightSide": false, 1917 "show": true, 1918 "sort": "current", 1919 "sortDesc": true, 1920 "total": false, 1921 "values": true 1922 }, 1923 "lines": true, 1924 "linewidth": 1, 1925 "nullPointMode": "null as zero", 1926 "options": { 1927 "alertThreshold": true 1928 }, 1929 "percentage": false, 1930 "pluginVersion": "9.2.4", 1931 "pointradius": 2, 1932 "points": false, 1933 "renderer": "flot", 1934 "seriesOverrides": [], 1935 "spaceLength": 10, 1936 "stack": false, 1937 "steppedLine": false, 1938 "targets": [ 1939 { 1940 "datasource": { 1941 "type": "prometheus", 1942 "uid": "prometheus" 1943 }, 1944 "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_blocks_per_chunk_bucket[5m])) by (le))", 1945 "interval": "", 1946 "legendFormat": "blocks", 1947 "refId": "A" 1948 } 1949 ], 1950 "thresholds": [], 1951 "timeRegions": [], 1952 "title": "Blocks / Chunk", 1953 "tooltip": { 1954 "shared": true, 1955 "sort": 0, 1956 "value_type": "individual" 1957 }, 1958 "type": "graph", 1959 "xaxis": { 1960 "mode": "time", 1961 "show": true, 1962 "values": [] 1963 }, 1964 "yaxes": [ 1965 { 1966 "$$hashKey": "object:155", 1967 "format": "none", 1968 "logBase": 1, 1969 "show": true 1970 }, 1971 { 1972 "$$hashKey": "object:156", 1973 "format": "short", 1974 "logBase": 1, 1975 "show": true 1976 } 1977 ], 1978 "yaxis": { 1979 "align": false 1980 } 1981 }, 1982 { 1983 "aliasColors": {}, 1984 "bars": false, 1985 "dashLength": 10, 1986 "dashes": false, 1987 "datasource": { 1988 "type": "prometheus", 1989 "uid": "prometheus" 1990 }, 1991 "fieldConfig": { 1992 "defaults": { 1993 "links": [] 1994 }, 1995 "overrides": [] 1996 }, 1997 "fill": 0, 1998 "fillGradient": 0, 1999 "gridPos": { 2000 "h": 6, 2001 "w": 8, 2002 "x": 8, 2003 "y": 38 2004 }, 2005 "hiddenSeries": false, 2006 "id": 68, 2007 "interval": "", 2008 "legend": { 2009 "alignAsTable": false, 2010 "avg": false, 2011 "current": true, 2012 "max": false, 2013 "min": false, 2014 "rightSide": false, 2015 "show": true, 2016 "sort": "current", 2017 "sortDesc": true, 2018 "total": false, 2019 "values": true 2020 }, 2021 "lines": true, 2022 "linewidth": 1, 2023 "nullPointMode": "null as zero", 2024 "options": { 2025 "alertThreshold": true 2026 }, 2027 "percentage": false, 2028 "pluginVersion": "9.2.4", 2029 "pointradius": 2, 2030 "points": false, 2031 "renderer": "flot", 2032 "seriesOverrides": [], 2033 "spaceLength": 10, 2034 "stack": false, 2035 "steppedLine": false, 2036 "targets": [ 2037 { 2038 "datasource": { 2039 "type": "prometheus", 2040 "uid": "prometheus" 2041 }, 2042 "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_size_bytes_bucket[5m])) by (le))", 2043 "interval": "", 2044 "legendFormat": "chunk size", 2045 "refId": "A" 2046 } 2047 ], 2048 "thresholds": [], 2049 "timeRegions": [], 2050 "title": "Chunk Size", 2051 "tooltip": { 2052 "shared": true, 2053 "sort": 0, 2054 "value_type": "individual" 2055 }, 2056 "type": "graph", 2057 "xaxis": { 2058 "mode": "time", 2059 "show": true, 2060 "values": [] 2061 }, 2062 "yaxes": [ 2063 { 2064 "$$hashKey": "object:155", 2065 "format": "decbytes", 2066 "logBase": 1, 2067 "show": true 2068 }, 2069 { 2070 "$$hashKey": "object:156", 2071 "format": "short", 2072 "logBase": 1, 2073 "show": true 2074 } 2075 ], 2076 "yaxis": { 2077 "align": false 2078 } 2079 }, 2080 { 2081 "aliasColors": {}, 2082 "bars": false, 2083 "dashLength": 10, 2084 "dashes": false, 2085 "datasource": { 2086 "type": "prometheus", 2087 "uid": "prometheus" 2088 }, 2089 "fieldConfig": { 2090 "defaults": { 2091 "links": [] 2092 }, 2093 "overrides": [] 2094 }, 2095 "fill": 0, 2096 "fillGradient": 0, 2097 "gridPos": { 2098 "h": 6, 2099 "w": 8, 2100 "x": 16, 2101 "y": 38 2102 }, 2103 "hiddenSeries": false, 2104 "id": 72, 2105 "interval": "", 2106 "legend": { 2107 "alignAsTable": false, 2108 "avg": false, 2109 "current": true, 2110 "max": false, 2111 "min": false, 2112 "rightSide": false, 2113 "show": true, 2114 "sort": "current", 2115 "sortDesc": true, 2116 "total": false, 2117 "values": true 2118 }, 2119 "lines": true, 2120 "linewidth": 1, 2121 "nullPointMode": "null as zero", 2122 "options": { 2123 "alertThreshold": true 2124 }, 2125 "percentage": false, 2126 "pluginVersion": "9.2.4", 2127 "pointradius": 2, 2128 "points": false, 2129 "renderer": "flot", 2130 "seriesOverrides": [], 2131 "spaceLength": 10, 2132 "stack": false, 2133 "steppedLine": false, 2134 "targets": [ 2135 { 2136 "datasource": { 2137 "type": "prometheus", 2138 "uid": "prometheus" 2139 }, 2140 "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_age_seconds_bucket[5m])) by (le))", 2141 "interval": "", 2142 "legendFormat": "ages", 2143 "refId": "A" 2144 } 2145 ], 2146 "thresholds": [], 2147 "timeRegions": [], 2148 "title": "Chunk Age", 2149 "tooltip": { 2150 "shared": true, 2151 "sort": 0, 2152 "value_type": "individual" 2153 }, 2154 "type": "graph", 2155 "xaxis": { 2156 "mode": "time", 2157 "show": true, 2158 "values": [] 2159 }, 2160 "yaxes": [ 2161 { 2162 "$$hashKey": "object:155", 2163 "format": "s", 2164 "logBase": 1, 2165 "show": true 2166 }, 2167 { 2168 "$$hashKey": "object:156", 2169 "format": "short", 2170 "logBase": 1, 2171 "show": true 2172 } 2173 ], 2174 "yaxis": { 2175 "align": false 2176 } 2177 }, 2178 { 2179 "aliasColors": {}, 2180 "bars": false, 2181 "dashLength": 10, 2182 "dashes": false, 2183 "datasource": { 2184 "type": "prometheus", 2185 "uid": "prometheus" 2186 }, 2187 "fieldConfig": { 2188 "defaults": { 2189 "links": [] 2190 }, 2191 "overrides": [] 2192 }, 2193 "fill": 0, 2194 "fillGradient": 0, 2195 "gridPos": { 2196 "h": 6, 2197 "w": 8, 2198 "x": 0, 2199 "y": 44 2200 }, 2201 "hiddenSeries": false, 2202 "id": 70, 2203 "interval": "", 2204 "legend": { 2205 "alignAsTable": false, 2206 "avg": false, 2207 "current": true, 2208 "max": false, 2209 "min": false, 2210 "rightSide": false, 2211 "show": true, 2212 "sort": "current", 2213 "sortDesc": true, 2214 "total": false, 2215 "values": true 2216 }, 2217 "lines": true, 2218 "linewidth": 1, 2219 "nullPointMode": "null as zero", 2220 "options": { 2221 "alertThreshold": true 2222 }, 2223 "percentage": false, 2224 "pluginVersion": "9.2.4", 2225 "pointradius": 2, 2226 "points": false, 2227 "renderer": "flot", 2228 "seriesOverrides": [], 2229 "spaceLength": 10, 2230 "stack": false, 2231 "steppedLine": false, 2232 "targets": [ 2233 { 2234 "datasource": { 2235 "type": "prometheus", 2236 "uid": "prometheus" 2237 }, 2238 "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_compression_ratio_bucket[5m])) by (le))", 2239 "interval": "", 2240 "legendFormat": "ratio", 2241 "refId": "A" 2242 } 2243 ], 2244 "thresholds": [], 2245 "timeRegions": [], 2246 "title": "Chunk Compression Ratios", 2247 "tooltip": { 2248 "shared": true, 2249 "sort": 0, 2250 "value_type": "individual" 2251 }, 2252 "type": "graph", 2253 "xaxis": { 2254 "mode": "time", 2255 "show": true, 2256 "values": [] 2257 }, 2258 "yaxes": [ 2259 { 2260 "$$hashKey": "object:155", 2261 "format": "percent", 2262 "logBase": 1, 2263 "show": true 2264 }, 2265 { 2266 "$$hashKey": "object:156", 2267 "format": "short", 2268 "logBase": 1, 2269 "show": true 2270 } 2271 ], 2272 "yaxis": { 2273 "align": false 2274 } 2275 }, 2276 { 2277 "aliasColors": {}, 2278 "bars": false, 2279 "dashLength": 10, 2280 "dashes": false, 2281 "datasource": { 2282 "type": "prometheus", 2283 "uid": "prometheus" 2284 }, 2285 "fieldConfig": { 2286 "defaults": { 2287 "links": [] 2288 }, 2289 "overrides": [] 2290 }, 2291 "fill": 0, 2292 "fillGradient": 0, 2293 "gridPos": { 2294 "h": 6, 2295 "w": 8, 2296 "x": 8, 2297 "y": 44 2298 }, 2299 "hiddenSeries": false, 2300 "id": 69, 2301 "interval": "", 2302 "legend": { 2303 "alignAsTable": false, 2304 "avg": false, 2305 "current": true, 2306 "max": false, 2307 "min": false, 2308 "rightSide": false, 2309 "show": true, 2310 "sort": "current", 2311 "sortDesc": true, 2312 "total": false, 2313 "values": true 2314 }, 2315 "lines": true, 2316 "linewidth": 1, 2317 "nullPointMode": "null as zero", 2318 "options": { 2319 "alertThreshold": true 2320 }, 2321 "percentage": false, 2322 "pluginVersion": "9.2.4", 2323 "pointradius": 2, 2324 "points": false, 2325 "renderer": "flot", 2326 "seriesOverrides": [], 2327 "spaceLength": 10, 2328 "stack": false, 2329 "steppedLine": false, 2330 "targets": [ 2331 { 2332 "datasource": { 2333 "type": "prometheus", 2334 "uid": "prometheus" 2335 }, 2336 "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_encode_time_seconds_bucket[5m])) by (le))", 2337 "interval": "", 2338 "legendFormat": "durations.", 2339 "refId": "A" 2340 } 2341 ], 2342 "thresholds": [], 2343 "timeRegions": [], 2344 "title": "Chunk Encode Time", 2345 "tooltip": { 2346 "shared": true, 2347 "sort": 0, 2348 "value_type": "individual" 2349 }, 2350 "type": "graph", 2351 "xaxis": { 2352 "mode": "time", 2353 "show": true, 2354 "values": [] 2355 }, 2356 "yaxes": [ 2357 { 2358 "$$hashKey": "object:155", 2359 "format": "s", 2360 "logBase": 1, 2361 "show": true 2362 }, 2363 { 2364 "$$hashKey": "object:156", 2365 "format": "short", 2366 "logBase": 1, 2367 "show": true 2368 } 2369 ], 2370 "yaxis": { 2371 "align": false 2372 } 2373 }, 2374 { 2375 "aliasColors": {}, 2376 "bars": false, 2377 "dashLength": 10, 2378 "dashes": false, 2379 "datasource": { 2380 "type": "prometheus", 2381 "uid": "prometheus" 2382 }, 2383 "fieldConfig": { 2384 "defaults": { 2385 "links": [] 2386 }, 2387 "overrides": [] 2388 }, 2389 "fill": 0, 2390 "fillGradient": 0, 2391 "gridPos": { 2392 "h": 6, 2393 "w": 8, 2394 "x": 16, 2395 "y": 44 2396 }, 2397 "hiddenSeries": false, 2398 "id": 73, 2399 "interval": "", 2400 "legend": { 2401 "alignAsTable": false, 2402 "avg": false, 2403 "current": true, 2404 "max": false, 2405 "min": false, 2406 "rightSide": false, 2407 "show": true, 2408 "sort": "current", 2409 "sortDesc": true, 2410 "total": false, 2411 "values": true 2412 }, 2413 "lines": true, 2414 "linewidth": 1, 2415 "nullPointMode": "null as zero", 2416 "options": { 2417 "alertThreshold": true 2418 }, 2419 "percentage": false, 2420 "pluginVersion": "9.2.4", 2421 "pointradius": 2, 2422 "points": false, 2423 "renderer": "flot", 2424 "seriesOverrides": [], 2425 "spaceLength": 10, 2426 "stack": false, 2427 "steppedLine": false, 2428 "targets": [ 2429 { 2430 "datasource": { 2431 "type": "prometheus", 2432 "uid": "prometheus" 2433 }, 2434 "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_entries_bucket[5m])) by (le))", 2435 "interval": "", 2436 "legendFormat": "lines", 2437 "refId": "A" 2438 } 2439 ], 2440 "thresholds": [], 2441 "timeRegions": [], 2442 "title": "Lines / Chunk", 2443 "tooltip": { 2444 "shared": true, 2445 "sort": 0, 2446 "value_type": "individual" 2447 }, 2448 "type": "graph", 2449 "xaxis": { 2450 "mode": "time", 2451 "show": true, 2452 "values": [] 2453 }, 2454 "yaxes": [ 2455 { 2456 "$$hashKey": "object:155", 2457 "format": "none", 2458 "logBase": 1, 2459 "show": true 2460 }, 2461 { 2462 "$$hashKey": "object:156", 2463 "format": "short", 2464 "logBase": 1, 2465 "show": true 2466 } 2467 ], 2468 "yaxis": { 2469 "align": false 2470 } 2471 }, 2472 { 2473 "aliasColors": {}, 2474 "bars": false, 2475 "dashLength": 10, 2476 "dashes": false, 2477 "datasource": { 2478 "type": "prometheus", 2479 "uid": "prometheus" 2480 }, 2481 "fill": 1, 2482 "fillGradient": 0, 2483 "gridPos": { 2484 "h": 6, 2485 "w": 12, 2486 "x": 0, 2487 "y": 50 2488 }, 2489 "hiddenSeries": false, 2490 "id": 60, 2491 "legend": { 2492 "alignAsTable": true, 2493 "avg": false, 2494 "current": true, 2495 "max": false, 2496 "min": false, 2497 "rightSide": true, 2498 "show": true, 2499 "total": false, 2500 "values": true 2501 }, 2502 "lines": true, 2503 "linewidth": 1, 2504 "nullPointMode": "null", 2505 "options": { 2506 "alertThreshold": true 2507 }, 2508 "percentage": false, 2509 "pluginVersion": "9.2.4", 2510 "pointradius": 2, 2511 "points": false, 2512 "renderer": "flot", 2513 "seriesOverrides": [], 2514 "spaceLength": 10, 2515 "stack": false, 2516 "steppedLine": false, 2517 "targets": [ 2518 { 2519 "datasource": { 2520 "type": "prometheus", 2521 "uid": "prometheus" 2522 }, 2523 "expr": "rate(loki_ingester_chunks_created_total[5m])", 2524 "interval": "", 2525 "legendFormat": "create", 2526 "refId": "A" 2527 }, 2528 { 2529 "datasource": { 2530 "type": "prometheus", 2531 "uid": "prometheus" 2532 }, 2533 "expr": "sum by (reason) (rate(loki_ingester_chunks_flushed_total[5m]))", 2534 "interval": "", 2535 "legendFormat": "flush / {{ reason }}", 2536 "refId": "B" 2537 }, 2538 { 2539 "datasource": { 2540 "type": "prometheus", 2541 "uid": "prometheus" 2542 }, 2543 "expr": "sum by (fake) (rate(loki_ingester_chunks_stored_total[5m]))", 2544 "interval": "", 2545 "legendFormat": "stored", 2546 "refId": "C" 2547 }, 2548 { 2549 "datasource": { 2550 "type": "prometheus", 2551 "uid": "prometheus" 2552 }, 2553 "expr": "", 2554 "interval": "", 2555 "legendFormat": "", 2556 "refId": "D" 2557 } 2558 ], 2559 "thresholds": [], 2560 "timeRegions": [], 2561 "title": "Chunk Status", 2562 "tooltip": { 2563 "shared": true, 2564 "sort": 0, 2565 "value_type": "individual" 2566 }, 2567 "type": "graph", 2568 "xaxis": { 2569 "mode": "time", 2570 "show": true, 2571 "values": [] 2572 }, 2573 "yaxes": [ 2574 { 2575 "$$hashKey": "object:1092", 2576 "format": "short", 2577 "logBase": 1, 2578 "show": true 2579 }, 2580 { 2581 "$$hashKey": "object:1093", 2582 "format": "short", 2583 "logBase": 1, 2584 "show": true 2585 } 2586 ], 2587 "yaxis": { 2588 "align": false 2589 } 2590 }, 2591 { 2592 "aliasColors": {}, 2593 "bars": false, 2594 "dashLength": 10, 2595 "dashes": false, 2596 "datasource": { 2597 "type": "prometheus", 2598 "uid": "prometheus" 2599 }, 2600 "fill": 1, 2601 "fillGradient": 0, 2602 "gridPos": { 2603 "h": 6, 2604 "w": 6, 2605 "x": 12, 2606 "y": 50 2607 }, 2608 "hiddenSeries": false, 2609 "id": 61, 2610 "legend": { 2611 "avg": false, 2612 "current": true, 2613 "max": false, 2614 "min": false, 2615 "show": true, 2616 "total": false, 2617 "values": true 2618 }, 2619 "lines": true, 2620 "linewidth": 1, 2621 "nullPointMode": "null", 2622 "options": { 2623 "alertThreshold": true 2624 }, 2625 "percentage": false, 2626 "pluginVersion": "9.2.4", 2627 "pointradius": 2, 2628 "points": false, 2629 "renderer": "flot", 2630 "seriesOverrides": [], 2631 "spaceLength": 10, 2632 "stack": false, 2633 "steppedLine": false, 2634 "targets": [ 2635 { 2636 "datasource": { 2637 "type": "prometheus", 2638 "uid": "prometheus" 2639 }, 2640 "expr": "loki_ingester_memory_chunks", 2641 "interval": "", 2642 "legendFormat": "chunks", 2643 "refId": "A" 2644 } 2645 ], 2646 "thresholds": [], 2647 "timeRegions": [], 2648 "title": "Chunks in Memory", 2649 "tooltip": { 2650 "shared": true, 2651 "sort": 0, 2652 "value_type": "individual" 2653 }, 2654 "type": "graph", 2655 "xaxis": { 2656 "mode": "time", 2657 "show": true, 2658 "values": [] 2659 }, 2660 "yaxes": [ 2661 { 2662 "$$hashKey": "object:1092", 2663 "format": "short", 2664 "logBase": 1, 2665 "show": true 2666 }, 2667 { 2668 "$$hashKey": "object:1093", 2669 "format": "short", 2670 "logBase": 1, 2671 "show": true 2672 } 2673 ], 2674 "yaxis": { 2675 "align": false 2676 } 2677 }, 2678 { 2679 "aliasColors": {}, 2680 "bars": false, 2681 "dashLength": 10, 2682 "dashes": false, 2683 "datasource": { 2684 "type": "prometheus", 2685 "uid": "prometheus" 2686 }, 2687 "fill": 1, 2688 "fillGradient": 0, 2689 "gridPos": { 2690 "h": 6, 2691 "w": 6, 2692 "x": 18, 2693 "y": 50 2694 }, 2695 "hiddenSeries": false, 2696 "id": 62, 2697 "legend": { 2698 "avg": false, 2699 "current": true, 2700 "max": false, 2701 "min": false, 2702 "show": true, 2703 "total": false, 2704 "values": true 2705 }, 2706 "lines": true, 2707 "linewidth": 1, 2708 "nullPointMode": "null", 2709 "options": { 2710 "alertThreshold": true 2711 }, 2712 "percentage": false, 2713 "pluginVersion": "9.2.4", 2714 "pointradius": 2, 2715 "points": false, 2716 "renderer": "flot", 2717 "seriesOverrides": [], 2718 "spaceLength": 10, 2719 "stack": false, 2720 "steppedLine": false, 2721 "targets": [ 2722 { 2723 "datasource": { 2724 "type": "prometheus", 2725 "uid": "prometheus" 2726 }, 2727 "expr": "sum by (tenan) (loki_ingester_memory_streams)", 2728 "interval": "", 2729 "legendFormat": "streams", 2730 "refId": "A" 2731 } 2732 ], 2733 "thresholds": [], 2734 "timeRegions": [], 2735 "title": "Chunks in Streams", 2736 "tooltip": { 2737 "shared": true, 2738 "sort": 0, 2739 "value_type": "individual" 2740 }, 2741 "type": "graph", 2742 "xaxis": { 2743 "mode": "time", 2744 "show": true, 2745 "values": [] 2746 }, 2747 "yaxes": [ 2748 { 2749 "$$hashKey": "object:1092", 2750 "format": "short", 2751 "logBase": 1, 2752 "show": true 2753 }, 2754 { 2755 "$$hashKey": "object:1093", 2756 "format": "short", 2757 "logBase": 1, 2758 "show": true 2759 } 2760 ], 2761 "yaxis": { 2762 "align": false 2763 } 2764 }, 2765 { 2766 "aliasColors": {}, 2767 "bars": false, 2768 "dashLength": 10, 2769 "dashes": false, 2770 "datasource": { 2771 "type": "prometheus", 2772 "uid": "prometheus" 2773 }, 2774 "fill": 1, 2775 "fillGradient": 0, 2776 "gridPos": { 2777 "h": 6, 2778 "w": 12, 2779 "x": 0, 2780 "y": 56 2781 }, 2782 "hiddenSeries": false, 2783 "id": 67, 2784 "legend": { 2785 "alignAsTable": true, 2786 "avg": false, 2787 "current": true, 2788 "max": false, 2789 "min": false, 2790 "rightSide": true, 2791 "show": true, 2792 "total": false, 2793 "values": true 2794 }, 2795 "lines": true, 2796 "linewidth": 1, 2797 "nullPointMode": "null", 2798 "options": { 2799 "alertThreshold": true 2800 }, 2801 "percentage": false, 2802 "pluginVersion": "9.2.4", 2803 "pointradius": 2, 2804 "points": false, 2805 "renderer": "flot", 2806 "seriesOverrides": [], 2807 "spaceLength": 10, 2808 "stack": false, 2809 "steppedLine": false, 2810 "targets": [ 2811 { 2812 "datasource": { 2813 "type": "prometheus", 2814 "uid": "prometheus" 2815 }, 2816 "expr": "rate(loki_ingester_streams_created_total[5m])", 2817 "interval": "", 2818 "legendFormat": "create", 2819 "refId": "A" 2820 }, 2821 { 2822 "datasource": { 2823 "type": "prometheus", 2824 "uid": "prometheus" 2825 }, 2826 "expr": "rate(loki_ingester_streams_removed_total[5m])", 2827 "interval": "", 2828 "legendFormat": "delete", 2829 "refId": "B" 2830 } 2831 ], 2832 "thresholds": [], 2833 "timeRegions": [], 2834 "title": "Strams Stats", 2835 "tooltip": { 2836 "shared": true, 2837 "sort": 0, 2838 "value_type": "individual" 2839 }, 2840 "type": "graph", 2841 "xaxis": { 2842 "mode": "time", 2843 "show": true, 2844 "values": [] 2845 }, 2846 "yaxes": [ 2847 { 2848 "$$hashKey": "object:1092", 2849 "format": "short", 2850 "logBase": 1, 2851 "show": true 2852 }, 2853 { 2854 "$$hashKey": "object:1093", 2855 "format": "short", 2856 "logBase": 1, 2857 "show": true 2858 } 2859 ], 2860 "yaxis": { 2861 "align": false 2862 } 2863 } 2864 ], 2865 "schemaVersion": 37, 2866 "style": "dark", 2867 "tags": [ 2868 "loki" 2869 ], 2870 "templating": { 2871 "list": [] 2872 }, 2873 "time": { 2874 "from": "now-1h", 2875 "to": "now" 2876 }, 2877 "timepicker": { 2878 "refresh_intervals": [ 2879 "5s", 2880 "10s", 2881 "30s", 2882 "1m", 2883 "5m", 2884 "15m", 2885 "30m", 2886 "1h", 2887 "2h", 2888 "1d" 2889 ] 2890 }, 2891 "timezone": "", 2892 "title": "Loki", 2893 "uid": "loki", 2894 "version": 1, 2895 "weekStart": "" 2896 }