github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/production/loki-mixin-compiled/dashboards/loki-chunks.json (about) 1 { 2 "annotations": { 3 "list": [ ] 4 }, 5 "editable": true, 6 "gnetId": null, 7 "graphTooltip": 0, 8 "hideControls": false, 9 "links": [ 10 { 11 "asDropdown": true, 12 "icon": "external link", 13 "includeVars": true, 14 "keepTime": true, 15 "tags": [ 16 "loki" 17 ], 18 "targetBlank": false, 19 "title": "Loki Dashboards", 20 "type": "dashboards" 21 } 22 ], 23 "refresh": "10s", 24 "rows": [ 25 { 26 "collapse": false, 27 "height": "250px", 28 "panels": [ 29 { 30 "aliasColors": { }, 31 "bars": false, 32 "dashLength": 10, 33 "dashes": false, 34 "datasource": "$datasource", 35 "fill": 1, 36 "id": 1, 37 "legend": { 38 "avg": false, 39 "current": false, 40 "max": false, 41 "min": false, 42 "show": true, 43 "total": false, 44 "values": false 45 }, 46 "lines": true, 47 "linewidth": 1, 48 "links": [ ], 49 "nullPointMode": "null as zero", 50 "percentage": false, 51 "pointradius": 5, 52 "points": false, 53 "renderer": "flot", 54 "seriesOverrides": [ ], 55 "spaceLength": 10, 56 "span": 6, 57 "stack": false, 58 "steppedLine": false, 59 "targets": [ 60 { 61 "expr": "sum(loki_ingester_memory_chunks{cluster=\"$cluster\", job=~\"$namespace/ingester\"})", 62 "format": "time_series", 63 "intervalFactor": 2, 64 "legendFormat": "series", 65 "legendLink": null, 66 "step": 10 67 } 68 ], 69 "thresholds": [ ], 70 "timeFrom": null, 71 "timeShift": null, 72 "title": "Series", 73 "tooltip": { 74 "shared": true, 75 "sort": 2, 76 "value_type": "individual" 77 }, 78 "type": "graph", 79 "xaxis": { 80 "buckets": null, 81 "mode": "time", 82 "name": null, 83 "show": true, 84 "values": [ ] 85 }, 86 "yaxes": [ 87 { 88 "format": "short", 89 "label": null, 90 "logBase": 1, 91 "max": null, 92 "min": 0, 93 "show": true 94 }, 95 { 96 "format": "short", 97 "label": null, 98 "logBase": 1, 99 "max": null, 100 "min": null, 101 "show": false 102 } 103 ] 104 }, 105 { 106 "aliasColors": { }, 107 "bars": false, 108 "dashLength": 10, 109 "dashes": false, 110 "datasource": "$datasource", 111 "fill": 1, 112 "id": 2, 113 "legend": { 114 "avg": false, 115 "current": false, 116 "max": false, 117 "min": false, 118 "show": true, 119 "total": false, 120 "values": false 121 }, 122 "lines": true, 123 "linewidth": 1, 124 "links": [ ], 125 "nullPointMode": "null as zero", 126 "percentage": false, 127 "pointradius": 5, 128 "points": false, 129 "renderer": "flot", 130 "seriesOverrides": [ ], 131 "spaceLength": 10, 132 "span": 6, 133 "stack": false, 134 "steppedLine": false, 135 "targets": [ 136 { 137 "expr": "sum(loki_ingester_memory_chunks{cluster=\"$cluster\", job=~\"$namespace/ingester\"}) / sum(loki_ingester_memory_streams{cluster=\"$cluster\", job=~\"$namespace/ingester\"})", 138 "format": "time_series", 139 "intervalFactor": 2, 140 "legendFormat": "chunks", 141 "legendLink": null, 142 "step": 10 143 } 144 ], 145 "thresholds": [ ], 146 "timeFrom": null, 147 "timeShift": null, 148 "title": "Chunks per series", 149 "tooltip": { 150 "shared": true, 151 "sort": 2, 152 "value_type": "individual" 153 }, 154 "type": "graph", 155 "xaxis": { 156 "buckets": null, 157 "mode": "time", 158 "name": null, 159 "show": true, 160 "values": [ ] 161 }, 162 "yaxes": [ 163 { 164 "format": "short", 165 "label": null, 166 "logBase": 1, 167 "max": null, 168 "min": 0, 169 "show": true 170 }, 171 { 172 "format": "short", 173 "label": null, 174 "logBase": 1, 175 "max": null, 176 "min": null, 177 "show": false 178 } 179 ] 180 } 181 ], 182 "repeat": null, 183 "repeatIteration": null, 184 "repeatRowId": null, 185 "showTitle": true, 186 "title": "Active Series / Chunks", 187 "titleSize": "h6" 188 }, 189 { 190 "collapse": false, 191 "height": "250px", 192 "panels": [ 193 { 194 "aliasColors": { }, 195 "bars": false, 196 "dashLength": 10, 197 "dashes": false, 198 "datasource": "$datasource", 199 "fill": 1, 200 "id": 3, 201 "legend": { 202 "avg": false, 203 "current": false, 204 "max": false, 205 "min": false, 206 "show": true, 207 "total": false, 208 "values": false 209 }, 210 "lines": true, 211 "linewidth": 1, 212 "links": [ ], 213 "nullPointMode": "null as zero", 214 "percentage": false, 215 "pointradius": 5, 216 "points": false, 217 "renderer": "flot", 218 "seriesOverrides": [ ], 219 "spaceLength": 10, 220 "span": 6, 221 "stack": false, 222 "steppedLine": false, 223 "targets": [ 224 { 225 "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[$__rate_interval])) by (le)) * 1", 226 "format": "time_series", 227 "intervalFactor": 2, 228 "legendFormat": "99th Percentile", 229 "refId": "A", 230 "step": 10 231 }, 232 { 233 "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[$__rate_interval])) by (le)) * 1", 234 "format": "time_series", 235 "intervalFactor": 2, 236 "legendFormat": "50th Percentile", 237 "refId": "B", 238 "step": 10 239 }, 240 { 241 "expr": "sum(rate(loki_ingester_chunk_utilization_sum{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_utilization_count{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[$__rate_interval]))", 242 "format": "time_series", 243 "intervalFactor": 2, 244 "legendFormat": "Average", 245 "refId": "C", 246 "step": 10 247 } 248 ], 249 "thresholds": [ ], 250 "timeFrom": null, 251 "timeShift": null, 252 "title": "Utilization", 253 "tooltip": { 254 "shared": true, 255 "sort": 2, 256 "value_type": "individual" 257 }, 258 "type": "graph", 259 "xaxis": { 260 "buckets": null, 261 "mode": "time", 262 "name": null, 263 "show": true, 264 "values": [ ] 265 }, 266 "yaxes": [ 267 { 268 "format": "percentunit", 269 "label": null, 270 "logBase": 1, 271 "max": null, 272 "min": 0, 273 "show": true 274 }, 275 { 276 "format": "short", 277 "label": null, 278 "logBase": 1, 279 "max": null, 280 "min": null, 281 "show": false 282 } 283 ] 284 }, 285 { 286 "aliasColors": { }, 287 "bars": false, 288 "dashLength": 10, 289 "dashes": false, 290 "datasource": "$datasource", 291 "fill": 1, 292 "id": 4, 293 "legend": { 294 "avg": false, 295 "current": false, 296 "max": false, 297 "min": false, 298 "show": true, 299 "total": false, 300 "values": false 301 }, 302 "lines": true, 303 "linewidth": 1, 304 "links": [ ], 305 "nullPointMode": "null as zero", 306 "percentage": false, 307 "pointradius": 5, 308 "points": false, 309 "renderer": "flot", 310 "seriesOverrides": [ ], 311 "spaceLength": 10, 312 "span": 6, 313 "stack": false, 314 "steppedLine": false, 315 "targets": [ 316 { 317 "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[$__rate_interval])) by (le)) * 1e3", 318 "format": "time_series", 319 "intervalFactor": 2, 320 "legendFormat": "99th Percentile", 321 "refId": "A", 322 "step": 10 323 }, 324 { 325 "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[$__rate_interval])) by (le)) * 1e3", 326 "format": "time_series", 327 "intervalFactor": 2, 328 "legendFormat": "50th Percentile", 329 "refId": "B", 330 "step": 10 331 }, 332 { 333 "expr": "sum(rate(loki_ingester_chunk_age_seconds_sum{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[$__rate_interval])) * 1e3 / sum(rate(loki_ingester_chunk_age_seconds_count{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[$__rate_interval]))", 334 "format": "time_series", 335 "intervalFactor": 2, 336 "legendFormat": "Average", 337 "refId": "C", 338 "step": 10 339 } 340 ], 341 "thresholds": [ ], 342 "timeFrom": null, 343 "timeShift": null, 344 "title": "Age", 345 "tooltip": { 346 "shared": true, 347 "sort": 2, 348 "value_type": "individual" 349 }, 350 "type": "graph", 351 "xaxis": { 352 "buckets": null, 353 "mode": "time", 354 "name": null, 355 "show": true, 356 "values": [ ] 357 }, 358 "yaxes": [ 359 { 360 "format": "ms", 361 "label": null, 362 "logBase": 1, 363 "max": null, 364 "min": 0, 365 "show": true 366 }, 367 { 368 "format": "short", 369 "label": null, 370 "logBase": 1, 371 "max": null, 372 "min": null, 373 "show": false 374 } 375 ] 376 } 377 ], 378 "repeat": null, 379 "repeatIteration": null, 380 "repeatRowId": null, 381 "showTitle": true, 382 "title": "Flush Stats", 383 "titleSize": "h6" 384 }, 385 { 386 "collapse": false, 387 "height": "250px", 388 "panels": [ 389 { 390 "aliasColors": { }, 391 "bars": false, 392 "dashLength": 10, 393 "dashes": false, 394 "datasource": "$datasource", 395 "fill": 1, 396 "id": 5, 397 "legend": { 398 "avg": false, 399 "current": false, 400 "max": false, 401 "min": false, 402 "show": true, 403 "total": false, 404 "values": false 405 }, 406 "lines": true, 407 "linewidth": 1, 408 "links": [ ], 409 "nullPointMode": "null as zero", 410 "percentage": false, 411 "pointradius": 5, 412 "points": false, 413 "renderer": "flot", 414 "seriesOverrides": [ ], 415 "spaceLength": 10, 416 "span": 6, 417 "stack": false, 418 "steppedLine": false, 419 "targets": [ 420 { 421 "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_entries_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[$__rate_interval])) by (le)) * 1", 422 "format": "time_series", 423 "intervalFactor": 2, 424 "legendFormat": "99th Percentile", 425 "refId": "A", 426 "step": 10 427 }, 428 { 429 "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_entries_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[$__rate_interval])) by (le)) * 1", 430 "format": "time_series", 431 "intervalFactor": 2, 432 "legendFormat": "50th Percentile", 433 "refId": "B", 434 "step": 10 435 }, 436 { 437 "expr": "sum(rate(loki_ingester_chunk_entries_sum{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_entries_count{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[$__rate_interval]))", 438 "format": "time_series", 439 "intervalFactor": 2, 440 "legendFormat": "Average", 441 "refId": "C", 442 "step": 10 443 } 444 ], 445 "thresholds": [ ], 446 "timeFrom": null, 447 "timeShift": null, 448 "title": "Log Entries Per Chunk", 449 "tooltip": { 450 "shared": true, 451 "sort": 2, 452 "value_type": "individual" 453 }, 454 "type": "graph", 455 "xaxis": { 456 "buckets": null, 457 "mode": "time", 458 "name": null, 459 "show": true, 460 "values": [ ] 461 }, 462 "yaxes": [ 463 { 464 "format": "short", 465 "label": null, 466 "logBase": 1, 467 "max": null, 468 "min": 0, 469 "show": true 470 }, 471 { 472 "format": "short", 473 "label": null, 474 "logBase": 1, 475 "max": null, 476 "min": null, 477 "show": false 478 } 479 ] 480 }, 481 { 482 "aliasColors": { }, 483 "bars": false, 484 "dashLength": 10, 485 "dashes": false, 486 "datasource": "$datasource", 487 "fill": 1, 488 "id": 6, 489 "legend": { 490 "avg": false, 491 "current": false, 492 "max": false, 493 "min": false, 494 "show": true, 495 "total": false, 496 "values": false 497 }, 498 "lines": true, 499 "linewidth": 1, 500 "links": [ ], 501 "nullPointMode": "null as zero", 502 "percentage": false, 503 "pointradius": 5, 504 "points": false, 505 "renderer": "flot", 506 "seriesOverrides": [ ], 507 "spaceLength": 10, 508 "span": 6, 509 "stack": false, 510 "steppedLine": false, 511 "targets": [ 512 { 513 "expr": "sum(rate(loki_chunk_store_index_entries_per_chunk_sum{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[5m])) / sum(rate(loki_chunk_store_index_entries_per_chunk_count{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[5m]))", 514 "format": "time_series", 515 "intervalFactor": 2, 516 "legendFormat": "Index Entries", 517 "legendLink": null, 518 "step": 10 519 } 520 ], 521 "thresholds": [ ], 522 "timeFrom": null, 523 "timeShift": null, 524 "title": "Index Entries Per Chunk", 525 "tooltip": { 526 "shared": true, 527 "sort": 2, 528 "value_type": "individual" 529 }, 530 "type": "graph", 531 "xaxis": { 532 "buckets": null, 533 "mode": "time", 534 "name": null, 535 "show": true, 536 "values": [ ] 537 }, 538 "yaxes": [ 539 { 540 "format": "short", 541 "label": null, 542 "logBase": 1, 543 "max": null, 544 "min": 0, 545 "show": true 546 }, 547 { 548 "format": "short", 549 "label": null, 550 "logBase": 1, 551 "max": null, 552 "min": null, 553 "show": false 554 } 555 ] 556 } 557 ], 558 "repeat": null, 559 "repeatIteration": null, 560 "repeatRowId": null, 561 "showTitle": true, 562 "title": "Flush Stats", 563 "titleSize": "h6" 564 }, 565 { 566 "collapse": false, 567 "height": "250px", 568 "panels": [ 569 { 570 "aliasColors": { }, 571 "bars": false, 572 "dashLength": 10, 573 "dashes": false, 574 "datasource": "$datasource", 575 "fill": 1, 576 "id": 7, 577 "legend": { 578 "avg": false, 579 "current": false, 580 "max": false, 581 "min": false, 582 "show": true, 583 "total": false, 584 "values": false 585 }, 586 "lines": true, 587 "linewidth": 1, 588 "links": [ ], 589 "nullPointMode": "null as zero", 590 "percentage": false, 591 "pointradius": 5, 592 "points": false, 593 "renderer": "flot", 594 "seriesOverrides": [ ], 595 "spaceLength": 10, 596 "span": 6, 597 "stack": false, 598 "steppedLine": false, 599 "targets": [ 600 { 601 "expr": "cortex_ingester_flush_queue_length{cluster=\"$cluster\", job=~\"$namespace/ingester\"}", 602 "format": "time_series", 603 "intervalFactor": 2, 604 "legendFormat": "{{pod}}", 605 "legendLink": null, 606 "step": 10 607 } 608 ], 609 "thresholds": [ ], 610 "timeFrom": null, 611 "timeShift": null, 612 "title": "Queue Length", 613 "tooltip": { 614 "shared": true, 615 "sort": 2, 616 "value_type": "individual" 617 }, 618 "type": "graph", 619 "xaxis": { 620 "buckets": null, 621 "mode": "time", 622 "name": null, 623 "show": true, 624 "values": [ ] 625 }, 626 "yaxes": [ 627 { 628 "format": "short", 629 "label": null, 630 "logBase": 1, 631 "max": null, 632 "min": 0, 633 "show": true 634 }, 635 { 636 "format": "short", 637 "label": null, 638 "logBase": 1, 639 "max": null, 640 "min": null, 641 "show": false 642 } 643 ] 644 }, 645 { 646 "aliasColors": { 647 "1xx": "#EAB839", 648 "2xx": "#7EB26D", 649 "3xx": "#6ED0E0", 650 "4xx": "#EF843C", 651 "5xx": "#E24D42", 652 "error": "#E24D42", 653 "success": "#7EB26D" 654 }, 655 "bars": false, 656 "dashLength": 10, 657 "dashes": false, 658 "datasource": "$datasource", 659 "fill": 10, 660 "id": 8, 661 "legend": { 662 "avg": false, 663 "current": false, 664 "max": false, 665 "min": false, 666 "show": true, 667 "total": false, 668 "values": false 669 }, 670 "lines": true, 671 "linewidth": 0, 672 "links": [ ], 673 "nullPointMode": "null as zero", 674 "percentage": false, 675 "pointradius": 5, 676 "points": false, 677 "renderer": "flot", 678 "seriesOverrides": [ ], 679 "spaceLength": 10, 680 "span": 6, 681 "stack": true, 682 "steppedLine": false, 683 "targets": [ 684 { 685 "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_ingester_chunk_age_seconds_count{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", 686 "format": "time_series", 687 "intervalFactor": 2, 688 "legendFormat": "{{status}}", 689 "refId": "A", 690 "step": 10 691 } 692 ], 693 "thresholds": [ ], 694 "timeFrom": null, 695 "timeShift": null, 696 "title": "Flush Rate", 697 "tooltip": { 698 "shared": true, 699 "sort": 2, 700 "value_type": "individual" 701 }, 702 "type": "graph", 703 "xaxis": { 704 "buckets": null, 705 "mode": "time", 706 "name": null, 707 "show": true, 708 "values": [ ] 709 }, 710 "yaxes": [ 711 { 712 "format": "short", 713 "label": null, 714 "logBase": 1, 715 "max": null, 716 "min": 0, 717 "show": true 718 }, 719 { 720 "format": "short", 721 "label": null, 722 "logBase": 1, 723 "max": null, 724 "min": null, 725 "show": false 726 } 727 ] 728 } 729 ], 730 "repeat": null, 731 "repeatIteration": null, 732 "repeatRowId": null, 733 "showTitle": true, 734 "title": "Flush Stats", 735 "titleSize": "h6" 736 }, 737 { 738 "collapse": false, 739 "height": "250px", 740 "panels": [ 741 { 742 "aliasColors": { }, 743 "bars": false, 744 "dashLength": 10, 745 "dashes": false, 746 "datasource": "$datasource", 747 "fill": 1, 748 "id": 9, 749 "legend": { 750 "avg": false, 751 "current": false, 752 "max": false, 753 "min": false, 754 "show": true, 755 "total": false, 756 "values": false 757 }, 758 "lines": true, 759 "linewidth": 1, 760 "links": [ ], 761 "nullPointMode": "null as zero", 762 "percentage": false, 763 "pointradius": 5, 764 "points": false, 765 "renderer": "flot", 766 "seriesOverrides": [ ], 767 "spaceLength": 10, 768 "span": 6, 769 "stack": false, 770 "steppedLine": false, 771 "targets": [ 772 { 773 "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[$__rate_interval]))", 774 "format": "time_series", 775 "intervalFactor": 2, 776 "legendFormat": "{{pod}}", 777 "legendLink": null, 778 "step": 10 779 } 780 ], 781 "thresholds": [ ], 782 "timeFrom": null, 783 "timeShift": null, 784 "title": "Chunks Flushed/Second", 785 "tooltip": { 786 "shared": true, 787 "sort": 2, 788 "value_type": "individual" 789 }, 790 "type": "graph", 791 "xaxis": { 792 "buckets": null, 793 "mode": "time", 794 "name": null, 795 "show": true, 796 "values": [ ] 797 }, 798 "yaxes": [ 799 { 800 "format": "short", 801 "label": null, 802 "logBase": 1, 803 "max": null, 804 "min": 0, 805 "show": true 806 }, 807 { 808 "format": "short", 809 "label": null, 810 "logBase": 1, 811 "max": null, 812 "min": null, 813 "show": false 814 } 815 ] 816 }, 817 { 818 "aliasColors": { }, 819 "bars": false, 820 "dashLength": 10, 821 "dashes": false, 822 "datasource": "$datasource", 823 "fill": 1, 824 "id": 10, 825 "legend": { 826 "avg": false, 827 "current": false, 828 "max": false, 829 "min": false, 830 "show": true, 831 "total": false, 832 "values": false 833 }, 834 "lines": true, 835 "linewidth": 1, 836 "links": [ ], 837 "nullPointMode": "null as zero", 838 "percentage": false, 839 "pointradius": 5, 840 "points": false, 841 "renderer": "flot", 842 "seriesOverrides": [ ], 843 "spaceLength": 10, 844 "span": 6, 845 "stack": true, 846 "steppedLine": false, 847 "targets": [ 848 { 849 "expr": "sum by (reason) (rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[$__rate_interval]))", 850 "format": "time_series", 851 "intervalFactor": 2, 852 "legendFormat": "{{reason}}", 853 "legendLink": null, 854 "step": 10 855 } 856 ], 857 "thresholds": [ ], 858 "timeFrom": null, 859 "timeShift": null, 860 "title": "Chunk Flush Reason", 861 "tooltip": { 862 "shared": true, 863 "sort": 2, 864 "value_type": "individual" 865 }, 866 "type": "graph", 867 "xaxis": { 868 "buckets": null, 869 "mode": "time", 870 "name": null, 871 "show": true, 872 "values": [ ] 873 }, 874 "yaxes": [ 875 { 876 "format": "short", 877 "label": null, 878 "logBase": 1, 879 "max": 1, 880 "min": 0, 881 "show": true 882 }, 883 { 884 "format": "short", 885 "label": null, 886 "logBase": 1, 887 "max": 1, 888 "min": null, 889 "show": false 890 } 891 ] 892 } 893 ], 894 "repeat": null, 895 "repeatIteration": null, 896 "repeatRowId": null, 897 "showTitle": true, 898 "title": "Flush Stats", 899 "titleSize": "h6" 900 }, 901 { 902 "collapse": false, 903 "height": "250px", 904 "panels": [ 905 { 906 "cards": { 907 "cardPadding": null, 908 "cardRound": null 909 }, 910 "color": { 911 "cardColor": "#b4ff00", 912 "colorScale": "sqrt", 913 "colorScheme": "interpolateSpectral", 914 "exponent": 0.5, 915 "mode": "spectrum" 916 }, 917 "dataFormat": "tsbuckets", 918 "datasource": "$datasource", 919 "heatmap": { }, 920 "hideZeroBuckets": false, 921 "highlightCards": true, 922 "id": 11, 923 "legend": { 924 "show": true 925 }, 926 "span": 12, 927 "targets": [ 928 { 929 "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[$__rate_interval]))", 930 "format": "heatmap", 931 "intervalFactor": 2, 932 "legendFormat": "{{le}}", 933 "refId": "A" 934 } 935 ], 936 "title": "Chunk Utilization", 937 "tooltip": { 938 "show": true, 939 "showHistogram": true 940 }, 941 "type": "heatmap", 942 "xAxis": { 943 "show": true 944 }, 945 "xBucketNumber": null, 946 "xBucketSize": null, 947 "yAxis": { 948 "decimals": 0, 949 "format": "percentunit", 950 "show": true, 951 "splitFactor": null 952 }, 953 "yBucketBound": "auto" 954 } 955 ], 956 "repeat": null, 957 "repeatIteration": null, 958 "repeatRowId": null, 959 "showTitle": true, 960 "title": "Utilization", 961 "titleSize": "h6" 962 }, 963 { 964 "collapse": false, 965 "height": "250px", 966 "panels": [ 967 { 968 "cards": { 969 "cardPadding": null, 970 "cardRound": null 971 }, 972 "color": { 973 "cardColor": "#b4ff00", 974 "colorScale": "sqrt", 975 "colorScheme": "interpolateSpectral", 976 "exponent": 0.5, 977 "mode": "spectrum" 978 }, 979 "dataFormat": "tsbuckets", 980 "datasource": "$datasource", 981 "heatmap": { }, 982 "hideZeroBuckets": false, 983 "highlightCards": true, 984 "id": 12, 985 "legend": { 986 "show": true 987 }, 988 "span": 12, 989 "targets": [ 990 { 991 "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[$__rate_interval])) by (le)", 992 "format": "heatmap", 993 "intervalFactor": 2, 994 "legendFormat": "{{le}}", 995 "refId": "A" 996 } 997 ], 998 "title": "Chunk Size Bytes", 999 "tooltip": { 1000 "show": true, 1001 "showHistogram": true 1002 }, 1003 "type": "heatmap", 1004 "xAxis": { 1005 "show": true 1006 }, 1007 "xBucketNumber": null, 1008 "xBucketSize": null, 1009 "yAxis": { 1010 "decimals": 0, 1011 "format": "bytes", 1012 "show": true, 1013 "splitFactor": null 1014 }, 1015 "yBucketBound": "auto" 1016 } 1017 ], 1018 "repeat": null, 1019 "repeatIteration": null, 1020 "repeatRowId": null, 1021 "showTitle": true, 1022 "title": "Utilization", 1023 "titleSize": "h6" 1024 }, 1025 { 1026 "collapse": false, 1027 "height": "250px", 1028 "panels": [ 1029 { 1030 "aliasColors": { }, 1031 "bars": false, 1032 "dashLength": 10, 1033 "dashes": false, 1034 "datasource": "$datasource", 1035 "fill": 1, 1036 "id": 13, 1037 "legend": { 1038 "avg": false, 1039 "current": false, 1040 "max": false, 1041 "min": false, 1042 "show": true, 1043 "total": false, 1044 "values": false 1045 }, 1046 "lines": true, 1047 "linewidth": 1, 1048 "links": [ ], 1049 "nullPointMode": "null as zero", 1050 "percentage": false, 1051 "pointradius": 5, 1052 "points": false, 1053 "renderer": "flot", 1054 "seriesOverrides": [ ], 1055 "spaceLength": 10, 1056 "span": 12, 1057 "stack": false, 1058 "steppedLine": false, 1059 "targets": [ 1060 { 1061 "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[1m])) by (le))", 1062 "format": "time_series", 1063 "intervalFactor": 2, 1064 "legendFormat": "p99", 1065 "legendLink": null, 1066 "step": 10 1067 }, 1068 { 1069 "expr": "histogram_quantile(0.90, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[1m])) by (le))", 1070 "format": "time_series", 1071 "intervalFactor": 2, 1072 "legendFormat": "p90", 1073 "legendLink": null, 1074 "step": 10 1075 }, 1076 { 1077 "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[1m])) by (le))", 1078 "format": "time_series", 1079 "intervalFactor": 2, 1080 "legendFormat": "p50", 1081 "legendLink": null, 1082 "step": 10 1083 } 1084 ], 1085 "thresholds": [ ], 1086 "timeFrom": null, 1087 "timeShift": null, 1088 "title": "Chunk Size Quantiles", 1089 "tooltip": { 1090 "shared": true, 1091 "sort": 2, 1092 "value_type": "individual" 1093 }, 1094 "type": "graph", 1095 "xaxis": { 1096 "buckets": null, 1097 "mode": "time", 1098 "name": null, 1099 "show": true, 1100 "values": [ ] 1101 }, 1102 "yaxes": [ 1103 { 1104 "format": "bytes", 1105 "label": null, 1106 "logBase": 1, 1107 "max": null, 1108 "min": 0, 1109 "show": true 1110 }, 1111 { 1112 "format": "short", 1113 "label": null, 1114 "logBase": 1, 1115 "max": null, 1116 "min": null, 1117 "show": false 1118 } 1119 ] 1120 } 1121 ], 1122 "repeat": null, 1123 "repeatIteration": null, 1124 "repeatRowId": null, 1125 "showTitle": true, 1126 "title": "Utilization", 1127 "titleSize": "h6" 1128 }, 1129 { 1130 "collapse": false, 1131 "height": "250px", 1132 "panels": [ 1133 { 1134 "aliasColors": { }, 1135 "bars": false, 1136 "dashLength": 10, 1137 "dashes": false, 1138 "datasource": "$datasource", 1139 "fill": 1, 1140 "id": 14, 1141 "legend": { 1142 "avg": false, 1143 "current": false, 1144 "max": false, 1145 "min": false, 1146 "show": true, 1147 "total": false, 1148 "values": false 1149 }, 1150 "lines": true, 1151 "linewidth": 1, 1152 "links": [ ], 1153 "nullPointMode": "null as zero", 1154 "percentage": false, 1155 "pointradius": 5, 1156 "points": false, 1157 "renderer": "flot", 1158 "seriesOverrides": [ ], 1159 "spaceLength": 10, 1160 "span": 12, 1161 "stack": false, 1162 "steppedLine": false, 1163 "targets": [ 1164 { 1165 "expr": "histogram_quantile(0.5, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[5m])) by (le))", 1166 "format": "time_series", 1167 "intervalFactor": 2, 1168 "legendFormat": "p50", 1169 "legendLink": null, 1170 "step": 10 1171 }, 1172 { 1173 "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[5m])) by (le))", 1174 "format": "time_series", 1175 "intervalFactor": 2, 1176 "legendFormat": "p99", 1177 "legendLink": null, 1178 "step": 10 1179 }, 1180 { 1181 "expr": "sum(rate(loki_ingester_chunk_bounds_hours_sum{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[5m])) / sum(rate(loki_ingester_chunk_bounds_hours_count{cluster=\"$cluster\", job=~\"$namespace/ingester\"}[5m]))", 1182 "format": "time_series", 1183 "intervalFactor": 2, 1184 "legendFormat": "avg", 1185 "legendLink": null, 1186 "step": 10 1187 } 1188 ], 1189 "thresholds": [ ], 1190 "timeFrom": null, 1191 "timeShift": null, 1192 "title": "Chunk Duration hours (end-start)", 1193 "tooltip": { 1194 "shared": true, 1195 "sort": 2, 1196 "value_type": "individual" 1197 }, 1198 "type": "graph", 1199 "xaxis": { 1200 "buckets": null, 1201 "mode": "time", 1202 "name": null, 1203 "show": true, 1204 "values": [ ] 1205 }, 1206 "yaxes": [ 1207 { 1208 "format": "short", 1209 "label": null, 1210 "logBase": 1, 1211 "max": null, 1212 "min": 0, 1213 "show": true 1214 }, 1215 { 1216 "format": "short", 1217 "label": null, 1218 "logBase": 1, 1219 "max": null, 1220 "min": null, 1221 "show": false 1222 } 1223 ] 1224 } 1225 ], 1226 "repeat": null, 1227 "repeatIteration": null, 1228 "repeatRowId": null, 1229 "showTitle": true, 1230 "title": "Duration", 1231 "titleSize": "h6" 1232 } 1233 ], 1234 "schemaVersion": 14, 1235 "style": "dark", 1236 "tags": [ 1237 "loki" 1238 ], 1239 "templating": { 1240 "list": [ 1241 { 1242 "current": { 1243 "text": "default", 1244 "value": "default" 1245 }, 1246 "hide": 0, 1247 "label": "Data Source", 1248 "name": "datasource", 1249 "options": [ ], 1250 "query": "prometheus", 1251 "refresh": 1, 1252 "regex": "", 1253 "type": "datasource" 1254 }, 1255 { 1256 "allValue": null, 1257 "current": { 1258 "text": "prod", 1259 "value": "prod" 1260 }, 1261 "datasource": "$datasource", 1262 "hide": 0, 1263 "includeAll": false, 1264 "label": "cluster", 1265 "multi": false, 1266 "name": "cluster", 1267 "options": [ ], 1268 "query": "label_values(loki_build_info, cluster)", 1269 "refresh": 1, 1270 "regex": "", 1271 "sort": 2, 1272 "tagValuesQuery": "", 1273 "tags": [ ], 1274 "tagsQuery": "", 1275 "type": "query", 1276 "useTags": false 1277 }, 1278 { 1279 "allValue": null, 1280 "current": { 1281 "text": "prod", 1282 "value": "prod" 1283 }, 1284 "datasource": "$datasource", 1285 "hide": 0, 1286 "includeAll": false, 1287 "label": "namespace", 1288 "multi": false, 1289 "name": "namespace", 1290 "options": [ ], 1291 "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", 1292 "refresh": 1, 1293 "regex": "", 1294 "sort": 2, 1295 "tagValuesQuery": "", 1296 "tags": [ ], 1297 "tagsQuery": "", 1298 "type": "query", 1299 "useTags": false 1300 } 1301 ] 1302 }, 1303 "time": { 1304 "from": "now-1h", 1305 "to": "now" 1306 }, 1307 "timepicker": { 1308 "refresh_intervals": [ 1309 "5s", 1310 "10s", 1311 "30s", 1312 "1m", 1313 "5m", 1314 "15m", 1315 "30m", 1316 "1h", 1317 "2h", 1318 "1d" 1319 ], 1320 "time_options": [ 1321 "5m", 1322 "15m", 1323 "1h", 1324 "6h", 1325 "12h", 1326 "24h", 1327 "2d", 1328 "7d", 1329 "30d" 1330 ] 1331 }, 1332 "timezone": "utc", 1333 "title": "Loki / Chunks", 1334 "uid": "chunks", 1335 "version": 0 1336 }