github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/production/loki-mixin-compiled/dashboards/loki-reads-resources.json (about) 1 { 2 "annotations": { 3 "list": [ ] 4 }, 5 "editable": true, 6 "gnetId": null, 7 "graphTooltip": 0, 8 "hideControls": false, 9 "links": [ 10 { 11 "asDropdown": true, 12 "icon": "external link", 13 "includeVars": true, 14 "keepTime": true, 15 "tags": [ 16 "loki" 17 ], 18 "targetBlank": false, 19 "title": "Loki Dashboards", 20 "type": "dashboards" 21 } 22 ], 23 "refresh": "10s", 24 "rows": [ 25 { 26 "collapse": false, 27 "height": "250px", 28 "panels": [ 29 { 30 "aliasColors": { }, 31 "bars": false, 32 "dashLength": 10, 33 "dashes": false, 34 "datasource": "$datasource", 35 "fill": 1, 36 "id": 1, 37 "legend": { 38 "avg": false, 39 "current": false, 40 "max": false, 41 "min": false, 42 "show": true, 43 "total": false, 44 "values": false 45 }, 46 "lines": true, 47 "linewidth": 1, 48 "links": [ ], 49 "nullPointMode": "null as zero", 50 "percentage": false, 51 "pointradius": 5, 52 "points": false, 53 "renderer": "flot", 54 "seriesOverrides": [ 55 { 56 "alias": "limit", 57 "color": "#E02F44", 58 "fill": 0 59 } 60 ], 61 "spaceLength": 10, 62 "span": 4, 63 "stack": false, 64 "steppedLine": false, 65 "targets": [ 66 { 67 "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"query-frontend\"}[$__rate_interval]))", 68 "format": "time_series", 69 "intervalFactor": 2, 70 "legendFormat": "{{pod}}", 71 "legendLink": null, 72 "step": 10 73 }, 74 { 75 "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"query-frontend\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"query-frontend\"})", 76 "format": "time_series", 77 "intervalFactor": 2, 78 "legendFormat": "limit", 79 "legendLink": null, 80 "step": 10 81 } 82 ], 83 "thresholds": [ ], 84 "timeFrom": null, 85 "timeShift": null, 86 "title": "CPU", 87 "tooltip": { 88 "sort": 2 89 }, 90 "type": "graph", 91 "xaxis": { 92 "buckets": null, 93 "mode": "time", 94 "name": null, 95 "show": true, 96 "values": [ ] 97 }, 98 "yaxes": [ 99 { 100 "format": "short", 101 "label": null, 102 "logBase": 1, 103 "max": null, 104 "min": 0, 105 "show": true 106 }, 107 { 108 "format": "short", 109 "label": null, 110 "logBase": 1, 111 "max": null, 112 "min": null, 113 "show": false 114 } 115 ] 116 }, 117 { 118 "aliasColors": { }, 119 "bars": false, 120 "dashLength": 10, 121 "dashes": false, 122 "datasource": "$datasource", 123 "fill": 1, 124 "id": 2, 125 "legend": { 126 "avg": false, 127 "current": false, 128 "max": false, 129 "min": false, 130 "show": true, 131 "total": false, 132 "values": false 133 }, 134 "lines": true, 135 "linewidth": 1, 136 "links": [ ], 137 "nullPointMode": "null as zero", 138 "percentage": false, 139 "pointradius": 5, 140 "points": false, 141 "renderer": "flot", 142 "seriesOverrides": [ 143 { 144 "alias": "limit", 145 "color": "#E02F44", 146 "fill": 0 147 } 148 ], 149 "spaceLength": 10, 150 "span": 4, 151 "stack": false, 152 "steppedLine": false, 153 "targets": [ 154 { 155 "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"query-frontend\"})", 156 "format": "time_series", 157 "intervalFactor": 2, 158 "legendFormat": "{{pod}}", 159 "legendLink": null, 160 "step": 10 161 }, 162 { 163 "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"query-frontend\"} > 0)", 164 "format": "time_series", 165 "intervalFactor": 2, 166 "legendFormat": "limit", 167 "legendLink": null, 168 "step": 10 169 } 170 ], 171 "thresholds": [ ], 172 "timeFrom": null, 173 "timeShift": null, 174 "title": "Memory (workingset)", 175 "tooltip": { 176 "sort": 2 177 }, 178 "type": "graph", 179 "xaxis": { 180 "buckets": null, 181 "mode": "time", 182 "name": null, 183 "show": true, 184 "values": [ ] 185 }, 186 "yaxes": [ 187 { 188 "format": "bytes", 189 "label": null, 190 "logBase": 1, 191 "max": null, 192 "min": 0, 193 "show": true 194 }, 195 { 196 "format": "short", 197 "label": null, 198 "logBase": 1, 199 "max": null, 200 "min": null, 201 "show": false 202 } 203 ] 204 }, 205 { 206 "aliasColors": { }, 207 "bars": false, 208 "dashLength": 10, 209 "dashes": false, 210 "datasource": "$datasource", 211 "fill": 1, 212 "id": 3, 213 "legend": { 214 "avg": false, 215 "current": false, 216 "max": false, 217 "min": false, 218 "show": true, 219 "total": false, 220 "values": false 221 }, 222 "lines": true, 223 "linewidth": 1, 224 "links": [ ], 225 "nullPointMode": "null as zero", 226 "percentage": false, 227 "pointradius": 5, 228 "points": false, 229 "renderer": "flot", 230 "seriesOverrides": [ ], 231 "spaceLength": 10, 232 "span": 4, 233 "stack": false, 234 "steppedLine": false, 235 "targets": [ 236 { 237 "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\"})", 238 "format": "time_series", 239 "intervalFactor": 2, 240 "legendFormat": "{{pod}}", 241 "legendLink": null, 242 "step": 10 243 } 244 ], 245 "thresholds": [ ], 246 "timeFrom": null, 247 "timeShift": null, 248 "title": "Memory (go heap inuse)", 249 "tooltip": { 250 "sort": 2 251 }, 252 "type": "graph", 253 "xaxis": { 254 "buckets": null, 255 "mode": "time", 256 "name": null, 257 "show": true, 258 "values": [ ] 259 }, 260 "yaxes": [ 261 { 262 "format": "bytes", 263 "label": null, 264 "logBase": 1, 265 "max": null, 266 "min": 0, 267 "show": true 268 }, 269 { 270 "format": "short", 271 "label": null, 272 "logBase": 1, 273 "max": null, 274 "min": null, 275 "show": false 276 } 277 ] 278 } 279 ], 280 "repeat": null, 281 "repeatIteration": null, 282 "repeatRowId": null, 283 "showTitle": true, 284 "title": "Query Frontend", 285 "titleSize": "h6" 286 }, 287 { 288 "collapse": false, 289 "height": "250px", 290 "panels": [ 291 { 292 "aliasColors": { }, 293 "bars": false, 294 "dashLength": 10, 295 "dashes": false, 296 "datasource": "$datasource", 297 "fill": 1, 298 "id": 4, 299 "legend": { 300 "avg": false, 301 "current": false, 302 "max": false, 303 "min": false, 304 "show": true, 305 "total": false, 306 "values": false 307 }, 308 "lines": true, 309 "linewidth": 1, 310 "links": [ ], 311 "nullPointMode": "null as zero", 312 "percentage": false, 313 "pointradius": 5, 314 "points": false, 315 "renderer": "flot", 316 "seriesOverrides": [ 317 { 318 "alias": "limit", 319 "color": "#E02F44", 320 "fill": 0 321 } 322 ], 323 "spaceLength": 10, 324 "span": 4, 325 "stack": false, 326 "steppedLine": false, 327 "targets": [ 328 { 329 "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"query-scheduler\"}[$__rate_interval]))", 330 "format": "time_series", 331 "intervalFactor": 2, 332 "legendFormat": "{{pod}}", 333 "legendLink": null, 334 "step": 10 335 }, 336 { 337 "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"query-scheduler\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"query-scheduler\"})", 338 "format": "time_series", 339 "intervalFactor": 2, 340 "legendFormat": "limit", 341 "legendLink": null, 342 "step": 10 343 } 344 ], 345 "thresholds": [ ], 346 "timeFrom": null, 347 "timeShift": null, 348 "title": "CPU", 349 "tooltip": { 350 "sort": 2 351 }, 352 "type": "graph", 353 "xaxis": { 354 "buckets": null, 355 "mode": "time", 356 "name": null, 357 "show": true, 358 "values": [ ] 359 }, 360 "yaxes": [ 361 { 362 "format": "short", 363 "label": null, 364 "logBase": 1, 365 "max": null, 366 "min": 0, 367 "show": true 368 }, 369 { 370 "format": "short", 371 "label": null, 372 "logBase": 1, 373 "max": null, 374 "min": null, 375 "show": false 376 } 377 ] 378 }, 379 { 380 "aliasColors": { }, 381 "bars": false, 382 "dashLength": 10, 383 "dashes": false, 384 "datasource": "$datasource", 385 "fill": 1, 386 "id": 5, 387 "legend": { 388 "avg": false, 389 "current": false, 390 "max": false, 391 "min": false, 392 "show": true, 393 "total": false, 394 "values": false 395 }, 396 "lines": true, 397 "linewidth": 1, 398 "links": [ ], 399 "nullPointMode": "null as zero", 400 "percentage": false, 401 "pointradius": 5, 402 "points": false, 403 "renderer": "flot", 404 "seriesOverrides": [ 405 { 406 "alias": "limit", 407 "color": "#E02F44", 408 "fill": 0 409 } 410 ], 411 "spaceLength": 10, 412 "span": 4, 413 "stack": false, 414 "steppedLine": false, 415 "targets": [ 416 { 417 "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"query-scheduler\"})", 418 "format": "time_series", 419 "intervalFactor": 2, 420 "legendFormat": "{{pod}}", 421 "legendLink": null, 422 "step": 10 423 }, 424 { 425 "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"query-scheduler\"} > 0)", 426 "format": "time_series", 427 "intervalFactor": 2, 428 "legendFormat": "limit", 429 "legendLink": null, 430 "step": 10 431 } 432 ], 433 "thresholds": [ ], 434 "timeFrom": null, 435 "timeShift": null, 436 "title": "Memory (workingset)", 437 "tooltip": { 438 "sort": 2 439 }, 440 "type": "graph", 441 "xaxis": { 442 "buckets": null, 443 "mode": "time", 444 "name": null, 445 "show": true, 446 "values": [ ] 447 }, 448 "yaxes": [ 449 { 450 "format": "bytes", 451 "label": null, 452 "logBase": 1, 453 "max": null, 454 "min": 0, 455 "show": true 456 }, 457 { 458 "format": "short", 459 "label": null, 460 "logBase": 1, 461 "max": null, 462 "min": null, 463 "show": false 464 } 465 ] 466 }, 467 { 468 "aliasColors": { }, 469 "bars": false, 470 "dashLength": 10, 471 "dashes": false, 472 "datasource": "$datasource", 473 "fill": 1, 474 "id": 6, 475 "legend": { 476 "avg": false, 477 "current": false, 478 "max": false, 479 "min": false, 480 "show": true, 481 "total": false, 482 "values": false 483 }, 484 "lines": true, 485 "linewidth": 1, 486 "links": [ ], 487 "nullPointMode": "null as zero", 488 "percentage": false, 489 "pointradius": 5, 490 "points": false, 491 "renderer": "flot", 492 "seriesOverrides": [ ], 493 "spaceLength": 10, 494 "span": 4, 495 "stack": false, 496 "steppedLine": false, 497 "targets": [ 498 { 499 "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/query-scheduler\"})", 500 "format": "time_series", 501 "intervalFactor": 2, 502 "legendFormat": "{{pod}}", 503 "legendLink": null, 504 "step": 10 505 } 506 ], 507 "thresholds": [ ], 508 "timeFrom": null, 509 "timeShift": null, 510 "title": "Memory (go heap inuse)", 511 "tooltip": { 512 "sort": 2 513 }, 514 "type": "graph", 515 "xaxis": { 516 "buckets": null, 517 "mode": "time", 518 "name": null, 519 "show": true, 520 "values": [ ] 521 }, 522 "yaxes": [ 523 { 524 "format": "bytes", 525 "label": null, 526 "logBase": 1, 527 "max": null, 528 "min": 0, 529 "show": true 530 }, 531 { 532 "format": "short", 533 "label": null, 534 "logBase": 1, 535 "max": null, 536 "min": null, 537 "show": false 538 } 539 ] 540 } 541 ], 542 "repeat": null, 543 "repeatIteration": null, 544 "repeatRowId": null, 545 "showTitle": true, 546 "title": "Query Scheduler", 547 "titleSize": "h6" 548 }, 549 { 550 "collapse": false, 551 "collapsed": false, 552 "panels": [ 553 { 554 "aliasColors": { }, 555 "bars": false, 556 "dashLength": 10, 557 "dashes": false, 558 "datasource": "$datasource", 559 "fill": 1, 560 "gridPos": { }, 561 "id": 7, 562 "legend": { 563 "avg": false, 564 "current": false, 565 "max": false, 566 "min": false, 567 "show": true, 568 "total": false, 569 "values": false 570 }, 571 "lines": true, 572 "linewidth": 1, 573 "links": [ ], 574 "nullPointMode": "null as zero", 575 "percentage": false, 576 "pointradius": 5, 577 "points": false, 578 "renderer": "flot", 579 "seriesOverrides": [ 580 { 581 "alias": "limit", 582 "color": "#E02F44", 583 "fill": 0 584 } 585 ], 586 "spaceLength": 10, 587 "span": 6, 588 "stack": false, 589 "steppedLine": false, 590 "targets": [ 591 { 592 "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"querier\"}[$__rate_interval]))", 593 "format": "time_series", 594 "intervalFactor": 2, 595 "legendFormat": "{{pod}}", 596 "legendLink": null, 597 "step": 10 598 }, 599 { 600 "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"querier\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"querier\"})", 601 "format": "time_series", 602 "intervalFactor": 2, 603 "legendFormat": "limit", 604 "legendLink": null, 605 "step": 10 606 } 607 ], 608 "thresholds": [ ], 609 "timeFrom": null, 610 "timeShift": null, 611 "title": "CPU", 612 "tooltip": { 613 "sort": 2 614 }, 615 "type": "graph", 616 "xaxis": { 617 "buckets": null, 618 "mode": "time", 619 "name": null, 620 "show": true, 621 "values": [ ] 622 }, 623 "yaxes": [ 624 { 625 "format": "short", 626 "label": null, 627 "logBase": 1, 628 "max": null, 629 "min": 0, 630 "show": true 631 }, 632 { 633 "format": "short", 634 "label": null, 635 "logBase": 1, 636 "max": null, 637 "min": null, 638 "show": false 639 } 640 ] 641 }, 642 { 643 "aliasColors": { }, 644 "bars": false, 645 "dashLength": 10, 646 "dashes": false, 647 "datasource": "$datasource", 648 "fill": 1, 649 "gridPos": { }, 650 "id": 8, 651 "legend": { 652 "avg": false, 653 "current": false, 654 "max": false, 655 "min": false, 656 "show": true, 657 "total": false, 658 "values": false 659 }, 660 "lines": true, 661 "linewidth": 1, 662 "links": [ ], 663 "nullPointMode": "null as zero", 664 "percentage": false, 665 "pointradius": 5, 666 "points": false, 667 "renderer": "flot", 668 "seriesOverrides": [ 669 { 670 "alias": "limit", 671 "color": "#E02F44", 672 "fill": 0 673 } 674 ], 675 "spaceLength": 10, 676 "span": 6, 677 "stack": false, 678 "steppedLine": false, 679 "targets": [ 680 { 681 "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"querier\"})", 682 "format": "time_series", 683 "intervalFactor": 2, 684 "legendFormat": "{{pod}}", 685 "legendLink": null, 686 "step": 10 687 }, 688 { 689 "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"querier\"} > 0)", 690 "format": "time_series", 691 "intervalFactor": 2, 692 "legendFormat": "limit", 693 "legendLink": null, 694 "step": 10 695 } 696 ], 697 "thresholds": [ ], 698 "timeFrom": null, 699 "timeShift": null, 700 "title": "Memory (workingset)", 701 "tooltip": { 702 "sort": 2 703 }, 704 "type": "graph", 705 "xaxis": { 706 "buckets": null, 707 "mode": "time", 708 "name": null, 709 "show": true, 710 "values": [ ] 711 }, 712 "yaxes": [ 713 { 714 "format": "bytes", 715 "label": null, 716 "logBase": 1, 717 "max": null, 718 "min": 0, 719 "show": true 720 }, 721 { 722 "format": "short", 723 "label": null, 724 "logBase": 1, 725 "max": null, 726 "min": null, 727 "show": false 728 } 729 ] 730 }, 731 { 732 "aliasColors": { }, 733 "bars": false, 734 "dashLength": 10, 735 "dashes": false, 736 "datasource": "$datasource", 737 "fill": 1, 738 "gridPos": { }, 739 "id": 9, 740 "legend": { 741 "avg": false, 742 "current": false, 743 "max": false, 744 "min": false, 745 "show": true, 746 "total": false, 747 "values": false 748 }, 749 "lines": true, 750 "linewidth": 1, 751 "links": [ ], 752 "nullPointMode": "null as zero", 753 "percentage": false, 754 "pointradius": 5, 755 "points": false, 756 "renderer": "flot", 757 "seriesOverrides": [ ], 758 "spaceLength": 10, 759 "span": 6, 760 "stack": false, 761 "steppedLine": false, 762 "targets": [ 763 { 764 "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/querier\"})", 765 "format": "time_series", 766 "intervalFactor": 2, 767 "legendFormat": "{{pod}}", 768 "legendLink": null, 769 "step": 10 770 } 771 ], 772 "thresholds": [ ], 773 "timeFrom": null, 774 "timeShift": null, 775 "title": "Memory (go heap inuse)", 776 "tooltip": { 777 "sort": 2 778 }, 779 "type": "graph", 780 "xaxis": { 781 "buckets": null, 782 "mode": "time", 783 "name": null, 784 "show": true, 785 "values": [ ] 786 }, 787 "yaxes": [ 788 { 789 "format": "bytes", 790 "label": null, 791 "logBase": 1, 792 "max": null, 793 "min": 0, 794 "show": true 795 }, 796 { 797 "format": "short", 798 "label": null, 799 "logBase": 1, 800 "max": null, 801 "min": null, 802 "show": false 803 } 804 ] 805 }, 806 { 807 "aliasColors": { }, 808 "bars": false, 809 "dashLength": 10, 810 "dashes": false, 811 "datasource": "$datasource", 812 "fill": 10, 813 "gridPos": { }, 814 "id": 10, 815 "legend": { 816 "avg": false, 817 "current": false, 818 "max": false, 819 "min": false, 820 "show": true, 821 "total": false, 822 "values": false 823 }, 824 "lines": true, 825 "linewidth": 0, 826 "links": [ ], 827 "nullPointMode": "null as zero", 828 "percentage": false, 829 "pointradius": 5, 830 "points": false, 831 "renderer": "flot", 832 "seriesOverrides": [ ], 833 "spaceLength": 10, 834 "span": 6, 835 "stack": true, 836 "steppedLine": false, 837 "targets": [ 838 { 839 "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"querier\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", 840 "format": "time_series", 841 "intervalFactor": 2, 842 "legendFormat": "{{pod}} - {{device}}", 843 "legendLink": null, 844 "step": 10 845 } 846 ], 847 "thresholds": [ ], 848 "timeFrom": null, 849 "timeShift": null, 850 "title": "Disk Writes", 851 "tooltip": { 852 "shared": true, 853 "sort": 2, 854 "value_type": "individual" 855 }, 856 "type": "graph", 857 "xaxis": { 858 "buckets": null, 859 "mode": "time", 860 "name": null, 861 "show": true, 862 "values": [ ] 863 }, 864 "yaxes": [ 865 { 866 "format": "Bps", 867 "label": null, 868 "logBase": 1, 869 "max": null, 870 "min": 0, 871 "show": true 872 }, 873 { 874 "format": "short", 875 "label": null, 876 "logBase": 1, 877 "max": null, 878 "min": null, 879 "show": false 880 } 881 ] 882 }, 883 { 884 "aliasColors": { }, 885 "bars": false, 886 "dashLength": 10, 887 "dashes": false, 888 "datasource": "$datasource", 889 "fill": 10, 890 "gridPos": { }, 891 "id": 11, 892 "legend": { 893 "avg": false, 894 "current": false, 895 "max": false, 896 "min": false, 897 "show": true, 898 "total": false, 899 "values": false 900 }, 901 "lines": true, 902 "linewidth": 0, 903 "links": [ ], 904 "nullPointMode": "null as zero", 905 "percentage": false, 906 "pointradius": 5, 907 "points": false, 908 "renderer": "flot", 909 "seriesOverrides": [ ], 910 "spaceLength": 10, 911 "span": 6, 912 "stack": true, 913 "steppedLine": false, 914 "targets": [ 915 { 916 "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"querier\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", 917 "format": "time_series", 918 "intervalFactor": 2, 919 "legendFormat": "{{pod}} - {{device}}", 920 "legendLink": null, 921 "step": 10 922 } 923 ], 924 "thresholds": [ ], 925 "timeFrom": null, 926 "timeShift": null, 927 "title": "Disk Reads", 928 "tooltip": { 929 "shared": true, 930 "sort": 2, 931 "value_type": "individual" 932 }, 933 "type": "graph", 934 "xaxis": { 935 "buckets": null, 936 "mode": "time", 937 "name": null, 938 "show": true, 939 "values": [ ] 940 }, 941 "yaxes": [ 942 { 943 "format": "Bps", 944 "label": null, 945 "logBase": 1, 946 "max": null, 947 "min": 0, 948 "show": true 949 }, 950 { 951 "format": "short", 952 "label": null, 953 "logBase": 1, 954 "max": null, 955 "min": null, 956 "show": false 957 } 958 ] 959 }, 960 { 961 "aliasColors": { }, 962 "bars": false, 963 "dashLength": 10, 964 "dashes": false, 965 "datasource": "$datasource", 966 "fill": 1, 967 "gridPos": { }, 968 "id": 12, 969 "legend": { 970 "avg": false, 971 "current": false, 972 "max": false, 973 "min": false, 974 "show": true, 975 "total": false, 976 "values": false 977 }, 978 "lines": true, 979 "linewidth": 1, 980 "links": [ ], 981 "nullPointMode": "null as zero", 982 "percentage": false, 983 "pointradius": 5, 984 "points": false, 985 "renderer": "flot", 986 "seriesOverrides": [ ], 987 "spaceLength": 10, 988 "span": 6, 989 "stack": false, 990 "steppedLine": false, 991 "targets": [ 992 { 993 "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"querier.*\"})", 994 "format": "time_series", 995 "intervalFactor": 2, 996 "legendFormat": "{{persistentvolumeclaim}}", 997 "legendLink": null, 998 "step": 10 999 } 1000 ], 1001 "thresholds": [ ], 1002 "timeFrom": null, 1003 "timeShift": null, 1004 "title": "Disk Space Utilization", 1005 "tooltip": { 1006 "shared": true, 1007 "sort": 2, 1008 "value_type": "individual" 1009 }, 1010 "type": "graph", 1011 "xaxis": { 1012 "buckets": null, 1013 "mode": "time", 1014 "name": null, 1015 "show": true, 1016 "values": [ ] 1017 }, 1018 "yaxes": [ 1019 { 1020 "format": "percentunit", 1021 "label": null, 1022 "logBase": 1, 1023 "max": null, 1024 "min": 0, 1025 "show": true 1026 }, 1027 { 1028 "format": "short", 1029 "label": null, 1030 "logBase": 1, 1031 "max": null, 1032 "min": null, 1033 "show": false 1034 } 1035 ] 1036 } 1037 ], 1038 "repeat": null, 1039 "repeatIteration": null, 1040 "repeatRowId": null, 1041 "showTitle": true, 1042 "title": "Querier", 1043 "titleSize": "h6", 1044 "type": "row" 1045 }, 1046 { 1047 "collapse": false, 1048 "collapsed": false, 1049 "panels": [ 1050 { 1051 "aliasColors": { }, 1052 "bars": false, 1053 "dashLength": 10, 1054 "dashes": false, 1055 "datasource": "$datasource", 1056 "fill": 1, 1057 "gridPos": { }, 1058 "id": 13, 1059 "legend": { 1060 "avg": false, 1061 "current": false, 1062 "max": false, 1063 "min": false, 1064 "show": true, 1065 "total": false, 1066 "values": false 1067 }, 1068 "lines": true, 1069 "linewidth": 1, 1070 "links": [ ], 1071 "nullPointMode": "null as zero", 1072 "percentage": false, 1073 "pointradius": 5, 1074 "points": false, 1075 "renderer": "flot", 1076 "seriesOverrides": [ 1077 { 1078 "alias": "limit", 1079 "color": "#E02F44", 1080 "fill": 0 1081 } 1082 ], 1083 "spaceLength": 10, 1084 "span": 6, 1085 "stack": false, 1086 "steppedLine": false, 1087 "targets": [ 1088 { 1089 "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\"}[$__rate_interval]))", 1090 "format": "time_series", 1091 "intervalFactor": 2, 1092 "legendFormat": "{{pod}}", 1093 "legendLink": null, 1094 "step": 10 1095 }, 1096 { 1097 "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\"})", 1098 "format": "time_series", 1099 "intervalFactor": 2, 1100 "legendFormat": "limit", 1101 "legendLink": null, 1102 "step": 10 1103 } 1104 ], 1105 "thresholds": [ ], 1106 "timeFrom": null, 1107 "timeShift": null, 1108 "title": "CPU", 1109 "tooltip": { 1110 "sort": 2 1111 }, 1112 "type": "graph", 1113 "xaxis": { 1114 "buckets": null, 1115 "mode": "time", 1116 "name": null, 1117 "show": true, 1118 "values": [ ] 1119 }, 1120 "yaxes": [ 1121 { 1122 "format": "short", 1123 "label": null, 1124 "logBase": 1, 1125 "max": null, 1126 "min": 0, 1127 "show": true 1128 }, 1129 { 1130 "format": "short", 1131 "label": null, 1132 "logBase": 1, 1133 "max": null, 1134 "min": null, 1135 "show": false 1136 } 1137 ] 1138 }, 1139 { 1140 "aliasColors": { }, 1141 "bars": false, 1142 "dashLength": 10, 1143 "dashes": false, 1144 "datasource": "$datasource", 1145 "fill": 1, 1146 "gridPos": { }, 1147 "id": 14, 1148 "legend": { 1149 "avg": false, 1150 "current": false, 1151 "max": false, 1152 "min": false, 1153 "show": true, 1154 "total": false, 1155 "values": false 1156 }, 1157 "lines": true, 1158 "linewidth": 1, 1159 "links": [ ], 1160 "nullPointMode": "null as zero", 1161 "percentage": false, 1162 "pointradius": 5, 1163 "points": false, 1164 "renderer": "flot", 1165 "seriesOverrides": [ 1166 { 1167 "alias": "limit", 1168 "color": "#E02F44", 1169 "fill": 0 1170 } 1171 ], 1172 "spaceLength": 10, 1173 "span": 6, 1174 "stack": false, 1175 "steppedLine": false, 1176 "targets": [ 1177 { 1178 "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\"})", 1179 "format": "time_series", 1180 "intervalFactor": 2, 1181 "legendFormat": "{{pod}}", 1182 "legendLink": null, 1183 "step": 10 1184 }, 1185 { 1186 "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\"} > 0)", 1187 "format": "time_series", 1188 "intervalFactor": 2, 1189 "legendFormat": "limit", 1190 "legendLink": null, 1191 "step": 10 1192 } 1193 ], 1194 "thresholds": [ ], 1195 "timeFrom": null, 1196 "timeShift": null, 1197 "title": "Memory (workingset)", 1198 "tooltip": { 1199 "sort": 2 1200 }, 1201 "type": "graph", 1202 "xaxis": { 1203 "buckets": null, 1204 "mode": "time", 1205 "name": null, 1206 "show": true, 1207 "values": [ ] 1208 }, 1209 "yaxes": [ 1210 { 1211 "format": "bytes", 1212 "label": null, 1213 "logBase": 1, 1214 "max": null, 1215 "min": 0, 1216 "show": true 1217 }, 1218 { 1219 "format": "short", 1220 "label": null, 1221 "logBase": 1, 1222 "max": null, 1223 "min": null, 1224 "show": false 1225 } 1226 ] 1227 }, 1228 { 1229 "aliasColors": { }, 1230 "bars": false, 1231 "dashLength": 10, 1232 "dashes": false, 1233 "datasource": "$datasource", 1234 "fill": 1, 1235 "gridPos": { }, 1236 "id": 15, 1237 "legend": { 1238 "avg": false, 1239 "current": false, 1240 "max": false, 1241 "min": false, 1242 "show": true, 1243 "total": false, 1244 "values": false 1245 }, 1246 "lines": true, 1247 "linewidth": 1, 1248 "links": [ ], 1249 "nullPointMode": "null as zero", 1250 "percentage": false, 1251 "pointradius": 5, 1252 "points": false, 1253 "renderer": "flot", 1254 "seriesOverrides": [ ], 1255 "spaceLength": 10, 1256 "span": 6, 1257 "stack": false, 1258 "steppedLine": false, 1259 "targets": [ 1260 { 1261 "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/index-gateway\"})", 1262 "format": "time_series", 1263 "intervalFactor": 2, 1264 "legendFormat": "{{pod}}", 1265 "legendLink": null, 1266 "step": 10 1267 } 1268 ], 1269 "thresholds": [ ], 1270 "timeFrom": null, 1271 "timeShift": null, 1272 "title": "Memory (go heap inuse)", 1273 "tooltip": { 1274 "sort": 2 1275 }, 1276 "type": "graph", 1277 "xaxis": { 1278 "buckets": null, 1279 "mode": "time", 1280 "name": null, 1281 "show": true, 1282 "values": [ ] 1283 }, 1284 "yaxes": [ 1285 { 1286 "format": "bytes", 1287 "label": null, 1288 "logBase": 1, 1289 "max": null, 1290 "min": 0, 1291 "show": true 1292 }, 1293 { 1294 "format": "short", 1295 "label": null, 1296 "logBase": 1, 1297 "max": null, 1298 "min": null, 1299 "show": false 1300 } 1301 ] 1302 }, 1303 { 1304 "aliasColors": { }, 1305 "bars": false, 1306 "dashLength": 10, 1307 "dashes": false, 1308 "datasource": "$datasource", 1309 "fill": 10, 1310 "gridPos": { }, 1311 "id": 16, 1312 "legend": { 1313 "avg": false, 1314 "current": false, 1315 "max": false, 1316 "min": false, 1317 "show": true, 1318 "total": false, 1319 "values": false 1320 }, 1321 "lines": true, 1322 "linewidth": 0, 1323 "links": [ ], 1324 "nullPointMode": "null as zero", 1325 "percentage": false, 1326 "pointradius": 5, 1327 "points": false, 1328 "renderer": "flot", 1329 "seriesOverrides": [ ], 1330 "spaceLength": 10, 1331 "span": 6, 1332 "stack": true, 1333 "steppedLine": false, 1334 "targets": [ 1335 { 1336 "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", 1337 "format": "time_series", 1338 "intervalFactor": 2, 1339 "legendFormat": "{{pod}} - {{device}}", 1340 "legendLink": null, 1341 "step": 10 1342 } 1343 ], 1344 "thresholds": [ ], 1345 "timeFrom": null, 1346 "timeShift": null, 1347 "title": "Disk Writes", 1348 "tooltip": { 1349 "shared": true, 1350 "sort": 2, 1351 "value_type": "individual" 1352 }, 1353 "type": "graph", 1354 "xaxis": { 1355 "buckets": null, 1356 "mode": "time", 1357 "name": null, 1358 "show": true, 1359 "values": [ ] 1360 }, 1361 "yaxes": [ 1362 { 1363 "format": "Bps", 1364 "label": null, 1365 "logBase": 1, 1366 "max": null, 1367 "min": 0, 1368 "show": true 1369 }, 1370 { 1371 "format": "short", 1372 "label": null, 1373 "logBase": 1, 1374 "max": null, 1375 "min": null, 1376 "show": false 1377 } 1378 ] 1379 }, 1380 { 1381 "aliasColors": { }, 1382 "bars": false, 1383 "dashLength": 10, 1384 "dashes": false, 1385 "datasource": "$datasource", 1386 "fill": 10, 1387 "gridPos": { }, 1388 "id": 17, 1389 "legend": { 1390 "avg": false, 1391 "current": false, 1392 "max": false, 1393 "min": false, 1394 "show": true, 1395 "total": false, 1396 "values": false 1397 }, 1398 "lines": true, 1399 "linewidth": 0, 1400 "links": [ ], 1401 "nullPointMode": "null as zero", 1402 "percentage": false, 1403 "pointradius": 5, 1404 "points": false, 1405 "renderer": "flot", 1406 "seriesOverrides": [ ], 1407 "spaceLength": 10, 1408 "span": 6, 1409 "stack": true, 1410 "steppedLine": false, 1411 "targets": [ 1412 { 1413 "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"index-gateway\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", 1414 "format": "time_series", 1415 "intervalFactor": 2, 1416 "legendFormat": "{{pod}} - {{device}}", 1417 "legendLink": null, 1418 "step": 10 1419 } 1420 ], 1421 "thresholds": [ ], 1422 "timeFrom": null, 1423 "timeShift": null, 1424 "title": "Disk Reads", 1425 "tooltip": { 1426 "shared": true, 1427 "sort": 2, 1428 "value_type": "individual" 1429 }, 1430 "type": "graph", 1431 "xaxis": { 1432 "buckets": null, 1433 "mode": "time", 1434 "name": null, 1435 "show": true, 1436 "values": [ ] 1437 }, 1438 "yaxes": [ 1439 { 1440 "format": "Bps", 1441 "label": null, 1442 "logBase": 1, 1443 "max": null, 1444 "min": 0, 1445 "show": true 1446 }, 1447 { 1448 "format": "short", 1449 "label": null, 1450 "logBase": 1, 1451 "max": null, 1452 "min": null, 1453 "show": false 1454 } 1455 ] 1456 }, 1457 { 1458 "aliasColors": { }, 1459 "bars": false, 1460 "dashLength": 10, 1461 "dashes": false, 1462 "datasource": "$datasource", 1463 "fill": 1, 1464 "gridPos": { }, 1465 "id": 18, 1466 "legend": { 1467 "avg": false, 1468 "current": false, 1469 "max": false, 1470 "min": false, 1471 "show": true, 1472 "total": false, 1473 "values": false 1474 }, 1475 "lines": true, 1476 "linewidth": 1, 1477 "links": [ ], 1478 "nullPointMode": "null as zero", 1479 "percentage": false, 1480 "pointradius": 5, 1481 "points": false, 1482 "renderer": "flot", 1483 "seriesOverrides": [ ], 1484 "spaceLength": 10, 1485 "span": 6, 1486 "stack": false, 1487 "steppedLine": false, 1488 "targets": [ 1489 { 1490 "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"index-gateway.*\"})", 1491 "format": "time_series", 1492 "intervalFactor": 2, 1493 "legendFormat": "{{persistentvolumeclaim}}", 1494 "legendLink": null, 1495 "step": 10 1496 } 1497 ], 1498 "thresholds": [ ], 1499 "timeFrom": null, 1500 "timeShift": null, 1501 "title": "Disk Space Utilization", 1502 "tooltip": { 1503 "shared": true, 1504 "sort": 2, 1505 "value_type": "individual" 1506 }, 1507 "type": "graph", 1508 "xaxis": { 1509 "buckets": null, 1510 "mode": "time", 1511 "name": null, 1512 "show": true, 1513 "values": [ ] 1514 }, 1515 "yaxes": [ 1516 { 1517 "format": "percentunit", 1518 "label": null, 1519 "logBase": 1, 1520 "max": null, 1521 "min": 0, 1522 "show": true 1523 }, 1524 { 1525 "format": "short", 1526 "label": null, 1527 "logBase": 1, 1528 "max": null, 1529 "min": null, 1530 "show": false 1531 } 1532 ] 1533 }, 1534 { 1535 "aliasColors": { }, 1536 "bars": false, 1537 "dashLength": 10, 1538 "dashes": false, 1539 "datasource": "$datasource", 1540 "fill": 1, 1541 "gridPos": { }, 1542 "id": 19, 1543 "legend": { 1544 "avg": false, 1545 "current": false, 1546 "max": false, 1547 "min": false, 1548 "show": true, 1549 "total": false, 1550 "values": false 1551 }, 1552 "lines": true, 1553 "linewidth": 1, 1554 "links": [ ], 1555 "nullPointMode": "null as zero", 1556 "percentage": false, 1557 "pointradius": 5, 1558 "points": false, 1559 "renderer": "flot", 1560 "seriesOverrides": [ ], 1561 "spaceLength": 10, 1562 "span": 6, 1563 "stack": false, 1564 "steppedLine": false, 1565 "targets": [ 1566 { 1567 "expr": "loki_boltdb_shipper_query_readiness_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}", 1568 "format": "time_series", 1569 "intervalFactor": 2, 1570 "legendFormat": "duration", 1571 "legendLink": null, 1572 "step": 10 1573 } 1574 ], 1575 "thresholds": [ ], 1576 "timeFrom": null, 1577 "timeShift": null, 1578 "title": "Query Readiness Duration", 1579 "tooltip": { 1580 "shared": true, 1581 "sort": 2, 1582 "value_type": "individual" 1583 }, 1584 "type": "graph", 1585 "xaxis": { 1586 "buckets": null, 1587 "mode": "time", 1588 "name": null, 1589 "show": true, 1590 "values": [ ] 1591 }, 1592 "yaxes": [ 1593 { 1594 "format": "s", 1595 "label": null, 1596 "logBase": 1, 1597 "max": null, 1598 "min": 0, 1599 "show": true 1600 }, 1601 { 1602 "format": "short", 1603 "label": null, 1604 "logBase": 1, 1605 "max": null, 1606 "min": null, 1607 "show": false 1608 } 1609 ] 1610 } 1611 ], 1612 "repeat": null, 1613 "repeatIteration": null, 1614 "repeatRowId": null, 1615 "showTitle": true, 1616 "title": "Index Gateway", 1617 "titleSize": "h6", 1618 "type": "row" 1619 }, 1620 { 1621 "collapse": false, 1622 "height": "250px", 1623 "panels": [ 1624 { 1625 "aliasColors": { }, 1626 "bars": false, 1627 "dashLength": 10, 1628 "dashes": false, 1629 "datasource": "$datasource", 1630 "fill": 1, 1631 "id": 20, 1632 "legend": { 1633 "avg": false, 1634 "current": false, 1635 "max": false, 1636 "min": false, 1637 "show": true, 1638 "total": false, 1639 "values": false 1640 }, 1641 "lines": true, 1642 "linewidth": 1, 1643 "links": [ ], 1644 "nullPointMode": "null as zero", 1645 "percentage": false, 1646 "pointradius": 5, 1647 "points": false, 1648 "renderer": "flot", 1649 "seriesOverrides": [ 1650 { 1651 "alias": "limit", 1652 "color": "#E02F44", 1653 "fill": 0 1654 } 1655 ], 1656 "spaceLength": 10, 1657 "span": 4, 1658 "stack": false, 1659 "steppedLine": false, 1660 "targets": [ 1661 { 1662 "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"}[$__rate_interval]))", 1663 "format": "time_series", 1664 "intervalFactor": 2, 1665 "legendFormat": "{{pod}}", 1666 "legendLink": null, 1667 "step": 10 1668 }, 1669 { 1670 "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"})", 1671 "format": "time_series", 1672 "intervalFactor": 2, 1673 "legendFormat": "limit", 1674 "legendLink": null, 1675 "step": 10 1676 } 1677 ], 1678 "thresholds": [ ], 1679 "timeFrom": null, 1680 "timeShift": null, 1681 "title": "CPU", 1682 "tooltip": { 1683 "sort": 2 1684 }, 1685 "type": "graph", 1686 "xaxis": { 1687 "buckets": null, 1688 "mode": "time", 1689 "name": null, 1690 "show": true, 1691 "values": [ ] 1692 }, 1693 "yaxes": [ 1694 { 1695 "format": "short", 1696 "label": null, 1697 "logBase": 1, 1698 "max": null, 1699 "min": 0, 1700 "show": true 1701 }, 1702 { 1703 "format": "short", 1704 "label": null, 1705 "logBase": 1, 1706 "max": null, 1707 "min": null, 1708 "show": false 1709 } 1710 ] 1711 }, 1712 { 1713 "aliasColors": { }, 1714 "bars": false, 1715 "dashLength": 10, 1716 "dashes": false, 1717 "datasource": "$datasource", 1718 "fill": 1, 1719 "id": 21, 1720 "legend": { 1721 "avg": false, 1722 "current": false, 1723 "max": false, 1724 "min": false, 1725 "show": true, 1726 "total": false, 1727 "values": false 1728 }, 1729 "lines": true, 1730 "linewidth": 1, 1731 "links": [ ], 1732 "nullPointMode": "null as zero", 1733 "percentage": false, 1734 "pointradius": 5, 1735 "points": false, 1736 "renderer": "flot", 1737 "seriesOverrides": [ 1738 { 1739 "alias": "limit", 1740 "color": "#E02F44", 1741 "fill": 0 1742 } 1743 ], 1744 "spaceLength": 10, 1745 "span": 4, 1746 "stack": false, 1747 "steppedLine": false, 1748 "targets": [ 1749 { 1750 "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"})", 1751 "format": "time_series", 1752 "intervalFactor": 2, 1753 "legendFormat": "{{pod}}", 1754 "legendLink": null, 1755 "step": 10 1756 }, 1757 { 1758 "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"} > 0)", 1759 "format": "time_series", 1760 "intervalFactor": 2, 1761 "legendFormat": "limit", 1762 "legendLink": null, 1763 "step": 10 1764 } 1765 ], 1766 "thresholds": [ ], 1767 "timeFrom": null, 1768 "timeShift": null, 1769 "title": "Memory (workingset)", 1770 "tooltip": { 1771 "sort": 2 1772 }, 1773 "type": "graph", 1774 "xaxis": { 1775 "buckets": null, 1776 "mode": "time", 1777 "name": null, 1778 "show": true, 1779 "values": [ ] 1780 }, 1781 "yaxes": [ 1782 { 1783 "format": "bytes", 1784 "label": null, 1785 "logBase": 1, 1786 "max": null, 1787 "min": 0, 1788 "show": true 1789 }, 1790 { 1791 "format": "short", 1792 "label": null, 1793 "logBase": 1, 1794 "max": null, 1795 "min": null, 1796 "show": false 1797 } 1798 ] 1799 }, 1800 { 1801 "aliasColors": { }, 1802 "bars": false, 1803 "dashLength": 10, 1804 "dashes": false, 1805 "datasource": "$datasource", 1806 "fill": 1, 1807 "id": 22, 1808 "legend": { 1809 "avg": false, 1810 "current": false, 1811 "max": false, 1812 "min": false, 1813 "show": true, 1814 "total": false, 1815 "values": false 1816 }, 1817 "lines": true, 1818 "linewidth": 1, 1819 "links": [ ], 1820 "nullPointMode": "null as zero", 1821 "percentage": false, 1822 "pointradius": 5, 1823 "points": false, 1824 "renderer": "flot", 1825 "seriesOverrides": [ ], 1826 "spaceLength": 10, 1827 "span": 4, 1828 "stack": false, 1829 "steppedLine": false, 1830 "targets": [ 1831 { 1832 "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/ingester\"})", 1833 "format": "time_series", 1834 "intervalFactor": 2, 1835 "legendFormat": "{{pod}}", 1836 "legendLink": null, 1837 "step": 10 1838 } 1839 ], 1840 "thresholds": [ ], 1841 "timeFrom": null, 1842 "timeShift": null, 1843 "title": "Memory (go heap inuse)", 1844 "tooltip": { 1845 "sort": 2 1846 }, 1847 "type": "graph", 1848 "xaxis": { 1849 "buckets": null, 1850 "mode": "time", 1851 "name": null, 1852 "show": true, 1853 "values": [ ] 1854 }, 1855 "yaxes": [ 1856 { 1857 "format": "bytes", 1858 "label": null, 1859 "logBase": 1, 1860 "max": null, 1861 "min": 0, 1862 "show": true 1863 }, 1864 { 1865 "format": "short", 1866 "label": null, 1867 "logBase": 1, 1868 "max": null, 1869 "min": null, 1870 "show": false 1871 } 1872 ] 1873 } 1874 ], 1875 "repeat": null, 1876 "repeatIteration": null, 1877 "repeatRowId": null, 1878 "showTitle": true, 1879 "title": "Ingester", 1880 "titleSize": "h6" 1881 }, 1882 { 1883 "collapse": false, 1884 "collapsed": false, 1885 "panels": [ 1886 { 1887 "aliasColors": { }, 1888 "bars": false, 1889 "dashLength": 10, 1890 "dashes": false, 1891 "datasource": "$datasource", 1892 "fill": 1, 1893 "gridPos": { }, 1894 "id": 23, 1895 "legend": { 1896 "avg": false, 1897 "current": false, 1898 "max": false, 1899 "min": false, 1900 "show": true, 1901 "total": false, 1902 "values": false 1903 }, 1904 "lines": true, 1905 "linewidth": 1, 1906 "links": [ ], 1907 "nullPointMode": "null as zero", 1908 "percentage": false, 1909 "pointradius": 5, 1910 "points": false, 1911 "renderer": "flot", 1912 "seriesOverrides": [ ], 1913 "spaceLength": 10, 1914 "span": 6, 1915 "stack": false, 1916 "steppedLine": false, 1917 "targets": [ 1918 { 1919 "expr": "sum by(pod) (cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"})", 1920 "format": "time_series", 1921 "intervalFactor": 2, 1922 "legendFormat": "{{pod}}", 1923 "legendLink": null, 1924 "step": 10 1925 } 1926 ], 1927 "thresholds": [ ], 1928 "timeFrom": null, 1929 "timeShift": null, 1930 "title": "Rules", 1931 "tooltip": { 1932 "shared": true, 1933 "sort": 2, 1934 "value_type": "individual" 1935 }, 1936 "type": "graph", 1937 "xaxis": { 1938 "buckets": null, 1939 "mode": "time", 1940 "name": null, 1941 "show": true, 1942 "values": [ ] 1943 }, 1944 "yaxes": [ 1945 { 1946 "format": "short", 1947 "label": null, 1948 "logBase": 1, 1949 "max": null, 1950 "min": 0, 1951 "show": true 1952 }, 1953 { 1954 "format": "short", 1955 "label": null, 1956 "logBase": 1, 1957 "max": null, 1958 "min": null, 1959 "show": false 1960 } 1961 ] 1962 }, 1963 { 1964 "aliasColors": { }, 1965 "bars": false, 1966 "dashLength": 10, 1967 "dashes": false, 1968 "datasource": "$datasource", 1969 "fill": 1, 1970 "gridPos": { }, 1971 "id": 24, 1972 "legend": { 1973 "avg": false, 1974 "current": false, 1975 "max": false, 1976 "min": false, 1977 "show": true, 1978 "total": false, 1979 "values": false 1980 }, 1981 "lines": true, 1982 "linewidth": 1, 1983 "links": [ ], 1984 "nullPointMode": "null as zero", 1985 "percentage": false, 1986 "pointradius": 5, 1987 "points": false, 1988 "renderer": "flot", 1989 "seriesOverrides": [ 1990 { 1991 "alias": "limit", 1992 "color": "#E02F44", 1993 "fill": 0 1994 } 1995 ], 1996 "spaceLength": 10, 1997 "span": 6, 1998 "stack": false, 1999 "steppedLine": false, 2000 "targets": [ 2001 { 2002 "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ruler\"}[$__rate_interval]))", 2003 "format": "time_series", 2004 "intervalFactor": 2, 2005 "legendFormat": "{{pod}}", 2006 "legendLink": null, 2007 "step": 10 2008 }, 2009 { 2010 "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ruler\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ruler\"})", 2011 "format": "time_series", 2012 "intervalFactor": 2, 2013 "legendFormat": "limit", 2014 "legendLink": null, 2015 "step": 10 2016 } 2017 ], 2018 "thresholds": [ ], 2019 "timeFrom": null, 2020 "timeShift": null, 2021 "title": "CPU", 2022 "tooltip": { 2023 "sort": 2 2024 }, 2025 "type": "graph", 2026 "xaxis": { 2027 "buckets": null, 2028 "mode": "time", 2029 "name": null, 2030 "show": true, 2031 "values": [ ] 2032 }, 2033 "yaxes": [ 2034 { 2035 "format": "short", 2036 "label": null, 2037 "logBase": 1, 2038 "max": null, 2039 "min": 0, 2040 "show": true 2041 }, 2042 { 2043 "format": "short", 2044 "label": null, 2045 "logBase": 1, 2046 "max": null, 2047 "min": null, 2048 "show": false 2049 } 2050 ] 2051 }, 2052 { 2053 "aliasColors": { }, 2054 "bars": false, 2055 "dashLength": 10, 2056 "dashes": false, 2057 "datasource": "$datasource", 2058 "fill": 1, 2059 "gridPos": { }, 2060 "id": 25, 2061 "legend": { 2062 "avg": false, 2063 "current": false, 2064 "max": false, 2065 "min": false, 2066 "show": true, 2067 "total": false, 2068 "values": false 2069 }, 2070 "lines": true, 2071 "linewidth": 1, 2072 "links": [ ], 2073 "nullPointMode": "null as zero", 2074 "percentage": false, 2075 "pointradius": 5, 2076 "points": false, 2077 "renderer": "flot", 2078 "seriesOverrides": [ 2079 { 2080 "alias": "limit", 2081 "color": "#E02F44", 2082 "fill": 0 2083 } 2084 ], 2085 "spaceLength": 10, 2086 "span": 6, 2087 "stack": false, 2088 "steppedLine": false, 2089 "targets": [ 2090 { 2091 "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ruler\"})", 2092 "format": "time_series", 2093 "intervalFactor": 2, 2094 "legendFormat": "{{pod}}", 2095 "legendLink": null, 2096 "step": 10 2097 }, 2098 { 2099 "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ruler\"} > 0)", 2100 "format": "time_series", 2101 "intervalFactor": 2, 2102 "legendFormat": "limit", 2103 "legendLink": null, 2104 "step": 10 2105 } 2106 ], 2107 "thresholds": [ ], 2108 "timeFrom": null, 2109 "timeShift": null, 2110 "title": "Memory (workingset)", 2111 "tooltip": { 2112 "sort": 2 2113 }, 2114 "type": "graph", 2115 "xaxis": { 2116 "buckets": null, 2117 "mode": "time", 2118 "name": null, 2119 "show": true, 2120 "values": [ ] 2121 }, 2122 "yaxes": [ 2123 { 2124 "format": "bytes", 2125 "label": null, 2126 "logBase": 1, 2127 "max": null, 2128 "min": 0, 2129 "show": true 2130 }, 2131 { 2132 "format": "short", 2133 "label": null, 2134 "logBase": 1, 2135 "max": null, 2136 "min": null, 2137 "show": false 2138 } 2139 ] 2140 }, 2141 { 2142 "aliasColors": { }, 2143 "bars": false, 2144 "dashLength": 10, 2145 "dashes": false, 2146 "datasource": "$datasource", 2147 "fill": 1, 2148 "gridPos": { }, 2149 "id": 26, 2150 "legend": { 2151 "avg": false, 2152 "current": false, 2153 "max": false, 2154 "min": false, 2155 "show": true, 2156 "total": false, 2157 "values": false 2158 }, 2159 "lines": true, 2160 "linewidth": 1, 2161 "links": [ ], 2162 "nullPointMode": "null as zero", 2163 "percentage": false, 2164 "pointradius": 5, 2165 "points": false, 2166 "renderer": "flot", 2167 "seriesOverrides": [ ], 2168 "spaceLength": 10, 2169 "span": 6, 2170 "stack": false, 2171 "steppedLine": false, 2172 "targets": [ 2173 { 2174 "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/ruler\"})", 2175 "format": "time_series", 2176 "intervalFactor": 2, 2177 "legendFormat": "{{pod}}", 2178 "legendLink": null, 2179 "step": 10 2180 } 2181 ], 2182 "thresholds": [ ], 2183 "timeFrom": null, 2184 "timeShift": null, 2185 "title": "Memory (go heap inuse)", 2186 "tooltip": { 2187 "sort": 2 2188 }, 2189 "type": "graph", 2190 "xaxis": { 2191 "buckets": null, 2192 "mode": "time", 2193 "name": null, 2194 "show": true, 2195 "values": [ ] 2196 }, 2197 "yaxes": [ 2198 { 2199 "format": "bytes", 2200 "label": null, 2201 "logBase": 1, 2202 "max": null, 2203 "min": 0, 2204 "show": true 2205 }, 2206 { 2207 "format": "short", 2208 "label": null, 2209 "logBase": 1, 2210 "max": null, 2211 "min": null, 2212 "show": false 2213 } 2214 ] 2215 } 2216 ], 2217 "repeat": null, 2218 "repeatIteration": null, 2219 "repeatRowId": null, 2220 "showTitle": true, 2221 "title": "Ruler", 2222 "titleSize": "h6", 2223 "type": "row" 2224 } 2225 ], 2226 "schemaVersion": 14, 2227 "style": "dark", 2228 "tags": [ 2229 "loki" 2230 ], 2231 "templating": { 2232 "list": [ 2233 { 2234 "current": { 2235 "text": "default", 2236 "value": "default" 2237 }, 2238 "hide": 0, 2239 "label": "Data Source", 2240 "name": "datasource", 2241 "options": [ ], 2242 "query": "prometheus", 2243 "refresh": 1, 2244 "regex": "", 2245 "type": "datasource" 2246 }, 2247 { 2248 "allValue": null, 2249 "current": { 2250 "text": "prod", 2251 "value": "prod" 2252 }, 2253 "datasource": "$datasource", 2254 "hide": 0, 2255 "includeAll": false, 2256 "label": "cluster", 2257 "multi": false, 2258 "name": "cluster", 2259 "options": [ ], 2260 "query": "label_values(loki_build_info, cluster)", 2261 "refresh": 1, 2262 "regex": "", 2263 "sort": 2, 2264 "tagValuesQuery": "", 2265 "tags": [ ], 2266 "tagsQuery": "", 2267 "type": "query", 2268 "useTags": false 2269 }, 2270 { 2271 "allValue": null, 2272 "current": { 2273 "text": "prod", 2274 "value": "prod" 2275 }, 2276 "datasource": "$datasource", 2277 "hide": 0, 2278 "includeAll": false, 2279 "label": "namespace", 2280 "multi": false, 2281 "name": "namespace", 2282 "options": [ ], 2283 "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", 2284 "refresh": 1, 2285 "regex": "", 2286 "sort": 2, 2287 "tagValuesQuery": "", 2288 "tags": [ ], 2289 "tagsQuery": "", 2290 "type": "query", 2291 "useTags": false 2292 } 2293 ] 2294 }, 2295 "time": { 2296 "from": "now-1h", 2297 "to": "now" 2298 }, 2299 "timepicker": { 2300 "refresh_intervals": [ 2301 "5s", 2302 "10s", 2303 "30s", 2304 "1m", 2305 "5m", 2306 "15m", 2307 "30m", 2308 "1h", 2309 "2h", 2310 "1d" 2311 ], 2312 "time_options": [ 2313 "5m", 2314 "15m", 2315 "1h", 2316 "6h", 2317 "12h", 2318 "24h", 2319 "2d", 2320 "7d", 2321 "30d" 2322 ] 2323 }, 2324 "timezone": "utc", 2325 "title": "Loki / Reads Resources", 2326 "uid": "reads-resources", 2327 "version": 0 2328 }