github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/production/loki-mixin-compiled/dashboards/loki-writes-resources.json (about) 1 { 2 "annotations": { 3 "list": [ ] 4 }, 5 "editable": true, 6 "gnetId": null, 7 "graphTooltip": 0, 8 "hideControls": false, 9 "links": [ 10 { 11 "asDropdown": true, 12 "icon": "external link", 13 "includeVars": true, 14 "keepTime": true, 15 "tags": [ 16 "loki" 17 ], 18 "targetBlank": false, 19 "title": "Loki Dashboards", 20 "type": "dashboards" 21 } 22 ], 23 "refresh": "10s", 24 "rows": [ 25 { 26 "collapse": false, 27 "height": "250px", 28 "panels": [ 29 { 30 "aliasColors": { }, 31 "bars": false, 32 "dashLength": 10, 33 "dashes": false, 34 "datasource": "$datasource", 35 "fill": 1, 36 "id": 1, 37 "legend": { 38 "avg": false, 39 "current": false, 40 "max": false, 41 "min": false, 42 "show": true, 43 "total": false, 44 "values": false 45 }, 46 "lines": true, 47 "linewidth": 1, 48 "links": [ ], 49 "nullPointMode": "null as zero", 50 "percentage": false, 51 "pointradius": 5, 52 "points": false, 53 "renderer": "flot", 54 "seriesOverrides": [ 55 { 56 "alias": "limit", 57 "color": "#E02F44", 58 "fill": 0 59 } 60 ], 61 "spaceLength": 10, 62 "span": 4, 63 "stack": false, 64 "steppedLine": false, 65 "targets": [ 66 { 67 "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"distributor\"}[$__rate_interval]))", 68 "format": "time_series", 69 "intervalFactor": 2, 70 "legendFormat": "{{pod}}", 71 "legendLink": null, 72 "step": 10 73 }, 74 { 75 "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"distributor\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"distributor\"})", 76 "format": "time_series", 77 "intervalFactor": 2, 78 "legendFormat": "limit", 79 "legendLink": null, 80 "step": 10 81 } 82 ], 83 "thresholds": [ ], 84 "timeFrom": null, 85 "timeShift": null, 86 "title": "CPU", 87 "tooltip": { 88 "sort": 2 89 }, 90 "type": "graph", 91 "xaxis": { 92 "buckets": null, 93 "mode": "time", 94 "name": null, 95 "show": true, 96 "values": [ ] 97 }, 98 "yaxes": [ 99 { 100 "format": "short", 101 "label": null, 102 "logBase": 1, 103 "max": null, 104 "min": 0, 105 "show": true 106 }, 107 { 108 "format": "short", 109 "label": null, 110 "logBase": 1, 111 "max": null, 112 "min": null, 113 "show": false 114 } 115 ] 116 }, 117 { 118 "aliasColors": { }, 119 "bars": false, 120 "dashLength": 10, 121 "dashes": false, 122 "datasource": "$datasource", 123 "fill": 1, 124 "id": 2, 125 "legend": { 126 "avg": false, 127 "current": false, 128 "max": false, 129 "min": false, 130 "show": true, 131 "total": false, 132 "values": false 133 }, 134 "lines": true, 135 "linewidth": 1, 136 "links": [ ], 137 "nullPointMode": "null as zero", 138 "percentage": false, 139 "pointradius": 5, 140 "points": false, 141 "renderer": "flot", 142 "seriesOverrides": [ 143 { 144 "alias": "limit", 145 "color": "#E02F44", 146 "fill": 0 147 } 148 ], 149 "spaceLength": 10, 150 "span": 4, 151 "stack": false, 152 "steppedLine": false, 153 "targets": [ 154 { 155 "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"distributor\"})", 156 "format": "time_series", 157 "intervalFactor": 2, 158 "legendFormat": "{{pod}}", 159 "legendLink": null, 160 "step": 10 161 }, 162 { 163 "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"distributor\"} > 0)", 164 "format": "time_series", 165 "intervalFactor": 2, 166 "legendFormat": "limit", 167 "legendLink": null, 168 "step": 10 169 } 170 ], 171 "thresholds": [ ], 172 "timeFrom": null, 173 "timeShift": null, 174 "title": "Memory (workingset)", 175 "tooltip": { 176 "sort": 2 177 }, 178 "type": "graph", 179 "xaxis": { 180 "buckets": null, 181 "mode": "time", 182 "name": null, 183 "show": true, 184 "values": [ ] 185 }, 186 "yaxes": [ 187 { 188 "format": "bytes", 189 "label": null, 190 "logBase": 1, 191 "max": null, 192 "min": 0, 193 "show": true 194 }, 195 { 196 "format": "short", 197 "label": null, 198 "logBase": 1, 199 "max": null, 200 "min": null, 201 "show": false 202 } 203 ] 204 }, 205 { 206 "aliasColors": { }, 207 "bars": false, 208 "dashLength": 10, 209 "dashes": false, 210 "datasource": "$datasource", 211 "fill": 1, 212 "id": 3, 213 "legend": { 214 "avg": false, 215 "current": false, 216 "max": false, 217 "min": false, 218 "show": true, 219 "total": false, 220 "values": false 221 }, 222 "lines": true, 223 "linewidth": 1, 224 "links": [ ], 225 "nullPointMode": "null as zero", 226 "percentage": false, 227 "pointradius": 5, 228 "points": false, 229 "renderer": "flot", 230 "seriesOverrides": [ ], 231 "spaceLength": 10, 232 "span": 4, 233 "stack": false, 234 "steppedLine": false, 235 "targets": [ 236 { 237 "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/distributor\"})", 238 "format": "time_series", 239 "intervalFactor": 2, 240 "legendFormat": "{{pod}}", 241 "legendLink": null, 242 "step": 10 243 } 244 ], 245 "thresholds": [ ], 246 "timeFrom": null, 247 "timeShift": null, 248 "title": "Memory (go heap inuse)", 249 "tooltip": { 250 "sort": 2 251 }, 252 "type": "graph", 253 "xaxis": { 254 "buckets": null, 255 "mode": "time", 256 "name": null, 257 "show": true, 258 "values": [ ] 259 }, 260 "yaxes": [ 261 { 262 "format": "bytes", 263 "label": null, 264 "logBase": 1, 265 "max": null, 266 "min": 0, 267 "show": true 268 }, 269 { 270 "format": "short", 271 "label": null, 272 "logBase": 1, 273 "max": null, 274 "min": null, 275 "show": false 276 } 277 ] 278 } 279 ], 280 "repeat": null, 281 "repeatIteration": null, 282 "repeatRowId": null, 283 "showTitle": true, 284 "title": "Distributor", 285 "titleSize": "h6" 286 }, 287 { 288 "collapse": false, 289 "collapsed": false, 290 "panels": [ 291 { 292 "aliasColors": { }, 293 "bars": false, 294 "dashLength": 10, 295 "dashes": false, 296 "datasource": "$datasource", 297 "fill": 1, 298 "gridPos": { }, 299 "id": 4, 300 "legend": { 301 "avg": false, 302 "current": false, 303 "max": false, 304 "min": false, 305 "show": true, 306 "total": false, 307 "values": false 308 }, 309 "lines": true, 310 "linewidth": 1, 311 "links": [ ], 312 "nullPointMode": "null as zero", 313 "percentage": false, 314 "pointradius": 5, 315 "points": false, 316 "renderer": "flot", 317 "seriesOverrides": [ ], 318 "spaceLength": 10, 319 "span": 6, 320 "stack": false, 321 "steppedLine": false, 322 "targets": [ 323 { 324 "expr": "sum by(pod) (loki_ingester_memory_streams{cluster=~\"$cluster\", job=~\"($namespace)/ingester\"})", 325 "format": "time_series", 326 "intervalFactor": 2, 327 "legendFormat": "{{pod}}", 328 "legendLink": null, 329 "step": 10 330 } 331 ], 332 "thresholds": [ ], 333 "timeFrom": null, 334 "timeShift": null, 335 "title": "In-memory streams", 336 "tooltip": { 337 "sort": 2 338 }, 339 "type": "graph", 340 "xaxis": { 341 "buckets": null, 342 "mode": "time", 343 "name": null, 344 "show": true, 345 "values": [ ] 346 }, 347 "yaxes": [ 348 { 349 "format": "short", 350 "label": null, 351 "logBase": 1, 352 "max": null, 353 "min": 0, 354 "show": true 355 }, 356 { 357 "format": "short", 358 "label": null, 359 "logBase": 1, 360 "max": null, 361 "min": null, 362 "show": false 363 } 364 ] 365 }, 366 { 367 "aliasColors": { }, 368 "bars": false, 369 "dashLength": 10, 370 "dashes": false, 371 "datasource": "$datasource", 372 "fill": 1, 373 "gridPos": { }, 374 "id": 5, 375 "legend": { 376 "avg": false, 377 "current": false, 378 "max": false, 379 "min": false, 380 "show": true, 381 "total": false, 382 "values": false 383 }, 384 "lines": true, 385 "linewidth": 1, 386 "links": [ ], 387 "nullPointMode": "null as zero", 388 "percentage": false, 389 "pointradius": 5, 390 "points": false, 391 "renderer": "flot", 392 "seriesOverrides": [ 393 { 394 "alias": "limit", 395 "color": "#E02F44", 396 "fill": 0 397 } 398 ], 399 "spaceLength": 10, 400 "span": 6, 401 "stack": false, 402 "steppedLine": false, 403 "targets": [ 404 { 405 "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"}[$__rate_interval]))", 406 "format": "time_series", 407 "intervalFactor": 2, 408 "legendFormat": "{{pod}}", 409 "legendLink": null, 410 "step": 10 411 }, 412 { 413 "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"})", 414 "format": "time_series", 415 "intervalFactor": 2, 416 "legendFormat": "limit", 417 "legendLink": null, 418 "step": 10 419 } 420 ], 421 "thresholds": [ ], 422 "timeFrom": null, 423 "timeShift": null, 424 "title": "CPU", 425 "tooltip": { 426 "sort": 2 427 }, 428 "type": "graph", 429 "xaxis": { 430 "buckets": null, 431 "mode": "time", 432 "name": null, 433 "show": true, 434 "values": [ ] 435 }, 436 "yaxes": [ 437 { 438 "format": "short", 439 "label": null, 440 "logBase": 1, 441 "max": null, 442 "min": 0, 443 "show": true 444 }, 445 { 446 "format": "short", 447 "label": null, 448 "logBase": 1, 449 "max": null, 450 "min": null, 451 "show": false 452 } 453 ] 454 }, 455 { 456 "aliasColors": { }, 457 "bars": false, 458 "dashLength": 10, 459 "dashes": false, 460 "datasource": "$datasource", 461 "fill": 1, 462 "gridPos": { }, 463 "id": 6, 464 "legend": { 465 "avg": false, 466 "current": false, 467 "max": false, 468 "min": false, 469 "show": true, 470 "total": false, 471 "values": false 472 }, 473 "lines": true, 474 "linewidth": 1, 475 "links": [ ], 476 "nullPointMode": "null as zero", 477 "percentage": false, 478 "pointradius": 5, 479 "points": false, 480 "renderer": "flot", 481 "seriesOverrides": [ 482 { 483 "alias": "limit", 484 "color": "#E02F44", 485 "fill": 0 486 } 487 ], 488 "spaceLength": 10, 489 "span": 6, 490 "stack": false, 491 "steppedLine": false, 492 "targets": [ 493 { 494 "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"})", 495 "format": "time_series", 496 "intervalFactor": 2, 497 "legendFormat": "{{pod}}", 498 "legendLink": null, 499 "step": 10 500 }, 501 { 502 "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\"} > 0)", 503 "format": "time_series", 504 "intervalFactor": 2, 505 "legendFormat": "limit", 506 "legendLink": null, 507 "step": 10 508 } 509 ], 510 "thresholds": [ ], 511 "timeFrom": null, 512 "timeShift": null, 513 "title": "Memory (workingset)", 514 "tooltip": { 515 "sort": 2 516 }, 517 "type": "graph", 518 "xaxis": { 519 "buckets": null, 520 "mode": "time", 521 "name": null, 522 "show": true, 523 "values": [ ] 524 }, 525 "yaxes": [ 526 { 527 "format": "bytes", 528 "label": null, 529 "logBase": 1, 530 "max": null, 531 "min": 0, 532 "show": true 533 }, 534 { 535 "format": "short", 536 "label": null, 537 "logBase": 1, 538 "max": null, 539 "min": null, 540 "show": false 541 } 542 ] 543 }, 544 { 545 "aliasColors": { }, 546 "bars": false, 547 "dashLength": 10, 548 "dashes": false, 549 "datasource": "$datasource", 550 "fill": 1, 551 "gridPos": { }, 552 "id": 7, 553 "legend": { 554 "avg": false, 555 "current": false, 556 "max": false, 557 "min": false, 558 "show": true, 559 "total": false, 560 "values": false 561 }, 562 "lines": true, 563 "linewidth": 1, 564 "links": [ ], 565 "nullPointMode": "null as zero", 566 "percentage": false, 567 "pointradius": 5, 568 "points": false, 569 "renderer": "flot", 570 "seriesOverrides": [ ], 571 "spaceLength": 10, 572 "span": 6, 573 "stack": false, 574 "steppedLine": false, 575 "targets": [ 576 { 577 "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/ingester\"})", 578 "format": "time_series", 579 "intervalFactor": 2, 580 "legendFormat": "{{pod}}", 581 "legendLink": null, 582 "step": 10 583 } 584 ], 585 "thresholds": [ ], 586 "timeFrom": null, 587 "timeShift": null, 588 "title": "Memory (go heap inuse)", 589 "tooltip": { 590 "sort": 2 591 }, 592 "type": "graph", 593 "xaxis": { 594 "buckets": null, 595 "mode": "time", 596 "name": null, 597 "show": true, 598 "values": [ ] 599 }, 600 "yaxes": [ 601 { 602 "format": "bytes", 603 "label": null, 604 "logBase": 1, 605 "max": null, 606 "min": 0, 607 "show": true 608 }, 609 { 610 "format": "short", 611 "label": null, 612 "logBase": 1, 613 "max": null, 614 "min": null, 615 "show": false 616 } 617 ] 618 }, 619 { 620 "aliasColors": { }, 621 "bars": false, 622 "dashLength": 10, 623 "dashes": false, 624 "datasource": "$datasource", 625 "fill": 10, 626 "gridPos": { }, 627 "id": 8, 628 "legend": { 629 "avg": false, 630 "current": false, 631 "max": false, 632 "min": false, 633 "show": true, 634 "total": false, 635 "values": false 636 }, 637 "lines": true, 638 "linewidth": 0, 639 "links": [ ], 640 "nullPointMode": "null as zero", 641 "percentage": false, 642 "pointradius": 5, 643 "points": false, 644 "renderer": "flot", 645 "seriesOverrides": [ ], 646 "spaceLength": 10, 647 "span": 6, 648 "stack": true, 649 "steppedLine": false, 650 "targets": [ 651 { 652 "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", 653 "format": "time_series", 654 "intervalFactor": 2, 655 "legendFormat": "{{pod}} - {{device}}", 656 "legendLink": null, 657 "step": 10 658 } 659 ], 660 "thresholds": [ ], 661 "timeFrom": null, 662 "timeShift": null, 663 "title": "Disk Writes", 664 "tooltip": { 665 "shared": true, 666 "sort": 2, 667 "value_type": "individual" 668 }, 669 "type": "graph", 670 "xaxis": { 671 "buckets": null, 672 "mode": "time", 673 "name": null, 674 "show": true, 675 "values": [ ] 676 }, 677 "yaxes": [ 678 { 679 "format": "Bps", 680 "label": null, 681 "logBase": 1, 682 "max": null, 683 "min": 0, 684 "show": true 685 }, 686 { 687 "format": "short", 688 "label": null, 689 "logBase": 1, 690 "max": null, 691 "min": null, 692 "show": false 693 } 694 ] 695 }, 696 { 697 "aliasColors": { }, 698 "bars": false, 699 "dashLength": 10, 700 "dashes": false, 701 "datasource": "$datasource", 702 "fill": 10, 703 "gridPos": { }, 704 "id": 9, 705 "legend": { 706 "avg": false, 707 "current": false, 708 "max": false, 709 "min": false, 710 "show": true, 711 "total": false, 712 "values": false 713 }, 714 "lines": true, 715 "linewidth": 0, 716 "links": [ ], 717 "nullPointMode": "null as zero", 718 "percentage": false, 719 "pointradius": 5, 720 "points": false, 721 "renderer": "flot", 722 "seriesOverrides": [ ], 723 "spaceLength": 10, 724 "span": 6, 725 "stack": true, 726 "steppedLine": false, 727 "targets": [ 728 { 729 "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"ingester\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", 730 "format": "time_series", 731 "intervalFactor": 2, 732 "legendFormat": "{{pod}} - {{device}}", 733 "legendLink": null, 734 "step": 10 735 } 736 ], 737 "thresholds": [ ], 738 "timeFrom": null, 739 "timeShift": null, 740 "title": "Disk Reads", 741 "tooltip": { 742 "shared": true, 743 "sort": 2, 744 "value_type": "individual" 745 }, 746 "type": "graph", 747 "xaxis": { 748 "buckets": null, 749 "mode": "time", 750 "name": null, 751 "show": true, 752 "values": [ ] 753 }, 754 "yaxes": [ 755 { 756 "format": "Bps", 757 "label": null, 758 "logBase": 1, 759 "max": null, 760 "min": 0, 761 "show": true 762 }, 763 { 764 "format": "short", 765 "label": null, 766 "logBase": 1, 767 "max": null, 768 "min": null, 769 "show": false 770 } 771 ] 772 }, 773 { 774 "aliasColors": { }, 775 "bars": false, 776 "dashLength": 10, 777 "dashes": false, 778 "datasource": "$datasource", 779 "fill": 1, 780 "gridPos": { }, 781 "id": 10, 782 "legend": { 783 "avg": false, 784 "current": false, 785 "max": false, 786 "min": false, 787 "show": true, 788 "total": false, 789 "values": false 790 }, 791 "lines": true, 792 "linewidth": 1, 793 "links": [ ], 794 "nullPointMode": "null as zero", 795 "percentage": false, 796 "pointradius": 5, 797 "points": false, 798 "renderer": "flot", 799 "seriesOverrides": [ ], 800 "spaceLength": 10, 801 "span": 6, 802 "stack": false, 803 "steppedLine": false, 804 "targets": [ 805 { 806 "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"ingester.*\"})", 807 "format": "time_series", 808 "intervalFactor": 2, 809 "legendFormat": "{{persistentvolumeclaim}}", 810 "legendLink": null, 811 "step": 10 812 } 813 ], 814 "thresholds": [ ], 815 "timeFrom": null, 816 "timeShift": null, 817 "title": "Disk Space Utilization", 818 "tooltip": { 819 "shared": true, 820 "sort": 2, 821 "value_type": "individual" 822 }, 823 "type": "graph", 824 "xaxis": { 825 "buckets": null, 826 "mode": "time", 827 "name": null, 828 "show": true, 829 "values": [ ] 830 }, 831 "yaxes": [ 832 { 833 "format": "percentunit", 834 "label": null, 835 "logBase": 1, 836 "max": null, 837 "min": 0, 838 "show": true 839 }, 840 { 841 "format": "short", 842 "label": null, 843 "logBase": 1, 844 "max": null, 845 "min": null, 846 "show": false 847 } 848 ] 849 } 850 ], 851 "repeat": null, 852 "repeatIteration": null, 853 "repeatRowId": null, 854 "showTitle": true, 855 "title": "Ingester", 856 "titleSize": "h6", 857 "type": "row" 858 } 859 ], 860 "schemaVersion": 14, 861 "style": "dark", 862 "tags": [ 863 "loki" 864 ], 865 "templating": { 866 "list": [ 867 { 868 "current": { 869 "text": "default", 870 "value": "default" 871 }, 872 "hide": 0, 873 "label": "Data Source", 874 "name": "datasource", 875 "options": [ ], 876 "query": "prometheus", 877 "refresh": 1, 878 "regex": "", 879 "type": "datasource" 880 }, 881 { 882 "allValue": null, 883 "current": { 884 "text": "prod", 885 "value": "prod" 886 }, 887 "datasource": "$datasource", 888 "hide": 0, 889 "includeAll": false, 890 "label": "cluster", 891 "multi": false, 892 "name": "cluster", 893 "options": [ ], 894 "query": "label_values(loki_build_info, cluster)", 895 "refresh": 1, 896 "regex": "", 897 "sort": 2, 898 "tagValuesQuery": "", 899 "tags": [ ], 900 "tagsQuery": "", 901 "type": "query", 902 "useTags": false 903 }, 904 { 905 "allValue": null, 906 "current": { 907 "text": "prod", 908 "value": "prod" 909 }, 910 "datasource": "$datasource", 911 "hide": 0, 912 "includeAll": false, 913 "label": "namespace", 914 "multi": false, 915 "name": "namespace", 916 "options": [ ], 917 "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", 918 "refresh": 1, 919 "regex": "", 920 "sort": 2, 921 "tagValuesQuery": "", 922 "tags": [ ], 923 "tagsQuery": "", 924 "type": "query", 925 "useTags": false 926 } 927 ] 928 }, 929 "time": { 930 "from": "now-1h", 931 "to": "now" 932 }, 933 "timepicker": { 934 "refresh_intervals": [ 935 "5s", 936 "10s", 937 "30s", 938 "1m", 939 "5m", 940 "15m", 941 "30m", 942 "1h", 943 "2h", 944 "1d" 945 ], 946 "time_options": [ 947 "5m", 948 "15m", 949 "1h", 950 "6h", 951 "12h", 952 "24h", 953 "2d", 954 "7d", 955 "30d" 956 ] 957 }, 958 "timezone": "utc", 959 "title": "Loki / Writes Resources", 960 "uid": "writes-resources", 961 "version": 0 962 }