github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/production/loki-mixin-compiled-ssd/dashboards/loki-writes-resources.json (about) 1 { 2 "annotations": { 3 "list": [ ] 4 }, 5 "editable": true, 6 "gnetId": null, 7 "graphTooltip": 0, 8 "hideControls": false, 9 "links": [ 10 { 11 "asDropdown": true, 12 "icon": "external link", 13 "includeVars": true, 14 "keepTime": true, 15 "tags": [ 16 "loki" 17 ], 18 "targetBlank": false, 19 "title": "Loki Dashboards", 20 "type": "dashboards" 21 } 22 ], 23 "refresh": "10s", 24 "rows": [ 25 { 26 "collapse": false, 27 "collapsed": false, 28 "panels": [ 29 { 30 "aliasColors": { }, 31 "bars": false, 32 "dashLength": 10, 33 "dashes": false, 34 "datasource": "$datasource", 35 "fill": 1, 36 "gridPos": { }, 37 "id": 1, 38 "legend": { 39 "avg": false, 40 "current": false, 41 "max": false, 42 "min": false, 43 "show": true, 44 "total": false, 45 "values": false 46 }, 47 "lines": true, 48 "linewidth": 1, 49 "links": [ ], 50 "nullPointMode": "null as zero", 51 "percentage": false, 52 "pointradius": 5, 53 "points": false, 54 "renderer": "flot", 55 "seriesOverrides": [ ], 56 "spaceLength": 10, 57 "span": 6, 58 "stack": false, 59 "steppedLine": false, 60 "targets": [ 61 { 62 "expr": "sum by(pod) (loki_ingester_memory_streams{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})", 63 "format": "time_series", 64 "intervalFactor": 2, 65 "legendFormat": "{{pod}}", 66 "legendLink": null, 67 "step": 10 68 } 69 ], 70 "thresholds": [ ], 71 "timeFrom": null, 72 "timeShift": null, 73 "title": "In-memory streams", 74 "tooltip": { 75 "sort": 2 76 }, 77 "type": "graph", 78 "xaxis": { 79 "buckets": null, 80 "mode": "time", 81 "name": null, 82 "show": true, 83 "values": [ ] 84 }, 85 "yaxes": [ 86 { 87 "format": "short", 88 "label": null, 89 "logBase": 1, 90 "max": null, 91 "min": 0, 92 "show": true 93 }, 94 { 95 "format": "short", 96 "label": null, 97 "logBase": 1, 98 "max": null, 99 "min": null, 100 "show": false 101 } 102 ] 103 }, 104 { 105 "aliasColors": { }, 106 "bars": false, 107 "dashLength": 10, 108 "dashes": false, 109 "datasource": "$datasource", 110 "fill": 1, 111 "gridPos": { }, 112 "id": 2, 113 "legend": { 114 "avg": false, 115 "current": false, 116 "max": false, 117 "min": false, 118 "show": true, 119 "total": false, 120 "values": false 121 }, 122 "lines": true, 123 "linewidth": 1, 124 "links": [ ], 125 "nullPointMode": "null as zero", 126 "percentage": false, 127 "pointradius": 5, 128 "points": false, 129 "renderer": "flot", 130 "seriesOverrides": [ 131 { 132 "alias": "limit", 133 "color": "#E02F44", 134 "fill": 0 135 } 136 ], 137 "spaceLength": 10, 138 "span": 6, 139 "stack": false, 140 "steppedLine": false, 141 "targets": [ 142 { 143 "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"}[$__rate_interval]))", 144 "format": "time_series", 145 "intervalFactor": 2, 146 "legendFormat": "{{pod}}", 147 "legendLink": null, 148 "step": 10 149 }, 150 { 151 "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", 152 "format": "time_series", 153 "intervalFactor": 2, 154 "legendFormat": "limit", 155 "legendLink": null, 156 "step": 10 157 } 158 ], 159 "thresholds": [ ], 160 "timeFrom": null, 161 "timeShift": null, 162 "title": "CPU", 163 "tooltip": { 164 "sort": 2 165 }, 166 "type": "graph", 167 "xaxis": { 168 "buckets": null, 169 "mode": "time", 170 "name": null, 171 "show": true, 172 "values": [ ] 173 }, 174 "yaxes": [ 175 { 176 "format": "short", 177 "label": null, 178 "logBase": 1, 179 "max": null, 180 "min": 0, 181 "show": true 182 }, 183 { 184 "format": "short", 185 "label": null, 186 "logBase": 1, 187 "max": null, 188 "min": null, 189 "show": false 190 } 191 ] 192 }, 193 { 194 "aliasColors": { }, 195 "bars": false, 196 "dashLength": 10, 197 "dashes": false, 198 "datasource": "$datasource", 199 "fill": 1, 200 "gridPos": { }, 201 "id": 3, 202 "legend": { 203 "avg": false, 204 "current": false, 205 "max": false, 206 "min": false, 207 "show": true, 208 "total": false, 209 "values": false 210 }, 211 "lines": true, 212 "linewidth": 1, 213 "links": [ ], 214 "nullPointMode": "null as zero", 215 "percentage": false, 216 "pointradius": 5, 217 "points": false, 218 "renderer": "flot", 219 "seriesOverrides": [ 220 { 221 "alias": "limit", 222 "color": "#E02F44", 223 "fill": 0 224 } 225 ], 226 "spaceLength": 10, 227 "span": 6, 228 "stack": false, 229 "steppedLine": false, 230 "targets": [ 231 { 232 "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", 233 "format": "time_series", 234 "intervalFactor": 2, 235 "legendFormat": "{{pod}}", 236 "legendLink": null, 237 "step": 10 238 }, 239 { 240 "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} > 0)", 241 "format": "time_series", 242 "intervalFactor": 2, 243 "legendFormat": "limit", 244 "legendLink": null, 245 "step": 10 246 } 247 ], 248 "thresholds": [ ], 249 "timeFrom": null, 250 "timeShift": null, 251 "title": "Memory (workingset)", 252 "tooltip": { 253 "sort": 2 254 }, 255 "type": "graph", 256 "xaxis": { 257 "buckets": null, 258 "mode": "time", 259 "name": null, 260 "show": true, 261 "values": [ ] 262 }, 263 "yaxes": [ 264 { 265 "format": "bytes", 266 "label": null, 267 "logBase": 1, 268 "max": null, 269 "min": 0, 270 "show": true 271 }, 272 { 273 "format": "short", 274 "label": null, 275 "logBase": 1, 276 "max": null, 277 "min": null, 278 "show": false 279 } 280 ] 281 }, 282 { 283 "aliasColors": { }, 284 "bars": false, 285 "dashLength": 10, 286 "dashes": false, 287 "datasource": "$datasource", 288 "fill": 1, 289 "gridPos": { }, 290 "id": 4, 291 "legend": { 292 "avg": false, 293 "current": false, 294 "max": false, 295 "min": false, 296 "show": true, 297 "total": false, 298 "values": false 299 }, 300 "lines": true, 301 "linewidth": 1, 302 "links": [ ], 303 "nullPointMode": "null as zero", 304 "percentage": false, 305 "pointradius": 5, 306 "points": false, 307 "renderer": "flot", 308 "seriesOverrides": [ ], 309 "spaceLength": 10, 310 "span": 6, 311 "stack": false, 312 "steppedLine": false, 313 "targets": [ 314 { 315 "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})", 316 "format": "time_series", 317 "intervalFactor": 2, 318 "legendFormat": "{{pod}}", 319 "legendLink": null, 320 "step": 10 321 } 322 ], 323 "thresholds": [ ], 324 "timeFrom": null, 325 "timeShift": null, 326 "title": "Memory (go heap inuse)", 327 "tooltip": { 328 "sort": 2 329 }, 330 "type": "graph", 331 "xaxis": { 332 "buckets": null, 333 "mode": "time", 334 "name": null, 335 "show": true, 336 "values": [ ] 337 }, 338 "yaxes": [ 339 { 340 "format": "bytes", 341 "label": null, 342 "logBase": 1, 343 "max": null, 344 "min": 0, 345 "show": true 346 }, 347 { 348 "format": "short", 349 "label": null, 350 "logBase": 1, 351 "max": null, 352 "min": null, 353 "show": false 354 } 355 ] 356 }, 357 { 358 "aliasColors": { }, 359 "bars": false, 360 "dashLength": 10, 361 "dashes": false, 362 "datasource": "$datasource", 363 "fill": 10, 364 "gridPos": { }, 365 "id": 5, 366 "legend": { 367 "avg": false, 368 "current": false, 369 "max": false, 370 "min": false, 371 "show": true, 372 "total": false, 373 "values": false 374 }, 375 "lines": true, 376 "linewidth": 0, 377 "links": [ ], 378 "nullPointMode": "null as zero", 379 "percentage": false, 380 "pointradius": 5, 381 "points": false, 382 "renderer": "flot", 383 "seriesOverrides": [ ], 384 "spaceLength": 10, 385 "span": 6, 386 "stack": true, 387 "steppedLine": false, 388 "targets": [ 389 { 390 "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", 391 "format": "time_series", 392 "intervalFactor": 2, 393 "legendFormat": "{{pod}} - {{device}}", 394 "legendLink": null, 395 "step": 10 396 } 397 ], 398 "thresholds": [ ], 399 "timeFrom": null, 400 "timeShift": null, 401 "title": "Disk Writes", 402 "tooltip": { 403 "shared": true, 404 "sort": 2, 405 "value_type": "individual" 406 }, 407 "type": "graph", 408 "xaxis": { 409 "buckets": null, 410 "mode": "time", 411 "name": null, 412 "show": true, 413 "values": [ ] 414 }, 415 "yaxes": [ 416 { 417 "format": "Bps", 418 "label": null, 419 "logBase": 1, 420 "max": null, 421 "min": 0, 422 "show": true 423 }, 424 { 425 "format": "short", 426 "label": null, 427 "logBase": 1, 428 "max": null, 429 "min": null, 430 "show": false 431 } 432 ] 433 }, 434 { 435 "aliasColors": { }, 436 "bars": false, 437 "dashLength": 10, 438 "dashes": false, 439 "datasource": "$datasource", 440 "fill": 10, 441 "gridPos": { }, 442 "id": 6, 443 "legend": { 444 "avg": false, 445 "current": false, 446 "max": false, 447 "min": false, 448 "show": true, 449 "total": false, 450 "values": false 451 }, 452 "lines": true, 453 "linewidth": 0, 454 "links": [ ], 455 "nullPointMode": "null as zero", 456 "percentage": false, 457 "pointradius": 5, 458 "points": false, 459 "renderer": "flot", 460 "seriesOverrides": [ ], 461 "spaceLength": 10, 462 "span": 6, 463 "stack": true, 464 "steppedLine": false, 465 "targets": [ 466 { 467 "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", 468 "format": "time_series", 469 "intervalFactor": 2, 470 "legendFormat": "{{pod}} - {{device}}", 471 "legendLink": null, 472 "step": 10 473 } 474 ], 475 "thresholds": [ ], 476 "timeFrom": null, 477 "timeShift": null, 478 "title": "Disk Reads", 479 "tooltip": { 480 "shared": true, 481 "sort": 2, 482 "value_type": "individual" 483 }, 484 "type": "graph", 485 "xaxis": { 486 "buckets": null, 487 "mode": "time", 488 "name": null, 489 "show": true, 490 "values": [ ] 491 }, 492 "yaxes": [ 493 { 494 "format": "Bps", 495 "label": null, 496 "logBase": 1, 497 "max": null, 498 "min": 0, 499 "show": true 500 }, 501 { 502 "format": "short", 503 "label": null, 504 "logBase": 1, 505 "max": null, 506 "min": null, 507 "show": false 508 } 509 ] 510 }, 511 { 512 "aliasColors": { }, 513 "bars": false, 514 "dashLength": 10, 515 "dashes": false, 516 "datasource": "$datasource", 517 "fill": 1, 518 "gridPos": { }, 519 "id": 7, 520 "legend": { 521 "avg": false, 522 "current": false, 523 "max": false, 524 "min": false, 525 "show": true, 526 "total": false, 527 "values": false 528 }, 529 "lines": true, 530 "linewidth": 1, 531 "links": [ ], 532 "nullPointMode": "null as zero", 533 "percentage": false, 534 "pointradius": 5, 535 "points": false, 536 "renderer": "flot", 537 "seriesOverrides": [ ], 538 "spaceLength": 10, 539 "span": 6, 540 "stack": false, 541 "steppedLine": false, 542 "targets": [ 543 { 544 "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"(loki|enterprise-logs)-write.*\"})", 545 "format": "time_series", 546 "intervalFactor": 2, 547 "legendFormat": "{{persistentvolumeclaim}}", 548 "legendLink": null, 549 "step": 10 550 } 551 ], 552 "thresholds": [ ], 553 "timeFrom": null, 554 "timeShift": null, 555 "title": "Disk Space Utilization", 556 "tooltip": { 557 "shared": true, 558 "sort": 2, 559 "value_type": "individual" 560 }, 561 "type": "graph", 562 "xaxis": { 563 "buckets": null, 564 "mode": "time", 565 "name": null, 566 "show": true, 567 "values": [ ] 568 }, 569 "yaxes": [ 570 { 571 "format": "percentunit", 572 "label": null, 573 "logBase": 1, 574 "max": null, 575 "min": 0, 576 "show": true 577 }, 578 { 579 "format": "short", 580 "label": null, 581 "logBase": 1, 582 "max": null, 583 "min": null, 584 "show": false 585 } 586 ] 587 } 588 ], 589 "repeat": null, 590 "repeatIteration": null, 591 "repeatRowId": null, 592 "showTitle": true, 593 "title": "Write path", 594 "titleSize": "h6", 595 "type": "row" 596 } 597 ], 598 "schemaVersion": 14, 599 "style": "dark", 600 "tags": [ 601 "loki" 602 ], 603 "templating": { 604 "list": [ 605 { 606 "current": { 607 "text": "default", 608 "value": "default" 609 }, 610 "hide": 0, 611 "label": "Data Source", 612 "name": "datasource", 613 "options": [ ], 614 "query": "prometheus", 615 "refresh": 1, 616 "regex": "", 617 "type": "datasource" 618 }, 619 { 620 "allValue": null, 621 "current": { 622 "text": "prod", 623 "value": "prod" 624 }, 625 "datasource": "$datasource", 626 "hide": 0, 627 "includeAll": false, 628 "label": "cluster", 629 "multi": false, 630 "name": "cluster", 631 "options": [ ], 632 "query": "label_values(loki_build_info, cluster)", 633 "refresh": 1, 634 "regex": "", 635 "sort": 2, 636 "tagValuesQuery": "", 637 "tags": [ ], 638 "tagsQuery": "", 639 "type": "query", 640 "useTags": false 641 }, 642 { 643 "allValue": null, 644 "current": { 645 "text": "prod", 646 "value": "prod" 647 }, 648 "datasource": "$datasource", 649 "hide": 0, 650 "includeAll": false, 651 "label": "namespace", 652 "multi": false, 653 "name": "namespace", 654 "options": [ ], 655 "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", 656 "refresh": 1, 657 "regex": "", 658 "sort": 2, 659 "tagValuesQuery": "", 660 "tags": [ ], 661 "tagsQuery": "", 662 "type": "query", 663 "useTags": false 664 } 665 ] 666 }, 667 "time": { 668 "from": "now-1h", 669 "to": "now" 670 }, 671 "timepicker": { 672 "refresh_intervals": [ 673 "5s", 674 "10s", 675 "30s", 676 "1m", 677 "5m", 678 "15m", 679 "30m", 680 "1h", 681 "2h", 682 "1d" 683 ], 684 "time_options": [ 685 "5m", 686 "15m", 687 "1h", 688 "6h", 689 "12h", 690 "24h", 691 "2d", 692 "7d", 693 "30d" 694 ] 695 }, 696 "timezone": "utc", 697 "title": "Loki / Writes Resources", 698 "uid": "writes-resources", 699 "version": 0 700 }