github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/production/loki-mixin-compiled-ssd/dashboards/loki-operational.json (about) 1 { 2 "annotations": { 3 "list": [ ] 4 }, 5 "editable": true, 6 "gnetId": null, 7 "graphTooltip": 0, 8 "hideControls": false, 9 "id": 68, 10 "iteration": 1588704280892, 11 "links": [ 12 { 13 "asDropdown": true, 14 "icon": "external link", 15 "includeVars": true, 16 "keepTime": true, 17 "tags": [ 18 "loki" 19 ], 20 "targetBlank": false, 21 "title": "Loki Dashboards", 22 "type": "dashboards" 23 } 24 ], 25 "panels": [ 26 { 27 "collapsed": false, 28 "datasource": null, 29 "gridPos": { 30 "h": 1, 31 "w": 24, 32 "x": 0, 33 "y": 0 34 }, 35 "id": 17, 36 "panels": [ ], 37 "targets": [ ], 38 "title": "Main", 39 "type": "row" 40 }, 41 { 42 "aliasColors": { 43 "5xx": "red" 44 }, 45 "bars": false, 46 "dashLength": 10, 47 "dashes": false, 48 "datasource": "$datasource", 49 "fieldConfig": { 50 "defaults": { 51 "custom": { } 52 }, 53 "overrides": [ ] 54 }, 55 "fill": 1, 56 "fillGradient": 0, 57 "gridPos": { 58 "h": 5, 59 "w": 4, 60 "x": 0, 61 "y": 1 62 }, 63 "hiddenSeries": false, 64 "id": 6, 65 "legend": { 66 "avg": false, 67 "current": false, 68 "max": false, 69 "min": false, 70 "show": true, 71 "total": false, 72 "values": false 73 }, 74 "lines": true, 75 "linewidth": 1, 76 "nullPointMode": "null", 77 "options": { 78 "dataLinks": [ ] 79 }, 80 "panels": [ ], 81 "percentage": false, 82 "pointradius": 2, 83 "points": false, 84 "renderer": "flot", 85 "seriesOverrides": [ ], 86 "spaceLength": 10, 87 "stack": false, 88 "steppedLine": false, 89 "targets": [ 90 { 91 "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=~\"($namespace)/cortex-gw\", route=~\"api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}[5m]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)", 92 "legendFormat": "{{status}}", 93 "refId": "A" 94 } 95 ], 96 "thresholds": [ ], 97 "timeFrom": null, 98 "timeRegions": [ ], 99 "timeShift": null, 100 "title": "Queries/Second", 101 "tooltip": { 102 "shared": true, 103 "sort": 0, 104 "value_type": "individual" 105 }, 106 "type": "graph", 107 "xaxis": { 108 "buckets": null, 109 "mode": "time", 110 "name": null, 111 "show": true, 112 "values": [ ] 113 }, 114 "yaxes": [ 115 { 116 "format": "short", 117 "label": null, 118 "logBase": 10, 119 "max": null, 120 "min": null, 121 "show": true 122 }, 123 { 124 "format": "short", 125 "label": null, 126 "logBase": 1, 127 "max": null, 128 "min": null, 129 "show": true 130 } 131 ], 132 "yaxis": { 133 "align": false, 134 "alignLevel": null 135 } 136 }, 137 { 138 "aliasColors": { 139 "5xx": "red" 140 }, 141 "bars": false, 142 "dashLength": 10, 143 "dashes": false, 144 "datasource": "$datasource", 145 "fieldConfig": { 146 "defaults": { 147 "custom": { } 148 }, 149 "overrides": [ ] 150 }, 151 "fill": 1, 152 "fillGradient": 0, 153 "gridPos": { 154 "h": 5, 155 "w": 4, 156 "x": 4, 157 "y": 1 158 }, 159 "hiddenSeries": false, 160 "id": 7, 161 "legend": { 162 "avg": false, 163 "current": false, 164 "max": false, 165 "min": false, 166 "show": true, 167 "total": false, 168 "values": false 169 }, 170 "lines": true, 171 "linewidth": 1, 172 "nullPointMode": "null", 173 "options": { 174 "dataLinks": [ ] 175 }, 176 "panels": [ ], 177 "percentage": false, 178 "pointradius": 2, 179 "points": false, 180 "renderer": "flot", 181 "seriesOverrides": [ ], 182 "spaceLength": 10, 183 "stack": false, 184 "steppedLine": false, 185 "targets": [ 186 { 187 "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=~\"($namespace)/cortex-gw\", route=~\"api_prom_push|loki_api_v1_push\"}[5m]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", 188 "legendFormat": "{{status}}", 189 "refId": "A" 190 } 191 ], 192 "thresholds": [ ], 193 "timeFrom": null, 194 "timeRegions": [ ], 195 "timeShift": null, 196 "title": "Pushes/Second", 197 "tooltip": { 198 "shared": true, 199 "sort": 0, 200 "value_type": "individual" 201 }, 202 "type": "graph", 203 "xaxis": { 204 "buckets": null, 205 "mode": "time", 206 "name": null, 207 "show": true, 208 "values": [ ] 209 }, 210 "yaxes": [ 211 { 212 "format": "short", 213 "label": null, 214 "logBase": 10, 215 "max": null, 216 "min": null, 217 "show": true 218 }, 219 { 220 "format": "short", 221 "label": null, 222 "logBase": 1, 223 "max": null, 224 "min": null, 225 "show": true 226 } 227 ], 228 "yaxis": { 229 "align": false, 230 "alignLevel": null 231 } 232 }, 233 { 234 "aliasColors": { }, 235 "bars": false, 236 "dashLength": 10, 237 "dashes": false, 238 "datasource": "$datasource", 239 "fieldConfig": { 240 "defaults": { 241 "custom": { } 242 }, 243 "overrides": [ ] 244 }, 245 "fill": 1, 246 "fillGradient": 0, 247 "gridPos": { 248 "h": 5, 249 "w": 4, 250 "x": 8, 251 "y": 1 252 }, 253 "hiddenSeries": false, 254 "id": 11, 255 "legend": { 256 "avg": false, 257 "current": false, 258 "hideEmpty": false, 259 "hideZero": false, 260 "max": false, 261 "min": false, 262 "show": false, 263 "total": false, 264 "values": false 265 }, 266 "lines": true, 267 "linewidth": 1, 268 "nullPointMode": "null", 269 "options": { 270 "dataLinks": [ ] 271 }, 272 "panels": [ ], 273 "percentage": false, 274 "pointradius": 2, 275 "points": false, 276 "renderer": "flot", 277 "seriesOverrides": [ ], 278 "spaceLength": 10, 279 "stack": false, 280 "steppedLine": false, 281 "targets": [ 282 { 283 "expr": "topk(5, sum by (name,level) (rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\"}[$__interval])) - \nsum by (name,level) (rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\"}[$__interval] offset 1h)))", 284 "legendFormat": "{{name}}-{{level}}", 285 "refId": "A" 286 } 287 ], 288 "thresholds": [ ], 289 "timeFrom": null, 290 "timeRegions": [ ], 291 "timeShift": null, 292 "title": "Bad Words", 293 "tooltip": { 294 "shared": true, 295 "sort": 2, 296 "value_type": "individual" 297 }, 298 "type": "graph", 299 "xaxis": { 300 "buckets": null, 301 "mode": "time", 302 "name": null, 303 "show": true, 304 "values": [ ] 305 }, 306 "yaxes": [ 307 { 308 "format": "short", 309 "label": null, 310 "logBase": 1, 311 "max": null, 312 "min": null, 313 "show": true 314 }, 315 { 316 "format": "short", 317 "label": null, 318 "logBase": 1, 319 "max": null, 320 "min": null, 321 "show": true 322 } 323 ], 324 "yaxis": { 325 "align": false, 326 "alignLevel": null 327 } 328 }, 329 { 330 "aliasColors": { }, 331 "bars": false, 332 "dashLength": 10, 333 "dashes": false, 334 "datasource": "$datasource", 335 "fieldConfig": { 336 "defaults": { 337 "custom": { } 338 }, 339 "overrides": [ ] 340 }, 341 "fill": 1, 342 "fillGradient": 0, 343 "gridPos": { 344 "h": 5, 345 "w": 4, 346 "x": 12, 347 "y": 1 348 }, 349 "hiddenSeries": false, 350 "id": 2, 351 "interval": "", 352 "legend": { 353 "avg": false, 354 "current": false, 355 "max": false, 356 "min": false, 357 "show": false, 358 "total": false, 359 "values": false 360 }, 361 "lines": true, 362 "linewidth": 1, 363 "nullPointMode": "null", 364 "options": { 365 "dataLinks": [ ] 366 }, 367 "panels": [ ], 368 "percentage": false, 369 "pointradius": 2, 370 "points": false, 371 "renderer": "flot", 372 "seriesOverrides": [ ], 373 "spaceLength": 10, 374 "stack": false, 375 "steppedLine": false, 376 "targets": [ 377 { 378 "expr": "topk(10, sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (tenant))", 379 "legendFormat": "{{tenant}}", 380 "refId": "A" 381 } 382 ], 383 "thresholds": [ ], 384 "timeFrom": null, 385 "timeRegions": [ ], 386 "timeShift": null, 387 "title": "Lines Per Tenant (top 10)", 388 "tooltip": { 389 "shared": false, 390 "sort": 0, 391 "value_type": "individual" 392 }, 393 "type": "graph", 394 "xaxis": { 395 "buckets": null, 396 "mode": "time", 397 "name": null, 398 "show": true, 399 "values": [ ] 400 }, 401 "yaxes": [ 402 { 403 "format": "short", 404 "label": null, 405 "logBase": 1, 406 "max": null, 407 "min": null, 408 "show": true 409 }, 410 { 411 "format": "short", 412 "label": null, 413 "logBase": 1, 414 "max": null, 415 "min": null, 416 "show": true 417 } 418 ], 419 "yaxis": { 420 "align": false, 421 "alignLevel": null 422 } 423 }, 424 { 425 "aliasColors": { }, 426 "bars": false, 427 "dashLength": 10, 428 "dashes": false, 429 "datasource": "$datasource", 430 "fieldConfig": { 431 "defaults": { 432 "custom": { } 433 }, 434 "overrides": [ ] 435 }, 436 "fill": 1, 437 "fillGradient": 0, 438 "gridPos": { 439 "h": 5, 440 "w": 4, 441 "x": 16, 442 "y": 1 443 }, 444 "hiddenSeries": false, 445 "id": 4, 446 "legend": { 447 "avg": false, 448 "current": false, 449 "hideEmpty": true, 450 "hideZero": true, 451 "max": false, 452 "min": false, 453 "show": false, 454 "total": false, 455 "values": false 456 }, 457 "lines": true, 458 "linewidth": 1, 459 "nullPointMode": "null", 460 "options": { 461 "dataLinks": [ ] 462 }, 463 "panels": [ ], 464 "percentage": false, 465 "pointradius": 2, 466 "points": false, 467 "renderer": "flot", 468 "seriesOverrides": [ ], 469 "spaceLength": 10, 470 "stack": false, 471 "steppedLine": false, 472 "targets": [ 473 { 474 "expr": "topk(10, sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (tenant)) / 1024 / 1024", 475 "legendFormat": "{{tenant}}", 476 "refId": "A" 477 } 478 ], 479 "thresholds": [ ], 480 "timeFrom": null, 481 "timeRegions": [ ], 482 "timeShift": null, 483 "title": "MBs Per Tenant (Top 10)", 484 "tooltip": { 485 "shared": false, 486 "sort": 0, 487 "value_type": "individual" 488 }, 489 "type": "graph", 490 "xaxis": { 491 "buckets": null, 492 "mode": "time", 493 "name": null, 494 "show": true, 495 "values": [ ] 496 }, 497 "yaxes": [ 498 { 499 "format": "short", 500 "label": null, 501 "logBase": 1, 502 "max": null, 503 "min": null, 504 "show": true 505 }, 506 { 507 "format": "short", 508 "label": null, 509 "logBase": 1, 510 "max": null, 511 "min": null, 512 "show": true 513 } 514 ], 515 "yaxis": { 516 "align": false, 517 "alignLevel": null 518 } 519 }, 520 { 521 "aliasColors": { }, 522 "bars": false, 523 "dashLength": 10, 524 "dashes": false, 525 "datasource": "$datasource", 526 "fieldConfig": { 527 "defaults": { 528 "custom": { } 529 }, 530 "overrides": [ ] 531 }, 532 "fill": 1, 533 "fillGradient": 0, 534 "gridPos": { 535 "h": 5, 536 "w": 4, 537 "x": 20, 538 "y": 1 539 }, 540 "hiddenSeries": false, 541 "id": 24, 542 "legend": { 543 "avg": false, 544 "current": false, 545 "max": false, 546 "min": false, 547 "show": false, 548 "total": false, 549 "values": false 550 }, 551 "lines": true, 552 "linewidth": 1, 553 "nullPointMode": "null", 554 "options": { 555 "dataLinks": [ ] 556 }, 557 "panels": [ ], 558 "percentage": false, 559 "pointradius": 2, 560 "points": false, 561 "renderer": "flot", 562 "seriesOverrides": [ ], 563 "spaceLength": 10, 564 "stack": false, 565 "steppedLine": false, 566 "targets": [ 567 { 568 "expr": "increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\"}[10m]) > 0", 569 "hide": false, 570 "interval": "", 571 "legendFormat": "{{container}}-{{pod}}", 572 "refId": "B" 573 } 574 ], 575 "thresholds": [ ], 576 "timeFrom": null, 577 "timeRegions": [ ], 578 "timeShift": null, 579 "title": "Container Restarts", 580 "tooltip": { 581 "shared": true, 582 "sort": 2, 583 "value_type": "individual" 584 }, 585 "type": "graph", 586 "xaxis": { 587 "buckets": null, 588 "mode": "time", 589 "name": null, 590 "show": true, 591 "values": [ ] 592 }, 593 "yaxes": [ 594 { 595 "format": "short", 596 "label": null, 597 "logBase": 1, 598 "max": null, 599 "min": null, 600 "show": true 601 }, 602 { 603 "format": "short", 604 "label": null, 605 "logBase": 1, 606 "max": null, 607 "min": null, 608 "show": true 609 } 610 ], 611 "yaxis": { 612 "align": false, 613 "alignLevel": null 614 } 615 }, 616 { 617 "aliasColors": { }, 618 "bars": false, 619 "dashLength": 10, 620 "dashes": false, 621 "datasource": "$datasource", 622 "fieldConfig": { 623 "defaults": { 624 "custom": { } 625 }, 626 "overrides": [ ] 627 }, 628 "fill": 1, 629 "fillGradient": 0, 630 "gridPos": { 631 "h": 10, 632 "w": 12, 633 "x": 0, 634 "y": 6 635 }, 636 "hiddenSeries": false, 637 "id": 9, 638 "legend": { 639 "alignAsTable": true, 640 "avg": false, 641 "current": false, 642 "max": false, 643 "min": false, 644 "rightSide": true, 645 "show": true, 646 "total": false, 647 "values": false 648 }, 649 "lines": true, 650 "linewidth": 1, 651 "nullPointMode": "null", 652 "options": { 653 "dataLinks": [ ] 654 }, 655 "panels": [ ], 656 "percentage": false, 657 "pointradius": 2, 658 "points": false, 659 "renderer": "flot", 660 "seriesOverrides": [ ], 661 "spaceLength": 10, 662 "stack": false, 663 "steppedLine": false, 664 "targets": [ 665 { 666 "expr": "histogram_quantile(0.99, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/cortex-gw\", route=~\"api_prom_push|loki_api_v1_push\", cluster=~\"$cluster\"})) * 1e3", 667 "legendFormat": ".99", 668 "refId": "A" 669 }, 670 { 671 "expr": "histogram_quantile(0.75, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/cortex-gw\", route=~\"api_prom_push|loki_api_v1_push\", cluster=~\"$cluster\"})) * 1e3", 672 "legendFormat": ".9", 673 "refId": "B" 674 }, 675 { 676 "expr": "histogram_quantile(0.5, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/cortex-gw\", route=~\"api_prom_push|loki_api_v1_push\", cluster=~\"$cluster\"})) * 1e3", 677 "legendFormat": ".5", 678 "refId": "C" 679 } 680 ], 681 "thresholds": [ ], 682 "timeFrom": null, 683 "timeRegions": [ ], 684 "timeShift": null, 685 "title": "Push Latency", 686 "tooltip": { 687 "shared": true, 688 "sort": 0, 689 "value_type": "individual" 690 }, 691 "type": "graph", 692 "xaxis": { 693 "buckets": null, 694 "mode": "time", 695 "name": null, 696 "show": true, 697 "values": [ ] 698 }, 699 "yaxes": [ 700 { 701 "format": "short", 702 "label": null, 703 "logBase": 1, 704 "max": null, 705 "min": null, 706 "show": true 707 }, 708 { 709 "format": "short", 710 "label": null, 711 "logBase": 1, 712 "max": null, 713 "min": null, 714 "show": true 715 } 716 ], 717 "yaxis": { 718 "align": false, 719 "alignLevel": null 720 } 721 }, 722 { 723 "aliasColors": { }, 724 "bars": false, 725 "dashLength": 10, 726 "dashes": false, 727 "datasource": "$datasource", 728 "fieldConfig": { 729 "defaults": { 730 "custom": { } 731 }, 732 "overrides": [ ] 733 }, 734 "fill": 1, 735 "fillGradient": 0, 736 "gridPos": { 737 "h": 5, 738 "w": 6, 739 "x": 12, 740 "y": 6 741 }, 742 "hiddenSeries": false, 743 "id": 12, 744 "legend": { 745 "alignAsTable": true, 746 "avg": false, 747 "current": false, 748 "max": false, 749 "min": false, 750 "rightSide": true, 751 "show": false, 752 "total": false, 753 "values": false 754 }, 755 "lines": true, 756 "linewidth": 1, 757 "nullPointMode": "null", 758 "options": { 759 "dataLinks": [ ] 760 }, 761 "panels": [ ], 762 "percentage": false, 763 "pointradius": 2, 764 "points": false, 765 "renderer": "flot", 766 "seriesOverrides": [ ], 767 "spaceLength": 10, 768 "stack": false, 769 "steppedLine": false, 770 "targets": [ 771 { 772 "expr": "histogram_quantile(0.99, sum by (le) (job:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", cluster=~\"$cluster\"})) * 1e3", 773 "legendFormat": ".99", 774 "refId": "A" 775 }, 776 { 777 "expr": "histogram_quantile(0.9, sum by (le) (job:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", cluster=~\"$cluster\"})) * 1e3", 778 "legendFormat": ".9", 779 "refId": "B" 780 }, 781 { 782 "expr": "histogram_quantile(0.5, sum by (le) (job:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", cluster=~\"$cluster\"})) * 1e3", 783 "legendFormat": ".5", 784 "refId": "C" 785 } 786 ], 787 "thresholds": [ ], 788 "timeFrom": null, 789 "timeRegions": [ ], 790 "timeShift": null, 791 "title": "Distributor Latency", 792 "tooltip": { 793 "shared": true, 794 "sort": 0, 795 "value_type": "individual" 796 }, 797 "type": "graph", 798 "xaxis": { 799 "buckets": null, 800 "mode": "time", 801 "name": null, 802 "show": true, 803 "values": [ ] 804 }, 805 "yaxes": [ 806 { 807 "format": "short", 808 "label": null, 809 "logBase": 1, 810 "max": null, 811 "min": null, 812 "show": true 813 }, 814 { 815 "format": "short", 816 "label": null, 817 "logBase": 1, 818 "max": null, 819 "min": null, 820 "show": true 821 } 822 ], 823 "yaxis": { 824 "align": false, 825 "alignLevel": null 826 } 827 }, 828 { 829 "aliasColors": { }, 830 "bars": false, 831 "dashLength": 10, 832 "dashes": false, 833 "datasource": "$datasource", 834 "fieldConfig": { 835 "defaults": { 836 "custom": { } 837 }, 838 "overrides": [ ] 839 }, 840 "fill": 0, 841 "fillGradient": 0, 842 "gridPos": { 843 "h": 5, 844 "w": 6, 845 "x": 18, 846 "y": 6 847 }, 848 "hiddenSeries": false, 849 "id": 71, 850 "legend": { 851 "alignAsTable": true, 852 "avg": false, 853 "current": false, 854 "max": false, 855 "min": false, 856 "rightSide": true, 857 "show": false, 858 "total": false, 859 "values": false 860 }, 861 "lines": true, 862 "linewidth": 1, 863 "nullPointMode": "null", 864 "options": { 865 "dataLinks": [ ] 866 }, 867 "panels": [ ], 868 "percentage": false, 869 "pointradius": 2, 870 "points": false, 871 "renderer": "flot", 872 "seriesOverrides": [ ], 873 "spaceLength": 10, 874 "stack": false, 875 "steppedLine": false, 876 "targets": [ 877 { 878 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[5m])) by (route)", 879 "interval": "", 880 "legendFormat": "{{route}}", 881 "refId": "A" 882 } 883 ], 884 "thresholds": [ ], 885 "timeFrom": null, 886 "timeRegions": [ ], 887 "timeShift": null, 888 "title": "Distributor Success Rate", 889 "tooltip": { 890 "shared": true, 891 "sort": 0, 892 "value_type": "individual" 893 }, 894 "type": "graph", 895 "xaxis": { 896 "buckets": null, 897 "mode": "time", 898 "name": null, 899 "show": true, 900 "values": [ ] 901 }, 902 "yaxes": [ 903 { 904 "decimals": null, 905 "format": "percentunit", 906 "label": "", 907 "logBase": 1, 908 "max": "1", 909 "min": null, 910 "show": true 911 }, 912 { 913 "format": "short", 914 "label": null, 915 "logBase": 1, 916 "max": null, 917 "min": null, 918 "show": true 919 } 920 ], 921 "yaxis": { 922 "align": false, 923 "alignLevel": null 924 } 925 }, 926 { 927 "aliasColors": { }, 928 "bars": false, 929 "dashLength": 10, 930 "dashes": false, 931 "datasource": "$datasource", 932 "fieldConfig": { 933 "defaults": { 934 "custom": { } 935 }, 936 "overrides": [ ] 937 }, 938 "fill": 1, 939 "fillGradient": 0, 940 "gridPos": { 941 "h": 5, 942 "w": 6, 943 "x": 12, 944 "y": 11 945 }, 946 "hiddenSeries": false, 947 "id": 13, 948 "legend": { 949 "alignAsTable": true, 950 "avg": false, 951 "current": false, 952 "max": false, 953 "min": false, 954 "rightSide": true, 955 "show": false, 956 "total": false, 957 "values": false 958 }, 959 "lines": true, 960 "linewidth": 1, 961 "nullPointMode": "null", 962 "options": { 963 "dataLinks": [ ] 964 }, 965 "panels": [ ], 966 "percentage": false, 967 "pointradius": 2, 968 "points": false, 969 "renderer": "flot", 970 "seriesOverrides": [ ], 971 "spaceLength": 10, 972 "stack": false, 973 "steppedLine": false, 974 "targets": [ 975 { 976 "expr": "histogram_quantile(0.99, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3", 977 "legendFormat": ".99", 978 "refId": "A" 979 }, 980 { 981 "expr": "histogram_quantile(0.9, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3", 982 "hide": false, 983 "legendFormat": ".9", 984 "refId": "B" 985 }, 986 { 987 "expr": "histogram_quantile(0.5, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3", 988 "hide": false, 989 "legendFormat": ".5", 990 "refId": "C" 991 } 992 ], 993 "thresholds": [ ], 994 "timeFrom": null, 995 "timeRegions": [ ], 996 "timeShift": null, 997 "title": "Ingester Latency Write", 998 "tooltip": { 999 "shared": true, 1000 "sort": 0, 1001 "value_type": "individual" 1002 }, 1003 "type": "graph", 1004 "xaxis": { 1005 "buckets": null, 1006 "mode": "time", 1007 "name": null, 1008 "show": true, 1009 "values": [ ] 1010 }, 1011 "yaxes": [ 1012 { 1013 "format": "short", 1014 "label": null, 1015 "logBase": 1, 1016 "max": null, 1017 "min": null, 1018 "show": true 1019 }, 1020 { 1021 "format": "short", 1022 "label": null, 1023 "logBase": 1, 1024 "max": null, 1025 "min": null, 1026 "show": true 1027 } 1028 ], 1029 "yaxis": { 1030 "align": false, 1031 "alignLevel": null 1032 } 1033 }, 1034 { 1035 "aliasColors": { }, 1036 "bars": false, 1037 "dashLength": 10, 1038 "dashes": false, 1039 "datasource": "$datasource", 1040 "fieldConfig": { 1041 "defaults": { 1042 "custom": { } 1043 }, 1044 "overrides": [ ] 1045 }, 1046 "fill": 0, 1047 "fillGradient": 0, 1048 "gridPos": { 1049 "h": 5, 1050 "w": 6, 1051 "x": 18, 1052 "y": 11 1053 }, 1054 "hiddenSeries": false, 1055 "id": 72, 1056 "legend": { 1057 "alignAsTable": true, 1058 "avg": false, 1059 "current": false, 1060 "max": false, 1061 "min": false, 1062 "rightSide": true, 1063 "show": false, 1064 "total": false, 1065 "values": false 1066 }, 1067 "lines": true, 1068 "linewidth": 1, 1069 "nullPointMode": "null", 1070 "options": { 1071 "dataLinks": [ ] 1072 }, 1073 "panels": [ ], 1074 "percentage": false, 1075 "pointradius": 2, 1076 "points": false, 1077 "renderer": "flot", 1078 "seriesOverrides": [ ], 1079 "spaceLength": 10, 1080 "stack": false, 1081 "steppedLine": false, 1082 "targets": [ 1083 { 1084 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\", route=\"/logproto.Pusher/Push\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\"}[5m])) by (route)", 1085 "interval": "", 1086 "legendFormat": "{{route}}", 1087 "refId": "A" 1088 } 1089 ], 1090 "thresholds": [ ], 1091 "timeFrom": null, 1092 "timeRegions": [ ], 1093 "timeShift": null, 1094 "title": "Ingester Success Rate Write", 1095 "tooltip": { 1096 "shared": true, 1097 "sort": 0, 1098 "value_type": "individual" 1099 }, 1100 "type": "graph", 1101 "xaxis": { 1102 "buckets": null, 1103 "mode": "time", 1104 "name": null, 1105 "show": true, 1106 "values": [ ] 1107 }, 1108 "yaxes": [ 1109 { 1110 "decimals": null, 1111 "format": "percentunit", 1112 "label": "", 1113 "logBase": 1, 1114 "max": "1", 1115 "min": null, 1116 "show": true 1117 }, 1118 { 1119 "format": "short", 1120 "label": null, 1121 "logBase": 1, 1122 "max": null, 1123 "min": null, 1124 "show": true 1125 } 1126 ], 1127 "yaxis": { 1128 "align": false, 1129 "alignLevel": null 1130 } 1131 }, 1132 { 1133 "aliasColors": { }, 1134 "bars": false, 1135 "dashLength": 10, 1136 "dashes": false, 1137 "datasource": "$datasource", 1138 "fieldConfig": { 1139 "defaults": { 1140 "custom": { } 1141 }, 1142 "overrides": [ ] 1143 }, 1144 "fill": 1, 1145 "fillGradient": 0, 1146 "gridPos": { 1147 "h": 10, 1148 "w": 12, 1149 "x": 0, 1150 "y": 16 1151 }, 1152 "hiddenSeries": false, 1153 "id": 10, 1154 "legend": { 1155 "alignAsTable": true, 1156 "avg": false, 1157 "current": false, 1158 "hideEmpty": true, 1159 "hideZero": false, 1160 "max": false, 1161 "min": false, 1162 "rightSide": true, 1163 "show": true, 1164 "total": false, 1165 "values": false 1166 }, 1167 "lines": true, 1168 "linewidth": 1, 1169 "nullPointMode": "null", 1170 "options": { 1171 "dataLinks": [ ] 1172 }, 1173 "panels": [ ], 1174 "percentage": false, 1175 "pointradius": 2, 1176 "points": false, 1177 "renderer": "flot", 1178 "seriesOverrides": [ ], 1179 "spaceLength": 10, 1180 "stack": false, 1181 "steppedLine": false, 1182 "targets": [ 1183 { 1184 "expr": "histogram_quantile(0.99, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", 1185 "legendFormat": "{{route}}-.99", 1186 "refId": "A" 1187 }, 1188 { 1189 "expr": "histogram_quantile(0.9, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", 1190 "legendFormat": "{{route}}-.9", 1191 "refId": "B" 1192 }, 1193 { 1194 "expr": "histogram_quantile(0.5, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", 1195 "legendFormat": "{{route}}-.5", 1196 "refId": "C" 1197 } 1198 ], 1199 "thresholds": [ ], 1200 "timeFrom": null, 1201 "timeRegions": [ ], 1202 "timeShift": null, 1203 "title": "Query Latency", 1204 "tooltip": { 1205 "shared": true, 1206 "sort": 0, 1207 "value_type": "individual" 1208 }, 1209 "type": "graph", 1210 "xaxis": { 1211 "buckets": null, 1212 "mode": "time", 1213 "name": null, 1214 "show": true, 1215 "values": [ ] 1216 }, 1217 "yaxes": [ 1218 { 1219 "format": "short", 1220 "label": null, 1221 "logBase": 1, 1222 "max": null, 1223 "min": null, 1224 "show": true 1225 }, 1226 { 1227 "format": "short", 1228 "label": null, 1229 "logBase": 1, 1230 "max": null, 1231 "min": null, 1232 "show": true 1233 } 1234 ], 1235 "yaxis": { 1236 "align": false, 1237 "alignLevel": null 1238 } 1239 }, 1240 { 1241 "aliasColors": { }, 1242 "bars": false, 1243 "dashLength": 10, 1244 "dashes": false, 1245 "datasource": "$datasource", 1246 "fieldConfig": { 1247 "defaults": { 1248 "custom": { } 1249 }, 1250 "overrides": [ ] 1251 }, 1252 "fill": 1, 1253 "fillGradient": 0, 1254 "gridPos": { 1255 "h": 5, 1256 "w": 6, 1257 "x": 12, 1258 "y": 16 1259 }, 1260 "hiddenSeries": false, 1261 "id": 14, 1262 "legend": { 1263 "alignAsTable": true, 1264 "avg": false, 1265 "current": false, 1266 "max": false, 1267 "min": false, 1268 "rightSide": true, 1269 "show": false, 1270 "total": false, 1271 "values": false 1272 }, 1273 "lines": true, 1274 "linewidth": 1, 1275 "nullPointMode": "null", 1276 "options": { 1277 "dataLinks": [ ] 1278 }, 1279 "panels": [ ], 1280 "percentage": false, 1281 "pointradius": 2, 1282 "points": false, 1283 "renderer": "flot", 1284 "seriesOverrides": [ ], 1285 "spaceLength": 10, 1286 "stack": false, 1287 "steppedLine": false, 1288 "targets": [ 1289 { 1290 "expr": "histogram_quantile(0.99, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", 1291 "legendFormat": ".99-{{route}}", 1292 "refId": "A" 1293 }, 1294 { 1295 "expr": "histogram_quantile(0.9, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", 1296 "legendFormat": ".9-{{route}}", 1297 "refId": "B" 1298 }, 1299 { 1300 "expr": "histogram_quantile(0.5, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", 1301 "legendFormat": ".5-{{route}}", 1302 "refId": "C" 1303 } 1304 ], 1305 "thresholds": [ ], 1306 "timeFrom": null, 1307 "timeRegions": [ ], 1308 "timeShift": null, 1309 "title": "Querier Latency", 1310 "tooltip": { 1311 "shared": true, 1312 "sort": 0, 1313 "value_type": "individual" 1314 }, 1315 "type": "graph", 1316 "xaxis": { 1317 "buckets": null, 1318 "mode": "time", 1319 "name": null, 1320 "show": true, 1321 "values": [ ] 1322 }, 1323 "yaxes": [ 1324 { 1325 "format": "short", 1326 "label": null, 1327 "logBase": 1, 1328 "max": null, 1329 "min": null, 1330 "show": true 1331 }, 1332 { 1333 "format": "short", 1334 "label": null, 1335 "logBase": 1, 1336 "max": null, 1337 "min": null, 1338 "show": true 1339 } 1340 ], 1341 "yaxis": { 1342 "align": false, 1343 "alignLevel": null 1344 } 1345 }, 1346 { 1347 "aliasColors": { }, 1348 "bars": false, 1349 "dashLength": 10, 1350 "dashes": false, 1351 "datasource": "$datasource", 1352 "fieldConfig": { 1353 "defaults": { 1354 "custom": { } 1355 }, 1356 "overrides": [ ] 1357 }, 1358 "fill": 0, 1359 "fillGradient": 0, 1360 "gridPos": { 1361 "h": 5, 1362 "w": 6, 1363 "x": 18, 1364 "y": 16 1365 }, 1366 "hiddenSeries": false, 1367 "id": 73, 1368 "legend": { 1369 "alignAsTable": true, 1370 "avg": false, 1371 "current": false, 1372 "max": false, 1373 "min": false, 1374 "rightSide": true, 1375 "show": false, 1376 "total": false, 1377 "values": false 1378 }, 1379 "lines": true, 1380 "linewidth": 1, 1381 "nullPointMode": "null", 1382 "options": { 1383 "dataLinks": [ ] 1384 }, 1385 "panels": [ ], 1386 "percentage": false, 1387 "pointradius": 2, 1388 "points": false, 1389 "renderer": "flot", 1390 "seriesOverrides": [ ], 1391 "spaceLength": 10, 1392 "stack": false, 1393 "steppedLine": false, 1394 "targets": [ 1395 { 1396 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\", status_code!~\"5[0-9]{2}\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"}[5m])) by (route)", 1397 "interval": "", 1398 "legendFormat": "{{route}}", 1399 "refId": "A" 1400 } 1401 ], 1402 "thresholds": [ ], 1403 "timeFrom": null, 1404 "timeRegions": [ ], 1405 "timeShift": null, 1406 "title": "Querier Success Rate", 1407 "tooltip": { 1408 "shared": true, 1409 "sort": 0, 1410 "value_type": "individual" 1411 }, 1412 "type": "graph", 1413 "xaxis": { 1414 "buckets": null, 1415 "mode": "time", 1416 "name": null, 1417 "show": true, 1418 "values": [ ] 1419 }, 1420 "yaxes": [ 1421 { 1422 "decimals": null, 1423 "format": "percentunit", 1424 "label": "", 1425 "logBase": 1, 1426 "max": "1", 1427 "min": null, 1428 "show": true 1429 }, 1430 { 1431 "format": "short", 1432 "label": null, 1433 "logBase": 1, 1434 "max": null, 1435 "min": null, 1436 "show": true 1437 } 1438 ], 1439 "yaxis": { 1440 "align": false, 1441 "alignLevel": null 1442 } 1443 }, 1444 { 1445 "aliasColors": { }, 1446 "bars": false, 1447 "dashLength": 10, 1448 "dashes": false, 1449 "datasource": "$datasource", 1450 "description": "", 1451 "fieldConfig": { 1452 "defaults": { 1453 "custom": { } 1454 }, 1455 "overrides": [ ] 1456 }, 1457 "fill": 1, 1458 "fillGradient": 0, 1459 "gridPos": { 1460 "h": 5, 1461 "w": 6, 1462 "x": 12, 1463 "y": 21 1464 }, 1465 "hiddenSeries": false, 1466 "id": 15, 1467 "legend": { 1468 "alignAsTable": true, 1469 "avg": false, 1470 "current": false, 1471 "max": false, 1472 "min": false, 1473 "rightSide": true, 1474 "show": false, 1475 "total": false, 1476 "values": false 1477 }, 1478 "lines": true, 1479 "linewidth": 1, 1480 "nullPointMode": "null", 1481 "options": { 1482 "dataLinks": [ ] 1483 }, 1484 "panels": [ ], 1485 "percentage": false, 1486 "pointradius": 2, 1487 "points": false, 1488 "renderer": "flot", 1489 "seriesOverrides": [ ], 1490 "spaceLength": 10, 1491 "stack": false, 1492 "steppedLine": false, 1493 "targets": [ 1494 { 1495 "expr": "histogram_quantile(0.99, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3", 1496 "legendFormat": ".99-{{route}}", 1497 "refId": "A" 1498 }, 1499 { 1500 "expr": "histogram_quantile(0.9, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3", 1501 "legendFormat": ".9-{{route}}", 1502 "refId": "B" 1503 }, 1504 { 1505 "expr": "histogram_quantile(0.5, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3", 1506 "legendFormat": ".5-{{route}}", 1507 "refId": "C" 1508 } 1509 ], 1510 "thresholds": [ ], 1511 "timeFrom": null, 1512 "timeRegions": [ ], 1513 "timeShift": null, 1514 "title": "Ingester Latency Read", 1515 "tooltip": { 1516 "shared": true, 1517 "sort": 0, 1518 "value_type": "individual" 1519 }, 1520 "type": "graph", 1521 "xaxis": { 1522 "buckets": null, 1523 "mode": "time", 1524 "name": null, 1525 "show": true, 1526 "values": [ ] 1527 }, 1528 "yaxes": [ 1529 { 1530 "format": "short", 1531 "label": null, 1532 "logBase": 1, 1533 "max": null, 1534 "min": null, 1535 "show": true 1536 }, 1537 { 1538 "format": "short", 1539 "label": null, 1540 "logBase": 1, 1541 "max": null, 1542 "min": null, 1543 "show": true 1544 } 1545 ], 1546 "yaxis": { 1547 "align": false, 1548 "alignLevel": null 1549 } 1550 }, 1551 { 1552 "aliasColors": { }, 1553 "bars": false, 1554 "dashLength": 10, 1555 "dashes": false, 1556 "datasource": "$datasource", 1557 "fieldConfig": { 1558 "defaults": { 1559 "custom": { } 1560 }, 1561 "overrides": [ ] 1562 }, 1563 "fill": 0, 1564 "fillGradient": 0, 1565 "gridPos": { 1566 "h": 5, 1567 "w": 6, 1568 "x": 18, 1569 "y": 21 1570 }, 1571 "hiddenSeries": false, 1572 "id": 74, 1573 "legend": { 1574 "alignAsTable": true, 1575 "avg": false, 1576 "current": false, 1577 "max": false, 1578 "min": false, 1579 "rightSide": true, 1580 "show": false, 1581 "total": false, 1582 "values": false 1583 }, 1584 "lines": true, 1585 "linewidth": 1, 1586 "nullPointMode": "null", 1587 "options": { 1588 "dataLinks": [ ] 1589 }, 1590 "panels": [ ], 1591 "percentage": false, 1592 "pointradius": 2, 1593 "points": false, 1594 "renderer": "flot", 1595 "seriesOverrides": [ ], 1596 "spaceLength": 10, 1597 "stack": false, 1598 "steppedLine": false, 1599 "targets": [ 1600 { 1601 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[5m])) by (route)", 1602 "interval": "", 1603 "legendFormat": "{{route}}", 1604 "refId": "A" 1605 } 1606 ], 1607 "thresholds": [ ], 1608 "timeFrom": null, 1609 "timeRegions": [ ], 1610 "timeShift": null, 1611 "title": "Ingester Success Rate Read", 1612 "tooltip": { 1613 "shared": true, 1614 "sort": 0, 1615 "value_type": "individual" 1616 }, 1617 "type": "graph", 1618 "xaxis": { 1619 "buckets": null, 1620 "mode": "time", 1621 "name": null, 1622 "show": true, 1623 "values": [ ] 1624 }, 1625 "yaxes": [ 1626 { 1627 "decimals": null, 1628 "format": "percentunit", 1629 "label": "", 1630 "logBase": 1, 1631 "max": "1", 1632 "min": null, 1633 "show": true 1634 }, 1635 { 1636 "format": "short", 1637 "label": null, 1638 "logBase": 1, 1639 "max": null, 1640 "min": null, 1641 "show": true 1642 } 1643 ], 1644 "yaxis": { 1645 "align": false, 1646 "alignLevel": null 1647 } 1648 }, 1649 { 1650 "collapsed": true, 1651 "datasource": null, 1652 "gridPos": { 1653 "h": 1, 1654 "w": 24, 1655 "x": 0, 1656 "y": 26 1657 }, 1658 "id": 110, 1659 "panels": [ 1660 { 1661 "aliasColors": { }, 1662 "bars": false, 1663 "dashLength": 10, 1664 "dashes": false, 1665 "datasource": "$datasource", 1666 "fill": 1, 1667 "fillGradient": 0, 1668 "gridPos": { 1669 "h": 8, 1670 "w": 12, 1671 "x": 0, 1672 "y": 27 1673 }, 1674 "hiddenSeries": false, 1675 "id": 112, 1676 "legend": { 1677 "avg": false, 1678 "current": false, 1679 "max": false, 1680 "min": false, 1681 "show": false, 1682 "total": false, 1683 "values": false 1684 }, 1685 "lines": true, 1686 "linewidth": 1, 1687 "nullPointMode": "null", 1688 "options": { 1689 "dataLinks": [ ] 1690 }, 1691 "panels": [ ], 1692 "percentage": false, 1693 "pointradius": 2, 1694 "points": false, 1695 "renderer": "flot", 1696 "seriesOverrides": [ ], 1697 "spaceLength": 10, 1698 "stack": false, 1699 "steppedLine": false, 1700 "targets": [ 1701 { 1702 "expr": "topk(10,sum by (tenant, reason) (rate(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[1m])))", 1703 "interval": "", 1704 "legendFormat": "{{ tenant }} - {{ reason }}", 1705 "refId": "A" 1706 } 1707 ], 1708 "thresholds": [ ], 1709 "timeFrom": null, 1710 "timeRegions": [ ], 1711 "timeShift": null, 1712 "title": "Discarded Lines", 1713 "tooltip": { 1714 "shared": false, 1715 "sort": 0, 1716 "value_type": "individual" 1717 }, 1718 "type": "graph", 1719 "xaxis": { 1720 "buckets": null, 1721 "mode": "time", 1722 "name": null, 1723 "show": true, 1724 "values": [ ] 1725 }, 1726 "yaxes": [ 1727 { 1728 "format": "short", 1729 "label": null, 1730 "logBase": 1, 1731 "max": null, 1732 "min": null, 1733 "show": true 1734 }, 1735 { 1736 "format": "short", 1737 "label": null, 1738 "logBase": 1, 1739 "max": null, 1740 "min": null, 1741 "show": true 1742 } 1743 ], 1744 "yaxis": { 1745 "align": false, 1746 "alignLevel": null 1747 } 1748 }, 1749 { 1750 "columns": [ ], 1751 "datasource": "$datasource", 1752 "fontSize": "100%", 1753 "gridPos": { 1754 "h": 8, 1755 "w": 12, 1756 "x": 12, 1757 "y": 27 1758 }, 1759 "id": 113, 1760 "pageSize": null, 1761 "panels": [ ], 1762 "showHeader": true, 1763 "sort": { 1764 "col": 3, 1765 "desc": true 1766 }, 1767 "styles": [ 1768 { 1769 "alias": "Time", 1770 "align": "auto", 1771 "dateFormat": "YYYY-MM-DD HH:mm:ss", 1772 "pattern": "Time", 1773 "type": "hidden" 1774 }, 1775 { 1776 "alias": "", 1777 "align": "auto", 1778 "colorMode": null, 1779 "colors": [ 1780 "rgba(245, 54, 54, 0.9)", 1781 "rgba(237, 129, 40, 0.89)", 1782 "rgba(50, 172, 45, 0.97)" 1783 ], 1784 "dateFormat": "YYYY-MM-DD HH:mm:ss", 1785 "decimals": 2, 1786 "mappingType": 1, 1787 "pattern": "tenant", 1788 "thresholds": [ ], 1789 "type": "string", 1790 "unit": "short" 1791 }, 1792 { 1793 "alias": "", 1794 "align": "auto", 1795 "colorMode": null, 1796 "colors": [ 1797 "rgba(245, 54, 54, 0.9)", 1798 "rgba(237, 129, 40, 0.89)", 1799 "rgba(50, 172, 45, 0.97)" 1800 ], 1801 "dateFormat": "YYYY-MM-DD HH:mm:ss", 1802 "decimals": 2, 1803 "mappingType": 1, 1804 "pattern": "reason", 1805 "thresholds": [ ], 1806 "type": "number", 1807 "unit": "short" 1808 }, 1809 { 1810 "alias": "", 1811 "align": "right", 1812 "colorMode": null, 1813 "colors": [ 1814 "rgba(245, 54, 54, 0.9)", 1815 "rgba(237, 129, 40, 0.89)", 1816 "rgba(50, 172, 45, 0.97)" 1817 ], 1818 "decimals": 2, 1819 "pattern": "/.*/", 1820 "thresholds": [ ], 1821 "type": "number", 1822 "unit": "short" 1823 } 1824 ], 1825 "targets": [ 1826 { 1827 "expr": "topk(10, sum by (tenant, reason) (sum_over_time(increase(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[1m])[$__range:1m])))", 1828 "format": "table", 1829 "instant": true, 1830 "interval": "", 1831 "legendFormat": "{{ tenant }} - {{ reason }}", 1832 "refId": "A" 1833 } 1834 ], 1835 "timeFrom": null, 1836 "timeShift": null, 1837 "title": "Discarded Lines Per Interval", 1838 "transform": "table", 1839 "type": "table-old" 1840 } 1841 ], 1842 "targets": [ ], 1843 "title": "Limits", 1844 "type": "row" 1845 }, 1846 { 1847 "collapsed": true, 1848 "datasource": null, 1849 "gridPos": { 1850 "h": 1, 1851 "w": 24, 1852 "x": 0, 1853 "y": 27 1854 }, 1855 "id": 23, 1856 "panels": [ 1857 { 1858 "aliasColors": { }, 1859 "bars": false, 1860 "dashLength": 10, 1861 "dashes": false, 1862 "datasource": "$datasource", 1863 "fill": 1, 1864 "fillGradient": 0, 1865 "gridPos": { 1866 "h": 7, 1867 "w": 6, 1868 "x": 0, 1869 "y": 28 1870 }, 1871 "hiddenSeries": false, 1872 "id": 26, 1873 "legend": { 1874 "avg": false, 1875 "current": false, 1876 "max": false, 1877 "min": false, 1878 "show": false, 1879 "total": false, 1880 "values": false 1881 }, 1882 "lines": false, 1883 "linewidth": 1, 1884 "nullPointMode": "null", 1885 "options": { 1886 "dataLinks": [ ] 1887 }, 1888 "panels": [ ], 1889 "percentage": false, 1890 "pointradius": 1, 1891 "points": true, 1892 "renderer": "flot", 1893 "seriesOverrides": [ ], 1894 "spaceLength": 10, 1895 "stack": false, 1896 "steppedLine": false, 1897 "targets": [ 1898 { 1899 "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-write.*\"}", 1900 "intervalFactor": 3, 1901 "legendFormat": "{{pod}}-{{container}}", 1902 "refId": "A" 1903 } 1904 ], 1905 "thresholds": [ ], 1906 "timeFrom": null, 1907 "timeRegions": [ ], 1908 "timeShift": null, 1909 "title": "CPU Usage", 1910 "tooltip": { 1911 "shared": true, 1912 "sort": 2, 1913 "value_type": "individual" 1914 }, 1915 "type": "graph", 1916 "xaxis": { 1917 "buckets": null, 1918 "mode": "time", 1919 "name": null, 1920 "show": true, 1921 "values": [ ] 1922 }, 1923 "yaxes": [ 1924 { 1925 "format": "short", 1926 "label": null, 1927 "logBase": 1, 1928 "max": null, 1929 "min": null, 1930 "show": true 1931 }, 1932 { 1933 "format": "short", 1934 "label": null, 1935 "logBase": 1, 1936 "max": null, 1937 "min": null, 1938 "show": true 1939 } 1940 ], 1941 "yaxis": { 1942 "align": false, 1943 "alignLevel": null 1944 } 1945 }, 1946 { 1947 "aliasColors": { }, 1948 "bars": false, 1949 "dashLength": 10, 1950 "dashes": false, 1951 "datasource": "$datasource", 1952 "fill": 1, 1953 "fillGradient": 0, 1954 "gridPos": { 1955 "h": 7, 1956 "w": 6, 1957 "x": 6, 1958 "y": 28 1959 }, 1960 "hiddenSeries": false, 1961 "id": 27, 1962 "legend": { 1963 "avg": false, 1964 "current": false, 1965 "hideEmpty": false, 1966 "hideZero": false, 1967 "max": false, 1968 "min": false, 1969 "show": false, 1970 "total": false, 1971 "values": false 1972 }, 1973 "lines": false, 1974 "linewidth": 1, 1975 "nullPointMode": "null", 1976 "options": { 1977 "dataLinks": [ ] 1978 }, 1979 "panels": [ ], 1980 "percentage": false, 1981 "pointradius": 1, 1982 "points": true, 1983 "renderer": "flot", 1984 "seriesOverrides": [ ], 1985 "spaceLength": 10, 1986 "stack": false, 1987 "steppedLine": false, 1988 "targets": [ 1989 { 1990 "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-write.*\"}", 1991 "instant": false, 1992 "intervalFactor": 3, 1993 "legendFormat": "{{pod}}", 1994 "refId": "A" 1995 } 1996 ], 1997 "thresholds": [ ], 1998 "timeFrom": null, 1999 "timeRegions": [ ], 2000 "timeShift": null, 2001 "title": "Memory Usage", 2002 "tooltip": { 2003 "shared": true, 2004 "sort": 2, 2005 "value_type": "individual" 2006 }, 2007 "type": "graph", 2008 "xaxis": { 2009 "buckets": null, 2010 "mode": "time", 2011 "name": null, 2012 "show": true, 2013 "values": [ ] 2014 }, 2015 "yaxes": [ 2016 { 2017 "format": "bytes", 2018 "label": null, 2019 "logBase": 1, 2020 "max": null, 2021 "min": null, 2022 "show": true 2023 }, 2024 { 2025 "format": "short", 2026 "label": null, 2027 "logBase": 1, 2028 "max": null, 2029 "min": null, 2030 "show": true 2031 } 2032 ], 2033 "yaxis": { 2034 "align": false, 2035 "alignLevel": null 2036 } 2037 }, 2038 { 2039 "aliasColors": { }, 2040 "bars": true, 2041 "dashLength": 10, 2042 "dashes": false, 2043 "datasource": "$logs", 2044 "fill": 1, 2045 "fillGradient": 0, 2046 "gridPos": { 2047 "h": 4, 2048 "w": 12, 2049 "x": 12, 2050 "y": 28 2051 }, 2052 "hiddenSeries": false, 2053 "id": 31, 2054 "legend": { 2055 "avg": false, 2056 "current": false, 2057 "max": false, 2058 "min": false, 2059 "show": false, 2060 "total": false, 2061 "values": false 2062 }, 2063 "lines": false, 2064 "linewidth": 1, 2065 "nullPointMode": "null", 2066 "options": { 2067 "dataLinks": [ ] 2068 }, 2069 "panels": [ ], 2070 "percentage": false, 2071 "pointradius": 2, 2072 "points": false, 2073 "renderer": "flot", 2074 "seriesOverrides": [ 2075 { 2076 "alias": "{}", 2077 "color": "#C4162A" 2078 } 2079 ], 2080 "spaceLength": 10, 2081 "stack": false, 2082 "steppedLine": false, 2083 "targets": [ 2084 { 2085 "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"} | logfmt | level=\"error\"[1m]))", 2086 "refId": "A" 2087 } 2088 ], 2089 "thresholds": [ ], 2090 "timeFrom": null, 2091 "timeRegions": [ ], 2092 "timeShift": null, 2093 "title": "Error Log Rate", 2094 "tooltip": { 2095 "shared": true, 2096 "sort": 2, 2097 "value_type": "individual" 2098 }, 2099 "type": "graph", 2100 "xaxis": { 2101 "buckets": null, 2102 "mode": "time", 2103 "name": null, 2104 "show": false, 2105 "values": [ ] 2106 }, 2107 "yaxes": [ 2108 { 2109 "format": "short", 2110 "label": null, 2111 "logBase": 1, 2112 "max": null, 2113 "min": null, 2114 "show": true 2115 }, 2116 { 2117 "format": "short", 2118 "label": null, 2119 "logBase": 1, 2120 "max": null, 2121 "min": null, 2122 "show": false 2123 } 2124 ], 2125 "yaxis": { 2126 "align": false, 2127 "alignLevel": null 2128 } 2129 }, 2130 { 2131 "datasource": "$logs", 2132 "gridPos": { 2133 "h": 18, 2134 "w": 12, 2135 "x": 12, 2136 "y": 32 2137 }, 2138 "id": 29, 2139 "options": { 2140 "showLabels": false, 2141 "showTime": false, 2142 "sortOrder": "Descending", 2143 "wrapLogMessage": true 2144 }, 2145 "panels": [ ], 2146 "targets": [ 2147 { 2148 "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"} | logfmt | level=\"error\"", 2149 "refId": "A" 2150 } 2151 ], 2152 "timeFrom": null, 2153 "timeShift": null, 2154 "title": "Logs", 2155 "type": "logs" 2156 }, 2157 { 2158 "aliasColors": { }, 2159 "bars": false, 2160 "dashLength": 10, 2161 "dashes": false, 2162 "datasource": "$datasource", 2163 "fill": 0, 2164 "fillGradient": 0, 2165 "gridPos": { 2166 "h": 7, 2167 "w": 6, 2168 "x": 0, 2169 "y": 35 2170 }, 2171 "hiddenSeries": false, 2172 "id": 33, 2173 "legend": { 2174 "avg": false, 2175 "current": false, 2176 "max": false, 2177 "min": false, 2178 "show": false, 2179 "total": false, 2180 "values": false 2181 }, 2182 "lines": true, 2183 "linewidth": 1, 2184 "nullPointMode": "null", 2185 "options": { 2186 "dataLinks": [ ] 2187 }, 2188 "panels": [ ], 2189 "percentage": false, 2190 "pointradius": 1, 2191 "points": false, 2192 "renderer": "flot", 2193 "seriesOverrides": [ ], 2194 "spaceLength": 10, 2195 "stack": false, 2196 "steppedLine": false, 2197 "targets": [ 2198 { 2199 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[5m])) by (route)", 2200 "interval": "", 2201 "intervalFactor": 1, 2202 "legendFormat": "{{route}}", 2203 "refId": "A" 2204 } 2205 ], 2206 "thresholds": [ ], 2207 "timeFrom": null, 2208 "timeRegions": [ ], 2209 "timeShift": null, 2210 "title": "Success Rate", 2211 "tooltip": { 2212 "shared": true, 2213 "sort": 2, 2214 "value_type": "individual" 2215 }, 2216 "type": "graph", 2217 "xaxis": { 2218 "buckets": null, 2219 "mode": "time", 2220 "name": null, 2221 "show": true, 2222 "values": [ ] 2223 }, 2224 "yaxes": [ 2225 { 2226 "format": "short", 2227 "label": null, 2228 "logBase": 1, 2229 "max": null, 2230 "min": null, 2231 "show": true 2232 }, 2233 { 2234 "format": "short", 2235 "label": null, 2236 "logBase": 1, 2237 "max": null, 2238 "min": null, 2239 "show": true 2240 } 2241 ], 2242 "yaxis": { 2243 "align": false, 2244 "alignLevel": null 2245 } 2246 }, 2247 { 2248 "aliasColors": { }, 2249 "bars": false, 2250 "dashLength": 10, 2251 "dashes": false, 2252 "datasource": "$datasource", 2253 "fill": 1, 2254 "fillGradient": 0, 2255 "gridPos": { 2256 "h": 7, 2257 "w": 6, 2258 "x": 6, 2259 "y": 35 2260 }, 2261 "hiddenSeries": false, 2262 "id": 32, 2263 "legend": { 2264 "avg": false, 2265 "current": false, 2266 "max": false, 2267 "min": false, 2268 "show": false, 2269 "total": false, 2270 "values": false 2271 }, 2272 "lines": true, 2273 "linewidth": 1, 2274 "nullPointMode": "null", 2275 "options": { 2276 "dataLinks": [ ] 2277 }, 2278 "panels": [ ], 2279 "percentage": false, 2280 "pointradius": 1, 2281 "points": false, 2282 "renderer": "flot", 2283 "seriesOverrides": [ ], 2284 "spaceLength": 10, 2285 "stack": false, 2286 "steppedLine": false, 2287 "targets": [ 2288 { 2289 "expr": "sum(rate(loki_distributor_ingester_append_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (ingester)", 2290 "intervalFactor": 1, 2291 "legendFormat": "{{ingester}}", 2292 "refId": "A" 2293 } 2294 ], 2295 "thresholds": [ ], 2296 "timeFrom": null, 2297 "timeRegions": [ ], 2298 "timeShift": null, 2299 "title": "Append Failures By Ingester", 2300 "tooltip": { 2301 "shared": true, 2302 "sort": 2, 2303 "value_type": "individual" 2304 }, 2305 "type": "graph", 2306 "xaxis": { 2307 "buckets": null, 2308 "mode": "time", 2309 "name": null, 2310 "show": true, 2311 "values": [ ] 2312 }, 2313 "yaxes": [ 2314 { 2315 "format": "short", 2316 "label": null, 2317 "logBase": 1, 2318 "max": null, 2319 "min": null, 2320 "show": true 2321 }, 2322 { 2323 "format": "short", 2324 "label": null, 2325 "logBase": 1, 2326 "max": null, 2327 "min": null, 2328 "show": true 2329 } 2330 ], 2331 "yaxis": { 2332 "align": false, 2333 "alignLevel": null 2334 } 2335 }, 2336 { 2337 "aliasColors": { }, 2338 "bars": false, 2339 "dashLength": 10, 2340 "dashes": false, 2341 "datasource": "$datasource", 2342 "fill": 1, 2343 "fillGradient": 0, 2344 "gridPos": { 2345 "h": 7, 2346 "w": 6, 2347 "x": 0, 2348 "y": 42 2349 }, 2350 "hiddenSeries": false, 2351 "id": 34, 2352 "legend": { 2353 "avg": false, 2354 "current": false, 2355 "max": false, 2356 "min": false, 2357 "show": false, 2358 "total": false, 2359 "values": false 2360 }, 2361 "lines": true, 2362 "linewidth": 1, 2363 "nullPointMode": "null", 2364 "options": { 2365 "dataLinks": [ ] 2366 }, 2367 "panels": [ ], 2368 "percentage": false, 2369 "pointradius": 1, 2370 "points": false, 2371 "renderer": "flot", 2372 "seriesOverrides": [ ], 2373 "spaceLength": 10, 2374 "stack": false, 2375 "steppedLine": false, 2376 "targets": [ 2377 { 2378 "expr": "sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", 2379 "intervalFactor": 1, 2380 "legendFormat": "{{pod}}", 2381 "refId": "A" 2382 } 2383 ], 2384 "thresholds": [ ], 2385 "timeFrom": null, 2386 "timeRegions": [ ], 2387 "timeShift": null, 2388 "title": "Bytes Received/Second", 2389 "tooltip": { 2390 "shared": true, 2391 "sort": 2, 2392 "value_type": "individual" 2393 }, 2394 "type": "graph", 2395 "xaxis": { 2396 "buckets": null, 2397 "mode": "time", 2398 "name": null, 2399 "show": true, 2400 "values": [ ] 2401 }, 2402 "yaxes": [ 2403 { 2404 "format": "short", 2405 "label": null, 2406 "logBase": 1, 2407 "max": null, 2408 "min": null, 2409 "show": true 2410 }, 2411 { 2412 "format": "short", 2413 "label": null, 2414 "logBase": 1, 2415 "max": null, 2416 "min": null, 2417 "show": true 2418 } 2419 ], 2420 "yaxis": { 2421 "align": false, 2422 "alignLevel": null 2423 } 2424 }, 2425 { 2426 "aliasColors": { }, 2427 "bars": false, 2428 "dashLength": 10, 2429 "dashes": false, 2430 "datasource": "$datasource", 2431 "fill": 1, 2432 "fillGradient": 0, 2433 "gridPos": { 2434 "h": 7, 2435 "w": 6, 2436 "x": 6, 2437 "y": 42 2438 }, 2439 "hiddenSeries": false, 2440 "id": 35, 2441 "legend": { 2442 "avg": false, 2443 "current": false, 2444 "max": false, 2445 "min": false, 2446 "show": false, 2447 "total": false, 2448 "values": false 2449 }, 2450 "lines": true, 2451 "linewidth": 1, 2452 "nullPointMode": "null", 2453 "options": { 2454 "dataLinks": [ ] 2455 }, 2456 "panels": [ ], 2457 "percentage": false, 2458 "pointradius": 1, 2459 "points": false, 2460 "renderer": "flot", 2461 "seriesOverrides": [ ], 2462 "spaceLength": 10, 2463 "stack": false, 2464 "steppedLine": false, 2465 "targets": [ 2466 { 2467 "expr": "sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", 2468 "intervalFactor": 1, 2469 "legendFormat": "{{pod}}", 2470 "refId": "A" 2471 } 2472 ], 2473 "thresholds": [ ], 2474 "timeFrom": null, 2475 "timeRegions": [ ], 2476 "timeShift": null, 2477 "title": "Lines Received/Second", 2478 "tooltip": { 2479 "shared": true, 2480 "sort": 2, 2481 "value_type": "individual" 2482 }, 2483 "type": "graph", 2484 "xaxis": { 2485 "buckets": null, 2486 "mode": "time", 2487 "name": null, 2488 "show": true, 2489 "values": [ ] 2490 }, 2491 "yaxes": [ 2492 { 2493 "format": "short", 2494 "label": null, 2495 "logBase": 1, 2496 "max": null, 2497 "min": null, 2498 "show": true 2499 }, 2500 { 2501 "format": "short", 2502 "label": null, 2503 "logBase": 1, 2504 "max": null, 2505 "min": null, 2506 "show": true 2507 } 2508 ], 2509 "yaxis": { 2510 "align": false, 2511 "alignLevel": null 2512 } 2513 } 2514 ], 2515 "targets": [ ], 2516 "title": "Write Path", 2517 "type": "row" 2518 }, 2519 { 2520 "collapsed": true, 2521 "datasource": null, 2522 "gridPos": { 2523 "h": 1, 2524 "w": 24, 2525 "x": 0, 2526 "y": 29 2527 }, 2528 "id": 104, 2529 "panels": [ 2530 { 2531 "aliasColors": { }, 2532 "bars": false, 2533 "dashLength": 10, 2534 "dashes": false, 2535 "datasource": "$datasource", 2536 "fill": 1, 2537 "fillGradient": 0, 2538 "gridPos": { 2539 "h": 8, 2540 "w": 12, 2541 "x": 0, 2542 "y": 30 2543 }, 2544 "hiddenSeries": false, 2545 "id": 106, 2546 "legend": { 2547 "avg": false, 2548 "current": false, 2549 "hideEmpty": true, 2550 "hideZero": true, 2551 "max": false, 2552 "min": false, 2553 "show": false, 2554 "total": false, 2555 "values": false 2556 }, 2557 "lines": true, 2558 "linewidth": 1, 2559 "nullPointMode": "null", 2560 "options": { 2561 "dataLinks": [ ] 2562 }, 2563 "panels": [ ], 2564 "percentage": false, 2565 "pointradius": 2, 2566 "points": false, 2567 "renderer": "flot", 2568 "seriesOverrides": [ ], 2569 "spaceLength": 10, 2570 "stack": false, 2571 "steppedLine": false, 2572 "targets": [ 2573 { 2574 "expr": "topk(10,sum by (tenant) (loki_ingester_memory_streams{cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}))", 2575 "interval": "", 2576 "legendFormat": "{{ tenant }}", 2577 "refId": "A" 2578 } 2579 ], 2580 "thresholds": [ ], 2581 "timeFrom": null, 2582 "timeRegions": [ ], 2583 "timeShift": null, 2584 "title": "Active Streams", 2585 "tooltip": { 2586 "shared": false, 2587 "sort": 0, 2588 "value_type": "individual" 2589 }, 2590 "type": "graph", 2591 "xaxis": { 2592 "buckets": null, 2593 "mode": "time", 2594 "name": null, 2595 "show": true, 2596 "values": [ ] 2597 }, 2598 "yaxes": [ 2599 { 2600 "format": "short", 2601 "label": null, 2602 "logBase": 1, 2603 "max": null, 2604 "min": null, 2605 "show": true 2606 }, 2607 { 2608 "format": "short", 2609 "label": null, 2610 "logBase": 1, 2611 "max": null, 2612 "min": null, 2613 "show": true 2614 } 2615 ], 2616 "yaxis": { 2617 "align": false, 2618 "alignLevel": null 2619 } 2620 }, 2621 { 2622 "aliasColors": { }, 2623 "bars": false, 2624 "dashLength": 10, 2625 "dashes": false, 2626 "datasource": "$datasource", 2627 "fill": 1, 2628 "fillGradient": 0, 2629 "gridPos": { 2630 "h": 8, 2631 "w": 12, 2632 "x": 12, 2633 "y": 30 2634 }, 2635 "hiddenSeries": false, 2636 "id": 108, 2637 "legend": { 2638 "avg": false, 2639 "current": false, 2640 "hideEmpty": true, 2641 "hideZero": true, 2642 "max": false, 2643 "min": false, 2644 "show": false, 2645 "total": false, 2646 "values": false 2647 }, 2648 "lines": true, 2649 "linewidth": 1, 2650 "nullPointMode": "null", 2651 "options": { 2652 "dataLinks": [ ] 2653 }, 2654 "panels": [ ], 2655 "percentage": false, 2656 "pointradius": 2, 2657 "points": false, 2658 "renderer": "flot", 2659 "seriesOverrides": [ ], 2660 "spaceLength": 10, 2661 "stack": false, 2662 "steppedLine": false, 2663 "targets": [ 2664 { 2665 "expr": "topk(10, sum by (tenant) (rate(loki_ingester_streams_created_total{cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]) > 0))", 2666 "interval": "", 2667 "legendFormat": "{{ tenant }}", 2668 "refId": "A" 2669 } 2670 ], 2671 "thresholds": [ ], 2672 "timeFrom": null, 2673 "timeRegions": [ ], 2674 "timeShift": null, 2675 "title": "Streams Created/Sec", 2676 "tooltip": { 2677 "shared": false, 2678 "sort": 0, 2679 "value_type": "individual" 2680 }, 2681 "type": "graph", 2682 "xaxis": { 2683 "buckets": null, 2684 "mode": "time", 2685 "name": null, 2686 "show": true, 2687 "values": [ ] 2688 }, 2689 "yaxes": [ 2690 { 2691 "format": "short", 2692 "label": null, 2693 "logBase": 1, 2694 "max": null, 2695 "min": null, 2696 "show": true 2697 }, 2698 { 2699 "format": "short", 2700 "label": null, 2701 "logBase": 1, 2702 "max": null, 2703 "min": null, 2704 "show": true 2705 } 2706 ], 2707 "yaxis": { 2708 "align": false, 2709 "alignLevel": null 2710 } 2711 } 2712 ], 2713 "targets": [ ], 2714 "title": "Streams", 2715 "type": "row" 2716 }, 2717 { 2718 "collapsed": true, 2719 "datasource": null, 2720 "gridPos": { 2721 "h": 1, 2722 "w": 24, 2723 "x": 0, 2724 "y": 30 2725 }, 2726 "id": 94, 2727 "panels": [ 2728 { 2729 "aliasColors": { }, 2730 "bars": false, 2731 "dashLength": 10, 2732 "dashes": false, 2733 "datasource": "$datasource", 2734 "fill": 1, 2735 "fillGradient": 0, 2736 "gridPos": { 2737 "h": 8, 2738 "w": 12, 2739 "x": 0, 2740 "y": 31 2741 }, 2742 "hiddenSeries": false, 2743 "id": 102, 2744 "legend": { 2745 "avg": false, 2746 "current": false, 2747 "max": false, 2748 "min": false, 2749 "show": true, 2750 "total": false, 2751 "values": false 2752 }, 2753 "lines": true, 2754 "linewidth": 1, 2755 "nullPointMode": "null", 2756 "options": { 2757 "dataLinks": [ ] 2758 }, 2759 "panels": [ ], 2760 "percentage": false, 2761 "pointradius": 2, 2762 "points": false, 2763 "renderer": "flot", 2764 "seriesOverrides": [ 2765 { 2766 "alias": "De-Dupe Ratio", 2767 "yaxis": 2 2768 } 2769 ], 2770 "spaceLength": 10, 2771 "stack": false, 2772 "steppedLine": false, 2773 "targets": [ 2774 { 2775 "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]))", 2776 "interval": "", 2777 "legendFormat": "Chunks", 2778 "refId": "A" 2779 }, 2780 { 2781 "expr": "sum(increase(loki_chunk_store_deduped_chunks_total{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]))/sum(increase(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m])) < 1", 2782 "interval": "", 2783 "legendFormat": "De-Dupe Ratio", 2784 "refId": "B" 2785 } 2786 ], 2787 "thresholds": [ ], 2788 "timeFrom": null, 2789 "timeRegions": [ ], 2790 "timeShift": null, 2791 "title": "Chunks Flushed/Sec", 2792 "tooltip": { 2793 "shared": true, 2794 "sort": 0, 2795 "value_type": "individual" 2796 }, 2797 "type": "graph", 2798 "xaxis": { 2799 "buckets": null, 2800 "mode": "time", 2801 "name": null, 2802 "show": true, 2803 "values": [ ] 2804 }, 2805 "yaxes": [ 2806 { 2807 "format": "short", 2808 "label": null, 2809 "logBase": 1, 2810 "max": null, 2811 "min": null, 2812 "show": true 2813 }, 2814 { 2815 "format": "short", 2816 "label": null, 2817 "logBase": 1, 2818 "max": null, 2819 "min": null, 2820 "show": true 2821 } 2822 ], 2823 "yaxis": { 2824 "align": false, 2825 "alignLevel": null 2826 } 2827 }, 2828 { 2829 "cards": { 2830 "cardPadding": null, 2831 "cardRound": null 2832 }, 2833 "color": { 2834 "cardColor": "#b4ff00", 2835 "colorScale": "sqrt", 2836 "colorScheme": "interpolateSpectral", 2837 "exponent": 0.5, 2838 "mode": "spectrum" 2839 }, 2840 "dataFormat": "tsbuckets", 2841 "datasource": "$datasource", 2842 "gridPos": { 2843 "h": 8, 2844 "w": 12, 2845 "x": 12, 2846 "y": 31 2847 }, 2848 "heatmap": { }, 2849 "hideZeroBuckets": false, 2850 "highlightCards": true, 2851 "id": 100, 2852 "legend": { 2853 "show": true 2854 }, 2855 "panels": [ ], 2856 "reverseYBuckets": false, 2857 "targets": [ 2858 { 2859 "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m])) by (le)", 2860 "format": "heatmap", 2861 "instant": false, 2862 "interval": "", 2863 "legendFormat": "{{ le }}", 2864 "refId": "A" 2865 } 2866 ], 2867 "timeFrom": null, 2868 "timeShift": null, 2869 "title": "Chunk Size Bytes", 2870 "tooltip": { 2871 "show": true, 2872 "showHistogram": false 2873 }, 2874 "type": "heatmap", 2875 "xAxis": { 2876 "show": true 2877 }, 2878 "xBucketNumber": null, 2879 "xBucketSize": null, 2880 "yAxis": { 2881 "decimals": 0, 2882 "format": "bytes", 2883 "logBase": 1, 2884 "max": null, 2885 "min": null, 2886 "show": true, 2887 "splitFactor": null 2888 }, 2889 "yBucketBound": "auto", 2890 "yBucketNumber": null, 2891 "yBucketSize": null 2892 }, 2893 { 2894 "aliasColors": { }, 2895 "bars": false, 2896 "dashLength": 10, 2897 "dashes": false, 2898 "datasource": "$datasource", 2899 "fill": 7, 2900 "fillGradient": 0, 2901 "gridPos": { 2902 "h": 9, 2903 "w": 12, 2904 "x": 0, 2905 "y": 39 2906 }, 2907 "hiddenSeries": false, 2908 "id": 96, 2909 "legend": { 2910 "avg": false, 2911 "current": false, 2912 "max": false, 2913 "min": false, 2914 "show": true, 2915 "total": false, 2916 "values": false 2917 }, 2918 "lines": true, 2919 "linewidth": 1, 2920 "nullPointMode": "null", 2921 "options": { 2922 "dataLinks": [ ] 2923 }, 2924 "panels": [ ], 2925 "percentage": false, 2926 "pointradius": 2, 2927 "points": false, 2928 "renderer": "flot", 2929 "seriesOverrides": [ ], 2930 "spaceLength": 10, 2931 "stack": true, 2932 "steppedLine": false, 2933 "targets": [ 2934 { 2935 "expr": "sum by(reason) (rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\",job=~\"$namespace/ingester\", namespace=~\"$namespace\"}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\",job=~\"$namespace/ingester\", namespace=~\"$namespace\"}[$__rate_interval]))", 2936 "interval": "", 2937 "legendFormat": "{{ reason }}" 2938 } 2939 ], 2940 "thresholds": [ ], 2941 "timeFrom": null, 2942 "timeRegions": [ ], 2943 "timeShift": null, 2944 "title": "Chunk Flush Reason %", 2945 "tooltip": { 2946 "shared": true, 2947 "sort": 0, 2948 "value_type": "individual" 2949 }, 2950 "type": "graph", 2951 "xaxis": { 2952 "buckets": null, 2953 "mode": "time", 2954 "name": null, 2955 "show": true, 2956 "values": [ ] 2957 }, 2958 "yaxes": [ 2959 { 2960 "format": "percentunit", 2961 "label": null, 2962 "logBase": 1, 2963 "max": "1", 2964 "min": "0", 2965 "show": true 2966 }, 2967 { 2968 "format": "short", 2969 "label": null, 2970 "logBase": 1, 2971 "max": null, 2972 "min": null, 2973 "show": true 2974 } 2975 ], 2976 "yaxis": { 2977 "align": false, 2978 "alignLevel": null 2979 } 2980 }, 2981 { 2982 "cards": { 2983 "cardPadding": null, 2984 "cardRound": null 2985 }, 2986 "color": { 2987 "cardColor": "#b4ff00", 2988 "colorScale": "sqrt", 2989 "colorScheme": "interpolateSpectral", 2990 "exponent": 0.5, 2991 "max": null, 2992 "min": null, 2993 "mode": "spectrum" 2994 }, 2995 "dataFormat": "tsbuckets", 2996 "datasource": "$datasource", 2997 "gridPos": { 2998 "h": 9, 2999 "w": 12, 3000 "x": 12, 3001 "y": 39 3002 }, 3003 "heatmap": { }, 3004 "hideZeroBuckets": true, 3005 "highlightCards": true, 3006 "id": 98, 3007 "legend": { 3008 "show": true 3009 }, 3010 "panels": [ ], 3011 "reverseYBuckets": false, 3012 "targets": [ 3013 { 3014 "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]))", 3015 "format": "heatmap", 3016 "instant": false, 3017 "interval": "", 3018 "legendFormat": "{{ le }}", 3019 "refId": "A" 3020 } 3021 ], 3022 "timeFrom": null, 3023 "timeShift": null, 3024 "title": "Chunk Utilization", 3025 "tooltip": { 3026 "show": true, 3027 "showHistogram": false 3028 }, 3029 "type": "heatmap", 3030 "xAxis": { 3031 "show": true 3032 }, 3033 "xBucketNumber": null, 3034 "xBucketSize": null, 3035 "yAxis": { 3036 "decimals": 0, 3037 "format": "percentunit", 3038 "logBase": 1, 3039 "max": null, 3040 "min": null, 3041 "show": true, 3042 "splitFactor": null 3043 }, 3044 "yBucketBound": "auto", 3045 "yBucketNumber": null, 3046 "yBucketSize": null 3047 } 3048 ], 3049 "targets": [ ], 3050 "title": "Chunks", 3051 "type": "row" 3052 }, 3053 { 3054 "collapsed": true, 3055 "datasource": null, 3056 "gridPos": { 3057 "h": 1, 3058 "w": 24, 3059 "x": 0, 3060 "y": 31 3061 }, 3062 "id": 64, 3063 "panels": [ 3064 { 3065 "aliasColors": { }, 3066 "bars": false, 3067 "dashLength": 10, 3068 "dashes": false, 3069 "datasource": "$datasource", 3070 "fill": 1, 3071 "fillGradient": 0, 3072 "gridPos": { 3073 "h": 7, 3074 "w": 6, 3075 "x": 0, 3076 "y": 32 3077 }, 3078 "hiddenSeries": false, 3079 "id": 68, 3080 "legend": { 3081 "avg": false, 3082 "current": false, 3083 "max": false, 3084 "min": false, 3085 "show": false, 3086 "total": false, 3087 "values": false 3088 }, 3089 "lines": false, 3090 "linewidth": 1, 3091 "nullPointMode": "null", 3092 "options": { 3093 "dataLinks": [ ] 3094 }, 3095 "panels": [ ], 3096 "percentage": false, 3097 "pointradius": 1, 3098 "points": true, 3099 "renderer": "flot", 3100 "seriesOverrides": [ ], 3101 "spaceLength": 10, 3102 "stack": false, 3103 "steppedLine": false, 3104 "targets": [ 3105 { 3106 "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-read.*\"}", 3107 "intervalFactor": 3, 3108 "legendFormat": "{{pod}}-{{container}}", 3109 "refId": "A" 3110 } 3111 ], 3112 "thresholds": [ ], 3113 "timeFrom": null, 3114 "timeRegions": [ ], 3115 "timeShift": null, 3116 "title": "CPU Usage", 3117 "tooltip": { 3118 "shared": true, 3119 "sort": 2, 3120 "value_type": "individual" 3121 }, 3122 "type": "graph", 3123 "xaxis": { 3124 "buckets": null, 3125 "mode": "time", 3126 "name": null, 3127 "show": true, 3128 "values": [ ] 3129 }, 3130 "yaxes": [ 3131 { 3132 "format": "short", 3133 "label": null, 3134 "logBase": 1, 3135 "max": null, 3136 "min": null, 3137 "show": true 3138 }, 3139 { 3140 "format": "short", 3141 "label": null, 3142 "logBase": 1, 3143 "max": null, 3144 "min": null, 3145 "show": true 3146 } 3147 ], 3148 "yaxis": { 3149 "align": false, 3150 "alignLevel": null 3151 } 3152 }, 3153 { 3154 "aliasColors": { }, 3155 "bars": false, 3156 "dashLength": 10, 3157 "dashes": false, 3158 "datasource": "$datasource", 3159 "fill": 1, 3160 "fillGradient": 0, 3161 "gridPos": { 3162 "h": 7, 3163 "w": 6, 3164 "x": 0, 3165 "y": 39 3166 }, 3167 "hiddenSeries": false, 3168 "id": 69, 3169 "legend": { 3170 "avg": false, 3171 "current": false, 3172 "hideEmpty": false, 3173 "hideZero": false, 3174 "max": false, 3175 "min": false, 3176 "show": false, 3177 "total": false, 3178 "values": false 3179 }, 3180 "lines": false, 3181 "linewidth": 1, 3182 "nullPointMode": "null", 3183 "options": { 3184 "dataLinks": [ ] 3185 }, 3186 "panels": [ ], 3187 "percentage": false, 3188 "pointradius": 1, 3189 "points": true, 3190 "renderer": "flot", 3191 "seriesOverrides": [ ], 3192 "spaceLength": 10, 3193 "stack": false, 3194 "steppedLine": false, 3195 "targets": [ 3196 { 3197 "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-read.*\"}", 3198 "instant": false, 3199 "intervalFactor": 3, 3200 "legendFormat": "{{pod}}", 3201 "refId": "A" 3202 } 3203 ], 3204 "thresholds": [ ], 3205 "timeFrom": null, 3206 "timeRegions": [ ], 3207 "timeShift": null, 3208 "title": "Memory Usage", 3209 "tooltip": { 3210 "shared": true, 3211 "sort": 2, 3212 "value_type": "individual" 3213 }, 3214 "type": "graph", 3215 "xaxis": { 3216 "buckets": null, 3217 "mode": "time", 3218 "name": null, 3219 "show": true, 3220 "values": [ ] 3221 }, 3222 "yaxes": [ 3223 { 3224 "format": "bytes", 3225 "label": null, 3226 "logBase": 1, 3227 "max": null, 3228 "min": null, 3229 "show": true 3230 }, 3231 { 3232 "format": "short", 3233 "label": null, 3234 "logBase": 1, 3235 "max": null, 3236 "min": null, 3237 "show": true 3238 } 3239 ], 3240 "yaxis": { 3241 "align": false, 3242 "alignLevel": null 3243 } 3244 }, 3245 { 3246 "aliasColors": { }, 3247 "bars": true, 3248 "dashLength": 10, 3249 "dashes": false, 3250 "datasource": "$logs", 3251 "fill": 1, 3252 "fillGradient": 0, 3253 "gridPos": { 3254 "h": 3, 3255 "w": 18, 3256 "x": 12, 3257 "y": 32 3258 }, 3259 "hiddenSeries": false, 3260 "id": 65, 3261 "legend": { 3262 "avg": false, 3263 "current": false, 3264 "max": false, 3265 "min": false, 3266 "show": false, 3267 "total": false, 3268 "values": false 3269 }, 3270 "lines": false, 3271 "linewidth": 1, 3272 "nullPointMode": "null", 3273 "options": { 3274 "dataLinks": [ ] 3275 }, 3276 "panels": [ ], 3277 "percentage": false, 3278 "pointradius": 2, 3279 "points": false, 3280 "renderer": "flot", 3281 "seriesOverrides": [ 3282 { 3283 "alias": "{}", 3284 "color": "#F2495C" 3285 } 3286 ], 3287 "spaceLength": 10, 3288 "stack": false, 3289 "steppedLine": false, 3290 "targets": [ 3291 { 3292 "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"} | logfmt | level=\"error\"[1m]))", 3293 "refId": "A" 3294 } 3295 ], 3296 "thresholds": [ ], 3297 "timeFrom": null, 3298 "timeRegions": [ ], 3299 "timeShift": null, 3300 "title": "Error Log Rate", 3301 "tooltip": { 3302 "shared": true, 3303 "sort": 2, 3304 "value_type": "individual" 3305 }, 3306 "type": "graph", 3307 "xaxis": { 3308 "buckets": null, 3309 "mode": "time", 3310 "name": null, 3311 "show": false, 3312 "values": [ ] 3313 }, 3314 "yaxes": [ 3315 { 3316 "format": "short", 3317 "label": null, 3318 "logBase": 1, 3319 "max": null, 3320 "min": null, 3321 "show": true 3322 }, 3323 { 3324 "format": "short", 3325 "label": null, 3326 "logBase": 1, 3327 "max": null, 3328 "min": null, 3329 "show": false 3330 } 3331 ], 3332 "yaxis": { 3333 "align": false, 3334 "alignLevel": null 3335 } 3336 }, 3337 { 3338 "datasource": "$logs", 3339 "gridPos": { 3340 "h": 18, 3341 "w": 18, 3342 "x": 12, 3343 "y": 35 3344 }, 3345 "id": 66, 3346 "options": { 3347 "showLabels": false, 3348 "showTime": false, 3349 "sortOrder": "Descending", 3350 "wrapLogMessage": true 3351 }, 3352 "panels": [ ], 3353 "targets": [ 3354 { 3355 "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"} | logfmt | level=\"error\"", 3356 "refId": "A" 3357 } 3358 ], 3359 "timeFrom": null, 3360 "timeShift": null, 3361 "title": "Logs", 3362 "type": "logs" 3363 }, 3364 { 3365 "aliasColors": { }, 3366 "bars": false, 3367 "dashLength": 10, 3368 "dashes": false, 3369 "datasource": "$datasource", 3370 "fill": 0, 3371 "fillGradient": 0, 3372 "gridPos": { 3373 "h": 7, 3374 "w": 6, 3375 "x": 0, 3376 "y": 46 3377 }, 3378 "hiddenSeries": false, 3379 "id": 70, 3380 "legend": { 3381 "avg": false, 3382 "current": false, 3383 "max": false, 3384 "min": false, 3385 "show": false, 3386 "total": false, 3387 "values": false 3388 }, 3389 "lines": true, 3390 "linewidth": 1, 3391 "nullPointMode": "null", 3392 "options": { 3393 "dataLinks": [ ] 3394 }, 3395 "panels": [ ], 3396 "percentage": false, 3397 "pointradius": 1, 3398 "points": false, 3399 "renderer": "flot", 3400 "seriesOverrides": [ ], 3401 "spaceLength": 10, 3402 "stack": false, 3403 "steppedLine": false, 3404 "targets": [ 3405 { 3406 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\", status_code!~\"5[0-9]{2}\"}[1m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"}[1m])) by (route)", 3407 "interval": "", 3408 "intervalFactor": 1, 3409 "legendFormat": "{{route}}", 3410 "refId": "A" 3411 } 3412 ], 3413 "thresholds": [ ], 3414 "timeFrom": null, 3415 "timeRegions": [ ], 3416 "timeShift": null, 3417 "title": "Success Rate", 3418 "tooltip": { 3419 "shared": true, 3420 "sort": 2, 3421 "value_type": "individual" 3422 }, 3423 "type": "graph", 3424 "xaxis": { 3425 "buckets": null, 3426 "mode": "time", 3427 "name": null, 3428 "show": true, 3429 "values": [ ] 3430 }, 3431 "yaxes": [ 3432 { 3433 "format": "short", 3434 "label": null, 3435 "logBase": 1, 3436 "max": null, 3437 "min": null, 3438 "show": true 3439 }, 3440 { 3441 "format": "short", 3442 "label": null, 3443 "logBase": 1, 3444 "max": null, 3445 "min": null, 3446 "show": true 3447 } 3448 ], 3449 "yaxis": { 3450 "align": false, 3451 "alignLevel": null 3452 } 3453 } 3454 ], 3455 "targets": [ ], 3456 "title": "Read Path", 3457 "type": "row" 3458 }, 3459 { 3460 "collapsed": true, 3461 "datasource": null, 3462 "gridPos": { 3463 "h": 1, 3464 "w": 24, 3465 "x": 0, 3466 "y": 32 3467 }, 3468 "id": 52, 3469 "panels": [ 3470 { 3471 "aliasColors": { }, 3472 "bars": false, 3473 "dashLength": 10, 3474 "dashes": false, 3475 "datasource": "$datasource", 3476 "fill": 1, 3477 "fillGradient": 0, 3478 "gridPos": { 3479 "h": 8, 3480 "w": 24, 3481 "x": 0, 3482 "y": 30 3483 }, 3484 "hiddenSeries": false, 3485 "id": 53, 3486 "interval": "", 3487 "legend": { 3488 "alignAsTable": true, 3489 "avg": false, 3490 "current": false, 3491 "max": false, 3492 "min": false, 3493 "rightSide": true, 3494 "show": true, 3495 "total": false, 3496 "values": false 3497 }, 3498 "lines": true, 3499 "linewidth": 1, 3500 "nullPointMode": "null", 3501 "options": { 3502 "dataLinks": [ ] 3503 }, 3504 "panels": [ ], 3505 "percentage": false, 3506 "pointradius": 1, 3507 "points": false, 3508 "renderer": "flot", 3509 "seriesOverrides": [ ], 3510 "spaceLength": 10, 3511 "stack": false, 3512 "steppedLine": false, 3513 "targets": [ 3514 { 3515 "expr": "histogram_quantile(.99, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", 3516 "intervalFactor": 1, 3517 "legendFormat": "{{container}}: .99-{{method}}-{{name}}", 3518 "refId": "A" 3519 }, 3520 { 3521 "expr": "histogram_quantile(.9, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", 3522 "hide": false, 3523 "legendFormat": "{{container}}: .9-{{method}}-{{name}}", 3524 "refId": "B" 3525 }, 3526 { 3527 "expr": "histogram_quantile(.5, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", 3528 "hide": false, 3529 "legendFormat": "{{container}}: .5-{{method}}-{{name}}", 3530 "refId": "C" 3531 } 3532 ], 3533 "thresholds": [ ], 3534 "timeFrom": null, 3535 "timeRegions": [ ], 3536 "timeShift": null, 3537 "title": "Latency By Method", 3538 "tooltip": { 3539 "shared": true, 3540 "sort": 2, 3541 "value_type": "individual" 3542 }, 3543 "type": "graph", 3544 "xaxis": { 3545 "buckets": null, 3546 "mode": "time", 3547 "name": null, 3548 "show": true, 3549 "values": [ ] 3550 }, 3551 "yaxes": [ 3552 { 3553 "format": "short", 3554 "label": null, 3555 "logBase": 1, 3556 "max": null, 3557 "min": null, 3558 "show": true 3559 }, 3560 { 3561 "format": "short", 3562 "label": null, 3563 "logBase": 1, 3564 "max": null, 3565 "min": null, 3566 "show": true 3567 } 3568 ], 3569 "yaxis": { 3570 "align": false, 3571 "alignLevel": null 3572 } 3573 }, 3574 { 3575 "aliasColors": { }, 3576 "bars": false, 3577 "dashLength": 10, 3578 "dashes": false, 3579 "datasource": "$datasource", 3580 "fill": 1, 3581 "fillGradient": 0, 3582 "gridPos": { 3583 "h": 8, 3584 "w": 24, 3585 "x": 0, 3586 "y": 38 3587 }, 3588 "hiddenSeries": false, 3589 "id": 54, 3590 "interval": "", 3591 "legend": { 3592 "alignAsTable": true, 3593 "avg": false, 3594 "current": false, 3595 "max": false, 3596 "min": false, 3597 "rightSide": true, 3598 "show": true, 3599 "total": false, 3600 "values": false 3601 }, 3602 "lines": true, 3603 "linewidth": 1, 3604 "nullPointMode": "null", 3605 "options": { 3606 "dataLinks": [ ] 3607 }, 3608 "panels": [ ], 3609 "percentage": false, 3610 "pointradius": 1, 3611 "points": false, 3612 "renderer": "flot", 3613 "seriesOverrides": [ ], 3614 "spaceLength": 10, 3615 "stack": false, 3616 "steppedLine": false, 3617 "targets": [ 3618 { 3619 "expr": "sum(rate(loki_memcache_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, method, name, container)", 3620 "intervalFactor": 1, 3621 "legendFormat": "{{container}}: {{status_code}}-{{method}}-{{name}}", 3622 "refId": "A" 3623 } 3624 ], 3625 "thresholds": [ ], 3626 "timeFrom": null, 3627 "timeRegions": [ ], 3628 "timeShift": null, 3629 "title": "Status By Method", 3630 "tooltip": { 3631 "shared": true, 3632 "sort": 2, 3633 "value_type": "individual" 3634 }, 3635 "type": "graph", 3636 "xaxis": { 3637 "buckets": null, 3638 "mode": "time", 3639 "name": null, 3640 "show": true, 3641 "values": [ ] 3642 }, 3643 "yaxes": [ 3644 { 3645 "format": "short", 3646 "label": null, 3647 "logBase": 1, 3648 "max": null, 3649 "min": null, 3650 "show": true 3651 }, 3652 { 3653 "format": "short", 3654 "label": null, 3655 "logBase": 1, 3656 "max": null, 3657 "min": null, 3658 "show": true 3659 } 3660 ], 3661 "yaxis": { 3662 "align": false, 3663 "alignLevel": null 3664 } 3665 } 3666 ], 3667 "targets": [ ], 3668 "title": "Memcached", 3669 "type": "row" 3670 }, 3671 { 3672 "collapsed": true, 3673 "datasource": null, 3674 "gridPos": { 3675 "h": 1, 3676 "w": 24, 3677 "x": 0, 3678 "y": 33 3679 }, 3680 "id": 57, 3681 "panels": [ 3682 { 3683 "aliasColors": { }, 3684 "bars": false, 3685 "dashLength": 10, 3686 "dashes": false, 3687 "datasource": "$datasource", 3688 "fill": 1, 3689 "fillGradient": 0, 3690 "gridPos": { 3691 "h": 8, 3692 "w": 24, 3693 "x": 0, 3694 "y": 31 3695 }, 3696 "hiddenSeries": false, 3697 "id": 55, 3698 "interval": "", 3699 "legend": { 3700 "alignAsTable": true, 3701 "avg": false, 3702 "current": false, 3703 "max": false, 3704 "min": false, 3705 "rightSide": true, 3706 "show": true, 3707 "total": false, 3708 "values": false 3709 }, 3710 "lines": true, 3711 "linewidth": 1, 3712 "nullPointMode": "null", 3713 "options": { 3714 "dataLinks": [ ] 3715 }, 3716 "panels": [ ], 3717 "percentage": false, 3718 "pointradius": 1, 3719 "points": false, 3720 "renderer": "flot", 3721 "seriesOverrides": [ ], 3722 "spaceLength": 10, 3723 "stack": false, 3724 "steppedLine": false, 3725 "targets": [ 3726 { 3727 "expr": "histogram_quantile(.99, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 3728 "intervalFactor": 1, 3729 "legendFormat": ".99-{{operation}}", 3730 "refId": "A" 3731 }, 3732 { 3733 "expr": "histogram_quantile(.9, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 3734 "hide": false, 3735 "legendFormat": ".9-{{operation}}", 3736 "refId": "B" 3737 }, 3738 { 3739 "expr": "histogram_quantile(.5, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 3740 "hide": false, 3741 "legendFormat": ".5-{{operation}}", 3742 "refId": "C" 3743 } 3744 ], 3745 "thresholds": [ ], 3746 "timeFrom": null, 3747 "timeRegions": [ ], 3748 "timeShift": null, 3749 "title": "Latency By Operation", 3750 "tooltip": { 3751 "shared": true, 3752 "sort": 2, 3753 "value_type": "individual" 3754 }, 3755 "type": "graph", 3756 "xaxis": { 3757 "buckets": null, 3758 "mode": "time", 3759 "name": null, 3760 "show": true, 3761 "values": [ ] 3762 }, 3763 "yaxes": [ 3764 { 3765 "format": "short", 3766 "label": null, 3767 "logBase": 1, 3768 "max": null, 3769 "min": null, 3770 "show": true 3771 }, 3772 { 3773 "format": "short", 3774 "label": null, 3775 "logBase": 1, 3776 "max": null, 3777 "min": null, 3778 "show": true 3779 } 3780 ], 3781 "yaxis": { 3782 "align": false, 3783 "alignLevel": null 3784 } 3785 }, 3786 { 3787 "aliasColors": { }, 3788 "bars": false, 3789 "dashLength": 10, 3790 "dashes": false, 3791 "datasource": "$datasource", 3792 "fill": 1, 3793 "fillGradient": 0, 3794 "gridPos": { 3795 "h": 8, 3796 "w": 24, 3797 "x": 0, 3798 "y": 39 3799 }, 3800 "hiddenSeries": false, 3801 "id": 58, 3802 "interval": "", 3803 "legend": { 3804 "alignAsTable": true, 3805 "avg": false, 3806 "current": false, 3807 "max": false, 3808 "min": false, 3809 "rightSide": true, 3810 "show": true, 3811 "total": false, 3812 "values": false 3813 }, 3814 "lines": true, 3815 "linewidth": 1, 3816 "nullPointMode": "null", 3817 "options": { 3818 "dataLinks": [ ] 3819 }, 3820 "panels": [ ], 3821 "percentage": false, 3822 "pointradius": 1, 3823 "points": false, 3824 "renderer": "flot", 3825 "seriesOverrides": [ ], 3826 "spaceLength": 10, 3827 "stack": false, 3828 "steppedLine": false, 3829 "targets": [ 3830 { 3831 "expr": "sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, status_code, method)", 3832 "intervalFactor": 1, 3833 "legendFormat": "{{status_code}}-{{operation}}", 3834 "refId": "A" 3835 } 3836 ], 3837 "thresholds": [ ], 3838 "timeFrom": null, 3839 "timeRegions": [ ], 3840 "timeShift": null, 3841 "title": "Status By Operation", 3842 "tooltip": { 3843 "shared": true, 3844 "sort": 2, 3845 "value_type": "individual" 3846 }, 3847 "type": "graph", 3848 "xaxis": { 3849 "buckets": null, 3850 "mode": "time", 3851 "name": null, 3852 "show": true, 3853 "values": [ ] 3854 }, 3855 "yaxes": [ 3856 { 3857 "format": "short", 3858 "label": null, 3859 "logBase": 1, 3860 "max": null, 3861 "min": null, 3862 "show": true 3863 }, 3864 { 3865 "format": "short", 3866 "label": null, 3867 "logBase": 1, 3868 "max": null, 3869 "min": null, 3870 "show": true 3871 } 3872 ], 3873 "yaxis": { 3874 "align": false, 3875 "alignLevel": null 3876 } 3877 } 3878 ], 3879 "targets": [ ], 3880 "title": "Consul", 3881 "type": "row" 3882 }, 3883 { 3884 "collapsed": true, 3885 "datasource": null, 3886 "gridPos": { 3887 "h": 1, 3888 "w": 24, 3889 "x": 0, 3890 "y": 34 3891 }, 3892 "id": 43, 3893 "panels": [ 3894 { 3895 "aliasColors": { }, 3896 "bars": false, 3897 "dashLength": 10, 3898 "dashes": false, 3899 "datasource": "$datasource", 3900 "fill": 1, 3901 "fillGradient": 0, 3902 "gridPos": { 3903 "h": 7, 3904 "w": 6, 3905 "x": 0, 3906 "y": 9 3907 }, 3908 "hiddenSeries": false, 3909 "id": 41, 3910 "interval": "", 3911 "legend": { 3912 "avg": false, 3913 "current": false, 3914 "max": false, 3915 "min": false, 3916 "show": false, 3917 "total": false, 3918 "values": false 3919 }, 3920 "lines": true, 3921 "linewidth": 1, 3922 "nullPointMode": "null", 3923 "options": { 3924 "dataLinks": [ ] 3925 }, 3926 "panels": [ ], 3927 "percentage": false, 3928 "pointradius": 1, 3929 "points": false, 3930 "renderer": "flot", 3931 "seriesOverrides": [ ], 3932 "spaceLength": 10, 3933 "stack": false, 3934 "steppedLine": false, 3935 "targets": [ 3936 { 3937 "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", 3938 "intervalFactor": 1, 3939 "legendFormat": ".9", 3940 "refId": "A" 3941 }, 3942 { 3943 "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", 3944 "refId": "B" 3945 }, 3946 { 3947 "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", 3948 "refId": "C" 3949 } 3950 ], 3951 "thresholds": [ ], 3952 "timeFrom": null, 3953 "timeRegions": [ ], 3954 "timeShift": null, 3955 "title": "MutateRows Latency", 3956 "tooltip": { 3957 "shared": true, 3958 "sort": 2, 3959 "value_type": "individual" 3960 }, 3961 "type": "graph", 3962 "xaxis": { 3963 "buckets": null, 3964 "mode": "time", 3965 "name": null, 3966 "show": true, 3967 "values": [ ] 3968 }, 3969 "yaxes": [ 3970 { 3971 "format": "short", 3972 "label": null, 3973 "logBase": 1, 3974 "max": null, 3975 "min": null, 3976 "show": true 3977 }, 3978 { 3979 "format": "short", 3980 "label": null, 3981 "logBase": 1, 3982 "max": null, 3983 "min": null, 3984 "show": true 3985 } 3986 ], 3987 "yaxis": { 3988 "align": false, 3989 "alignLevel": null 3990 } 3991 }, 3992 { 3993 "aliasColors": { }, 3994 "bars": false, 3995 "dashLength": 10, 3996 "dashes": false, 3997 "datasource": "$datasource", 3998 "fill": 1, 3999 "fillGradient": 0, 4000 "gridPos": { 4001 "h": 7, 4002 "w": 6, 4003 "x": 6, 4004 "y": 9 4005 }, 4006 "hiddenSeries": false, 4007 "id": 46, 4008 "interval": "", 4009 "legend": { 4010 "avg": false, 4011 "current": false, 4012 "max": false, 4013 "min": false, 4014 "show": false, 4015 "total": false, 4016 "values": false 4017 }, 4018 "lines": true, 4019 "linewidth": 1, 4020 "nullPointMode": "null", 4021 "options": { 4022 "dataLinks": [ ] 4023 }, 4024 "panels": [ ], 4025 "percentage": false, 4026 "pointradius": 1, 4027 "points": false, 4028 "renderer": "flot", 4029 "seriesOverrides": [ ], 4030 "spaceLength": 10, 4031 "stack": false, 4032 "steppedLine": false, 4033 "targets": [ 4034 { 4035 "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", 4036 "interval": "", 4037 "intervalFactor": 1, 4038 "legendFormat": "99%", 4039 "refId": "A" 4040 }, 4041 { 4042 "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", 4043 "interval": "", 4044 "legendFormat": "90%", 4045 "refId": "B" 4046 }, 4047 { 4048 "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", 4049 "interval": "", 4050 "legendFormat": "50%", 4051 "refId": "C" 4052 } 4053 ], 4054 "thresholds": [ ], 4055 "timeFrom": null, 4056 "timeRegions": [ ], 4057 "timeShift": null, 4058 "title": "ReadRows Latency", 4059 "tooltip": { 4060 "shared": true, 4061 "sort": 2, 4062 "value_type": "individual" 4063 }, 4064 "type": "graph", 4065 "xaxis": { 4066 "buckets": null, 4067 "mode": "time", 4068 "name": null, 4069 "show": true, 4070 "values": [ ] 4071 }, 4072 "yaxes": [ 4073 { 4074 "format": "short", 4075 "label": null, 4076 "logBase": 1, 4077 "max": null, 4078 "min": null, 4079 "show": true 4080 }, 4081 { 4082 "format": "short", 4083 "label": null, 4084 "logBase": 1, 4085 "max": null, 4086 "min": null, 4087 "show": true 4088 } 4089 ], 4090 "yaxis": { 4091 "align": false, 4092 "alignLevel": null 4093 } 4094 }, 4095 { 4096 "aliasColors": { }, 4097 "bars": false, 4098 "dashLength": 10, 4099 "dashes": false, 4100 "datasource": "$datasource", 4101 "fill": 1, 4102 "fillGradient": 0, 4103 "gridPos": { 4104 "h": 7, 4105 "w": 6, 4106 "x": 12, 4107 "y": 9 4108 }, 4109 "hiddenSeries": false, 4110 "id": 44, 4111 "interval": "", 4112 "legend": { 4113 "avg": false, 4114 "current": false, 4115 "max": false, 4116 "min": false, 4117 "show": false, 4118 "total": false, 4119 "values": false 4120 }, 4121 "lines": true, 4122 "linewidth": 1, 4123 "nullPointMode": "null", 4124 "options": { 4125 "dataLinks": [ ] 4126 }, 4127 "panels": [ ], 4128 "percentage": false, 4129 "pointradius": 1, 4130 "points": false, 4131 "renderer": "flot", 4132 "seriesOverrides": [ ], 4133 "spaceLength": 10, 4134 "stack": false, 4135 "steppedLine": false, 4136 "targets": [ 4137 { 4138 "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", 4139 "interval": "", 4140 "intervalFactor": 1, 4141 "legendFormat": "99%", 4142 "refId": "A" 4143 }, 4144 { 4145 "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", 4146 "interval": "", 4147 "legendFormat": "90%", 4148 "refId": "B" 4149 }, 4150 { 4151 "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", 4152 "interval": "", 4153 "legendFormat": "50%", 4154 "refId": "C" 4155 } 4156 ], 4157 "thresholds": [ ], 4158 "timeFrom": null, 4159 "timeRegions": [ ], 4160 "timeShift": null, 4161 "title": "GetTable Latency", 4162 "tooltip": { 4163 "shared": true, 4164 "sort": 2, 4165 "value_type": "individual" 4166 }, 4167 "type": "graph", 4168 "xaxis": { 4169 "buckets": null, 4170 "mode": "time", 4171 "name": null, 4172 "show": true, 4173 "values": [ ] 4174 }, 4175 "yaxes": [ 4176 { 4177 "format": "short", 4178 "label": null, 4179 "logBase": 1, 4180 "max": null, 4181 "min": null, 4182 "show": true 4183 }, 4184 { 4185 "format": "short", 4186 "label": null, 4187 "logBase": 1, 4188 "max": null, 4189 "min": null, 4190 "show": true 4191 } 4192 ], 4193 "yaxis": { 4194 "align": false, 4195 "alignLevel": null 4196 } 4197 }, 4198 { 4199 "aliasColors": { }, 4200 "bars": false, 4201 "dashLength": 10, 4202 "dashes": false, 4203 "datasource": "$datasource", 4204 "fill": 1, 4205 "fillGradient": 0, 4206 "gridPos": { 4207 "h": 7, 4208 "w": 6, 4209 "x": 18, 4210 "y": 9 4211 }, 4212 "hiddenSeries": false, 4213 "id": 45, 4214 "interval": "", 4215 "legend": { 4216 "avg": false, 4217 "current": false, 4218 "max": false, 4219 "min": false, 4220 "show": false, 4221 "total": false, 4222 "values": false 4223 }, 4224 "lines": true, 4225 "linewidth": 1, 4226 "nullPointMode": "null", 4227 "options": { 4228 "dataLinks": [ ] 4229 }, 4230 "panels": [ ], 4231 "percentage": false, 4232 "pointradius": 1, 4233 "points": false, 4234 "renderer": "flot", 4235 "seriesOverrides": [ ], 4236 "spaceLength": 10, 4237 "stack": false, 4238 "steppedLine": false, 4239 "targets": [ 4240 { 4241 "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", 4242 "intervalFactor": 1, 4243 "legendFormat": ".9", 4244 "refId": "A" 4245 }, 4246 { 4247 "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", 4248 "refId": "B" 4249 }, 4250 { 4251 "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", 4252 "refId": "C" 4253 } 4254 ], 4255 "thresholds": [ ], 4256 "timeFrom": null, 4257 "timeRegions": [ ], 4258 "timeShift": null, 4259 "title": "ListTables Latency", 4260 "tooltip": { 4261 "shared": true, 4262 "sort": 2, 4263 "value_type": "individual" 4264 }, 4265 "type": "graph", 4266 "xaxis": { 4267 "buckets": null, 4268 "mode": "time", 4269 "name": null, 4270 "show": true, 4271 "values": [ ] 4272 }, 4273 "yaxes": [ 4274 { 4275 "format": "short", 4276 "label": null, 4277 "logBase": 1, 4278 "max": null, 4279 "min": null, 4280 "show": true 4281 }, 4282 { 4283 "format": "short", 4284 "label": null, 4285 "logBase": 1, 4286 "max": null, 4287 "min": null, 4288 "show": true 4289 } 4290 ], 4291 "yaxis": { 4292 "align": false, 4293 "alignLevel": null 4294 } 4295 }, 4296 { 4297 "aliasColors": { }, 4298 "bars": false, 4299 "dashLength": 10, 4300 "dashes": false, 4301 "datasource": "$datasource", 4302 "fill": 1, 4303 "fillGradient": 0, 4304 "gridPos": { 4305 "h": 7, 4306 "w": 6, 4307 "x": 0, 4308 "y": 16 4309 }, 4310 "hiddenSeries": false, 4311 "id": 47, 4312 "interval": "", 4313 "legend": { 4314 "avg": false, 4315 "current": false, 4316 "max": false, 4317 "min": false, 4318 "show": false, 4319 "total": false, 4320 "values": false 4321 }, 4322 "lines": true, 4323 "linewidth": 1, 4324 "nullPointMode": "null", 4325 "options": { 4326 "dataLinks": [ ] 4327 }, 4328 "panels": [ ], 4329 "percentage": false, 4330 "pointradius": 1, 4331 "points": false, 4332 "renderer": "flot", 4333 "seriesOverrides": [ ], 4334 "spaceLength": 10, 4335 "stack": false, 4336 "steppedLine": false, 4337 "targets": [ 4338 { 4339 "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (status_code)", 4340 "intervalFactor": 1, 4341 "legendFormat": "{{status_code}}", 4342 "refId": "A" 4343 } 4344 ], 4345 "thresholds": [ ], 4346 "timeFrom": null, 4347 "timeRegions": [ ], 4348 "timeShift": null, 4349 "title": "MutateRows Status", 4350 "tooltip": { 4351 "shared": true, 4352 "sort": 2, 4353 "value_type": "individual" 4354 }, 4355 "type": "graph", 4356 "xaxis": { 4357 "buckets": null, 4358 "mode": "time", 4359 "name": null, 4360 "show": true, 4361 "values": [ ] 4362 }, 4363 "yaxes": [ 4364 { 4365 "format": "short", 4366 "label": null, 4367 "logBase": 1, 4368 "max": null, 4369 "min": null, 4370 "show": true 4371 }, 4372 { 4373 "format": "short", 4374 "label": null, 4375 "logBase": 1, 4376 "max": null, 4377 "min": null, 4378 "show": true 4379 } 4380 ], 4381 "yaxis": { 4382 "align": false, 4383 "alignLevel": null 4384 } 4385 }, 4386 { 4387 "aliasColors": { }, 4388 "bars": false, 4389 "dashLength": 10, 4390 "dashes": false, 4391 "datasource": "$datasource", 4392 "fill": 1, 4393 "fillGradient": 0, 4394 "gridPos": { 4395 "h": 7, 4396 "w": 6, 4397 "x": 6, 4398 "y": 16 4399 }, 4400 "hiddenSeries": false, 4401 "id": 50, 4402 "interval": "", 4403 "legend": { 4404 "avg": false, 4405 "current": false, 4406 "max": false, 4407 "min": false, 4408 "show": false, 4409 "total": false, 4410 "values": false 4411 }, 4412 "lines": true, 4413 "linewidth": 1, 4414 "nullPointMode": "null", 4415 "options": { 4416 "dataLinks": [ ] 4417 }, 4418 "panels": [ ], 4419 "percentage": false, 4420 "pointradius": 1, 4421 "points": false, 4422 "renderer": "flot", 4423 "seriesOverrides": [ ], 4424 "spaceLength": 10, 4425 "stack": false, 4426 "steppedLine": false, 4427 "targets": [ 4428 { 4429 "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (status_code)", 4430 "intervalFactor": 1, 4431 "legendFormat": "{{status_code}}", 4432 "refId": "A" 4433 } 4434 ], 4435 "thresholds": [ ], 4436 "timeFrom": null, 4437 "timeRegions": [ ], 4438 "timeShift": null, 4439 "title": "ReadRows Status", 4440 "tooltip": { 4441 "shared": true, 4442 "sort": 2, 4443 "value_type": "individual" 4444 }, 4445 "type": "graph", 4446 "xaxis": { 4447 "buckets": null, 4448 "mode": "time", 4449 "name": null, 4450 "show": true, 4451 "values": [ ] 4452 }, 4453 "yaxes": [ 4454 { 4455 "format": "short", 4456 "label": null, 4457 "logBase": 1, 4458 "max": null, 4459 "min": null, 4460 "show": true 4461 }, 4462 { 4463 "format": "short", 4464 "label": null, 4465 "logBase": 1, 4466 "max": null, 4467 "min": null, 4468 "show": true 4469 } 4470 ], 4471 "yaxis": { 4472 "align": false, 4473 "alignLevel": null 4474 } 4475 }, 4476 { 4477 "aliasColors": { }, 4478 "bars": false, 4479 "dashLength": 10, 4480 "dashes": false, 4481 "datasource": "$datasource", 4482 "fill": 1, 4483 "fillGradient": 0, 4484 "gridPos": { 4485 "h": 7, 4486 "w": 6, 4487 "x": 12, 4488 "y": 16 4489 }, 4490 "hiddenSeries": false, 4491 "id": 48, 4492 "interval": "", 4493 "legend": { 4494 "avg": false, 4495 "current": false, 4496 "max": false, 4497 "min": false, 4498 "show": false, 4499 "total": false, 4500 "values": false 4501 }, 4502 "lines": true, 4503 "linewidth": 1, 4504 "nullPointMode": "null", 4505 "options": { 4506 "dataLinks": [ ] 4507 }, 4508 "panels": [ ], 4509 "percentage": false, 4510 "pointradius": 1, 4511 "points": false, 4512 "renderer": "flot", 4513 "seriesOverrides": [ ], 4514 "spaceLength": 10, 4515 "stack": false, 4516 "steppedLine": false, 4517 "targets": [ 4518 { 4519 "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (status_code)", 4520 "intervalFactor": 1, 4521 "legendFormat": "{{status_code}}", 4522 "refId": "A" 4523 } 4524 ], 4525 "thresholds": [ ], 4526 "timeFrom": null, 4527 "timeRegions": [ ], 4528 "timeShift": null, 4529 "title": "GetTable Status", 4530 "tooltip": { 4531 "shared": true, 4532 "sort": 2, 4533 "value_type": "individual" 4534 }, 4535 "type": "graph", 4536 "xaxis": { 4537 "buckets": null, 4538 "mode": "time", 4539 "name": null, 4540 "show": true, 4541 "values": [ ] 4542 }, 4543 "yaxes": [ 4544 { 4545 "format": "short", 4546 "label": null, 4547 "logBase": 1, 4548 "max": null, 4549 "min": null, 4550 "show": true 4551 }, 4552 { 4553 "format": "short", 4554 "label": null, 4555 "logBase": 1, 4556 "max": null, 4557 "min": null, 4558 "show": true 4559 } 4560 ], 4561 "yaxis": { 4562 "align": false, 4563 "alignLevel": null 4564 } 4565 }, 4566 { 4567 "aliasColors": { }, 4568 "bars": false, 4569 "dashLength": 10, 4570 "dashes": false, 4571 "datasource": "$datasource", 4572 "fill": 1, 4573 "fillGradient": 0, 4574 "gridPos": { 4575 "h": 7, 4576 "w": 6, 4577 "x": 18, 4578 "y": 16 4579 }, 4580 "hiddenSeries": false, 4581 "id": 49, 4582 "interval": "", 4583 "legend": { 4584 "avg": false, 4585 "current": false, 4586 "max": false, 4587 "min": false, 4588 "show": false, 4589 "total": false, 4590 "values": false 4591 }, 4592 "lines": true, 4593 "linewidth": 1, 4594 "nullPointMode": "null", 4595 "options": { 4596 "dataLinks": [ ] 4597 }, 4598 "panels": [ ], 4599 "percentage": false, 4600 "pointradius": 1, 4601 "points": false, 4602 "renderer": "flot", 4603 "seriesOverrides": [ ], 4604 "spaceLength": 10, 4605 "stack": false, 4606 "steppedLine": false, 4607 "targets": [ 4608 { 4609 "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (status_code)", 4610 "intervalFactor": 1, 4611 "legendFormat": "{{status_code}}", 4612 "refId": "A" 4613 } 4614 ], 4615 "thresholds": [ ], 4616 "timeFrom": null, 4617 "timeRegions": [ ], 4618 "timeShift": null, 4619 "title": "ListTables Status", 4620 "tooltip": { 4621 "shared": true, 4622 "sort": 2, 4623 "value_type": "individual" 4624 }, 4625 "type": "graph", 4626 "xaxis": { 4627 "buckets": null, 4628 "mode": "time", 4629 "name": null, 4630 "show": true, 4631 "values": [ ] 4632 }, 4633 "yaxes": [ 4634 { 4635 "format": "short", 4636 "label": null, 4637 "logBase": 1, 4638 "max": null, 4639 "min": null, 4640 "show": true 4641 }, 4642 { 4643 "format": "short", 4644 "label": null, 4645 "logBase": 1, 4646 "max": null, 4647 "min": null, 4648 "show": true 4649 } 4650 ], 4651 "yaxis": { 4652 "align": false, 4653 "alignLevel": null 4654 } 4655 } 4656 ], 4657 "targets": [ ], 4658 "title": "Big Table", 4659 "type": "row" 4660 }, 4661 { 4662 "collapsed": true, 4663 "datasource": null, 4664 "gridPos": { 4665 "h": 1, 4666 "w": 24, 4667 "x": 0, 4668 "y": 35 4669 }, 4670 "id": 60, 4671 "panels": [ 4672 { 4673 "aliasColors": { }, 4674 "bars": false, 4675 "dashLength": 10, 4676 "dashes": false, 4677 "datasource": "$datasource", 4678 "fill": 1, 4679 "fillGradient": 0, 4680 "gridPos": { 4681 "h": 8, 4682 "w": 24, 4683 "x": 0, 4684 "y": 33 4685 }, 4686 "hiddenSeries": false, 4687 "id": 61, 4688 "interval": "", 4689 "legend": { 4690 "alignAsTable": true, 4691 "avg": false, 4692 "current": false, 4693 "max": false, 4694 "min": false, 4695 "rightSide": true, 4696 "show": true, 4697 "total": false, 4698 "values": false 4699 }, 4700 "lines": true, 4701 "linewidth": 1, 4702 "nullPointMode": "null", 4703 "options": { 4704 "dataLinks": [ ] 4705 }, 4706 "panels": [ ], 4707 "percentage": false, 4708 "pointradius": 1, 4709 "points": false, 4710 "renderer": "flot", 4711 "seriesOverrides": [ ], 4712 "spaceLength": 10, 4713 "stack": false, 4714 "steppedLine": false, 4715 "targets": [ 4716 { 4717 "expr": "histogram_quantile(.99, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 4718 "intervalFactor": 1, 4719 "legendFormat": ".99-{{operation}}", 4720 "refId": "A" 4721 }, 4722 { 4723 "expr": "histogram_quantile(.9, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 4724 "hide": false, 4725 "legendFormat": ".9-{{operation}}", 4726 "refId": "B" 4727 }, 4728 { 4729 "expr": "histogram_quantile(.5, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 4730 "hide": false, 4731 "legendFormat": ".5-{{operation}}", 4732 "refId": "C" 4733 } 4734 ], 4735 "thresholds": [ ], 4736 "timeFrom": null, 4737 "timeRegions": [ ], 4738 "timeShift": null, 4739 "title": "Latency By Operation", 4740 "tooltip": { 4741 "shared": true, 4742 "sort": 2, 4743 "value_type": "individual" 4744 }, 4745 "type": "graph", 4746 "xaxis": { 4747 "buckets": null, 4748 "mode": "time", 4749 "name": null, 4750 "show": true, 4751 "values": [ ] 4752 }, 4753 "yaxes": [ 4754 { 4755 "format": "short", 4756 "label": null, 4757 "logBase": 1, 4758 "max": null, 4759 "min": null, 4760 "show": true 4761 }, 4762 { 4763 "format": "short", 4764 "label": null, 4765 "logBase": 1, 4766 "max": null, 4767 "min": null, 4768 "show": true 4769 } 4770 ], 4771 "yaxis": { 4772 "align": false, 4773 "alignLevel": null 4774 } 4775 }, 4776 { 4777 "aliasColors": { }, 4778 "bars": false, 4779 "dashLength": 10, 4780 "dashes": false, 4781 "datasource": "$datasource", 4782 "fill": 1, 4783 "fillGradient": 0, 4784 "gridPos": { 4785 "h": 8, 4786 "w": 24, 4787 "x": 0, 4788 "y": 41 4789 }, 4790 "hiddenSeries": false, 4791 "id": 62, 4792 "interval": "", 4793 "legend": { 4794 "alignAsTable": true, 4795 "avg": false, 4796 "current": false, 4797 "max": false, 4798 "min": false, 4799 "rightSide": true, 4800 "show": true, 4801 "total": false, 4802 "values": false 4803 }, 4804 "lines": true, 4805 "linewidth": 1, 4806 "nullPointMode": "null", 4807 "options": { 4808 "dataLinks": [ ] 4809 }, 4810 "panels": [ ], 4811 "percentage": false, 4812 "pointradius": 1, 4813 "points": false, 4814 "renderer": "flot", 4815 "seriesOverrides": [ ], 4816 "spaceLength": 10, 4817 "stack": false, 4818 "steppedLine": false, 4819 "targets": [ 4820 { 4821 "expr": "sum(rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", 4822 "intervalFactor": 1, 4823 "legendFormat": "{{status_code}}-{{operation}}", 4824 "refId": "A" 4825 } 4826 ], 4827 "thresholds": [ ], 4828 "timeFrom": null, 4829 "timeRegions": [ ], 4830 "timeShift": null, 4831 "title": "Status By Method", 4832 "tooltip": { 4833 "shared": true, 4834 "sort": 2, 4835 "value_type": "individual" 4836 }, 4837 "type": "graph", 4838 "xaxis": { 4839 "buckets": null, 4840 "mode": "time", 4841 "name": null, 4842 "show": true, 4843 "values": [ ] 4844 }, 4845 "yaxes": [ 4846 { 4847 "format": "short", 4848 "label": null, 4849 "logBase": 1, 4850 "max": null, 4851 "min": null, 4852 "show": true 4853 }, 4854 { 4855 "format": "short", 4856 "label": null, 4857 "logBase": 1, 4858 "max": null, 4859 "min": null, 4860 "show": true 4861 } 4862 ], 4863 "yaxis": { 4864 "align": false, 4865 "alignLevel": null 4866 } 4867 } 4868 ], 4869 "targets": [ ], 4870 "title": "GCS", 4871 "type": "row" 4872 }, 4873 { 4874 "collapsed": true, 4875 "datasource": null, 4876 "gridPos": { 4877 "h": 1, 4878 "w": 24, 4879 "x": 0, 4880 "y": 36 4881 }, 4882 "id": 76, 4883 "panels": [ 4884 { 4885 "aliasColors": { }, 4886 "bars": false, 4887 "dashLength": 10, 4888 "dashes": false, 4889 "datasource": null, 4890 "fill": 1, 4891 "fillGradient": 0, 4892 "gridPos": { 4893 "h": 6, 4894 "w": 6, 4895 "x": 0, 4896 "y": 9 4897 }, 4898 "id": 82, 4899 "legend": { 4900 "avg": false, 4901 "current": false, 4902 "max": false, 4903 "min": false, 4904 "show": true, 4905 "total": false, 4906 "values": false 4907 }, 4908 "lines": true, 4909 "linewidth": 1, 4910 "nullPointMode": "null", 4911 "options": { 4912 "dataLinks": [ ] 4913 }, 4914 "panels": [ ], 4915 "percentage": false, 4916 "pointradius": 2, 4917 "points": false, 4918 "renderer": "flot", 4919 "seriesOverrides": [ ], 4920 "spaceLength": 10, 4921 "stack": false, 4922 "steppedLine": false, 4923 "targets": [ 4924 { 4925 "expr": "sum(rate(cortex_dynamo_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", 4926 "refId": "A" 4927 } 4928 ], 4929 "thresholds": [ ], 4930 "timeFrom": null, 4931 "timeRegions": [ ], 4932 "timeShift": null, 4933 "title": "Failure Rate", 4934 "tooltip": { 4935 "shared": true, 4936 "sort": 0, 4937 "value_type": "individual" 4938 }, 4939 "type": "graph", 4940 "xaxis": { 4941 "buckets": null, 4942 "mode": "time", 4943 "name": null, 4944 "show": true, 4945 "values": [ ] 4946 }, 4947 "yaxes": [ 4948 { 4949 "format": "short", 4950 "label": null, 4951 "logBase": 1, 4952 "max": null, 4953 "min": null, 4954 "show": true 4955 }, 4956 { 4957 "format": "short", 4958 "label": null, 4959 "logBase": 1, 4960 "max": null, 4961 "min": null, 4962 "show": true 4963 } 4964 ], 4965 "yaxis": { 4966 "align": false, 4967 "alignLevel": null 4968 } 4969 }, 4970 { 4971 "aliasColors": { }, 4972 "bars": false, 4973 "dashLength": 10, 4974 "dashes": false, 4975 "datasource": null, 4976 "fill": 1, 4977 "fillGradient": 0, 4978 "gridPos": { 4979 "h": 6, 4980 "w": 6, 4981 "x": 6, 4982 "y": 9 4983 }, 4984 "id": 83, 4985 "legend": { 4986 "avg": false, 4987 "current": false, 4988 "max": false, 4989 "min": false, 4990 "show": true, 4991 "total": false, 4992 "values": false 4993 }, 4994 "lines": true, 4995 "linewidth": 1, 4996 "nullPointMode": "null", 4997 "options": { 4998 "dataLinks": [ ] 4999 }, 5000 "panels": [ ], 5001 "percentage": false, 5002 "pointradius": 2, 5003 "points": false, 5004 "renderer": "flot", 5005 "seriesOverrides": [ ], 5006 "spaceLength": 10, 5007 "stack": false, 5008 "steppedLine": false, 5009 "targets": [ 5010 { 5011 "expr": "sum(rate(cortex_dynamo_consumed_capacity_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", 5012 "refId": "A" 5013 } 5014 ], 5015 "thresholds": [ ], 5016 "timeFrom": null, 5017 "timeRegions": [ ], 5018 "timeShift": null, 5019 "title": "Consumed Capacity Rate", 5020 "tooltip": { 5021 "shared": true, 5022 "sort": 0, 5023 "value_type": "individual" 5024 }, 5025 "type": "graph", 5026 "xaxis": { 5027 "buckets": null, 5028 "mode": "time", 5029 "name": null, 5030 "show": true, 5031 "values": [ ] 5032 }, 5033 "yaxes": [ 5034 { 5035 "format": "short", 5036 "label": null, 5037 "logBase": 1, 5038 "max": null, 5039 "min": null, 5040 "show": true 5041 }, 5042 { 5043 "format": "short", 5044 "label": null, 5045 "logBase": 1, 5046 "max": null, 5047 "min": null, 5048 "show": true 5049 } 5050 ], 5051 "yaxis": { 5052 "align": false, 5053 "alignLevel": null 5054 } 5055 }, 5056 { 5057 "aliasColors": { }, 5058 "bars": false, 5059 "dashLength": 10, 5060 "dashes": false, 5061 "datasource": null, 5062 "fill": 1, 5063 "fillGradient": 0, 5064 "gridPos": { 5065 "h": 6, 5066 "w": 6, 5067 "x": 12, 5068 "y": 9 5069 }, 5070 "id": 84, 5071 "legend": { 5072 "avg": false, 5073 "current": false, 5074 "max": false, 5075 "min": false, 5076 "show": true, 5077 "total": false, 5078 "values": false 5079 }, 5080 "lines": true, 5081 "linewidth": 1, 5082 "nullPointMode": "null", 5083 "options": { 5084 "dataLinks": [ ] 5085 }, 5086 "panels": [ ], 5087 "percentage": false, 5088 "pointradius": 2, 5089 "points": false, 5090 "renderer": "flot", 5091 "seriesOverrides": [ ], 5092 "spaceLength": 10, 5093 "stack": false, 5094 "steppedLine": false, 5095 "targets": [ 5096 { 5097 "expr": "sum(rate(cortex_dynamo_throttled_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", 5098 "refId": "A" 5099 } 5100 ], 5101 "thresholds": [ ], 5102 "timeFrom": null, 5103 "timeRegions": [ ], 5104 "timeShift": null, 5105 "title": "Throttled Rate", 5106 "tooltip": { 5107 "shared": true, 5108 "sort": 0, 5109 "value_type": "individual" 5110 }, 5111 "type": "graph", 5112 "xaxis": { 5113 "buckets": null, 5114 "mode": "time", 5115 "name": null, 5116 "show": true, 5117 "values": [ ] 5118 }, 5119 "yaxes": [ 5120 { 5121 "format": "short", 5122 "label": null, 5123 "logBase": 1, 5124 "max": null, 5125 "min": null, 5126 "show": true 5127 }, 5128 { 5129 "format": "short", 5130 "label": null, 5131 "logBase": 1, 5132 "max": null, 5133 "min": null, 5134 "show": true 5135 } 5136 ], 5137 "yaxis": { 5138 "align": false, 5139 "alignLevel": null 5140 } 5141 }, 5142 { 5143 "aliasColors": { }, 5144 "bars": false, 5145 "dashLength": 10, 5146 "dashes": false, 5147 "datasource": null, 5148 "fill": 1, 5149 "fillGradient": 0, 5150 "gridPos": { 5151 "h": 6, 5152 "w": 6, 5153 "x": 18, 5154 "y": 9 5155 }, 5156 "id": 85, 5157 "legend": { 5158 "avg": false, 5159 "current": false, 5160 "max": false, 5161 "min": false, 5162 "show": true, 5163 "total": false, 5164 "values": false 5165 }, 5166 "lines": true, 5167 "linewidth": 1, 5168 "nullPointMode": "null", 5169 "options": { 5170 "dataLinks": [ ] 5171 }, 5172 "panels": [ ], 5173 "percentage": false, 5174 "pointradius": 2, 5175 "points": false, 5176 "renderer": "flot", 5177 "seriesOverrides": [ ], 5178 "spaceLength": 10, 5179 "stack": false, 5180 "steppedLine": false, 5181 "targets": [ 5182 { 5183 "expr": "sum(rate(cortex_dynamo_dropped_requests_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", 5184 "refId": "A" 5185 } 5186 ], 5187 "thresholds": [ ], 5188 "timeFrom": null, 5189 "timeRegions": [ ], 5190 "timeShift": null, 5191 "title": "Dropped Rate", 5192 "tooltip": { 5193 "shared": true, 5194 "sort": 0, 5195 "value_type": "individual" 5196 }, 5197 "type": "graph", 5198 "xaxis": { 5199 "buckets": null, 5200 "mode": "time", 5201 "name": null, 5202 "show": true, 5203 "values": [ ] 5204 }, 5205 "yaxes": [ 5206 { 5207 "format": "short", 5208 "label": null, 5209 "logBase": 1, 5210 "max": null, 5211 "min": null, 5212 "show": true 5213 }, 5214 { 5215 "format": "short", 5216 "label": null, 5217 "logBase": 1, 5218 "max": null, 5219 "min": null, 5220 "show": true 5221 } 5222 ], 5223 "yaxis": { 5224 "align": false, 5225 "alignLevel": null 5226 } 5227 }, 5228 { 5229 "aliasColors": { }, 5230 "bars": false, 5231 "dashLength": 10, 5232 "dashes": false, 5233 "datasource": null, 5234 "fill": 1, 5235 "fillGradient": 0, 5236 "gridPos": { 5237 "h": 6, 5238 "w": 6, 5239 "x": 0, 5240 "y": 15 5241 }, 5242 "id": 86, 5243 "legend": { 5244 "avg": false, 5245 "current": false, 5246 "max": false, 5247 "min": false, 5248 "show": true, 5249 "total": false, 5250 "values": false 5251 }, 5252 "lines": true, 5253 "linewidth": 1, 5254 "nullPointMode": "null", 5255 "options": { 5256 "dataLinks": [ ] 5257 }, 5258 "panels": [ ], 5259 "percentage": false, 5260 "pointradius": 2, 5261 "points": false, 5262 "renderer": "flot", 5263 "seriesOverrides": [ ], 5264 "spaceLength": 10, 5265 "stack": false, 5266 "steppedLine": false, 5267 "targets": [ 5268 { 5269 "expr": "histogram_quantile(.99, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", 5270 "legendFormat": ".99", 5271 "refId": "A" 5272 }, 5273 { 5274 "expr": "histogram_quantile(.9, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", 5275 "legendFormat": ".9", 5276 "refId": "B" 5277 }, 5278 { 5279 "expr": "histogram_quantile(.5, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", 5280 "legendFormat": ".5", 5281 "refId": "C" 5282 } 5283 ], 5284 "thresholds": [ ], 5285 "timeFrom": null, 5286 "timeRegions": [ ], 5287 "timeShift": null, 5288 "title": "Query Pages", 5289 "tooltip": { 5290 "shared": true, 5291 "sort": 0, 5292 "value_type": "individual" 5293 }, 5294 "type": "graph", 5295 "xaxis": { 5296 "buckets": null, 5297 "mode": "time", 5298 "name": null, 5299 "show": true, 5300 "values": [ ] 5301 }, 5302 "yaxes": [ 5303 { 5304 "format": "short", 5305 "label": null, 5306 "logBase": 1, 5307 "max": null, 5308 "min": null, 5309 "show": true 5310 }, 5311 { 5312 "format": "short", 5313 "label": null, 5314 "logBase": 1, 5315 "max": null, 5316 "min": null, 5317 "show": true 5318 } 5319 ], 5320 "yaxis": { 5321 "align": false, 5322 "alignLevel": null 5323 } 5324 }, 5325 { 5326 "aliasColors": { }, 5327 "bars": false, 5328 "dashLength": 10, 5329 "dashes": false, 5330 "datasource": "$datasource", 5331 "fill": 1, 5332 "fillGradient": 0, 5333 "gridPos": { 5334 "h": 6, 5335 "w": 9, 5336 "x": 6, 5337 "y": 15 5338 }, 5339 "id": 87, 5340 "interval": "", 5341 "legend": { 5342 "alignAsTable": true, 5343 "avg": false, 5344 "current": false, 5345 "max": false, 5346 "min": false, 5347 "rightSide": true, 5348 "show": true, 5349 "total": false, 5350 "values": false 5351 }, 5352 "lines": true, 5353 "linewidth": 1, 5354 "nullPointMode": "null", 5355 "options": { 5356 "dataLinks": [ ] 5357 }, 5358 "panels": [ ], 5359 "percentage": false, 5360 "pointradius": 1, 5361 "points": false, 5362 "renderer": "flot", 5363 "seriesOverrides": [ ], 5364 "spaceLength": 10, 5365 "stack": false, 5366 "steppedLine": false, 5367 "targets": [ 5368 { 5369 "expr": "histogram_quantile(.99, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5370 "intervalFactor": 1, 5371 "legendFormat": ".99-{{operation}}", 5372 "refId": "A" 5373 }, 5374 { 5375 "expr": "histogram_quantile(.9, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5376 "hide": false, 5377 "legendFormat": ".9-{{operation}}", 5378 "refId": "B" 5379 }, 5380 { 5381 "expr": "histogram_quantile(.5, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5382 "hide": false, 5383 "legendFormat": ".5-{{operation}}", 5384 "refId": "C" 5385 } 5386 ], 5387 "thresholds": [ ], 5388 "timeFrom": null, 5389 "timeRegions": [ ], 5390 "timeShift": null, 5391 "title": "Latency By Operation", 5392 "tooltip": { 5393 "shared": true, 5394 "sort": 2, 5395 "value_type": "individual" 5396 }, 5397 "type": "graph", 5398 "xaxis": { 5399 "buckets": null, 5400 "mode": "time", 5401 "name": null, 5402 "show": true, 5403 "values": [ ] 5404 }, 5405 "yaxes": [ 5406 { 5407 "format": "short", 5408 "label": null, 5409 "logBase": 1, 5410 "max": null, 5411 "min": null, 5412 "show": true 5413 }, 5414 { 5415 "format": "short", 5416 "label": null, 5417 "logBase": 1, 5418 "max": null, 5419 "min": null, 5420 "show": true 5421 } 5422 ], 5423 "yaxis": { 5424 "align": false, 5425 "alignLevel": null 5426 } 5427 }, 5428 { 5429 "aliasColors": { }, 5430 "bars": false, 5431 "dashLength": 10, 5432 "dashes": false, 5433 "datasource": "$datasource", 5434 "fill": 1, 5435 "fillGradient": 0, 5436 "gridPos": { 5437 "h": 6, 5438 "w": 9, 5439 "x": 15, 5440 "y": 15 5441 }, 5442 "id": 88, 5443 "interval": "", 5444 "legend": { 5445 "alignAsTable": true, 5446 "avg": false, 5447 "current": false, 5448 "max": false, 5449 "min": false, 5450 "rightSide": true, 5451 "show": true, 5452 "total": false, 5453 "values": false 5454 }, 5455 "lines": true, 5456 "linewidth": 1, 5457 "nullPointMode": "null", 5458 "options": { 5459 "dataLinks": [ ] 5460 }, 5461 "panels": [ ], 5462 "percentage": false, 5463 "pointradius": 1, 5464 "points": false, 5465 "renderer": "flot", 5466 "seriesOverrides": [ ], 5467 "spaceLength": 10, 5468 "stack": false, 5469 "steppedLine": false, 5470 "targets": [ 5471 { 5472 "expr": "sum(rate(cortex_dynamo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", 5473 "intervalFactor": 1, 5474 "legendFormat": "{{status_code}}-{{operation}}", 5475 "refId": "A" 5476 } 5477 ], 5478 "thresholds": [ ], 5479 "timeFrom": null, 5480 "timeRegions": [ ], 5481 "timeShift": null, 5482 "title": "Status By Method", 5483 "tooltip": { 5484 "shared": true, 5485 "sort": 2, 5486 "value_type": "individual" 5487 }, 5488 "type": "graph", 5489 "xaxis": { 5490 "buckets": null, 5491 "mode": "time", 5492 "name": null, 5493 "show": true, 5494 "values": [ ] 5495 }, 5496 "yaxes": [ 5497 { 5498 "format": "short", 5499 "label": null, 5500 "logBase": 1, 5501 "max": null, 5502 "min": null, 5503 "show": true 5504 }, 5505 { 5506 "format": "short", 5507 "label": null, 5508 "logBase": 1, 5509 "max": null, 5510 "min": null, 5511 "show": true 5512 } 5513 ], 5514 "yaxis": { 5515 "align": false, 5516 "alignLevel": null 5517 } 5518 } 5519 ], 5520 "targets": [ ], 5521 "title": "Dynamo", 5522 "type": "row" 5523 }, 5524 { 5525 "collapsed": true, 5526 "datasource": null, 5527 "gridPos": { 5528 "h": 1, 5529 "w": 24, 5530 "x": 0, 5531 "y": 37 5532 }, 5533 "id": 78, 5534 "panels": [ 5535 { 5536 "aliasColors": { }, 5537 "bars": false, 5538 "dashLength": 10, 5539 "dashes": false, 5540 "datasource": "$datasource", 5541 "fill": 1, 5542 "fillGradient": 0, 5543 "gridPos": { 5544 "h": 8, 5545 "w": 24, 5546 "x": 0, 5547 "y": 10 5548 }, 5549 "id": 79, 5550 "interval": "", 5551 "legend": { 5552 "alignAsTable": true, 5553 "avg": false, 5554 "current": false, 5555 "max": false, 5556 "min": false, 5557 "rightSide": true, 5558 "show": true, 5559 "total": false, 5560 "values": false 5561 }, 5562 "lines": true, 5563 "linewidth": 1, 5564 "nullPointMode": "null", 5565 "options": { 5566 "dataLinks": [ ] 5567 }, 5568 "panels": [ ], 5569 "percentage": false, 5570 "pointradius": 1, 5571 "points": false, 5572 "renderer": "flot", 5573 "seriesOverrides": [ ], 5574 "spaceLength": 10, 5575 "stack": false, 5576 "steppedLine": false, 5577 "targets": [ 5578 { 5579 "expr": "histogram_quantile(.99, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5580 "intervalFactor": 1, 5581 "legendFormat": ".99-{{operation}}", 5582 "refId": "A" 5583 }, 5584 { 5585 "expr": "histogram_quantile(.9, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5586 "hide": false, 5587 "legendFormat": ".9-{{operation}}", 5588 "refId": "B" 5589 }, 5590 { 5591 "expr": "histogram_quantile(.5, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5592 "hide": false, 5593 "legendFormat": ".5-{{operation}}", 5594 "refId": "C" 5595 } 5596 ], 5597 "thresholds": [ ], 5598 "timeFrom": null, 5599 "timeRegions": [ ], 5600 "timeShift": null, 5601 "title": "Latency By Operation", 5602 "tooltip": { 5603 "shared": true, 5604 "sort": 2, 5605 "value_type": "individual" 5606 }, 5607 "type": "graph", 5608 "xaxis": { 5609 "buckets": null, 5610 "mode": "time", 5611 "name": null, 5612 "show": true, 5613 "values": [ ] 5614 }, 5615 "yaxes": [ 5616 { 5617 "format": "short", 5618 "label": null, 5619 "logBase": 1, 5620 "max": null, 5621 "min": null, 5622 "show": true 5623 }, 5624 { 5625 "format": "short", 5626 "label": null, 5627 "logBase": 1, 5628 "max": null, 5629 "min": null, 5630 "show": true 5631 } 5632 ], 5633 "yaxis": { 5634 "align": false, 5635 "alignLevel": null 5636 } 5637 }, 5638 { 5639 "aliasColors": { }, 5640 "bars": false, 5641 "dashLength": 10, 5642 "dashes": false, 5643 "datasource": "$datasource", 5644 "fill": 1, 5645 "fillGradient": 0, 5646 "gridPos": { 5647 "h": 8, 5648 "w": 24, 5649 "x": 0, 5650 "y": 18 5651 }, 5652 "id": 80, 5653 "interval": "", 5654 "legend": { 5655 "alignAsTable": true, 5656 "avg": false, 5657 "current": false, 5658 "max": false, 5659 "min": false, 5660 "rightSide": true, 5661 "show": true, 5662 "total": false, 5663 "values": false 5664 }, 5665 "lines": true, 5666 "linewidth": 1, 5667 "nullPointMode": "null", 5668 "options": { 5669 "dataLinks": [ ] 5670 }, 5671 "panels": [ ], 5672 "percentage": false, 5673 "pointradius": 1, 5674 "points": false, 5675 "renderer": "flot", 5676 "seriesOverrides": [ ], 5677 "spaceLength": 10, 5678 "stack": false, 5679 "steppedLine": false, 5680 "targets": [ 5681 { 5682 "expr": "sum(rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", 5683 "intervalFactor": 1, 5684 "legendFormat": "{{status_code}}-{{operation}}", 5685 "refId": "A" 5686 } 5687 ], 5688 "thresholds": [ ], 5689 "timeFrom": null, 5690 "timeRegions": [ ], 5691 "timeShift": null, 5692 "title": "Status By Method", 5693 "tooltip": { 5694 "shared": true, 5695 "sort": 2, 5696 "value_type": "individual" 5697 }, 5698 "type": "graph", 5699 "xaxis": { 5700 "buckets": null, 5701 "mode": "time", 5702 "name": null, 5703 "show": true, 5704 "values": [ ] 5705 }, 5706 "yaxes": [ 5707 { 5708 "format": "short", 5709 "label": null, 5710 "logBase": 1, 5711 "max": null, 5712 "min": null, 5713 "show": true 5714 }, 5715 { 5716 "format": "short", 5717 "label": null, 5718 "logBase": 1, 5719 "max": null, 5720 "min": null, 5721 "show": true 5722 } 5723 ], 5724 "yaxis": { 5725 "align": false, 5726 "alignLevel": null 5727 } 5728 } 5729 ], 5730 "targets": [ ], 5731 "title": "S3", 5732 "type": "row" 5733 }, 5734 { 5735 "collapsed": true, 5736 "datasource": null, 5737 "gridPos": { 5738 "h": 1, 5739 "w": 24, 5740 "x": 0, 5741 "y": 37 5742 }, 5743 "id": 78, 5744 "panels": [ 5745 { 5746 "aliasColors": { }, 5747 "bars": false, 5748 "dashLength": 10, 5749 "dashes": false, 5750 "datasource": "$datasource", 5751 "fill": 1, 5752 "fillGradient": 0, 5753 "gridPos": { 5754 "h": 8, 5755 "w": 24, 5756 "x": 0, 5757 "y": 10 5758 }, 5759 "id": 79, 5760 "interval": "", 5761 "legend": { 5762 "alignAsTable": true, 5763 "avg": false, 5764 "current": false, 5765 "max": false, 5766 "min": false, 5767 "rightSide": true, 5768 "show": true, 5769 "total": false, 5770 "values": false 5771 }, 5772 "lines": true, 5773 "linewidth": 1, 5774 "nullPointMode": "null", 5775 "options": { 5776 "dataLinks": [ ] 5777 }, 5778 "panels": [ ], 5779 "percentage": false, 5780 "pointradius": 1, 5781 "points": false, 5782 "renderer": "flot", 5783 "seriesOverrides": [ ], 5784 "spaceLength": 10, 5785 "stack": false, 5786 "steppedLine": false, 5787 "targets": [ 5788 { 5789 "expr": "histogram_quantile(.99, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5790 "intervalFactor": 1, 5791 "legendFormat": ".99-{{operation}}", 5792 "refId": "A" 5793 }, 5794 { 5795 "expr": "histogram_quantile(.9, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5796 "hide": false, 5797 "legendFormat": ".9-{{operation}}", 5798 "refId": "B" 5799 }, 5800 { 5801 "expr": "histogram_quantile(.5, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5802 "hide": false, 5803 "legendFormat": ".5-{{operation}}", 5804 "refId": "C" 5805 } 5806 ], 5807 "thresholds": [ ], 5808 "timeFrom": null, 5809 "timeRegions": [ ], 5810 "timeShift": null, 5811 "title": "Latency By Operation", 5812 "tooltip": { 5813 "shared": true, 5814 "sort": 2, 5815 "value_type": "individual" 5816 }, 5817 "type": "graph", 5818 "xaxis": { 5819 "buckets": null, 5820 "mode": "time", 5821 "name": null, 5822 "show": true, 5823 "values": [ ] 5824 }, 5825 "yaxes": [ 5826 { 5827 "format": "short", 5828 "label": null, 5829 "logBase": 1, 5830 "max": null, 5831 "min": null, 5832 "show": true 5833 }, 5834 { 5835 "format": "short", 5836 "label": null, 5837 "logBase": 1, 5838 "max": null, 5839 "min": null, 5840 "show": true 5841 } 5842 ], 5843 "yaxis": { 5844 "align": false, 5845 "alignLevel": null 5846 } 5847 }, 5848 { 5849 "aliasColors": { }, 5850 "bars": false, 5851 "dashLength": 10, 5852 "dashes": false, 5853 "datasource": "$datasource", 5854 "fill": 1, 5855 "fillGradient": 0, 5856 "gridPos": { 5857 "h": 8, 5858 "w": 24, 5859 "x": 0, 5860 "y": 18 5861 }, 5862 "id": 80, 5863 "interval": "", 5864 "legend": { 5865 "alignAsTable": true, 5866 "avg": false, 5867 "current": false, 5868 "max": false, 5869 "min": false, 5870 "rightSide": true, 5871 "show": true, 5872 "total": false, 5873 "values": false 5874 }, 5875 "lines": true, 5876 "linewidth": 1, 5877 "nullPointMode": "null", 5878 "options": { 5879 "dataLinks": [ ] 5880 }, 5881 "panels": [ ], 5882 "percentage": false, 5883 "pointradius": 1, 5884 "points": false, 5885 "renderer": "flot", 5886 "seriesOverrides": [ ], 5887 "spaceLength": 10, 5888 "stack": false, 5889 "steppedLine": false, 5890 "targets": [ 5891 { 5892 "expr": "sum(rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", 5893 "intervalFactor": 1, 5894 "legendFormat": "{{status_code}}-{{operation}}", 5895 "refId": "A" 5896 } 5897 ], 5898 "thresholds": [ ], 5899 "timeFrom": null, 5900 "timeRegions": [ ], 5901 "timeShift": null, 5902 "title": "Status By Method", 5903 "tooltip": { 5904 "shared": true, 5905 "sort": 2, 5906 "value_type": "individual" 5907 }, 5908 "type": "graph", 5909 "xaxis": { 5910 "buckets": null, 5911 "mode": "time", 5912 "name": null, 5913 "show": true, 5914 "values": [ ] 5915 }, 5916 "yaxes": [ 5917 { 5918 "format": "short", 5919 "label": null, 5920 "logBase": 1, 5921 "max": null, 5922 "min": null, 5923 "show": true 5924 }, 5925 { 5926 "format": "short", 5927 "label": null, 5928 "logBase": 1, 5929 "max": null, 5930 "min": null, 5931 "show": true 5932 } 5933 ], 5934 "yaxis": { 5935 "align": false, 5936 "alignLevel": null 5937 } 5938 } 5939 ], 5940 "targets": [ ], 5941 "title": "Azure Blob", 5942 "type": "row" 5943 }, 5944 { 5945 "collapsed": true, 5946 "datasource": null, 5947 "gridPos": { 5948 "h": 1, 5949 "w": 24, 5950 "x": 0, 5951 "y": 37 5952 }, 5953 "id": 114, 5954 "panels": [ 5955 { 5956 "aliasColors": { }, 5957 "bars": false, 5958 "dashLength": 10, 5959 "dashes": false, 5960 "datasource": "$datasource", 5961 "fill": 1, 5962 "fillGradient": 0, 5963 "gridPos": { 5964 "h": 8, 5965 "w": 24, 5966 "x": 0, 5967 "y": 10 5968 }, 5969 "id": 115, 5970 "interval": "", 5971 "legend": { 5972 "alignAsTable": true, 5973 "avg": false, 5974 "current": false, 5975 "max": false, 5976 "min": false, 5977 "rightSide": true, 5978 "show": true, 5979 "total": false, 5980 "values": false 5981 }, 5982 "lines": true, 5983 "linewidth": 1, 5984 "nullPointMode": "null", 5985 "options": { 5986 "dataLinks": [ ] 5987 }, 5988 "panels": [ ], 5989 "percentage": false, 5990 "pointradius": 1, 5991 "points": false, 5992 "renderer": "flot", 5993 "seriesOverrides": [ ], 5994 "spaceLength": 10, 5995 "stack": false, 5996 "steppedLine": false, 5997 "targets": [ 5998 { 5999 "expr": "histogram_quantile(.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 6000 "intervalFactor": 1, 6001 "legendFormat": ".99-{{operation}}", 6002 "refId": "A" 6003 }, 6004 { 6005 "expr": "histogram_quantile(.9, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 6006 "hide": false, 6007 "legendFormat": ".9-{{operation}}", 6008 "refId": "B" 6009 }, 6010 { 6011 "expr": "histogram_quantile(.5, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 6012 "hide": false, 6013 "legendFormat": ".5-{{operation}}", 6014 "refId": "C" 6015 } 6016 ], 6017 "thresholds": [ ], 6018 "timeFrom": null, 6019 "timeRegions": [ ], 6020 "timeShift": null, 6021 "title": "Latency By Operation", 6022 "tooltip": { 6023 "shared": true, 6024 "sort": 2, 6025 "value_type": "individual" 6026 }, 6027 "type": "graph", 6028 "xaxis": { 6029 "buckets": null, 6030 "mode": "time", 6031 "name": null, 6032 "show": true, 6033 "values": [ ] 6034 }, 6035 "yaxes": [ 6036 { 6037 "format": "short", 6038 "label": null, 6039 "logBase": 1, 6040 "max": null, 6041 "min": null, 6042 "show": true 6043 }, 6044 { 6045 "format": "short", 6046 "label": null, 6047 "logBase": 1, 6048 "max": null, 6049 "min": null, 6050 "show": true 6051 } 6052 ], 6053 "yaxis": { 6054 "align": false, 6055 "alignLevel": null 6056 } 6057 }, 6058 { 6059 "aliasColors": { }, 6060 "bars": false, 6061 "dashLength": 10, 6062 "dashes": false, 6063 "datasource": "$datasource", 6064 "fill": 1, 6065 "fillGradient": 0, 6066 "gridPos": { 6067 "h": 8, 6068 "w": 24, 6069 "x": 0, 6070 "y": 18 6071 }, 6072 "id": 116, 6073 "interval": "", 6074 "legend": { 6075 "alignAsTable": true, 6076 "avg": false, 6077 "current": false, 6078 "max": false, 6079 "min": false, 6080 "rightSide": true, 6081 "show": true, 6082 "total": false, 6083 "values": false 6084 }, 6085 "lines": true, 6086 "linewidth": 1, 6087 "nullPointMode": "null", 6088 "options": { 6089 "dataLinks": [ ] 6090 }, 6091 "panels": [ ], 6092 "percentage": false, 6093 "pointradius": 1, 6094 "points": false, 6095 "renderer": "flot", 6096 "seriesOverrides": [ ], 6097 "spaceLength": 10, 6098 "stack": false, 6099 "steppedLine": false, 6100 "targets": [ 6101 { 6102 "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", 6103 "intervalFactor": 1, 6104 "legendFormat": "{{status_code}}-{{operation}}", 6105 "refId": "A" 6106 } 6107 ], 6108 "thresholds": [ ], 6109 "timeFrom": null, 6110 "timeRegions": [ ], 6111 "timeShift": null, 6112 "title": "Status By Method", 6113 "tooltip": { 6114 "shared": true, 6115 "sort": 2, 6116 "value_type": "individual" 6117 }, 6118 "type": "graph", 6119 "xaxis": { 6120 "buckets": null, 6121 "mode": "time", 6122 "name": null, 6123 "show": true, 6124 "values": [ ] 6125 }, 6126 "yaxes": [ 6127 { 6128 "format": "short", 6129 "label": null, 6130 "logBase": 1, 6131 "max": null, 6132 "min": null, 6133 "show": true 6134 }, 6135 { 6136 "format": "short", 6137 "label": null, 6138 "logBase": 1, 6139 "max": null, 6140 "min": null, 6141 "show": true 6142 } 6143 ], 6144 "yaxis": { 6145 "align": false, 6146 "alignLevel": null 6147 } 6148 } 6149 ], 6150 "targets": [ ], 6151 "title": "BoltDB Shipper", 6152 "type": "row" 6153 } 6154 ], 6155 "refresh": "10s", 6156 "rows": [ ], 6157 "schemaVersion": 14, 6158 "style": "dark", 6159 "tags": [ 6160 "loki" 6161 ], 6162 "templating": { 6163 "list": [ 6164 { 6165 "current": { 6166 "text": "default", 6167 "value": "default" 6168 }, 6169 "hide": 0, 6170 "label": "Data Source", 6171 "name": "datasource", 6172 "options": [ ], 6173 "query": "prometheus", 6174 "refresh": 1, 6175 "regex": "", 6176 "type": "datasource" 6177 }, 6178 { 6179 "hide": 0, 6180 "label": null, 6181 "name": "logs", 6182 "options": [ ], 6183 "query": "loki", 6184 "refresh": 1, 6185 "regex": "", 6186 "type": "datasource" 6187 }, 6188 { 6189 "allValue": null, 6190 "current": { 6191 "text": "prod", 6192 "value": "prod" 6193 }, 6194 "datasource": "$datasource", 6195 "hide": 0, 6196 "includeAll": false, 6197 "label": "cluster", 6198 "multi": false, 6199 "name": "cluster", 6200 "options": [ ], 6201 "query": "label_values(loki_build_info, cluster)", 6202 "refresh": 1, 6203 "regex": "", 6204 "sort": 2, 6205 "tagValuesQuery": "", 6206 "tags": [ ], 6207 "tagsQuery": "", 6208 "type": "query", 6209 "useTags": false 6210 }, 6211 { 6212 "allValue": null, 6213 "current": { 6214 "text": "prod", 6215 "value": "prod" 6216 }, 6217 "datasource": "$datasource", 6218 "hide": 0, 6219 "includeAll": false, 6220 "label": "namespace", 6221 "multi": false, 6222 "name": "namespace", 6223 "options": [ ], 6224 "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", 6225 "refresh": 1, 6226 "regex": "", 6227 "sort": 2, 6228 "tagValuesQuery": "", 6229 "tags": [ ], 6230 "tagsQuery": "", 6231 "type": "query", 6232 "useTags": false 6233 } 6234 ] 6235 }, 6236 "time": { 6237 "from": "now-1h", 6238 "to": "now" 6239 }, 6240 "timepicker": { 6241 "refresh_intervals": [ 6242 "5s", 6243 "10s", 6244 "30s", 6245 "1m", 6246 "5m", 6247 "15m", 6248 "30m", 6249 "1h", 6250 "2h", 6251 "1d" 6252 ], 6253 "time_options": [ 6254 "5m", 6255 "15m", 6256 "1h", 6257 "6h", 6258 "12h", 6259 "24h", 6260 "2d", 6261 "7d", 6262 "30d" 6263 ] 6264 }, 6265 "timezone": "utc", 6266 "title": "Loki / Operational", 6267 "uid": "operational", 6268 "version": 0 6269 }