github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/production/loki-mixin-compiled/dashboards/loki-operational.json (about) 1 { 2 "annotations": { 3 "list": [ ] 4 }, 5 "editable": true, 6 "gnetId": null, 7 "graphTooltip": 0, 8 "hideControls": false, 9 "id": 68, 10 "iteration": 1588704280892, 11 "links": [ 12 { 13 "asDropdown": true, 14 "icon": "external link", 15 "includeVars": true, 16 "keepTime": true, 17 "tags": [ 18 "loki" 19 ], 20 "targetBlank": false, 21 "title": "Loki Dashboards", 22 "type": "dashboards" 23 } 24 ], 25 "panels": [ 26 { 27 "collapsed": false, 28 "datasource": null, 29 "gridPos": { 30 "h": 1, 31 "w": 24, 32 "x": 0, 33 "y": 0 34 }, 35 "id": 17, 36 "panels": [ ], 37 "targets": [ ], 38 "title": "Main", 39 "type": "row" 40 }, 41 { 42 "aliasColors": { 43 "5xx": "red" 44 }, 45 "bars": false, 46 "dashLength": 10, 47 "dashes": false, 48 "datasource": "$datasource", 49 "fieldConfig": { 50 "defaults": { 51 "custom": { } 52 }, 53 "overrides": [ ] 54 }, 55 "fill": 1, 56 "fillGradient": 0, 57 "gridPos": { 58 "h": 5, 59 "w": 4, 60 "x": 0, 61 "y": 1 62 }, 63 "hiddenSeries": false, 64 "id": 6, 65 "legend": { 66 "avg": false, 67 "current": false, 68 "max": false, 69 "min": false, 70 "show": true, 71 "total": false, 72 "values": false 73 }, 74 "lines": true, 75 "linewidth": 1, 76 "nullPointMode": "null", 77 "options": { 78 "dataLinks": [ ] 79 }, 80 "panels": [ ], 81 "percentage": false, 82 "pointradius": 2, 83 "points": false, 84 "renderer": "flot", 85 "seriesOverrides": [ ], 86 "spaceLength": 10, 87 "stack": false, 88 "steppedLine": false, 89 "targets": [ 90 { 91 "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=~\"($namespace)/cortex-gw\", route=~\"api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}[5m]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)", 92 "legendFormat": "{{status}}", 93 "refId": "A" 94 } 95 ], 96 "thresholds": [ ], 97 "timeFrom": null, 98 "timeRegions": [ ], 99 "timeShift": null, 100 "title": "Queries/Second", 101 "tooltip": { 102 "shared": true, 103 "sort": 0, 104 "value_type": "individual" 105 }, 106 "type": "graph", 107 "xaxis": { 108 "buckets": null, 109 "mode": "time", 110 "name": null, 111 "show": true, 112 "values": [ ] 113 }, 114 "yaxes": [ 115 { 116 "format": "short", 117 "label": null, 118 "logBase": 10, 119 "max": null, 120 "min": null, 121 "show": true 122 }, 123 { 124 "format": "short", 125 "label": null, 126 "logBase": 1, 127 "max": null, 128 "min": null, 129 "show": true 130 } 131 ], 132 "yaxis": { 133 "align": false, 134 "alignLevel": null 135 } 136 }, 137 { 138 "aliasColors": { 139 "5xx": "red" 140 }, 141 "bars": false, 142 "dashLength": 10, 143 "dashes": false, 144 "datasource": "$datasource", 145 "fieldConfig": { 146 "defaults": { 147 "custom": { } 148 }, 149 "overrides": [ ] 150 }, 151 "fill": 1, 152 "fillGradient": 0, 153 "gridPos": { 154 "h": 5, 155 "w": 4, 156 "x": 4, 157 "y": 1 158 }, 159 "hiddenSeries": false, 160 "id": 7, 161 "legend": { 162 "avg": false, 163 "current": false, 164 "max": false, 165 "min": false, 166 "show": true, 167 "total": false, 168 "values": false 169 }, 170 "lines": true, 171 "linewidth": 1, 172 "nullPointMode": "null", 173 "options": { 174 "dataLinks": [ ] 175 }, 176 "panels": [ ], 177 "percentage": false, 178 "pointradius": 2, 179 "points": false, 180 "renderer": "flot", 181 "seriesOverrides": [ ], 182 "spaceLength": 10, 183 "stack": false, 184 "steppedLine": false, 185 "targets": [ 186 { 187 "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=~\"($namespace)/cortex-gw\", route=~\"api_prom_push|loki_api_v1_push\"}[5m]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", 188 "legendFormat": "{{status}}", 189 "refId": "A" 190 } 191 ], 192 "thresholds": [ ], 193 "timeFrom": null, 194 "timeRegions": [ ], 195 "timeShift": null, 196 "title": "Pushes/Second", 197 "tooltip": { 198 "shared": true, 199 "sort": 0, 200 "value_type": "individual" 201 }, 202 "type": "graph", 203 "xaxis": { 204 "buckets": null, 205 "mode": "time", 206 "name": null, 207 "show": true, 208 "values": [ ] 209 }, 210 "yaxes": [ 211 { 212 "format": "short", 213 "label": null, 214 "logBase": 10, 215 "max": null, 216 "min": null, 217 "show": true 218 }, 219 { 220 "format": "short", 221 "label": null, 222 "logBase": 1, 223 "max": null, 224 "min": null, 225 "show": true 226 } 227 ], 228 "yaxis": { 229 "align": false, 230 "alignLevel": null 231 } 232 }, 233 { 234 "aliasColors": { }, 235 "bars": false, 236 "dashLength": 10, 237 "dashes": false, 238 "datasource": "$datasource", 239 "fieldConfig": { 240 "defaults": { 241 "custom": { } 242 }, 243 "overrides": [ ] 244 }, 245 "fill": 1, 246 "fillGradient": 0, 247 "gridPos": { 248 "h": 5, 249 "w": 4, 250 "x": 8, 251 "y": 1 252 }, 253 "hiddenSeries": false, 254 "id": 11, 255 "legend": { 256 "avg": false, 257 "current": false, 258 "hideEmpty": false, 259 "hideZero": false, 260 "max": false, 261 "min": false, 262 "show": false, 263 "total": false, 264 "values": false 265 }, 266 "lines": true, 267 "linewidth": 1, 268 "nullPointMode": "null", 269 "options": { 270 "dataLinks": [ ] 271 }, 272 "panels": [ ], 273 "percentage": false, 274 "pointradius": 2, 275 "points": false, 276 "renderer": "flot", 277 "seriesOverrides": [ ], 278 "spaceLength": 10, 279 "stack": false, 280 "steppedLine": false, 281 "targets": [ 282 { 283 "expr": "topk(5, sum by (name,level) (rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\"}[$__interval])) - \nsum by (name,level) (rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\"}[$__interval] offset 1h)))", 284 "legendFormat": "{{name}}-{{level}}", 285 "refId": "A" 286 } 287 ], 288 "thresholds": [ ], 289 "timeFrom": null, 290 "timeRegions": [ ], 291 "timeShift": null, 292 "title": "Bad Words", 293 "tooltip": { 294 "shared": true, 295 "sort": 2, 296 "value_type": "individual" 297 }, 298 "type": "graph", 299 "xaxis": { 300 "buckets": null, 301 "mode": "time", 302 "name": null, 303 "show": true, 304 "values": [ ] 305 }, 306 "yaxes": [ 307 { 308 "format": "short", 309 "label": null, 310 "logBase": 1, 311 "max": null, 312 "min": null, 313 "show": true 314 }, 315 { 316 "format": "short", 317 "label": null, 318 "logBase": 1, 319 "max": null, 320 "min": null, 321 "show": true 322 } 323 ], 324 "yaxis": { 325 "align": false, 326 "alignLevel": null 327 } 328 }, 329 { 330 "aliasColors": { }, 331 "bars": false, 332 "dashLength": 10, 333 "dashes": false, 334 "datasource": "$datasource", 335 "fieldConfig": { 336 "defaults": { 337 "custom": { } 338 }, 339 "overrides": [ ] 340 }, 341 "fill": 1, 342 "fillGradient": 0, 343 "gridPos": { 344 "h": 5, 345 "w": 4, 346 "x": 12, 347 "y": 1 348 }, 349 "hiddenSeries": false, 350 "id": 2, 351 "interval": "", 352 "legend": { 353 "avg": false, 354 "current": false, 355 "max": false, 356 "min": false, 357 "show": false, 358 "total": false, 359 "values": false 360 }, 361 "lines": true, 362 "linewidth": 1, 363 "nullPointMode": "null", 364 "options": { 365 "dataLinks": [ ] 366 }, 367 "panels": [ ], 368 "percentage": false, 369 "pointradius": 2, 370 "points": false, 371 "renderer": "flot", 372 "seriesOverrides": [ ], 373 "spaceLength": 10, 374 "stack": false, 375 "steppedLine": false, 376 "targets": [ 377 { 378 "expr": "topk(10, sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (tenant))", 379 "legendFormat": "{{tenant}}", 380 "refId": "A" 381 } 382 ], 383 "thresholds": [ ], 384 "timeFrom": null, 385 "timeRegions": [ ], 386 "timeShift": null, 387 "title": "Lines Per Tenant (top 10)", 388 "tooltip": { 389 "shared": false, 390 "sort": 0, 391 "value_type": "individual" 392 }, 393 "type": "graph", 394 "xaxis": { 395 "buckets": null, 396 "mode": "time", 397 "name": null, 398 "show": true, 399 "values": [ ] 400 }, 401 "yaxes": [ 402 { 403 "format": "short", 404 "label": null, 405 "logBase": 1, 406 "max": null, 407 "min": null, 408 "show": true 409 }, 410 { 411 "format": "short", 412 "label": null, 413 "logBase": 1, 414 "max": null, 415 "min": null, 416 "show": true 417 } 418 ], 419 "yaxis": { 420 "align": false, 421 "alignLevel": null 422 } 423 }, 424 { 425 "aliasColors": { }, 426 "bars": false, 427 "dashLength": 10, 428 "dashes": false, 429 "datasource": "$datasource", 430 "fieldConfig": { 431 "defaults": { 432 "custom": { } 433 }, 434 "overrides": [ ] 435 }, 436 "fill": 1, 437 "fillGradient": 0, 438 "gridPos": { 439 "h": 5, 440 "w": 4, 441 "x": 16, 442 "y": 1 443 }, 444 "hiddenSeries": false, 445 "id": 4, 446 "legend": { 447 "avg": false, 448 "current": false, 449 "hideEmpty": true, 450 "hideZero": true, 451 "max": false, 452 "min": false, 453 "show": false, 454 "total": false, 455 "values": false 456 }, 457 "lines": true, 458 "linewidth": 1, 459 "nullPointMode": "null", 460 "options": { 461 "dataLinks": [ ] 462 }, 463 "panels": [ ], 464 "percentage": false, 465 "pointradius": 2, 466 "points": false, 467 "renderer": "flot", 468 "seriesOverrides": [ ], 469 "spaceLength": 10, 470 "stack": false, 471 "steppedLine": false, 472 "targets": [ 473 { 474 "expr": "topk(10, sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (tenant)) / 1024 / 1024", 475 "legendFormat": "{{tenant}}", 476 "refId": "A" 477 } 478 ], 479 "thresholds": [ ], 480 "timeFrom": null, 481 "timeRegions": [ ], 482 "timeShift": null, 483 "title": "MBs Per Tenant (Top 10)", 484 "tooltip": { 485 "shared": false, 486 "sort": 0, 487 "value_type": "individual" 488 }, 489 "type": "graph", 490 "xaxis": { 491 "buckets": null, 492 "mode": "time", 493 "name": null, 494 "show": true, 495 "values": [ ] 496 }, 497 "yaxes": [ 498 { 499 "format": "short", 500 "label": null, 501 "logBase": 1, 502 "max": null, 503 "min": null, 504 "show": true 505 }, 506 { 507 "format": "short", 508 "label": null, 509 "logBase": 1, 510 "max": null, 511 "min": null, 512 "show": true 513 } 514 ], 515 "yaxis": { 516 "align": false, 517 "alignLevel": null 518 } 519 }, 520 { 521 "aliasColors": { }, 522 "bars": false, 523 "dashLength": 10, 524 "dashes": false, 525 "datasource": "$datasource", 526 "fieldConfig": { 527 "defaults": { 528 "custom": { } 529 }, 530 "overrides": [ ] 531 }, 532 "fill": 1, 533 "fillGradient": 0, 534 "gridPos": { 535 "h": 5, 536 "w": 4, 537 "x": 20, 538 "y": 1 539 }, 540 "hiddenSeries": false, 541 "id": 24, 542 "legend": { 543 "avg": false, 544 "current": false, 545 "max": false, 546 "min": false, 547 "show": false, 548 "total": false, 549 "values": false 550 }, 551 "lines": true, 552 "linewidth": 1, 553 "nullPointMode": "null", 554 "options": { 555 "dataLinks": [ ] 556 }, 557 "panels": [ ], 558 "percentage": false, 559 "pointradius": 2, 560 "points": false, 561 "renderer": "flot", 562 "seriesOverrides": [ ], 563 "spaceLength": 10, 564 "stack": false, 565 "steppedLine": false, 566 "targets": [ 567 { 568 "expr": "increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\"}[10m]) > 0", 569 "hide": false, 570 "interval": "", 571 "legendFormat": "{{container}}-{{pod}}", 572 "refId": "B" 573 } 574 ], 575 "thresholds": [ ], 576 "timeFrom": null, 577 "timeRegions": [ ], 578 "timeShift": null, 579 "title": "Container Restarts", 580 "tooltip": { 581 "shared": true, 582 "sort": 2, 583 "value_type": "individual" 584 }, 585 "type": "graph", 586 "xaxis": { 587 "buckets": null, 588 "mode": "time", 589 "name": null, 590 "show": true, 591 "values": [ ] 592 }, 593 "yaxes": [ 594 { 595 "format": "short", 596 "label": null, 597 "logBase": 1, 598 "max": null, 599 "min": null, 600 "show": true 601 }, 602 { 603 "format": "short", 604 "label": null, 605 "logBase": 1, 606 "max": null, 607 "min": null, 608 "show": true 609 } 610 ], 611 "yaxis": { 612 "align": false, 613 "alignLevel": null 614 } 615 }, 616 { 617 "aliasColors": { }, 618 "bars": false, 619 "dashLength": 10, 620 "dashes": false, 621 "datasource": "$datasource", 622 "fieldConfig": { 623 "defaults": { 624 "custom": { } 625 }, 626 "overrides": [ ] 627 }, 628 "fill": 1, 629 "fillGradient": 0, 630 "gridPos": { 631 "h": 10, 632 "w": 12, 633 "x": 0, 634 "y": 6 635 }, 636 "hiddenSeries": false, 637 "id": 9, 638 "legend": { 639 "alignAsTable": true, 640 "avg": false, 641 "current": false, 642 "max": false, 643 "min": false, 644 "rightSide": true, 645 "show": true, 646 "total": false, 647 "values": false 648 }, 649 "lines": true, 650 "linewidth": 1, 651 "nullPointMode": "null", 652 "options": { 653 "dataLinks": [ ] 654 }, 655 "panels": [ ], 656 "percentage": false, 657 "pointradius": 2, 658 "points": false, 659 "renderer": "flot", 660 "seriesOverrides": [ ], 661 "spaceLength": 10, 662 "stack": false, 663 "steppedLine": false, 664 "targets": [ 665 { 666 "expr": "histogram_quantile(0.99, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/cortex-gw\", route=~\"api_prom_push|loki_api_v1_push\", cluster=~\"$cluster\"})) * 1e3", 667 "legendFormat": ".99", 668 "refId": "A" 669 }, 670 { 671 "expr": "histogram_quantile(0.75, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/cortex-gw\", route=~\"api_prom_push|loki_api_v1_push\", cluster=~\"$cluster\"})) * 1e3", 672 "legendFormat": ".9", 673 "refId": "B" 674 }, 675 { 676 "expr": "histogram_quantile(0.5, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/cortex-gw\", route=~\"api_prom_push|loki_api_v1_push\", cluster=~\"$cluster\"})) * 1e3", 677 "legendFormat": ".5", 678 "refId": "C" 679 } 680 ], 681 "thresholds": [ ], 682 "timeFrom": null, 683 "timeRegions": [ ], 684 "timeShift": null, 685 "title": "Push Latency", 686 "tooltip": { 687 "shared": true, 688 "sort": 0, 689 "value_type": "individual" 690 }, 691 "type": "graph", 692 "xaxis": { 693 "buckets": null, 694 "mode": "time", 695 "name": null, 696 "show": true, 697 "values": [ ] 698 }, 699 "yaxes": [ 700 { 701 "format": "short", 702 "label": null, 703 "logBase": 1, 704 "max": null, 705 "min": null, 706 "show": true 707 }, 708 { 709 "format": "short", 710 "label": null, 711 "logBase": 1, 712 "max": null, 713 "min": null, 714 "show": true 715 } 716 ], 717 "yaxis": { 718 "align": false, 719 "alignLevel": null 720 } 721 }, 722 { 723 "aliasColors": { }, 724 "bars": false, 725 "dashLength": 10, 726 "dashes": false, 727 "datasource": "$datasource", 728 "fieldConfig": { 729 "defaults": { 730 "custom": { } 731 }, 732 "overrides": [ ] 733 }, 734 "fill": 1, 735 "fillGradient": 0, 736 "gridPos": { 737 "h": 5, 738 "w": 6, 739 "x": 12, 740 "y": 6 741 }, 742 "hiddenSeries": false, 743 "id": 12, 744 "legend": { 745 "alignAsTable": true, 746 "avg": false, 747 "current": false, 748 "max": false, 749 "min": false, 750 "rightSide": true, 751 "show": false, 752 "total": false, 753 "values": false 754 }, 755 "lines": true, 756 "linewidth": 1, 757 "nullPointMode": "null", 758 "options": { 759 "dataLinks": [ ] 760 }, 761 "panels": [ ], 762 "percentage": false, 763 "pointradius": 2, 764 "points": false, 765 "renderer": "flot", 766 "seriesOverrides": [ ], 767 "spaceLength": 10, 768 "stack": false, 769 "steppedLine": false, 770 "targets": [ 771 { 772 "expr": "histogram_quantile(0.99, sum by (le) (job:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/distributor\", cluster=~\"$cluster\"})) * 1e3", 773 "legendFormat": ".99", 774 "refId": "A" 775 }, 776 { 777 "expr": "histogram_quantile(0.9, sum by (le) (job:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/distributor\", cluster=~\"$cluster\"})) * 1e3", 778 "legendFormat": ".9", 779 "refId": "B" 780 }, 781 { 782 "expr": "histogram_quantile(0.5, sum by (le) (job:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/distributor\", cluster=~\"$cluster\"})) * 1e3", 783 "legendFormat": ".5", 784 "refId": "C" 785 } 786 ], 787 "thresholds": [ ], 788 "timeFrom": null, 789 "timeRegions": [ ], 790 "timeShift": null, 791 "title": "Distributor Latency", 792 "tooltip": { 793 "shared": true, 794 "sort": 0, 795 "value_type": "individual" 796 }, 797 "type": "graph", 798 "xaxis": { 799 "buckets": null, 800 "mode": "time", 801 "name": null, 802 "show": true, 803 "values": [ ] 804 }, 805 "yaxes": [ 806 { 807 "format": "short", 808 "label": null, 809 "logBase": 1, 810 "max": null, 811 "min": null, 812 "show": true 813 }, 814 { 815 "format": "short", 816 "label": null, 817 "logBase": 1, 818 "max": null, 819 "min": null, 820 "show": true 821 } 822 ], 823 "yaxis": { 824 "align": false, 825 "alignLevel": null 826 } 827 }, 828 { 829 "aliasColors": { }, 830 "bars": false, 831 "dashLength": 10, 832 "dashes": false, 833 "datasource": "$datasource", 834 "fieldConfig": { 835 "defaults": { 836 "custom": { } 837 }, 838 "overrides": [ ] 839 }, 840 "fill": 0, 841 "fillGradient": 0, 842 "gridPos": { 843 "h": 5, 844 "w": 6, 845 "x": 18, 846 "y": 6 847 }, 848 "hiddenSeries": false, 849 "id": 71, 850 "legend": { 851 "alignAsTable": true, 852 "avg": false, 853 "current": false, 854 "max": false, 855 "min": false, 856 "rightSide": true, 857 "show": false, 858 "total": false, 859 "values": false 860 }, 861 "lines": true, 862 "linewidth": 1, 863 "nullPointMode": "null", 864 "options": { 865 "dataLinks": [ ] 866 }, 867 "panels": [ ], 868 "percentage": false, 869 "pointradius": 2, 870 "points": false, 871 "renderer": "flot", 872 "seriesOverrides": [ ], 873 "spaceLength": 10, 874 "stack": false, 875 "steppedLine": false, 876 "targets": [ 877 { 878 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/distributor\", status_code!~\"5[0-9]{2}\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/distributor\"}[5m])) by (route)", 879 "interval": "", 880 "legendFormat": "{{route}}", 881 "refId": "A" 882 } 883 ], 884 "thresholds": [ ], 885 "timeFrom": null, 886 "timeRegions": [ ], 887 "timeShift": null, 888 "title": "Distributor Success Rate", 889 "tooltip": { 890 "shared": true, 891 "sort": 0, 892 "value_type": "individual" 893 }, 894 "type": "graph", 895 "xaxis": { 896 "buckets": null, 897 "mode": "time", 898 "name": null, 899 "show": true, 900 "values": [ ] 901 }, 902 "yaxes": [ 903 { 904 "decimals": null, 905 "format": "percentunit", 906 "label": "", 907 "logBase": 1, 908 "max": "1", 909 "min": null, 910 "show": true 911 }, 912 { 913 "format": "short", 914 "label": null, 915 "logBase": 1, 916 "max": null, 917 "min": null, 918 "show": true 919 } 920 ], 921 "yaxis": { 922 "align": false, 923 "alignLevel": null 924 } 925 }, 926 { 927 "aliasColors": { }, 928 "bars": false, 929 "dashLength": 10, 930 "dashes": false, 931 "datasource": "$datasource", 932 "fieldConfig": { 933 "defaults": { 934 "custom": { } 935 }, 936 "overrides": [ ] 937 }, 938 "fill": 1, 939 "fillGradient": 0, 940 "gridPos": { 941 "h": 5, 942 "w": 6, 943 "x": 12, 944 "y": 11 945 }, 946 "hiddenSeries": false, 947 "id": 13, 948 "legend": { 949 "alignAsTable": true, 950 "avg": false, 951 "current": false, 952 "max": false, 953 "min": false, 954 "rightSide": true, 955 "show": false, 956 "total": false, 957 "values": false 958 }, 959 "lines": true, 960 "linewidth": 1, 961 "nullPointMode": "null", 962 "options": { 963 "dataLinks": [ ] 964 }, 965 "panels": [ ], 966 "percentage": false, 967 "pointradius": 2, 968 "points": false, 969 "renderer": "flot", 970 "seriesOverrides": [ ], 971 "spaceLength": 10, 972 "stack": false, 973 "steppedLine": false, 974 "targets": [ 975 { 976 "expr": "histogram_quantile(0.99, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/ingester\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3", 977 "legendFormat": ".99", 978 "refId": "A" 979 }, 980 { 981 "expr": "histogram_quantile(0.9, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/ingester\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3", 982 "hide": false, 983 "legendFormat": ".9", 984 "refId": "B" 985 }, 986 { 987 "expr": "histogram_quantile(0.5, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/ingester\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3", 988 "hide": false, 989 "legendFormat": ".5", 990 "refId": "C" 991 } 992 ], 993 "thresholds": [ ], 994 "timeFrom": null, 995 "timeRegions": [ ], 996 "timeShift": null, 997 "title": "Ingester Latency Write", 998 "tooltip": { 999 "shared": true, 1000 "sort": 0, 1001 "value_type": "individual" 1002 }, 1003 "type": "graph", 1004 "xaxis": { 1005 "buckets": null, 1006 "mode": "time", 1007 "name": null, 1008 "show": true, 1009 "values": [ ] 1010 }, 1011 "yaxes": [ 1012 { 1013 "format": "short", 1014 "label": null, 1015 "logBase": 1, 1016 "max": null, 1017 "min": null, 1018 "show": true 1019 }, 1020 { 1021 "format": "short", 1022 "label": null, 1023 "logBase": 1, 1024 "max": null, 1025 "min": null, 1026 "show": true 1027 } 1028 ], 1029 "yaxis": { 1030 "align": false, 1031 "alignLevel": null 1032 } 1033 }, 1034 { 1035 "aliasColors": { }, 1036 "bars": false, 1037 "dashLength": 10, 1038 "dashes": false, 1039 "datasource": "$datasource", 1040 "fieldConfig": { 1041 "defaults": { 1042 "custom": { } 1043 }, 1044 "overrides": [ ] 1045 }, 1046 "fill": 0, 1047 "fillGradient": 0, 1048 "gridPos": { 1049 "h": 5, 1050 "w": 6, 1051 "x": 18, 1052 "y": 11 1053 }, 1054 "hiddenSeries": false, 1055 "id": 72, 1056 "legend": { 1057 "alignAsTable": true, 1058 "avg": false, 1059 "current": false, 1060 "max": false, 1061 "min": false, 1062 "rightSide": true, 1063 "show": false, 1064 "total": false, 1065 "values": false 1066 }, 1067 "lines": true, 1068 "linewidth": 1, 1069 "nullPointMode": "null", 1070 "options": { 1071 "dataLinks": [ ] 1072 }, 1073 "panels": [ ], 1074 "percentage": false, 1075 "pointradius": 2, 1076 "points": false, 1077 "renderer": "flot", 1078 "seriesOverrides": [ ], 1079 "spaceLength": 10, 1080 "stack": false, 1081 "steppedLine": false, 1082 "targets": [ 1083 { 1084 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/ingester\", status_code!~\"5[0-9]{2}\", route=\"/logproto.Pusher/Push\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/ingester\", route=\"/logproto.Pusher/Push\"}[5m])) by (route)", 1085 "interval": "", 1086 "legendFormat": "{{route}}", 1087 "refId": "A" 1088 } 1089 ], 1090 "thresholds": [ ], 1091 "timeFrom": null, 1092 "timeRegions": [ ], 1093 "timeShift": null, 1094 "title": "Ingester Success Rate Write", 1095 "tooltip": { 1096 "shared": true, 1097 "sort": 0, 1098 "value_type": "individual" 1099 }, 1100 "type": "graph", 1101 "xaxis": { 1102 "buckets": null, 1103 "mode": "time", 1104 "name": null, 1105 "show": true, 1106 "values": [ ] 1107 }, 1108 "yaxes": [ 1109 { 1110 "decimals": null, 1111 "format": "percentunit", 1112 "label": "", 1113 "logBase": 1, 1114 "max": "1", 1115 "min": null, 1116 "show": true 1117 }, 1118 { 1119 "format": "short", 1120 "label": null, 1121 "logBase": 1, 1122 "max": null, 1123 "min": null, 1124 "show": true 1125 } 1126 ], 1127 "yaxis": { 1128 "align": false, 1129 "alignLevel": null 1130 } 1131 }, 1132 { 1133 "aliasColors": { }, 1134 "bars": false, 1135 "dashLength": 10, 1136 "dashes": false, 1137 "datasource": "$datasource", 1138 "fieldConfig": { 1139 "defaults": { 1140 "custom": { } 1141 }, 1142 "overrides": [ ] 1143 }, 1144 "fill": 1, 1145 "fillGradient": 0, 1146 "gridPos": { 1147 "h": 10, 1148 "w": 12, 1149 "x": 0, 1150 "y": 16 1151 }, 1152 "hiddenSeries": false, 1153 "id": 10, 1154 "legend": { 1155 "alignAsTable": true, 1156 "avg": false, 1157 "current": false, 1158 "hideEmpty": true, 1159 "hideZero": false, 1160 "max": false, 1161 "min": false, 1162 "rightSide": true, 1163 "show": true, 1164 "total": false, 1165 "values": false 1166 }, 1167 "lines": true, 1168 "linewidth": 1, 1169 "nullPointMode": "null", 1170 "options": { 1171 "dataLinks": [ ] 1172 }, 1173 "panels": [ ], 1174 "percentage": false, 1175 "pointradius": 2, 1176 "points": false, 1177 "renderer": "flot", 1178 "seriesOverrides": [ ], 1179 "spaceLength": 10, 1180 "stack": false, 1181 "steppedLine": false, 1182 "targets": [ 1183 { 1184 "expr": "histogram_quantile(0.99, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", 1185 "legendFormat": "{{route}}-.99", 1186 "refId": "A" 1187 }, 1188 { 1189 "expr": "histogram_quantile(0.9, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", 1190 "legendFormat": "{{route}}-.9", 1191 "refId": "B" 1192 }, 1193 { 1194 "expr": "histogram_quantile(0.5, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", 1195 "legendFormat": "{{route}}-.5", 1196 "refId": "C" 1197 } 1198 ], 1199 "thresholds": [ ], 1200 "timeFrom": null, 1201 "timeRegions": [ ], 1202 "timeShift": null, 1203 "title": "Query Latency", 1204 "tooltip": { 1205 "shared": true, 1206 "sort": 0, 1207 "value_type": "individual" 1208 }, 1209 "type": "graph", 1210 "xaxis": { 1211 "buckets": null, 1212 "mode": "time", 1213 "name": null, 1214 "show": true, 1215 "values": [ ] 1216 }, 1217 "yaxes": [ 1218 { 1219 "format": "short", 1220 "label": null, 1221 "logBase": 1, 1222 "max": null, 1223 "min": null, 1224 "show": true 1225 }, 1226 { 1227 "format": "short", 1228 "label": null, 1229 "logBase": 1, 1230 "max": null, 1231 "min": null, 1232 "show": true 1233 } 1234 ], 1235 "yaxis": { 1236 "align": false, 1237 "alignLevel": null 1238 } 1239 }, 1240 { 1241 "aliasColors": { }, 1242 "bars": false, 1243 "dashLength": 10, 1244 "dashes": false, 1245 "datasource": "$datasource", 1246 "fieldConfig": { 1247 "defaults": { 1248 "custom": { } 1249 }, 1250 "overrides": [ ] 1251 }, 1252 "fill": 1, 1253 "fillGradient": 0, 1254 "gridPos": { 1255 "h": 5, 1256 "w": 6, 1257 "x": 12, 1258 "y": 16 1259 }, 1260 "hiddenSeries": false, 1261 "id": 14, 1262 "legend": { 1263 "alignAsTable": true, 1264 "avg": false, 1265 "current": false, 1266 "max": false, 1267 "min": false, 1268 "rightSide": true, 1269 "show": false, 1270 "total": false, 1271 "values": false 1272 }, 1273 "lines": true, 1274 "linewidth": 1, 1275 "nullPointMode": "null", 1276 "options": { 1277 "dataLinks": [ ] 1278 }, 1279 "panels": [ ], 1280 "percentage": false, 1281 "pointradius": 2, 1282 "points": false, 1283 "renderer": "flot", 1284 "seriesOverrides": [ ], 1285 "spaceLength": 10, 1286 "stack": false, 1287 "steppedLine": false, 1288 "targets": [ 1289 { 1290 "expr": "histogram_quantile(0.99, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", 1291 "legendFormat": ".99-{{route}}", 1292 "refId": "A" 1293 }, 1294 { 1295 "expr": "histogram_quantile(0.9, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", 1296 "legendFormat": ".9-{{route}}", 1297 "refId": "B" 1298 }, 1299 { 1300 "expr": "histogram_quantile(0.5, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", 1301 "legendFormat": ".5-{{route}}", 1302 "refId": "C" 1303 } 1304 ], 1305 "thresholds": [ ], 1306 "timeFrom": null, 1307 "timeRegions": [ ], 1308 "timeShift": null, 1309 "title": "Querier Latency", 1310 "tooltip": { 1311 "shared": true, 1312 "sort": 0, 1313 "value_type": "individual" 1314 }, 1315 "type": "graph", 1316 "xaxis": { 1317 "buckets": null, 1318 "mode": "time", 1319 "name": null, 1320 "show": true, 1321 "values": [ ] 1322 }, 1323 "yaxes": [ 1324 { 1325 "format": "short", 1326 "label": null, 1327 "logBase": 1, 1328 "max": null, 1329 "min": null, 1330 "show": true 1331 }, 1332 { 1333 "format": "short", 1334 "label": null, 1335 "logBase": 1, 1336 "max": null, 1337 "min": null, 1338 "show": true 1339 } 1340 ], 1341 "yaxis": { 1342 "align": false, 1343 "alignLevel": null 1344 } 1345 }, 1346 { 1347 "aliasColors": { }, 1348 "bars": false, 1349 "dashLength": 10, 1350 "dashes": false, 1351 "datasource": "$datasource", 1352 "fieldConfig": { 1353 "defaults": { 1354 "custom": { } 1355 }, 1356 "overrides": [ ] 1357 }, 1358 "fill": 0, 1359 "fillGradient": 0, 1360 "gridPos": { 1361 "h": 5, 1362 "w": 6, 1363 "x": 18, 1364 "y": 16 1365 }, 1366 "hiddenSeries": false, 1367 "id": 73, 1368 "legend": { 1369 "alignAsTable": true, 1370 "avg": false, 1371 "current": false, 1372 "max": false, 1373 "min": false, 1374 "rightSide": true, 1375 "show": false, 1376 "total": false, 1377 "values": false 1378 }, 1379 "lines": true, 1380 "linewidth": 1, 1381 "nullPointMode": "null", 1382 "options": { 1383 "dataLinks": [ ] 1384 }, 1385 "panels": [ ], 1386 "percentage": false, 1387 "pointradius": 2, 1388 "points": false, 1389 "renderer": "flot", 1390 "seriesOverrides": [ ], 1391 "spaceLength": 10, 1392 "stack": false, 1393 "steppedLine": false, 1394 "targets": [ 1395 { 1396 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/querier\", status_code!~\"5[0-9]{2}\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/querier\"}[5m])) by (route)", 1397 "interval": "", 1398 "legendFormat": "{{route}}", 1399 "refId": "A" 1400 } 1401 ], 1402 "thresholds": [ ], 1403 "timeFrom": null, 1404 "timeRegions": [ ], 1405 "timeShift": null, 1406 "title": "Querier Success Rate", 1407 "tooltip": { 1408 "shared": true, 1409 "sort": 0, 1410 "value_type": "individual" 1411 }, 1412 "type": "graph", 1413 "xaxis": { 1414 "buckets": null, 1415 "mode": "time", 1416 "name": null, 1417 "show": true, 1418 "values": [ ] 1419 }, 1420 "yaxes": [ 1421 { 1422 "decimals": null, 1423 "format": "percentunit", 1424 "label": "", 1425 "logBase": 1, 1426 "max": "1", 1427 "min": null, 1428 "show": true 1429 }, 1430 { 1431 "format": "short", 1432 "label": null, 1433 "logBase": 1, 1434 "max": null, 1435 "min": null, 1436 "show": true 1437 } 1438 ], 1439 "yaxis": { 1440 "align": false, 1441 "alignLevel": null 1442 } 1443 }, 1444 { 1445 "aliasColors": { }, 1446 "bars": false, 1447 "dashLength": 10, 1448 "dashes": false, 1449 "datasource": "$datasource", 1450 "description": "", 1451 "fieldConfig": { 1452 "defaults": { 1453 "custom": { } 1454 }, 1455 "overrides": [ ] 1456 }, 1457 "fill": 1, 1458 "fillGradient": 0, 1459 "gridPos": { 1460 "h": 5, 1461 "w": 6, 1462 "x": 12, 1463 "y": 21 1464 }, 1465 "hiddenSeries": false, 1466 "id": 15, 1467 "legend": { 1468 "alignAsTable": true, 1469 "avg": false, 1470 "current": false, 1471 "max": false, 1472 "min": false, 1473 "rightSide": true, 1474 "show": false, 1475 "total": false, 1476 "values": false 1477 }, 1478 "lines": true, 1479 "linewidth": 1, 1480 "nullPointMode": "null", 1481 "options": { 1482 "dataLinks": [ ] 1483 }, 1484 "panels": [ ], 1485 "percentage": false, 1486 "pointradius": 2, 1487 "points": false, 1488 "renderer": "flot", 1489 "seriesOverrides": [ ], 1490 "spaceLength": 10, 1491 "stack": false, 1492 "steppedLine": false, 1493 "targets": [ 1494 { 1495 "expr": "histogram_quantile(0.99, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/ingester\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3", 1496 "legendFormat": ".99-{{route}}", 1497 "refId": "A" 1498 }, 1499 { 1500 "expr": "histogram_quantile(0.9, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/ingester\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3", 1501 "legendFormat": ".9-{{route}}", 1502 "refId": "B" 1503 }, 1504 { 1505 "expr": "histogram_quantile(0.5, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/ingester\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3", 1506 "legendFormat": ".5-{{route}}", 1507 "refId": "C" 1508 } 1509 ], 1510 "thresholds": [ ], 1511 "timeFrom": null, 1512 "timeRegions": [ ], 1513 "timeShift": null, 1514 "title": "Ingester Latency Read", 1515 "tooltip": { 1516 "shared": true, 1517 "sort": 0, 1518 "value_type": "individual" 1519 }, 1520 "type": "graph", 1521 "xaxis": { 1522 "buckets": null, 1523 "mode": "time", 1524 "name": null, 1525 "show": true, 1526 "values": [ ] 1527 }, 1528 "yaxes": [ 1529 { 1530 "format": "short", 1531 "label": null, 1532 "logBase": 1, 1533 "max": null, 1534 "min": null, 1535 "show": true 1536 }, 1537 { 1538 "format": "short", 1539 "label": null, 1540 "logBase": 1, 1541 "max": null, 1542 "min": null, 1543 "show": true 1544 } 1545 ], 1546 "yaxis": { 1547 "align": false, 1548 "alignLevel": null 1549 } 1550 }, 1551 { 1552 "aliasColors": { }, 1553 "bars": false, 1554 "dashLength": 10, 1555 "dashes": false, 1556 "datasource": "$datasource", 1557 "fieldConfig": { 1558 "defaults": { 1559 "custom": { } 1560 }, 1561 "overrides": [ ] 1562 }, 1563 "fill": 0, 1564 "fillGradient": 0, 1565 "gridPos": { 1566 "h": 5, 1567 "w": 6, 1568 "x": 18, 1569 "y": 21 1570 }, 1571 "hiddenSeries": false, 1572 "id": 74, 1573 "legend": { 1574 "alignAsTable": true, 1575 "avg": false, 1576 "current": false, 1577 "max": false, 1578 "min": false, 1579 "rightSide": true, 1580 "show": false, 1581 "total": false, 1582 "values": false 1583 }, 1584 "lines": true, 1585 "linewidth": 1, 1586 "nullPointMode": "null", 1587 "options": { 1588 "dataLinks": [ ] 1589 }, 1590 "panels": [ ], 1591 "percentage": false, 1592 "pointradius": 2, 1593 "points": false, 1594 "renderer": "flot", 1595 "seriesOverrides": [ ], 1596 "spaceLength": 10, 1597 "stack": false, 1598 "steppedLine": false, 1599 "targets": [ 1600 { 1601 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/ingester\", status_code!~\"5[0-9]{2}\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/ingester\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[5m])) by (route)", 1602 "interval": "", 1603 "legendFormat": "{{route}}", 1604 "refId": "A" 1605 } 1606 ], 1607 "thresholds": [ ], 1608 "timeFrom": null, 1609 "timeRegions": [ ], 1610 "timeShift": null, 1611 "title": "Ingester Success Rate Read", 1612 "tooltip": { 1613 "shared": true, 1614 "sort": 0, 1615 "value_type": "individual" 1616 }, 1617 "type": "graph", 1618 "xaxis": { 1619 "buckets": null, 1620 "mode": "time", 1621 "name": null, 1622 "show": true, 1623 "values": [ ] 1624 }, 1625 "yaxes": [ 1626 { 1627 "decimals": null, 1628 "format": "percentunit", 1629 "label": "", 1630 "logBase": 1, 1631 "max": "1", 1632 "min": null, 1633 "show": true 1634 }, 1635 { 1636 "format": "short", 1637 "label": null, 1638 "logBase": 1, 1639 "max": null, 1640 "min": null, 1641 "show": true 1642 } 1643 ], 1644 "yaxis": { 1645 "align": false, 1646 "alignLevel": null 1647 } 1648 }, 1649 { 1650 "collapsed": true, 1651 "datasource": null, 1652 "gridPos": { 1653 "h": 1, 1654 "w": 24, 1655 "x": 0, 1656 "y": 26 1657 }, 1658 "id": 110, 1659 "panels": [ 1660 { 1661 "aliasColors": { }, 1662 "bars": false, 1663 "dashLength": 10, 1664 "dashes": false, 1665 "datasource": "$datasource", 1666 "fill": 1, 1667 "fillGradient": 0, 1668 "gridPos": { 1669 "h": 8, 1670 "w": 12, 1671 "x": 0, 1672 "y": 27 1673 }, 1674 "hiddenSeries": false, 1675 "id": 112, 1676 "legend": { 1677 "avg": false, 1678 "current": false, 1679 "max": false, 1680 "min": false, 1681 "show": false, 1682 "total": false, 1683 "values": false 1684 }, 1685 "lines": true, 1686 "linewidth": 1, 1687 "nullPointMode": "null", 1688 "options": { 1689 "dataLinks": [ ] 1690 }, 1691 "panels": [ ], 1692 "percentage": false, 1693 "pointradius": 2, 1694 "points": false, 1695 "renderer": "flot", 1696 "seriesOverrides": [ ], 1697 "spaceLength": 10, 1698 "stack": false, 1699 "steppedLine": false, 1700 "targets": [ 1701 { 1702 "expr": "topk(10,sum by (tenant, reason) (rate(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[1m])))", 1703 "interval": "", 1704 "legendFormat": "{{ tenant }} - {{ reason }}", 1705 "refId": "A" 1706 } 1707 ], 1708 "thresholds": [ ], 1709 "timeFrom": null, 1710 "timeRegions": [ ], 1711 "timeShift": null, 1712 "title": "Discarded Lines", 1713 "tooltip": { 1714 "shared": false, 1715 "sort": 0, 1716 "value_type": "individual" 1717 }, 1718 "type": "graph", 1719 "xaxis": { 1720 "buckets": null, 1721 "mode": "time", 1722 "name": null, 1723 "show": true, 1724 "values": [ ] 1725 }, 1726 "yaxes": [ 1727 { 1728 "format": "short", 1729 "label": null, 1730 "logBase": 1, 1731 "max": null, 1732 "min": null, 1733 "show": true 1734 }, 1735 { 1736 "format": "short", 1737 "label": null, 1738 "logBase": 1, 1739 "max": null, 1740 "min": null, 1741 "show": true 1742 } 1743 ], 1744 "yaxis": { 1745 "align": false, 1746 "alignLevel": null 1747 } 1748 }, 1749 { 1750 "columns": [ ], 1751 "datasource": "$datasource", 1752 "fontSize": "100%", 1753 "gridPos": { 1754 "h": 8, 1755 "w": 12, 1756 "x": 12, 1757 "y": 27 1758 }, 1759 "id": 113, 1760 "pageSize": null, 1761 "panels": [ ], 1762 "showHeader": true, 1763 "sort": { 1764 "col": 3, 1765 "desc": true 1766 }, 1767 "styles": [ 1768 { 1769 "alias": "Time", 1770 "align": "auto", 1771 "dateFormat": "YYYY-MM-DD HH:mm:ss", 1772 "pattern": "Time", 1773 "type": "hidden" 1774 }, 1775 { 1776 "alias": "", 1777 "align": "auto", 1778 "colorMode": null, 1779 "colors": [ 1780 "rgba(245, 54, 54, 0.9)", 1781 "rgba(237, 129, 40, 0.89)", 1782 "rgba(50, 172, 45, 0.97)" 1783 ], 1784 "dateFormat": "YYYY-MM-DD HH:mm:ss", 1785 "decimals": 2, 1786 "mappingType": 1, 1787 "pattern": "tenant", 1788 "thresholds": [ ], 1789 "type": "string", 1790 "unit": "short" 1791 }, 1792 { 1793 "alias": "", 1794 "align": "auto", 1795 "colorMode": null, 1796 "colors": [ 1797 "rgba(245, 54, 54, 0.9)", 1798 "rgba(237, 129, 40, 0.89)", 1799 "rgba(50, 172, 45, 0.97)" 1800 ], 1801 "dateFormat": "YYYY-MM-DD HH:mm:ss", 1802 "decimals": 2, 1803 "mappingType": 1, 1804 "pattern": "reason", 1805 "thresholds": [ ], 1806 "type": "number", 1807 "unit": "short" 1808 }, 1809 { 1810 "alias": "", 1811 "align": "right", 1812 "colorMode": null, 1813 "colors": [ 1814 "rgba(245, 54, 54, 0.9)", 1815 "rgba(237, 129, 40, 0.89)", 1816 "rgba(50, 172, 45, 0.97)" 1817 ], 1818 "decimals": 2, 1819 "pattern": "/.*/", 1820 "thresholds": [ ], 1821 "type": "number", 1822 "unit": "short" 1823 } 1824 ], 1825 "targets": [ 1826 { 1827 "expr": "topk(10, sum by (tenant, reason) (sum_over_time(increase(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[1m])[$__range:1m])))", 1828 "format": "table", 1829 "instant": true, 1830 "interval": "", 1831 "legendFormat": "{{ tenant }} - {{ reason }}", 1832 "refId": "A" 1833 } 1834 ], 1835 "timeFrom": null, 1836 "timeShift": null, 1837 "title": "Discarded Lines Per Interval", 1838 "transform": "table", 1839 "type": "table-old" 1840 } 1841 ], 1842 "targets": [ ], 1843 "title": "Limits", 1844 "type": "row" 1845 }, 1846 { 1847 "collapsed": true, 1848 "datasource": null, 1849 "gridPos": { 1850 "h": 1, 1851 "w": 24, 1852 "x": 0, 1853 "y": 27 1854 }, 1855 "id": 23, 1856 "panels": [ 1857 { 1858 "aliasColors": { }, 1859 "bars": false, 1860 "dashLength": 10, 1861 "dashes": false, 1862 "datasource": "$datasource", 1863 "fill": 1, 1864 "fillGradient": 0, 1865 "gridPos": { 1866 "h": 7, 1867 "w": 6, 1868 "x": 0, 1869 "y": 28 1870 }, 1871 "hiddenSeries": false, 1872 "id": 26, 1873 "legend": { 1874 "avg": false, 1875 "current": false, 1876 "max": false, 1877 "min": false, 1878 "show": false, 1879 "total": false, 1880 "values": false 1881 }, 1882 "lines": false, 1883 "linewidth": 1, 1884 "nullPointMode": "null", 1885 "options": { 1886 "dataLinks": [ ] 1887 }, 1888 "panels": [ ], 1889 "percentage": false, 1890 "pointradius": 1, 1891 "points": true, 1892 "renderer": "flot", 1893 "seriesOverrides": [ ], 1894 "spaceLength": 10, 1895 "stack": false, 1896 "steppedLine": false, 1897 "targets": [ 1898 { 1899 "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"distributor.*\"}", 1900 "intervalFactor": 3, 1901 "legendFormat": "{{pod}}-{{container}}", 1902 "refId": "A" 1903 } 1904 ], 1905 "thresholds": [ ], 1906 "timeFrom": null, 1907 "timeRegions": [ ], 1908 "timeShift": null, 1909 "title": "CPU Usage", 1910 "tooltip": { 1911 "shared": true, 1912 "sort": 2, 1913 "value_type": "individual" 1914 }, 1915 "type": "graph", 1916 "xaxis": { 1917 "buckets": null, 1918 "mode": "time", 1919 "name": null, 1920 "show": true, 1921 "values": [ ] 1922 }, 1923 "yaxes": [ 1924 { 1925 "format": "short", 1926 "label": null, 1927 "logBase": 1, 1928 "max": null, 1929 "min": null, 1930 "show": true 1931 }, 1932 { 1933 "format": "short", 1934 "label": null, 1935 "logBase": 1, 1936 "max": null, 1937 "min": null, 1938 "show": true 1939 } 1940 ], 1941 "yaxis": { 1942 "align": false, 1943 "alignLevel": null 1944 } 1945 }, 1946 { 1947 "aliasColors": { }, 1948 "bars": false, 1949 "dashLength": 10, 1950 "dashes": false, 1951 "datasource": "$datasource", 1952 "fill": 1, 1953 "fillGradient": 0, 1954 "gridPos": { 1955 "h": 7, 1956 "w": 6, 1957 "x": 6, 1958 "y": 28 1959 }, 1960 "hiddenSeries": false, 1961 "id": 27, 1962 "legend": { 1963 "avg": false, 1964 "current": false, 1965 "hideEmpty": false, 1966 "hideZero": false, 1967 "max": false, 1968 "min": false, 1969 "show": false, 1970 "total": false, 1971 "values": false 1972 }, 1973 "lines": false, 1974 "linewidth": 1, 1975 "nullPointMode": "null", 1976 "options": { 1977 "dataLinks": [ ] 1978 }, 1979 "panels": [ ], 1980 "percentage": false, 1981 "pointradius": 1, 1982 "points": true, 1983 "renderer": "flot", 1984 "seriesOverrides": [ ], 1985 "spaceLength": 10, 1986 "stack": false, 1987 "steppedLine": false, 1988 "targets": [ 1989 { 1990 "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"distributor.*\"}", 1991 "instant": false, 1992 "intervalFactor": 3, 1993 "legendFormat": "{{pod}}", 1994 "refId": "A" 1995 } 1996 ], 1997 "thresholds": [ ], 1998 "timeFrom": null, 1999 "timeRegions": [ ], 2000 "timeShift": null, 2001 "title": "Memory Usage", 2002 "tooltip": { 2003 "shared": true, 2004 "sort": 2, 2005 "value_type": "individual" 2006 }, 2007 "type": "graph", 2008 "xaxis": { 2009 "buckets": null, 2010 "mode": "time", 2011 "name": null, 2012 "show": true, 2013 "values": [ ] 2014 }, 2015 "yaxes": [ 2016 { 2017 "format": "bytes", 2018 "label": null, 2019 "logBase": 1, 2020 "max": null, 2021 "min": null, 2022 "show": true 2023 }, 2024 { 2025 "format": "short", 2026 "label": null, 2027 "logBase": 1, 2028 "max": null, 2029 "min": null, 2030 "show": true 2031 } 2032 ], 2033 "yaxis": { 2034 "align": false, 2035 "alignLevel": null 2036 } 2037 }, 2038 { 2039 "aliasColors": { }, 2040 "bars": true, 2041 "dashLength": 10, 2042 "dashes": false, 2043 "datasource": "$logs", 2044 "fill": 1, 2045 "fillGradient": 0, 2046 "gridPos": { 2047 "h": 4, 2048 "w": 12, 2049 "x": 12, 2050 "y": 28 2051 }, 2052 "hiddenSeries": false, 2053 "id": 31, 2054 "legend": { 2055 "avg": false, 2056 "current": false, 2057 "max": false, 2058 "min": false, 2059 "show": false, 2060 "total": false, 2061 "values": false 2062 }, 2063 "lines": false, 2064 "linewidth": 1, 2065 "nullPointMode": "null", 2066 "options": { 2067 "dataLinks": [ ] 2068 }, 2069 "panels": [ ], 2070 "percentage": false, 2071 "pointradius": 2, 2072 "points": false, 2073 "renderer": "flot", 2074 "seriesOverrides": [ 2075 { 2076 "alias": "{}", 2077 "color": "#C4162A" 2078 } 2079 ], 2080 "spaceLength": 10, 2081 "stack": false, 2082 "steppedLine": false, 2083 "targets": [ 2084 { 2085 "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/distributor\"} | logfmt | level=\"error\"[1m]))", 2086 "refId": "A" 2087 } 2088 ], 2089 "thresholds": [ ], 2090 "timeFrom": null, 2091 "timeRegions": [ ], 2092 "timeShift": null, 2093 "title": "Error Log Rate", 2094 "tooltip": { 2095 "shared": true, 2096 "sort": 2, 2097 "value_type": "individual" 2098 }, 2099 "type": "graph", 2100 "xaxis": { 2101 "buckets": null, 2102 "mode": "time", 2103 "name": null, 2104 "show": false, 2105 "values": [ ] 2106 }, 2107 "yaxes": [ 2108 { 2109 "format": "short", 2110 "label": null, 2111 "logBase": 1, 2112 "max": null, 2113 "min": null, 2114 "show": true 2115 }, 2116 { 2117 "format": "short", 2118 "label": null, 2119 "logBase": 1, 2120 "max": null, 2121 "min": null, 2122 "show": false 2123 } 2124 ], 2125 "yaxis": { 2126 "align": false, 2127 "alignLevel": null 2128 } 2129 }, 2130 { 2131 "datasource": "$logs", 2132 "gridPos": { 2133 "h": 18, 2134 "w": 12, 2135 "x": 12, 2136 "y": 32 2137 }, 2138 "id": 29, 2139 "options": { 2140 "showLabels": false, 2141 "showTime": false, 2142 "sortOrder": "Descending", 2143 "wrapLogMessage": true 2144 }, 2145 "panels": [ ], 2146 "targets": [ 2147 { 2148 "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/distributor\"} | logfmt | level=\"error\"", 2149 "refId": "A" 2150 } 2151 ], 2152 "timeFrom": null, 2153 "timeShift": null, 2154 "title": "Logs", 2155 "type": "logs" 2156 }, 2157 { 2158 "aliasColors": { }, 2159 "bars": false, 2160 "dashLength": 10, 2161 "dashes": false, 2162 "datasource": "$datasource", 2163 "fill": 0, 2164 "fillGradient": 0, 2165 "gridPos": { 2166 "h": 7, 2167 "w": 6, 2168 "x": 0, 2169 "y": 35 2170 }, 2171 "hiddenSeries": false, 2172 "id": 33, 2173 "legend": { 2174 "avg": false, 2175 "current": false, 2176 "max": false, 2177 "min": false, 2178 "show": false, 2179 "total": false, 2180 "values": false 2181 }, 2182 "lines": true, 2183 "linewidth": 1, 2184 "nullPointMode": "null", 2185 "options": { 2186 "dataLinks": [ ] 2187 }, 2188 "panels": [ ], 2189 "percentage": false, 2190 "pointradius": 1, 2191 "points": false, 2192 "renderer": "flot", 2193 "seriesOverrides": [ ], 2194 "spaceLength": 10, 2195 "stack": false, 2196 "steppedLine": false, 2197 "targets": [ 2198 { 2199 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/distributor\", status_code!~\"5[0-9]{2}\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/distributor\"}[5m])) by (route)", 2200 "interval": "", 2201 "intervalFactor": 1, 2202 "legendFormat": "{{route}}", 2203 "refId": "A" 2204 } 2205 ], 2206 "thresholds": [ ], 2207 "timeFrom": null, 2208 "timeRegions": [ ], 2209 "timeShift": null, 2210 "title": "Success Rate", 2211 "tooltip": { 2212 "shared": true, 2213 "sort": 2, 2214 "value_type": "individual" 2215 }, 2216 "type": "graph", 2217 "xaxis": { 2218 "buckets": null, 2219 "mode": "time", 2220 "name": null, 2221 "show": true, 2222 "values": [ ] 2223 }, 2224 "yaxes": [ 2225 { 2226 "format": "short", 2227 "label": null, 2228 "logBase": 1, 2229 "max": null, 2230 "min": null, 2231 "show": true 2232 }, 2233 { 2234 "format": "short", 2235 "label": null, 2236 "logBase": 1, 2237 "max": null, 2238 "min": null, 2239 "show": true 2240 } 2241 ], 2242 "yaxis": { 2243 "align": false, 2244 "alignLevel": null 2245 } 2246 }, 2247 { 2248 "aliasColors": { }, 2249 "bars": false, 2250 "dashLength": 10, 2251 "dashes": false, 2252 "datasource": "$datasource", 2253 "fill": 1, 2254 "fillGradient": 0, 2255 "gridPos": { 2256 "h": 7, 2257 "w": 6, 2258 "x": 6, 2259 "y": 35 2260 }, 2261 "hiddenSeries": false, 2262 "id": 32, 2263 "legend": { 2264 "avg": false, 2265 "current": false, 2266 "max": false, 2267 "min": false, 2268 "show": false, 2269 "total": false, 2270 "values": false 2271 }, 2272 "lines": true, 2273 "linewidth": 1, 2274 "nullPointMode": "null", 2275 "options": { 2276 "dataLinks": [ ] 2277 }, 2278 "panels": [ ], 2279 "percentage": false, 2280 "pointradius": 1, 2281 "points": false, 2282 "renderer": "flot", 2283 "seriesOverrides": [ ], 2284 "spaceLength": 10, 2285 "stack": false, 2286 "steppedLine": false, 2287 "targets": [ 2288 { 2289 "expr": "sum(rate(loki_distributor_ingester_append_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (ingester)", 2290 "intervalFactor": 1, 2291 "legendFormat": "{{ingester}}", 2292 "refId": "A" 2293 } 2294 ], 2295 "thresholds": [ ], 2296 "timeFrom": null, 2297 "timeRegions": [ ], 2298 "timeShift": null, 2299 "title": "Append Failures By Ingester", 2300 "tooltip": { 2301 "shared": true, 2302 "sort": 2, 2303 "value_type": "individual" 2304 }, 2305 "type": "graph", 2306 "xaxis": { 2307 "buckets": null, 2308 "mode": "time", 2309 "name": null, 2310 "show": true, 2311 "values": [ ] 2312 }, 2313 "yaxes": [ 2314 { 2315 "format": "short", 2316 "label": null, 2317 "logBase": 1, 2318 "max": null, 2319 "min": null, 2320 "show": true 2321 }, 2322 { 2323 "format": "short", 2324 "label": null, 2325 "logBase": 1, 2326 "max": null, 2327 "min": null, 2328 "show": true 2329 } 2330 ], 2331 "yaxis": { 2332 "align": false, 2333 "alignLevel": null 2334 } 2335 }, 2336 { 2337 "aliasColors": { }, 2338 "bars": false, 2339 "dashLength": 10, 2340 "dashes": false, 2341 "datasource": "$datasource", 2342 "fill": 1, 2343 "fillGradient": 0, 2344 "gridPos": { 2345 "h": 7, 2346 "w": 6, 2347 "x": 0, 2348 "y": 42 2349 }, 2350 "hiddenSeries": false, 2351 "id": 34, 2352 "legend": { 2353 "avg": false, 2354 "current": false, 2355 "max": false, 2356 "min": false, 2357 "show": false, 2358 "total": false, 2359 "values": false 2360 }, 2361 "lines": true, 2362 "linewidth": 1, 2363 "nullPointMode": "null", 2364 "options": { 2365 "dataLinks": [ ] 2366 }, 2367 "panels": [ ], 2368 "percentage": false, 2369 "pointradius": 1, 2370 "points": false, 2371 "renderer": "flot", 2372 "seriesOverrides": [ ], 2373 "spaceLength": 10, 2374 "stack": false, 2375 "steppedLine": false, 2376 "targets": [ 2377 { 2378 "expr": "sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", 2379 "intervalFactor": 1, 2380 "legendFormat": "{{pod}}", 2381 "refId": "A" 2382 } 2383 ], 2384 "thresholds": [ ], 2385 "timeFrom": null, 2386 "timeRegions": [ ], 2387 "timeShift": null, 2388 "title": "Bytes Received/Second", 2389 "tooltip": { 2390 "shared": true, 2391 "sort": 2, 2392 "value_type": "individual" 2393 }, 2394 "type": "graph", 2395 "xaxis": { 2396 "buckets": null, 2397 "mode": "time", 2398 "name": null, 2399 "show": true, 2400 "values": [ ] 2401 }, 2402 "yaxes": [ 2403 { 2404 "format": "short", 2405 "label": null, 2406 "logBase": 1, 2407 "max": null, 2408 "min": null, 2409 "show": true 2410 }, 2411 { 2412 "format": "short", 2413 "label": null, 2414 "logBase": 1, 2415 "max": null, 2416 "min": null, 2417 "show": true 2418 } 2419 ], 2420 "yaxis": { 2421 "align": false, 2422 "alignLevel": null 2423 } 2424 }, 2425 { 2426 "aliasColors": { }, 2427 "bars": false, 2428 "dashLength": 10, 2429 "dashes": false, 2430 "datasource": "$datasource", 2431 "fill": 1, 2432 "fillGradient": 0, 2433 "gridPos": { 2434 "h": 7, 2435 "w": 6, 2436 "x": 6, 2437 "y": 42 2438 }, 2439 "hiddenSeries": false, 2440 "id": 35, 2441 "legend": { 2442 "avg": false, 2443 "current": false, 2444 "max": false, 2445 "min": false, 2446 "show": false, 2447 "total": false, 2448 "values": false 2449 }, 2450 "lines": true, 2451 "linewidth": 1, 2452 "nullPointMode": "null", 2453 "options": { 2454 "dataLinks": [ ] 2455 }, 2456 "panels": [ ], 2457 "percentage": false, 2458 "pointradius": 1, 2459 "points": false, 2460 "renderer": "flot", 2461 "seriesOverrides": [ ], 2462 "spaceLength": 10, 2463 "stack": false, 2464 "steppedLine": false, 2465 "targets": [ 2466 { 2467 "expr": "sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", 2468 "intervalFactor": 1, 2469 "legendFormat": "{{pod}}", 2470 "refId": "A" 2471 } 2472 ], 2473 "thresholds": [ ], 2474 "timeFrom": null, 2475 "timeRegions": [ ], 2476 "timeShift": null, 2477 "title": "Lines Received/Second", 2478 "tooltip": { 2479 "shared": true, 2480 "sort": 2, 2481 "value_type": "individual" 2482 }, 2483 "type": "graph", 2484 "xaxis": { 2485 "buckets": null, 2486 "mode": "time", 2487 "name": null, 2488 "show": true, 2489 "values": [ ] 2490 }, 2491 "yaxes": [ 2492 { 2493 "format": "short", 2494 "label": null, 2495 "logBase": 1, 2496 "max": null, 2497 "min": null, 2498 "show": true 2499 }, 2500 { 2501 "format": "short", 2502 "label": null, 2503 "logBase": 1, 2504 "max": null, 2505 "min": null, 2506 "show": true 2507 } 2508 ], 2509 "yaxis": { 2510 "align": false, 2511 "alignLevel": null 2512 } 2513 } 2514 ], 2515 "targets": [ ], 2516 "title": "Distributor", 2517 "type": "row" 2518 }, 2519 { 2520 "collapsed": true, 2521 "datasource": null, 2522 "gridPos": { 2523 "h": 1, 2524 "w": 24, 2525 "x": 0, 2526 "y": 28 2527 }, 2528 "id": 19, 2529 "panels": [ 2530 { 2531 "aliasColors": { }, 2532 "bars": false, 2533 "dashLength": 10, 2534 "dashes": false, 2535 "datasource": "$datasource", 2536 "fill": 1, 2537 "fillGradient": 0, 2538 "gridPos": { 2539 "h": 7, 2540 "w": 6, 2541 "x": 0, 2542 "y": 29 2543 }, 2544 "hiddenSeries": false, 2545 "id": 36, 2546 "legend": { 2547 "avg": false, 2548 "current": false, 2549 "max": false, 2550 "min": false, 2551 "show": false, 2552 "total": false, 2553 "values": false 2554 }, 2555 "lines": false, 2556 "linewidth": 1, 2557 "nullPointMode": "null", 2558 "options": { 2559 "dataLinks": [ ] 2560 }, 2561 "panels": [ ], 2562 "percentage": false, 2563 "pointradius": 1, 2564 "points": true, 2565 "renderer": "flot", 2566 "seriesOverrides": [ ], 2567 "spaceLength": 10, 2568 "stack": false, 2569 "steppedLine": false, 2570 "targets": [ 2571 { 2572 "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"ingester.*\"}", 2573 "intervalFactor": 3, 2574 "legendFormat": "{{pod}}-{{container}}", 2575 "refId": "A" 2576 } 2577 ], 2578 "thresholds": [ ], 2579 "timeFrom": null, 2580 "timeRegions": [ ], 2581 "timeShift": null, 2582 "title": "CPU Usage", 2583 "tooltip": { 2584 "shared": true, 2585 "sort": 2, 2586 "value_type": "individual" 2587 }, 2588 "type": "graph", 2589 "xaxis": { 2590 "buckets": null, 2591 "mode": "time", 2592 "name": null, 2593 "show": true, 2594 "values": [ ] 2595 }, 2596 "yaxes": [ 2597 { 2598 "format": "short", 2599 "label": null, 2600 "logBase": 1, 2601 "max": null, 2602 "min": null, 2603 "show": true 2604 }, 2605 { 2606 "format": "short", 2607 "label": null, 2608 "logBase": 1, 2609 "max": null, 2610 "min": null, 2611 "show": true 2612 } 2613 ], 2614 "yaxis": { 2615 "align": false, 2616 "alignLevel": null 2617 } 2618 }, 2619 { 2620 "aliasColors": { }, 2621 "bars": false, 2622 "dashLength": 10, 2623 "dashes": false, 2624 "datasource": "$datasource", 2625 "fill": 1, 2626 "fillGradient": 0, 2627 "gridPos": { 2628 "h": 7, 2629 "w": 6, 2630 "x": 0, 2631 "y": 36 2632 }, 2633 "hiddenSeries": false, 2634 "id": 37, 2635 "legend": { 2636 "avg": false, 2637 "current": false, 2638 "hideEmpty": false, 2639 "hideZero": false, 2640 "max": false, 2641 "min": false, 2642 "show": false, 2643 "total": false, 2644 "values": false 2645 }, 2646 "lines": false, 2647 "linewidth": 1, 2648 "nullPointMode": "null", 2649 "options": { 2650 "dataLinks": [ ] 2651 }, 2652 "panels": [ ], 2653 "percentage": false, 2654 "pointradius": 1, 2655 "points": true, 2656 "renderer": "flot", 2657 "seriesOverrides": [ ], 2658 "spaceLength": 10, 2659 "stack": false, 2660 "steppedLine": false, 2661 "targets": [ 2662 { 2663 "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"ingester.*\"}", 2664 "instant": false, 2665 "intervalFactor": 3, 2666 "legendFormat": "{{pod}}", 2667 "refId": "A" 2668 } 2669 ], 2670 "thresholds": [ ], 2671 "timeFrom": null, 2672 "timeRegions": [ ], 2673 "timeShift": null, 2674 "title": "Memory Usage", 2675 "tooltip": { 2676 "shared": true, 2677 "sort": 2, 2678 "value_type": "individual" 2679 }, 2680 "type": "graph", 2681 "xaxis": { 2682 "buckets": null, 2683 "mode": "time", 2684 "name": null, 2685 "show": true, 2686 "values": [ ] 2687 }, 2688 "yaxes": [ 2689 { 2690 "format": "bytes", 2691 "label": null, 2692 "logBase": 1, 2693 "max": null, 2694 "min": null, 2695 "show": true 2696 }, 2697 { 2698 "format": "short", 2699 "label": null, 2700 "logBase": 1, 2701 "max": null, 2702 "min": null, 2703 "show": true 2704 } 2705 ], 2706 "yaxis": { 2707 "align": false, 2708 "alignLevel": null 2709 } 2710 }, 2711 { 2712 "aliasColors": { }, 2713 "bars": true, 2714 "dashLength": 10, 2715 "dashes": false, 2716 "datasource": "$logs", 2717 "fill": 1, 2718 "fillGradient": 0, 2719 "gridPos": { 2720 "h": 3, 2721 "w": 18, 2722 "x": 12, 2723 "y": 29 2724 }, 2725 "hiddenSeries": false, 2726 "id": 38, 2727 "legend": { 2728 "avg": false, 2729 "current": false, 2730 "max": false, 2731 "min": false, 2732 "show": false, 2733 "total": false, 2734 "values": false 2735 }, 2736 "lines": false, 2737 "linewidth": 1, 2738 "nullPointMode": "null", 2739 "options": { 2740 "dataLinks": [ ] 2741 }, 2742 "panels": [ ], 2743 "percentage": false, 2744 "pointradius": 2, 2745 "points": false, 2746 "renderer": "flot", 2747 "seriesOverrides": [ 2748 { 2749 "alias": "{}", 2750 "color": "#F2495C" 2751 } 2752 ], 2753 "spaceLength": 10, 2754 "stack": false, 2755 "steppedLine": false, 2756 "targets": [ 2757 { 2758 "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/ingester\"} | logfmt | level=\"error\"[1m]))", 2759 "refId": "A" 2760 } 2761 ], 2762 "thresholds": [ ], 2763 "timeFrom": null, 2764 "timeRegions": [ ], 2765 "timeShift": null, 2766 "title": "Error Log Rate", 2767 "tooltip": { 2768 "shared": true, 2769 "sort": 2, 2770 "value_type": "individual" 2771 }, 2772 "type": "graph", 2773 "xaxis": { 2774 "buckets": null, 2775 "mode": "time", 2776 "name": null, 2777 "show": false, 2778 "values": [ ] 2779 }, 2780 "yaxes": [ 2781 { 2782 "format": "short", 2783 "label": null, 2784 "logBase": 1, 2785 "max": null, 2786 "min": null, 2787 "show": true 2788 }, 2789 { 2790 "format": "short", 2791 "label": null, 2792 "logBase": 1, 2793 "max": null, 2794 "min": null, 2795 "show": false 2796 } 2797 ], 2798 "yaxis": { 2799 "align": false, 2800 "alignLevel": null 2801 } 2802 }, 2803 { 2804 "datasource": "$logs", 2805 "gridPos": { 2806 "h": 18, 2807 "w": 18, 2808 "x": 12, 2809 "y": 32 2810 }, 2811 "id": 39, 2812 "options": { 2813 "showLabels": false, 2814 "showTime": false, 2815 "sortOrder": "Descending", 2816 "wrapLogMessage": true 2817 }, 2818 "panels": [ ], 2819 "targets": [ 2820 { 2821 "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/ingester\"} | logfmt | level=\"error\"", 2822 "refId": "A" 2823 } 2824 ], 2825 "timeFrom": null, 2826 "timeShift": null, 2827 "title": "Logs", 2828 "type": "logs" 2829 }, 2830 { 2831 "aliasColors": { }, 2832 "bars": false, 2833 "dashLength": 10, 2834 "dashes": false, 2835 "datasource": "$datasource", 2836 "fill": 0, 2837 "fillGradient": 0, 2838 "gridPos": { 2839 "h": 7, 2840 "w": 6, 2841 "x": 0, 2842 "y": 41 2843 }, 2844 "hiddenSeries": false, 2845 "id": 67, 2846 "legend": { 2847 "avg": false, 2848 "current": false, 2849 "max": false, 2850 "min": false, 2851 "show": false, 2852 "total": false, 2853 "values": false 2854 }, 2855 "lines": true, 2856 "linewidth": 1, 2857 "nullPointMode": "null", 2858 "options": { 2859 "dataLinks": [ ] 2860 }, 2861 "panels": [ ], 2862 "percentage": false, 2863 "pointradius": 1, 2864 "points": false, 2865 "renderer": "flot", 2866 "seriesOverrides": [ ], 2867 "spaceLength": 10, 2868 "stack": false, 2869 "steppedLine": false, 2870 "targets": [ 2871 { 2872 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/ingester\", status_code!~\"5[0-9]{2}\"}[1m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/ingester\"}[1m])) by (route)", 2873 "interval": "", 2874 "intervalFactor": 1, 2875 "legendFormat": "{{route}}", 2876 "refId": "A" 2877 } 2878 ], 2879 "thresholds": [ ], 2880 "timeFrom": null, 2881 "timeRegions": [ ], 2882 "timeShift": null, 2883 "title": "Success Rate", 2884 "tooltip": { 2885 "shared": true, 2886 "sort": 2, 2887 "value_type": "individual" 2888 }, 2889 "type": "graph", 2890 "xaxis": { 2891 "buckets": null, 2892 "mode": "time", 2893 "name": null, 2894 "show": true, 2895 "values": [ ] 2896 }, 2897 "yaxes": [ 2898 { 2899 "format": "short", 2900 "label": null, 2901 "logBase": 1, 2902 "max": null, 2903 "min": null, 2904 "show": true 2905 }, 2906 { 2907 "format": "short", 2908 "label": null, 2909 "logBase": 1, 2910 "max": null, 2911 "min": null, 2912 "show": true 2913 } 2914 ], 2915 "yaxis": { 2916 "align": false, 2917 "alignLevel": null 2918 } 2919 } 2920 ], 2921 "targets": [ ], 2922 "title": "Ingester", 2923 "type": "row" 2924 }, 2925 { 2926 "collapsed": true, 2927 "datasource": null, 2928 "gridPos": { 2929 "h": 1, 2930 "w": 24, 2931 "x": 0, 2932 "y": 29 2933 }, 2934 "id": 104, 2935 "panels": [ 2936 { 2937 "aliasColors": { }, 2938 "bars": false, 2939 "dashLength": 10, 2940 "dashes": false, 2941 "datasource": "$datasource", 2942 "fill": 1, 2943 "fillGradient": 0, 2944 "gridPos": { 2945 "h": 8, 2946 "w": 12, 2947 "x": 0, 2948 "y": 30 2949 }, 2950 "hiddenSeries": false, 2951 "id": 106, 2952 "legend": { 2953 "avg": false, 2954 "current": false, 2955 "hideEmpty": true, 2956 "hideZero": true, 2957 "max": false, 2958 "min": false, 2959 "show": false, 2960 "total": false, 2961 "values": false 2962 }, 2963 "lines": true, 2964 "linewidth": 1, 2965 "nullPointMode": "null", 2966 "options": { 2967 "dataLinks": [ ] 2968 }, 2969 "panels": [ ], 2970 "percentage": false, 2971 "pointradius": 2, 2972 "points": false, 2973 "renderer": "flot", 2974 "seriesOverrides": [ ], 2975 "spaceLength": 10, 2976 "stack": false, 2977 "steppedLine": false, 2978 "targets": [ 2979 { 2980 "expr": "topk(10,sum by (tenant) (loki_ingester_memory_streams{cluster=\"$cluster\",job=~\"($namespace)/ingester\"}))", 2981 "interval": "", 2982 "legendFormat": "{{ tenant }}", 2983 "refId": "A" 2984 } 2985 ], 2986 "thresholds": [ ], 2987 "timeFrom": null, 2988 "timeRegions": [ ], 2989 "timeShift": null, 2990 "title": "Active Streams", 2991 "tooltip": { 2992 "shared": false, 2993 "sort": 0, 2994 "value_type": "individual" 2995 }, 2996 "type": "graph", 2997 "xaxis": { 2998 "buckets": null, 2999 "mode": "time", 3000 "name": null, 3001 "show": true, 3002 "values": [ ] 3003 }, 3004 "yaxes": [ 3005 { 3006 "format": "short", 3007 "label": null, 3008 "logBase": 1, 3009 "max": null, 3010 "min": null, 3011 "show": true 3012 }, 3013 { 3014 "format": "short", 3015 "label": null, 3016 "logBase": 1, 3017 "max": null, 3018 "min": null, 3019 "show": true 3020 } 3021 ], 3022 "yaxis": { 3023 "align": false, 3024 "alignLevel": null 3025 } 3026 }, 3027 { 3028 "aliasColors": { }, 3029 "bars": false, 3030 "dashLength": 10, 3031 "dashes": false, 3032 "datasource": "$datasource", 3033 "fill": 1, 3034 "fillGradient": 0, 3035 "gridPos": { 3036 "h": 8, 3037 "w": 12, 3038 "x": 12, 3039 "y": 30 3040 }, 3041 "hiddenSeries": false, 3042 "id": 108, 3043 "legend": { 3044 "avg": false, 3045 "current": false, 3046 "hideEmpty": true, 3047 "hideZero": true, 3048 "max": false, 3049 "min": false, 3050 "show": false, 3051 "total": false, 3052 "values": false 3053 }, 3054 "lines": true, 3055 "linewidth": 1, 3056 "nullPointMode": "null", 3057 "options": { 3058 "dataLinks": [ ] 3059 }, 3060 "panels": [ ], 3061 "percentage": false, 3062 "pointradius": 2, 3063 "points": false, 3064 "renderer": "flot", 3065 "seriesOverrides": [ ], 3066 "spaceLength": 10, 3067 "stack": false, 3068 "steppedLine": false, 3069 "targets": [ 3070 { 3071 "expr": "topk(10, sum by (tenant) (rate(loki_ingester_streams_created_total{cluster=\"$cluster\",job=~\"($namespace)/ingester\"}[1m]) > 0))", 3072 "interval": "", 3073 "legendFormat": "{{ tenant }}", 3074 "refId": "A" 3075 } 3076 ], 3077 "thresholds": [ ], 3078 "timeFrom": null, 3079 "timeRegions": [ ], 3080 "timeShift": null, 3081 "title": "Streams Created/Sec", 3082 "tooltip": { 3083 "shared": false, 3084 "sort": 0, 3085 "value_type": "individual" 3086 }, 3087 "type": "graph", 3088 "xaxis": { 3089 "buckets": null, 3090 "mode": "time", 3091 "name": null, 3092 "show": true, 3093 "values": [ ] 3094 }, 3095 "yaxes": [ 3096 { 3097 "format": "short", 3098 "label": null, 3099 "logBase": 1, 3100 "max": null, 3101 "min": null, 3102 "show": true 3103 }, 3104 { 3105 "format": "short", 3106 "label": null, 3107 "logBase": 1, 3108 "max": null, 3109 "min": null, 3110 "show": true 3111 } 3112 ], 3113 "yaxis": { 3114 "align": false, 3115 "alignLevel": null 3116 } 3117 } 3118 ], 3119 "targets": [ ], 3120 "title": "Streams", 3121 "type": "row" 3122 }, 3123 { 3124 "collapsed": true, 3125 "datasource": null, 3126 "gridPos": { 3127 "h": 1, 3128 "w": 24, 3129 "x": 0, 3130 "y": 30 3131 }, 3132 "id": 94, 3133 "panels": [ 3134 { 3135 "aliasColors": { }, 3136 "bars": false, 3137 "dashLength": 10, 3138 "dashes": false, 3139 "datasource": "$datasource", 3140 "fill": 1, 3141 "fillGradient": 0, 3142 "gridPos": { 3143 "h": 8, 3144 "w": 12, 3145 "x": 0, 3146 "y": 31 3147 }, 3148 "hiddenSeries": false, 3149 "id": 102, 3150 "legend": { 3151 "avg": false, 3152 "current": false, 3153 "max": false, 3154 "min": false, 3155 "show": true, 3156 "total": false, 3157 "values": false 3158 }, 3159 "lines": true, 3160 "linewidth": 1, 3161 "nullPointMode": "null", 3162 "options": { 3163 "dataLinks": [ ] 3164 }, 3165 "panels": [ ], 3166 "percentage": false, 3167 "pointradius": 2, 3168 "points": false, 3169 "renderer": "flot", 3170 "seriesOverrides": [ 3171 { 3172 "alias": "De-Dupe Ratio", 3173 "yaxis": 2 3174 } 3175 ], 3176 "spaceLength": 10, 3177 "stack": false, 3178 "steppedLine": false, 3179 "targets": [ 3180 { 3181 "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\",job=~\"($namespace)/ingester\"}[1m]))", 3182 "interval": "", 3183 "legendFormat": "Chunks", 3184 "refId": "A" 3185 }, 3186 { 3187 "expr": "sum(increase(loki_chunk_store_deduped_chunks_total{cluster=\"$cluster\", job=~\"($namespace)/ingester\"}[1m]))/sum(increase(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"($namespace)/ingester\"}[1m])) < 1", 3188 "interval": "", 3189 "legendFormat": "De-Dupe Ratio", 3190 "refId": "B" 3191 } 3192 ], 3193 "thresholds": [ ], 3194 "timeFrom": null, 3195 "timeRegions": [ ], 3196 "timeShift": null, 3197 "title": "Chunks Flushed/Sec", 3198 "tooltip": { 3199 "shared": true, 3200 "sort": 0, 3201 "value_type": "individual" 3202 }, 3203 "type": "graph", 3204 "xaxis": { 3205 "buckets": null, 3206 "mode": "time", 3207 "name": null, 3208 "show": true, 3209 "values": [ ] 3210 }, 3211 "yaxes": [ 3212 { 3213 "format": "short", 3214 "label": null, 3215 "logBase": 1, 3216 "max": null, 3217 "min": null, 3218 "show": true 3219 }, 3220 { 3221 "format": "short", 3222 "label": null, 3223 "logBase": 1, 3224 "max": null, 3225 "min": null, 3226 "show": true 3227 } 3228 ], 3229 "yaxis": { 3230 "align": false, 3231 "alignLevel": null 3232 } 3233 }, 3234 { 3235 "cards": { 3236 "cardPadding": null, 3237 "cardRound": null 3238 }, 3239 "color": { 3240 "cardColor": "#b4ff00", 3241 "colorScale": "sqrt", 3242 "colorScheme": "interpolateSpectral", 3243 "exponent": 0.5, 3244 "mode": "spectrum" 3245 }, 3246 "dataFormat": "tsbuckets", 3247 "datasource": "$datasource", 3248 "gridPos": { 3249 "h": 8, 3250 "w": 12, 3251 "x": 12, 3252 "y": 31 3253 }, 3254 "heatmap": { }, 3255 "hideZeroBuckets": false, 3256 "highlightCards": true, 3257 "id": 100, 3258 "legend": { 3259 "show": true 3260 }, 3261 "panels": [ ], 3262 "reverseYBuckets": false, 3263 "targets": [ 3264 { 3265 "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\",job=~\"($namespace)/ingester\"}[1m])) by (le)", 3266 "format": "heatmap", 3267 "instant": false, 3268 "interval": "", 3269 "legendFormat": "{{ le }}", 3270 "refId": "A" 3271 } 3272 ], 3273 "timeFrom": null, 3274 "timeShift": null, 3275 "title": "Chunk Size Bytes", 3276 "tooltip": { 3277 "show": true, 3278 "showHistogram": false 3279 }, 3280 "type": "heatmap", 3281 "xAxis": { 3282 "show": true 3283 }, 3284 "xBucketNumber": null, 3285 "xBucketSize": null, 3286 "yAxis": { 3287 "decimals": 0, 3288 "format": "bytes", 3289 "logBase": 1, 3290 "max": null, 3291 "min": null, 3292 "show": true, 3293 "splitFactor": null 3294 }, 3295 "yBucketBound": "auto", 3296 "yBucketNumber": null, 3297 "yBucketSize": null 3298 }, 3299 { 3300 "aliasColors": { }, 3301 "bars": false, 3302 "dashLength": 10, 3303 "dashes": false, 3304 "datasource": "$datasource", 3305 "fill": 7, 3306 "fillGradient": 0, 3307 "gridPos": { 3308 "h": 9, 3309 "w": 12, 3310 "x": 0, 3311 "y": 39 3312 }, 3313 "hiddenSeries": false, 3314 "id": 96, 3315 "legend": { 3316 "avg": false, 3317 "current": false, 3318 "max": false, 3319 "min": false, 3320 "show": true, 3321 "total": false, 3322 "values": false 3323 }, 3324 "lines": true, 3325 "linewidth": 1, 3326 "nullPointMode": "null", 3327 "options": { 3328 "dataLinks": [ ] 3329 }, 3330 "panels": [ ], 3331 "percentage": false, 3332 "pointradius": 2, 3333 "points": false, 3334 "renderer": "flot", 3335 "seriesOverrides": [ ], 3336 "spaceLength": 10, 3337 "stack": true, 3338 "steppedLine": false, 3339 "targets": [ 3340 { 3341 "expr": "sum by(reason) (rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\",job=~\"$namespace/ingester\", namespace=~\"$namespace\"}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\",job=~\"$namespace/ingester\", namespace=~\"$namespace\"}[$__rate_interval]))", 3342 "interval": "", 3343 "legendFormat": "{{ reason }}" 3344 } 3345 ], 3346 "thresholds": [ ], 3347 "timeFrom": null, 3348 "timeRegions": [ ], 3349 "timeShift": null, 3350 "title": "Chunk Flush Reason %", 3351 "tooltip": { 3352 "shared": true, 3353 "sort": 0, 3354 "value_type": "individual" 3355 }, 3356 "type": "graph", 3357 "xaxis": { 3358 "buckets": null, 3359 "mode": "time", 3360 "name": null, 3361 "show": true, 3362 "values": [ ] 3363 }, 3364 "yaxes": [ 3365 { 3366 "format": "percentunit", 3367 "label": null, 3368 "logBase": 1, 3369 "max": "1", 3370 "min": "0", 3371 "show": true 3372 }, 3373 { 3374 "format": "short", 3375 "label": null, 3376 "logBase": 1, 3377 "max": null, 3378 "min": null, 3379 "show": true 3380 } 3381 ], 3382 "yaxis": { 3383 "align": false, 3384 "alignLevel": null 3385 } 3386 }, 3387 { 3388 "cards": { 3389 "cardPadding": null, 3390 "cardRound": null 3391 }, 3392 "color": { 3393 "cardColor": "#b4ff00", 3394 "colorScale": "sqrt", 3395 "colorScheme": "interpolateSpectral", 3396 "exponent": 0.5, 3397 "max": null, 3398 "min": null, 3399 "mode": "spectrum" 3400 }, 3401 "dataFormat": "tsbuckets", 3402 "datasource": "$datasource", 3403 "gridPos": { 3404 "h": 9, 3405 "w": 12, 3406 "x": 12, 3407 "y": 39 3408 }, 3409 "heatmap": { }, 3410 "hideZeroBuckets": true, 3411 "highlightCards": true, 3412 "id": 98, 3413 "legend": { 3414 "show": true 3415 }, 3416 "panels": [ ], 3417 "reverseYBuckets": false, 3418 "targets": [ 3419 { 3420 "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"($namespace)/ingester\"}[1m]))", 3421 "format": "heatmap", 3422 "instant": false, 3423 "interval": "", 3424 "legendFormat": "{{ le }}", 3425 "refId": "A" 3426 } 3427 ], 3428 "timeFrom": null, 3429 "timeShift": null, 3430 "title": "Chunk Utilization", 3431 "tooltip": { 3432 "show": true, 3433 "showHistogram": false 3434 }, 3435 "type": "heatmap", 3436 "xAxis": { 3437 "show": true 3438 }, 3439 "xBucketNumber": null, 3440 "xBucketSize": null, 3441 "yAxis": { 3442 "decimals": 0, 3443 "format": "percentunit", 3444 "logBase": 1, 3445 "max": null, 3446 "min": null, 3447 "show": true, 3448 "splitFactor": null 3449 }, 3450 "yBucketBound": "auto", 3451 "yBucketNumber": null, 3452 "yBucketSize": null 3453 } 3454 ], 3455 "targets": [ ], 3456 "title": "Chunks", 3457 "type": "row" 3458 }, 3459 { 3460 "collapsed": true, 3461 "datasource": null, 3462 "gridPos": { 3463 "h": 1, 3464 "w": 24, 3465 "x": 0, 3466 "y": 31 3467 }, 3468 "id": 64, 3469 "panels": [ 3470 { 3471 "aliasColors": { }, 3472 "bars": false, 3473 "dashLength": 10, 3474 "dashes": false, 3475 "datasource": "$datasource", 3476 "fill": 1, 3477 "fillGradient": 0, 3478 "gridPos": { 3479 "h": 7, 3480 "w": 6, 3481 "x": 0, 3482 "y": 32 3483 }, 3484 "hiddenSeries": false, 3485 "id": 68, 3486 "legend": { 3487 "avg": false, 3488 "current": false, 3489 "max": false, 3490 "min": false, 3491 "show": false, 3492 "total": false, 3493 "values": false 3494 }, 3495 "lines": false, 3496 "linewidth": 1, 3497 "nullPointMode": "null", 3498 "options": { 3499 "dataLinks": [ ] 3500 }, 3501 "panels": [ ], 3502 "percentage": false, 3503 "pointradius": 1, 3504 "points": true, 3505 "renderer": "flot", 3506 "seriesOverrides": [ ], 3507 "spaceLength": 10, 3508 "stack": false, 3509 "steppedLine": false, 3510 "targets": [ 3511 { 3512 "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"querier.*\"}", 3513 "intervalFactor": 3, 3514 "legendFormat": "{{pod}}-{{container}}", 3515 "refId": "A" 3516 } 3517 ], 3518 "thresholds": [ ], 3519 "timeFrom": null, 3520 "timeRegions": [ ], 3521 "timeShift": null, 3522 "title": "CPU Usage", 3523 "tooltip": { 3524 "shared": true, 3525 "sort": 2, 3526 "value_type": "individual" 3527 }, 3528 "type": "graph", 3529 "xaxis": { 3530 "buckets": null, 3531 "mode": "time", 3532 "name": null, 3533 "show": true, 3534 "values": [ ] 3535 }, 3536 "yaxes": [ 3537 { 3538 "format": "short", 3539 "label": null, 3540 "logBase": 1, 3541 "max": null, 3542 "min": null, 3543 "show": true 3544 }, 3545 { 3546 "format": "short", 3547 "label": null, 3548 "logBase": 1, 3549 "max": null, 3550 "min": null, 3551 "show": true 3552 } 3553 ], 3554 "yaxis": { 3555 "align": false, 3556 "alignLevel": null 3557 } 3558 }, 3559 { 3560 "aliasColors": { }, 3561 "bars": false, 3562 "dashLength": 10, 3563 "dashes": false, 3564 "datasource": "$datasource", 3565 "fill": 1, 3566 "fillGradient": 0, 3567 "gridPos": { 3568 "h": 7, 3569 "w": 6, 3570 "x": 0, 3571 "y": 39 3572 }, 3573 "hiddenSeries": false, 3574 "id": 69, 3575 "legend": { 3576 "avg": false, 3577 "current": false, 3578 "hideEmpty": false, 3579 "hideZero": false, 3580 "max": false, 3581 "min": false, 3582 "show": false, 3583 "total": false, 3584 "values": false 3585 }, 3586 "lines": false, 3587 "linewidth": 1, 3588 "nullPointMode": "null", 3589 "options": { 3590 "dataLinks": [ ] 3591 }, 3592 "panels": [ ], 3593 "percentage": false, 3594 "pointradius": 1, 3595 "points": true, 3596 "renderer": "flot", 3597 "seriesOverrides": [ ], 3598 "spaceLength": 10, 3599 "stack": false, 3600 "steppedLine": false, 3601 "targets": [ 3602 { 3603 "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"querier.*\"}", 3604 "instant": false, 3605 "intervalFactor": 3, 3606 "legendFormat": "{{pod}}", 3607 "refId": "A" 3608 } 3609 ], 3610 "thresholds": [ ], 3611 "timeFrom": null, 3612 "timeRegions": [ ], 3613 "timeShift": null, 3614 "title": "Memory Usage", 3615 "tooltip": { 3616 "shared": true, 3617 "sort": 2, 3618 "value_type": "individual" 3619 }, 3620 "type": "graph", 3621 "xaxis": { 3622 "buckets": null, 3623 "mode": "time", 3624 "name": null, 3625 "show": true, 3626 "values": [ ] 3627 }, 3628 "yaxes": [ 3629 { 3630 "format": "bytes", 3631 "label": null, 3632 "logBase": 1, 3633 "max": null, 3634 "min": null, 3635 "show": true 3636 }, 3637 { 3638 "format": "short", 3639 "label": null, 3640 "logBase": 1, 3641 "max": null, 3642 "min": null, 3643 "show": true 3644 } 3645 ], 3646 "yaxis": { 3647 "align": false, 3648 "alignLevel": null 3649 } 3650 }, 3651 { 3652 "aliasColors": { }, 3653 "bars": true, 3654 "dashLength": 10, 3655 "dashes": false, 3656 "datasource": "$logs", 3657 "fill": 1, 3658 "fillGradient": 0, 3659 "gridPos": { 3660 "h": 3, 3661 "w": 18, 3662 "x": 12, 3663 "y": 32 3664 }, 3665 "hiddenSeries": false, 3666 "id": 65, 3667 "legend": { 3668 "avg": false, 3669 "current": false, 3670 "max": false, 3671 "min": false, 3672 "show": false, 3673 "total": false, 3674 "values": false 3675 }, 3676 "lines": false, 3677 "linewidth": 1, 3678 "nullPointMode": "null", 3679 "options": { 3680 "dataLinks": [ ] 3681 }, 3682 "panels": [ ], 3683 "percentage": false, 3684 "pointradius": 2, 3685 "points": false, 3686 "renderer": "flot", 3687 "seriesOverrides": [ 3688 { 3689 "alias": "{}", 3690 "color": "#F2495C" 3691 } 3692 ], 3693 "spaceLength": 10, 3694 "stack": false, 3695 "steppedLine": false, 3696 "targets": [ 3697 { 3698 "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/querier\"} | logfmt | level=\"error\"[1m]))", 3699 "refId": "A" 3700 } 3701 ], 3702 "thresholds": [ ], 3703 "timeFrom": null, 3704 "timeRegions": [ ], 3705 "timeShift": null, 3706 "title": "Error Log Rate", 3707 "tooltip": { 3708 "shared": true, 3709 "sort": 2, 3710 "value_type": "individual" 3711 }, 3712 "type": "graph", 3713 "xaxis": { 3714 "buckets": null, 3715 "mode": "time", 3716 "name": null, 3717 "show": false, 3718 "values": [ ] 3719 }, 3720 "yaxes": [ 3721 { 3722 "format": "short", 3723 "label": null, 3724 "logBase": 1, 3725 "max": null, 3726 "min": null, 3727 "show": true 3728 }, 3729 { 3730 "format": "short", 3731 "label": null, 3732 "logBase": 1, 3733 "max": null, 3734 "min": null, 3735 "show": false 3736 } 3737 ], 3738 "yaxis": { 3739 "align": false, 3740 "alignLevel": null 3741 } 3742 }, 3743 { 3744 "datasource": "$logs", 3745 "gridPos": { 3746 "h": 18, 3747 "w": 18, 3748 "x": 12, 3749 "y": 35 3750 }, 3751 "id": 66, 3752 "options": { 3753 "showLabels": false, 3754 "showTime": false, 3755 "sortOrder": "Descending", 3756 "wrapLogMessage": true 3757 }, 3758 "panels": [ ], 3759 "targets": [ 3760 { 3761 "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/querier\"} | logfmt | level=\"error\"", 3762 "refId": "A" 3763 } 3764 ], 3765 "timeFrom": null, 3766 "timeShift": null, 3767 "title": "Logs", 3768 "type": "logs" 3769 }, 3770 { 3771 "aliasColors": { }, 3772 "bars": false, 3773 "dashLength": 10, 3774 "dashes": false, 3775 "datasource": "$datasource", 3776 "fill": 0, 3777 "fillGradient": 0, 3778 "gridPos": { 3779 "h": 7, 3780 "w": 6, 3781 "x": 0, 3782 "y": 46 3783 }, 3784 "hiddenSeries": false, 3785 "id": 70, 3786 "legend": { 3787 "avg": false, 3788 "current": false, 3789 "max": false, 3790 "min": false, 3791 "show": false, 3792 "total": false, 3793 "values": false 3794 }, 3795 "lines": true, 3796 "linewidth": 1, 3797 "nullPointMode": "null", 3798 "options": { 3799 "dataLinks": [ ] 3800 }, 3801 "panels": [ ], 3802 "percentage": false, 3803 "pointradius": 1, 3804 "points": false, 3805 "renderer": "flot", 3806 "seriesOverrides": [ ], 3807 "spaceLength": 10, 3808 "stack": false, 3809 "steppedLine": false, 3810 "targets": [ 3811 { 3812 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/querier\", status_code!~\"5[0-9]{2}\"}[1m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/querier\"}[1m])) by (route)", 3813 "interval": "", 3814 "intervalFactor": 1, 3815 "legendFormat": "{{route}}", 3816 "refId": "A" 3817 } 3818 ], 3819 "thresholds": [ ], 3820 "timeFrom": null, 3821 "timeRegions": [ ], 3822 "timeShift": null, 3823 "title": "Success Rate", 3824 "tooltip": { 3825 "shared": true, 3826 "sort": 2, 3827 "value_type": "individual" 3828 }, 3829 "type": "graph", 3830 "xaxis": { 3831 "buckets": null, 3832 "mode": "time", 3833 "name": null, 3834 "show": true, 3835 "values": [ ] 3836 }, 3837 "yaxes": [ 3838 { 3839 "format": "short", 3840 "label": null, 3841 "logBase": 1, 3842 "max": null, 3843 "min": null, 3844 "show": true 3845 }, 3846 { 3847 "format": "short", 3848 "label": null, 3849 "logBase": 1, 3850 "max": null, 3851 "min": null, 3852 "show": true 3853 } 3854 ], 3855 "yaxis": { 3856 "align": false, 3857 "alignLevel": null 3858 } 3859 } 3860 ], 3861 "targets": [ ], 3862 "title": "Querier", 3863 "type": "row" 3864 }, 3865 { 3866 "collapsed": true, 3867 "datasource": null, 3868 "gridPos": { 3869 "h": 1, 3870 "w": 24, 3871 "x": 0, 3872 "y": 32 3873 }, 3874 "id": 52, 3875 "panels": [ 3876 { 3877 "aliasColors": { }, 3878 "bars": false, 3879 "dashLength": 10, 3880 "dashes": false, 3881 "datasource": "$datasource", 3882 "fill": 1, 3883 "fillGradient": 0, 3884 "gridPos": { 3885 "h": 8, 3886 "w": 24, 3887 "x": 0, 3888 "y": 30 3889 }, 3890 "hiddenSeries": false, 3891 "id": 53, 3892 "interval": "", 3893 "legend": { 3894 "alignAsTable": true, 3895 "avg": false, 3896 "current": false, 3897 "max": false, 3898 "min": false, 3899 "rightSide": true, 3900 "show": true, 3901 "total": false, 3902 "values": false 3903 }, 3904 "lines": true, 3905 "linewidth": 1, 3906 "nullPointMode": "null", 3907 "options": { 3908 "dataLinks": [ ] 3909 }, 3910 "panels": [ ], 3911 "percentage": false, 3912 "pointradius": 1, 3913 "points": false, 3914 "renderer": "flot", 3915 "seriesOverrides": [ ], 3916 "spaceLength": 10, 3917 "stack": false, 3918 "steppedLine": false, 3919 "targets": [ 3920 { 3921 "expr": "histogram_quantile(.99, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", 3922 "intervalFactor": 1, 3923 "legendFormat": "{{container}}: .99-{{method}}-{{name}}", 3924 "refId": "A" 3925 }, 3926 { 3927 "expr": "histogram_quantile(.9, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", 3928 "hide": false, 3929 "legendFormat": "{{container}}: .9-{{method}}-{{name}}", 3930 "refId": "B" 3931 }, 3932 { 3933 "expr": "histogram_quantile(.5, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", 3934 "hide": false, 3935 "legendFormat": "{{container}}: .5-{{method}}-{{name}}", 3936 "refId": "C" 3937 } 3938 ], 3939 "thresholds": [ ], 3940 "timeFrom": null, 3941 "timeRegions": [ ], 3942 "timeShift": null, 3943 "title": "Latency By Method", 3944 "tooltip": { 3945 "shared": true, 3946 "sort": 2, 3947 "value_type": "individual" 3948 }, 3949 "type": "graph", 3950 "xaxis": { 3951 "buckets": null, 3952 "mode": "time", 3953 "name": null, 3954 "show": true, 3955 "values": [ ] 3956 }, 3957 "yaxes": [ 3958 { 3959 "format": "short", 3960 "label": null, 3961 "logBase": 1, 3962 "max": null, 3963 "min": null, 3964 "show": true 3965 }, 3966 { 3967 "format": "short", 3968 "label": null, 3969 "logBase": 1, 3970 "max": null, 3971 "min": null, 3972 "show": true 3973 } 3974 ], 3975 "yaxis": { 3976 "align": false, 3977 "alignLevel": null 3978 } 3979 }, 3980 { 3981 "aliasColors": { }, 3982 "bars": false, 3983 "dashLength": 10, 3984 "dashes": false, 3985 "datasource": "$datasource", 3986 "fill": 1, 3987 "fillGradient": 0, 3988 "gridPos": { 3989 "h": 8, 3990 "w": 24, 3991 "x": 0, 3992 "y": 38 3993 }, 3994 "hiddenSeries": false, 3995 "id": 54, 3996 "interval": "", 3997 "legend": { 3998 "alignAsTable": true, 3999 "avg": false, 4000 "current": false, 4001 "max": false, 4002 "min": false, 4003 "rightSide": true, 4004 "show": true, 4005 "total": false, 4006 "values": false 4007 }, 4008 "lines": true, 4009 "linewidth": 1, 4010 "nullPointMode": "null", 4011 "options": { 4012 "dataLinks": [ ] 4013 }, 4014 "panels": [ ], 4015 "percentage": false, 4016 "pointradius": 1, 4017 "points": false, 4018 "renderer": "flot", 4019 "seriesOverrides": [ ], 4020 "spaceLength": 10, 4021 "stack": false, 4022 "steppedLine": false, 4023 "targets": [ 4024 { 4025 "expr": "sum(rate(loki_memcache_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, method, name, container)", 4026 "intervalFactor": 1, 4027 "legendFormat": "{{container}}: {{status_code}}-{{method}}-{{name}}", 4028 "refId": "A" 4029 } 4030 ], 4031 "thresholds": [ ], 4032 "timeFrom": null, 4033 "timeRegions": [ ], 4034 "timeShift": null, 4035 "title": "Status By Method", 4036 "tooltip": { 4037 "shared": true, 4038 "sort": 2, 4039 "value_type": "individual" 4040 }, 4041 "type": "graph", 4042 "xaxis": { 4043 "buckets": null, 4044 "mode": "time", 4045 "name": null, 4046 "show": true, 4047 "values": [ ] 4048 }, 4049 "yaxes": [ 4050 { 4051 "format": "short", 4052 "label": null, 4053 "logBase": 1, 4054 "max": null, 4055 "min": null, 4056 "show": true 4057 }, 4058 { 4059 "format": "short", 4060 "label": null, 4061 "logBase": 1, 4062 "max": null, 4063 "min": null, 4064 "show": true 4065 } 4066 ], 4067 "yaxis": { 4068 "align": false, 4069 "alignLevel": null 4070 } 4071 } 4072 ], 4073 "targets": [ ], 4074 "title": "Memcached", 4075 "type": "row" 4076 }, 4077 { 4078 "collapsed": true, 4079 "datasource": null, 4080 "gridPos": { 4081 "h": 1, 4082 "w": 24, 4083 "x": 0, 4084 "y": 33 4085 }, 4086 "id": 57, 4087 "panels": [ 4088 { 4089 "aliasColors": { }, 4090 "bars": false, 4091 "dashLength": 10, 4092 "dashes": false, 4093 "datasource": "$datasource", 4094 "fill": 1, 4095 "fillGradient": 0, 4096 "gridPos": { 4097 "h": 8, 4098 "w": 24, 4099 "x": 0, 4100 "y": 31 4101 }, 4102 "hiddenSeries": false, 4103 "id": 55, 4104 "interval": "", 4105 "legend": { 4106 "alignAsTable": true, 4107 "avg": false, 4108 "current": false, 4109 "max": false, 4110 "min": false, 4111 "rightSide": true, 4112 "show": true, 4113 "total": false, 4114 "values": false 4115 }, 4116 "lines": true, 4117 "linewidth": 1, 4118 "nullPointMode": "null", 4119 "options": { 4120 "dataLinks": [ ] 4121 }, 4122 "panels": [ ], 4123 "percentage": false, 4124 "pointradius": 1, 4125 "points": false, 4126 "renderer": "flot", 4127 "seriesOverrides": [ ], 4128 "spaceLength": 10, 4129 "stack": false, 4130 "steppedLine": false, 4131 "targets": [ 4132 { 4133 "expr": "histogram_quantile(.99, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 4134 "intervalFactor": 1, 4135 "legendFormat": ".99-{{operation}}", 4136 "refId": "A" 4137 }, 4138 { 4139 "expr": "histogram_quantile(.9, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 4140 "hide": false, 4141 "legendFormat": ".9-{{operation}}", 4142 "refId": "B" 4143 }, 4144 { 4145 "expr": "histogram_quantile(.5, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 4146 "hide": false, 4147 "legendFormat": ".5-{{operation}}", 4148 "refId": "C" 4149 } 4150 ], 4151 "thresholds": [ ], 4152 "timeFrom": null, 4153 "timeRegions": [ ], 4154 "timeShift": null, 4155 "title": "Latency By Operation", 4156 "tooltip": { 4157 "shared": true, 4158 "sort": 2, 4159 "value_type": "individual" 4160 }, 4161 "type": "graph", 4162 "xaxis": { 4163 "buckets": null, 4164 "mode": "time", 4165 "name": null, 4166 "show": true, 4167 "values": [ ] 4168 }, 4169 "yaxes": [ 4170 { 4171 "format": "short", 4172 "label": null, 4173 "logBase": 1, 4174 "max": null, 4175 "min": null, 4176 "show": true 4177 }, 4178 { 4179 "format": "short", 4180 "label": null, 4181 "logBase": 1, 4182 "max": null, 4183 "min": null, 4184 "show": true 4185 } 4186 ], 4187 "yaxis": { 4188 "align": false, 4189 "alignLevel": null 4190 } 4191 }, 4192 { 4193 "aliasColors": { }, 4194 "bars": false, 4195 "dashLength": 10, 4196 "dashes": false, 4197 "datasource": "$datasource", 4198 "fill": 1, 4199 "fillGradient": 0, 4200 "gridPos": { 4201 "h": 8, 4202 "w": 24, 4203 "x": 0, 4204 "y": 39 4205 }, 4206 "hiddenSeries": false, 4207 "id": 58, 4208 "interval": "", 4209 "legend": { 4210 "alignAsTable": true, 4211 "avg": false, 4212 "current": false, 4213 "max": false, 4214 "min": false, 4215 "rightSide": true, 4216 "show": true, 4217 "total": false, 4218 "values": false 4219 }, 4220 "lines": true, 4221 "linewidth": 1, 4222 "nullPointMode": "null", 4223 "options": { 4224 "dataLinks": [ ] 4225 }, 4226 "panels": [ ], 4227 "percentage": false, 4228 "pointradius": 1, 4229 "points": false, 4230 "renderer": "flot", 4231 "seriesOverrides": [ ], 4232 "spaceLength": 10, 4233 "stack": false, 4234 "steppedLine": false, 4235 "targets": [ 4236 { 4237 "expr": "sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, status_code, method)", 4238 "intervalFactor": 1, 4239 "legendFormat": "{{status_code}}-{{operation}}", 4240 "refId": "A" 4241 } 4242 ], 4243 "thresholds": [ ], 4244 "timeFrom": null, 4245 "timeRegions": [ ], 4246 "timeShift": null, 4247 "title": "Status By Operation", 4248 "tooltip": { 4249 "shared": true, 4250 "sort": 2, 4251 "value_type": "individual" 4252 }, 4253 "type": "graph", 4254 "xaxis": { 4255 "buckets": null, 4256 "mode": "time", 4257 "name": null, 4258 "show": true, 4259 "values": [ ] 4260 }, 4261 "yaxes": [ 4262 { 4263 "format": "short", 4264 "label": null, 4265 "logBase": 1, 4266 "max": null, 4267 "min": null, 4268 "show": true 4269 }, 4270 { 4271 "format": "short", 4272 "label": null, 4273 "logBase": 1, 4274 "max": null, 4275 "min": null, 4276 "show": true 4277 } 4278 ], 4279 "yaxis": { 4280 "align": false, 4281 "alignLevel": null 4282 } 4283 } 4284 ], 4285 "targets": [ ], 4286 "title": "Consul", 4287 "type": "row" 4288 }, 4289 { 4290 "collapsed": true, 4291 "datasource": null, 4292 "gridPos": { 4293 "h": 1, 4294 "w": 24, 4295 "x": 0, 4296 "y": 34 4297 }, 4298 "id": 43, 4299 "panels": [ 4300 { 4301 "aliasColors": { }, 4302 "bars": false, 4303 "dashLength": 10, 4304 "dashes": false, 4305 "datasource": "$datasource", 4306 "fill": 1, 4307 "fillGradient": 0, 4308 "gridPos": { 4309 "h": 7, 4310 "w": 6, 4311 "x": 0, 4312 "y": 9 4313 }, 4314 "hiddenSeries": false, 4315 "id": 41, 4316 "interval": "", 4317 "legend": { 4318 "avg": false, 4319 "current": false, 4320 "max": false, 4321 "min": false, 4322 "show": false, 4323 "total": false, 4324 "values": false 4325 }, 4326 "lines": true, 4327 "linewidth": 1, 4328 "nullPointMode": "null", 4329 "options": { 4330 "dataLinks": [ ] 4331 }, 4332 "panels": [ ], 4333 "percentage": false, 4334 "pointradius": 1, 4335 "points": false, 4336 "renderer": "flot", 4337 "seriesOverrides": [ ], 4338 "spaceLength": 10, 4339 "stack": false, 4340 "steppedLine": false, 4341 "targets": [ 4342 { 4343 "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", 4344 "intervalFactor": 1, 4345 "legendFormat": ".9", 4346 "refId": "A" 4347 }, 4348 { 4349 "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", 4350 "refId": "B" 4351 }, 4352 { 4353 "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", 4354 "refId": "C" 4355 } 4356 ], 4357 "thresholds": [ ], 4358 "timeFrom": null, 4359 "timeRegions": [ ], 4360 "timeShift": null, 4361 "title": "MutateRows Latency", 4362 "tooltip": { 4363 "shared": true, 4364 "sort": 2, 4365 "value_type": "individual" 4366 }, 4367 "type": "graph", 4368 "xaxis": { 4369 "buckets": null, 4370 "mode": "time", 4371 "name": null, 4372 "show": true, 4373 "values": [ ] 4374 }, 4375 "yaxes": [ 4376 { 4377 "format": "short", 4378 "label": null, 4379 "logBase": 1, 4380 "max": null, 4381 "min": null, 4382 "show": true 4383 }, 4384 { 4385 "format": "short", 4386 "label": null, 4387 "logBase": 1, 4388 "max": null, 4389 "min": null, 4390 "show": true 4391 } 4392 ], 4393 "yaxis": { 4394 "align": false, 4395 "alignLevel": null 4396 } 4397 }, 4398 { 4399 "aliasColors": { }, 4400 "bars": false, 4401 "dashLength": 10, 4402 "dashes": false, 4403 "datasource": "$datasource", 4404 "fill": 1, 4405 "fillGradient": 0, 4406 "gridPos": { 4407 "h": 7, 4408 "w": 6, 4409 "x": 6, 4410 "y": 9 4411 }, 4412 "hiddenSeries": false, 4413 "id": 46, 4414 "interval": "", 4415 "legend": { 4416 "avg": false, 4417 "current": false, 4418 "max": false, 4419 "min": false, 4420 "show": false, 4421 "total": false, 4422 "values": false 4423 }, 4424 "lines": true, 4425 "linewidth": 1, 4426 "nullPointMode": "null", 4427 "options": { 4428 "dataLinks": [ ] 4429 }, 4430 "panels": [ ], 4431 "percentage": false, 4432 "pointradius": 1, 4433 "points": false, 4434 "renderer": "flot", 4435 "seriesOverrides": [ ], 4436 "spaceLength": 10, 4437 "stack": false, 4438 "steppedLine": false, 4439 "targets": [ 4440 { 4441 "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", 4442 "interval": "", 4443 "intervalFactor": 1, 4444 "legendFormat": "99%", 4445 "refId": "A" 4446 }, 4447 { 4448 "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", 4449 "interval": "", 4450 "legendFormat": "90%", 4451 "refId": "B" 4452 }, 4453 { 4454 "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", 4455 "interval": "", 4456 "legendFormat": "50%", 4457 "refId": "C" 4458 } 4459 ], 4460 "thresholds": [ ], 4461 "timeFrom": null, 4462 "timeRegions": [ ], 4463 "timeShift": null, 4464 "title": "ReadRows Latency", 4465 "tooltip": { 4466 "shared": true, 4467 "sort": 2, 4468 "value_type": "individual" 4469 }, 4470 "type": "graph", 4471 "xaxis": { 4472 "buckets": null, 4473 "mode": "time", 4474 "name": null, 4475 "show": true, 4476 "values": [ ] 4477 }, 4478 "yaxes": [ 4479 { 4480 "format": "short", 4481 "label": null, 4482 "logBase": 1, 4483 "max": null, 4484 "min": null, 4485 "show": true 4486 }, 4487 { 4488 "format": "short", 4489 "label": null, 4490 "logBase": 1, 4491 "max": null, 4492 "min": null, 4493 "show": true 4494 } 4495 ], 4496 "yaxis": { 4497 "align": false, 4498 "alignLevel": null 4499 } 4500 }, 4501 { 4502 "aliasColors": { }, 4503 "bars": false, 4504 "dashLength": 10, 4505 "dashes": false, 4506 "datasource": "$datasource", 4507 "fill": 1, 4508 "fillGradient": 0, 4509 "gridPos": { 4510 "h": 7, 4511 "w": 6, 4512 "x": 12, 4513 "y": 9 4514 }, 4515 "hiddenSeries": false, 4516 "id": 44, 4517 "interval": "", 4518 "legend": { 4519 "avg": false, 4520 "current": false, 4521 "max": false, 4522 "min": false, 4523 "show": false, 4524 "total": false, 4525 "values": false 4526 }, 4527 "lines": true, 4528 "linewidth": 1, 4529 "nullPointMode": "null", 4530 "options": { 4531 "dataLinks": [ ] 4532 }, 4533 "panels": [ ], 4534 "percentage": false, 4535 "pointradius": 1, 4536 "points": false, 4537 "renderer": "flot", 4538 "seriesOverrides": [ ], 4539 "spaceLength": 10, 4540 "stack": false, 4541 "steppedLine": false, 4542 "targets": [ 4543 { 4544 "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", 4545 "interval": "", 4546 "intervalFactor": 1, 4547 "legendFormat": "99%", 4548 "refId": "A" 4549 }, 4550 { 4551 "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", 4552 "interval": "", 4553 "legendFormat": "90%", 4554 "refId": "B" 4555 }, 4556 { 4557 "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", 4558 "interval": "", 4559 "legendFormat": "50%", 4560 "refId": "C" 4561 } 4562 ], 4563 "thresholds": [ ], 4564 "timeFrom": null, 4565 "timeRegions": [ ], 4566 "timeShift": null, 4567 "title": "GetTable Latency", 4568 "tooltip": { 4569 "shared": true, 4570 "sort": 2, 4571 "value_type": "individual" 4572 }, 4573 "type": "graph", 4574 "xaxis": { 4575 "buckets": null, 4576 "mode": "time", 4577 "name": null, 4578 "show": true, 4579 "values": [ ] 4580 }, 4581 "yaxes": [ 4582 { 4583 "format": "short", 4584 "label": null, 4585 "logBase": 1, 4586 "max": null, 4587 "min": null, 4588 "show": true 4589 }, 4590 { 4591 "format": "short", 4592 "label": null, 4593 "logBase": 1, 4594 "max": null, 4595 "min": null, 4596 "show": true 4597 } 4598 ], 4599 "yaxis": { 4600 "align": false, 4601 "alignLevel": null 4602 } 4603 }, 4604 { 4605 "aliasColors": { }, 4606 "bars": false, 4607 "dashLength": 10, 4608 "dashes": false, 4609 "datasource": "$datasource", 4610 "fill": 1, 4611 "fillGradient": 0, 4612 "gridPos": { 4613 "h": 7, 4614 "w": 6, 4615 "x": 18, 4616 "y": 9 4617 }, 4618 "hiddenSeries": false, 4619 "id": 45, 4620 "interval": "", 4621 "legend": { 4622 "avg": false, 4623 "current": false, 4624 "max": false, 4625 "min": false, 4626 "show": false, 4627 "total": false, 4628 "values": false 4629 }, 4630 "lines": true, 4631 "linewidth": 1, 4632 "nullPointMode": "null", 4633 "options": { 4634 "dataLinks": [ ] 4635 }, 4636 "panels": [ ], 4637 "percentage": false, 4638 "pointradius": 1, 4639 "points": false, 4640 "renderer": "flot", 4641 "seriesOverrides": [ ], 4642 "spaceLength": 10, 4643 "stack": false, 4644 "steppedLine": false, 4645 "targets": [ 4646 { 4647 "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", 4648 "intervalFactor": 1, 4649 "legendFormat": ".9", 4650 "refId": "A" 4651 }, 4652 { 4653 "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", 4654 "refId": "B" 4655 }, 4656 { 4657 "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", 4658 "refId": "C" 4659 } 4660 ], 4661 "thresholds": [ ], 4662 "timeFrom": null, 4663 "timeRegions": [ ], 4664 "timeShift": null, 4665 "title": "ListTables Latency", 4666 "tooltip": { 4667 "shared": true, 4668 "sort": 2, 4669 "value_type": "individual" 4670 }, 4671 "type": "graph", 4672 "xaxis": { 4673 "buckets": null, 4674 "mode": "time", 4675 "name": null, 4676 "show": true, 4677 "values": [ ] 4678 }, 4679 "yaxes": [ 4680 { 4681 "format": "short", 4682 "label": null, 4683 "logBase": 1, 4684 "max": null, 4685 "min": null, 4686 "show": true 4687 }, 4688 { 4689 "format": "short", 4690 "label": null, 4691 "logBase": 1, 4692 "max": null, 4693 "min": null, 4694 "show": true 4695 } 4696 ], 4697 "yaxis": { 4698 "align": false, 4699 "alignLevel": null 4700 } 4701 }, 4702 { 4703 "aliasColors": { }, 4704 "bars": false, 4705 "dashLength": 10, 4706 "dashes": false, 4707 "datasource": "$datasource", 4708 "fill": 1, 4709 "fillGradient": 0, 4710 "gridPos": { 4711 "h": 7, 4712 "w": 6, 4713 "x": 0, 4714 "y": 16 4715 }, 4716 "hiddenSeries": false, 4717 "id": 47, 4718 "interval": "", 4719 "legend": { 4720 "avg": false, 4721 "current": false, 4722 "max": false, 4723 "min": false, 4724 "show": false, 4725 "total": false, 4726 "values": false 4727 }, 4728 "lines": true, 4729 "linewidth": 1, 4730 "nullPointMode": "null", 4731 "options": { 4732 "dataLinks": [ ] 4733 }, 4734 "panels": [ ], 4735 "percentage": false, 4736 "pointradius": 1, 4737 "points": false, 4738 "renderer": "flot", 4739 "seriesOverrides": [ ], 4740 "spaceLength": 10, 4741 "stack": false, 4742 "steppedLine": false, 4743 "targets": [ 4744 { 4745 "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (status_code)", 4746 "intervalFactor": 1, 4747 "legendFormat": "{{status_code}}", 4748 "refId": "A" 4749 } 4750 ], 4751 "thresholds": [ ], 4752 "timeFrom": null, 4753 "timeRegions": [ ], 4754 "timeShift": null, 4755 "title": "MutateRows Status", 4756 "tooltip": { 4757 "shared": true, 4758 "sort": 2, 4759 "value_type": "individual" 4760 }, 4761 "type": "graph", 4762 "xaxis": { 4763 "buckets": null, 4764 "mode": "time", 4765 "name": null, 4766 "show": true, 4767 "values": [ ] 4768 }, 4769 "yaxes": [ 4770 { 4771 "format": "short", 4772 "label": null, 4773 "logBase": 1, 4774 "max": null, 4775 "min": null, 4776 "show": true 4777 }, 4778 { 4779 "format": "short", 4780 "label": null, 4781 "logBase": 1, 4782 "max": null, 4783 "min": null, 4784 "show": true 4785 } 4786 ], 4787 "yaxis": { 4788 "align": false, 4789 "alignLevel": null 4790 } 4791 }, 4792 { 4793 "aliasColors": { }, 4794 "bars": false, 4795 "dashLength": 10, 4796 "dashes": false, 4797 "datasource": "$datasource", 4798 "fill": 1, 4799 "fillGradient": 0, 4800 "gridPos": { 4801 "h": 7, 4802 "w": 6, 4803 "x": 6, 4804 "y": 16 4805 }, 4806 "hiddenSeries": false, 4807 "id": 50, 4808 "interval": "", 4809 "legend": { 4810 "avg": false, 4811 "current": false, 4812 "max": false, 4813 "min": false, 4814 "show": false, 4815 "total": false, 4816 "values": false 4817 }, 4818 "lines": true, 4819 "linewidth": 1, 4820 "nullPointMode": "null", 4821 "options": { 4822 "dataLinks": [ ] 4823 }, 4824 "panels": [ ], 4825 "percentage": false, 4826 "pointradius": 1, 4827 "points": false, 4828 "renderer": "flot", 4829 "seriesOverrides": [ ], 4830 "spaceLength": 10, 4831 "stack": false, 4832 "steppedLine": false, 4833 "targets": [ 4834 { 4835 "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (status_code)", 4836 "intervalFactor": 1, 4837 "legendFormat": "{{status_code}}", 4838 "refId": "A" 4839 } 4840 ], 4841 "thresholds": [ ], 4842 "timeFrom": null, 4843 "timeRegions": [ ], 4844 "timeShift": null, 4845 "title": "ReadRows Status", 4846 "tooltip": { 4847 "shared": true, 4848 "sort": 2, 4849 "value_type": "individual" 4850 }, 4851 "type": "graph", 4852 "xaxis": { 4853 "buckets": null, 4854 "mode": "time", 4855 "name": null, 4856 "show": true, 4857 "values": [ ] 4858 }, 4859 "yaxes": [ 4860 { 4861 "format": "short", 4862 "label": null, 4863 "logBase": 1, 4864 "max": null, 4865 "min": null, 4866 "show": true 4867 }, 4868 { 4869 "format": "short", 4870 "label": null, 4871 "logBase": 1, 4872 "max": null, 4873 "min": null, 4874 "show": true 4875 } 4876 ], 4877 "yaxis": { 4878 "align": false, 4879 "alignLevel": null 4880 } 4881 }, 4882 { 4883 "aliasColors": { }, 4884 "bars": false, 4885 "dashLength": 10, 4886 "dashes": false, 4887 "datasource": "$datasource", 4888 "fill": 1, 4889 "fillGradient": 0, 4890 "gridPos": { 4891 "h": 7, 4892 "w": 6, 4893 "x": 12, 4894 "y": 16 4895 }, 4896 "hiddenSeries": false, 4897 "id": 48, 4898 "interval": "", 4899 "legend": { 4900 "avg": false, 4901 "current": false, 4902 "max": false, 4903 "min": false, 4904 "show": false, 4905 "total": false, 4906 "values": false 4907 }, 4908 "lines": true, 4909 "linewidth": 1, 4910 "nullPointMode": "null", 4911 "options": { 4912 "dataLinks": [ ] 4913 }, 4914 "panels": [ ], 4915 "percentage": false, 4916 "pointradius": 1, 4917 "points": false, 4918 "renderer": "flot", 4919 "seriesOverrides": [ ], 4920 "spaceLength": 10, 4921 "stack": false, 4922 "steppedLine": false, 4923 "targets": [ 4924 { 4925 "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (status_code)", 4926 "intervalFactor": 1, 4927 "legendFormat": "{{status_code}}", 4928 "refId": "A" 4929 } 4930 ], 4931 "thresholds": [ ], 4932 "timeFrom": null, 4933 "timeRegions": [ ], 4934 "timeShift": null, 4935 "title": "GetTable Status", 4936 "tooltip": { 4937 "shared": true, 4938 "sort": 2, 4939 "value_type": "individual" 4940 }, 4941 "type": "graph", 4942 "xaxis": { 4943 "buckets": null, 4944 "mode": "time", 4945 "name": null, 4946 "show": true, 4947 "values": [ ] 4948 }, 4949 "yaxes": [ 4950 { 4951 "format": "short", 4952 "label": null, 4953 "logBase": 1, 4954 "max": null, 4955 "min": null, 4956 "show": true 4957 }, 4958 { 4959 "format": "short", 4960 "label": null, 4961 "logBase": 1, 4962 "max": null, 4963 "min": null, 4964 "show": true 4965 } 4966 ], 4967 "yaxis": { 4968 "align": false, 4969 "alignLevel": null 4970 } 4971 }, 4972 { 4973 "aliasColors": { }, 4974 "bars": false, 4975 "dashLength": 10, 4976 "dashes": false, 4977 "datasource": "$datasource", 4978 "fill": 1, 4979 "fillGradient": 0, 4980 "gridPos": { 4981 "h": 7, 4982 "w": 6, 4983 "x": 18, 4984 "y": 16 4985 }, 4986 "hiddenSeries": false, 4987 "id": 49, 4988 "interval": "", 4989 "legend": { 4990 "avg": false, 4991 "current": false, 4992 "max": false, 4993 "min": false, 4994 "show": false, 4995 "total": false, 4996 "values": false 4997 }, 4998 "lines": true, 4999 "linewidth": 1, 5000 "nullPointMode": "null", 5001 "options": { 5002 "dataLinks": [ ] 5003 }, 5004 "panels": [ ], 5005 "percentage": false, 5006 "pointradius": 1, 5007 "points": false, 5008 "renderer": "flot", 5009 "seriesOverrides": [ ], 5010 "spaceLength": 10, 5011 "stack": false, 5012 "steppedLine": false, 5013 "targets": [ 5014 { 5015 "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (status_code)", 5016 "intervalFactor": 1, 5017 "legendFormat": "{{status_code}}", 5018 "refId": "A" 5019 } 5020 ], 5021 "thresholds": [ ], 5022 "timeFrom": null, 5023 "timeRegions": [ ], 5024 "timeShift": null, 5025 "title": "ListTables Status", 5026 "tooltip": { 5027 "shared": true, 5028 "sort": 2, 5029 "value_type": "individual" 5030 }, 5031 "type": "graph", 5032 "xaxis": { 5033 "buckets": null, 5034 "mode": "time", 5035 "name": null, 5036 "show": true, 5037 "values": [ ] 5038 }, 5039 "yaxes": [ 5040 { 5041 "format": "short", 5042 "label": null, 5043 "logBase": 1, 5044 "max": null, 5045 "min": null, 5046 "show": true 5047 }, 5048 { 5049 "format": "short", 5050 "label": null, 5051 "logBase": 1, 5052 "max": null, 5053 "min": null, 5054 "show": true 5055 } 5056 ], 5057 "yaxis": { 5058 "align": false, 5059 "alignLevel": null 5060 } 5061 } 5062 ], 5063 "targets": [ ], 5064 "title": "Big Table", 5065 "type": "row" 5066 }, 5067 { 5068 "collapsed": true, 5069 "datasource": null, 5070 "gridPos": { 5071 "h": 1, 5072 "w": 24, 5073 "x": 0, 5074 "y": 35 5075 }, 5076 "id": 60, 5077 "panels": [ 5078 { 5079 "aliasColors": { }, 5080 "bars": false, 5081 "dashLength": 10, 5082 "dashes": false, 5083 "datasource": "$datasource", 5084 "fill": 1, 5085 "fillGradient": 0, 5086 "gridPos": { 5087 "h": 8, 5088 "w": 24, 5089 "x": 0, 5090 "y": 33 5091 }, 5092 "hiddenSeries": false, 5093 "id": 61, 5094 "interval": "", 5095 "legend": { 5096 "alignAsTable": true, 5097 "avg": false, 5098 "current": false, 5099 "max": false, 5100 "min": false, 5101 "rightSide": true, 5102 "show": true, 5103 "total": false, 5104 "values": false 5105 }, 5106 "lines": true, 5107 "linewidth": 1, 5108 "nullPointMode": "null", 5109 "options": { 5110 "dataLinks": [ ] 5111 }, 5112 "panels": [ ], 5113 "percentage": false, 5114 "pointradius": 1, 5115 "points": false, 5116 "renderer": "flot", 5117 "seriesOverrides": [ ], 5118 "spaceLength": 10, 5119 "stack": false, 5120 "steppedLine": false, 5121 "targets": [ 5122 { 5123 "expr": "histogram_quantile(.99, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5124 "intervalFactor": 1, 5125 "legendFormat": ".99-{{operation}}", 5126 "refId": "A" 5127 }, 5128 { 5129 "expr": "histogram_quantile(.9, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5130 "hide": false, 5131 "legendFormat": ".9-{{operation}}", 5132 "refId": "B" 5133 }, 5134 { 5135 "expr": "histogram_quantile(.5, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5136 "hide": false, 5137 "legendFormat": ".5-{{operation}}", 5138 "refId": "C" 5139 } 5140 ], 5141 "thresholds": [ ], 5142 "timeFrom": null, 5143 "timeRegions": [ ], 5144 "timeShift": null, 5145 "title": "Latency By Operation", 5146 "tooltip": { 5147 "shared": true, 5148 "sort": 2, 5149 "value_type": "individual" 5150 }, 5151 "type": "graph", 5152 "xaxis": { 5153 "buckets": null, 5154 "mode": "time", 5155 "name": null, 5156 "show": true, 5157 "values": [ ] 5158 }, 5159 "yaxes": [ 5160 { 5161 "format": "short", 5162 "label": null, 5163 "logBase": 1, 5164 "max": null, 5165 "min": null, 5166 "show": true 5167 }, 5168 { 5169 "format": "short", 5170 "label": null, 5171 "logBase": 1, 5172 "max": null, 5173 "min": null, 5174 "show": true 5175 } 5176 ], 5177 "yaxis": { 5178 "align": false, 5179 "alignLevel": null 5180 } 5181 }, 5182 { 5183 "aliasColors": { }, 5184 "bars": false, 5185 "dashLength": 10, 5186 "dashes": false, 5187 "datasource": "$datasource", 5188 "fill": 1, 5189 "fillGradient": 0, 5190 "gridPos": { 5191 "h": 8, 5192 "w": 24, 5193 "x": 0, 5194 "y": 41 5195 }, 5196 "hiddenSeries": false, 5197 "id": 62, 5198 "interval": "", 5199 "legend": { 5200 "alignAsTable": true, 5201 "avg": false, 5202 "current": false, 5203 "max": false, 5204 "min": false, 5205 "rightSide": true, 5206 "show": true, 5207 "total": false, 5208 "values": false 5209 }, 5210 "lines": true, 5211 "linewidth": 1, 5212 "nullPointMode": "null", 5213 "options": { 5214 "dataLinks": [ ] 5215 }, 5216 "panels": [ ], 5217 "percentage": false, 5218 "pointradius": 1, 5219 "points": false, 5220 "renderer": "flot", 5221 "seriesOverrides": [ ], 5222 "spaceLength": 10, 5223 "stack": false, 5224 "steppedLine": false, 5225 "targets": [ 5226 { 5227 "expr": "sum(rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", 5228 "intervalFactor": 1, 5229 "legendFormat": "{{status_code}}-{{operation}}", 5230 "refId": "A" 5231 } 5232 ], 5233 "thresholds": [ ], 5234 "timeFrom": null, 5235 "timeRegions": [ ], 5236 "timeShift": null, 5237 "title": "Status By Method", 5238 "tooltip": { 5239 "shared": true, 5240 "sort": 2, 5241 "value_type": "individual" 5242 }, 5243 "type": "graph", 5244 "xaxis": { 5245 "buckets": null, 5246 "mode": "time", 5247 "name": null, 5248 "show": true, 5249 "values": [ ] 5250 }, 5251 "yaxes": [ 5252 { 5253 "format": "short", 5254 "label": null, 5255 "logBase": 1, 5256 "max": null, 5257 "min": null, 5258 "show": true 5259 }, 5260 { 5261 "format": "short", 5262 "label": null, 5263 "logBase": 1, 5264 "max": null, 5265 "min": null, 5266 "show": true 5267 } 5268 ], 5269 "yaxis": { 5270 "align": false, 5271 "alignLevel": null 5272 } 5273 } 5274 ], 5275 "targets": [ ], 5276 "title": "GCS", 5277 "type": "row" 5278 }, 5279 { 5280 "collapsed": true, 5281 "datasource": null, 5282 "gridPos": { 5283 "h": 1, 5284 "w": 24, 5285 "x": 0, 5286 "y": 36 5287 }, 5288 "id": 76, 5289 "panels": [ 5290 { 5291 "aliasColors": { }, 5292 "bars": false, 5293 "dashLength": 10, 5294 "dashes": false, 5295 "datasource": null, 5296 "fill": 1, 5297 "fillGradient": 0, 5298 "gridPos": { 5299 "h": 6, 5300 "w": 6, 5301 "x": 0, 5302 "y": 9 5303 }, 5304 "id": 82, 5305 "legend": { 5306 "avg": false, 5307 "current": false, 5308 "max": false, 5309 "min": false, 5310 "show": true, 5311 "total": false, 5312 "values": false 5313 }, 5314 "lines": true, 5315 "linewidth": 1, 5316 "nullPointMode": "null", 5317 "options": { 5318 "dataLinks": [ ] 5319 }, 5320 "panels": [ ], 5321 "percentage": false, 5322 "pointradius": 2, 5323 "points": false, 5324 "renderer": "flot", 5325 "seriesOverrides": [ ], 5326 "spaceLength": 10, 5327 "stack": false, 5328 "steppedLine": false, 5329 "targets": [ 5330 { 5331 "expr": "sum(rate(cortex_dynamo_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", 5332 "refId": "A" 5333 } 5334 ], 5335 "thresholds": [ ], 5336 "timeFrom": null, 5337 "timeRegions": [ ], 5338 "timeShift": null, 5339 "title": "Failure Rate", 5340 "tooltip": { 5341 "shared": true, 5342 "sort": 0, 5343 "value_type": "individual" 5344 }, 5345 "type": "graph", 5346 "xaxis": { 5347 "buckets": null, 5348 "mode": "time", 5349 "name": null, 5350 "show": true, 5351 "values": [ ] 5352 }, 5353 "yaxes": [ 5354 { 5355 "format": "short", 5356 "label": null, 5357 "logBase": 1, 5358 "max": null, 5359 "min": null, 5360 "show": true 5361 }, 5362 { 5363 "format": "short", 5364 "label": null, 5365 "logBase": 1, 5366 "max": null, 5367 "min": null, 5368 "show": true 5369 } 5370 ], 5371 "yaxis": { 5372 "align": false, 5373 "alignLevel": null 5374 } 5375 }, 5376 { 5377 "aliasColors": { }, 5378 "bars": false, 5379 "dashLength": 10, 5380 "dashes": false, 5381 "datasource": null, 5382 "fill": 1, 5383 "fillGradient": 0, 5384 "gridPos": { 5385 "h": 6, 5386 "w": 6, 5387 "x": 6, 5388 "y": 9 5389 }, 5390 "id": 83, 5391 "legend": { 5392 "avg": false, 5393 "current": false, 5394 "max": false, 5395 "min": false, 5396 "show": true, 5397 "total": false, 5398 "values": false 5399 }, 5400 "lines": true, 5401 "linewidth": 1, 5402 "nullPointMode": "null", 5403 "options": { 5404 "dataLinks": [ ] 5405 }, 5406 "panels": [ ], 5407 "percentage": false, 5408 "pointradius": 2, 5409 "points": false, 5410 "renderer": "flot", 5411 "seriesOverrides": [ ], 5412 "spaceLength": 10, 5413 "stack": false, 5414 "steppedLine": false, 5415 "targets": [ 5416 { 5417 "expr": "sum(rate(cortex_dynamo_consumed_capacity_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", 5418 "refId": "A" 5419 } 5420 ], 5421 "thresholds": [ ], 5422 "timeFrom": null, 5423 "timeRegions": [ ], 5424 "timeShift": null, 5425 "title": "Consumed Capacity Rate", 5426 "tooltip": { 5427 "shared": true, 5428 "sort": 0, 5429 "value_type": "individual" 5430 }, 5431 "type": "graph", 5432 "xaxis": { 5433 "buckets": null, 5434 "mode": "time", 5435 "name": null, 5436 "show": true, 5437 "values": [ ] 5438 }, 5439 "yaxes": [ 5440 { 5441 "format": "short", 5442 "label": null, 5443 "logBase": 1, 5444 "max": null, 5445 "min": null, 5446 "show": true 5447 }, 5448 { 5449 "format": "short", 5450 "label": null, 5451 "logBase": 1, 5452 "max": null, 5453 "min": null, 5454 "show": true 5455 } 5456 ], 5457 "yaxis": { 5458 "align": false, 5459 "alignLevel": null 5460 } 5461 }, 5462 { 5463 "aliasColors": { }, 5464 "bars": false, 5465 "dashLength": 10, 5466 "dashes": false, 5467 "datasource": null, 5468 "fill": 1, 5469 "fillGradient": 0, 5470 "gridPos": { 5471 "h": 6, 5472 "w": 6, 5473 "x": 12, 5474 "y": 9 5475 }, 5476 "id": 84, 5477 "legend": { 5478 "avg": false, 5479 "current": false, 5480 "max": false, 5481 "min": false, 5482 "show": true, 5483 "total": false, 5484 "values": false 5485 }, 5486 "lines": true, 5487 "linewidth": 1, 5488 "nullPointMode": "null", 5489 "options": { 5490 "dataLinks": [ ] 5491 }, 5492 "panels": [ ], 5493 "percentage": false, 5494 "pointradius": 2, 5495 "points": false, 5496 "renderer": "flot", 5497 "seriesOverrides": [ ], 5498 "spaceLength": 10, 5499 "stack": false, 5500 "steppedLine": false, 5501 "targets": [ 5502 { 5503 "expr": "sum(rate(cortex_dynamo_throttled_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", 5504 "refId": "A" 5505 } 5506 ], 5507 "thresholds": [ ], 5508 "timeFrom": null, 5509 "timeRegions": [ ], 5510 "timeShift": null, 5511 "title": "Throttled Rate", 5512 "tooltip": { 5513 "shared": true, 5514 "sort": 0, 5515 "value_type": "individual" 5516 }, 5517 "type": "graph", 5518 "xaxis": { 5519 "buckets": null, 5520 "mode": "time", 5521 "name": null, 5522 "show": true, 5523 "values": [ ] 5524 }, 5525 "yaxes": [ 5526 { 5527 "format": "short", 5528 "label": null, 5529 "logBase": 1, 5530 "max": null, 5531 "min": null, 5532 "show": true 5533 }, 5534 { 5535 "format": "short", 5536 "label": null, 5537 "logBase": 1, 5538 "max": null, 5539 "min": null, 5540 "show": true 5541 } 5542 ], 5543 "yaxis": { 5544 "align": false, 5545 "alignLevel": null 5546 } 5547 }, 5548 { 5549 "aliasColors": { }, 5550 "bars": false, 5551 "dashLength": 10, 5552 "dashes": false, 5553 "datasource": null, 5554 "fill": 1, 5555 "fillGradient": 0, 5556 "gridPos": { 5557 "h": 6, 5558 "w": 6, 5559 "x": 18, 5560 "y": 9 5561 }, 5562 "id": 85, 5563 "legend": { 5564 "avg": false, 5565 "current": false, 5566 "max": false, 5567 "min": false, 5568 "show": true, 5569 "total": false, 5570 "values": false 5571 }, 5572 "lines": true, 5573 "linewidth": 1, 5574 "nullPointMode": "null", 5575 "options": { 5576 "dataLinks": [ ] 5577 }, 5578 "panels": [ ], 5579 "percentage": false, 5580 "pointradius": 2, 5581 "points": false, 5582 "renderer": "flot", 5583 "seriesOverrides": [ ], 5584 "spaceLength": 10, 5585 "stack": false, 5586 "steppedLine": false, 5587 "targets": [ 5588 { 5589 "expr": "sum(rate(cortex_dynamo_dropped_requests_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", 5590 "refId": "A" 5591 } 5592 ], 5593 "thresholds": [ ], 5594 "timeFrom": null, 5595 "timeRegions": [ ], 5596 "timeShift": null, 5597 "title": "Dropped Rate", 5598 "tooltip": { 5599 "shared": true, 5600 "sort": 0, 5601 "value_type": "individual" 5602 }, 5603 "type": "graph", 5604 "xaxis": { 5605 "buckets": null, 5606 "mode": "time", 5607 "name": null, 5608 "show": true, 5609 "values": [ ] 5610 }, 5611 "yaxes": [ 5612 { 5613 "format": "short", 5614 "label": null, 5615 "logBase": 1, 5616 "max": null, 5617 "min": null, 5618 "show": true 5619 }, 5620 { 5621 "format": "short", 5622 "label": null, 5623 "logBase": 1, 5624 "max": null, 5625 "min": null, 5626 "show": true 5627 } 5628 ], 5629 "yaxis": { 5630 "align": false, 5631 "alignLevel": null 5632 } 5633 }, 5634 { 5635 "aliasColors": { }, 5636 "bars": false, 5637 "dashLength": 10, 5638 "dashes": false, 5639 "datasource": null, 5640 "fill": 1, 5641 "fillGradient": 0, 5642 "gridPos": { 5643 "h": 6, 5644 "w": 6, 5645 "x": 0, 5646 "y": 15 5647 }, 5648 "id": 86, 5649 "legend": { 5650 "avg": false, 5651 "current": false, 5652 "max": false, 5653 "min": false, 5654 "show": true, 5655 "total": false, 5656 "values": false 5657 }, 5658 "lines": true, 5659 "linewidth": 1, 5660 "nullPointMode": "null", 5661 "options": { 5662 "dataLinks": [ ] 5663 }, 5664 "panels": [ ], 5665 "percentage": false, 5666 "pointradius": 2, 5667 "points": false, 5668 "renderer": "flot", 5669 "seriesOverrides": [ ], 5670 "spaceLength": 10, 5671 "stack": false, 5672 "steppedLine": false, 5673 "targets": [ 5674 { 5675 "expr": "histogram_quantile(.99, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", 5676 "legendFormat": ".99", 5677 "refId": "A" 5678 }, 5679 { 5680 "expr": "histogram_quantile(.9, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", 5681 "legendFormat": ".9", 5682 "refId": "B" 5683 }, 5684 { 5685 "expr": "histogram_quantile(.5, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", 5686 "legendFormat": ".5", 5687 "refId": "C" 5688 } 5689 ], 5690 "thresholds": [ ], 5691 "timeFrom": null, 5692 "timeRegions": [ ], 5693 "timeShift": null, 5694 "title": "Query Pages", 5695 "tooltip": { 5696 "shared": true, 5697 "sort": 0, 5698 "value_type": "individual" 5699 }, 5700 "type": "graph", 5701 "xaxis": { 5702 "buckets": null, 5703 "mode": "time", 5704 "name": null, 5705 "show": true, 5706 "values": [ ] 5707 }, 5708 "yaxes": [ 5709 { 5710 "format": "short", 5711 "label": null, 5712 "logBase": 1, 5713 "max": null, 5714 "min": null, 5715 "show": true 5716 }, 5717 { 5718 "format": "short", 5719 "label": null, 5720 "logBase": 1, 5721 "max": null, 5722 "min": null, 5723 "show": true 5724 } 5725 ], 5726 "yaxis": { 5727 "align": false, 5728 "alignLevel": null 5729 } 5730 }, 5731 { 5732 "aliasColors": { }, 5733 "bars": false, 5734 "dashLength": 10, 5735 "dashes": false, 5736 "datasource": "$datasource", 5737 "fill": 1, 5738 "fillGradient": 0, 5739 "gridPos": { 5740 "h": 6, 5741 "w": 9, 5742 "x": 6, 5743 "y": 15 5744 }, 5745 "id": 87, 5746 "interval": "", 5747 "legend": { 5748 "alignAsTable": true, 5749 "avg": false, 5750 "current": false, 5751 "max": false, 5752 "min": false, 5753 "rightSide": true, 5754 "show": true, 5755 "total": false, 5756 "values": false 5757 }, 5758 "lines": true, 5759 "linewidth": 1, 5760 "nullPointMode": "null", 5761 "options": { 5762 "dataLinks": [ ] 5763 }, 5764 "panels": [ ], 5765 "percentage": false, 5766 "pointradius": 1, 5767 "points": false, 5768 "renderer": "flot", 5769 "seriesOverrides": [ ], 5770 "spaceLength": 10, 5771 "stack": false, 5772 "steppedLine": false, 5773 "targets": [ 5774 { 5775 "expr": "histogram_quantile(.99, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5776 "intervalFactor": 1, 5777 "legendFormat": ".99-{{operation}}", 5778 "refId": "A" 5779 }, 5780 { 5781 "expr": "histogram_quantile(.9, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5782 "hide": false, 5783 "legendFormat": ".9-{{operation}}", 5784 "refId": "B" 5785 }, 5786 { 5787 "expr": "histogram_quantile(.5, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5788 "hide": false, 5789 "legendFormat": ".5-{{operation}}", 5790 "refId": "C" 5791 } 5792 ], 5793 "thresholds": [ ], 5794 "timeFrom": null, 5795 "timeRegions": [ ], 5796 "timeShift": null, 5797 "title": "Latency By Operation", 5798 "tooltip": { 5799 "shared": true, 5800 "sort": 2, 5801 "value_type": "individual" 5802 }, 5803 "type": "graph", 5804 "xaxis": { 5805 "buckets": null, 5806 "mode": "time", 5807 "name": null, 5808 "show": true, 5809 "values": [ ] 5810 }, 5811 "yaxes": [ 5812 { 5813 "format": "short", 5814 "label": null, 5815 "logBase": 1, 5816 "max": null, 5817 "min": null, 5818 "show": true 5819 }, 5820 { 5821 "format": "short", 5822 "label": null, 5823 "logBase": 1, 5824 "max": null, 5825 "min": null, 5826 "show": true 5827 } 5828 ], 5829 "yaxis": { 5830 "align": false, 5831 "alignLevel": null 5832 } 5833 }, 5834 { 5835 "aliasColors": { }, 5836 "bars": false, 5837 "dashLength": 10, 5838 "dashes": false, 5839 "datasource": "$datasource", 5840 "fill": 1, 5841 "fillGradient": 0, 5842 "gridPos": { 5843 "h": 6, 5844 "w": 9, 5845 "x": 15, 5846 "y": 15 5847 }, 5848 "id": 88, 5849 "interval": "", 5850 "legend": { 5851 "alignAsTable": true, 5852 "avg": false, 5853 "current": false, 5854 "max": false, 5855 "min": false, 5856 "rightSide": true, 5857 "show": true, 5858 "total": false, 5859 "values": false 5860 }, 5861 "lines": true, 5862 "linewidth": 1, 5863 "nullPointMode": "null", 5864 "options": { 5865 "dataLinks": [ ] 5866 }, 5867 "panels": [ ], 5868 "percentage": false, 5869 "pointradius": 1, 5870 "points": false, 5871 "renderer": "flot", 5872 "seriesOverrides": [ ], 5873 "spaceLength": 10, 5874 "stack": false, 5875 "steppedLine": false, 5876 "targets": [ 5877 { 5878 "expr": "sum(rate(cortex_dynamo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", 5879 "intervalFactor": 1, 5880 "legendFormat": "{{status_code}}-{{operation}}", 5881 "refId": "A" 5882 } 5883 ], 5884 "thresholds": [ ], 5885 "timeFrom": null, 5886 "timeRegions": [ ], 5887 "timeShift": null, 5888 "title": "Status By Method", 5889 "tooltip": { 5890 "shared": true, 5891 "sort": 2, 5892 "value_type": "individual" 5893 }, 5894 "type": "graph", 5895 "xaxis": { 5896 "buckets": null, 5897 "mode": "time", 5898 "name": null, 5899 "show": true, 5900 "values": [ ] 5901 }, 5902 "yaxes": [ 5903 { 5904 "format": "short", 5905 "label": null, 5906 "logBase": 1, 5907 "max": null, 5908 "min": null, 5909 "show": true 5910 }, 5911 { 5912 "format": "short", 5913 "label": null, 5914 "logBase": 1, 5915 "max": null, 5916 "min": null, 5917 "show": true 5918 } 5919 ], 5920 "yaxis": { 5921 "align": false, 5922 "alignLevel": null 5923 } 5924 } 5925 ], 5926 "targets": [ ], 5927 "title": "Dynamo", 5928 "type": "row" 5929 }, 5930 { 5931 "collapsed": true, 5932 "datasource": null, 5933 "gridPos": { 5934 "h": 1, 5935 "w": 24, 5936 "x": 0, 5937 "y": 37 5938 }, 5939 "id": 78, 5940 "panels": [ 5941 { 5942 "aliasColors": { }, 5943 "bars": false, 5944 "dashLength": 10, 5945 "dashes": false, 5946 "datasource": "$datasource", 5947 "fill": 1, 5948 "fillGradient": 0, 5949 "gridPos": { 5950 "h": 8, 5951 "w": 24, 5952 "x": 0, 5953 "y": 10 5954 }, 5955 "id": 79, 5956 "interval": "", 5957 "legend": { 5958 "alignAsTable": true, 5959 "avg": false, 5960 "current": false, 5961 "max": false, 5962 "min": false, 5963 "rightSide": true, 5964 "show": true, 5965 "total": false, 5966 "values": false 5967 }, 5968 "lines": true, 5969 "linewidth": 1, 5970 "nullPointMode": "null", 5971 "options": { 5972 "dataLinks": [ ] 5973 }, 5974 "panels": [ ], 5975 "percentage": false, 5976 "pointradius": 1, 5977 "points": false, 5978 "renderer": "flot", 5979 "seriesOverrides": [ ], 5980 "spaceLength": 10, 5981 "stack": false, 5982 "steppedLine": false, 5983 "targets": [ 5984 { 5985 "expr": "histogram_quantile(.99, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5986 "intervalFactor": 1, 5987 "legendFormat": ".99-{{operation}}", 5988 "refId": "A" 5989 }, 5990 { 5991 "expr": "histogram_quantile(.9, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5992 "hide": false, 5993 "legendFormat": ".9-{{operation}}", 5994 "refId": "B" 5995 }, 5996 { 5997 "expr": "histogram_quantile(.5, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5998 "hide": false, 5999 "legendFormat": ".5-{{operation}}", 6000 "refId": "C" 6001 } 6002 ], 6003 "thresholds": [ ], 6004 "timeFrom": null, 6005 "timeRegions": [ ], 6006 "timeShift": null, 6007 "title": "Latency By Operation", 6008 "tooltip": { 6009 "shared": true, 6010 "sort": 2, 6011 "value_type": "individual" 6012 }, 6013 "type": "graph", 6014 "xaxis": { 6015 "buckets": null, 6016 "mode": "time", 6017 "name": null, 6018 "show": true, 6019 "values": [ ] 6020 }, 6021 "yaxes": [ 6022 { 6023 "format": "short", 6024 "label": null, 6025 "logBase": 1, 6026 "max": null, 6027 "min": null, 6028 "show": true 6029 }, 6030 { 6031 "format": "short", 6032 "label": null, 6033 "logBase": 1, 6034 "max": null, 6035 "min": null, 6036 "show": true 6037 } 6038 ], 6039 "yaxis": { 6040 "align": false, 6041 "alignLevel": null 6042 } 6043 }, 6044 { 6045 "aliasColors": { }, 6046 "bars": false, 6047 "dashLength": 10, 6048 "dashes": false, 6049 "datasource": "$datasource", 6050 "fill": 1, 6051 "fillGradient": 0, 6052 "gridPos": { 6053 "h": 8, 6054 "w": 24, 6055 "x": 0, 6056 "y": 18 6057 }, 6058 "id": 80, 6059 "interval": "", 6060 "legend": { 6061 "alignAsTable": true, 6062 "avg": false, 6063 "current": false, 6064 "max": false, 6065 "min": false, 6066 "rightSide": true, 6067 "show": true, 6068 "total": false, 6069 "values": false 6070 }, 6071 "lines": true, 6072 "linewidth": 1, 6073 "nullPointMode": "null", 6074 "options": { 6075 "dataLinks": [ ] 6076 }, 6077 "panels": [ ], 6078 "percentage": false, 6079 "pointradius": 1, 6080 "points": false, 6081 "renderer": "flot", 6082 "seriesOverrides": [ ], 6083 "spaceLength": 10, 6084 "stack": false, 6085 "steppedLine": false, 6086 "targets": [ 6087 { 6088 "expr": "sum(rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", 6089 "intervalFactor": 1, 6090 "legendFormat": "{{status_code}}-{{operation}}", 6091 "refId": "A" 6092 } 6093 ], 6094 "thresholds": [ ], 6095 "timeFrom": null, 6096 "timeRegions": [ ], 6097 "timeShift": null, 6098 "title": "Status By Method", 6099 "tooltip": { 6100 "shared": true, 6101 "sort": 2, 6102 "value_type": "individual" 6103 }, 6104 "type": "graph", 6105 "xaxis": { 6106 "buckets": null, 6107 "mode": "time", 6108 "name": null, 6109 "show": true, 6110 "values": [ ] 6111 }, 6112 "yaxes": [ 6113 { 6114 "format": "short", 6115 "label": null, 6116 "logBase": 1, 6117 "max": null, 6118 "min": null, 6119 "show": true 6120 }, 6121 { 6122 "format": "short", 6123 "label": null, 6124 "logBase": 1, 6125 "max": null, 6126 "min": null, 6127 "show": true 6128 } 6129 ], 6130 "yaxis": { 6131 "align": false, 6132 "alignLevel": null 6133 } 6134 } 6135 ], 6136 "targets": [ ], 6137 "title": "S3", 6138 "type": "row" 6139 }, 6140 { 6141 "collapsed": true, 6142 "datasource": null, 6143 "gridPos": { 6144 "h": 1, 6145 "w": 24, 6146 "x": 0, 6147 "y": 37 6148 }, 6149 "id": 78, 6150 "panels": [ 6151 { 6152 "aliasColors": { }, 6153 "bars": false, 6154 "dashLength": 10, 6155 "dashes": false, 6156 "datasource": "$datasource", 6157 "fill": 1, 6158 "fillGradient": 0, 6159 "gridPos": { 6160 "h": 8, 6161 "w": 24, 6162 "x": 0, 6163 "y": 10 6164 }, 6165 "id": 79, 6166 "interval": "", 6167 "legend": { 6168 "alignAsTable": true, 6169 "avg": false, 6170 "current": false, 6171 "max": false, 6172 "min": false, 6173 "rightSide": true, 6174 "show": true, 6175 "total": false, 6176 "values": false 6177 }, 6178 "lines": true, 6179 "linewidth": 1, 6180 "nullPointMode": "null", 6181 "options": { 6182 "dataLinks": [ ] 6183 }, 6184 "panels": [ ], 6185 "percentage": false, 6186 "pointradius": 1, 6187 "points": false, 6188 "renderer": "flot", 6189 "seriesOverrides": [ ], 6190 "spaceLength": 10, 6191 "stack": false, 6192 "steppedLine": false, 6193 "targets": [ 6194 { 6195 "expr": "histogram_quantile(.99, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 6196 "intervalFactor": 1, 6197 "legendFormat": ".99-{{operation}}", 6198 "refId": "A" 6199 }, 6200 { 6201 "expr": "histogram_quantile(.9, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 6202 "hide": false, 6203 "legendFormat": ".9-{{operation}}", 6204 "refId": "B" 6205 }, 6206 { 6207 "expr": "histogram_quantile(.5, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 6208 "hide": false, 6209 "legendFormat": ".5-{{operation}}", 6210 "refId": "C" 6211 } 6212 ], 6213 "thresholds": [ ], 6214 "timeFrom": null, 6215 "timeRegions": [ ], 6216 "timeShift": null, 6217 "title": "Latency By Operation", 6218 "tooltip": { 6219 "shared": true, 6220 "sort": 2, 6221 "value_type": "individual" 6222 }, 6223 "type": "graph", 6224 "xaxis": { 6225 "buckets": null, 6226 "mode": "time", 6227 "name": null, 6228 "show": true, 6229 "values": [ ] 6230 }, 6231 "yaxes": [ 6232 { 6233 "format": "short", 6234 "label": null, 6235 "logBase": 1, 6236 "max": null, 6237 "min": null, 6238 "show": true 6239 }, 6240 { 6241 "format": "short", 6242 "label": null, 6243 "logBase": 1, 6244 "max": null, 6245 "min": null, 6246 "show": true 6247 } 6248 ], 6249 "yaxis": { 6250 "align": false, 6251 "alignLevel": null 6252 } 6253 }, 6254 { 6255 "aliasColors": { }, 6256 "bars": false, 6257 "dashLength": 10, 6258 "dashes": false, 6259 "datasource": "$datasource", 6260 "fill": 1, 6261 "fillGradient": 0, 6262 "gridPos": { 6263 "h": 8, 6264 "w": 24, 6265 "x": 0, 6266 "y": 18 6267 }, 6268 "id": 80, 6269 "interval": "", 6270 "legend": { 6271 "alignAsTable": true, 6272 "avg": false, 6273 "current": false, 6274 "max": false, 6275 "min": false, 6276 "rightSide": true, 6277 "show": true, 6278 "total": false, 6279 "values": false 6280 }, 6281 "lines": true, 6282 "linewidth": 1, 6283 "nullPointMode": "null", 6284 "options": { 6285 "dataLinks": [ ] 6286 }, 6287 "panels": [ ], 6288 "percentage": false, 6289 "pointradius": 1, 6290 "points": false, 6291 "renderer": "flot", 6292 "seriesOverrides": [ ], 6293 "spaceLength": 10, 6294 "stack": false, 6295 "steppedLine": false, 6296 "targets": [ 6297 { 6298 "expr": "sum(rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", 6299 "intervalFactor": 1, 6300 "legendFormat": "{{status_code}}-{{operation}}", 6301 "refId": "A" 6302 } 6303 ], 6304 "thresholds": [ ], 6305 "timeFrom": null, 6306 "timeRegions": [ ], 6307 "timeShift": null, 6308 "title": "Status By Method", 6309 "tooltip": { 6310 "shared": true, 6311 "sort": 2, 6312 "value_type": "individual" 6313 }, 6314 "type": "graph", 6315 "xaxis": { 6316 "buckets": null, 6317 "mode": "time", 6318 "name": null, 6319 "show": true, 6320 "values": [ ] 6321 }, 6322 "yaxes": [ 6323 { 6324 "format": "short", 6325 "label": null, 6326 "logBase": 1, 6327 "max": null, 6328 "min": null, 6329 "show": true 6330 }, 6331 { 6332 "format": "short", 6333 "label": null, 6334 "logBase": 1, 6335 "max": null, 6336 "min": null, 6337 "show": true 6338 } 6339 ], 6340 "yaxis": { 6341 "align": false, 6342 "alignLevel": null 6343 } 6344 } 6345 ], 6346 "targets": [ ], 6347 "title": "Azure Blob", 6348 "type": "row" 6349 }, 6350 { 6351 "collapsed": true, 6352 "datasource": null, 6353 "gridPos": { 6354 "h": 1, 6355 "w": 24, 6356 "x": 0, 6357 "y": 37 6358 }, 6359 "id": 114, 6360 "panels": [ 6361 { 6362 "aliasColors": { }, 6363 "bars": false, 6364 "dashLength": 10, 6365 "dashes": false, 6366 "datasource": "$datasource", 6367 "fill": 1, 6368 "fillGradient": 0, 6369 "gridPos": { 6370 "h": 8, 6371 "w": 24, 6372 "x": 0, 6373 "y": 10 6374 }, 6375 "id": 115, 6376 "interval": "", 6377 "legend": { 6378 "alignAsTable": true, 6379 "avg": false, 6380 "current": false, 6381 "max": false, 6382 "min": false, 6383 "rightSide": true, 6384 "show": true, 6385 "total": false, 6386 "values": false 6387 }, 6388 "lines": true, 6389 "linewidth": 1, 6390 "nullPointMode": "null", 6391 "options": { 6392 "dataLinks": [ ] 6393 }, 6394 "panels": [ ], 6395 "percentage": false, 6396 "pointradius": 1, 6397 "points": false, 6398 "renderer": "flot", 6399 "seriesOverrides": [ ], 6400 "spaceLength": 10, 6401 "stack": false, 6402 "steppedLine": false, 6403 "targets": [ 6404 { 6405 "expr": "histogram_quantile(.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 6406 "intervalFactor": 1, 6407 "legendFormat": ".99-{{operation}}", 6408 "refId": "A" 6409 }, 6410 { 6411 "expr": "histogram_quantile(.9, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 6412 "hide": false, 6413 "legendFormat": ".9-{{operation}}", 6414 "refId": "B" 6415 }, 6416 { 6417 "expr": "histogram_quantile(.5, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 6418 "hide": false, 6419 "legendFormat": ".5-{{operation}}", 6420 "refId": "C" 6421 } 6422 ], 6423 "thresholds": [ ], 6424 "timeFrom": null, 6425 "timeRegions": [ ], 6426 "timeShift": null, 6427 "title": "Latency By Operation", 6428 "tooltip": { 6429 "shared": true, 6430 "sort": 2, 6431 "value_type": "individual" 6432 }, 6433 "type": "graph", 6434 "xaxis": { 6435 "buckets": null, 6436 "mode": "time", 6437 "name": null, 6438 "show": true, 6439 "values": [ ] 6440 }, 6441 "yaxes": [ 6442 { 6443 "format": "short", 6444 "label": null, 6445 "logBase": 1, 6446 "max": null, 6447 "min": null, 6448 "show": true 6449 }, 6450 { 6451 "format": "short", 6452 "label": null, 6453 "logBase": 1, 6454 "max": null, 6455 "min": null, 6456 "show": true 6457 } 6458 ], 6459 "yaxis": { 6460 "align": false, 6461 "alignLevel": null 6462 } 6463 }, 6464 { 6465 "aliasColors": { }, 6466 "bars": false, 6467 "dashLength": 10, 6468 "dashes": false, 6469 "datasource": "$datasource", 6470 "fill": 1, 6471 "fillGradient": 0, 6472 "gridPos": { 6473 "h": 8, 6474 "w": 24, 6475 "x": 0, 6476 "y": 18 6477 }, 6478 "id": 116, 6479 "interval": "", 6480 "legend": { 6481 "alignAsTable": true, 6482 "avg": false, 6483 "current": false, 6484 "max": false, 6485 "min": false, 6486 "rightSide": true, 6487 "show": true, 6488 "total": false, 6489 "values": false 6490 }, 6491 "lines": true, 6492 "linewidth": 1, 6493 "nullPointMode": "null", 6494 "options": { 6495 "dataLinks": [ ] 6496 }, 6497 "panels": [ ], 6498 "percentage": false, 6499 "pointradius": 1, 6500 "points": false, 6501 "renderer": "flot", 6502 "seriesOverrides": [ ], 6503 "spaceLength": 10, 6504 "stack": false, 6505 "steppedLine": false, 6506 "targets": [ 6507 { 6508 "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", 6509 "intervalFactor": 1, 6510 "legendFormat": "{{status_code}}-{{operation}}", 6511 "refId": "A" 6512 } 6513 ], 6514 "thresholds": [ ], 6515 "timeFrom": null, 6516 "timeRegions": [ ], 6517 "timeShift": null, 6518 "title": "Status By Method", 6519 "tooltip": { 6520 "shared": true, 6521 "sort": 2, 6522 "value_type": "individual" 6523 }, 6524 "type": "graph", 6525 "xaxis": { 6526 "buckets": null, 6527 "mode": "time", 6528 "name": null, 6529 "show": true, 6530 "values": [ ] 6531 }, 6532 "yaxes": [ 6533 { 6534 "format": "short", 6535 "label": null, 6536 "logBase": 1, 6537 "max": null, 6538 "min": null, 6539 "show": true 6540 }, 6541 { 6542 "format": "short", 6543 "label": null, 6544 "logBase": 1, 6545 "max": null, 6546 "min": null, 6547 "show": true 6548 } 6549 ], 6550 "yaxis": { 6551 "align": false, 6552 "alignLevel": null 6553 } 6554 } 6555 ], 6556 "targets": [ ], 6557 "title": "BoltDB Shipper", 6558 "type": "row" 6559 } 6560 ], 6561 "refresh": "10s", 6562 "rows": [ ], 6563 "schemaVersion": 14, 6564 "style": "dark", 6565 "tags": [ 6566 "loki" 6567 ], 6568 "templating": { 6569 "list": [ 6570 { 6571 "current": { 6572 "text": "default", 6573 "value": "default" 6574 }, 6575 "hide": 0, 6576 "label": "Data Source", 6577 "name": "datasource", 6578 "options": [ ], 6579 "query": "prometheus", 6580 "refresh": 1, 6581 "regex": "", 6582 "type": "datasource" 6583 }, 6584 { 6585 "hide": 0, 6586 "label": null, 6587 "name": "logs", 6588 "options": [ ], 6589 "query": "loki", 6590 "refresh": 1, 6591 "regex": "", 6592 "type": "datasource" 6593 }, 6594 { 6595 "allValue": null, 6596 "current": { 6597 "text": "prod", 6598 "value": "prod" 6599 }, 6600 "datasource": "$datasource", 6601 "hide": 0, 6602 "includeAll": false, 6603 "label": "cluster", 6604 "multi": false, 6605 "name": "cluster", 6606 "options": [ ], 6607 "query": "label_values(loki_build_info, cluster)", 6608 "refresh": 1, 6609 "regex": "", 6610 "sort": 2, 6611 "tagValuesQuery": "", 6612 "tags": [ ], 6613 "tagsQuery": "", 6614 "type": "query", 6615 "useTags": false 6616 }, 6617 { 6618 "allValue": null, 6619 "current": { 6620 "text": "prod", 6621 "value": "prod" 6622 }, 6623 "datasource": "$datasource", 6624 "hide": 0, 6625 "includeAll": false, 6626 "label": "namespace", 6627 "multi": false, 6628 "name": "namespace", 6629 "options": [ ], 6630 "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", 6631 "refresh": 1, 6632 "regex": "", 6633 "sort": 2, 6634 "tagValuesQuery": "", 6635 "tags": [ ], 6636 "tagsQuery": "", 6637 "type": "query", 6638 "useTags": false 6639 } 6640 ] 6641 }, 6642 "time": { 6643 "from": "now-1h", 6644 "to": "now" 6645 }, 6646 "timepicker": { 6647 "refresh_intervals": [ 6648 "5s", 6649 "10s", 6650 "30s", 6651 "1m", 6652 "5m", 6653 "15m", 6654 "30m", 6655 "1h", 6656 "2h", 6657 "1d" 6658 ], 6659 "time_options": [ 6660 "5m", 6661 "15m", 6662 "1h", 6663 "6h", 6664 "12h", 6665 "24h", 6666 "2d", 6667 "7d", 6668 "30d" 6669 ] 6670 }, 6671 "timezone": "utc", 6672 "title": "Loki / Operational", 6673 "uid": "operational", 6674 "version": 0 6675 }