github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/production/loki-mixin/dashboards/dashboard-loki-operational.json (about) 1 { 2 "annotations": { 3 "list": [ 4 { 5 "builtIn": 1, 6 "datasource": "-- Grafana --", 7 "enable": true, 8 "hide": true, 9 "iconColor": "rgba(0, 211, 255, 1)", 10 "name": "Annotations & Alerts", 11 "type": "dashboard" 12 }, 13 { 14 "datasource": "$logs", 15 "enable": true, 16 "expr": "{cluster=\"$cluster\", container=\"kube-diff-logger\"} | json | namespace_extracted=\"$namespace\"", 17 "hide": true, 18 "iconColor": "rgba(255, 96, 96, 1)", 19 "name": "deployments", 20 "showIn": 0, 21 "target": {} 22 } 23 ] 24 }, 25 "editable": true, 26 "gnetId": null, 27 "graphTooltip": 1, 28 "id": 68, 29 "iteration": 1588704280892, 30 "links": [ 31 { 32 "icon": "dashboard", 33 "includeVars": true, 34 "keepTime": true, 35 "tags": [], 36 "targetBlank": true, 37 "title": "Chunks", 38 "tooltip": "", 39 "type": "link", 40 "url": "/grafana/d/586c2916e59d3f81282a776dbe8333e9/loki-chunks" 41 }, 42 { 43 "icon": "dashboard", 44 "includeVars": true, 45 "keepTime": true, 46 "tags": [], 47 "targetBlank": true, 48 "title": "Memcached", 49 "tooltip": "", 50 "type": "link", 51 "url": "/grafana/d/8127d7f60e3a3230d7c633c8b5ddf68c/memcached" 52 }, 53 { 54 "icon": "dashboard", 55 "includeVars": false, 56 "keepTime": true, 57 "tags": [], 58 "targetBlank": true, 59 "title": "Promtail", 60 "type": "link", 61 "url": "/grafana/d/739c15a23c04c6ace0d03f6e14ba26e8/loki-promtail?var-datasource=$datasource&var-cluster=$cluster&var-namespace=default" 62 }, 63 { 64 "icon": "dashboard", 65 "includeVars": true, 66 "keepTime": true, 67 "tags": [], 68 "targetBlank": true, 69 "title": "BigTable Backup", 70 "type": "link", 71 "url": "/grafana/d/da5ce918801d6d9f775a8c7e9f9f339a/loki-bigtable-backup" 72 }, 73 { 74 "icon": "dashboard", 75 "keepTime": true, 76 "tags": [], 77 "targetBlank": true, 78 "title": "Consul", 79 "type": "link", 80 "url": "/grafana/d/836793349c8ce200baa98f045123d19b/consul?var-datasource=$datasource&var-job=$namespace%2Fconsul" 81 } 82 ], 83 "panels": [ 84 { 85 "collapsed": false, 86 "datasource": null, 87 "gridPos": { 88 "h": 1, 89 "w": 24, 90 "x": 0, 91 "y": 0 92 }, 93 "id": 17, 94 "panels": [], 95 "title": "Main", 96 "type": "row" 97 }, 98 { 99 "aliasColors": { 100 "5xx": "red" 101 }, 102 "bars": false, 103 "dashLength": 10, 104 "dashes": false, 105 "datasource": "$datasource", 106 "fieldConfig": { 107 "defaults": { 108 "custom": {} 109 }, 110 "overrides": [] 111 }, 112 "fill": 1, 113 "fillGradient": 0, 114 "gridPos": { 115 "h": 5, 116 "w": 4, 117 "x": 0, 118 "y": 1 119 }, 120 "hiddenSeries": false, 121 "id": 6, 122 "legend": { 123 "avg": false, 124 "current": false, 125 "max": false, 126 "min": false, 127 "show": true, 128 "total": false, 129 "values": false 130 }, 131 "lines": true, 132 "linewidth": 1, 133 "nullPointMode": "null", 134 "options": { 135 "dataLinks": [] 136 }, 137 "percentage": false, 138 "pointradius": 2, 139 "points": false, 140 "renderer": "flot", 141 "seriesOverrides": [], 142 "spaceLength": 10, 143 "stack": false, 144 "steppedLine": false, 145 "targets": [ 146 { 147 "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/cortex-gw\", route=~\"api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}[5m]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)", 148 "legendFormat": "{{status}}", 149 "refId": "A" 150 } 151 ], 152 "thresholds": [], 153 "timeFrom": null, 154 "timeRegions": [], 155 "timeShift": null, 156 "title": "Queries/Second", 157 "tooltip": { 158 "shared": true, 159 "sort": 0, 160 "value_type": "individual" 161 }, 162 "type": "graph", 163 "xaxis": { 164 "buckets": null, 165 "mode": "time", 166 "name": null, 167 "show": true, 168 "values": [] 169 }, 170 "yaxes": [ 171 { 172 "format": "short", 173 "label": null, 174 "logBase": 10, 175 "max": null, 176 "min": null, 177 "show": true 178 }, 179 { 180 "format": "short", 181 "label": null, 182 "logBase": 1, 183 "max": null, 184 "min": null, 185 "show": true 186 } 187 ], 188 "yaxis": { 189 "align": false, 190 "alignLevel": null 191 } 192 }, 193 { 194 "aliasColors": { 195 "5xx": "red" 196 }, 197 "bars": false, 198 "dashLength": 10, 199 "dashes": false, 200 "datasource": "$datasource", 201 "fieldConfig": { 202 "defaults": { 203 "custom": {} 204 }, 205 "overrides": [] 206 }, 207 "fill": 1, 208 "fillGradient": 0, 209 "gridPos": { 210 "h": 5, 211 "w": 4, 212 "x": 4, 213 "y": 1 214 }, 215 "hiddenSeries": false, 216 "id": 7, 217 "legend": { 218 "avg": false, 219 "current": false, 220 "max": false, 221 "min": false, 222 "show": true, 223 "total": false, 224 "values": false 225 }, 226 "lines": true, 227 "linewidth": 1, 228 "nullPointMode": "null", 229 "options": { 230 "dataLinks": [] 231 }, 232 "percentage": false, 233 "pointradius": 2, 234 "points": false, 235 "renderer": "flot", 236 "seriesOverrides": [], 237 "spaceLength": 10, 238 "stack": false, 239 "steppedLine": false, 240 "targets": [ 241 { 242 "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/cortex-gw\", route=~\"api_prom_push|loki_api_v1_push\"}[5m]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", 243 "legendFormat": "{{status}}", 244 "refId": "A" 245 } 246 ], 247 "thresholds": [], 248 "timeFrom": null, 249 "timeRegions": [], 250 "timeShift": null, 251 "title": "Pushes/Second", 252 "tooltip": { 253 "shared": true, 254 "sort": 0, 255 "value_type": "individual" 256 }, 257 "type": "graph", 258 "xaxis": { 259 "buckets": null, 260 "mode": "time", 261 "name": null, 262 "show": true, 263 "values": [] 264 }, 265 "yaxes": [ 266 { 267 "format": "short", 268 "label": null, 269 "logBase": 10, 270 "max": null, 271 "min": null, 272 "show": true 273 }, 274 { 275 "format": "short", 276 "label": null, 277 "logBase": 1, 278 "max": null, 279 "min": null, 280 "show": true 281 } 282 ], 283 "yaxis": { 284 "align": false, 285 "alignLevel": null 286 } 287 }, 288 { 289 "aliasColors": {}, 290 "bars": false, 291 "dashLength": 10, 292 "dashes": false, 293 "datasource": "$datasource", 294 "fieldConfig": { 295 "defaults": { 296 "custom": {} 297 }, 298 "overrides": [] 299 }, 300 "fill": 1, 301 "fillGradient": 0, 302 "gridPos": { 303 "h": 5, 304 "w": 4, 305 "x": 8, 306 "y": 1 307 }, 308 "hiddenSeries": false, 309 "id": 11, 310 "legend": { 311 "avg": false, 312 "current": false, 313 "hideEmpty": false, 314 "hideZero": false, 315 "max": false, 316 "min": false, 317 "show": false, 318 "total": false, 319 "values": false 320 }, 321 "lines": true, 322 "linewidth": 1, 323 "nullPointMode": "null", 324 "options": { 325 "dataLinks": [] 326 }, 327 "percentage": false, 328 "pointradius": 2, 329 "points": false, 330 "renderer": "flot", 331 "seriesOverrides": [], 332 "spaceLength": 10, 333 "stack": false, 334 "steppedLine": false, 335 "targets": [ 336 { 337 "expr": "topk(5, sum by (name,level) (rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\"}[$__interval])) - \nsum by (name,level) (rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\"}[$__interval] offset 1h)))", 338 "legendFormat": "{{name}}-{{level}}", 339 "refId": "A" 340 } 341 ], 342 "thresholds": [], 343 "timeFrom": null, 344 "timeRegions": [], 345 "timeShift": null, 346 "title": "Bad Words", 347 "tooltip": { 348 "shared": true, 349 "sort": 2, 350 "value_type": "individual" 351 }, 352 "type": "graph", 353 "xaxis": { 354 "buckets": null, 355 "mode": "time", 356 "name": null, 357 "show": true, 358 "values": [] 359 }, 360 "yaxes": [ 361 { 362 "format": "short", 363 "label": null, 364 "logBase": 1, 365 "max": null, 366 "min": null, 367 "show": true 368 }, 369 { 370 "format": "short", 371 "label": null, 372 "logBase": 1, 373 "max": null, 374 "min": null, 375 "show": true 376 } 377 ], 378 "yaxis": { 379 "align": false, 380 "alignLevel": null 381 } 382 }, 383 { 384 "aliasColors": {}, 385 "bars": false, 386 "dashLength": 10, 387 "dashes": false, 388 "datasource": "$datasource", 389 "fieldConfig": { 390 "defaults": { 391 "custom": {} 392 }, 393 "overrides": [] 394 }, 395 "fill": 1, 396 "fillGradient": 0, 397 "gridPos": { 398 "h": 5, 399 "w": 4, 400 "x": 12, 401 "y": 1 402 }, 403 "hiddenSeries": false, 404 "id": 2, 405 "interval": "", 406 "legend": { 407 "avg": false, 408 "current": false, 409 "max": false, 410 "min": false, 411 "show": false, 412 "total": false, 413 "values": false 414 }, 415 "lines": true, 416 "linewidth": 1, 417 "nullPointMode": "null", 418 "options": { 419 "dataLinks": [] 420 }, 421 "percentage": false, 422 "pointradius": 2, 423 "points": false, 424 "renderer": "flot", 425 "seriesOverrides": [], 426 "spaceLength": 10, 427 "stack": false, 428 "steppedLine": false, 429 "targets": [ 430 { 431 "expr": "topk(10, sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (tenant))", 432 "legendFormat": "{{tenant}}", 433 "refId": "A" 434 } 435 ], 436 "thresholds": [], 437 "timeFrom": null, 438 "timeRegions": [], 439 "timeShift": null, 440 "title": "Lines Per Tenant (top 10)", 441 "tooltip": { 442 "shared": false, 443 "sort": 0, 444 "value_type": "individual" 445 }, 446 "type": "graph", 447 "xaxis": { 448 "buckets": null, 449 "mode": "time", 450 "name": null, 451 "show": true, 452 "values": [] 453 }, 454 "yaxes": [ 455 { 456 "format": "short", 457 "label": null, 458 "logBase": 1, 459 "max": null, 460 "min": null, 461 "show": true 462 }, 463 { 464 "format": "short", 465 "label": null, 466 "logBase": 1, 467 "max": null, 468 "min": null, 469 "show": true 470 } 471 ], 472 "yaxis": { 473 "align": false, 474 "alignLevel": null 475 } 476 }, 477 { 478 "aliasColors": {}, 479 "bars": false, 480 "dashLength": 10, 481 "dashes": false, 482 "datasource": "$datasource", 483 "fieldConfig": { 484 "defaults": { 485 "custom": {} 486 }, 487 "overrides": [] 488 }, 489 "fill": 1, 490 "fillGradient": 0, 491 "gridPos": { 492 "h": 5, 493 "w": 4, 494 "x": 16, 495 "y": 1 496 }, 497 "hiddenSeries": false, 498 "id": 4, 499 "legend": { 500 "avg": false, 501 "current": false, 502 "hideEmpty": true, 503 "hideZero": true, 504 "max": false, 505 "min": false, 506 "show": false, 507 "total": false, 508 "values": false 509 }, 510 "lines": true, 511 "linewidth": 1, 512 "nullPointMode": "null", 513 "options": { 514 "dataLinks": [] 515 }, 516 "percentage": false, 517 "pointradius": 2, 518 "points": false, 519 "renderer": "flot", 520 "seriesOverrides": [], 521 "spaceLength": 10, 522 "stack": false, 523 "steppedLine": false, 524 "targets": [ 525 { 526 "expr": "topk(10, sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (tenant)) / 1024 / 1024", 527 "legendFormat": "{{tenant}}", 528 "refId": "A" 529 } 530 ], 531 "thresholds": [], 532 "timeFrom": null, 533 "timeRegions": [], 534 "timeShift": null, 535 "title": "MBs Per Tenant (Top 10)", 536 "tooltip": { 537 "shared": false, 538 "sort": 0, 539 "value_type": "individual" 540 }, 541 "type": "graph", 542 "xaxis": { 543 "buckets": null, 544 "mode": "time", 545 "name": null, 546 "show": true, 547 "values": [] 548 }, 549 "yaxes": [ 550 { 551 "format": "short", 552 "label": null, 553 "logBase": 1, 554 "max": null, 555 "min": null, 556 "show": true 557 }, 558 { 559 "format": "short", 560 "label": null, 561 "logBase": 1, 562 "max": null, 563 "min": null, 564 "show": true 565 } 566 ], 567 "yaxis": { 568 "align": false, 569 "alignLevel": null 570 } 571 }, 572 { 573 "aliasColors": {}, 574 "bars": false, 575 "dashLength": 10, 576 "dashes": false, 577 "datasource": "$datasource", 578 "fieldConfig": { 579 "defaults": { 580 "custom": {} 581 }, 582 "overrides": [] 583 }, 584 "fill": 1, 585 "fillGradient": 0, 586 "gridPos": { 587 "h": 5, 588 "w": 4, 589 "x": 20, 590 "y": 1 591 }, 592 "hiddenSeries": false, 593 "id": 24, 594 "legend": { 595 "avg": false, 596 "current": false, 597 "max": false, 598 "min": false, 599 "show": false, 600 "total": false, 601 "values": false 602 }, 603 "lines": true, 604 "linewidth": 1, 605 "nullPointMode": "null", 606 "options": { 607 "dataLinks": [] 608 }, 609 "percentage": false, 610 "pointradius": 2, 611 "points": false, 612 "renderer": "flot", 613 "seriesOverrides": [], 614 "spaceLength": 10, 615 "stack": false, 616 "steppedLine": false, 617 "targets": [ 618 { 619 "expr": "increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\"}[10m]) > 0", 620 "hide": false, 621 "interval": "", 622 "legendFormat": "{{container}}-{{pod}}", 623 "refId": "B" 624 } 625 ], 626 "thresholds": [], 627 "timeFrom": null, 628 "timeRegions": [], 629 "timeShift": null, 630 "title": "Container Restarts", 631 "tooltip": { 632 "shared": true, 633 "sort": 2, 634 "value_type": "individual" 635 }, 636 "type": "graph", 637 "xaxis": { 638 "buckets": null, 639 "mode": "time", 640 "name": null, 641 "show": true, 642 "values": [] 643 }, 644 "yaxes": [ 645 { 646 "format": "short", 647 "label": null, 648 "logBase": 1, 649 "max": null, 650 "min": null, 651 "show": true 652 }, 653 { 654 "format": "short", 655 "label": null, 656 "logBase": 1, 657 "max": null, 658 "min": null, 659 "show": true 660 } 661 ], 662 "yaxis": { 663 "align": false, 664 "alignLevel": null 665 } 666 }, 667 { 668 "aliasColors": {}, 669 "bars": false, 670 "dashLength": 10, 671 "dashes": false, 672 "datasource": "$datasource", 673 "fieldConfig": { 674 "defaults": { 675 "custom": {} 676 }, 677 "overrides": [] 678 }, 679 "fill": 1, 680 "fillGradient": 0, 681 "gridPos": { 682 "h": 10, 683 "w": 12, 684 "x": 0, 685 "y": 6 686 }, 687 "hiddenSeries": false, 688 "id": 9, 689 "legend": { 690 "alignAsTable": true, 691 "avg": false, 692 "current": false, 693 "max": false, 694 "min": false, 695 "rightSide": true, 696 "show": true, 697 "total": false, 698 "values": false 699 }, 700 "lines": true, 701 "linewidth": 1, 702 "nullPointMode": "null", 703 "options": { 704 "dataLinks": [] 705 }, 706 "percentage": false, 707 "pointradius": 2, 708 "points": false, 709 "renderer": "flot", 710 "seriesOverrides": [], 711 "spaceLength": 10, 712 "stack": false, 713 "steppedLine": false, 714 "targets": [ 715 { 716 "expr": "histogram_quantile(0.99, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/cortex-gw\", route=~\"api_prom_push|loki_api_v1_push\", cluster=~\"$cluster\"})) * 1e3", 717 "legendFormat": ".99", 718 "refId": "A" 719 }, 720 { 721 "expr": "histogram_quantile(0.75, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/cortex-gw\", route=~\"api_prom_push|loki_api_v1_push\", cluster=~\"$cluster\"})) * 1e3", 722 "legendFormat": ".9", 723 "refId": "B" 724 }, 725 { 726 "expr": "histogram_quantile(0.5, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/cortex-gw\", route=~\"api_prom_push|loki_api_v1_push\", cluster=~\"$cluster\"})) * 1e3", 727 "legendFormat": ".5", 728 "refId": "C" 729 } 730 ], 731 "thresholds": [], 732 "timeFrom": null, 733 "timeRegions": [], 734 "timeShift": null, 735 "title": "Push Latency", 736 "tooltip": { 737 "shared": true, 738 "sort": 0, 739 "value_type": "individual" 740 }, 741 "type": "graph", 742 "xaxis": { 743 "buckets": null, 744 "mode": "time", 745 "name": null, 746 "show": true, 747 "values": [] 748 }, 749 "yaxes": [ 750 { 751 "format": "short", 752 "label": null, 753 "logBase": 1, 754 "max": null, 755 "min": null, 756 "show": true 757 }, 758 { 759 "format": "short", 760 "label": null, 761 "logBase": 1, 762 "max": null, 763 "min": null, 764 "show": true 765 } 766 ], 767 "yaxis": { 768 "align": false, 769 "alignLevel": null 770 } 771 }, 772 { 773 "aliasColors": {}, 774 "bars": false, 775 "dashLength": 10, 776 "dashes": false, 777 "datasource": "$datasource", 778 "fieldConfig": { 779 "defaults": { 780 "custom": {} 781 }, 782 "overrides": [] 783 }, 784 "fill": 1, 785 "fillGradient": 0, 786 "gridPos": { 787 "h": 5, 788 "w": 6, 789 "x": 12, 790 "y": 6 791 }, 792 "hiddenSeries": false, 793 "id": 12, 794 "legend": { 795 "alignAsTable": true, 796 "avg": false, 797 "current": false, 798 "max": false, 799 "min": false, 800 "rightSide": true, 801 "show": false, 802 "total": false, 803 "values": false 804 }, 805 "lines": true, 806 "linewidth": 1, 807 "nullPointMode": "null", 808 "options": { 809 "dataLinks": [] 810 }, 811 "percentage": false, 812 "pointradius": 2, 813 "points": false, 814 "renderer": "flot", 815 "seriesOverrides": [], 816 "spaceLength": 10, 817 "stack": false, 818 "steppedLine": false, 819 "targets": [ 820 { 821 "expr": "histogram_quantile(0.99, sum by (le) (job:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/distributor\", cluster=~\"$cluster\"})) * 1e3", 822 "legendFormat": ".99", 823 "refId": "A" 824 }, 825 { 826 "expr": "histogram_quantile(0.9, sum by (le) (job:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/distributor\", cluster=~\"$cluster\"})) * 1e3", 827 "legendFormat": ".9", 828 "refId": "B" 829 }, 830 { 831 "expr": "histogram_quantile(0.5, sum by (le) (job:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/distributor\", cluster=~\"$cluster\"})) * 1e3", 832 "legendFormat": ".5", 833 "refId": "C" 834 } 835 ], 836 "thresholds": [], 837 "timeFrom": null, 838 "timeRegions": [], 839 "timeShift": null, 840 "title": "Distributor Latency", 841 "tooltip": { 842 "shared": true, 843 "sort": 0, 844 "value_type": "individual" 845 }, 846 "type": "graph", 847 "xaxis": { 848 "buckets": null, 849 "mode": "time", 850 "name": null, 851 "show": true, 852 "values": [] 853 }, 854 "yaxes": [ 855 { 856 "format": "short", 857 "label": null, 858 "logBase": 1, 859 "max": null, 860 "min": null, 861 "show": true 862 }, 863 { 864 "format": "short", 865 "label": null, 866 "logBase": 1, 867 "max": null, 868 "min": null, 869 "show": true 870 } 871 ], 872 "yaxis": { 873 "align": false, 874 "alignLevel": null 875 } 876 }, 877 { 878 "aliasColors": {}, 879 "bars": false, 880 "dashLength": 10, 881 "dashes": false, 882 "datasource": "$datasource", 883 "fieldConfig": { 884 "defaults": { 885 "custom": {} 886 }, 887 "overrides": [] 888 }, 889 "fill": 0, 890 "fillGradient": 0, 891 "gridPos": { 892 "h": 5, 893 "w": 6, 894 "x": 18, 895 "y": 6 896 }, 897 "hiddenSeries": false, 898 "id": 71, 899 "legend": { 900 "alignAsTable": true, 901 "avg": false, 902 "current": false, 903 "max": false, 904 "min": false, 905 "rightSide": true, 906 "show": false, 907 "total": false, 908 "values": false 909 }, 910 "lines": true, 911 "linewidth": 1, 912 "nullPointMode": "null", 913 "options": { 914 "dataLinks": [] 915 }, 916 "percentage": false, 917 "pointradius": 2, 918 "points": false, 919 "renderer": "flot", 920 "seriesOverrides": [], 921 "spaceLength": 10, 922 "stack": false, 923 "steppedLine": false, 924 "targets": [ 925 { 926 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/distributor\", status_code!~\"5[0-9]{2}\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/distributor\"}[5m])) by (route)", 927 "interval": "", 928 "legendFormat": "{{route}}", 929 "refId": "A" 930 } 931 ], 932 "thresholds": [], 933 "timeFrom": null, 934 "timeRegions": [], 935 "timeShift": null, 936 "title": "Distributor Success Rate", 937 "tooltip": { 938 "shared": true, 939 "sort": 0, 940 "value_type": "individual" 941 }, 942 "type": "graph", 943 "xaxis": { 944 "buckets": null, 945 "mode": "time", 946 "name": null, 947 "show": true, 948 "values": [] 949 }, 950 "yaxes": [ 951 { 952 "decimals": null, 953 "format": "percentunit", 954 "label": "", 955 "logBase": 1, 956 "max": "1", 957 "min": null, 958 "show": true 959 }, 960 { 961 "format": "short", 962 "label": null, 963 "logBase": 1, 964 "max": null, 965 "min": null, 966 "show": true 967 } 968 ], 969 "yaxis": { 970 "align": false, 971 "alignLevel": null 972 } 973 }, 974 { 975 "aliasColors": {}, 976 "bars": false, 977 "dashLength": 10, 978 "dashes": false, 979 "datasource": "$datasource", 980 "fieldConfig": { 981 "defaults": { 982 "custom": {} 983 }, 984 "overrides": [] 985 }, 986 "fill": 1, 987 "fillGradient": 0, 988 "gridPos": { 989 "h": 5, 990 "w": 6, 991 "x": 12, 992 "y": 11 993 }, 994 "hiddenSeries": false, 995 "id": 13, 996 "legend": { 997 "alignAsTable": true, 998 "avg": false, 999 "current": false, 1000 "max": false, 1001 "min": false, 1002 "rightSide": true, 1003 "show": false, 1004 "total": false, 1005 "values": false 1006 }, 1007 "lines": true, 1008 "linewidth": 1, 1009 "nullPointMode": "null", 1010 "options": { 1011 "dataLinks": [] 1012 }, 1013 "percentage": false, 1014 "pointradius": 2, 1015 "points": false, 1016 "renderer": "flot", 1017 "seriesOverrides": [], 1018 "spaceLength": 10, 1019 "stack": false, 1020 "steppedLine": false, 1021 "targets": [ 1022 { 1023 "expr": "histogram_quantile(0.99, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/ingester\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3", 1024 "legendFormat": ".99", 1025 "refId": "A" 1026 }, 1027 { 1028 "expr": "histogram_quantile(0.9, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/ingester\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3", 1029 "hide": false, 1030 "legendFormat": ".9", 1031 "refId": "B" 1032 }, 1033 { 1034 "expr": "histogram_quantile(0.5, sum by (le) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/ingester\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3", 1035 "hide": false, 1036 "legendFormat": ".5", 1037 "refId": "C" 1038 } 1039 ], 1040 "thresholds": [], 1041 "timeFrom": null, 1042 "timeRegions": [], 1043 "timeShift": null, 1044 "title": "Ingester Latency Write", 1045 "tooltip": { 1046 "shared": true, 1047 "sort": 0, 1048 "value_type": "individual" 1049 }, 1050 "type": "graph", 1051 "xaxis": { 1052 "buckets": null, 1053 "mode": "time", 1054 "name": null, 1055 "show": true, 1056 "values": [] 1057 }, 1058 "yaxes": [ 1059 { 1060 "format": "short", 1061 "label": null, 1062 "logBase": 1, 1063 "max": null, 1064 "min": null, 1065 "show": true 1066 }, 1067 { 1068 "format": "short", 1069 "label": null, 1070 "logBase": 1, 1071 "max": null, 1072 "min": null, 1073 "show": true 1074 } 1075 ], 1076 "yaxis": { 1077 "align": false, 1078 "alignLevel": null 1079 } 1080 }, 1081 { 1082 "aliasColors": {}, 1083 "bars": false, 1084 "dashLength": 10, 1085 "dashes": false, 1086 "datasource": "$datasource", 1087 "fieldConfig": { 1088 "defaults": { 1089 "custom": {} 1090 }, 1091 "overrides": [] 1092 }, 1093 "fill": 0, 1094 "fillGradient": 0, 1095 "gridPos": { 1096 "h": 5, 1097 "w": 6, 1098 "x": 18, 1099 "y": 11 1100 }, 1101 "hiddenSeries": false, 1102 "id": 72, 1103 "legend": { 1104 "alignAsTable": true, 1105 "avg": false, 1106 "current": false, 1107 "max": false, 1108 "min": false, 1109 "rightSide": true, 1110 "show": false, 1111 "total": false, 1112 "values": false 1113 }, 1114 "lines": true, 1115 "linewidth": 1, 1116 "nullPointMode": "null", 1117 "options": { 1118 "dataLinks": [] 1119 }, 1120 "percentage": false, 1121 "pointradius": 2, 1122 "points": false, 1123 "renderer": "flot", 1124 "seriesOverrides": [], 1125 "spaceLength": 10, 1126 "stack": false, 1127 "steppedLine": false, 1128 "targets": [ 1129 { 1130 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", status_code!~\"5[0-9]{2}\", route=\"/logproto.Pusher/Push\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=\"/logproto.Pusher/Push\"}[5m])) by (route)", 1131 "interval": "", 1132 "legendFormat": "{{route}}", 1133 "refId": "A" 1134 } 1135 ], 1136 "thresholds": [], 1137 "timeFrom": null, 1138 "timeRegions": [], 1139 "timeShift": null, 1140 "title": "Ingester Success Rate Write", 1141 "tooltip": { 1142 "shared": true, 1143 "sort": 0, 1144 "value_type": "individual" 1145 }, 1146 "type": "graph", 1147 "xaxis": { 1148 "buckets": null, 1149 "mode": "time", 1150 "name": null, 1151 "show": true, 1152 "values": [] 1153 }, 1154 "yaxes": [ 1155 { 1156 "decimals": null, 1157 "format": "percentunit", 1158 "label": "", 1159 "logBase": 1, 1160 "max": "1", 1161 "min": null, 1162 "show": true 1163 }, 1164 { 1165 "format": "short", 1166 "label": null, 1167 "logBase": 1, 1168 "max": null, 1169 "min": null, 1170 "show": true 1171 } 1172 ], 1173 "yaxis": { 1174 "align": false, 1175 "alignLevel": null 1176 } 1177 }, 1178 { 1179 "aliasColors": {}, 1180 "bars": false, 1181 "dashLength": 10, 1182 "dashes": false, 1183 "datasource": "$datasource", 1184 "fieldConfig": { 1185 "defaults": { 1186 "custom": {} 1187 }, 1188 "overrides": [] 1189 }, 1190 "fill": 1, 1191 "fillGradient": 0, 1192 "gridPos": { 1193 "h": 10, 1194 "w": 12, 1195 "x": 0, 1196 "y": 16 1197 }, 1198 "hiddenSeries": false, 1199 "id": 10, 1200 "legend": { 1201 "alignAsTable": true, 1202 "avg": false, 1203 "current": false, 1204 "hideEmpty": true, 1205 "hideZero": false, 1206 "max": false, 1207 "min": false, 1208 "rightSide": true, 1209 "show": true, 1210 "total": false, 1211 "values": false 1212 }, 1213 "lines": true, 1214 "linewidth": 1, 1215 "nullPointMode": "null", 1216 "options": { 1217 "dataLinks": [] 1218 }, 1219 "percentage": false, 1220 "pointradius": 2, 1221 "points": false, 1222 "renderer": "flot", 1223 "seriesOverrides": [], 1224 "spaceLength": 10, 1225 "stack": false, 1226 "steppedLine": false, 1227 "targets": [ 1228 { 1229 "expr": "histogram_quantile(0.99, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=\"$namespace/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", 1230 "legendFormat": "{{route}}-.99", 1231 "refId": "A" 1232 }, 1233 { 1234 "expr": "histogram_quantile(0.9, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=\"$namespace/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", 1235 "legendFormat": "{{route}}-.9", 1236 "refId": "B" 1237 }, 1238 { 1239 "expr": "histogram_quantile(0.5, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=\"$namespace/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", 1240 "legendFormat": "{{route}}-.5", 1241 "refId": "C" 1242 } 1243 ], 1244 "thresholds": [], 1245 "timeFrom": null, 1246 "timeRegions": [], 1247 "timeShift": null, 1248 "title": "Query Latency", 1249 "tooltip": { 1250 "shared": true, 1251 "sort": 0, 1252 "value_type": "individual" 1253 }, 1254 "type": "graph", 1255 "xaxis": { 1256 "buckets": null, 1257 "mode": "time", 1258 "name": null, 1259 "show": true, 1260 "values": [] 1261 }, 1262 "yaxes": [ 1263 { 1264 "format": "short", 1265 "label": null, 1266 "logBase": 1, 1267 "max": null, 1268 "min": null, 1269 "show": true 1270 }, 1271 { 1272 "format": "short", 1273 "label": null, 1274 "logBase": 1, 1275 "max": null, 1276 "min": null, 1277 "show": true 1278 } 1279 ], 1280 "yaxis": { 1281 "align": false, 1282 "alignLevel": null 1283 } 1284 }, 1285 { 1286 "aliasColors": {}, 1287 "bars": false, 1288 "dashLength": 10, 1289 "dashes": false, 1290 "datasource": "$datasource", 1291 "fieldConfig": { 1292 "defaults": { 1293 "custom": {} 1294 }, 1295 "overrides": [] 1296 }, 1297 "fill": 1, 1298 "fillGradient": 0, 1299 "gridPos": { 1300 "h": 5, 1301 "w": 6, 1302 "x": 12, 1303 "y": 16 1304 }, 1305 "hiddenSeries": false, 1306 "id": 14, 1307 "legend": { 1308 "alignAsTable": true, 1309 "avg": false, 1310 "current": false, 1311 "max": false, 1312 "min": false, 1313 "rightSide": true, 1314 "show": false, 1315 "total": false, 1316 "values": false 1317 }, 1318 "lines": true, 1319 "linewidth": 1, 1320 "nullPointMode": "null", 1321 "options": { 1322 "dataLinks": [] 1323 }, 1324 "percentage": false, 1325 "pointradius": 2, 1326 "points": false, 1327 "renderer": "flot", 1328 "seriesOverrides": [], 1329 "spaceLength": 10, 1330 "stack": false, 1331 "steppedLine": false, 1332 "targets": [ 1333 { 1334 "expr": "histogram_quantile(0.99, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=\"$namespace/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", 1335 "legendFormat": ".99-{{route}}", 1336 "refId": "A" 1337 }, 1338 { 1339 "expr": "histogram_quantile(0.9, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=\"$namespace/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", 1340 "legendFormat": ".9-{{route}}", 1341 "refId": "B" 1342 }, 1343 { 1344 "expr": "histogram_quantile(0.5, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=\"$namespace/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", 1345 "legendFormat": ".5-{{route}}", 1346 "refId": "C" 1347 } 1348 ], 1349 "thresholds": [], 1350 "timeFrom": null, 1351 "timeRegions": [], 1352 "timeShift": null, 1353 "title": "Querier Latency", 1354 "tooltip": { 1355 "shared": true, 1356 "sort": 0, 1357 "value_type": "individual" 1358 }, 1359 "type": "graph", 1360 "xaxis": { 1361 "buckets": null, 1362 "mode": "time", 1363 "name": null, 1364 "show": true, 1365 "values": [] 1366 }, 1367 "yaxes": [ 1368 { 1369 "format": "short", 1370 "label": null, 1371 "logBase": 1, 1372 "max": null, 1373 "min": null, 1374 "show": true 1375 }, 1376 { 1377 "format": "short", 1378 "label": null, 1379 "logBase": 1, 1380 "max": null, 1381 "min": null, 1382 "show": true 1383 } 1384 ], 1385 "yaxis": { 1386 "align": false, 1387 "alignLevel": null 1388 } 1389 }, 1390 { 1391 "aliasColors": {}, 1392 "bars": false, 1393 "dashLength": 10, 1394 "dashes": false, 1395 "datasource": "$datasource", 1396 "fieldConfig": { 1397 "defaults": { 1398 "custom": {} 1399 }, 1400 "overrides": [] 1401 }, 1402 "fill": 0, 1403 "fillGradient": 0, 1404 "gridPos": { 1405 "h": 5, 1406 "w": 6, 1407 "x": 18, 1408 "y": 16 1409 }, 1410 "hiddenSeries": false, 1411 "id": 73, 1412 "legend": { 1413 "alignAsTable": true, 1414 "avg": false, 1415 "current": false, 1416 "max": false, 1417 "min": false, 1418 "rightSide": true, 1419 "show": false, 1420 "total": false, 1421 "values": false 1422 }, 1423 "lines": true, 1424 "linewidth": 1, 1425 "nullPointMode": "null", 1426 "options": { 1427 "dataLinks": [] 1428 }, 1429 "percentage": false, 1430 "pointradius": 2, 1431 "points": false, 1432 "renderer": "flot", 1433 "seriesOverrides": [], 1434 "spaceLength": 10, 1435 "stack": false, 1436 "steppedLine": false, 1437 "targets": [ 1438 { 1439 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", status_code!~\"5[0-9]{2}\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\"}[5m])) by (route)", 1440 "interval": "", 1441 "legendFormat": "{{route}}", 1442 "refId": "A" 1443 } 1444 ], 1445 "thresholds": [], 1446 "timeFrom": null, 1447 "timeRegions": [], 1448 "timeShift": null, 1449 "title": "Querier Success Rate", 1450 "tooltip": { 1451 "shared": true, 1452 "sort": 0, 1453 "value_type": "individual" 1454 }, 1455 "type": "graph", 1456 "xaxis": { 1457 "buckets": null, 1458 "mode": "time", 1459 "name": null, 1460 "show": true, 1461 "values": [] 1462 }, 1463 "yaxes": [ 1464 { 1465 "decimals": null, 1466 "format": "percentunit", 1467 "label": "", 1468 "logBase": 1, 1469 "max": "1", 1470 "min": null, 1471 "show": true 1472 }, 1473 { 1474 "format": "short", 1475 "label": null, 1476 "logBase": 1, 1477 "max": null, 1478 "min": null, 1479 "show": true 1480 } 1481 ], 1482 "yaxis": { 1483 "align": false, 1484 "alignLevel": null 1485 } 1486 }, 1487 { 1488 "aliasColors": {}, 1489 "bars": false, 1490 "dashLength": 10, 1491 "dashes": false, 1492 "datasource": "$datasource", 1493 "description": "", 1494 "fieldConfig": { 1495 "defaults": { 1496 "custom": {} 1497 }, 1498 "overrides": [] 1499 }, 1500 "fill": 1, 1501 "fillGradient": 0, 1502 "gridPos": { 1503 "h": 5, 1504 "w": 6, 1505 "x": 12, 1506 "y": 21 1507 }, 1508 "hiddenSeries": false, 1509 "id": 15, 1510 "legend": { 1511 "alignAsTable": true, 1512 "avg": false, 1513 "current": false, 1514 "max": false, 1515 "min": false, 1516 "rightSide": true, 1517 "show": false, 1518 "total": false, 1519 "values": false 1520 }, 1521 "lines": true, 1522 "linewidth": 1, 1523 "nullPointMode": "null", 1524 "options": { 1525 "dataLinks": [] 1526 }, 1527 "percentage": false, 1528 "pointradius": 2, 1529 "points": false, 1530 "renderer": "flot", 1531 "seriesOverrides": [], 1532 "spaceLength": 10, 1533 "stack": false, 1534 "steppedLine": false, 1535 "targets": [ 1536 { 1537 "expr": "histogram_quantile(0.99, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=\"$namespace/ingester\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3", 1538 "legendFormat": ".99-{{route}}", 1539 "refId": "A" 1540 }, 1541 { 1542 "expr": "histogram_quantile(0.9, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=\"$namespace/ingester\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3", 1543 "legendFormat": ".9-{{route}}", 1544 "refId": "B" 1545 }, 1546 { 1547 "expr": "histogram_quantile(0.5, sum by (le,route) (job_route:loki_request_duration_seconds_bucket:sum_rate{job=\"$namespace/ingester\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3", 1548 "legendFormat": ".5-{{route}}", 1549 "refId": "C" 1550 } 1551 ], 1552 "thresholds": [], 1553 "timeFrom": null, 1554 "timeRegions": [], 1555 "timeShift": null, 1556 "title": "Ingester Latency Read", 1557 "tooltip": { 1558 "shared": true, 1559 "sort": 0, 1560 "value_type": "individual" 1561 }, 1562 "type": "graph", 1563 "xaxis": { 1564 "buckets": null, 1565 "mode": "time", 1566 "name": null, 1567 "show": true, 1568 "values": [] 1569 }, 1570 "yaxes": [ 1571 { 1572 "format": "short", 1573 "label": null, 1574 "logBase": 1, 1575 "max": null, 1576 "min": null, 1577 "show": true 1578 }, 1579 { 1580 "format": "short", 1581 "label": null, 1582 "logBase": 1, 1583 "max": null, 1584 "min": null, 1585 "show": true 1586 } 1587 ], 1588 "yaxis": { 1589 "align": false, 1590 "alignLevel": null 1591 } 1592 }, 1593 { 1594 "aliasColors": {}, 1595 "bars": false, 1596 "dashLength": 10, 1597 "dashes": false, 1598 "datasource": "$datasource", 1599 "fieldConfig": { 1600 "defaults": { 1601 "custom": {} 1602 }, 1603 "overrides": [] 1604 }, 1605 "fill": 0, 1606 "fillGradient": 0, 1607 "gridPos": { 1608 "h": 5, 1609 "w": 6, 1610 "x": 18, 1611 "y": 21 1612 }, 1613 "hiddenSeries": false, 1614 "id": 74, 1615 "legend": { 1616 "alignAsTable": true, 1617 "avg": false, 1618 "current": false, 1619 "max": false, 1620 "min": false, 1621 "rightSide": true, 1622 "show": false, 1623 "total": false, 1624 "values": false 1625 }, 1626 "lines": true, 1627 "linewidth": 1, 1628 "nullPointMode": "null", 1629 "options": { 1630 "dataLinks": [] 1631 }, 1632 "percentage": false, 1633 "pointradius": 2, 1634 "points": false, 1635 "renderer": "flot", 1636 "seriesOverrides": [], 1637 "spaceLength": 10, 1638 "stack": false, 1639 "steppedLine": false, 1640 "targets": [ 1641 { 1642 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", status_code!~\"5[0-9]{2}\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[5m])) by (route)", 1643 "interval": "", 1644 "legendFormat": "{{route}}", 1645 "refId": "A" 1646 } 1647 ], 1648 "thresholds": [], 1649 "timeFrom": null, 1650 "timeRegions": [], 1651 "timeShift": null, 1652 "title": "Ingester Success Rate Read", 1653 "tooltip": { 1654 "shared": true, 1655 "sort": 0, 1656 "value_type": "individual" 1657 }, 1658 "type": "graph", 1659 "xaxis": { 1660 "buckets": null, 1661 "mode": "time", 1662 "name": null, 1663 "show": true, 1664 "values": [] 1665 }, 1666 "yaxes": [ 1667 { 1668 "decimals": null, 1669 "format": "percentunit", 1670 "label": "", 1671 "logBase": 1, 1672 "max": "1", 1673 "min": null, 1674 "show": true 1675 }, 1676 { 1677 "format": "short", 1678 "label": null, 1679 "logBase": 1, 1680 "max": null, 1681 "min": null, 1682 "show": true 1683 } 1684 ], 1685 "yaxis": { 1686 "align": false, 1687 "alignLevel": null 1688 } 1689 }, 1690 { 1691 "collapsed": true, 1692 "datasource": null, 1693 "gridPos": { 1694 "h": 1, 1695 "w": 24, 1696 "x": 0, 1697 "y": 26 1698 }, 1699 "id": 110, 1700 "panels": [ 1701 { 1702 "aliasColors": {}, 1703 "bars": false, 1704 "dashLength": 10, 1705 "dashes": false, 1706 "datasource": "$datasource", 1707 "fill": 1, 1708 "fillGradient": 0, 1709 "gridPos": { 1710 "h": 8, 1711 "w": 12, 1712 "x": 0, 1713 "y": 27 1714 }, 1715 "hiddenSeries": false, 1716 "id": 112, 1717 "legend": { 1718 "avg": false, 1719 "current": false, 1720 "max": false, 1721 "min": false, 1722 "show": false, 1723 "total": false, 1724 "values": false 1725 }, 1726 "lines": true, 1727 "linewidth": 1, 1728 "nullPointMode": "null", 1729 "options": { 1730 "dataLinks": [] 1731 }, 1732 "percentage": false, 1733 "pointradius": 2, 1734 "points": false, 1735 "renderer": "flot", 1736 "seriesOverrides": [], 1737 "spaceLength": 10, 1738 "stack": false, 1739 "steppedLine": false, 1740 "targets": [ 1741 { 1742 "expr": "topk(10,sum by (tenant, reason) (rate(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[1m])))", 1743 "interval": "", 1744 "legendFormat": "{{ tenant }} - {{ reason }}", 1745 "refId": "A" 1746 } 1747 ], 1748 "thresholds": [], 1749 "timeFrom": null, 1750 "timeRegions": [], 1751 "timeShift": null, 1752 "title": "Discarded Lines", 1753 "tooltip": { 1754 "shared": false, 1755 "sort": 0, 1756 "value_type": "individual" 1757 }, 1758 "type": "graph", 1759 "xaxis": { 1760 "buckets": null, 1761 "mode": "time", 1762 "name": null, 1763 "show": true, 1764 "values": [] 1765 }, 1766 "yaxes": [ 1767 { 1768 "format": "short", 1769 "label": null, 1770 "logBase": 1, 1771 "max": null, 1772 "min": null, 1773 "show": true 1774 }, 1775 { 1776 "format": "short", 1777 "label": null, 1778 "logBase": 1, 1779 "max": null, 1780 "min": null, 1781 "show": true 1782 } 1783 ], 1784 "yaxis": { 1785 "align": false, 1786 "alignLevel": null 1787 } 1788 }, 1789 { 1790 "columns": [], 1791 "datasource": "$datasource", 1792 "fontSize": "100%", 1793 "gridPos": { 1794 "h": 8, 1795 "w": 12, 1796 "x": 12, 1797 "y": 27 1798 }, 1799 "id": 113, 1800 "pageSize": null, 1801 "showHeader": true, 1802 "sort": { 1803 "col": 3, 1804 "desc": true 1805 }, 1806 "styles": [ 1807 { 1808 "alias": "Time", 1809 "align": "auto", 1810 "dateFormat": "YYYY-MM-DD HH:mm:ss", 1811 "pattern": "Time", 1812 "type": "hidden" 1813 }, 1814 { 1815 "alias": "", 1816 "align": "auto", 1817 "colorMode": null, 1818 "colors": [ 1819 "rgba(245, 54, 54, 0.9)", 1820 "rgba(237, 129, 40, 0.89)", 1821 "rgba(50, 172, 45, 0.97)" 1822 ], 1823 "dateFormat": "YYYY-MM-DD HH:mm:ss", 1824 "decimals": 2, 1825 "mappingType": 1, 1826 "pattern": "tenant", 1827 "thresholds": [], 1828 "type": "string", 1829 "unit": "short" 1830 }, 1831 { 1832 "alias": "", 1833 "align": "auto", 1834 "colorMode": null, 1835 "colors": [ 1836 "rgba(245, 54, 54, 0.9)", 1837 "rgba(237, 129, 40, 0.89)", 1838 "rgba(50, 172, 45, 0.97)" 1839 ], 1840 "dateFormat": "YYYY-MM-DD HH:mm:ss", 1841 "decimals": 2, 1842 "mappingType": 1, 1843 "pattern": "reason", 1844 "thresholds": [], 1845 "type": "number", 1846 "unit": "short" 1847 }, 1848 { 1849 "alias": "", 1850 "align": "right", 1851 "colorMode": null, 1852 "colors": [ 1853 "rgba(245, 54, 54, 0.9)", 1854 "rgba(237, 129, 40, 0.89)", 1855 "rgba(50, 172, 45, 0.97)" 1856 ], 1857 "decimals": 2, 1858 "pattern": "/.*/", 1859 "thresholds": [], 1860 "type": "number", 1861 "unit": "short" 1862 } 1863 ], 1864 "targets": [ 1865 { 1866 "expr": "topk(10, sum by (tenant, reason) (sum_over_time(increase(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[1m])[$__range:1m])))", 1867 "format": "table", 1868 "instant": true, 1869 "interval": "", 1870 "legendFormat": "{{ tenant }} - {{ reason }}", 1871 "refId": "A" 1872 } 1873 ], 1874 "timeFrom": null, 1875 "timeShift": null, 1876 "title": "Discarded Lines Per Interval", 1877 "transform": "table", 1878 "type": "table-old" 1879 } 1880 ], 1881 "title": "Limits", 1882 "type": "row" 1883 }, 1884 { 1885 "collapsed": true, 1886 "datasource": null, 1887 "gridPos": { 1888 "h": 1, 1889 "w": 24, 1890 "x": 0, 1891 "y": 27 1892 }, 1893 "id": 23, 1894 "panels": [ 1895 { 1896 "aliasColors": {}, 1897 "bars": false, 1898 "dashLength": 10, 1899 "dashes": false, 1900 "datasource": "$datasource", 1901 "fill": 1, 1902 "fillGradient": 0, 1903 "gridPos": { 1904 "h": 7, 1905 "w": 6, 1906 "x": 0, 1907 "y": 28 1908 }, 1909 "hiddenSeries": false, 1910 "id": 26, 1911 "legend": { 1912 "avg": false, 1913 "current": false, 1914 "max": false, 1915 "min": false, 1916 "show": false, 1917 "total": false, 1918 "values": false 1919 }, 1920 "lines": false, 1921 "linewidth": 1, 1922 "nullPointMode": "null", 1923 "options": { 1924 "dataLinks": [] 1925 }, 1926 "percentage": false, 1927 "pointradius": 1, 1928 "points": true, 1929 "renderer": "flot", 1930 "seriesOverrides": [], 1931 "spaceLength": 10, 1932 "stack": false, 1933 "steppedLine": false, 1934 "targets": [ 1935 { 1936 "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"distributor.*\"}", 1937 "intervalFactor": 3, 1938 "legendFormat": "{{pod}}-{{container}}", 1939 "refId": "A" 1940 } 1941 ], 1942 "thresholds": [], 1943 "timeFrom": null, 1944 "timeRegions": [], 1945 "timeShift": null, 1946 "title": "CPU Usage", 1947 "tooltip": { 1948 "shared": true, 1949 "sort": 2, 1950 "value_type": "individual" 1951 }, 1952 "type": "graph", 1953 "xaxis": { 1954 "buckets": null, 1955 "mode": "time", 1956 "name": null, 1957 "show": true, 1958 "values": [] 1959 }, 1960 "yaxes": [ 1961 { 1962 "format": "short", 1963 "label": null, 1964 "logBase": 1, 1965 "max": null, 1966 "min": null, 1967 "show": true 1968 }, 1969 { 1970 "format": "short", 1971 "label": null, 1972 "logBase": 1, 1973 "max": null, 1974 "min": null, 1975 "show": true 1976 } 1977 ], 1978 "yaxis": { 1979 "align": false, 1980 "alignLevel": null 1981 } 1982 }, 1983 { 1984 "aliasColors": {}, 1985 "bars": false, 1986 "dashLength": 10, 1987 "dashes": false, 1988 "datasource": "$datasource", 1989 "fill": 1, 1990 "fillGradient": 0, 1991 "gridPos": { 1992 "h": 7, 1993 "w": 6, 1994 "x": 6, 1995 "y": 28 1996 }, 1997 "hiddenSeries": false, 1998 "id": 27, 1999 "legend": { 2000 "avg": false, 2001 "current": false, 2002 "hideEmpty": false, 2003 "hideZero": false, 2004 "max": false, 2005 "min": false, 2006 "show": false, 2007 "total": false, 2008 "values": false 2009 }, 2010 "lines": false, 2011 "linewidth": 1, 2012 "nullPointMode": "null", 2013 "options": { 2014 "dataLinks": [] 2015 }, 2016 "percentage": false, 2017 "pointradius": 1, 2018 "points": true, 2019 "renderer": "flot", 2020 "seriesOverrides": [], 2021 "spaceLength": 10, 2022 "stack": false, 2023 "steppedLine": false, 2024 "targets": [ 2025 { 2026 "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"distributor.*\"}", 2027 "instant": false, 2028 "intervalFactor": 3, 2029 "legendFormat": "{{pod}}", 2030 "refId": "A" 2031 } 2032 ], 2033 "thresholds": [], 2034 "timeFrom": null, 2035 "timeRegions": [], 2036 "timeShift": null, 2037 "title": "Memory Usage", 2038 "tooltip": { 2039 "shared": true, 2040 "sort": 2, 2041 "value_type": "individual" 2042 }, 2043 "type": "graph", 2044 "xaxis": { 2045 "buckets": null, 2046 "mode": "time", 2047 "name": null, 2048 "show": true, 2049 "values": [] 2050 }, 2051 "yaxes": [ 2052 { 2053 "format": "bytes", 2054 "label": null, 2055 "logBase": 1, 2056 "max": null, 2057 "min": null, 2058 "show": true 2059 }, 2060 { 2061 "format": "short", 2062 "label": null, 2063 "logBase": 1, 2064 "max": null, 2065 "min": null, 2066 "show": true 2067 } 2068 ], 2069 "yaxis": { 2070 "align": false, 2071 "alignLevel": null 2072 } 2073 }, 2074 { 2075 "aliasColors": {}, 2076 "bars": true, 2077 "dashLength": 10, 2078 "dashes": false, 2079 "datasource": "$logs", 2080 "fill": 1, 2081 "fillGradient": 0, 2082 "gridPos": { 2083 "h": 4, 2084 "w": 12, 2085 "x": 12, 2086 "y": 28 2087 }, 2088 "hiddenSeries": false, 2089 "id": 31, 2090 "legend": { 2091 "avg": false, 2092 "current": false, 2093 "max": false, 2094 "min": false, 2095 "show": false, 2096 "total": false, 2097 "values": false 2098 }, 2099 "lines": false, 2100 "linewidth": 1, 2101 "nullPointMode": "null", 2102 "options": { 2103 "dataLinks": [] 2104 }, 2105 "percentage": false, 2106 "pointradius": 2, 2107 "points": false, 2108 "renderer": "flot", 2109 "seriesOverrides": [ 2110 { 2111 "alias": "{}", 2112 "color": "#C4162A" 2113 } 2114 ], 2115 "spaceLength": 10, 2116 "stack": false, 2117 "steppedLine": false, 2118 "targets": [ 2119 { 2120 "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/distributor\"} | logfmt | level=\"error\"[1m]))", 2121 "refId": "A" 2122 } 2123 ], 2124 "thresholds": [], 2125 "timeFrom": null, 2126 "timeRegions": [], 2127 "timeShift": null, 2128 "title": "Error Log Rate", 2129 "tooltip": { 2130 "shared": true, 2131 "sort": 2, 2132 "value_type": "individual" 2133 }, 2134 "type": "graph", 2135 "xaxis": { 2136 "buckets": null, 2137 "mode": "time", 2138 "name": null, 2139 "show": false, 2140 "values": [] 2141 }, 2142 "yaxes": [ 2143 { 2144 "format": "short", 2145 "label": null, 2146 "logBase": 1, 2147 "max": null, 2148 "min": null, 2149 "show": true 2150 }, 2151 { 2152 "format": "short", 2153 "label": null, 2154 "logBase": 1, 2155 "max": null, 2156 "min": null, 2157 "show": false 2158 } 2159 ], 2160 "yaxis": { 2161 "align": false, 2162 "alignLevel": null 2163 } 2164 }, 2165 { 2166 "datasource": "$logs", 2167 "gridPos": { 2168 "h": 18, 2169 "w": 12, 2170 "x": 12, 2171 "y": 32 2172 }, 2173 "id": 29, 2174 "options": { 2175 "showLabels": false, 2176 "showTime": false, 2177 "sortOrder": "Descending", 2178 "wrapLogMessage": true 2179 }, 2180 "targets": [ 2181 { 2182 "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/distributor\"} | logfmt | level=\"error\"", 2183 "refId": "A" 2184 } 2185 ], 2186 "timeFrom": null, 2187 "timeShift": null, 2188 "title": "Logs", 2189 "type": "logs" 2190 }, 2191 { 2192 "aliasColors": {}, 2193 "bars": false, 2194 "dashLength": 10, 2195 "dashes": false, 2196 "datasource": "$datasource", 2197 "fill": 0, 2198 "fillGradient": 0, 2199 "gridPos": { 2200 "h": 7, 2201 "w": 6, 2202 "x": 0, 2203 "y": 35 2204 }, 2205 "hiddenSeries": false, 2206 "id": 33, 2207 "legend": { 2208 "avg": false, 2209 "current": false, 2210 "max": false, 2211 "min": false, 2212 "show": false, 2213 "total": false, 2214 "values": false 2215 }, 2216 "lines": true, 2217 "linewidth": 1, 2218 "nullPointMode": "null", 2219 "options": { 2220 "dataLinks": [] 2221 }, 2222 "percentage": false, 2223 "pointradius": 1, 2224 "points": false, 2225 "renderer": "flot", 2226 "seriesOverrides": [], 2227 "spaceLength": 10, 2228 "stack": false, 2229 "steppedLine": false, 2230 "targets": [ 2231 { 2232 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/distributor\", status_code!~\"5[0-9]{2}\"}[5m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/distributor\"}[5m])) by (route)", 2233 "interval": "", 2234 "intervalFactor": 1, 2235 "legendFormat": "{{route}}", 2236 "refId": "A" 2237 } 2238 ], 2239 "thresholds": [], 2240 "timeFrom": null, 2241 "timeRegions": [], 2242 "timeShift": null, 2243 "title": "Success Rate", 2244 "tooltip": { 2245 "shared": true, 2246 "sort": 2, 2247 "value_type": "individual" 2248 }, 2249 "type": "graph", 2250 "xaxis": { 2251 "buckets": null, 2252 "mode": "time", 2253 "name": null, 2254 "show": true, 2255 "values": [] 2256 }, 2257 "yaxes": [ 2258 { 2259 "format": "short", 2260 "label": null, 2261 "logBase": 1, 2262 "max": null, 2263 "min": null, 2264 "show": true 2265 }, 2266 { 2267 "format": "short", 2268 "label": null, 2269 "logBase": 1, 2270 "max": null, 2271 "min": null, 2272 "show": true 2273 } 2274 ], 2275 "yaxis": { 2276 "align": false, 2277 "alignLevel": null 2278 } 2279 }, 2280 { 2281 "aliasColors": {}, 2282 "bars": false, 2283 "dashLength": 10, 2284 "dashes": false, 2285 "datasource": "$datasource", 2286 "fill": 1, 2287 "fillGradient": 0, 2288 "gridPos": { 2289 "h": 7, 2290 "w": 6, 2291 "x": 6, 2292 "y": 35 2293 }, 2294 "hiddenSeries": false, 2295 "id": 32, 2296 "legend": { 2297 "avg": false, 2298 "current": false, 2299 "max": false, 2300 "min": false, 2301 "show": false, 2302 "total": false, 2303 "values": false 2304 }, 2305 "lines": true, 2306 "linewidth": 1, 2307 "nullPointMode": "null", 2308 "options": { 2309 "dataLinks": [] 2310 }, 2311 "percentage": false, 2312 "pointradius": 1, 2313 "points": false, 2314 "renderer": "flot", 2315 "seriesOverrides": [], 2316 "spaceLength": 10, 2317 "stack": false, 2318 "steppedLine": false, 2319 "targets": [ 2320 { 2321 "expr": "sum(rate(loki_distributor_ingester_append_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (ingester)", 2322 "intervalFactor": 1, 2323 "legendFormat": "{{ingester}}", 2324 "refId": "A" 2325 } 2326 ], 2327 "thresholds": [], 2328 "timeFrom": null, 2329 "timeRegions": [], 2330 "timeShift": null, 2331 "title": "Append Failures By Ingester", 2332 "tooltip": { 2333 "shared": true, 2334 "sort": 2, 2335 "value_type": "individual" 2336 }, 2337 "type": "graph", 2338 "xaxis": { 2339 "buckets": null, 2340 "mode": "time", 2341 "name": null, 2342 "show": true, 2343 "values": [] 2344 }, 2345 "yaxes": [ 2346 { 2347 "format": "short", 2348 "label": null, 2349 "logBase": 1, 2350 "max": null, 2351 "min": null, 2352 "show": true 2353 }, 2354 { 2355 "format": "short", 2356 "label": null, 2357 "logBase": 1, 2358 "max": null, 2359 "min": null, 2360 "show": true 2361 } 2362 ], 2363 "yaxis": { 2364 "align": false, 2365 "alignLevel": null 2366 } 2367 }, 2368 { 2369 "aliasColors": {}, 2370 "bars": false, 2371 "dashLength": 10, 2372 "dashes": false, 2373 "datasource": "$datasource", 2374 "fill": 1, 2375 "fillGradient": 0, 2376 "gridPos": { 2377 "h": 7, 2378 "w": 6, 2379 "x": 0, 2380 "y": 42 2381 }, 2382 "hiddenSeries": false, 2383 "id": 34, 2384 "legend": { 2385 "avg": false, 2386 "current": false, 2387 "max": false, 2388 "min": false, 2389 "show": false, 2390 "total": false, 2391 "values": false 2392 }, 2393 "lines": true, 2394 "linewidth": 1, 2395 "nullPointMode": "null", 2396 "options": { 2397 "dataLinks": [] 2398 }, 2399 "percentage": false, 2400 "pointradius": 1, 2401 "points": false, 2402 "renderer": "flot", 2403 "seriesOverrides": [], 2404 "spaceLength": 10, 2405 "stack": false, 2406 "steppedLine": false, 2407 "targets": [ 2408 { 2409 "expr": "sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", 2410 "intervalFactor": 1, 2411 "legendFormat": "{{pod}}", 2412 "refId": "A" 2413 } 2414 ], 2415 "thresholds": [], 2416 "timeFrom": null, 2417 "timeRegions": [], 2418 "timeShift": null, 2419 "title": "Bytes Received/Second", 2420 "tooltip": { 2421 "shared": true, 2422 "sort": 2, 2423 "value_type": "individual" 2424 }, 2425 "type": "graph", 2426 "xaxis": { 2427 "buckets": null, 2428 "mode": "time", 2429 "name": null, 2430 "show": true, 2431 "values": [] 2432 }, 2433 "yaxes": [ 2434 { 2435 "format": "short", 2436 "label": null, 2437 "logBase": 1, 2438 "max": null, 2439 "min": null, 2440 "show": true 2441 }, 2442 { 2443 "format": "short", 2444 "label": null, 2445 "logBase": 1, 2446 "max": null, 2447 "min": null, 2448 "show": true 2449 } 2450 ], 2451 "yaxis": { 2452 "align": false, 2453 "alignLevel": null 2454 } 2455 }, 2456 { 2457 "aliasColors": {}, 2458 "bars": false, 2459 "dashLength": 10, 2460 "dashes": false, 2461 "datasource": "$datasource", 2462 "fill": 1, 2463 "fillGradient": 0, 2464 "gridPos": { 2465 "h": 7, 2466 "w": 6, 2467 "x": 6, 2468 "y": 42 2469 }, 2470 "hiddenSeries": false, 2471 "id": 35, 2472 "legend": { 2473 "avg": false, 2474 "current": false, 2475 "max": false, 2476 "min": false, 2477 "show": false, 2478 "total": false, 2479 "values": false 2480 }, 2481 "lines": true, 2482 "linewidth": 1, 2483 "nullPointMode": "null", 2484 "options": { 2485 "dataLinks": [] 2486 }, 2487 "percentage": false, 2488 "pointradius": 1, 2489 "points": false, 2490 "renderer": "flot", 2491 "seriesOverrides": [], 2492 "spaceLength": 10, 2493 "stack": false, 2494 "steppedLine": false, 2495 "targets": [ 2496 { 2497 "expr": "sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", 2498 "intervalFactor": 1, 2499 "legendFormat": "{{pod}}", 2500 "refId": "A" 2501 } 2502 ], 2503 "thresholds": [], 2504 "timeFrom": null, 2505 "timeRegions": [], 2506 "timeShift": null, 2507 "title": "Lines Received/Second", 2508 "tooltip": { 2509 "shared": true, 2510 "sort": 2, 2511 "value_type": "individual" 2512 }, 2513 "type": "graph", 2514 "xaxis": { 2515 "buckets": null, 2516 "mode": "time", 2517 "name": null, 2518 "show": true, 2519 "values": [] 2520 }, 2521 "yaxes": [ 2522 { 2523 "format": "short", 2524 "label": null, 2525 "logBase": 1, 2526 "max": null, 2527 "min": null, 2528 "show": true 2529 }, 2530 { 2531 "format": "short", 2532 "label": null, 2533 "logBase": 1, 2534 "max": null, 2535 "min": null, 2536 "show": true 2537 } 2538 ], 2539 "yaxis": { 2540 "align": false, 2541 "alignLevel": null 2542 } 2543 } 2544 ], 2545 "title": "Distributor", 2546 "type": "row" 2547 }, 2548 { 2549 "collapsed": true, 2550 "datasource": null, 2551 "gridPos": { 2552 "h": 1, 2553 "w": 24, 2554 "x": 0, 2555 "y": 28 2556 }, 2557 "id": 19, 2558 "panels": [ 2559 { 2560 "aliasColors": {}, 2561 "bars": false, 2562 "dashLength": 10, 2563 "dashes": false, 2564 "datasource": "$datasource", 2565 "fill": 1, 2566 "fillGradient": 0, 2567 "gridPos": { 2568 "h": 7, 2569 "w": 6, 2570 "x": 0, 2571 "y": 29 2572 }, 2573 "hiddenSeries": false, 2574 "id": 36, 2575 "legend": { 2576 "avg": false, 2577 "current": false, 2578 "max": false, 2579 "min": false, 2580 "show": false, 2581 "total": false, 2582 "values": false 2583 }, 2584 "lines": false, 2585 "linewidth": 1, 2586 "nullPointMode": "null", 2587 "options": { 2588 "dataLinks": [] 2589 }, 2590 "percentage": false, 2591 "pointradius": 1, 2592 "points": true, 2593 "renderer": "flot", 2594 "seriesOverrides": [], 2595 "spaceLength": 10, 2596 "stack": false, 2597 "steppedLine": false, 2598 "targets": [ 2599 { 2600 "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"ingester.*\"}", 2601 "intervalFactor": 3, 2602 "legendFormat": "{{pod}}-{{container}}", 2603 "refId": "A" 2604 } 2605 ], 2606 "thresholds": [], 2607 "timeFrom": null, 2608 "timeRegions": [], 2609 "timeShift": null, 2610 "title": "CPU Usage", 2611 "tooltip": { 2612 "shared": true, 2613 "sort": 2, 2614 "value_type": "individual" 2615 }, 2616 "type": "graph", 2617 "xaxis": { 2618 "buckets": null, 2619 "mode": "time", 2620 "name": null, 2621 "show": true, 2622 "values": [] 2623 }, 2624 "yaxes": [ 2625 { 2626 "format": "short", 2627 "label": null, 2628 "logBase": 1, 2629 "max": null, 2630 "min": null, 2631 "show": true 2632 }, 2633 { 2634 "format": "short", 2635 "label": null, 2636 "logBase": 1, 2637 "max": null, 2638 "min": null, 2639 "show": true 2640 } 2641 ], 2642 "yaxis": { 2643 "align": false, 2644 "alignLevel": null 2645 } 2646 }, 2647 { 2648 "aliasColors": {}, 2649 "bars": false, 2650 "dashLength": 10, 2651 "dashes": false, 2652 "datasource": "$datasource", 2653 "fill": 1, 2654 "fillGradient": 0, 2655 "gridPos": { 2656 "h": 7, 2657 "w": 6, 2658 "x": 0, 2659 "y": 36 2660 }, 2661 "hiddenSeries": false, 2662 "id": 37, 2663 "legend": { 2664 "avg": false, 2665 "current": false, 2666 "hideEmpty": false, 2667 "hideZero": false, 2668 "max": false, 2669 "min": false, 2670 "show": false, 2671 "total": false, 2672 "values": false 2673 }, 2674 "lines": false, 2675 "linewidth": 1, 2676 "nullPointMode": "null", 2677 "options": { 2678 "dataLinks": [] 2679 }, 2680 "percentage": false, 2681 "pointradius": 1, 2682 "points": true, 2683 "renderer": "flot", 2684 "seriesOverrides": [], 2685 "spaceLength": 10, 2686 "stack": false, 2687 "steppedLine": false, 2688 "targets": [ 2689 { 2690 "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"ingester.*\"}", 2691 "instant": false, 2692 "intervalFactor": 3, 2693 "legendFormat": "{{pod}}", 2694 "refId": "A" 2695 } 2696 ], 2697 "thresholds": [], 2698 "timeFrom": null, 2699 "timeRegions": [], 2700 "timeShift": null, 2701 "title": "Memory Usage", 2702 "tooltip": { 2703 "shared": true, 2704 "sort": 2, 2705 "value_type": "individual" 2706 }, 2707 "type": "graph", 2708 "xaxis": { 2709 "buckets": null, 2710 "mode": "time", 2711 "name": null, 2712 "show": true, 2713 "values": [] 2714 }, 2715 "yaxes": [ 2716 { 2717 "format": "bytes", 2718 "label": null, 2719 "logBase": 1, 2720 "max": null, 2721 "min": null, 2722 "show": true 2723 }, 2724 { 2725 "format": "short", 2726 "label": null, 2727 "logBase": 1, 2728 "max": null, 2729 "min": null, 2730 "show": true 2731 } 2732 ], 2733 "yaxis": { 2734 "align": false, 2735 "alignLevel": null 2736 } 2737 }, 2738 { 2739 "aliasColors": {}, 2740 "bars": true, 2741 "dashLength": 10, 2742 "dashes": false, 2743 "datasource": "$logs", 2744 "fill": 1, 2745 "fillGradient": 0, 2746 "gridPos": { 2747 "h": 3, 2748 "w": 18, 2749 "x": 12, 2750 "y": 29 2751 }, 2752 "hiddenSeries": false, 2753 "id": 38, 2754 "legend": { 2755 "avg": false, 2756 "current": false, 2757 "max": false, 2758 "min": false, 2759 "show": false, 2760 "total": false, 2761 "values": false 2762 }, 2763 "lines": false, 2764 "linewidth": 1, 2765 "nullPointMode": "null", 2766 "options": { 2767 "dataLinks": [] 2768 }, 2769 "percentage": false, 2770 "pointradius": 2, 2771 "points": false, 2772 "renderer": "flot", 2773 "seriesOverrides": [ 2774 { 2775 "alias": "{}", 2776 "color": "#F2495C" 2777 } 2778 ], 2779 "spaceLength": 10, 2780 "stack": false, 2781 "steppedLine": false, 2782 "targets": [ 2783 { 2784 "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\"} | logfmt | level=\"error\"[1m]))", 2785 "refId": "A" 2786 } 2787 ], 2788 "thresholds": [], 2789 "timeFrom": null, 2790 "timeRegions": [], 2791 "timeShift": null, 2792 "title": "Error Log Rate", 2793 "tooltip": { 2794 "shared": true, 2795 "sort": 2, 2796 "value_type": "individual" 2797 }, 2798 "type": "graph", 2799 "xaxis": { 2800 "buckets": null, 2801 "mode": "time", 2802 "name": null, 2803 "show": false, 2804 "values": [] 2805 }, 2806 "yaxes": [ 2807 { 2808 "format": "short", 2809 "label": null, 2810 "logBase": 1, 2811 "max": null, 2812 "min": null, 2813 "show": true 2814 }, 2815 { 2816 "format": "short", 2817 "label": null, 2818 "logBase": 1, 2819 "max": null, 2820 "min": null, 2821 "show": false 2822 } 2823 ], 2824 "yaxis": { 2825 "align": false, 2826 "alignLevel": null 2827 } 2828 }, 2829 { 2830 "datasource": "$logs", 2831 "gridPos": { 2832 "h": 18, 2833 "w": 18, 2834 "x": 12, 2835 "y": 32 2836 }, 2837 "id": 39, 2838 "options": { 2839 "showLabels": false, 2840 "showTime": false, 2841 "sortOrder": "Descending", 2842 "wrapLogMessage": true 2843 }, 2844 "targets": [ 2845 { 2846 "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\"} | logfmt | level=\"error\"", 2847 "refId": "A" 2848 } 2849 ], 2850 "timeFrom": null, 2851 "timeShift": null, 2852 "title": "Logs", 2853 "type": "logs" 2854 }, 2855 { 2856 "aliasColors": {}, 2857 "bars": false, 2858 "dashLength": 10, 2859 "dashes": false, 2860 "datasource": "$datasource", 2861 "fill": 0, 2862 "fillGradient": 0, 2863 "gridPos": { 2864 "h": 7, 2865 "w": 6, 2866 "x": 0, 2867 "y": 41 2868 }, 2869 "hiddenSeries": false, 2870 "id": 67, 2871 "legend": { 2872 "avg": false, 2873 "current": false, 2874 "max": false, 2875 "min": false, 2876 "show": false, 2877 "total": false, 2878 "values": false 2879 }, 2880 "lines": true, 2881 "linewidth": 1, 2882 "nullPointMode": "null", 2883 "options": { 2884 "dataLinks": [] 2885 }, 2886 "percentage": false, 2887 "pointradius": 1, 2888 "points": false, 2889 "renderer": "flot", 2890 "seriesOverrides": [], 2891 "spaceLength": 10, 2892 "stack": false, 2893 "steppedLine": false, 2894 "targets": [ 2895 { 2896 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", status_code!~\"5[0-9]{2}\"}[1m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\"}[1m])) by (route)", 2897 "interval": "", 2898 "intervalFactor": 1, 2899 "legendFormat": "{{route}}", 2900 "refId": "A" 2901 } 2902 ], 2903 "thresholds": [], 2904 "timeFrom": null, 2905 "timeRegions": [], 2906 "timeShift": null, 2907 "title": "Success Rate", 2908 "tooltip": { 2909 "shared": true, 2910 "sort": 2, 2911 "value_type": "individual" 2912 }, 2913 "type": "graph", 2914 "xaxis": { 2915 "buckets": null, 2916 "mode": "time", 2917 "name": null, 2918 "show": true, 2919 "values": [] 2920 }, 2921 "yaxes": [ 2922 { 2923 "format": "short", 2924 "label": null, 2925 "logBase": 1, 2926 "max": null, 2927 "min": null, 2928 "show": true 2929 }, 2930 { 2931 "format": "short", 2932 "label": null, 2933 "logBase": 1, 2934 "max": null, 2935 "min": null, 2936 "show": true 2937 } 2938 ], 2939 "yaxis": { 2940 "align": false, 2941 "alignLevel": null 2942 } 2943 } 2944 ], 2945 "title": "Ingester", 2946 "type": "row" 2947 }, 2948 { 2949 "collapsed": true, 2950 "datasource": null, 2951 "gridPos": { 2952 "h": 1, 2953 "w": 24, 2954 "x": 0, 2955 "y": 29 2956 }, 2957 "id": 104, 2958 "panels": [ 2959 { 2960 "aliasColors": {}, 2961 "bars": false, 2962 "dashLength": 10, 2963 "dashes": false, 2964 "datasource": "$datasource", 2965 "fill": 1, 2966 "fillGradient": 0, 2967 "gridPos": { 2968 "h": 8, 2969 "w": 12, 2970 "x": 0, 2971 "y": 30 2972 }, 2973 "hiddenSeries": false, 2974 "id": 106, 2975 "legend": { 2976 "avg": false, 2977 "current": false, 2978 "hideEmpty": true, 2979 "hideZero": true, 2980 "max": false, 2981 "min": false, 2982 "show": false, 2983 "total": false, 2984 "values": false 2985 }, 2986 "lines": true, 2987 "linewidth": 1, 2988 "nullPointMode": "null", 2989 "options": { 2990 "dataLinks": [] 2991 }, 2992 "percentage": false, 2993 "pointradius": 2, 2994 "points": false, 2995 "renderer": "flot", 2996 "seriesOverrides": [], 2997 "spaceLength": 10, 2998 "stack": false, 2999 "steppedLine": false, 3000 "targets": [ 3001 { 3002 "expr": "topk(10,sum by (tenant) (loki_ingester_memory_streams{cluster=\"$cluster\",job=\"$namespace/ingester\"}))", 3003 "interval": "", 3004 "legendFormat": "{{ tenant }}", 3005 "refId": "A" 3006 } 3007 ], 3008 "thresholds": [], 3009 "timeFrom": null, 3010 "timeRegions": [], 3011 "timeShift": null, 3012 "title": "Active Streams", 3013 "tooltip": { 3014 "shared": false, 3015 "sort": 0, 3016 "value_type": "individual" 3017 }, 3018 "type": "graph", 3019 "xaxis": { 3020 "buckets": null, 3021 "mode": "time", 3022 "name": null, 3023 "show": true, 3024 "values": [] 3025 }, 3026 "yaxes": [ 3027 { 3028 "format": "short", 3029 "label": null, 3030 "logBase": 1, 3031 "max": null, 3032 "min": null, 3033 "show": true 3034 }, 3035 { 3036 "format": "short", 3037 "label": null, 3038 "logBase": 1, 3039 "max": null, 3040 "min": null, 3041 "show": true 3042 } 3043 ], 3044 "yaxis": { 3045 "align": false, 3046 "alignLevel": null 3047 } 3048 }, 3049 { 3050 "aliasColors": {}, 3051 "bars": false, 3052 "dashLength": 10, 3053 "dashes": false, 3054 "datasource": "$datasource", 3055 "fill": 1, 3056 "fillGradient": 0, 3057 "gridPos": { 3058 "h": 8, 3059 "w": 12, 3060 "x": 12, 3061 "y": 30 3062 }, 3063 "hiddenSeries": false, 3064 "id": 108, 3065 "legend": { 3066 "avg": false, 3067 "current": false, 3068 "hideEmpty": true, 3069 "hideZero": true, 3070 "max": false, 3071 "min": false, 3072 "show": false, 3073 "total": false, 3074 "values": false 3075 }, 3076 "lines": true, 3077 "linewidth": 1, 3078 "nullPointMode": "null", 3079 "options": { 3080 "dataLinks": [] 3081 }, 3082 "percentage": false, 3083 "pointradius": 2, 3084 "points": false, 3085 "renderer": "flot", 3086 "seriesOverrides": [], 3087 "spaceLength": 10, 3088 "stack": false, 3089 "steppedLine": false, 3090 "targets": [ 3091 { 3092 "expr": "topk(10, sum by (tenant) (rate(loki_ingester_streams_created_total{cluster=\"$cluster\",job=\"$namespace/ingester\"}[1m]) > 0))", 3093 "interval": "", 3094 "legendFormat": "{{ tenant }}", 3095 "refId": "A" 3096 } 3097 ], 3098 "thresholds": [], 3099 "timeFrom": null, 3100 "timeRegions": [], 3101 "timeShift": null, 3102 "title": "Streams Created/Sec", 3103 "tooltip": { 3104 "shared": false, 3105 "sort": 0, 3106 "value_type": "individual" 3107 }, 3108 "type": "graph", 3109 "xaxis": { 3110 "buckets": null, 3111 "mode": "time", 3112 "name": null, 3113 "show": true, 3114 "values": [] 3115 }, 3116 "yaxes": [ 3117 { 3118 "format": "short", 3119 "label": null, 3120 "logBase": 1, 3121 "max": null, 3122 "min": null, 3123 "show": true 3124 }, 3125 { 3126 "format": "short", 3127 "label": null, 3128 "logBase": 1, 3129 "max": null, 3130 "min": null, 3131 "show": true 3132 } 3133 ], 3134 "yaxis": { 3135 "align": false, 3136 "alignLevel": null 3137 } 3138 } 3139 ], 3140 "title": "Streams", 3141 "type": "row" 3142 }, 3143 { 3144 "collapsed": true, 3145 "datasource": null, 3146 "gridPos": { 3147 "h": 1, 3148 "w": 24, 3149 "x": 0, 3150 "y": 30 3151 }, 3152 "id": 94, 3153 "panels": [ 3154 { 3155 "aliasColors": {}, 3156 "bars": false, 3157 "dashLength": 10, 3158 "dashes": false, 3159 "datasource": "$datasource", 3160 "fill": 1, 3161 "fillGradient": 0, 3162 "gridPos": { 3163 "h": 8, 3164 "w": 12, 3165 "x": 0, 3166 "y": 31 3167 }, 3168 "hiddenSeries": false, 3169 "id": 102, 3170 "legend": { 3171 "avg": false, 3172 "current": false, 3173 "max": false, 3174 "min": false, 3175 "show": true, 3176 "total": false, 3177 "values": false 3178 }, 3179 "lines": true, 3180 "linewidth": 1, 3181 "nullPointMode": "null", 3182 "options": { 3183 "dataLinks": [] 3184 }, 3185 "percentage": false, 3186 "pointradius": 2, 3187 "points": false, 3188 "renderer": "flot", 3189 "seriesOverrides": [ 3190 { 3191 "alias": "De-Dupe Ratio", 3192 "yaxis": 2 3193 } 3194 ], 3195 "spaceLength": 10, 3196 "stack": false, 3197 "steppedLine": false, 3198 "targets": [ 3199 { 3200 "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\",job=\"$namespace/ingester\"}[1m]))", 3201 "interval": "", 3202 "legendFormat": "Chunks", 3203 "refId": "A" 3204 }, 3205 { 3206 "expr": "sum(increase(loki_chunk_store_deduped_chunks_total{cluster=\"$cluster\", job=\"$namespace/ingester\"}[1m]))/sum(increase(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=\"$namespace/ingester\"}[1m])) < 1", 3207 "interval": "", 3208 "legendFormat": "De-Dupe Ratio", 3209 "refId": "B" 3210 } 3211 ], 3212 "thresholds": [], 3213 "timeFrom": null, 3214 "timeRegions": [], 3215 "timeShift": null, 3216 "title": "Chunks Flushed/Sec", 3217 "tooltip": { 3218 "shared": true, 3219 "sort": 0, 3220 "value_type": "individual" 3221 }, 3222 "type": "graph", 3223 "xaxis": { 3224 "buckets": null, 3225 "mode": "time", 3226 "name": null, 3227 "show": true, 3228 "values": [] 3229 }, 3230 "yaxes": [ 3231 { 3232 "format": "short", 3233 "label": null, 3234 "logBase": 1, 3235 "max": null, 3236 "min": null, 3237 "show": true 3238 }, 3239 { 3240 "format": "short", 3241 "label": null, 3242 "logBase": 1, 3243 "max": null, 3244 "min": null, 3245 "show": true 3246 } 3247 ], 3248 "yaxis": { 3249 "align": false, 3250 "alignLevel": null 3251 } 3252 }, 3253 { 3254 "cards": { 3255 "cardPadding": null, 3256 "cardRound": null 3257 }, 3258 "color": { 3259 "cardColor": "#b4ff00", 3260 "colorScale": "sqrt", 3261 "colorScheme": "interpolateSpectral", 3262 "exponent": 0.5, 3263 "mode": "spectrum" 3264 }, 3265 "dataFormat": "tsbuckets", 3266 "datasource": "$datasource", 3267 "gridPos": { 3268 "h": 8, 3269 "w": 12, 3270 "x": 12, 3271 "y": 31 3272 }, 3273 "heatmap": {}, 3274 "hideZeroBuckets": false, 3275 "highlightCards": true, 3276 "id": 100, 3277 "legend": { 3278 "show": true 3279 }, 3280 "reverseYBuckets": false, 3281 "targets": [ 3282 { 3283 "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\",job=\"$namespace/ingester\"}[1m])) by (le)", 3284 "format": "heatmap", 3285 "instant": false, 3286 "interval": "", 3287 "legendFormat": "{{ le }}", 3288 "refId": "A" 3289 } 3290 ], 3291 "timeFrom": null, 3292 "timeShift": null, 3293 "title": "Chunk Size Bytes", 3294 "tooltip": { 3295 "show": true, 3296 "showHistogram": false 3297 }, 3298 "type": "heatmap", 3299 "xAxis": { 3300 "show": true 3301 }, 3302 "xBucketNumber": null, 3303 "xBucketSize": null, 3304 "yAxis": { 3305 "decimals": 0, 3306 "format": "bytes", 3307 "logBase": 1, 3308 "max": null, 3309 "min": null, 3310 "show": true, 3311 "splitFactor": null 3312 }, 3313 "yBucketBound": "auto", 3314 "yBucketNumber": null, 3315 "yBucketSize": null 3316 }, 3317 { 3318 "aliasColors": {}, 3319 "bars": false, 3320 "dashLength": 10, 3321 "dashes": false, 3322 "datasource": "$datasource", 3323 "fill": 7, 3324 "fillGradient": 0, 3325 "gridPos": { 3326 "h": 9, 3327 "w": 12, 3328 "x": 0, 3329 "y": 39 3330 }, 3331 "hiddenSeries": false, 3332 "id": 96, 3333 "legend": { 3334 "avg": false, 3335 "current": false, 3336 "max": false, 3337 "min": false, 3338 "show": true, 3339 "total": false, 3340 "values": false 3341 }, 3342 "lines": true, 3343 "linewidth": 1, 3344 "nullPointMode": "null", 3345 "options": { 3346 "dataLinks": [] 3347 }, 3348 "percentage": false, 3349 "pointradius": 2, 3350 "points": false, 3351 "renderer": "flot", 3352 "seriesOverrides": [], 3353 "spaceLength": 10, 3354 "stack": true, 3355 "steppedLine": false, 3356 "targets": [ 3357 { 3358 "expr": "sum by(reason) (rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\",job=~\"$namespace/ingester\", namespace=~\"$namespace\"}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\",job=~\"$namespace/ingester\", namespace=~\"$namespace\"}[$__rate_interval]))", 3359 "interval": "", 3360 "legendFormat": "{{ reason }}" 3361 } 3362 ], 3363 "thresholds": [], 3364 "timeFrom": null, 3365 "timeRegions": [], 3366 "timeShift": null, 3367 "title": "Chunk Flush Reason %", 3368 "tooltip": { 3369 "shared": true, 3370 "sort": 0, 3371 "value_type": "individual" 3372 }, 3373 "type": "graph", 3374 "xaxis": { 3375 "buckets": null, 3376 "mode": "time", 3377 "name": null, 3378 "show": true, 3379 "values": [] 3380 }, 3381 "yaxes": [ 3382 { 3383 "format": "percentunit", 3384 "label": null, 3385 "logBase": 1, 3386 "max": "1", 3387 "min": "0", 3388 "show": true 3389 }, 3390 { 3391 "format": "short", 3392 "label": null, 3393 "logBase": 1, 3394 "max": null, 3395 "min": null, 3396 "show": true 3397 } 3398 ], 3399 "yaxis": { 3400 "align": false, 3401 "alignLevel": null 3402 } 3403 }, 3404 { 3405 "cards": { 3406 "cardPadding": null, 3407 "cardRound": null 3408 }, 3409 "color": { 3410 "cardColor": "#b4ff00", 3411 "colorScale": "sqrt", 3412 "colorScheme": "interpolateSpectral", 3413 "exponent": 0.5, 3414 "max": null, 3415 "min": null, 3416 "mode": "spectrum" 3417 }, 3418 "dataFormat": "tsbuckets", 3419 "datasource": "$datasource", 3420 "gridPos": { 3421 "h": 9, 3422 "w": 12, 3423 "x": 12, 3424 "y": 39 3425 }, 3426 "heatmap": {}, 3427 "hideZeroBuckets": true, 3428 "highlightCards": true, 3429 "id": 98, 3430 "legend": { 3431 "show": true 3432 }, 3433 "reverseYBuckets": false, 3434 "targets": [ 3435 { 3436 "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=\"$namespace/ingester\"}[1m]))", 3437 "format": "heatmap", 3438 "instant": false, 3439 "interval": "", 3440 "legendFormat": "{{ le }}", 3441 "refId": "A" 3442 } 3443 ], 3444 "timeFrom": null, 3445 "timeShift": null, 3446 "title": "Chunk Utilization", 3447 "tooltip": { 3448 "show": true, 3449 "showHistogram": false 3450 }, 3451 "type": "heatmap", 3452 "xAxis": { 3453 "show": true 3454 }, 3455 "xBucketNumber": null, 3456 "xBucketSize": null, 3457 "yAxis": { 3458 "decimals": 0, 3459 "format": "percentunit", 3460 "logBase": 1, 3461 "max": null, 3462 "min": null, 3463 "show": true, 3464 "splitFactor": null 3465 }, 3466 "yBucketBound": "auto", 3467 "yBucketNumber": null, 3468 "yBucketSize": null 3469 } 3470 ], 3471 "title": "Chunks", 3472 "type": "row" 3473 }, 3474 { 3475 "collapsed": true, 3476 "datasource": null, 3477 "gridPos": { 3478 "h": 1, 3479 "w": 24, 3480 "x": 0, 3481 "y": 31 3482 }, 3483 "id": 64, 3484 "panels": [ 3485 { 3486 "aliasColors": {}, 3487 "bars": false, 3488 "dashLength": 10, 3489 "dashes": false, 3490 "datasource": "$datasource", 3491 "fill": 1, 3492 "fillGradient": 0, 3493 "gridPos": { 3494 "h": 7, 3495 "w": 6, 3496 "x": 0, 3497 "y": 32 3498 }, 3499 "hiddenSeries": false, 3500 "id": 68, 3501 "legend": { 3502 "avg": false, 3503 "current": false, 3504 "max": false, 3505 "min": false, 3506 "show": false, 3507 "total": false, 3508 "values": false 3509 }, 3510 "lines": false, 3511 "linewidth": 1, 3512 "nullPointMode": "null", 3513 "options": { 3514 "dataLinks": [] 3515 }, 3516 "percentage": false, 3517 "pointradius": 1, 3518 "points": true, 3519 "renderer": "flot", 3520 "seriesOverrides": [], 3521 "spaceLength": 10, 3522 "stack": false, 3523 "steppedLine": false, 3524 "targets": [ 3525 { 3526 "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"querier.*\"}", 3527 "intervalFactor": 3, 3528 "legendFormat": "{{pod}}-{{container}}", 3529 "refId": "A" 3530 } 3531 ], 3532 "thresholds": [], 3533 "timeFrom": null, 3534 "timeRegions": [], 3535 "timeShift": null, 3536 "title": "CPU Usage", 3537 "tooltip": { 3538 "shared": true, 3539 "sort": 2, 3540 "value_type": "individual" 3541 }, 3542 "type": "graph", 3543 "xaxis": { 3544 "buckets": null, 3545 "mode": "time", 3546 "name": null, 3547 "show": true, 3548 "values": [] 3549 }, 3550 "yaxes": [ 3551 { 3552 "format": "short", 3553 "label": null, 3554 "logBase": 1, 3555 "max": null, 3556 "min": null, 3557 "show": true 3558 }, 3559 { 3560 "format": "short", 3561 "label": null, 3562 "logBase": 1, 3563 "max": null, 3564 "min": null, 3565 "show": true 3566 } 3567 ], 3568 "yaxis": { 3569 "align": false, 3570 "alignLevel": null 3571 } 3572 }, 3573 { 3574 "aliasColors": {}, 3575 "bars": false, 3576 "dashLength": 10, 3577 "dashes": false, 3578 "datasource": "$datasource", 3579 "fill": 1, 3580 "fillGradient": 0, 3581 "gridPos": { 3582 "h": 7, 3583 "w": 6, 3584 "x": 0, 3585 "y": 39 3586 }, 3587 "hiddenSeries": false, 3588 "id": 69, 3589 "legend": { 3590 "avg": false, 3591 "current": false, 3592 "hideEmpty": false, 3593 "hideZero": false, 3594 "max": false, 3595 "min": false, 3596 "show": false, 3597 "total": false, 3598 "values": false 3599 }, 3600 "lines": false, 3601 "linewidth": 1, 3602 "nullPointMode": "null", 3603 "options": { 3604 "dataLinks": [] 3605 }, 3606 "percentage": false, 3607 "pointradius": 1, 3608 "points": true, 3609 "renderer": "flot", 3610 "seriesOverrides": [], 3611 "spaceLength": 10, 3612 "stack": false, 3613 "steppedLine": false, 3614 "targets": [ 3615 { 3616 "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"querier.*\"}", 3617 "instant": false, 3618 "intervalFactor": 3, 3619 "legendFormat": "{{pod}}", 3620 "refId": "A" 3621 } 3622 ], 3623 "thresholds": [], 3624 "timeFrom": null, 3625 "timeRegions": [], 3626 "timeShift": null, 3627 "title": "Memory Usage", 3628 "tooltip": { 3629 "shared": true, 3630 "sort": 2, 3631 "value_type": "individual" 3632 }, 3633 "type": "graph", 3634 "xaxis": { 3635 "buckets": null, 3636 "mode": "time", 3637 "name": null, 3638 "show": true, 3639 "values": [] 3640 }, 3641 "yaxes": [ 3642 { 3643 "format": "bytes", 3644 "label": null, 3645 "logBase": 1, 3646 "max": null, 3647 "min": null, 3648 "show": true 3649 }, 3650 { 3651 "format": "short", 3652 "label": null, 3653 "logBase": 1, 3654 "max": null, 3655 "min": null, 3656 "show": true 3657 } 3658 ], 3659 "yaxis": { 3660 "align": false, 3661 "alignLevel": null 3662 } 3663 }, 3664 { 3665 "aliasColors": {}, 3666 "bars": true, 3667 "dashLength": 10, 3668 "dashes": false, 3669 "datasource": "$logs", 3670 "fill": 1, 3671 "fillGradient": 0, 3672 "gridPos": { 3673 "h": 3, 3674 "w": 18, 3675 "x": 12, 3676 "y": 32 3677 }, 3678 "hiddenSeries": false, 3679 "id": 65, 3680 "legend": { 3681 "avg": false, 3682 "current": false, 3683 "max": false, 3684 "min": false, 3685 "show": false, 3686 "total": false, 3687 "values": false 3688 }, 3689 "lines": false, 3690 "linewidth": 1, 3691 "nullPointMode": "null", 3692 "options": { 3693 "dataLinks": [] 3694 }, 3695 "percentage": false, 3696 "pointradius": 2, 3697 "points": false, 3698 "renderer": "flot", 3699 "seriesOverrides": [ 3700 { 3701 "alias": "{}", 3702 "color": "#F2495C" 3703 } 3704 ], 3705 "spaceLength": 10, 3706 "stack": false, 3707 "steppedLine": false, 3708 "targets": [ 3709 { 3710 "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\"} | logfmt | level=\"error\"[1m]))", 3711 "refId": "A" 3712 } 3713 ], 3714 "thresholds": [], 3715 "timeFrom": null, 3716 "timeRegions": [], 3717 "timeShift": null, 3718 "title": "Error Log Rate", 3719 "tooltip": { 3720 "shared": true, 3721 "sort": 2, 3722 "value_type": "individual" 3723 }, 3724 "type": "graph", 3725 "xaxis": { 3726 "buckets": null, 3727 "mode": "time", 3728 "name": null, 3729 "show": false, 3730 "values": [] 3731 }, 3732 "yaxes": [ 3733 { 3734 "format": "short", 3735 "label": null, 3736 "logBase": 1, 3737 "max": null, 3738 "min": null, 3739 "show": true 3740 }, 3741 { 3742 "format": "short", 3743 "label": null, 3744 "logBase": 1, 3745 "max": null, 3746 "min": null, 3747 "show": false 3748 } 3749 ], 3750 "yaxis": { 3751 "align": false, 3752 "alignLevel": null 3753 } 3754 }, 3755 { 3756 "datasource": "$logs", 3757 "gridPos": { 3758 "h": 18, 3759 "w": 18, 3760 "x": 12, 3761 "y": 35 3762 }, 3763 "id": 66, 3764 "options": { 3765 "showLabels": false, 3766 "showTime": false, 3767 "sortOrder": "Descending", 3768 "wrapLogMessage": true 3769 }, 3770 "targets": [ 3771 { 3772 "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\"} | logfmt | level=\"error\"", 3773 "refId": "A" 3774 } 3775 ], 3776 "timeFrom": null, 3777 "timeShift": null, 3778 "title": "Logs", 3779 "type": "logs" 3780 }, 3781 { 3782 "aliasColors": {}, 3783 "bars": false, 3784 "dashLength": 10, 3785 "dashes": false, 3786 "datasource": "$datasource", 3787 "fill": 0, 3788 "fillGradient": 0, 3789 "gridPos": { 3790 "h": 7, 3791 "w": 6, 3792 "x": 0, 3793 "y": 46 3794 }, 3795 "hiddenSeries": false, 3796 "id": 70, 3797 "legend": { 3798 "avg": false, 3799 "current": false, 3800 "max": false, 3801 "min": false, 3802 "show": false, 3803 "total": false, 3804 "values": false 3805 }, 3806 "lines": true, 3807 "linewidth": 1, 3808 "nullPointMode": "null", 3809 "options": { 3810 "dataLinks": [] 3811 }, 3812 "percentage": false, 3813 "pointradius": 1, 3814 "points": false, 3815 "renderer": "flot", 3816 "seriesOverrides": [], 3817 "spaceLength": 10, 3818 "stack": false, 3819 "steppedLine": false, 3820 "targets": [ 3821 { 3822 "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", status_code!~\"5[0-9]{2}\"}[1m])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\"}[1m])) by (route)", 3823 "interval": "", 3824 "intervalFactor": 1, 3825 "legendFormat": "{{route}}", 3826 "refId": "A" 3827 } 3828 ], 3829 "thresholds": [], 3830 "timeFrom": null, 3831 "timeRegions": [], 3832 "timeShift": null, 3833 "title": "Success Rate", 3834 "tooltip": { 3835 "shared": true, 3836 "sort": 2, 3837 "value_type": "individual" 3838 }, 3839 "type": "graph", 3840 "xaxis": { 3841 "buckets": null, 3842 "mode": "time", 3843 "name": null, 3844 "show": true, 3845 "values": [] 3846 }, 3847 "yaxes": [ 3848 { 3849 "format": "short", 3850 "label": null, 3851 "logBase": 1, 3852 "max": null, 3853 "min": null, 3854 "show": true 3855 }, 3856 { 3857 "format": "short", 3858 "label": null, 3859 "logBase": 1, 3860 "max": null, 3861 "min": null, 3862 "show": true 3863 } 3864 ], 3865 "yaxis": { 3866 "align": false, 3867 "alignLevel": null 3868 } 3869 } 3870 ], 3871 "title": "Querier", 3872 "type": "row" 3873 }, 3874 { 3875 "collapsed": true, 3876 "datasource": null, 3877 "gridPos": { 3878 "h": 1, 3879 "w": 24, 3880 "x": 0, 3881 "y": 32 3882 }, 3883 "id": 52, 3884 "panels": [ 3885 { 3886 "aliasColors": {}, 3887 "bars": false, 3888 "dashLength": 10, 3889 "dashes": false, 3890 "datasource": "$datasource", 3891 "fill": 1, 3892 "fillGradient": 0, 3893 "gridPos": { 3894 "h": 8, 3895 "w": 24, 3896 "x": 0, 3897 "y": 30 3898 }, 3899 "hiddenSeries": false, 3900 "id": 53, 3901 "interval": "", 3902 "legend": { 3903 "alignAsTable": true, 3904 "avg": false, 3905 "current": false, 3906 "max": false, 3907 "min": false, 3908 "rightSide": true, 3909 "show": true, 3910 "total": false, 3911 "values": false 3912 }, 3913 "lines": true, 3914 "linewidth": 1, 3915 "nullPointMode": "null", 3916 "options": { 3917 "dataLinks": [] 3918 }, 3919 "percentage": false, 3920 "pointradius": 1, 3921 "points": false, 3922 "renderer": "flot", 3923 "seriesOverrides": [], 3924 "spaceLength": 10, 3925 "stack": false, 3926 "steppedLine": false, 3927 "targets": [ 3928 { 3929 "expr": "histogram_quantile(.99, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", 3930 "intervalFactor": 1, 3931 "legendFormat": "{{container}}: .99-{{method}}-{{name}}", 3932 "refId": "A" 3933 }, 3934 { 3935 "expr": "histogram_quantile(.9, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", 3936 "hide": false, 3937 "legendFormat": "{{container}}: .9-{{method}}-{{name}}", 3938 "refId": "B" 3939 }, 3940 { 3941 "expr": "histogram_quantile(.5, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", 3942 "hide": false, 3943 "legendFormat": "{{container}}: .5-{{method}}-{{name}}", 3944 "refId": "C" 3945 } 3946 ], 3947 "thresholds": [], 3948 "timeFrom": null, 3949 "timeRegions": [], 3950 "timeShift": null, 3951 "title": "Latency By Method", 3952 "tooltip": { 3953 "shared": true, 3954 "sort": 2, 3955 "value_type": "individual" 3956 }, 3957 "type": "graph", 3958 "xaxis": { 3959 "buckets": null, 3960 "mode": "time", 3961 "name": null, 3962 "show": true, 3963 "values": [] 3964 }, 3965 "yaxes": [ 3966 { 3967 "format": "short", 3968 "label": null, 3969 "logBase": 1, 3970 "max": null, 3971 "min": null, 3972 "show": true 3973 }, 3974 { 3975 "format": "short", 3976 "label": null, 3977 "logBase": 1, 3978 "max": null, 3979 "min": null, 3980 "show": true 3981 } 3982 ], 3983 "yaxis": { 3984 "align": false, 3985 "alignLevel": null 3986 } 3987 }, 3988 { 3989 "aliasColors": {}, 3990 "bars": false, 3991 "dashLength": 10, 3992 "dashes": false, 3993 "datasource": "$datasource", 3994 "fill": 1, 3995 "fillGradient": 0, 3996 "gridPos": { 3997 "h": 8, 3998 "w": 24, 3999 "x": 0, 4000 "y": 38 4001 }, 4002 "hiddenSeries": false, 4003 "id": 54, 4004 "interval": "", 4005 "legend": { 4006 "alignAsTable": true, 4007 "avg": false, 4008 "current": false, 4009 "max": false, 4010 "min": false, 4011 "rightSide": true, 4012 "show": true, 4013 "total": false, 4014 "values": false 4015 }, 4016 "lines": true, 4017 "linewidth": 1, 4018 "nullPointMode": "null", 4019 "options": { 4020 "dataLinks": [] 4021 }, 4022 "percentage": false, 4023 "pointradius": 1, 4024 "points": false, 4025 "renderer": "flot", 4026 "seriesOverrides": [], 4027 "spaceLength": 10, 4028 "stack": false, 4029 "steppedLine": false, 4030 "targets": [ 4031 { 4032 "expr": "sum(rate(loki_memcache_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, method, name, container)", 4033 "intervalFactor": 1, 4034 "legendFormat": "{{container}}: {{status_code}}-{{method}}-{{name}}", 4035 "refId": "A" 4036 } 4037 ], 4038 "thresholds": [], 4039 "timeFrom": null, 4040 "timeRegions": [], 4041 "timeShift": null, 4042 "title": "Status By Method", 4043 "tooltip": { 4044 "shared": true, 4045 "sort": 2, 4046 "value_type": "individual" 4047 }, 4048 "type": "graph", 4049 "xaxis": { 4050 "buckets": null, 4051 "mode": "time", 4052 "name": null, 4053 "show": true, 4054 "values": [] 4055 }, 4056 "yaxes": [ 4057 { 4058 "format": "short", 4059 "label": null, 4060 "logBase": 1, 4061 "max": null, 4062 "min": null, 4063 "show": true 4064 }, 4065 { 4066 "format": "short", 4067 "label": null, 4068 "logBase": 1, 4069 "max": null, 4070 "min": null, 4071 "show": true 4072 } 4073 ], 4074 "yaxis": { 4075 "align": false, 4076 "alignLevel": null 4077 } 4078 } 4079 ], 4080 "title": "Memcached", 4081 "type": "row" 4082 }, 4083 { 4084 "collapsed": true, 4085 "datasource": null, 4086 "gridPos": { 4087 "h": 1, 4088 "w": 24, 4089 "x": 0, 4090 "y": 33 4091 }, 4092 "id": 57, 4093 "panels": [ 4094 { 4095 "aliasColors": {}, 4096 "bars": false, 4097 "dashLength": 10, 4098 "dashes": false, 4099 "datasource": "$datasource", 4100 "fill": 1, 4101 "fillGradient": 0, 4102 "gridPos": { 4103 "h": 8, 4104 "w": 24, 4105 "x": 0, 4106 "y": 31 4107 }, 4108 "hiddenSeries": false, 4109 "id": 55, 4110 "interval": "", 4111 "legend": { 4112 "alignAsTable": true, 4113 "avg": false, 4114 "current": false, 4115 "max": false, 4116 "min": false, 4117 "rightSide": true, 4118 "show": true, 4119 "total": false, 4120 "values": false 4121 }, 4122 "lines": true, 4123 "linewidth": 1, 4124 "nullPointMode": "null", 4125 "options": { 4126 "dataLinks": [] 4127 }, 4128 "percentage": false, 4129 "pointradius": 1, 4130 "points": false, 4131 "renderer": "flot", 4132 "seriesOverrides": [], 4133 "spaceLength": 10, 4134 "stack": false, 4135 "steppedLine": false, 4136 "targets": [ 4137 { 4138 "expr": "histogram_quantile(.99, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 4139 "intervalFactor": 1, 4140 "legendFormat": ".99-{{operation}}", 4141 "refId": "A" 4142 }, 4143 { 4144 "expr": "histogram_quantile(.9, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 4145 "hide": false, 4146 "legendFormat": ".9-{{operation}}", 4147 "refId": "B" 4148 }, 4149 { 4150 "expr": "histogram_quantile(.5, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 4151 "hide": false, 4152 "legendFormat": ".5-{{operation}}", 4153 "refId": "C" 4154 } 4155 ], 4156 "thresholds": [], 4157 "timeFrom": null, 4158 "timeRegions": [], 4159 "timeShift": null, 4160 "title": "Latency By Operation", 4161 "tooltip": { 4162 "shared": true, 4163 "sort": 2, 4164 "value_type": "individual" 4165 }, 4166 "type": "graph", 4167 "xaxis": { 4168 "buckets": null, 4169 "mode": "time", 4170 "name": null, 4171 "show": true, 4172 "values": [] 4173 }, 4174 "yaxes": [ 4175 { 4176 "format": "short", 4177 "label": null, 4178 "logBase": 1, 4179 "max": null, 4180 "min": null, 4181 "show": true 4182 }, 4183 { 4184 "format": "short", 4185 "label": null, 4186 "logBase": 1, 4187 "max": null, 4188 "min": null, 4189 "show": true 4190 } 4191 ], 4192 "yaxis": { 4193 "align": false, 4194 "alignLevel": null 4195 } 4196 }, 4197 { 4198 "aliasColors": {}, 4199 "bars": false, 4200 "dashLength": 10, 4201 "dashes": false, 4202 "datasource": "$datasource", 4203 "fill": 1, 4204 "fillGradient": 0, 4205 "gridPos": { 4206 "h": 8, 4207 "w": 24, 4208 "x": 0, 4209 "y": 39 4210 }, 4211 "hiddenSeries": false, 4212 "id": 58, 4213 "interval": "", 4214 "legend": { 4215 "alignAsTable": true, 4216 "avg": false, 4217 "current": false, 4218 "max": false, 4219 "min": false, 4220 "rightSide": true, 4221 "show": true, 4222 "total": false, 4223 "values": false 4224 }, 4225 "lines": true, 4226 "linewidth": 1, 4227 "nullPointMode": "null", 4228 "options": { 4229 "dataLinks": [] 4230 }, 4231 "percentage": false, 4232 "pointradius": 1, 4233 "points": false, 4234 "renderer": "flot", 4235 "seriesOverrides": [], 4236 "spaceLength": 10, 4237 "stack": false, 4238 "steppedLine": false, 4239 "targets": [ 4240 { 4241 "expr": "sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, status_code, method)", 4242 "intervalFactor": 1, 4243 "legendFormat": "{{status_code}}-{{operation}}", 4244 "refId": "A" 4245 } 4246 ], 4247 "thresholds": [], 4248 "timeFrom": null, 4249 "timeRegions": [], 4250 "timeShift": null, 4251 "title": "Status By Operation", 4252 "tooltip": { 4253 "shared": true, 4254 "sort": 2, 4255 "value_type": "individual" 4256 }, 4257 "type": "graph", 4258 "xaxis": { 4259 "buckets": null, 4260 "mode": "time", 4261 "name": null, 4262 "show": true, 4263 "values": [] 4264 }, 4265 "yaxes": [ 4266 { 4267 "format": "short", 4268 "label": null, 4269 "logBase": 1, 4270 "max": null, 4271 "min": null, 4272 "show": true 4273 }, 4274 { 4275 "format": "short", 4276 "label": null, 4277 "logBase": 1, 4278 "max": null, 4279 "min": null, 4280 "show": true 4281 } 4282 ], 4283 "yaxis": { 4284 "align": false, 4285 "alignLevel": null 4286 } 4287 } 4288 ], 4289 "title": "Consul", 4290 "type": "row" 4291 }, 4292 { 4293 "collapsed": true, 4294 "datasource": null, 4295 "gridPos": { 4296 "h": 1, 4297 "w": 24, 4298 "x": 0, 4299 "y": 34 4300 }, 4301 "id": 43, 4302 "panels": [ 4303 { 4304 "aliasColors": {}, 4305 "bars": false, 4306 "dashLength": 10, 4307 "dashes": false, 4308 "datasource": "$datasource", 4309 "fill": 1, 4310 "fillGradient": 0, 4311 "gridPos": { 4312 "h": 7, 4313 "w": 6, 4314 "x": 0, 4315 "y": 9 4316 }, 4317 "hiddenSeries": false, 4318 "id": 41, 4319 "interval": "", 4320 "legend": { 4321 "avg": false, 4322 "current": false, 4323 "max": false, 4324 "min": false, 4325 "show": false, 4326 "total": false, 4327 "values": false 4328 }, 4329 "lines": true, 4330 "linewidth": 1, 4331 "nullPointMode": "null", 4332 "options": { 4333 "dataLinks": [] 4334 }, 4335 "percentage": false, 4336 "pointradius": 1, 4337 "points": false, 4338 "renderer": "flot", 4339 "seriesOverrides": [], 4340 "spaceLength": 10, 4341 "stack": false, 4342 "steppedLine": false, 4343 "targets": [ 4344 { 4345 "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", 4346 "intervalFactor": 1, 4347 "legendFormat": ".9", 4348 "refId": "A" 4349 }, 4350 { 4351 "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", 4352 "refId": "B" 4353 }, 4354 { 4355 "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", 4356 "refId": "C" 4357 } 4358 ], 4359 "thresholds": [], 4360 "timeFrom": null, 4361 "timeRegions": [], 4362 "timeShift": null, 4363 "title": "MutateRows Latency", 4364 "tooltip": { 4365 "shared": true, 4366 "sort": 2, 4367 "value_type": "individual" 4368 }, 4369 "type": "graph", 4370 "xaxis": { 4371 "buckets": null, 4372 "mode": "time", 4373 "name": null, 4374 "show": true, 4375 "values": [] 4376 }, 4377 "yaxes": [ 4378 { 4379 "format": "short", 4380 "label": null, 4381 "logBase": 1, 4382 "max": null, 4383 "min": null, 4384 "show": true 4385 }, 4386 { 4387 "format": "short", 4388 "label": null, 4389 "logBase": 1, 4390 "max": null, 4391 "min": null, 4392 "show": true 4393 } 4394 ], 4395 "yaxis": { 4396 "align": false, 4397 "alignLevel": null 4398 } 4399 }, 4400 { 4401 "aliasColors": {}, 4402 "bars": false, 4403 "dashLength": 10, 4404 "dashes": false, 4405 "datasource": "$datasource", 4406 "fill": 1, 4407 "fillGradient": 0, 4408 "gridPos": { 4409 "h": 7, 4410 "w": 6, 4411 "x": 6, 4412 "y": 9 4413 }, 4414 "hiddenSeries": false, 4415 "id": 46, 4416 "interval": "", 4417 "legend": { 4418 "avg": false, 4419 "current": false, 4420 "max": false, 4421 "min": false, 4422 "show": false, 4423 "total": false, 4424 "values": false 4425 }, 4426 "lines": true, 4427 "linewidth": 1, 4428 "nullPointMode": "null", 4429 "options": { 4430 "dataLinks": [] 4431 }, 4432 "percentage": false, 4433 "pointradius": 1, 4434 "points": false, 4435 "renderer": "flot", 4436 "seriesOverrides": [], 4437 "spaceLength": 10, 4438 "stack": false, 4439 "steppedLine": false, 4440 "targets": [ 4441 { 4442 "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", 4443 "interval": "", 4444 "intervalFactor": 1, 4445 "legendFormat": "99%", 4446 "refId": "A" 4447 }, 4448 { 4449 "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", 4450 "interval": "", 4451 "legendFormat": "90%", 4452 "refId": "B" 4453 }, 4454 { 4455 "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", 4456 "interval": "", 4457 "legendFormat": "50%", 4458 "refId": "C" 4459 } 4460 ], 4461 "thresholds": [], 4462 "timeFrom": null, 4463 "timeRegions": [], 4464 "timeShift": null, 4465 "title": "ReadRows Latency", 4466 "tooltip": { 4467 "shared": true, 4468 "sort": 2, 4469 "value_type": "individual" 4470 }, 4471 "type": "graph", 4472 "xaxis": { 4473 "buckets": null, 4474 "mode": "time", 4475 "name": null, 4476 "show": true, 4477 "values": [] 4478 }, 4479 "yaxes": [ 4480 { 4481 "format": "short", 4482 "label": null, 4483 "logBase": 1, 4484 "max": null, 4485 "min": null, 4486 "show": true 4487 }, 4488 { 4489 "format": "short", 4490 "label": null, 4491 "logBase": 1, 4492 "max": null, 4493 "min": null, 4494 "show": true 4495 } 4496 ], 4497 "yaxis": { 4498 "align": false, 4499 "alignLevel": null 4500 } 4501 }, 4502 { 4503 "aliasColors": {}, 4504 "bars": false, 4505 "dashLength": 10, 4506 "dashes": false, 4507 "datasource": "$datasource", 4508 "fill": 1, 4509 "fillGradient": 0, 4510 "gridPos": { 4511 "h": 7, 4512 "w": 6, 4513 "x": 12, 4514 "y": 9 4515 }, 4516 "hiddenSeries": false, 4517 "id": 44, 4518 "interval": "", 4519 "legend": { 4520 "avg": false, 4521 "current": false, 4522 "max": false, 4523 "min": false, 4524 "show": false, 4525 "total": false, 4526 "values": false 4527 }, 4528 "lines": true, 4529 "linewidth": 1, 4530 "nullPointMode": "null", 4531 "options": { 4532 "dataLinks": [] 4533 }, 4534 "percentage": false, 4535 "pointradius": 1, 4536 "points": false, 4537 "renderer": "flot", 4538 "seriesOverrides": [], 4539 "spaceLength": 10, 4540 "stack": false, 4541 "steppedLine": false, 4542 "targets": [ 4543 { 4544 "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", 4545 "interval": "", 4546 "intervalFactor": 1, 4547 "legendFormat": "99%", 4548 "refId": "A" 4549 }, 4550 { 4551 "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", 4552 "interval": "", 4553 "legendFormat": "90%", 4554 "refId": "B" 4555 }, 4556 { 4557 "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", 4558 "interval": "", 4559 "legendFormat": "50%", 4560 "refId": "C" 4561 } 4562 ], 4563 "thresholds": [], 4564 "timeFrom": null, 4565 "timeRegions": [], 4566 "timeShift": null, 4567 "title": "GetTable Latency", 4568 "tooltip": { 4569 "shared": true, 4570 "sort": 2, 4571 "value_type": "individual" 4572 }, 4573 "type": "graph", 4574 "xaxis": { 4575 "buckets": null, 4576 "mode": "time", 4577 "name": null, 4578 "show": true, 4579 "values": [] 4580 }, 4581 "yaxes": [ 4582 { 4583 "format": "short", 4584 "label": null, 4585 "logBase": 1, 4586 "max": null, 4587 "min": null, 4588 "show": true 4589 }, 4590 { 4591 "format": "short", 4592 "label": null, 4593 "logBase": 1, 4594 "max": null, 4595 "min": null, 4596 "show": true 4597 } 4598 ], 4599 "yaxis": { 4600 "align": false, 4601 "alignLevel": null 4602 } 4603 }, 4604 { 4605 "aliasColors": {}, 4606 "bars": false, 4607 "dashLength": 10, 4608 "dashes": false, 4609 "datasource": "$datasource", 4610 "fill": 1, 4611 "fillGradient": 0, 4612 "gridPos": { 4613 "h": 7, 4614 "w": 6, 4615 "x": 18, 4616 "y": 9 4617 }, 4618 "hiddenSeries": false, 4619 "id": 45, 4620 "interval": "", 4621 "legend": { 4622 "avg": false, 4623 "current": false, 4624 "max": false, 4625 "min": false, 4626 "show": false, 4627 "total": false, 4628 "values": false 4629 }, 4630 "lines": true, 4631 "linewidth": 1, 4632 "nullPointMode": "null", 4633 "options": { 4634 "dataLinks": [] 4635 }, 4636 "percentage": false, 4637 "pointradius": 1, 4638 "points": false, 4639 "renderer": "flot", 4640 "seriesOverrides": [], 4641 "spaceLength": 10, 4642 "stack": false, 4643 "steppedLine": false, 4644 "targets": [ 4645 { 4646 "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", 4647 "intervalFactor": 1, 4648 "legendFormat": ".9", 4649 "refId": "A" 4650 }, 4651 { 4652 "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", 4653 "refId": "B" 4654 }, 4655 { 4656 "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", 4657 "refId": "C" 4658 } 4659 ], 4660 "thresholds": [], 4661 "timeFrom": null, 4662 "timeRegions": [], 4663 "timeShift": null, 4664 "title": "ListTables Latency", 4665 "tooltip": { 4666 "shared": true, 4667 "sort": 2, 4668 "value_type": "individual" 4669 }, 4670 "type": "graph", 4671 "xaxis": { 4672 "buckets": null, 4673 "mode": "time", 4674 "name": null, 4675 "show": true, 4676 "values": [] 4677 }, 4678 "yaxes": [ 4679 { 4680 "format": "short", 4681 "label": null, 4682 "logBase": 1, 4683 "max": null, 4684 "min": null, 4685 "show": true 4686 }, 4687 { 4688 "format": "short", 4689 "label": null, 4690 "logBase": 1, 4691 "max": null, 4692 "min": null, 4693 "show": true 4694 } 4695 ], 4696 "yaxis": { 4697 "align": false, 4698 "alignLevel": null 4699 } 4700 }, 4701 { 4702 "aliasColors": {}, 4703 "bars": false, 4704 "dashLength": 10, 4705 "dashes": false, 4706 "datasource": "$datasource", 4707 "fill": 1, 4708 "fillGradient": 0, 4709 "gridPos": { 4710 "h": 7, 4711 "w": 6, 4712 "x": 0, 4713 "y": 16 4714 }, 4715 "hiddenSeries": false, 4716 "id": 47, 4717 "interval": "", 4718 "legend": { 4719 "avg": false, 4720 "current": false, 4721 "max": false, 4722 "min": false, 4723 "show": false, 4724 "total": false, 4725 "values": false 4726 }, 4727 "lines": true, 4728 "linewidth": 1, 4729 "nullPointMode": "null", 4730 "options": { 4731 "dataLinks": [] 4732 }, 4733 "percentage": false, 4734 "pointradius": 1, 4735 "points": false, 4736 "renderer": "flot", 4737 "seriesOverrides": [], 4738 "spaceLength": 10, 4739 "stack": false, 4740 "steppedLine": false, 4741 "targets": [ 4742 { 4743 "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (status_code)", 4744 "intervalFactor": 1, 4745 "legendFormat": "{{status_code}}", 4746 "refId": "A" 4747 } 4748 ], 4749 "thresholds": [], 4750 "timeFrom": null, 4751 "timeRegions": [], 4752 "timeShift": null, 4753 "title": "MutateRows Status", 4754 "tooltip": { 4755 "shared": true, 4756 "sort": 2, 4757 "value_type": "individual" 4758 }, 4759 "type": "graph", 4760 "xaxis": { 4761 "buckets": null, 4762 "mode": "time", 4763 "name": null, 4764 "show": true, 4765 "values": [] 4766 }, 4767 "yaxes": [ 4768 { 4769 "format": "short", 4770 "label": null, 4771 "logBase": 1, 4772 "max": null, 4773 "min": null, 4774 "show": true 4775 }, 4776 { 4777 "format": "short", 4778 "label": null, 4779 "logBase": 1, 4780 "max": null, 4781 "min": null, 4782 "show": true 4783 } 4784 ], 4785 "yaxis": { 4786 "align": false, 4787 "alignLevel": null 4788 } 4789 }, 4790 { 4791 "aliasColors": {}, 4792 "bars": false, 4793 "dashLength": 10, 4794 "dashes": false, 4795 "datasource": "$datasource", 4796 "fill": 1, 4797 "fillGradient": 0, 4798 "gridPos": { 4799 "h": 7, 4800 "w": 6, 4801 "x": 6, 4802 "y": 16 4803 }, 4804 "hiddenSeries": false, 4805 "id": 50, 4806 "interval": "", 4807 "legend": { 4808 "avg": false, 4809 "current": false, 4810 "max": false, 4811 "min": false, 4812 "show": false, 4813 "total": false, 4814 "values": false 4815 }, 4816 "lines": true, 4817 "linewidth": 1, 4818 "nullPointMode": "null", 4819 "options": { 4820 "dataLinks": [] 4821 }, 4822 "percentage": false, 4823 "pointradius": 1, 4824 "points": false, 4825 "renderer": "flot", 4826 "seriesOverrides": [], 4827 "spaceLength": 10, 4828 "stack": false, 4829 "steppedLine": false, 4830 "targets": [ 4831 { 4832 "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (status_code)", 4833 "intervalFactor": 1, 4834 "legendFormat": "{{status_code}}", 4835 "refId": "A" 4836 } 4837 ], 4838 "thresholds": [], 4839 "timeFrom": null, 4840 "timeRegions": [], 4841 "timeShift": null, 4842 "title": "ReadRows Status", 4843 "tooltip": { 4844 "shared": true, 4845 "sort": 2, 4846 "value_type": "individual" 4847 }, 4848 "type": "graph", 4849 "xaxis": { 4850 "buckets": null, 4851 "mode": "time", 4852 "name": null, 4853 "show": true, 4854 "values": [] 4855 }, 4856 "yaxes": [ 4857 { 4858 "format": "short", 4859 "label": null, 4860 "logBase": 1, 4861 "max": null, 4862 "min": null, 4863 "show": true 4864 }, 4865 { 4866 "format": "short", 4867 "label": null, 4868 "logBase": 1, 4869 "max": null, 4870 "min": null, 4871 "show": true 4872 } 4873 ], 4874 "yaxis": { 4875 "align": false, 4876 "alignLevel": null 4877 } 4878 }, 4879 { 4880 "aliasColors": {}, 4881 "bars": false, 4882 "dashLength": 10, 4883 "dashes": false, 4884 "datasource": "$datasource", 4885 "fill": 1, 4886 "fillGradient": 0, 4887 "gridPos": { 4888 "h": 7, 4889 "w": 6, 4890 "x": 12, 4891 "y": 16 4892 }, 4893 "hiddenSeries": false, 4894 "id": 48, 4895 "interval": "", 4896 "legend": { 4897 "avg": false, 4898 "current": false, 4899 "max": false, 4900 "min": false, 4901 "show": false, 4902 "total": false, 4903 "values": false 4904 }, 4905 "lines": true, 4906 "linewidth": 1, 4907 "nullPointMode": "null", 4908 "options": { 4909 "dataLinks": [] 4910 }, 4911 "percentage": false, 4912 "pointradius": 1, 4913 "points": false, 4914 "renderer": "flot", 4915 "seriesOverrides": [], 4916 "spaceLength": 10, 4917 "stack": false, 4918 "steppedLine": false, 4919 "targets": [ 4920 { 4921 "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (status_code)", 4922 "intervalFactor": 1, 4923 "legendFormat": "{{status_code}}", 4924 "refId": "A" 4925 } 4926 ], 4927 "thresholds": [], 4928 "timeFrom": null, 4929 "timeRegions": [], 4930 "timeShift": null, 4931 "title": "GetTable Status", 4932 "tooltip": { 4933 "shared": true, 4934 "sort": 2, 4935 "value_type": "individual" 4936 }, 4937 "type": "graph", 4938 "xaxis": { 4939 "buckets": null, 4940 "mode": "time", 4941 "name": null, 4942 "show": true, 4943 "values": [] 4944 }, 4945 "yaxes": [ 4946 { 4947 "format": "short", 4948 "label": null, 4949 "logBase": 1, 4950 "max": null, 4951 "min": null, 4952 "show": true 4953 }, 4954 { 4955 "format": "short", 4956 "label": null, 4957 "logBase": 1, 4958 "max": null, 4959 "min": null, 4960 "show": true 4961 } 4962 ], 4963 "yaxis": { 4964 "align": false, 4965 "alignLevel": null 4966 } 4967 }, 4968 { 4969 "aliasColors": {}, 4970 "bars": false, 4971 "dashLength": 10, 4972 "dashes": false, 4973 "datasource": "$datasource", 4974 "fill": 1, 4975 "fillGradient": 0, 4976 "gridPos": { 4977 "h": 7, 4978 "w": 6, 4979 "x": 18, 4980 "y": 16 4981 }, 4982 "hiddenSeries": false, 4983 "id": 49, 4984 "interval": "", 4985 "legend": { 4986 "avg": false, 4987 "current": false, 4988 "max": false, 4989 "min": false, 4990 "show": false, 4991 "total": false, 4992 "values": false 4993 }, 4994 "lines": true, 4995 "linewidth": 1, 4996 "nullPointMode": "null", 4997 "options": { 4998 "dataLinks": [] 4999 }, 5000 "percentage": false, 5001 "pointradius": 1, 5002 "points": false, 5003 "renderer": "flot", 5004 "seriesOverrides": [], 5005 "spaceLength": 10, 5006 "stack": false, 5007 "steppedLine": false, 5008 "targets": [ 5009 { 5010 "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (status_code)", 5011 "intervalFactor": 1, 5012 "legendFormat": "{{status_code}}", 5013 "refId": "A" 5014 } 5015 ], 5016 "thresholds": [], 5017 "timeFrom": null, 5018 "timeRegions": [], 5019 "timeShift": null, 5020 "title": "ListTables Status", 5021 "tooltip": { 5022 "shared": true, 5023 "sort": 2, 5024 "value_type": "individual" 5025 }, 5026 "type": "graph", 5027 "xaxis": { 5028 "buckets": null, 5029 "mode": "time", 5030 "name": null, 5031 "show": true, 5032 "values": [] 5033 }, 5034 "yaxes": [ 5035 { 5036 "format": "short", 5037 "label": null, 5038 "logBase": 1, 5039 "max": null, 5040 "min": null, 5041 "show": true 5042 }, 5043 { 5044 "format": "short", 5045 "label": null, 5046 "logBase": 1, 5047 "max": null, 5048 "min": null, 5049 "show": true 5050 } 5051 ], 5052 "yaxis": { 5053 "align": false, 5054 "alignLevel": null 5055 } 5056 } 5057 ], 5058 "title": "Big Table", 5059 "type": "row" 5060 }, 5061 { 5062 "collapsed": true, 5063 "datasource": null, 5064 "gridPos": { 5065 "h": 1, 5066 "w": 24, 5067 "x": 0, 5068 "y": 35 5069 }, 5070 "id": 60, 5071 "panels": [ 5072 { 5073 "aliasColors": {}, 5074 "bars": false, 5075 "dashLength": 10, 5076 "dashes": false, 5077 "datasource": "$datasource", 5078 "fill": 1, 5079 "fillGradient": 0, 5080 "gridPos": { 5081 "h": 8, 5082 "w": 24, 5083 "x": 0, 5084 "y": 33 5085 }, 5086 "hiddenSeries": false, 5087 "id": 61, 5088 "interval": "", 5089 "legend": { 5090 "alignAsTable": true, 5091 "avg": false, 5092 "current": false, 5093 "max": false, 5094 "min": false, 5095 "rightSide": true, 5096 "show": true, 5097 "total": false, 5098 "values": false 5099 }, 5100 "lines": true, 5101 "linewidth": 1, 5102 "nullPointMode": "null", 5103 "options": { 5104 "dataLinks": [] 5105 }, 5106 "percentage": false, 5107 "pointradius": 1, 5108 "points": false, 5109 "renderer": "flot", 5110 "seriesOverrides": [], 5111 "spaceLength": 10, 5112 "stack": false, 5113 "steppedLine": false, 5114 "targets": [ 5115 { 5116 "expr": "histogram_quantile(.99, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5117 "intervalFactor": 1, 5118 "legendFormat": ".99-{{operation}}", 5119 "refId": "A" 5120 }, 5121 { 5122 "expr": "histogram_quantile(.9, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5123 "hide": false, 5124 "legendFormat": ".9-{{operation}}", 5125 "refId": "B" 5126 }, 5127 { 5128 "expr": "histogram_quantile(.5, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5129 "hide": false, 5130 "legendFormat": ".5-{{operation}}", 5131 "refId": "C" 5132 } 5133 ], 5134 "thresholds": [], 5135 "timeFrom": null, 5136 "timeRegions": [], 5137 "timeShift": null, 5138 "title": "Latency By Operation", 5139 "tooltip": { 5140 "shared": true, 5141 "sort": 2, 5142 "value_type": "individual" 5143 }, 5144 "type": "graph", 5145 "xaxis": { 5146 "buckets": null, 5147 "mode": "time", 5148 "name": null, 5149 "show": true, 5150 "values": [] 5151 }, 5152 "yaxes": [ 5153 { 5154 "format": "short", 5155 "label": null, 5156 "logBase": 1, 5157 "max": null, 5158 "min": null, 5159 "show": true 5160 }, 5161 { 5162 "format": "short", 5163 "label": null, 5164 "logBase": 1, 5165 "max": null, 5166 "min": null, 5167 "show": true 5168 } 5169 ], 5170 "yaxis": { 5171 "align": false, 5172 "alignLevel": null 5173 } 5174 }, 5175 { 5176 "aliasColors": {}, 5177 "bars": false, 5178 "dashLength": 10, 5179 "dashes": false, 5180 "datasource": "$datasource", 5181 "fill": 1, 5182 "fillGradient": 0, 5183 "gridPos": { 5184 "h": 8, 5185 "w": 24, 5186 "x": 0, 5187 "y": 41 5188 }, 5189 "hiddenSeries": false, 5190 "id": 62, 5191 "interval": "", 5192 "legend": { 5193 "alignAsTable": true, 5194 "avg": false, 5195 "current": false, 5196 "max": false, 5197 "min": false, 5198 "rightSide": true, 5199 "show": true, 5200 "total": false, 5201 "values": false 5202 }, 5203 "lines": true, 5204 "linewidth": 1, 5205 "nullPointMode": "null", 5206 "options": { 5207 "dataLinks": [] 5208 }, 5209 "percentage": false, 5210 "pointradius": 1, 5211 "points": false, 5212 "renderer": "flot", 5213 "seriesOverrides": [], 5214 "spaceLength": 10, 5215 "stack": false, 5216 "steppedLine": false, 5217 "targets": [ 5218 { 5219 "expr": "sum(rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", 5220 "intervalFactor": 1, 5221 "legendFormat": "{{status_code}}-{{operation}}", 5222 "refId": "A" 5223 } 5224 ], 5225 "thresholds": [], 5226 "timeFrom": null, 5227 "timeRegions": [], 5228 "timeShift": null, 5229 "title": "Status By Method", 5230 "tooltip": { 5231 "shared": true, 5232 "sort": 2, 5233 "value_type": "individual" 5234 }, 5235 "type": "graph", 5236 "xaxis": { 5237 "buckets": null, 5238 "mode": "time", 5239 "name": null, 5240 "show": true, 5241 "values": [] 5242 }, 5243 "yaxes": [ 5244 { 5245 "format": "short", 5246 "label": null, 5247 "logBase": 1, 5248 "max": null, 5249 "min": null, 5250 "show": true 5251 }, 5252 { 5253 "format": "short", 5254 "label": null, 5255 "logBase": 1, 5256 "max": null, 5257 "min": null, 5258 "show": true 5259 } 5260 ], 5261 "yaxis": { 5262 "align": false, 5263 "alignLevel": null 5264 } 5265 } 5266 ], 5267 "title": "GCS", 5268 "type": "row" 5269 }, 5270 { 5271 "collapsed": true, 5272 "datasource": null, 5273 "gridPos": { 5274 "h": 1, 5275 "w": 24, 5276 "x": 0, 5277 "y": 36 5278 }, 5279 "id": 76, 5280 "panels": [ 5281 { 5282 "aliasColors": {}, 5283 "bars": false, 5284 "dashLength": 10, 5285 "dashes": false, 5286 "datasource": null, 5287 "fill": 1, 5288 "fillGradient": 0, 5289 "gridPos": { 5290 "h": 6, 5291 "w": 6, 5292 "x": 0, 5293 "y": 9 5294 }, 5295 "id": 82, 5296 "legend": { 5297 "avg": false, 5298 "current": false, 5299 "max": false, 5300 "min": false, 5301 "show": true, 5302 "total": false, 5303 "values": false 5304 }, 5305 "lines": true, 5306 "linewidth": 1, 5307 "nullPointMode": "null", 5308 "options": { 5309 "dataLinks": [] 5310 }, 5311 "percentage": false, 5312 "pointradius": 2, 5313 "points": false, 5314 "renderer": "flot", 5315 "seriesOverrides": [], 5316 "spaceLength": 10, 5317 "stack": false, 5318 "steppedLine": false, 5319 "targets": [ 5320 { 5321 "expr": "sum(rate(cortex_dynamo_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", 5322 "refId": "A" 5323 } 5324 ], 5325 "thresholds": [], 5326 "timeFrom": null, 5327 "timeRegions": [], 5328 "timeShift": null, 5329 "title": "Failure Rate", 5330 "tooltip": { 5331 "shared": true, 5332 "sort": 0, 5333 "value_type": "individual" 5334 }, 5335 "type": "graph", 5336 "xaxis": { 5337 "buckets": null, 5338 "mode": "time", 5339 "name": null, 5340 "show": true, 5341 "values": [] 5342 }, 5343 "yaxes": [ 5344 { 5345 "format": "short", 5346 "label": null, 5347 "logBase": 1, 5348 "max": null, 5349 "min": null, 5350 "show": true 5351 }, 5352 { 5353 "format": "short", 5354 "label": null, 5355 "logBase": 1, 5356 "max": null, 5357 "min": null, 5358 "show": true 5359 } 5360 ], 5361 "yaxis": { 5362 "align": false, 5363 "alignLevel": null 5364 } 5365 }, 5366 { 5367 "aliasColors": {}, 5368 "bars": false, 5369 "dashLength": 10, 5370 "dashes": false, 5371 "datasource": null, 5372 "fill": 1, 5373 "fillGradient": 0, 5374 "gridPos": { 5375 "h": 6, 5376 "w": 6, 5377 "x": 6, 5378 "y": 9 5379 }, 5380 "id": 83, 5381 "legend": { 5382 "avg": false, 5383 "current": false, 5384 "max": false, 5385 "min": false, 5386 "show": true, 5387 "total": false, 5388 "values": false 5389 }, 5390 "lines": true, 5391 "linewidth": 1, 5392 "nullPointMode": "null", 5393 "options": { 5394 "dataLinks": [] 5395 }, 5396 "percentage": false, 5397 "pointradius": 2, 5398 "points": false, 5399 "renderer": "flot", 5400 "seriesOverrides": [], 5401 "spaceLength": 10, 5402 "stack": false, 5403 "steppedLine": false, 5404 "targets": [ 5405 { 5406 "expr": "sum(rate(cortex_dynamo_consumed_capacity_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", 5407 "refId": "A" 5408 } 5409 ], 5410 "thresholds": [], 5411 "timeFrom": null, 5412 "timeRegions": [], 5413 "timeShift": null, 5414 "title": "Consumed Capacity Rate", 5415 "tooltip": { 5416 "shared": true, 5417 "sort": 0, 5418 "value_type": "individual" 5419 }, 5420 "type": "graph", 5421 "xaxis": { 5422 "buckets": null, 5423 "mode": "time", 5424 "name": null, 5425 "show": true, 5426 "values": [] 5427 }, 5428 "yaxes": [ 5429 { 5430 "format": "short", 5431 "label": null, 5432 "logBase": 1, 5433 "max": null, 5434 "min": null, 5435 "show": true 5436 }, 5437 { 5438 "format": "short", 5439 "label": null, 5440 "logBase": 1, 5441 "max": null, 5442 "min": null, 5443 "show": true 5444 } 5445 ], 5446 "yaxis": { 5447 "align": false, 5448 "alignLevel": null 5449 } 5450 }, 5451 { 5452 "aliasColors": {}, 5453 "bars": false, 5454 "dashLength": 10, 5455 "dashes": false, 5456 "datasource": null, 5457 "fill": 1, 5458 "fillGradient": 0, 5459 "gridPos": { 5460 "h": 6, 5461 "w": 6, 5462 "x": 12, 5463 "y": 9 5464 }, 5465 "id": 84, 5466 "legend": { 5467 "avg": false, 5468 "current": false, 5469 "max": false, 5470 "min": false, 5471 "show": true, 5472 "total": false, 5473 "values": false 5474 }, 5475 "lines": true, 5476 "linewidth": 1, 5477 "nullPointMode": "null", 5478 "options": { 5479 "dataLinks": [] 5480 }, 5481 "percentage": false, 5482 "pointradius": 2, 5483 "points": false, 5484 "renderer": "flot", 5485 "seriesOverrides": [], 5486 "spaceLength": 10, 5487 "stack": false, 5488 "steppedLine": false, 5489 "targets": [ 5490 { 5491 "expr": "sum(rate(cortex_dynamo_throttled_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", 5492 "refId": "A" 5493 } 5494 ], 5495 "thresholds": [], 5496 "timeFrom": null, 5497 "timeRegions": [], 5498 "timeShift": null, 5499 "title": "Throttled Rate", 5500 "tooltip": { 5501 "shared": true, 5502 "sort": 0, 5503 "value_type": "individual" 5504 }, 5505 "type": "graph", 5506 "xaxis": { 5507 "buckets": null, 5508 "mode": "time", 5509 "name": null, 5510 "show": true, 5511 "values": [] 5512 }, 5513 "yaxes": [ 5514 { 5515 "format": "short", 5516 "label": null, 5517 "logBase": 1, 5518 "max": null, 5519 "min": null, 5520 "show": true 5521 }, 5522 { 5523 "format": "short", 5524 "label": null, 5525 "logBase": 1, 5526 "max": null, 5527 "min": null, 5528 "show": true 5529 } 5530 ], 5531 "yaxis": { 5532 "align": false, 5533 "alignLevel": null 5534 } 5535 }, 5536 { 5537 "aliasColors": {}, 5538 "bars": false, 5539 "dashLength": 10, 5540 "dashes": false, 5541 "datasource": null, 5542 "fill": 1, 5543 "fillGradient": 0, 5544 "gridPos": { 5545 "h": 6, 5546 "w": 6, 5547 "x": 18, 5548 "y": 9 5549 }, 5550 "id": 85, 5551 "legend": { 5552 "avg": false, 5553 "current": false, 5554 "max": false, 5555 "min": false, 5556 "show": true, 5557 "total": false, 5558 "values": false 5559 }, 5560 "lines": true, 5561 "linewidth": 1, 5562 "nullPointMode": "null", 5563 "options": { 5564 "dataLinks": [] 5565 }, 5566 "percentage": false, 5567 "pointradius": 2, 5568 "points": false, 5569 "renderer": "flot", 5570 "seriesOverrides": [], 5571 "spaceLength": 10, 5572 "stack": false, 5573 "steppedLine": false, 5574 "targets": [ 5575 { 5576 "expr": "sum(rate(cortex_dynamo_dropped_requests_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", 5577 "refId": "A" 5578 } 5579 ], 5580 "thresholds": [], 5581 "timeFrom": null, 5582 "timeRegions": [], 5583 "timeShift": null, 5584 "title": "Dropped Rate", 5585 "tooltip": { 5586 "shared": true, 5587 "sort": 0, 5588 "value_type": "individual" 5589 }, 5590 "type": "graph", 5591 "xaxis": { 5592 "buckets": null, 5593 "mode": "time", 5594 "name": null, 5595 "show": true, 5596 "values": [] 5597 }, 5598 "yaxes": [ 5599 { 5600 "format": "short", 5601 "label": null, 5602 "logBase": 1, 5603 "max": null, 5604 "min": null, 5605 "show": true 5606 }, 5607 { 5608 "format": "short", 5609 "label": null, 5610 "logBase": 1, 5611 "max": null, 5612 "min": null, 5613 "show": true 5614 } 5615 ], 5616 "yaxis": { 5617 "align": false, 5618 "alignLevel": null 5619 } 5620 }, 5621 { 5622 "aliasColors": {}, 5623 "bars": false, 5624 "dashLength": 10, 5625 "dashes": false, 5626 "datasource": null, 5627 "fill": 1, 5628 "fillGradient": 0, 5629 "gridPos": { 5630 "h": 6, 5631 "w": 6, 5632 "x": 0, 5633 "y": 15 5634 }, 5635 "id": 86, 5636 "legend": { 5637 "avg": false, 5638 "current": false, 5639 "max": false, 5640 "min": false, 5641 "show": true, 5642 "total": false, 5643 "values": false 5644 }, 5645 "lines": true, 5646 "linewidth": 1, 5647 "nullPointMode": "null", 5648 "options": { 5649 "dataLinks": [] 5650 }, 5651 "percentage": false, 5652 "pointradius": 2, 5653 "points": false, 5654 "renderer": "flot", 5655 "seriesOverrides": [], 5656 "spaceLength": 10, 5657 "stack": false, 5658 "steppedLine": false, 5659 "targets": [ 5660 { 5661 "expr": "histogram_quantile(.99, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", 5662 "legendFormat": ".99", 5663 "refId": "A" 5664 }, 5665 { 5666 "expr": "histogram_quantile(.9, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", 5667 "legendFormat": ".9", 5668 "refId": "B" 5669 }, 5670 { 5671 "expr": "histogram_quantile(.5, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", 5672 "legendFormat": ".5", 5673 "refId": "C" 5674 } 5675 ], 5676 "thresholds": [], 5677 "timeFrom": null, 5678 "timeRegions": [], 5679 "timeShift": null, 5680 "title": "Query Pages", 5681 "tooltip": { 5682 "shared": true, 5683 "sort": 0, 5684 "value_type": "individual" 5685 }, 5686 "type": "graph", 5687 "xaxis": { 5688 "buckets": null, 5689 "mode": "time", 5690 "name": null, 5691 "show": true, 5692 "values": [] 5693 }, 5694 "yaxes": [ 5695 { 5696 "format": "short", 5697 "label": null, 5698 "logBase": 1, 5699 "max": null, 5700 "min": null, 5701 "show": true 5702 }, 5703 { 5704 "format": "short", 5705 "label": null, 5706 "logBase": 1, 5707 "max": null, 5708 "min": null, 5709 "show": true 5710 } 5711 ], 5712 "yaxis": { 5713 "align": false, 5714 "alignLevel": null 5715 } 5716 }, 5717 { 5718 "aliasColors": {}, 5719 "bars": false, 5720 "dashLength": 10, 5721 "dashes": false, 5722 "datasource": "$datasource", 5723 "fill": 1, 5724 "fillGradient": 0, 5725 "gridPos": { 5726 "h": 6, 5727 "w": 9, 5728 "x": 6, 5729 "y": 15 5730 }, 5731 "id": 87, 5732 "interval": "", 5733 "legend": { 5734 "alignAsTable": true, 5735 "avg": false, 5736 "current": false, 5737 "max": false, 5738 "min": false, 5739 "rightSide": true, 5740 "show": true, 5741 "total": false, 5742 "values": false 5743 }, 5744 "lines": true, 5745 "linewidth": 1, 5746 "nullPointMode": "null", 5747 "options": { 5748 "dataLinks": [] 5749 }, 5750 "percentage": false, 5751 "pointradius": 1, 5752 "points": false, 5753 "renderer": "flot", 5754 "seriesOverrides": [], 5755 "spaceLength": 10, 5756 "stack": false, 5757 "steppedLine": false, 5758 "targets": [ 5759 { 5760 "expr": "histogram_quantile(.99, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5761 "intervalFactor": 1, 5762 "legendFormat": ".99-{{operation}}", 5763 "refId": "A" 5764 }, 5765 { 5766 "expr": "histogram_quantile(.9, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5767 "hide": false, 5768 "legendFormat": ".9-{{operation}}", 5769 "refId": "B" 5770 }, 5771 { 5772 "expr": "histogram_quantile(.5, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5773 "hide": false, 5774 "legendFormat": ".5-{{operation}}", 5775 "refId": "C" 5776 } 5777 ], 5778 "thresholds": [], 5779 "timeFrom": null, 5780 "timeRegions": [], 5781 "timeShift": null, 5782 "title": "Latency By Operation", 5783 "tooltip": { 5784 "shared": true, 5785 "sort": 2, 5786 "value_type": "individual" 5787 }, 5788 "type": "graph", 5789 "xaxis": { 5790 "buckets": null, 5791 "mode": "time", 5792 "name": null, 5793 "show": true, 5794 "values": [] 5795 }, 5796 "yaxes": [ 5797 { 5798 "format": "short", 5799 "label": null, 5800 "logBase": 1, 5801 "max": null, 5802 "min": null, 5803 "show": true 5804 }, 5805 { 5806 "format": "short", 5807 "label": null, 5808 "logBase": 1, 5809 "max": null, 5810 "min": null, 5811 "show": true 5812 } 5813 ], 5814 "yaxis": { 5815 "align": false, 5816 "alignLevel": null 5817 } 5818 }, 5819 { 5820 "aliasColors": {}, 5821 "bars": false, 5822 "dashLength": 10, 5823 "dashes": false, 5824 "datasource": "$datasource", 5825 "fill": 1, 5826 "fillGradient": 0, 5827 "gridPos": { 5828 "h": 6, 5829 "w": 9, 5830 "x": 15, 5831 "y": 15 5832 }, 5833 "id": 88, 5834 "interval": "", 5835 "legend": { 5836 "alignAsTable": true, 5837 "avg": false, 5838 "current": false, 5839 "max": false, 5840 "min": false, 5841 "rightSide": true, 5842 "show": true, 5843 "total": false, 5844 "values": false 5845 }, 5846 "lines": true, 5847 "linewidth": 1, 5848 "nullPointMode": "null", 5849 "options": { 5850 "dataLinks": [] 5851 }, 5852 "percentage": false, 5853 "pointradius": 1, 5854 "points": false, 5855 "renderer": "flot", 5856 "seriesOverrides": [], 5857 "spaceLength": 10, 5858 "stack": false, 5859 "steppedLine": false, 5860 "targets": [ 5861 { 5862 "expr": "sum(rate(cortex_dynamo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", 5863 "intervalFactor": 1, 5864 "legendFormat": "{{status_code}}-{{operation}}", 5865 "refId": "A" 5866 } 5867 ], 5868 "thresholds": [], 5869 "timeFrom": null, 5870 "timeRegions": [], 5871 "timeShift": null, 5872 "title": "Status By Method", 5873 "tooltip": { 5874 "shared": true, 5875 "sort": 2, 5876 "value_type": "individual" 5877 }, 5878 "type": "graph", 5879 "xaxis": { 5880 "buckets": null, 5881 "mode": "time", 5882 "name": null, 5883 "show": true, 5884 "values": [] 5885 }, 5886 "yaxes": [ 5887 { 5888 "format": "short", 5889 "label": null, 5890 "logBase": 1, 5891 "max": null, 5892 "min": null, 5893 "show": true 5894 }, 5895 { 5896 "format": "short", 5897 "label": null, 5898 "logBase": 1, 5899 "max": null, 5900 "min": null, 5901 "show": true 5902 } 5903 ], 5904 "yaxis": { 5905 "align": false, 5906 "alignLevel": null 5907 } 5908 } 5909 ], 5910 "title": "Dynamo", 5911 "type": "row" 5912 }, 5913 { 5914 "collapsed": true, 5915 "datasource": null, 5916 "gridPos": { 5917 "h": 1, 5918 "w": 24, 5919 "x": 0, 5920 "y": 37 5921 }, 5922 "id": 78, 5923 "panels": [ 5924 { 5925 "aliasColors": {}, 5926 "bars": false, 5927 "dashLength": 10, 5928 "dashes": false, 5929 "datasource": "$datasource", 5930 "fill": 1, 5931 "fillGradient": 0, 5932 "gridPos": { 5933 "h": 8, 5934 "w": 24, 5935 "x": 0, 5936 "y": 10 5937 }, 5938 "id": 79, 5939 "interval": "", 5940 "legend": { 5941 "alignAsTable": true, 5942 "avg": false, 5943 "current": false, 5944 "max": false, 5945 "min": false, 5946 "rightSide": true, 5947 "show": true, 5948 "total": false, 5949 "values": false 5950 }, 5951 "lines": true, 5952 "linewidth": 1, 5953 "nullPointMode": "null", 5954 "options": { 5955 "dataLinks": [] 5956 }, 5957 "percentage": false, 5958 "pointradius": 1, 5959 "points": false, 5960 "renderer": "flot", 5961 "seriesOverrides": [], 5962 "spaceLength": 10, 5963 "stack": false, 5964 "steppedLine": false, 5965 "targets": [ 5966 { 5967 "expr": "histogram_quantile(.99, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5968 "intervalFactor": 1, 5969 "legendFormat": ".99-{{operation}}", 5970 "refId": "A" 5971 }, 5972 { 5973 "expr": "histogram_quantile(.9, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5974 "hide": false, 5975 "legendFormat": ".9-{{operation}}", 5976 "refId": "B" 5977 }, 5978 { 5979 "expr": "histogram_quantile(.5, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 5980 "hide": false, 5981 "legendFormat": ".5-{{operation}}", 5982 "refId": "C" 5983 } 5984 ], 5985 "thresholds": [], 5986 "timeFrom": null, 5987 "timeRegions": [], 5988 "timeShift": null, 5989 "title": "Latency By Operation", 5990 "tooltip": { 5991 "shared": true, 5992 "sort": 2, 5993 "value_type": "individual" 5994 }, 5995 "type": "graph", 5996 "xaxis": { 5997 "buckets": null, 5998 "mode": "time", 5999 "name": null, 6000 "show": true, 6001 "values": [] 6002 }, 6003 "yaxes": [ 6004 { 6005 "format": "short", 6006 "label": null, 6007 "logBase": 1, 6008 "max": null, 6009 "min": null, 6010 "show": true 6011 }, 6012 { 6013 "format": "short", 6014 "label": null, 6015 "logBase": 1, 6016 "max": null, 6017 "min": null, 6018 "show": true 6019 } 6020 ], 6021 "yaxis": { 6022 "align": false, 6023 "alignLevel": null 6024 } 6025 }, 6026 { 6027 "aliasColors": {}, 6028 "bars": false, 6029 "dashLength": 10, 6030 "dashes": false, 6031 "datasource": "$datasource", 6032 "fill": 1, 6033 "fillGradient": 0, 6034 "gridPos": { 6035 "h": 8, 6036 "w": 24, 6037 "x": 0, 6038 "y": 18 6039 }, 6040 "id": 80, 6041 "interval": "", 6042 "legend": { 6043 "alignAsTable": true, 6044 "avg": false, 6045 "current": false, 6046 "max": false, 6047 "min": false, 6048 "rightSide": true, 6049 "show": true, 6050 "total": false, 6051 "values": false 6052 }, 6053 "lines": true, 6054 "linewidth": 1, 6055 "nullPointMode": "null", 6056 "options": { 6057 "dataLinks": [] 6058 }, 6059 "percentage": false, 6060 "pointradius": 1, 6061 "points": false, 6062 "renderer": "flot", 6063 "seriesOverrides": [], 6064 "spaceLength": 10, 6065 "stack": false, 6066 "steppedLine": false, 6067 "targets": [ 6068 { 6069 "expr": "sum(rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", 6070 "intervalFactor": 1, 6071 "legendFormat": "{{status_code}}-{{operation}}", 6072 "refId": "A" 6073 } 6074 ], 6075 "thresholds": [], 6076 "timeFrom": null, 6077 "timeRegions": [], 6078 "timeShift": null, 6079 "title": "Status By Method", 6080 "tooltip": { 6081 "shared": true, 6082 "sort": 2, 6083 "value_type": "individual" 6084 }, 6085 "type": "graph", 6086 "xaxis": { 6087 "buckets": null, 6088 "mode": "time", 6089 "name": null, 6090 "show": true, 6091 "values": [] 6092 }, 6093 "yaxes": [ 6094 { 6095 "format": "short", 6096 "label": null, 6097 "logBase": 1, 6098 "max": null, 6099 "min": null, 6100 "show": true 6101 }, 6102 { 6103 "format": "short", 6104 "label": null, 6105 "logBase": 1, 6106 "max": null, 6107 "min": null, 6108 "show": true 6109 } 6110 ], 6111 "yaxis": { 6112 "align": false, 6113 "alignLevel": null 6114 } 6115 } 6116 ], 6117 "title": "S3", 6118 "type": "row" 6119 }, 6120 { 6121 "collapsed": true, 6122 "datasource": null, 6123 "gridPos": { 6124 "h": 1, 6125 "w": 24, 6126 "x": 0, 6127 "y": 37 6128 }, 6129 "id": 78, 6130 "panels": [ 6131 { 6132 "aliasColors": {}, 6133 "bars": false, 6134 "dashLength": 10, 6135 "dashes": false, 6136 "datasource": "$datasource", 6137 "fill": 1, 6138 "fillGradient": 0, 6139 "gridPos": { 6140 "h": 8, 6141 "w": 24, 6142 "x": 0, 6143 "y": 10 6144 }, 6145 "id": 79, 6146 "interval": "", 6147 "legend": { 6148 "alignAsTable": true, 6149 "avg": false, 6150 "current": false, 6151 "max": false, 6152 "min": false, 6153 "rightSide": true, 6154 "show": true, 6155 "total": false, 6156 "values": false 6157 }, 6158 "lines": true, 6159 "linewidth": 1, 6160 "nullPointMode": "null", 6161 "options": { 6162 "dataLinks": [] 6163 }, 6164 "percentage": false, 6165 "pointradius": 1, 6166 "points": false, 6167 "renderer": "flot", 6168 "seriesOverrides": [], 6169 "spaceLength": 10, 6170 "stack": false, 6171 "steppedLine": false, 6172 "targets": [ 6173 { 6174 "expr": "histogram_quantile(.99, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 6175 "intervalFactor": 1, 6176 "legendFormat": ".99-{{operation}}", 6177 "refId": "A" 6178 }, 6179 { 6180 "expr": "histogram_quantile(.9, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 6181 "hide": false, 6182 "legendFormat": ".9-{{operation}}", 6183 "refId": "B" 6184 }, 6185 { 6186 "expr": "histogram_quantile(.5, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 6187 "hide": false, 6188 "legendFormat": ".5-{{operation}}", 6189 "refId": "C" 6190 } 6191 ], 6192 "thresholds": [], 6193 "timeFrom": null, 6194 "timeRegions": [], 6195 "timeShift": null, 6196 "title": "Latency By Operation", 6197 "tooltip": { 6198 "shared": true, 6199 "sort": 2, 6200 "value_type": "individual" 6201 }, 6202 "type": "graph", 6203 "xaxis": { 6204 "buckets": null, 6205 "mode": "time", 6206 "name": null, 6207 "show": true, 6208 "values": [] 6209 }, 6210 "yaxes": [ 6211 { 6212 "format": "short", 6213 "label": null, 6214 "logBase": 1, 6215 "max": null, 6216 "min": null, 6217 "show": true 6218 }, 6219 { 6220 "format": "short", 6221 "label": null, 6222 "logBase": 1, 6223 "max": null, 6224 "min": null, 6225 "show": true 6226 } 6227 ], 6228 "yaxis": { 6229 "align": false, 6230 "alignLevel": null 6231 } 6232 }, 6233 { 6234 "aliasColors": {}, 6235 "bars": false, 6236 "dashLength": 10, 6237 "dashes": false, 6238 "datasource": "$datasource", 6239 "fill": 1, 6240 "fillGradient": 0, 6241 "gridPos": { 6242 "h": 8, 6243 "w": 24, 6244 "x": 0, 6245 "y": 18 6246 }, 6247 "id": 80, 6248 "interval": "", 6249 "legend": { 6250 "alignAsTable": true, 6251 "avg": false, 6252 "current": false, 6253 "max": false, 6254 "min": false, 6255 "rightSide": true, 6256 "show": true, 6257 "total": false, 6258 "values": false 6259 }, 6260 "lines": true, 6261 "linewidth": 1, 6262 "nullPointMode": "null", 6263 "options": { 6264 "dataLinks": [] 6265 }, 6266 "percentage": false, 6267 "pointradius": 1, 6268 "points": false, 6269 "renderer": "flot", 6270 "seriesOverrides": [], 6271 "spaceLength": 10, 6272 "stack": false, 6273 "steppedLine": false, 6274 "targets": [ 6275 { 6276 "expr": "sum(rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", 6277 "intervalFactor": 1, 6278 "legendFormat": "{{status_code}}-{{operation}}", 6279 "refId": "A" 6280 } 6281 ], 6282 "thresholds": [], 6283 "timeFrom": null, 6284 "timeRegions": [], 6285 "timeShift": null, 6286 "title": "Status By Method", 6287 "tooltip": { 6288 "shared": true, 6289 "sort": 2, 6290 "value_type": "individual" 6291 }, 6292 "type": "graph", 6293 "xaxis": { 6294 "buckets": null, 6295 "mode": "time", 6296 "name": null, 6297 "show": true, 6298 "values": [] 6299 }, 6300 "yaxes": [ 6301 { 6302 "format": "short", 6303 "label": null, 6304 "logBase": 1, 6305 "max": null, 6306 "min": null, 6307 "show": true 6308 }, 6309 { 6310 "format": "short", 6311 "label": null, 6312 "logBase": 1, 6313 "max": null, 6314 "min": null, 6315 "show": true 6316 } 6317 ], 6318 "yaxis": { 6319 "align": false, 6320 "alignLevel": null 6321 } 6322 } 6323 ], 6324 "title": "Azure Blob", 6325 "type": "row" 6326 }, 6327 { 6328 "collapsed": true, 6329 "datasource": null, 6330 "gridPos": { 6331 "h": 1, 6332 "w": 24, 6333 "x": 0, 6334 "y": 38 6335 }, 6336 "id": 90, 6337 "panels": [ 6338 { 6339 "aliasColors": {}, 6340 "bars": false, 6341 "dashLength": 10, 6342 "dashes": false, 6343 "datasource": "$datasource", 6344 "fill": 1, 6345 "fillGradient": 0, 6346 "gridPos": { 6347 "h": 8, 6348 "w": 24, 6349 "x": 0, 6350 "y": 36 6351 }, 6352 "id": 91, 6353 "interval": "", 6354 "legend": { 6355 "alignAsTable": true, 6356 "avg": false, 6357 "current": false, 6358 "max": false, 6359 "min": false, 6360 "rightSide": true, 6361 "show": true, 6362 "total": false, 6363 "values": false 6364 }, 6365 "lines": true, 6366 "linewidth": 1, 6367 "nullPointMode": "null", 6368 "options": { 6369 "dataLinks": [] 6370 }, 6371 "percentage": false, 6372 "pointradius": 1, 6373 "points": false, 6374 "renderer": "flot", 6375 "seriesOverrides": [], 6376 "spaceLength": 10, 6377 "stack": false, 6378 "steppedLine": false, 6379 "targets": [ 6380 { 6381 "expr": "histogram_quantile(.99, sum(rate(cortex_cassandra_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 6382 "intervalFactor": 1, 6383 "legendFormat": ".99-{{operation}}", 6384 "refId": "A" 6385 }, 6386 { 6387 "expr": "histogram_quantile(.9, sum(rate(cortex_cassandra_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 6388 "hide": false, 6389 "legendFormat": ".9-{{operation}}", 6390 "refId": "B" 6391 }, 6392 { 6393 "expr": "histogram_quantile(.5, sum(rate(cortex_cassandra_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 6394 "hide": false, 6395 "legendFormat": ".5-{{operation}}", 6396 "refId": "C" 6397 } 6398 ], 6399 "thresholds": [], 6400 "timeFrom": null, 6401 "timeRegions": [], 6402 "timeShift": null, 6403 "title": "Latency By Operation", 6404 "tooltip": { 6405 "shared": true, 6406 "sort": 2, 6407 "value_type": "individual" 6408 }, 6409 "type": "graph", 6410 "xaxis": { 6411 "buckets": null, 6412 "mode": "time", 6413 "name": null, 6414 "show": true, 6415 "values": [] 6416 }, 6417 "yaxes": [ 6418 { 6419 "format": "short", 6420 "label": null, 6421 "logBase": 1, 6422 "max": null, 6423 "min": null, 6424 "show": true 6425 }, 6426 { 6427 "format": "short", 6428 "label": null, 6429 "logBase": 1, 6430 "max": null, 6431 "min": null, 6432 "show": true 6433 } 6434 ], 6435 "yaxis": { 6436 "align": false, 6437 "alignLevel": null 6438 } 6439 }, 6440 { 6441 "aliasColors": {}, 6442 "bars": false, 6443 "dashLength": 10, 6444 "dashes": false, 6445 "datasource": "$datasource", 6446 "fill": 1, 6447 "fillGradient": 0, 6448 "gridPos": { 6449 "h": 8, 6450 "w": 24, 6451 "x": 0, 6452 "y": 44 6453 }, 6454 "id": 92, 6455 "interval": "", 6456 "legend": { 6457 "alignAsTable": true, 6458 "avg": false, 6459 "current": false, 6460 "max": false, 6461 "min": false, 6462 "rightSide": true, 6463 "show": true, 6464 "total": false, 6465 "values": false 6466 }, 6467 "lines": true, 6468 "linewidth": 1, 6469 "nullPointMode": "null", 6470 "options": { 6471 "dataLinks": [] 6472 }, 6473 "percentage": false, 6474 "pointradius": 1, 6475 "points": false, 6476 "renderer": "flot", 6477 "seriesOverrides": [], 6478 "spaceLength": 10, 6479 "stack": false, 6480 "steppedLine": false, 6481 "targets": [ 6482 { 6483 "expr": "sum(rate(cortex_cassandra_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", 6484 "intervalFactor": 1, 6485 "legendFormat": "{{status_code}}-{{operation}}", 6486 "refId": "A" 6487 } 6488 ], 6489 "thresholds": [], 6490 "timeFrom": null, 6491 "timeRegions": [], 6492 "timeShift": null, 6493 "title": "Status By Method", 6494 "tooltip": { 6495 "shared": true, 6496 "sort": 2, 6497 "value_type": "individual" 6498 }, 6499 "type": "graph", 6500 "xaxis": { 6501 "buckets": null, 6502 "mode": "time", 6503 "name": null, 6504 "show": true, 6505 "values": [] 6506 }, 6507 "yaxes": [ 6508 { 6509 "format": "short", 6510 "label": null, 6511 "logBase": 1, 6512 "max": null, 6513 "min": null, 6514 "show": true 6515 }, 6516 { 6517 "format": "short", 6518 "label": null, 6519 "logBase": 1, 6520 "max": null, 6521 "min": null, 6522 "show": true 6523 } 6524 ], 6525 "yaxis": { 6526 "align": false, 6527 "alignLevel": null 6528 } 6529 } 6530 ], 6531 "title": "Cassandra", 6532 "type": "row" 6533 }, 6534 { 6535 "collapsed": true, 6536 "datasource": null, 6537 "gridPos": { 6538 "h": 1, 6539 "w": 24, 6540 "x": 0, 6541 "y": 37 6542 }, 6543 "id": 114, 6544 "panels": [ 6545 { 6546 "aliasColors": {}, 6547 "bars": false, 6548 "dashLength": 10, 6549 "dashes": false, 6550 "datasource": "$datasource", 6551 "fill": 1, 6552 "fillGradient": 0, 6553 "gridPos": { 6554 "h": 8, 6555 "w": 24, 6556 "x": 0, 6557 "y": 10 6558 }, 6559 "id": 115, 6560 "interval": "", 6561 "legend": { 6562 "alignAsTable": true, 6563 "avg": false, 6564 "current": false, 6565 "max": false, 6566 "min": false, 6567 "rightSide": true, 6568 "show": true, 6569 "total": false, 6570 "values": false 6571 }, 6572 "lines": true, 6573 "linewidth": 1, 6574 "nullPointMode": "null", 6575 "options": { 6576 "dataLinks": [] 6577 }, 6578 "percentage": false, 6579 "pointradius": 1, 6580 "points": false, 6581 "renderer": "flot", 6582 "seriesOverrides": [], 6583 "spaceLength": 10, 6584 "stack": false, 6585 "steppedLine": false, 6586 "targets": [ 6587 { 6588 "expr": "histogram_quantile(.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 6589 "intervalFactor": 1, 6590 "legendFormat": ".99-{{operation}}", 6591 "refId": "A" 6592 }, 6593 { 6594 "expr": "histogram_quantile(.9, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 6595 "hide": false, 6596 "legendFormat": ".9-{{operation}}", 6597 "refId": "B" 6598 }, 6599 { 6600 "expr": "histogram_quantile(.5, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", 6601 "hide": false, 6602 "legendFormat": ".5-{{operation}}", 6603 "refId": "C" 6604 } 6605 ], 6606 "thresholds": [], 6607 "timeFrom": null, 6608 "timeRegions": [], 6609 "timeShift": null, 6610 "title": "Latency By Operation", 6611 "tooltip": { 6612 "shared": true, 6613 "sort": 2, 6614 "value_type": "individual" 6615 }, 6616 "type": "graph", 6617 "xaxis": { 6618 "buckets": null, 6619 "mode": "time", 6620 "name": null, 6621 "show": true, 6622 "values": [] 6623 }, 6624 "yaxes": [ 6625 { 6626 "format": "short", 6627 "label": null, 6628 "logBase": 1, 6629 "max": null, 6630 "min": null, 6631 "show": true 6632 }, 6633 { 6634 "format": "short", 6635 "label": null, 6636 "logBase": 1, 6637 "max": null, 6638 "min": null, 6639 "show": true 6640 } 6641 ], 6642 "yaxis": { 6643 "align": false, 6644 "alignLevel": null 6645 } 6646 }, 6647 { 6648 "aliasColors": {}, 6649 "bars": false, 6650 "dashLength": 10, 6651 "dashes": false, 6652 "datasource": "$datasource", 6653 "fill": 1, 6654 "fillGradient": 0, 6655 "gridPos": { 6656 "h": 8, 6657 "w": 24, 6658 "x": 0, 6659 "y": 18 6660 }, 6661 "id": 116, 6662 "interval": "", 6663 "legend": { 6664 "alignAsTable": true, 6665 "avg": false, 6666 "current": false, 6667 "max": false, 6668 "min": false, 6669 "rightSide": true, 6670 "show": true, 6671 "total": false, 6672 "values": false 6673 }, 6674 "lines": true, 6675 "linewidth": 1, 6676 "nullPointMode": "null", 6677 "options": { 6678 "dataLinks": [] 6679 }, 6680 "percentage": false, 6681 "pointradius": 1, 6682 "points": false, 6683 "renderer": "flot", 6684 "seriesOverrides": [], 6685 "spaceLength": 10, 6686 "stack": false, 6687 "steppedLine": false, 6688 "targets": [ 6689 { 6690 "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", 6691 "intervalFactor": 1, 6692 "legendFormat": "{{status_code}}-{{operation}}", 6693 "refId": "A" 6694 } 6695 ], 6696 "thresholds": [], 6697 "timeFrom": null, 6698 "timeRegions": [], 6699 "timeShift": null, 6700 "title": "Status By Method", 6701 "tooltip": { 6702 "shared": true, 6703 "sort": 2, 6704 "value_type": "individual" 6705 }, 6706 "type": "graph", 6707 "xaxis": { 6708 "buckets": null, 6709 "mode": "time", 6710 "name": null, 6711 "show": true, 6712 "values": [] 6713 }, 6714 "yaxes": [ 6715 { 6716 "format": "short", 6717 "label": null, 6718 "logBase": 1, 6719 "max": null, 6720 "min": null, 6721 "show": true 6722 }, 6723 { 6724 "format": "short", 6725 "label": null, 6726 "logBase": 1, 6727 "max": null, 6728 "min": null, 6729 "show": true 6730 } 6731 ], 6732 "yaxis": { 6733 "align": false, 6734 "alignLevel": null 6735 } 6736 } 6737 ], 6738 "title": "BoltDB Shipper", 6739 "type": "row" 6740 } 6741 ], 6742 "refresh": "30s", 6743 "schemaVersion": 25, 6744 "style": "dark", 6745 "tags": [], 6746 "time": { 6747 "from": "now-3h", 6748 "to": "now" 6749 }, 6750 "timepicker": { 6751 "refresh_intervals": [ 6752 "10s", 6753 "30s", 6754 "1m", 6755 "5m", 6756 "15m", 6757 "30m", 6758 "1h", 6759 "2h", 6760 "1d" 6761 ] 6762 }, 6763 "timezone": "", 6764 "title": "Loki Operational", 6765 "uid": "f6fe30815b172c9da7e813c15ddfe607", 6766 "version": 1 6767 }