github.com/thanos-io/thanos@v0.32.5/examples/dashboards/receive.json (about) 1 { 2 "annotations": { 3 "list": [ ] 4 }, 5 "editable": true, 6 "gnetId": null, 7 "graphTooltip": 0, 8 "hideControls": false, 9 "links": [ ], 10 "refresh": "10s", 11 "rows": [ 12 { 13 "collapse": false, 14 "height": "250px", 15 "panels": [ 16 { 17 "aliasColors": { }, 18 "bars": false, 19 "dashLength": 10, 20 "dashes": false, 21 "datasource": "$datasource", 22 "description": "Shows rate of incoming requests.", 23 "fill": 10, 24 "id": 1, 25 "legend": { 26 "avg": false, 27 "current": false, 28 "max": false, 29 "min": false, 30 "show": true, 31 "total": false, 32 "values": false 33 }, 34 "lines": true, 35 "linewidth": 0, 36 "links": [ ], 37 "nullPointMode": "null as zero", 38 "percentage": false, 39 "pointradius": 5, 40 "points": false, 41 "renderer": "flot", 42 "seriesOverrides": [ 43 { 44 "alias": "/1../", 45 "color": "#EAB839" 46 }, 47 { 48 "alias": "/2../", 49 "color": "#37872D" 50 }, 51 { 52 "alias": "/3../", 53 "color": "#E0B400" 54 }, 55 { 56 "alias": "/4../", 57 "color": "#1F60C4" 58 }, 59 { 60 "alias": "/5../", 61 "color": "#C4162A" 62 } 63 ], 64 "spaceLength": 10, 65 "span": 4, 66 "stack": true, 67 "steppedLine": false, 68 "targets": [ 69 { 70 "expr": "sum by (job, handler, code) (rate(http_requests_total{job=~\"$job\", handler=\"receive\"}[$interval]))", 71 "format": "time_series", 72 "intervalFactor": 2, 73 "legendFormat": "{{job}} {{handler}} {{code}}", 74 "step": 10 75 } 76 ], 77 "thresholds": [ ], 78 "timeFrom": null, 79 "timeShift": null, 80 "title": "Rate", 81 "tooltip": { 82 "shared": false, 83 "sort": 0, 84 "value_type": "individual" 85 }, 86 "type": "graph", 87 "xaxis": { 88 "buckets": null, 89 "mode": "time", 90 "name": null, 91 "show": true, 92 "values": [ ] 93 }, 94 "yaxes": [ 95 { 96 "format": "short", 97 "label": null, 98 "logBase": 1, 99 "max": null, 100 "min": 0, 101 "show": true 102 }, 103 { 104 "format": "short", 105 "label": null, 106 "logBase": 1, 107 "max": null, 108 "min": null, 109 "show": false 110 } 111 ] 112 }, 113 { 114 "aliasColors": { 115 "error": "#E24D42" 116 }, 117 "bars": false, 118 "dashLength": 10, 119 "dashes": false, 120 "datasource": "$datasource", 121 "description": "Shows ratio of errors compared to the total number of handled incoming requests.", 122 "fill": 10, 123 "id": 2, 124 "legend": { 125 "avg": false, 126 "current": false, 127 "max": false, 128 "min": false, 129 "show": true, 130 "total": false, 131 "values": false 132 }, 133 "lines": true, 134 "linewidth": 0, 135 "links": [ ], 136 "nullPointMode": "null as zero", 137 "percentage": false, 138 "pointradius": 5, 139 "points": false, 140 "renderer": "flot", 141 "seriesOverrides": [ ], 142 "spaceLength": 10, 143 "span": 4, 144 "stack": true, 145 "steppedLine": false, 146 "targets": [ 147 { 148 "expr": "sum by (job, code) (rate(http_requests_total{job=~\"$job\", handler=\"receive\",code=~\"5..\"}[$interval])) / ignoring (code) group_left() sum by (job) (rate(http_requests_total{job=~\"$job\", handler=\"receive\"}[$interval]))", 149 "format": "time_series", 150 "intervalFactor": 2, 151 "step": 10 152 } 153 ], 154 "thresholds": [ ], 155 "timeFrom": null, 156 "timeShift": null, 157 "title": "Errors", 158 "tooltip": { 159 "shared": false, 160 "sort": 0, 161 "value_type": "individual" 162 }, 163 "type": "graph", 164 "xaxis": { 165 "buckets": null, 166 "mode": "time", 167 "name": null, 168 "show": true, 169 "values": [ ] 170 }, 171 "yaxes": [ 172 { 173 "format": "percentunit", 174 "label": null, 175 "logBase": 1, 176 "max": null, 177 "min": 0, 178 "show": true 179 }, 180 { 181 "format": "short", 182 "label": null, 183 "logBase": 1, 184 "max": null, 185 "min": null, 186 "show": false 187 } 188 ] 189 }, 190 { 191 "aliasColors": { }, 192 "bars": false, 193 "dashLength": 10, 194 "dashes": false, 195 "datasource": "$datasource", 196 "description": "Shows how long has it taken to handle incoming requests in quantiles.", 197 "fill": 1, 198 "id": 3, 199 "legend": { 200 "avg": false, 201 "current": false, 202 "max": false, 203 "min": false, 204 "show": true, 205 "total": false, 206 "values": false 207 }, 208 "lines": true, 209 "linewidth": 1, 210 "links": [ ], 211 "nullPointMode": "null as zero", 212 "percentage": false, 213 "pointradius": 5, 214 "points": false, 215 "renderer": "flot", 216 "seriesOverrides": [ 217 { 218 "alias": "p99", 219 "color": "#FA6400", 220 "fill": 1, 221 "fillGradient": 1 222 }, 223 { 224 "alias": "p90", 225 "color": "#E0B400", 226 "fill": 1, 227 "fillGradient": 1 228 }, 229 { 230 "alias": "p50", 231 "color": "#37872D", 232 "fill": 10, 233 "fillGradient": 0 234 } 235 ], 236 "spaceLength": 10, 237 "span": 4, 238 "stack": false, 239 "steppedLine": false, 240 "targets": [ 241 { 242 "expr": "histogram_quantile(0.50, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=~\"$job\", handler=\"receive\"}[$interval]))) * 1", 243 "format": "time_series", 244 "intervalFactor": 2, 245 "legendFormat": "p50 {{job}}", 246 "logBase": 10, 247 "max": null, 248 "min": null, 249 "step": 10 250 }, 251 { 252 "expr": "histogram_quantile(0.90, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=~\"$job\", handler=\"receive\"}[$interval]))) * 1", 253 "format": "time_series", 254 "intervalFactor": 2, 255 "legendFormat": "p90 {{job}}", 256 "logBase": 10, 257 "max": null, 258 "min": null, 259 "step": 10 260 }, 261 { 262 "expr": "histogram_quantile(0.99, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=~\"$job\", handler=\"receive\"}[$interval]))) * 1", 263 "format": "time_series", 264 "intervalFactor": 2, 265 "legendFormat": "p99 {{job}}", 266 "logBase": 10, 267 "max": null, 268 "min": null, 269 "step": 10 270 } 271 ], 272 "thresholds": [ ], 273 "timeFrom": null, 274 "timeShift": null, 275 "title": "Duration", 276 "tooltip": { 277 "shared": false, 278 "sort": 0, 279 "value_type": "individual" 280 }, 281 "type": "graph", 282 "xaxis": { 283 "buckets": null, 284 "mode": "time", 285 "name": null, 286 "show": true, 287 "values": [ ] 288 }, 289 "yaxes": [ 290 { 291 "format": "s", 292 "label": null, 293 "logBase": 1, 294 "max": null, 295 "min": 0, 296 "show": true 297 }, 298 { 299 "format": "short", 300 "label": null, 301 "logBase": 1, 302 "max": null, 303 "min": null, 304 "show": false 305 } 306 ] 307 } 308 ], 309 "repeat": null, 310 "repeatIteration": null, 311 "repeatRowId": null, 312 "showTitle": true, 313 "title": "WRITE - Incoming Request", 314 "titleSize": "h6" 315 }, 316 { 317 "collapse": false, 318 "height": "250px", 319 "panels": [ 320 { 321 "aliasColors": { }, 322 "bars": false, 323 "dashLength": 10, 324 "dashes": false, 325 "datasource": "$datasource", 326 "fill": 1, 327 "id": 4, 328 "legend": { 329 "avg": false, 330 "current": false, 331 "max": false, 332 "min": false, 333 "show": true, 334 "total": false, 335 "values": false 336 }, 337 "lines": true, 338 "linewidth": 1, 339 "links": [ ], 340 "nullPointMode": "null as zero", 341 "percentage": false, 342 "pointradius": 5, 343 "points": false, 344 "renderer": "flot", 345 "seriesOverrides": [ ], 346 "spaceLength": 10, 347 "span": 4, 348 "stack": false, 349 "steppedLine": false, 350 "targets": [ 351 { 352 "expr": "sum by (tenant, code) (rate(http_requests_total{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\"}[$__rate_interval]))", 353 "format": "time_series", 354 "intervalFactor": 2, 355 "legendFormat": "{{code}} - {{tenant}}", 356 "legendLink": null, 357 "step": 10 358 } 359 ], 360 "thresholds": [ ], 361 "timeFrom": null, 362 "timeShift": null, 363 "title": "Rate of write requests (by tenant and code)", 364 "tooltip": { 365 "shared": false, 366 "sort": 0, 367 "value_type": "individual" 368 }, 369 "type": "graph", 370 "xaxis": { 371 "buckets": null, 372 "mode": "time", 373 "name": null, 374 "show": true, 375 "values": [ ] 376 }, 377 "yaxes": [ 378 { 379 "format": "short", 380 "label": null, 381 "logBase": 1, 382 "max": null, 383 "min": 0, 384 "show": true 385 }, 386 { 387 "format": "short", 388 "label": null, 389 "logBase": 1, 390 "max": null, 391 "min": null, 392 "show": false 393 } 394 ] 395 }, 396 { 397 "aliasColors": { }, 398 "bars": false, 399 "dashLength": 10, 400 "dashes": false, 401 "datasource": "$datasource", 402 "fill": 1, 403 "id": 5, 404 "legend": { 405 "avg": false, 406 "current": false, 407 "max": false, 408 "min": false, 409 "show": true, 410 "total": false, 411 "values": false 412 }, 413 "lines": true, 414 "linewidth": 1, 415 "links": [ ], 416 "nullPointMode": "null as zero", 417 "percentage": false, 418 "pointradius": 5, 419 "points": false, 420 "renderer": "flot", 421 "seriesOverrides": [ ], 422 "spaceLength": 10, 423 "span": 4, 424 "stack": false, 425 "steppedLine": false, 426 "targets": [ 427 { 428 "expr": "sum by (tenant, code) (rate(http_requests_total{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\", code!~\"2..\"}[$__rate_interval]))", 429 "format": "time_series", 430 "intervalFactor": 2, 431 "legendFormat": "{{code}} - {{tenant}}", 432 "legendLink": null, 433 "step": 10 434 } 435 ], 436 "thresholds": [ ], 437 "timeFrom": null, 438 "timeShift": null, 439 "title": "Number of errors (by tenant and code)", 440 "tooltip": { 441 "shared": false, 442 "sort": 0, 443 "value_type": "individual" 444 }, 445 "type": "graph", 446 "xaxis": { 447 "buckets": null, 448 "mode": "time", 449 "name": null, 450 "show": true, 451 "values": [ ] 452 }, 453 "yaxes": [ 454 { 455 "format": "short", 456 "label": null, 457 "logBase": 1, 458 "max": null, 459 "min": 0, 460 "show": true 461 }, 462 { 463 "format": "short", 464 "label": null, 465 "logBase": 1, 466 "max": null, 467 "min": null, 468 "show": false 469 } 470 ] 471 }, 472 { 473 "aliasColors": { }, 474 "bars": false, 475 "dashLength": 10, 476 "dashes": false, 477 "datasource": "$datasource", 478 "fill": 1, 479 "id": 6, 480 "legend": { 481 "avg": false, 482 "current": false, 483 "max": false, 484 "min": false, 485 "show": true, 486 "total": false, 487 "values": false 488 }, 489 "lines": true, 490 "linewidth": 1, 491 "links": [ ], 492 "nullPointMode": "null as zero", 493 "percentage": false, 494 "pointradius": 5, 495 "points": false, 496 "renderer": "flot", 497 "seriesOverrides": [ ], 498 "spaceLength": 10, 499 "span": 4, 500 "stack": false, 501 "steppedLine": false, 502 "targets": [ 503 { 504 "expr": "sum by (job, tenant) (rate(http_request_duration_seconds_sum{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\"}[$__rate_interval])) / sum by (job, tenant) (http_request_duration_seconds_count{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\"})", 505 "format": "time_series", 506 "intervalFactor": 2, 507 "legendFormat": "{{tenant}}", 508 "legendLink": null, 509 "step": 10 510 } 511 ], 512 "thresholds": [ ], 513 "timeFrom": null, 514 "timeShift": null, 515 "title": "Average request duration (by tenant)", 516 "tooltip": { 517 "shared": false, 518 "sort": 0, 519 "value_type": "individual" 520 }, 521 "type": "graph", 522 "xaxis": { 523 "buckets": null, 524 "mode": "time", 525 "name": null, 526 "show": true, 527 "values": [ ] 528 }, 529 "yaxes": [ 530 { 531 "format": "short", 532 "label": null, 533 "logBase": 1, 534 "max": null, 535 "min": 0, 536 "show": true 537 }, 538 { 539 "format": "short", 540 "label": null, 541 "logBase": 1, 542 "max": null, 543 "min": null, 544 "show": false 545 } 546 ] 547 } 548 ], 549 "repeat": null, 550 "repeatIteration": null, 551 "repeatRowId": null, 552 "showTitle": true, 553 "title": "WRITE - Incoming Request (tenant focus)", 554 "titleSize": "h6" 555 }, 556 { 557 "collapse": false, 558 "height": "250px", 559 "panels": [ 560 { 561 "aliasColors": { }, 562 "bars": false, 563 "dashLength": 10, 564 "dashes": false, 565 "datasource": "$datasource", 566 "fill": 1, 567 "id": 7, 568 "legend": { 569 "avg": false, 570 "current": false, 571 "max": false, 572 "min": false, 573 "show": true, 574 "total": false, 575 "values": false 576 }, 577 "lines": true, 578 "linewidth": 1, 579 "links": [ ], 580 "nullPointMode": "null as zero", 581 "percentage": false, 582 "pointradius": 5, 583 "points": false, 584 "renderer": "flot", 585 "seriesOverrides": [ ], 586 "spaceLength": 10, 587 "span": 4, 588 "stack": false, 589 "steppedLine": false, 590 "targets": [ 591 { 592 "expr": "sum by (job, tenant) (rate(http_request_size_bytes_sum{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\", code=~\"2..\"}[$__rate_interval])) / sum by (job, tenant) (rate(http_request_size_bytes_count{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\", code=~\"2..\"}[$__rate_interval]))", 593 "format": "time_series", 594 "intervalFactor": 2, 595 "legendFormat": "{{tenant}}", 596 "legendLink": null, 597 "step": 10 598 } 599 ], 600 "thresholds": [ ], 601 "timeFrom": null, 602 "timeShift": null, 603 "title": "Average successful HTTP request size (per tenant and code, only 2XX)", 604 "tooltip": { 605 "shared": false, 606 "sort": 0, 607 "value_type": "individual" 608 }, 609 "type": "graph", 610 "xaxis": { 611 "buckets": null, 612 "mode": "time", 613 "name": null, 614 "show": true, 615 "values": [ ] 616 }, 617 "yaxes": [ 618 { 619 "format": "short", 620 "label": null, 621 "logBase": 1, 622 "max": null, 623 "min": 0, 624 "show": true 625 }, 626 { 627 "format": "short", 628 "label": null, 629 "logBase": 1, 630 "max": null, 631 "min": null, 632 "show": false 633 } 634 ] 635 }, 636 { 637 "aliasColors": { }, 638 "bars": false, 639 "dashLength": 10, 640 "dashes": false, 641 "datasource": "$datasource", 642 "fill": 1, 643 "id": 8, 644 "legend": { 645 "avg": false, 646 "current": false, 647 "max": false, 648 "min": false, 649 "show": true, 650 "total": false, 651 "values": false 652 }, 653 "lines": true, 654 "linewidth": 1, 655 "links": [ ], 656 "nullPointMode": "null as zero", 657 "percentage": false, 658 "pointradius": 5, 659 "points": false, 660 "renderer": "flot", 661 "seriesOverrides": [ ], 662 "spaceLength": 10, 663 "span": 4, 664 "stack": false, 665 "steppedLine": false, 666 "targets": [ 667 { 668 "expr": "sum by (job, tenant) (rate(http_request_size_bytes_sum{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\", code!~\"2..\"}[$__rate_interval])) / sum by (job, tenant) (rate(http_request_size_bytes_count{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\", code!~\"2..\"}[$__rate_interval]))", 669 "format": "time_series", 670 "intervalFactor": 2, 671 "legendFormat": "{{tenant}}", 672 "legendLink": null, 673 "step": 10 674 } 675 ], 676 "thresholds": [ ], 677 "timeFrom": null, 678 "timeShift": null, 679 "title": "Average failed HTTP request size (per tenant and code, non 2XX)", 680 "tooltip": { 681 "shared": false, 682 "sort": 0, 683 "value_type": "individual" 684 }, 685 "type": "graph", 686 "xaxis": { 687 "buckets": null, 688 "mode": "time", 689 "name": null, 690 "show": true, 691 "values": [ ] 692 }, 693 "yaxes": [ 694 { 695 "format": "short", 696 "label": null, 697 "logBase": 1, 698 "max": null, 699 "min": 0, 700 "show": true 701 }, 702 { 703 "format": "short", 704 "label": null, 705 "logBase": 1, 706 "max": null, 707 "min": null, 708 "show": false 709 } 710 ] 711 }, 712 { 713 "aliasColors": { }, 714 "bars": false, 715 "dashLength": 10, 716 "dashes": false, 717 "datasource": "$datasource", 718 "fill": 1, 719 "id": 9, 720 "legend": { 721 "avg": false, 722 "current": false, 723 "max": false, 724 "min": false, 725 "show": true, 726 "total": false, 727 "values": false 728 }, 729 "lines": true, 730 "linewidth": 1, 731 "links": [ ], 732 "nullPointMode": "null as zero", 733 "percentage": false, 734 "pointradius": 5, 735 "points": false, 736 "renderer": "flot", 737 "seriesOverrides": [ ], 738 "spaceLength": 10, 739 "span": 4, 740 "stack": false, 741 "steppedLine": false, 742 "targets": [ 743 { 744 "expr": "sum by (job, tenant, method) (http_inflight_requests{job=~\"$job\", tenant=~\"$tenant\", handler=\"receive\"})", 745 "format": "time_series", 746 "intervalFactor": 2, 747 "legendFormat": "{{method}} - {{tenant}}", 748 "legendLink": null, 749 "step": 10 750 } 751 ], 752 "thresholds": [ ], 753 "timeFrom": null, 754 "timeShift": null, 755 "title": "Inflight requests (per tenant and method)", 756 "tooltip": { 757 "shared": false, 758 "sort": 0, 759 "value_type": "individual" 760 }, 761 "type": "graph", 762 "xaxis": { 763 "buckets": null, 764 "mode": "time", 765 "name": null, 766 "show": true, 767 "values": [ ] 768 }, 769 "yaxes": [ 770 { 771 "format": "short", 772 "label": null, 773 "logBase": 1, 774 "max": null, 775 "min": 0, 776 "show": true 777 }, 778 { 779 "format": "short", 780 "label": null, 781 "logBase": 1, 782 "max": null, 783 "min": null, 784 "show": false 785 } 786 ] 787 } 788 ], 789 "repeat": null, 790 "repeatIteration": null, 791 "repeatRowId": null, 792 "showTitle": true, 793 "title": "HTTP requests (tenant focus)", 794 "titleSize": "h6" 795 }, 796 { 797 "collapse": false, 798 "height": "250px", 799 "panels": [ 800 { 801 "aliasColors": { }, 802 "bars": false, 803 "dashLength": 10, 804 "dashes": false, 805 "datasource": "$datasource", 806 "fill": 1, 807 "id": 10, 808 "legend": { 809 "avg": false, 810 "current": false, 811 "max": false, 812 "min": false, 813 "show": true, 814 "total": false, 815 "values": false 816 }, 817 "lines": true, 818 "linewidth": 1, 819 "links": [ ], 820 "nullPointMode": "null as zero", 821 "percentage": false, 822 "pointradius": 5, 823 "points": false, 824 "renderer": "flot", 825 "seriesOverrides": [ ], 826 "spaceLength": 10, 827 "span": 3, 828 "stack": false, 829 "steppedLine": false, 830 "targets": [ 831 { 832 "expr": "sum(rate(thanos_receive_write_timeseries_sum{job=~\"$job\", tenant=~\"$tenant\", code=~\"2..\"}[$__rate_interval])) by (job, tenant) ", 833 "format": "time_series", 834 "intervalFactor": 2, 835 "legendFormat": "{{tenant}}", 836 "legendLink": null, 837 "step": 10 838 } 839 ], 840 "thresholds": [ ], 841 "timeFrom": null, 842 "timeShift": null, 843 "title": "Rate of series received (per tenant, only 2XX)", 844 "tooltip": { 845 "shared": false, 846 "sort": 0, 847 "value_type": "individual" 848 }, 849 "type": "graph", 850 "xaxis": { 851 "buckets": null, 852 "mode": "time", 853 "name": null, 854 "show": true, 855 "values": [ ] 856 }, 857 "yaxes": [ 858 { 859 "format": "short", 860 "label": null, 861 "logBase": 1, 862 "max": null, 863 "min": 0, 864 "show": true 865 }, 866 { 867 "format": "short", 868 "label": null, 869 "logBase": 1, 870 "max": null, 871 "min": null, 872 "show": false 873 } 874 ] 875 }, 876 { 877 "aliasColors": { }, 878 "bars": false, 879 "dashLength": 10, 880 "dashes": false, 881 "datasource": "$datasource", 882 "fill": 1, 883 "id": 11, 884 "legend": { 885 "avg": false, 886 "current": false, 887 "max": false, 888 "min": false, 889 "show": true, 890 "total": false, 891 "values": false 892 }, 893 "lines": true, 894 "linewidth": 1, 895 "links": [ ], 896 "nullPointMode": "null as zero", 897 "percentage": false, 898 "pointradius": 5, 899 "points": false, 900 "renderer": "flot", 901 "seriesOverrides": [ ], 902 "spaceLength": 10, 903 "span": 3, 904 "stack": false, 905 "steppedLine": false, 906 "targets": [ 907 { 908 "expr": "sum(rate(thanos_receive_write_timeseries_sum{job=~\"$job\", tenant=~\"$tenant\", code!~\"2..\"}[$__rate_interval])) by (tenant, code) ", 909 "format": "time_series", 910 "intervalFactor": 2, 911 "legendFormat": "{{code}} - {{tenant}}", 912 "legendLink": null, 913 "step": 10 914 } 915 ], 916 "thresholds": [ ], 917 "timeFrom": null, 918 "timeShift": null, 919 "title": "Rate of series not written (per tenant and code, non 2XX)", 920 "tooltip": { 921 "shared": false, 922 "sort": 0, 923 "value_type": "individual" 924 }, 925 "type": "graph", 926 "xaxis": { 927 "buckets": null, 928 "mode": "time", 929 "name": null, 930 "show": true, 931 "values": [ ] 932 }, 933 "yaxes": [ 934 { 935 "format": "short", 936 "label": null, 937 "logBase": 1, 938 "max": null, 939 "min": 0, 940 "show": true 941 }, 942 { 943 "format": "short", 944 "label": null, 945 "logBase": 1, 946 "max": null, 947 "min": null, 948 "show": false 949 } 950 ] 951 }, 952 { 953 "aliasColors": { }, 954 "bars": false, 955 "dashLength": 10, 956 "dashes": false, 957 "datasource": "$datasource", 958 "fill": 1, 959 "id": 12, 960 "legend": { 961 "avg": false, 962 "current": false, 963 "max": false, 964 "min": false, 965 "show": true, 966 "total": false, 967 "values": false 968 }, 969 "lines": true, 970 "linewidth": 1, 971 "links": [ ], 972 "nullPointMode": "null as zero", 973 "percentage": false, 974 "pointradius": 5, 975 "points": false, 976 "renderer": "flot", 977 "seriesOverrides": [ ], 978 "spaceLength": 10, 979 "span": 3, 980 "stack": false, 981 "steppedLine": false, 982 "targets": [ 983 { 984 "expr": "sum(rate(thanos_receive_write_samples_sum{job=~\"$job\", tenant=~\"$tenant\", code=~\"2..\"}[$__rate_interval])) by (job, tenant) ", 985 "format": "time_series", 986 "intervalFactor": 2, 987 "legendFormat": "{{tenant}}", 988 "legendLink": null, 989 "step": 10 990 } 991 ], 992 "thresholds": [ ], 993 "timeFrom": null, 994 "timeShift": null, 995 "title": "Rate of samples received (per tenant, only 2XX)", 996 "tooltip": { 997 "shared": false, 998 "sort": 0, 999 "value_type": "individual" 1000 }, 1001 "type": "graph", 1002 "xaxis": { 1003 "buckets": null, 1004 "mode": "time", 1005 "name": null, 1006 "show": true, 1007 "values": [ ] 1008 }, 1009 "yaxes": [ 1010 { 1011 "format": "short", 1012 "label": null, 1013 "logBase": 1, 1014 "max": null, 1015 "min": 0, 1016 "show": true 1017 }, 1018 { 1019 "format": "short", 1020 "label": null, 1021 "logBase": 1, 1022 "max": null, 1023 "min": null, 1024 "show": false 1025 } 1026 ] 1027 }, 1028 { 1029 "aliasColors": { }, 1030 "bars": false, 1031 "dashLength": 10, 1032 "dashes": false, 1033 "datasource": "$datasource", 1034 "fill": 1, 1035 "id": 13, 1036 "legend": { 1037 "avg": false, 1038 "current": false, 1039 "max": false, 1040 "min": false, 1041 "show": true, 1042 "total": false, 1043 "values": false 1044 }, 1045 "lines": true, 1046 "linewidth": 1, 1047 "links": [ ], 1048 "nullPointMode": "null as zero", 1049 "percentage": false, 1050 "pointradius": 5, 1051 "points": false, 1052 "renderer": "flot", 1053 "seriesOverrides": [ ], 1054 "spaceLength": 10, 1055 "span": 3, 1056 "stack": false, 1057 "steppedLine": false, 1058 "targets": [ 1059 { 1060 "expr": "sum(rate(thanos_receive_write_samples_sum{job=~\"$job\", tenant=~\"$tenant\", code!~\"2..\"}[$__rate_interval])) by (tenant, code) ", 1061 "format": "time_series", 1062 "intervalFactor": 2, 1063 "legendFormat": "{{code}} - {{tenant}}", 1064 "legendLink": null, 1065 "step": 10 1066 } 1067 ], 1068 "thresholds": [ ], 1069 "timeFrom": null, 1070 "timeShift": null, 1071 "title": "Rate of samples not written (per tenant and code, non 2XX)", 1072 "tooltip": { 1073 "shared": false, 1074 "sort": 0, 1075 "value_type": "individual" 1076 }, 1077 "type": "graph", 1078 "xaxis": { 1079 "buckets": null, 1080 "mode": "time", 1081 "name": null, 1082 "show": true, 1083 "values": [ ] 1084 }, 1085 "yaxes": [ 1086 { 1087 "format": "short", 1088 "label": null, 1089 "logBase": 1, 1090 "max": null, 1091 "min": 0, 1092 "show": true 1093 }, 1094 { 1095 "format": "short", 1096 "label": null, 1097 "logBase": 1, 1098 "max": null, 1099 "min": null, 1100 "show": false 1101 } 1102 ] 1103 } 1104 ], 1105 "repeat": null, 1106 "repeatIteration": null, 1107 "repeatRowId": null, 1108 "showTitle": true, 1109 "title": "Series & Samples (tenant focus)", 1110 "titleSize": "h6" 1111 }, 1112 { 1113 "collapse": false, 1114 "height": "250px", 1115 "panels": [ 1116 { 1117 "aliasColors": { }, 1118 "bars": false, 1119 "dashLength": 10, 1120 "dashes": false, 1121 "datasource": "$datasource", 1122 "description": "Shows rate of replications to other receive nodes.", 1123 "fill": 1, 1124 "id": 14, 1125 "legend": { 1126 "avg": false, 1127 "current": false, 1128 "max": false, 1129 "min": false, 1130 "show": true, 1131 "total": false, 1132 "values": false 1133 }, 1134 "lines": true, 1135 "linewidth": 1, 1136 "links": [ ], 1137 "nullPointMode": "null as zero", 1138 "percentage": false, 1139 "pointradius": 5, 1140 "points": false, 1141 "renderer": "flot", 1142 "seriesOverrides": [ ], 1143 "spaceLength": 10, 1144 "span": 6, 1145 "stack": false, 1146 "steppedLine": false, 1147 "targets": [ 1148 { 1149 "expr": "sum by (job) (rate(thanos_receive_replications_total{job=~\"$job\"}[$__rate_interval]))", 1150 "format": "time_series", 1151 "intervalFactor": 2, 1152 "legendFormat": "all {{job}}", 1153 "legendLink": null, 1154 "step": 10 1155 } 1156 ], 1157 "thresholds": [ ], 1158 "timeFrom": null, 1159 "timeShift": null, 1160 "title": "Rate", 1161 "tooltip": { 1162 "shared": false, 1163 "sort": 0, 1164 "value_type": "individual" 1165 }, 1166 "type": "graph", 1167 "xaxis": { 1168 "buckets": null, 1169 "mode": "time", 1170 "name": null, 1171 "show": true, 1172 "values": [ ] 1173 }, 1174 "yaxes": [ 1175 { 1176 "format": "short", 1177 "label": null, 1178 "logBase": 1, 1179 "max": null, 1180 "min": 0, 1181 "show": true 1182 }, 1183 { 1184 "format": "short", 1185 "label": null, 1186 "logBase": 1, 1187 "max": null, 1188 "min": null, 1189 "show": false 1190 } 1191 ] 1192 }, 1193 { 1194 "aliasColors": { 1195 "error": "#E24D42" 1196 }, 1197 "bars": false, 1198 "dashLength": 10, 1199 "dashes": false, 1200 "datasource": "$datasource", 1201 "description": "Shows ratio of errors compared to the total number of replications to other receive nodes.", 1202 "fill": 10, 1203 "id": 15, 1204 "legend": { 1205 "avg": false, 1206 "current": false, 1207 "max": false, 1208 "min": false, 1209 "show": true, 1210 "total": false, 1211 "values": false 1212 }, 1213 "lines": true, 1214 "linewidth": 0, 1215 "links": [ ], 1216 "nullPointMode": "null as zero", 1217 "percentage": false, 1218 "pointradius": 5, 1219 "points": false, 1220 "renderer": "flot", 1221 "seriesOverrides": [ ], 1222 "spaceLength": 10, 1223 "span": 6, 1224 "stack": true, 1225 "steppedLine": false, 1226 "targets": [ 1227 { 1228 "expr": "sum by (job) (rate(thanos_receive_replications_total{job=~\"$job\", result=\"error\"}[$interval])) / sum by (job) (rate(thanos_receive_replications_total{job=~\"$job\"}[$interval]))", 1229 "format": "time_series", 1230 "intervalFactor": 2, 1231 "legendFormat": "error", 1232 "step": 10 1233 } 1234 ], 1235 "thresholds": [ ], 1236 "timeFrom": null, 1237 "timeShift": null, 1238 "title": "Errors", 1239 "tooltip": { 1240 "shared": false, 1241 "sort": 0, 1242 "value_type": "individual" 1243 }, 1244 "type": "graph", 1245 "xaxis": { 1246 "buckets": null, 1247 "mode": "time", 1248 "name": null, 1249 "show": true, 1250 "values": [ ] 1251 }, 1252 "yaxes": [ 1253 { 1254 "format": "percentunit", 1255 "label": null, 1256 "logBase": 1, 1257 "max": null, 1258 "min": 0, 1259 "show": true 1260 }, 1261 { 1262 "format": "short", 1263 "label": null, 1264 "logBase": 1, 1265 "max": null, 1266 "min": null, 1267 "show": false 1268 } 1269 ] 1270 } 1271 ], 1272 "repeat": null, 1273 "repeatIteration": null, 1274 "repeatRowId": null, 1275 "showTitle": true, 1276 "title": "WRITE - Replication", 1277 "titleSize": "h6" 1278 }, 1279 { 1280 "collapse": false, 1281 "height": "250px", 1282 "panels": [ 1283 { 1284 "aliasColors": { }, 1285 "bars": false, 1286 "dashLength": 10, 1287 "dashes": false, 1288 "datasource": "$datasource", 1289 "description": "Shows rate of forwarded requests to other receive nodes.", 1290 "fill": 1, 1291 "id": 16, 1292 "legend": { 1293 "avg": false, 1294 "current": false, 1295 "max": false, 1296 "min": false, 1297 "show": true, 1298 "total": false, 1299 "values": false 1300 }, 1301 "lines": true, 1302 "linewidth": 1, 1303 "links": [ ], 1304 "nullPointMode": "null as zero", 1305 "percentage": false, 1306 "pointradius": 5, 1307 "points": false, 1308 "renderer": "flot", 1309 "seriesOverrides": [ ], 1310 "spaceLength": 10, 1311 "span": 6, 1312 "stack": false, 1313 "steppedLine": false, 1314 "targets": [ 1315 { 1316 "expr": "sum by (job) (rate(thanos_receive_forward_requests_total{job=~\"$job\"}[$__rate_interval]))", 1317 "format": "time_series", 1318 "intervalFactor": 2, 1319 "legendFormat": "all {{job}}", 1320 "legendLink": null, 1321 "step": 10 1322 } 1323 ], 1324 "thresholds": [ ], 1325 "timeFrom": null, 1326 "timeShift": null, 1327 "title": "Rate", 1328 "tooltip": { 1329 "shared": false, 1330 "sort": 0, 1331 "value_type": "individual" 1332 }, 1333 "type": "graph", 1334 "xaxis": { 1335 "buckets": null, 1336 "mode": "time", 1337 "name": null, 1338 "show": true, 1339 "values": [ ] 1340 }, 1341 "yaxes": [ 1342 { 1343 "format": "short", 1344 "label": null, 1345 "logBase": 1, 1346 "max": null, 1347 "min": 0, 1348 "show": true 1349 }, 1350 { 1351 "format": "short", 1352 "label": null, 1353 "logBase": 1, 1354 "max": null, 1355 "min": null, 1356 "show": false 1357 } 1358 ] 1359 }, 1360 { 1361 "aliasColors": { 1362 "error": "#E24D42" 1363 }, 1364 "bars": false, 1365 "dashLength": 10, 1366 "dashes": false, 1367 "datasource": "$datasource", 1368 "description": "Shows ratio of errors compared to the total number of forwareded requests to other receive nodes.", 1369 "fill": 10, 1370 "id": 17, 1371 "legend": { 1372 "avg": false, 1373 "current": false, 1374 "max": false, 1375 "min": false, 1376 "show": true, 1377 "total": false, 1378 "values": false 1379 }, 1380 "lines": true, 1381 "linewidth": 0, 1382 "links": [ ], 1383 "nullPointMode": "null as zero", 1384 "percentage": false, 1385 "pointradius": 5, 1386 "points": false, 1387 "renderer": "flot", 1388 "seriesOverrides": [ ], 1389 "spaceLength": 10, 1390 "span": 6, 1391 "stack": true, 1392 "steppedLine": false, 1393 "targets": [ 1394 { 1395 "expr": "sum by (job) (rate(thanos_receive_forward_requests_total{job=~\"$job\", result=\"error\"}[$interval])) / sum by (job) (rate(thanos_receive_forward_requests_total{job=~\"$job\"}[$interval]))", 1396 "format": "time_series", 1397 "intervalFactor": 2, 1398 "legendFormat": "error", 1399 "step": 10 1400 } 1401 ], 1402 "thresholds": [ ], 1403 "timeFrom": null, 1404 "timeShift": null, 1405 "title": "Errors", 1406 "tooltip": { 1407 "shared": false, 1408 "sort": 0, 1409 "value_type": "individual" 1410 }, 1411 "type": "graph", 1412 "xaxis": { 1413 "buckets": null, 1414 "mode": "time", 1415 "name": null, 1416 "show": true, 1417 "values": [ ] 1418 }, 1419 "yaxes": [ 1420 { 1421 "format": "percentunit", 1422 "label": null, 1423 "logBase": 1, 1424 "max": null, 1425 "min": 0, 1426 "show": true 1427 }, 1428 { 1429 "format": "short", 1430 "label": null, 1431 "logBase": 1, 1432 "max": null, 1433 "min": null, 1434 "show": false 1435 } 1436 ] 1437 } 1438 ], 1439 "repeat": null, 1440 "repeatIteration": null, 1441 "repeatRowId": null, 1442 "showTitle": true, 1443 "title": "WRITE - Forward Request", 1444 "titleSize": "h6" 1445 }, 1446 { 1447 "collapse": false, 1448 "height": "250px", 1449 "panels": [ 1450 { 1451 "aliasColors": { }, 1452 "bars": false, 1453 "dashLength": 10, 1454 "dashes": false, 1455 "datasource": "$datasource", 1456 "description": "Shows rate of handled Unary gRPC requests from queriers.", 1457 "fill": 10, 1458 "id": 18, 1459 "legend": { 1460 "avg": false, 1461 "current": false, 1462 "max": false, 1463 "min": false, 1464 "show": true, 1465 "total": false, 1466 "values": false 1467 }, 1468 "lines": true, 1469 "linewidth": 0, 1470 "links": [ ], 1471 "nullPointMode": "null as zero", 1472 "percentage": false, 1473 "pointradius": 5, 1474 "points": false, 1475 "renderer": "flot", 1476 "seriesOverrides": [ 1477 { 1478 "alias": "/Aborted/", 1479 "color": "#EAB839" 1480 }, 1481 { 1482 "alias": "/AlreadyExists/", 1483 "color": "#37872D" 1484 }, 1485 { 1486 "alias": "/FailedPrecondition/", 1487 "color": "#E0B400" 1488 }, 1489 { 1490 "alias": "/Unimplemented/", 1491 "color": "#E0B400" 1492 }, 1493 { 1494 "alias": "/InvalidArgument/", 1495 "color": "#1F60C4" 1496 }, 1497 { 1498 "alias": "/NotFound/", 1499 "color": "#1F60C4" 1500 }, 1501 { 1502 "alias": "/PermissionDenied/", 1503 "color": "#1F60C4" 1504 }, 1505 { 1506 "alias": "/Unauthenticated/", 1507 "color": "#1F60C4" 1508 }, 1509 { 1510 "alias": "/Canceled/", 1511 "color": "#C4162A" 1512 }, 1513 { 1514 "alias": "/DataLoss/", 1515 "color": "#C4162A" 1516 }, 1517 { 1518 "alias": "/DeadlineExceeded/", 1519 "color": "#C4162A" 1520 }, 1521 { 1522 "alias": "/Internal/", 1523 "color": "#C4162A" 1524 }, 1525 { 1526 "alias": "/OutOfRange/", 1527 "color": "#C4162A" 1528 }, 1529 { 1530 "alias": "/ResourceExhausted/", 1531 "color": "#C4162A" 1532 }, 1533 { 1534 "alias": "/Unavailable/", 1535 "color": "#C4162A" 1536 }, 1537 { 1538 "alias": "/Unknown/", 1539 "color": "#C4162A" 1540 }, 1541 { 1542 "alias": "/OK/", 1543 "color": "#37872D" 1544 }, 1545 { 1546 "alias": "error", 1547 "color": "#C4162A" 1548 } 1549 ], 1550 "spaceLength": 10, 1551 "span": 4, 1552 "stack": true, 1553 "steppedLine": false, 1554 "targets": [ 1555 { 1556 "expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))", 1557 "format": "time_series", 1558 "intervalFactor": 2, 1559 "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", 1560 "step": 10 1561 } 1562 ], 1563 "thresholds": [ ], 1564 "timeFrom": null, 1565 "timeShift": null, 1566 "title": "Rate", 1567 "tooltip": { 1568 "shared": false, 1569 "sort": 0, 1570 "value_type": "individual" 1571 }, 1572 "type": "graph", 1573 "xaxis": { 1574 "buckets": null, 1575 "mode": "time", 1576 "name": null, 1577 "show": true, 1578 "values": [ ] 1579 }, 1580 "yaxes": [ 1581 { 1582 "format": "short", 1583 "label": null, 1584 "logBase": 1, 1585 "max": null, 1586 "min": 0, 1587 "show": true 1588 }, 1589 { 1590 "format": "short", 1591 "label": null, 1592 "logBase": 1, 1593 "max": null, 1594 "min": null, 1595 "show": false 1596 } 1597 ] 1598 }, 1599 { 1600 "aliasColors": { 1601 "error": "#E24D42" 1602 }, 1603 "bars": false, 1604 "dashLength": 10, 1605 "dashes": false, 1606 "datasource": "$datasource", 1607 "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", 1608 "fill": 10, 1609 "id": 19, 1610 "legend": { 1611 "avg": false, 1612 "current": false, 1613 "max": false, 1614 "min": false, 1615 "show": true, 1616 "total": false, 1617 "values": false 1618 }, 1619 "lines": true, 1620 "linewidth": 0, 1621 "links": [ ], 1622 "nullPointMode": "null as zero", 1623 "percentage": false, 1624 "pointradius": 5, 1625 "points": false, 1626 "renderer": "flot", 1627 "seriesOverrides": [ ], 1628 "spaceLength": 10, 1629 "span": 4, 1630 "stack": true, 1631 "steppedLine": false, 1632 "targets": [ 1633 { 1634 "expr": "sum by (job, grpc_code) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval])) / ignoring (grpc_code) group_left() sum by (job) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))", 1635 "format": "time_series", 1636 "intervalFactor": 2, 1637 "step": 10 1638 } 1639 ], 1640 "thresholds": [ ], 1641 "timeFrom": null, 1642 "timeShift": null, 1643 "title": "Errors", 1644 "tooltip": { 1645 "shared": false, 1646 "sort": 0, 1647 "value_type": "individual" 1648 }, 1649 "type": "graph", 1650 "xaxis": { 1651 "buckets": null, 1652 "mode": "time", 1653 "name": null, 1654 "show": true, 1655 "values": [ ] 1656 }, 1657 "yaxes": [ 1658 { 1659 "format": "percentunit", 1660 "label": null, 1661 "logBase": 1, 1662 "max": null, 1663 "min": 0, 1664 "show": true 1665 }, 1666 { 1667 "format": "short", 1668 "label": null, 1669 "logBase": 1, 1670 "max": null, 1671 "min": null, 1672 "show": false 1673 } 1674 ] 1675 }, 1676 { 1677 "aliasColors": { }, 1678 "bars": false, 1679 "dashLength": 10, 1680 "dashes": false, 1681 "datasource": "$datasource", 1682 "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", 1683 "fill": 1, 1684 "id": 20, 1685 "legend": { 1686 "avg": false, 1687 "current": false, 1688 "max": false, 1689 "min": false, 1690 "show": true, 1691 "total": false, 1692 "values": false 1693 }, 1694 "lines": true, 1695 "linewidth": 1, 1696 "links": [ ], 1697 "nullPointMode": "null as zero", 1698 "percentage": false, 1699 "pointradius": 5, 1700 "points": false, 1701 "renderer": "flot", 1702 "seriesOverrides": [ 1703 { 1704 "alias": "p99", 1705 "color": "#FA6400", 1706 "fill": 1, 1707 "fillGradient": 1 1708 }, 1709 { 1710 "alias": "p90", 1711 "color": "#E0B400", 1712 "fill": 1, 1713 "fillGradient": 1 1714 }, 1715 { 1716 "alias": "p50", 1717 "color": "#37872D", 1718 "fill": 10, 1719 "fillGradient": 0 1720 } 1721 ], 1722 "spaceLength": 10, 1723 "span": 4, 1724 "stack": false, 1725 "steppedLine": false, 1726 "targets": [ 1727 { 1728 "expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))) * 1", 1729 "format": "time_series", 1730 "intervalFactor": 2, 1731 "legendFormat": "p50 {{job}}", 1732 "logBase": 10, 1733 "max": null, 1734 "min": null, 1735 "step": 10 1736 }, 1737 { 1738 "expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))) * 1", 1739 "format": "time_series", 1740 "intervalFactor": 2, 1741 "legendFormat": "p90 {{job}}", 1742 "logBase": 10, 1743 "max": null, 1744 "min": null, 1745 "step": 10 1746 }, 1747 { 1748 "expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))) * 1", 1749 "format": "time_series", 1750 "intervalFactor": 2, 1751 "legendFormat": "p99 {{job}}", 1752 "logBase": 10, 1753 "max": null, 1754 "min": null, 1755 "step": 10 1756 } 1757 ], 1758 "thresholds": [ ], 1759 "timeFrom": null, 1760 "timeShift": null, 1761 "title": "Duration", 1762 "tooltip": { 1763 "shared": false, 1764 "sort": 0, 1765 "value_type": "individual" 1766 }, 1767 "type": "graph", 1768 "xaxis": { 1769 "buckets": null, 1770 "mode": "time", 1771 "name": null, 1772 "show": true, 1773 "values": [ ] 1774 }, 1775 "yaxes": [ 1776 { 1777 "format": "s", 1778 "label": null, 1779 "logBase": 1, 1780 "max": null, 1781 "min": 0, 1782 "show": true 1783 }, 1784 { 1785 "format": "short", 1786 "label": null, 1787 "logBase": 1, 1788 "max": null, 1789 "min": null, 1790 "show": false 1791 } 1792 ] 1793 } 1794 ], 1795 "repeat": null, 1796 "repeatIteration": null, 1797 "repeatRowId": null, 1798 "showTitle": true, 1799 "title": "WRITE - gRPC (Unary)", 1800 "titleSize": "h6" 1801 }, 1802 { 1803 "collapse": false, 1804 "height": "250px", 1805 "panels": [ 1806 { 1807 "aliasColors": { }, 1808 "bars": false, 1809 "dashLength": 10, 1810 "dashes": false, 1811 "datasource": "$datasource", 1812 "description": "Shows rate of handled Unary gRPC requests from queriers.", 1813 "fill": 10, 1814 "id": 21, 1815 "legend": { 1816 "avg": false, 1817 "current": false, 1818 "max": false, 1819 "min": false, 1820 "show": true, 1821 "total": false, 1822 "values": false 1823 }, 1824 "lines": true, 1825 "linewidth": 0, 1826 "links": [ ], 1827 "nullPointMode": "null as zero", 1828 "percentage": false, 1829 "pointradius": 5, 1830 "points": false, 1831 "renderer": "flot", 1832 "seriesOverrides": [ 1833 { 1834 "alias": "/Aborted/", 1835 "color": "#EAB839" 1836 }, 1837 { 1838 "alias": "/AlreadyExists/", 1839 "color": "#37872D" 1840 }, 1841 { 1842 "alias": "/FailedPrecondition/", 1843 "color": "#E0B400" 1844 }, 1845 { 1846 "alias": "/Unimplemented/", 1847 "color": "#E0B400" 1848 }, 1849 { 1850 "alias": "/InvalidArgument/", 1851 "color": "#1F60C4" 1852 }, 1853 { 1854 "alias": "/NotFound/", 1855 "color": "#1F60C4" 1856 }, 1857 { 1858 "alias": "/PermissionDenied/", 1859 "color": "#1F60C4" 1860 }, 1861 { 1862 "alias": "/Unauthenticated/", 1863 "color": "#1F60C4" 1864 }, 1865 { 1866 "alias": "/Canceled/", 1867 "color": "#C4162A" 1868 }, 1869 { 1870 "alias": "/DataLoss/", 1871 "color": "#C4162A" 1872 }, 1873 { 1874 "alias": "/DeadlineExceeded/", 1875 "color": "#C4162A" 1876 }, 1877 { 1878 "alias": "/Internal/", 1879 "color": "#C4162A" 1880 }, 1881 { 1882 "alias": "/OutOfRange/", 1883 "color": "#C4162A" 1884 }, 1885 { 1886 "alias": "/ResourceExhausted/", 1887 "color": "#C4162A" 1888 }, 1889 { 1890 "alias": "/Unavailable/", 1891 "color": "#C4162A" 1892 }, 1893 { 1894 "alias": "/Unknown/", 1895 "color": "#C4162A" 1896 }, 1897 { 1898 "alias": "/OK/", 1899 "color": "#37872D" 1900 }, 1901 { 1902 "alias": "error", 1903 "color": "#C4162A" 1904 } 1905 ], 1906 "spaceLength": 10, 1907 "span": 4, 1908 "stack": true, 1909 "steppedLine": false, 1910 "targets": [ 1911 { 1912 "expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))", 1913 "format": "time_series", 1914 "intervalFactor": 2, 1915 "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", 1916 "step": 10 1917 } 1918 ], 1919 "thresholds": [ ], 1920 "timeFrom": null, 1921 "timeShift": null, 1922 "title": "Rate", 1923 "tooltip": { 1924 "shared": false, 1925 "sort": 0, 1926 "value_type": "individual" 1927 }, 1928 "type": "graph", 1929 "xaxis": { 1930 "buckets": null, 1931 "mode": "time", 1932 "name": null, 1933 "show": true, 1934 "values": [ ] 1935 }, 1936 "yaxes": [ 1937 { 1938 "format": "short", 1939 "label": null, 1940 "logBase": 1, 1941 "max": null, 1942 "min": 0, 1943 "show": true 1944 }, 1945 { 1946 "format": "short", 1947 "label": null, 1948 "logBase": 1, 1949 "max": null, 1950 "min": null, 1951 "show": false 1952 } 1953 ] 1954 }, 1955 { 1956 "aliasColors": { 1957 "error": "#E24D42" 1958 }, 1959 "bars": false, 1960 "dashLength": 10, 1961 "dashes": false, 1962 "datasource": "$datasource", 1963 "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", 1964 "fill": 10, 1965 "id": 22, 1966 "legend": { 1967 "avg": false, 1968 "current": false, 1969 "max": false, 1970 "min": false, 1971 "show": true, 1972 "total": false, 1973 "values": false 1974 }, 1975 "lines": true, 1976 "linewidth": 0, 1977 "links": [ ], 1978 "nullPointMode": "null as zero", 1979 "percentage": false, 1980 "pointradius": 5, 1981 "points": false, 1982 "renderer": "flot", 1983 "seriesOverrides": [ ], 1984 "spaceLength": 10, 1985 "span": 4, 1986 "stack": true, 1987 "steppedLine": false, 1988 "targets": [ 1989 { 1990 "expr": "sum by (job, grpc_code) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval])) / ignoring (grpc_code) group_left() sum by (job) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))", 1991 "format": "time_series", 1992 "intervalFactor": 2, 1993 "step": 10 1994 } 1995 ], 1996 "thresholds": [ ], 1997 "timeFrom": null, 1998 "timeShift": null, 1999 "title": "Errors", 2000 "tooltip": { 2001 "shared": false, 2002 "sort": 0, 2003 "value_type": "individual" 2004 }, 2005 "type": "graph", 2006 "xaxis": { 2007 "buckets": null, 2008 "mode": "time", 2009 "name": null, 2010 "show": true, 2011 "values": [ ] 2012 }, 2013 "yaxes": [ 2014 { 2015 "format": "percentunit", 2016 "label": null, 2017 "logBase": 1, 2018 "max": null, 2019 "min": 0, 2020 "show": true 2021 }, 2022 { 2023 "format": "short", 2024 "label": null, 2025 "logBase": 1, 2026 "max": null, 2027 "min": null, 2028 "show": false 2029 } 2030 ] 2031 }, 2032 { 2033 "aliasColors": { }, 2034 "bars": false, 2035 "dashLength": 10, 2036 "dashes": false, 2037 "datasource": "$datasource", 2038 "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", 2039 "fill": 1, 2040 "id": 23, 2041 "legend": { 2042 "avg": false, 2043 "current": false, 2044 "max": false, 2045 "min": false, 2046 "show": true, 2047 "total": false, 2048 "values": false 2049 }, 2050 "lines": true, 2051 "linewidth": 1, 2052 "links": [ ], 2053 "nullPointMode": "null as zero", 2054 "percentage": false, 2055 "pointradius": 5, 2056 "points": false, 2057 "renderer": "flot", 2058 "seriesOverrides": [ 2059 { 2060 "alias": "p99", 2061 "color": "#FA6400", 2062 "fill": 1, 2063 "fillGradient": 1 2064 }, 2065 { 2066 "alias": "p90", 2067 "color": "#E0B400", 2068 "fill": 1, 2069 "fillGradient": 1 2070 }, 2071 { 2072 "alias": "p50", 2073 "color": "#37872D", 2074 "fill": 10, 2075 "fillGradient": 0 2076 } 2077 ], 2078 "spaceLength": 10, 2079 "span": 4, 2080 "stack": false, 2081 "steppedLine": false, 2082 "targets": [ 2083 { 2084 "expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))) * 1", 2085 "format": "time_series", 2086 "intervalFactor": 2, 2087 "legendFormat": "p50 {{job}}", 2088 "logBase": 10, 2089 "max": null, 2090 "min": null, 2091 "step": 10 2092 }, 2093 { 2094 "expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))) * 1", 2095 "format": "time_series", 2096 "intervalFactor": 2, 2097 "legendFormat": "p90 {{job}}", 2098 "logBase": 10, 2099 "max": null, 2100 "min": null, 2101 "step": 10 2102 }, 2103 { 2104 "expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))) * 1", 2105 "format": "time_series", 2106 "intervalFactor": 2, 2107 "legendFormat": "p99 {{job}}", 2108 "logBase": 10, 2109 "max": null, 2110 "min": null, 2111 "step": 10 2112 } 2113 ], 2114 "thresholds": [ ], 2115 "timeFrom": null, 2116 "timeShift": null, 2117 "title": "Duration", 2118 "tooltip": { 2119 "shared": false, 2120 "sort": 0, 2121 "value_type": "individual" 2122 }, 2123 "type": "graph", 2124 "xaxis": { 2125 "buckets": null, 2126 "mode": "time", 2127 "name": null, 2128 "show": true, 2129 "values": [ ] 2130 }, 2131 "yaxes": [ 2132 { 2133 "format": "s", 2134 "label": null, 2135 "logBase": 1, 2136 "max": null, 2137 "min": 0, 2138 "show": true 2139 }, 2140 { 2141 "format": "short", 2142 "label": null, 2143 "logBase": 1, 2144 "max": null, 2145 "min": null, 2146 "show": false 2147 } 2148 ] 2149 } 2150 ], 2151 "repeat": null, 2152 "repeatIteration": null, 2153 "repeatRowId": null, 2154 "showTitle": true, 2155 "title": "READ - gRPC (Unary)", 2156 "titleSize": "h6" 2157 }, 2158 { 2159 "collapse": false, 2160 "height": "250px", 2161 "panels": [ 2162 { 2163 "aliasColors": { }, 2164 "bars": false, 2165 "dashLength": 10, 2166 "dashes": false, 2167 "datasource": "$datasource", 2168 "description": "Shows rate of handled Streamed gRPC requests from queriers.", 2169 "fill": 10, 2170 "id": 24, 2171 "legend": { 2172 "avg": false, 2173 "current": false, 2174 "max": false, 2175 "min": false, 2176 "show": true, 2177 "total": false, 2178 "values": false 2179 }, 2180 "lines": true, 2181 "linewidth": 0, 2182 "links": [ ], 2183 "nullPointMode": "null as zero", 2184 "percentage": false, 2185 "pointradius": 5, 2186 "points": false, 2187 "renderer": "flot", 2188 "seriesOverrides": [ 2189 { 2190 "alias": "/Aborted/", 2191 "color": "#EAB839" 2192 }, 2193 { 2194 "alias": "/AlreadyExists/", 2195 "color": "#37872D" 2196 }, 2197 { 2198 "alias": "/FailedPrecondition/", 2199 "color": "#E0B400" 2200 }, 2201 { 2202 "alias": "/Unimplemented/", 2203 "color": "#E0B400" 2204 }, 2205 { 2206 "alias": "/InvalidArgument/", 2207 "color": "#1F60C4" 2208 }, 2209 { 2210 "alias": "/NotFound/", 2211 "color": "#1F60C4" 2212 }, 2213 { 2214 "alias": "/PermissionDenied/", 2215 "color": "#1F60C4" 2216 }, 2217 { 2218 "alias": "/Unauthenticated/", 2219 "color": "#1F60C4" 2220 }, 2221 { 2222 "alias": "/Canceled/", 2223 "color": "#C4162A" 2224 }, 2225 { 2226 "alias": "/DataLoss/", 2227 "color": "#C4162A" 2228 }, 2229 { 2230 "alias": "/DeadlineExceeded/", 2231 "color": "#C4162A" 2232 }, 2233 { 2234 "alias": "/Internal/", 2235 "color": "#C4162A" 2236 }, 2237 { 2238 "alias": "/OutOfRange/", 2239 "color": "#C4162A" 2240 }, 2241 { 2242 "alias": "/ResourceExhausted/", 2243 "color": "#C4162A" 2244 }, 2245 { 2246 "alias": "/Unavailable/", 2247 "color": "#C4162A" 2248 }, 2249 { 2250 "alias": "/Unknown/", 2251 "color": "#C4162A" 2252 }, 2253 { 2254 "alias": "/OK/", 2255 "color": "#37872D" 2256 }, 2257 { 2258 "alias": "error", 2259 "color": "#C4162A" 2260 } 2261 ], 2262 "spaceLength": 10, 2263 "span": 4, 2264 "stack": true, 2265 "steppedLine": false, 2266 "targets": [ 2267 { 2268 "expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))", 2269 "format": "time_series", 2270 "intervalFactor": 2, 2271 "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", 2272 "step": 10 2273 } 2274 ], 2275 "thresholds": [ ], 2276 "timeFrom": null, 2277 "timeShift": null, 2278 "title": "Rate", 2279 "tooltip": { 2280 "shared": false, 2281 "sort": 0, 2282 "value_type": "individual" 2283 }, 2284 "type": "graph", 2285 "xaxis": { 2286 "buckets": null, 2287 "mode": "time", 2288 "name": null, 2289 "show": true, 2290 "values": [ ] 2291 }, 2292 "yaxes": [ 2293 { 2294 "format": "short", 2295 "label": null, 2296 "logBase": 1, 2297 "max": null, 2298 "min": 0, 2299 "show": true 2300 }, 2301 { 2302 "format": "short", 2303 "label": null, 2304 "logBase": 1, 2305 "max": null, 2306 "min": null, 2307 "show": false 2308 } 2309 ] 2310 }, 2311 { 2312 "aliasColors": { 2313 "error": "#E24D42" 2314 }, 2315 "bars": false, 2316 "dashLength": 10, 2317 "dashes": false, 2318 "datasource": "$datasource", 2319 "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", 2320 "fill": 10, 2321 "id": 25, 2322 "legend": { 2323 "avg": false, 2324 "current": false, 2325 "max": false, 2326 "min": false, 2327 "show": true, 2328 "total": false, 2329 "values": false 2330 }, 2331 "lines": true, 2332 "linewidth": 0, 2333 "links": [ ], 2334 "nullPointMode": "null as zero", 2335 "percentage": false, 2336 "pointradius": 5, 2337 "points": false, 2338 "renderer": "flot", 2339 "seriesOverrides": [ ], 2340 "spaceLength": 10, 2341 "span": 4, 2342 "stack": true, 2343 "steppedLine": false, 2344 "targets": [ 2345 { 2346 "expr": "sum by (job, grpc_code) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=~\"$job\", grpc_type=\"server_stream\"}[$interval])) / ignoring (grpc_code) group_left() sum by (job) (rate(grpc_server_handled_total{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))", 2347 "format": "time_series", 2348 "intervalFactor": 2, 2349 "step": 10 2350 } 2351 ], 2352 "thresholds": [ ], 2353 "timeFrom": null, 2354 "timeShift": null, 2355 "title": "Errors", 2356 "tooltip": { 2357 "shared": false, 2358 "sort": 0, 2359 "value_type": "individual" 2360 }, 2361 "type": "graph", 2362 "xaxis": { 2363 "buckets": null, 2364 "mode": "time", 2365 "name": null, 2366 "show": true, 2367 "values": [ ] 2368 }, 2369 "yaxes": [ 2370 { 2371 "format": "percentunit", 2372 "label": null, 2373 "logBase": 1, 2374 "max": null, 2375 "min": 0, 2376 "show": true 2377 }, 2378 { 2379 "format": "short", 2380 "label": null, 2381 "logBase": 1, 2382 "max": null, 2383 "min": null, 2384 "show": false 2385 } 2386 ] 2387 }, 2388 { 2389 "aliasColors": { }, 2390 "bars": false, 2391 "dashLength": 10, 2392 "dashes": false, 2393 "datasource": "$datasource", 2394 "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", 2395 "fill": 1, 2396 "id": 26, 2397 "legend": { 2398 "avg": false, 2399 "current": false, 2400 "max": false, 2401 "min": false, 2402 "show": true, 2403 "total": false, 2404 "values": false 2405 }, 2406 "lines": true, 2407 "linewidth": 1, 2408 "links": [ ], 2409 "nullPointMode": "null as zero", 2410 "percentage": false, 2411 "pointradius": 5, 2412 "points": false, 2413 "renderer": "flot", 2414 "seriesOverrides": [ 2415 { 2416 "alias": "p99", 2417 "color": "#FA6400", 2418 "fill": 1, 2419 "fillGradient": 1 2420 }, 2421 { 2422 "alias": "p90", 2423 "color": "#E0B400", 2424 "fill": 1, 2425 "fillGradient": 1 2426 }, 2427 { 2428 "alias": "p50", 2429 "color": "#37872D", 2430 "fill": 10, 2431 "fillGradient": 0 2432 } 2433 ], 2434 "spaceLength": 10, 2435 "span": 4, 2436 "stack": false, 2437 "steppedLine": false, 2438 "targets": [ 2439 { 2440 "expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1", 2441 "format": "time_series", 2442 "intervalFactor": 2, 2443 "legendFormat": "p50 {{job}}", 2444 "logBase": 10, 2445 "max": null, 2446 "min": null, 2447 "step": 10 2448 }, 2449 { 2450 "expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1", 2451 "format": "time_series", 2452 "intervalFactor": 2, 2453 "legendFormat": "p90 {{job}}", 2454 "logBase": 10, 2455 "max": null, 2456 "min": null, 2457 "step": 10 2458 }, 2459 { 2460 "expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=~\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1", 2461 "format": "time_series", 2462 "intervalFactor": 2, 2463 "legendFormat": "p99 {{job}}", 2464 "logBase": 10, 2465 "max": null, 2466 "min": null, 2467 "step": 10 2468 } 2469 ], 2470 "thresholds": [ ], 2471 "timeFrom": null, 2472 "timeShift": null, 2473 "title": "Duration", 2474 "tooltip": { 2475 "shared": false, 2476 "sort": 0, 2477 "value_type": "individual" 2478 }, 2479 "type": "graph", 2480 "xaxis": { 2481 "buckets": null, 2482 "mode": "time", 2483 "name": null, 2484 "show": true, 2485 "values": [ ] 2486 }, 2487 "yaxes": [ 2488 { 2489 "format": "s", 2490 "label": null, 2491 "logBase": 1, 2492 "max": null, 2493 "min": 0, 2494 "show": true 2495 }, 2496 { 2497 "format": "short", 2498 "label": null, 2499 "logBase": 1, 2500 "max": null, 2501 "min": null, 2502 "show": false 2503 } 2504 ] 2505 } 2506 ], 2507 "repeat": null, 2508 "repeatIteration": null, 2509 "repeatRowId": null, 2510 "showTitle": true, 2511 "title": "READ - gRPC (Stream)", 2512 "titleSize": "h6" 2513 }, 2514 { 2515 "collapse": false, 2516 "height": "250px", 2517 "panels": [ 2518 { 2519 "aliasColors": { }, 2520 "bars": false, 2521 "dashLength": 10, 2522 "dashes": false, 2523 "datasource": "$datasource", 2524 "description": "Shows the relative time of last successful upload to the object-store bucket.", 2525 "fill": 1, 2526 "id": 27, 2527 "legend": { 2528 "avg": false, 2529 "current": false, 2530 "max": false, 2531 "min": false, 2532 "show": true, 2533 "total": false, 2534 "values": false 2535 }, 2536 "lines": true, 2537 "linewidth": 1, 2538 "links": [ ], 2539 "nullPointMode": "null as zero", 2540 "percentage": false, 2541 "pointradius": 5, 2542 "points": false, 2543 "renderer": "flot", 2544 "seriesOverrides": [ ], 2545 "spaceLength": 10, 2546 "span": 12, 2547 "stack": false, 2548 "steppedLine": false, 2549 "styles": [ 2550 { 2551 "alias": "Time", 2552 "dateFormat": "YYYY-MM-DD HH:mm:ss", 2553 "pattern": "Time", 2554 "type": "hidden" 2555 }, 2556 { 2557 "alias": "Uploaded Ago", 2558 "colorMode": null, 2559 "colors": [ ], 2560 "dateFormat": "YYYY-MM-DD HH:mm:ss", 2561 "decimals": 2, 2562 "link": false, 2563 "linkTargetBlank": false, 2564 "linkTooltip": "Drill down", 2565 "linkUrl": "", 2566 "pattern": "Value", 2567 "thresholds": [ ], 2568 "type": "number", 2569 "unit": "s" 2570 }, 2571 { 2572 "alias": "", 2573 "colorMode": null, 2574 "colors": [ ], 2575 "dateFormat": "YYYY-MM-DD HH:mm:ss", 2576 "decimals": 2, 2577 "pattern": "/.*/", 2578 "thresholds": [ ], 2579 "type": "string", 2580 "unit": "short" 2581 } 2582 ], 2583 "targets": [ 2584 { 2585 "expr": "time() - max by (job, bucket) (thanos_objstore_bucket_last_successful_upload_time{job=~\"$job\"})", 2586 "format": "table", 2587 "instant": true, 2588 "intervalFactor": 2, 2589 "legendFormat": "", 2590 "refId": "A", 2591 "step": 10 2592 } 2593 ], 2594 "thresholds": [ ], 2595 "timeFrom": null, 2596 "timeShift": null, 2597 "title": "Successful Upload", 2598 "tooltip": { 2599 "shared": false, 2600 "sort": 0, 2601 "value_type": "individual" 2602 }, 2603 "transform": "table", 2604 "type": "table", 2605 "xaxis": { 2606 "buckets": null, 2607 "mode": "time", 2608 "name": null, 2609 "show": true, 2610 "values": [ ] 2611 }, 2612 "yaxes": [ 2613 { 2614 "format": "short", 2615 "label": null, 2616 "logBase": 1, 2617 "max": null, 2618 "min": 0, 2619 "show": true 2620 }, 2621 { 2622 "format": "short", 2623 "label": null, 2624 "logBase": 1, 2625 "max": null, 2626 "min": null, 2627 "show": false 2628 } 2629 ] 2630 } 2631 ], 2632 "repeat": null, 2633 "repeatIteration": null, 2634 "repeatRowId": null, 2635 "showTitle": true, 2636 "title": "Last Updated", 2637 "titleSize": "h6" 2638 }, 2639 { 2640 "collapse": true, 2641 "height": "250px", 2642 "panels": [ 2643 { 2644 "aliasColors": { }, 2645 "bars": false, 2646 "dashLength": 10, 2647 "dashes": false, 2648 "datasource": "$datasource", 2649 "fill": 1, 2650 "id": 28, 2651 "legend": { 2652 "avg": false, 2653 "current": false, 2654 "max": false, 2655 "min": false, 2656 "show": true, 2657 "total": false, 2658 "values": false 2659 }, 2660 "lines": true, 2661 "linewidth": 1, 2662 "links": [ ], 2663 "nullPointMode": "null as zero", 2664 "percentage": false, 2665 "pointradius": 5, 2666 "points": false, 2667 "renderer": "flot", 2668 "seriesOverrides": [ ], 2669 "spaceLength": 10, 2670 "span": 4, 2671 "stack": false, 2672 "steppedLine": false, 2673 "targets": [ 2674 { 2675 "expr": "go_memstats_alloc_bytes{job=~\"$job\"}", 2676 "format": "time_series", 2677 "intervalFactor": 2, 2678 "legendFormat": "alloc all {{instance}}", 2679 "legendLink": null, 2680 "step": 10 2681 }, 2682 { 2683 "expr": "go_memstats_heap_alloc_bytes{job=~\"$job\"}", 2684 "format": "time_series", 2685 "intervalFactor": 2, 2686 "legendFormat": "alloc heap {{instance}}", 2687 "legendLink": null, 2688 "step": 10 2689 }, 2690 { 2691 "expr": "rate(go_memstats_alloc_bytes_total{job=~\"$job\"}[30s])", 2692 "format": "time_series", 2693 "intervalFactor": 2, 2694 "legendFormat": "alloc rate all {{instance}}", 2695 "legendLink": null, 2696 "step": 10 2697 }, 2698 { 2699 "expr": "rate(go_memstats_heap_alloc_bytes{job=~\"$job\"}[30s])", 2700 "format": "time_series", 2701 "intervalFactor": 2, 2702 "legendFormat": "alloc rate heap {{instance}}", 2703 "legendLink": null, 2704 "step": 10 2705 }, 2706 { 2707 "expr": "go_memstats_stack_inuse_bytes{job=~\"$job\"}", 2708 "format": "time_series", 2709 "intervalFactor": 2, 2710 "legendFormat": "inuse heap {{instance}}", 2711 "legendLink": null, 2712 "step": 10 2713 }, 2714 { 2715 "expr": "go_memstats_heap_inuse_bytes{job=~\"$job\"}", 2716 "format": "time_series", 2717 "intervalFactor": 2, 2718 "legendFormat": "inuse stack {{instance}}", 2719 "legendLink": null, 2720 "step": 10 2721 } 2722 ], 2723 "thresholds": [ ], 2724 "timeFrom": null, 2725 "timeShift": null, 2726 "title": "Memory Used", 2727 "tooltip": { 2728 "shared": false, 2729 "sort": 0, 2730 "value_type": "individual" 2731 }, 2732 "type": "graph", 2733 "xaxis": { 2734 "buckets": null, 2735 "mode": "time", 2736 "name": null, 2737 "show": true, 2738 "values": [ ] 2739 }, 2740 "yaxes": [ 2741 { 2742 "format": "bytes", 2743 "label": null, 2744 "logBase": 1, 2745 "max": null, 2746 "min": 0, 2747 "show": true 2748 }, 2749 { 2750 "format": "short", 2751 "label": null, 2752 "logBase": 1, 2753 "max": null, 2754 "min": null, 2755 "show": false 2756 } 2757 ] 2758 }, 2759 { 2760 "aliasColors": { }, 2761 "bars": false, 2762 "dashLength": 10, 2763 "dashes": false, 2764 "datasource": "$datasource", 2765 "fill": 1, 2766 "id": 29, 2767 "legend": { 2768 "avg": false, 2769 "current": false, 2770 "max": false, 2771 "min": false, 2772 "show": true, 2773 "total": false, 2774 "values": false 2775 }, 2776 "lines": true, 2777 "linewidth": 1, 2778 "links": [ ], 2779 "nullPointMode": "null as zero", 2780 "percentage": false, 2781 "pointradius": 5, 2782 "points": false, 2783 "renderer": "flot", 2784 "seriesOverrides": [ ], 2785 "spaceLength": 10, 2786 "span": 4, 2787 "stack": false, 2788 "steppedLine": false, 2789 "targets": [ 2790 { 2791 "expr": "go_goroutines{job=~\"$job\"}", 2792 "format": "time_series", 2793 "intervalFactor": 2, 2794 "legendFormat": "{{instance}}", 2795 "legendLink": null, 2796 "step": 10 2797 } 2798 ], 2799 "thresholds": [ ], 2800 "timeFrom": null, 2801 "timeShift": null, 2802 "title": "Goroutines", 2803 "tooltip": { 2804 "shared": false, 2805 "sort": 0, 2806 "value_type": "individual" 2807 }, 2808 "type": "graph", 2809 "xaxis": { 2810 "buckets": null, 2811 "mode": "time", 2812 "name": null, 2813 "show": true, 2814 "values": [ ] 2815 }, 2816 "yaxes": [ 2817 { 2818 "format": "short", 2819 "label": null, 2820 "logBase": 1, 2821 "max": null, 2822 "min": 0, 2823 "show": true 2824 }, 2825 { 2826 "format": "short", 2827 "label": null, 2828 "logBase": 1, 2829 "max": null, 2830 "min": null, 2831 "show": false 2832 } 2833 ] 2834 }, 2835 { 2836 "aliasColors": { }, 2837 "bars": false, 2838 "dashLength": 10, 2839 "dashes": false, 2840 "datasource": "$datasource", 2841 "fill": 1, 2842 "id": 30, 2843 "legend": { 2844 "avg": false, 2845 "current": false, 2846 "max": false, 2847 "min": false, 2848 "show": true, 2849 "total": false, 2850 "values": false 2851 }, 2852 "lines": true, 2853 "linewidth": 1, 2854 "links": [ ], 2855 "nullPointMode": "null as zero", 2856 "percentage": false, 2857 "pointradius": 5, 2858 "points": false, 2859 "renderer": "flot", 2860 "seriesOverrides": [ ], 2861 "spaceLength": 10, 2862 "span": 4, 2863 "stack": false, 2864 "steppedLine": false, 2865 "targets": [ 2866 { 2867 "expr": "go_gc_duration_seconds{job=~\"$job\"}", 2868 "format": "time_series", 2869 "intervalFactor": 2, 2870 "legendFormat": "{{quantile}} {{instance}}", 2871 "legendLink": null, 2872 "step": 10 2873 } 2874 ], 2875 "thresholds": [ ], 2876 "timeFrom": null, 2877 "timeShift": null, 2878 "title": "GC Time Quantiles", 2879 "tooltip": { 2880 "shared": false, 2881 "sort": 0, 2882 "value_type": "individual" 2883 }, 2884 "type": "graph", 2885 "xaxis": { 2886 "buckets": null, 2887 "mode": "time", 2888 "name": null, 2889 "show": true, 2890 "values": [ ] 2891 }, 2892 "yaxes": [ 2893 { 2894 "format": "short", 2895 "label": null, 2896 "logBase": 1, 2897 "max": null, 2898 "min": 0, 2899 "show": true 2900 }, 2901 { 2902 "format": "short", 2903 "label": null, 2904 "logBase": 1, 2905 "max": null, 2906 "min": null, 2907 "show": false 2908 } 2909 ] 2910 } 2911 ], 2912 "repeat": null, 2913 "repeatIteration": null, 2914 "repeatRowId": null, 2915 "showTitle": true, 2916 "title": "Resources", 2917 "titleSize": "h6" 2918 } 2919 ], 2920 "schemaVersion": 14, 2921 "style": "dark", 2922 "tags": [ 2923 "thanos-mixin" 2924 ], 2925 "templating": { 2926 "list": [ 2927 { 2928 "current": { 2929 "text": "default", 2930 "value": "default" 2931 }, 2932 "hide": 0, 2933 "label": null, 2934 "name": "datasource", 2935 "options": [ ], 2936 "query": "prometheus", 2937 "refresh": 1, 2938 "regex": "", 2939 "type": "datasource" 2940 }, 2941 { 2942 "allValue": null, 2943 "current": { 2944 "text": "all", 2945 "value": "$__all" 2946 }, 2947 "datasource": "$datasource", 2948 "hide": 0, 2949 "includeAll": true, 2950 "label": "tenant", 2951 "multi": false, 2952 "name": "tenant", 2953 "options": [ ], 2954 "query": "label_values(http_requests_total{job=~\"$job\", tenant!=\"\"}, tenant)", 2955 "refresh": 1, 2956 "regex": "", 2957 "sort": 2, 2958 "tagValuesQuery": "", 2959 "tags": [ ], 2960 "tagsQuery": "", 2961 "type": "query", 2962 "useTags": false 2963 }, 2964 { 2965 "auto": true, 2966 "auto_count": 300, 2967 "auto_min": "10s", 2968 "current": { 2969 "text": "5m", 2970 "value": "5m" 2971 }, 2972 "hide": 0, 2973 "label": "interval", 2974 "name": "interval", 2975 "query": "5m,10m,30m,1h,6h,12h", 2976 "refresh": 2, 2977 "type": "interval" 2978 }, 2979 { 2980 "allValue": null, 2981 "current": { 2982 "text": "all", 2983 "value": "$__all" 2984 }, 2985 "datasource": "$datasource", 2986 "hide": 0, 2987 "includeAll": true, 2988 "label": "job", 2989 "multi": false, 2990 "name": "job", 2991 "options": [ ], 2992 "query": "label_values(up{job=~\".*thanos-receive.*\"}, job)", 2993 "refresh": 1, 2994 "regex": "", 2995 "sort": 2, 2996 "tagValuesQuery": "", 2997 "tags": [ ], 2998 "tagsQuery": "", 2999 "type": "query", 3000 "useTags": false 3001 } 3002 ] 3003 }, 3004 "time": { 3005 "from": "now-1h", 3006 "to": "now" 3007 }, 3008 "timepicker": { 3009 "refresh_intervals": [ 3010 "5s", 3011 "10s", 3012 "30s", 3013 "1m", 3014 "5m", 3015 "15m", 3016 "30m", 3017 "1h", 3018 "2h", 3019 "1d" 3020 ], 3021 "time_options": [ 3022 "5m", 3023 "15m", 3024 "1h", 3025 "6h", 3026 "12h", 3027 "24h", 3028 "2d", 3029 "7d", 3030 "30d" 3031 ] 3032 }, 3033 "timezone": "UTC", 3034 "title": "Thanos / Receive", 3035 "uid": "916a852b00ccc5ed81056644718fa4fb", 3036 "version": 0 3037 }