github.com/thanos-io/thanos@v0.32.5/examples/dashboards/compact.json (about) 1 { 2 "annotations": { 3 "list": [ ] 4 }, 5 "editable": true, 6 "gnetId": null, 7 "graphTooltip": 0, 8 "hideControls": false, 9 "links": [ ], 10 "refresh": "10s", 11 "rows": [ 12 { 13 "collapse": false, 14 "height": "250px", 15 "panels": [ 16 { 17 "aliasColors": { }, 18 "bars": false, 19 "dashLength": 10, 20 "dashes": false, 21 "datasource": "$datasource", 22 "description": "Shows rate of execution for compactions against blocks that are stored in the bucket by compaction resolution.", 23 "fill": 10, 24 "id": 1, 25 "legend": { 26 "avg": false, 27 "current": false, 28 "max": false, 29 "min": false, 30 "show": true, 31 "total": false, 32 "values": false 33 }, 34 "lines": true, 35 "linewidth": 0, 36 "links": [ ], 37 "nullPointMode": "null as zero", 38 "percentage": false, 39 "pointradius": 5, 40 "points": false, 41 "renderer": "flot", 42 "seriesOverrides": [ ], 43 "spaceLength": 10, 44 "span": 6, 45 "stack": true, 46 "steppedLine": false, 47 "targets": [ 48 { 49 "expr": "sum by (job, resolution) (rate(thanos_compact_group_compactions_total{job=~\"$job\"}[$__rate_interval]))", 50 "format": "time_series", 51 "intervalFactor": 2, 52 "legendFormat": "compaction {{job}} {{resolution}}", 53 "legendLink": null, 54 "step": 10 55 } 56 ], 57 "thresholds": [ ], 58 "timeFrom": null, 59 "timeShift": null, 60 "title": "Rate", 61 "tooltip": { 62 "shared": false, 63 "sort": 0, 64 "value_type": "individual" 65 }, 66 "type": "graph", 67 "xaxis": { 68 "buckets": null, 69 "mode": "time", 70 "name": null, 71 "show": true, 72 "values": [ ] 73 }, 74 "yaxes": [ 75 { 76 "format": "short", 77 "label": null, 78 "logBase": 1, 79 "max": null, 80 "min": 0, 81 "show": true 82 }, 83 { 84 "format": "short", 85 "label": null, 86 "logBase": 1, 87 "max": null, 88 "min": null, 89 "show": false 90 } 91 ] 92 }, 93 { 94 "aliasColors": { 95 "error": "#E24D42" 96 }, 97 "bars": false, 98 "dashLength": 10, 99 "dashes": false, 100 "datasource": "$datasource", 101 "description": "Shows ratio of errors compared to the total number of executed compactions against blocks that are stored in the bucket.", 102 "fill": 10, 103 "id": 2, 104 "legend": { 105 "avg": false, 106 "current": false, 107 "max": false, 108 "min": false, 109 "show": true, 110 "total": false, 111 "values": false 112 }, 113 "lines": true, 114 "linewidth": 0, 115 "links": [ ], 116 "nullPointMode": "null as zero", 117 "percentage": false, 118 "pointradius": 5, 119 "points": false, 120 "renderer": "flot", 121 "seriesOverrides": [ ], 122 "spaceLength": 10, 123 "span": 6, 124 "stack": true, 125 "steppedLine": false, 126 "targets": [ 127 { 128 "expr": "sum by (job) (rate(thanos_compact_group_compactions_failures_total{job=~\"$job\"}[$interval])) / sum by (job) (rate(thanos_compact_group_compactions_total{job=~\"$job\"}[$interval]))", 129 "format": "time_series", 130 "intervalFactor": 2, 131 "legendFormat": "error", 132 "step": 10 133 } 134 ], 135 "thresholds": [ ], 136 "timeFrom": null, 137 "timeShift": null, 138 "title": "Errors", 139 "tooltip": { 140 "shared": false, 141 "sort": 0, 142 "value_type": "individual" 143 }, 144 "type": "graph", 145 "xaxis": { 146 "buckets": null, 147 "mode": "time", 148 "name": null, 149 "show": true, 150 "values": [ ] 151 }, 152 "yaxes": [ 153 { 154 "format": "percentunit", 155 "label": null, 156 "logBase": 1, 157 "max": null, 158 "min": 0, 159 "show": true 160 }, 161 { 162 "format": "short", 163 "label": null, 164 "logBase": 1, 165 "max": null, 166 "min": null, 167 "show": false 168 } 169 ] 170 } 171 ], 172 "repeat": null, 173 "repeatIteration": null, 174 "repeatRowId": null, 175 "showTitle": true, 176 "title": "Group Compaction", 177 "titleSize": "h6" 178 }, 179 { 180 "collapse": false, 181 "height": "250px", 182 "panels": [ 183 { 184 "aliasColors": { }, 185 "bars": false, 186 "dashLength": 10, 187 "dashes": false, 188 "datasource": "$datasource", 189 "description": "Shows rate of execution for downsampling against blocks that are stored in the bucket by compaction resolution.", 190 "fill": 10, 191 "id": 3, 192 "legend": { 193 "avg": false, 194 "current": false, 195 "max": false, 196 "min": false, 197 "show": true, 198 "total": false, 199 "values": false 200 }, 201 "lines": true, 202 "linewidth": 0, 203 "links": [ ], 204 "nullPointMode": "null as zero", 205 "percentage": false, 206 "pointradius": 5, 207 "points": false, 208 "renderer": "flot", 209 "seriesOverrides": [ ], 210 "spaceLength": 10, 211 "span": 6, 212 "stack": true, 213 "steppedLine": false, 214 "targets": [ 215 { 216 "expr": "sum by (job, resolution) (rate(thanos_compact_downsample_total{job=~\"$job\"}[$__rate_interval]))", 217 "format": "time_series", 218 "intervalFactor": 2, 219 "legendFormat": "downsample {{job}} {{resolution}}", 220 "legendLink": null, 221 "step": 10 222 } 223 ], 224 "thresholds": [ ], 225 "timeFrom": null, 226 "timeShift": null, 227 "title": "Rate", 228 "tooltip": { 229 "shared": false, 230 "sort": 0, 231 "value_type": "individual" 232 }, 233 "type": "graph", 234 "xaxis": { 235 "buckets": null, 236 "mode": "time", 237 "name": null, 238 "show": true, 239 "values": [ ] 240 }, 241 "yaxes": [ 242 { 243 "format": "short", 244 "label": null, 245 "logBase": 1, 246 "max": null, 247 "min": 0, 248 "show": true 249 }, 250 { 251 "format": "short", 252 "label": null, 253 "logBase": 1, 254 "max": null, 255 "min": null, 256 "show": false 257 } 258 ] 259 }, 260 { 261 "aliasColors": { 262 "error": "#E24D42" 263 }, 264 "bars": false, 265 "dashLength": 10, 266 "dashes": false, 267 "datasource": "$datasource", 268 "description": "Shows ratio of errors compared to the total number of executed downsampling against blocks that are stored in the bucket.", 269 "fill": 10, 270 "id": 4, 271 "legend": { 272 "avg": false, 273 "current": false, 274 "max": false, 275 "min": false, 276 "show": true, 277 "total": false, 278 "values": false 279 }, 280 "lines": true, 281 "linewidth": 0, 282 "links": [ ], 283 "nullPointMode": "null as zero", 284 "percentage": false, 285 "pointradius": 5, 286 "points": false, 287 "renderer": "flot", 288 "seriesOverrides": [ ], 289 "spaceLength": 10, 290 "span": 6, 291 "stack": true, 292 "steppedLine": false, 293 "targets": [ 294 { 295 "expr": "sum by (job) (rate(thanos_compact_downsample_failed_total{job=~\"$job\"}[$interval])) / sum by (job) (rate(thanos_compact_downsample_total{job=~\"$job\"}[$interval]))", 296 "format": "time_series", 297 "intervalFactor": 2, 298 "legendFormat": "error", 299 "step": 10 300 } 301 ], 302 "thresholds": [ ], 303 "timeFrom": null, 304 "timeShift": null, 305 "title": "Errors", 306 "tooltip": { 307 "shared": false, 308 "sort": 0, 309 "value_type": "individual" 310 }, 311 "type": "graph", 312 "xaxis": { 313 "buckets": null, 314 "mode": "time", 315 "name": null, 316 "show": true, 317 "values": [ ] 318 }, 319 "yaxes": [ 320 { 321 "format": "percentunit", 322 "label": null, 323 "logBase": 1, 324 "max": null, 325 "min": 0, 326 "show": true 327 }, 328 { 329 "format": "short", 330 "label": null, 331 "logBase": 1, 332 "max": null, 333 "min": null, 334 "show": false 335 } 336 ] 337 } 338 ], 339 "repeat": null, 340 "repeatIteration": null, 341 "repeatRowId": null, 342 "showTitle": true, 343 "title": "Downsample", 344 "titleSize": "h6" 345 }, 346 { 347 "collapse": false, 348 "height": "250px", 349 "panels": [ 350 { 351 "aliasColors": { }, 352 "bars": false, 353 "dashLength": 10, 354 "dashes": false, 355 "datasource": "$datasource", 356 "description": "Shows rate of execution for removals of blocks if their data is available as part of a block with a higher compaction level.", 357 "fill": 10, 358 "id": 5, 359 "legend": { 360 "avg": false, 361 "current": false, 362 "max": false, 363 "min": false, 364 "show": true, 365 "total": false, 366 "values": false 367 }, 368 "lines": true, 369 "linewidth": 0, 370 "links": [ ], 371 "nullPointMode": "null as zero", 372 "percentage": false, 373 "pointradius": 5, 374 "points": false, 375 "renderer": "flot", 376 "seriesOverrides": [ ], 377 "spaceLength": 10, 378 "span": 4, 379 "stack": true, 380 "steppedLine": false, 381 "targets": [ 382 { 383 "expr": "sum by (job) (rate(thanos_compact_garbage_collection_total{job=~\"$job\"}[$__rate_interval]))", 384 "format": "time_series", 385 "intervalFactor": 2, 386 "legendFormat": "garbage collection {{job}}", 387 "legendLink": null, 388 "step": 10 389 } 390 ], 391 "thresholds": [ ], 392 "timeFrom": null, 393 "timeShift": null, 394 "title": "Rate", 395 "tooltip": { 396 "shared": false, 397 "sort": 0, 398 "value_type": "individual" 399 }, 400 "type": "graph", 401 "xaxis": { 402 "buckets": null, 403 "mode": "time", 404 "name": null, 405 "show": true, 406 "values": [ ] 407 }, 408 "yaxes": [ 409 { 410 "format": "short", 411 "label": null, 412 "logBase": 1, 413 "max": null, 414 "min": 0, 415 "show": true 416 }, 417 { 418 "format": "short", 419 "label": null, 420 "logBase": 1, 421 "max": null, 422 "min": null, 423 "show": false 424 } 425 ] 426 }, 427 { 428 "aliasColors": { 429 "error": "#E24D42" 430 }, 431 "bars": false, 432 "dashLength": 10, 433 "dashes": false, 434 "datasource": "$datasource", 435 "description": "Shows ratio of errors compared to the total number of executed garbage collections.", 436 "fill": 10, 437 "id": 6, 438 "legend": { 439 "avg": false, 440 "current": false, 441 "max": false, 442 "min": false, 443 "show": true, 444 "total": false, 445 "values": false 446 }, 447 "lines": true, 448 "linewidth": 0, 449 "links": [ ], 450 "nullPointMode": "null as zero", 451 "percentage": false, 452 "pointradius": 5, 453 "points": false, 454 "renderer": "flot", 455 "seriesOverrides": [ ], 456 "spaceLength": 10, 457 "span": 4, 458 "stack": true, 459 "steppedLine": false, 460 "targets": [ 461 { 462 "expr": "sum by (job) (rate(thanos_compact_garbage_collection_failures_total{job=~\"$job\"}[$interval])) / sum by (job) (rate(thanos_compact_garbage_collection_total{job=~\"$job\"}[$interval]))", 463 "format": "time_series", 464 "intervalFactor": 2, 465 "legendFormat": "error", 466 "step": 10 467 } 468 ], 469 "thresholds": [ ], 470 "timeFrom": null, 471 "timeShift": null, 472 "title": "Errors", 473 "tooltip": { 474 "shared": false, 475 "sort": 0, 476 "value_type": "individual" 477 }, 478 "type": "graph", 479 "xaxis": { 480 "buckets": null, 481 "mode": "time", 482 "name": null, 483 "show": true, 484 "values": [ ] 485 }, 486 "yaxes": [ 487 { 488 "format": "percentunit", 489 "label": null, 490 "logBase": 1, 491 "max": null, 492 "min": 0, 493 "show": true 494 }, 495 { 496 "format": "short", 497 "label": null, 498 "logBase": 1, 499 "max": null, 500 "min": null, 501 "show": false 502 } 503 ] 504 }, 505 { 506 "aliasColors": { }, 507 "bars": false, 508 "dashLength": 10, 509 "dashes": false, 510 "datasource": "$datasource", 511 "description": "Shows how long has it taken to execute garbage collection in quantiles.", 512 "fill": 1, 513 "id": 7, 514 "legend": { 515 "avg": false, 516 "current": false, 517 "max": false, 518 "min": false, 519 "show": true, 520 "total": false, 521 "values": false 522 }, 523 "lines": true, 524 "linewidth": 1, 525 "links": [ ], 526 "nullPointMode": "null as zero", 527 "percentage": false, 528 "pointradius": 5, 529 "points": false, 530 "renderer": "flot", 531 "seriesOverrides": [ 532 { 533 "alias": "p99", 534 "color": "#FA6400", 535 "fill": 1, 536 "fillGradient": 1 537 }, 538 { 539 "alias": "p90", 540 "color": "#E0B400", 541 "fill": 1, 542 "fillGradient": 1 543 }, 544 { 545 "alias": "p50", 546 "color": "#37872D", 547 "fill": 10, 548 "fillGradient": 0 549 } 550 ], 551 "spaceLength": 10, 552 "span": 4, 553 "stack": false, 554 "steppedLine": false, 555 "targets": [ 556 { 557 "expr": "histogram_quantile(0.50, sum by (job, le) (rate(thanos_compact_garbage_collection_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1", 558 "format": "time_series", 559 "intervalFactor": 2, 560 "legendFormat": "p50 {{job}}", 561 "logBase": 10, 562 "max": null, 563 "min": null, 564 "step": 10 565 }, 566 { 567 "expr": "histogram_quantile(0.90, sum by (job, le) (rate(thanos_compact_garbage_collection_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1", 568 "format": "time_series", 569 "intervalFactor": 2, 570 "legendFormat": "p90 {{job}}", 571 "logBase": 10, 572 "max": null, 573 "min": null, 574 "step": 10 575 }, 576 { 577 "expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_compact_garbage_collection_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1", 578 "format": "time_series", 579 "intervalFactor": 2, 580 "legendFormat": "p99 {{job}}", 581 "logBase": 10, 582 "max": null, 583 "min": null, 584 "step": 10 585 } 586 ], 587 "thresholds": [ ], 588 "timeFrom": null, 589 "timeShift": null, 590 "title": "Duration", 591 "tooltip": { 592 "shared": false, 593 "sort": 0, 594 "value_type": "individual" 595 }, 596 "type": "graph", 597 "xaxis": { 598 "buckets": null, 599 "mode": "time", 600 "name": null, 601 "show": true, 602 "values": [ ] 603 }, 604 "yaxes": [ 605 { 606 "format": "s", 607 "label": null, 608 "logBase": 1, 609 "max": null, 610 "min": 0, 611 "show": true 612 }, 613 { 614 "format": "short", 615 "label": null, 616 "logBase": 1, 617 "max": null, 618 "min": null, 619 "show": false 620 } 621 ] 622 } 623 ], 624 "repeat": null, 625 "repeatIteration": null, 626 "repeatRowId": null, 627 "showTitle": true, 628 "title": "Garbage Collection", 629 "titleSize": "h6" 630 }, 631 { 632 "collapse": false, 633 "height": "250px", 634 "panels": [ 635 { 636 "aliasColors": { }, 637 "bars": false, 638 "dashLength": 10, 639 "dashes": false, 640 "datasource": "$datasource", 641 "description": "Shows deletion rate of blocks already marked for deletion.", 642 "fill": 10, 643 "id": 8, 644 "legend": { 645 "avg": false, 646 "current": false, 647 "max": false, 648 "min": false, 649 "show": true, 650 "total": false, 651 "values": false 652 }, 653 "lines": true, 654 "linewidth": 0, 655 "links": [ ], 656 "nullPointMode": "null as zero", 657 "percentage": false, 658 "pointradius": 5, 659 "points": false, 660 "renderer": "flot", 661 "seriesOverrides": [ ], 662 "spaceLength": 10, 663 "span": 4, 664 "stack": true, 665 "steppedLine": false, 666 "targets": [ 667 { 668 "expr": "sum by (job) (rate(thanos_compact_blocks_cleaned_total{job=~\"$job\"}[$__rate_interval]))", 669 "format": "time_series", 670 "intervalFactor": 2, 671 "legendFormat": "Blocks cleanup {{job}}", 672 "legendLink": null, 673 "step": 10 674 } 675 ], 676 "thresholds": [ ], 677 "timeFrom": null, 678 "timeShift": null, 679 "title": "Deletion Rate", 680 "tooltip": { 681 "shared": false, 682 "sort": 0, 683 "value_type": "individual" 684 }, 685 "type": "graph", 686 "xaxis": { 687 "buckets": null, 688 "mode": "time", 689 "name": null, 690 "show": true, 691 "values": [ ] 692 }, 693 "yaxes": [ 694 { 695 "format": "short", 696 "label": null, 697 "logBase": 1, 698 "max": null, 699 "min": 0, 700 "show": true 701 }, 702 { 703 "format": "short", 704 "label": null, 705 "logBase": 1, 706 "max": null, 707 "min": null, 708 "show": false 709 } 710 ] 711 }, 712 { 713 "aliasColors": { }, 714 "bars": false, 715 "dashLength": 10, 716 "dashes": false, 717 "datasource": "$datasource", 718 "description": "Shows deletion failures rate of blocks already marked for deletion.", 719 "fill": 1, 720 "id": 9, 721 "legend": { 722 "avg": false, 723 "current": false, 724 "max": false, 725 "min": false, 726 "show": true, 727 "total": false, 728 "values": false 729 }, 730 "lines": true, 731 "linewidth": 1, 732 "links": [ ], 733 "nullPointMode": "null as zero", 734 "percentage": false, 735 "pointradius": 5, 736 "points": false, 737 "renderer": "flot", 738 "seriesOverrides": [ ], 739 "spaceLength": 10, 740 "span": 4, 741 "stack": false, 742 "steppedLine": false, 743 "targets": [ 744 { 745 "expr": "sum by (job) (rate(thanos_compact_block_cleanup_failures_total{job=~\"$job\"}[$__rate_interval]))", 746 "format": "time_series", 747 "intervalFactor": 2, 748 "legendFormat": "Blocks cleanup failures {{job}}", 749 "legendLink": null, 750 "step": 10 751 } 752 ], 753 "thresholds": [ ], 754 "timeFrom": null, 755 "timeShift": null, 756 "title": "Deletion Error Rate", 757 "tooltip": { 758 "shared": false, 759 "sort": 0, 760 "value_type": "individual" 761 }, 762 "type": "graph", 763 "xaxis": { 764 "buckets": null, 765 "mode": "time", 766 "name": null, 767 "show": true, 768 "values": [ ] 769 }, 770 "yaxes": [ 771 { 772 "format": "short", 773 "label": null, 774 "logBase": 1, 775 "max": null, 776 "min": 0, 777 "show": true 778 }, 779 { 780 "format": "short", 781 "label": null, 782 "logBase": 1, 783 "max": null, 784 "min": null, 785 "show": false 786 } 787 ] 788 }, 789 { 790 "aliasColors": { }, 791 "bars": false, 792 "dashLength": 10, 793 "dashes": false, 794 "datasource": "$datasource", 795 "description": "Shows rate at which blocks are marked for deletion (from GC and retention policy).", 796 "fill": 1, 797 "id": 10, 798 "legend": { 799 "avg": false, 800 "current": false, 801 "max": false, 802 "min": false, 803 "show": true, 804 "total": false, 805 "values": false 806 }, 807 "lines": true, 808 "linewidth": 1, 809 "links": [ ], 810 "nullPointMode": "null as zero", 811 "percentage": false, 812 "pointradius": 5, 813 "points": false, 814 "renderer": "flot", 815 "seriesOverrides": [ ], 816 "spaceLength": 10, 817 "span": 4, 818 "stack": false, 819 "steppedLine": false, 820 "targets": [ 821 { 822 "expr": "sum by (job) (rate(thanos_compact_blocks_marked_for_deletion_total{job=~\"$job\"}[$__rate_interval]))", 823 "format": "time_series", 824 "intervalFactor": 2, 825 "legendFormat": "Blocks marked {{job}}", 826 "legendLink": null, 827 "step": 10 828 } 829 ], 830 "thresholds": [ ], 831 "timeFrom": null, 832 "timeShift": null, 833 "title": "Marking Rate", 834 "tooltip": { 835 "shared": false, 836 "sort": 0, 837 "value_type": "individual" 838 }, 839 "type": "graph", 840 "xaxis": { 841 "buckets": null, 842 "mode": "time", 843 "name": null, 844 "show": true, 845 "values": [ ] 846 }, 847 "yaxes": [ 848 { 849 "format": "short", 850 "label": null, 851 "logBase": 1, 852 "max": null, 853 "min": 0, 854 "show": true 855 }, 856 { 857 "format": "short", 858 "label": null, 859 "logBase": 1, 860 "max": null, 861 "min": null, 862 "show": false 863 } 864 ] 865 } 866 ], 867 "repeat": null, 868 "repeatIteration": null, 869 "repeatRowId": null, 870 "showTitle": true, 871 "title": "Blocks deletion", 872 "titleSize": "h6" 873 }, 874 { 875 "collapse": false, 876 "height": "250px", 877 "panels": [ 878 { 879 "aliasColors": { }, 880 "bars": false, 881 "dashLength": 10, 882 "dashes": false, 883 "datasource": "$datasource", 884 "description": "Shows rate of execution for all meta files from blocks in the bucket into the memory.", 885 "fill": 10, 886 "id": 11, 887 "legend": { 888 "avg": false, 889 "current": false, 890 "max": false, 891 "min": false, 892 "show": true, 893 "total": false, 894 "values": false 895 }, 896 "lines": true, 897 "linewidth": 0, 898 "links": [ ], 899 "nullPointMode": "null as zero", 900 "percentage": false, 901 "pointradius": 5, 902 "points": false, 903 "renderer": "flot", 904 "seriesOverrides": [ ], 905 "spaceLength": 10, 906 "span": 4, 907 "stack": true, 908 "steppedLine": false, 909 "targets": [ 910 { 911 "expr": "sum by (job) (rate(thanos_blocks_meta_syncs_total{job=~\"$job\"}[$__rate_interval]))", 912 "format": "time_series", 913 "intervalFactor": 2, 914 "legendFormat": "sync {{job}}", 915 "legendLink": null, 916 "step": 10 917 } 918 ], 919 "thresholds": [ ], 920 "timeFrom": null, 921 "timeShift": null, 922 "title": "Rate", 923 "tooltip": { 924 "shared": false, 925 "sort": 0, 926 "value_type": "individual" 927 }, 928 "type": "graph", 929 "xaxis": { 930 "buckets": null, 931 "mode": "time", 932 "name": null, 933 "show": true, 934 "values": [ ] 935 }, 936 "yaxes": [ 937 { 938 "format": "short", 939 "label": null, 940 "logBase": 1, 941 "max": null, 942 "min": 0, 943 "show": true 944 }, 945 { 946 "format": "short", 947 "label": null, 948 "logBase": 1, 949 "max": null, 950 "min": null, 951 "show": false 952 } 953 ] 954 }, 955 { 956 "aliasColors": { 957 "error": "#E24D42" 958 }, 959 "bars": false, 960 "dashLength": 10, 961 "dashes": false, 962 "datasource": "$datasource", 963 "description": "Shows ratio of errors compared to the total number of executed meta file sync.", 964 "fill": 10, 965 "id": 12, 966 "legend": { 967 "avg": false, 968 "current": false, 969 "max": false, 970 "min": false, 971 "show": true, 972 "total": false, 973 "values": false 974 }, 975 "lines": true, 976 "linewidth": 0, 977 "links": [ ], 978 "nullPointMode": "null as zero", 979 "percentage": false, 980 "pointradius": 5, 981 "points": false, 982 "renderer": "flot", 983 "seriesOverrides": [ ], 984 "spaceLength": 10, 985 "span": 4, 986 "stack": true, 987 "steppedLine": false, 988 "targets": [ 989 { 990 "expr": "sum by (job) (rate(thanos_blocks_meta_sync_failures_total{job=~\"$job\"}[$interval])) / sum by (job) (rate(thanos_blocks_meta_syncs_total{job=~\"$job\"}[$interval]))", 991 "format": "time_series", 992 "intervalFactor": 2, 993 "legendFormat": "error", 994 "step": 10 995 } 996 ], 997 "thresholds": [ ], 998 "timeFrom": null, 999 "timeShift": null, 1000 "title": "Errors", 1001 "tooltip": { 1002 "shared": false, 1003 "sort": 0, 1004 "value_type": "individual" 1005 }, 1006 "type": "graph", 1007 "xaxis": { 1008 "buckets": null, 1009 "mode": "time", 1010 "name": null, 1011 "show": true, 1012 "values": [ ] 1013 }, 1014 "yaxes": [ 1015 { 1016 "format": "percentunit", 1017 "label": null, 1018 "logBase": 1, 1019 "max": null, 1020 "min": 0, 1021 "show": true 1022 }, 1023 { 1024 "format": "short", 1025 "label": null, 1026 "logBase": 1, 1027 "max": null, 1028 "min": null, 1029 "show": false 1030 } 1031 ] 1032 }, 1033 { 1034 "aliasColors": { }, 1035 "bars": false, 1036 "dashLength": 10, 1037 "dashes": false, 1038 "datasource": "$datasource", 1039 "description": "Shows how long has it taken to execute meta file sync, in quantiles.", 1040 "fill": 1, 1041 "id": 13, 1042 "legend": { 1043 "avg": false, 1044 "current": false, 1045 "max": false, 1046 "min": false, 1047 "show": true, 1048 "total": false, 1049 "values": false 1050 }, 1051 "lines": true, 1052 "linewidth": 1, 1053 "links": [ ], 1054 "nullPointMode": "null as zero", 1055 "percentage": false, 1056 "pointradius": 5, 1057 "points": false, 1058 "renderer": "flot", 1059 "seriesOverrides": [ 1060 { 1061 "alias": "p99", 1062 "color": "#FA6400", 1063 "fill": 1, 1064 "fillGradient": 1 1065 }, 1066 { 1067 "alias": "p90", 1068 "color": "#E0B400", 1069 "fill": 1, 1070 "fillGradient": 1 1071 }, 1072 { 1073 "alias": "p50", 1074 "color": "#37872D", 1075 "fill": 10, 1076 "fillGradient": 0 1077 } 1078 ], 1079 "spaceLength": 10, 1080 "span": 4, 1081 "stack": false, 1082 "steppedLine": false, 1083 "targets": [ 1084 { 1085 "expr": "histogram_quantile(0.50, sum by (job, le) (rate(thanos_blocks_meta_sync_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1", 1086 "format": "time_series", 1087 "intervalFactor": 2, 1088 "legendFormat": "p50 {{job}}", 1089 "logBase": 10, 1090 "max": null, 1091 "min": null, 1092 "step": 10 1093 }, 1094 { 1095 "expr": "histogram_quantile(0.90, sum by (job, le) (rate(thanos_blocks_meta_sync_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1", 1096 "format": "time_series", 1097 "intervalFactor": 2, 1098 "legendFormat": "p90 {{job}}", 1099 "logBase": 10, 1100 "max": null, 1101 "min": null, 1102 "step": 10 1103 }, 1104 { 1105 "expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_blocks_meta_sync_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1", 1106 "format": "time_series", 1107 "intervalFactor": 2, 1108 "legendFormat": "p99 {{job}}", 1109 "logBase": 10, 1110 "max": null, 1111 "min": null, 1112 "step": 10 1113 } 1114 ], 1115 "thresholds": [ ], 1116 "timeFrom": null, 1117 "timeShift": null, 1118 "title": "Duration", 1119 "tooltip": { 1120 "shared": false, 1121 "sort": 0, 1122 "value_type": "individual" 1123 }, 1124 "type": "graph", 1125 "xaxis": { 1126 "buckets": null, 1127 "mode": "time", 1128 "name": null, 1129 "show": true, 1130 "values": [ ] 1131 }, 1132 "yaxes": [ 1133 { 1134 "format": "s", 1135 "label": null, 1136 "logBase": 1, 1137 "max": null, 1138 "min": 0, 1139 "show": true 1140 }, 1141 { 1142 "format": "short", 1143 "label": null, 1144 "logBase": 1, 1145 "max": null, 1146 "min": null, 1147 "show": false 1148 } 1149 ] 1150 } 1151 ], 1152 "repeat": null, 1153 "repeatIteration": null, 1154 "repeatRowId": null, 1155 "showTitle": true, 1156 "title": "Sync Meta", 1157 "titleSize": "h6" 1158 }, 1159 { 1160 "collapse": false, 1161 "height": "250px", 1162 "panels": [ 1163 { 1164 "aliasColors": { }, 1165 "bars": false, 1166 "dashLength": 10, 1167 "dashes": false, 1168 "datasource": "$datasource", 1169 "description": "Shows rate of execution for operations against the bucket.", 1170 "fill": 10, 1171 "id": 14, 1172 "legend": { 1173 "avg": false, 1174 "current": false, 1175 "max": false, 1176 "min": false, 1177 "show": true, 1178 "total": false, 1179 "values": false 1180 }, 1181 "lines": true, 1182 "linewidth": 0, 1183 "links": [ ], 1184 "nullPointMode": "null as zero", 1185 "percentage": false, 1186 "pointradius": 5, 1187 "points": false, 1188 "renderer": "flot", 1189 "seriesOverrides": [ ], 1190 "spaceLength": 10, 1191 "span": 4, 1192 "stack": true, 1193 "steppedLine": false, 1194 "targets": [ 1195 { 1196 "expr": "sum by (job, operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\"}[$__rate_interval]))", 1197 "format": "time_series", 1198 "intervalFactor": 2, 1199 "legendFormat": "{{job}} {{operation}}", 1200 "legendLink": null, 1201 "step": 10 1202 } 1203 ], 1204 "thresholds": [ ], 1205 "timeFrom": null, 1206 "timeShift": null, 1207 "title": "Rate", 1208 "tooltip": { 1209 "shared": false, 1210 "sort": 0, 1211 "value_type": "individual" 1212 }, 1213 "type": "graph", 1214 "xaxis": { 1215 "buckets": null, 1216 "mode": "time", 1217 "name": null, 1218 "show": true, 1219 "values": [ ] 1220 }, 1221 "yaxes": [ 1222 { 1223 "format": "short", 1224 "label": null, 1225 "logBase": 1, 1226 "max": null, 1227 "min": 0, 1228 "show": true 1229 }, 1230 { 1231 "format": "short", 1232 "label": null, 1233 "logBase": 1, 1234 "max": null, 1235 "min": null, 1236 "show": false 1237 } 1238 ] 1239 }, 1240 { 1241 "aliasColors": { 1242 "error": "#E24D42" 1243 }, 1244 "bars": false, 1245 "dashLength": 10, 1246 "dashes": false, 1247 "datasource": "$datasource", 1248 "description": "Shows ratio of errors compared to the total number of executed operations against the bucket.", 1249 "fill": 10, 1250 "id": 15, 1251 "legend": { 1252 "avg": false, 1253 "current": false, 1254 "max": false, 1255 "min": false, 1256 "show": true, 1257 "total": false, 1258 "values": false 1259 }, 1260 "lines": true, 1261 "linewidth": 0, 1262 "links": [ ], 1263 "nullPointMode": "null as zero", 1264 "percentage": false, 1265 "pointradius": 5, 1266 "points": false, 1267 "renderer": "flot", 1268 "seriesOverrides": [ ], 1269 "spaceLength": 10, 1270 "span": 4, 1271 "stack": true, 1272 "steppedLine": false, 1273 "targets": [ 1274 { 1275 "expr": "sum by (job) (rate(thanos_objstore_bucket_operation_failures_total{job=~\"$job\"}[$interval])) / sum by (job) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\"}[$interval]))", 1276 "format": "time_series", 1277 "intervalFactor": 2, 1278 "legendFormat": "error", 1279 "step": 10 1280 } 1281 ], 1282 "thresholds": [ ], 1283 "timeFrom": null, 1284 "timeShift": null, 1285 "title": "Errors", 1286 "tooltip": { 1287 "shared": false, 1288 "sort": 0, 1289 "value_type": "individual" 1290 }, 1291 "type": "graph", 1292 "xaxis": { 1293 "buckets": null, 1294 "mode": "time", 1295 "name": null, 1296 "show": true, 1297 "values": [ ] 1298 }, 1299 "yaxes": [ 1300 { 1301 "format": "percentunit", 1302 "label": null, 1303 "logBase": 1, 1304 "max": null, 1305 "min": 0, 1306 "show": true 1307 }, 1308 { 1309 "format": "short", 1310 "label": null, 1311 "logBase": 1, 1312 "max": null, 1313 "min": null, 1314 "show": false 1315 } 1316 ] 1317 }, 1318 { 1319 "aliasColors": { }, 1320 "bars": false, 1321 "dashLength": 10, 1322 "dashes": false, 1323 "datasource": "$datasource", 1324 "description": "Shows how long has it taken to execute operations against the bucket, in quantiles.", 1325 "fill": 1, 1326 "id": 16, 1327 "legend": { 1328 "avg": false, 1329 "current": false, 1330 "max": false, 1331 "min": false, 1332 "show": true, 1333 "total": false, 1334 "values": false 1335 }, 1336 "lines": true, 1337 "linewidth": 1, 1338 "links": [ ], 1339 "nullPointMode": "null as zero", 1340 "percentage": false, 1341 "pointradius": 5, 1342 "points": false, 1343 "renderer": "flot", 1344 "seriesOverrides": [ 1345 { 1346 "alias": "p99", 1347 "color": "#FA6400", 1348 "fill": 1, 1349 "fillGradient": 1 1350 }, 1351 { 1352 "alias": "p90", 1353 "color": "#E0B400", 1354 "fill": 1, 1355 "fillGradient": 1 1356 }, 1357 { 1358 "alias": "p50", 1359 "color": "#37872D", 1360 "fill": 10, 1361 "fillGradient": 0 1362 } 1363 ], 1364 "spaceLength": 10, 1365 "span": 4, 1366 "stack": false, 1367 "steppedLine": false, 1368 "targets": [ 1369 { 1370 "expr": "histogram_quantile(0.50, sum by (job, le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1", 1371 "format": "time_series", 1372 "intervalFactor": 2, 1373 "legendFormat": "p50 {{job}}", 1374 "logBase": 10, 1375 "max": null, 1376 "min": null, 1377 "step": 10 1378 }, 1379 { 1380 "expr": "histogram_quantile(0.90, sum by (job, le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1", 1381 "format": "time_series", 1382 "intervalFactor": 2, 1383 "legendFormat": "p90 {{job}}", 1384 "logBase": 10, 1385 "max": null, 1386 "min": null, 1387 "step": 10 1388 }, 1389 { 1390 "expr": "histogram_quantile(0.99, sum by (job, le) (rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\"}[$interval]))) * 1", 1391 "format": "time_series", 1392 "intervalFactor": 2, 1393 "legendFormat": "p99 {{job}}", 1394 "logBase": 10, 1395 "max": null, 1396 "min": null, 1397 "step": 10 1398 } 1399 ], 1400 "thresholds": [ ], 1401 "timeFrom": null, 1402 "timeShift": null, 1403 "title": "Duration", 1404 "tooltip": { 1405 "shared": false, 1406 "sort": 0, 1407 "value_type": "individual" 1408 }, 1409 "type": "graph", 1410 "xaxis": { 1411 "buckets": null, 1412 "mode": "time", 1413 "name": null, 1414 "show": true, 1415 "values": [ ] 1416 }, 1417 "yaxes": [ 1418 { 1419 "format": "s", 1420 "label": null, 1421 "logBase": 1, 1422 "max": null, 1423 "min": 0, 1424 "show": true 1425 }, 1426 { 1427 "format": "short", 1428 "label": null, 1429 "logBase": 1, 1430 "max": null, 1431 "min": null, 1432 "show": false 1433 } 1434 ] 1435 } 1436 ], 1437 "repeat": null, 1438 "repeatIteration": null, 1439 "repeatRowId": null, 1440 "showTitle": true, 1441 "title": "Object Store Operations", 1442 "titleSize": "h6" 1443 }, 1444 { 1445 "collapse": true, 1446 "height": "250px", 1447 "panels": [ 1448 { 1449 "aliasColors": { }, 1450 "bars": false, 1451 "dashLength": 10, 1452 "dashes": false, 1453 "datasource": "$datasource", 1454 "fill": 1, 1455 "id": 17, 1456 "legend": { 1457 "avg": false, 1458 "current": false, 1459 "max": false, 1460 "min": false, 1461 "show": true, 1462 "total": false, 1463 "values": false 1464 }, 1465 "lines": true, 1466 "linewidth": 1, 1467 "links": [ ], 1468 "nullPointMode": "null as zero", 1469 "percentage": false, 1470 "pointradius": 5, 1471 "points": false, 1472 "renderer": "flot", 1473 "seriesOverrides": [ ], 1474 "spaceLength": 10, 1475 "span": 4, 1476 "stack": false, 1477 "steppedLine": false, 1478 "targets": [ 1479 { 1480 "expr": "go_memstats_alloc_bytes{job=~\"$job\"}", 1481 "format": "time_series", 1482 "intervalFactor": 2, 1483 "legendFormat": "alloc all {{instance}}", 1484 "legendLink": null, 1485 "step": 10 1486 }, 1487 { 1488 "expr": "go_memstats_heap_alloc_bytes{job=~\"$job\"}", 1489 "format": "time_series", 1490 "intervalFactor": 2, 1491 "legendFormat": "alloc heap {{instance}}", 1492 "legendLink": null, 1493 "step": 10 1494 }, 1495 { 1496 "expr": "rate(go_memstats_alloc_bytes_total{job=~\"$job\"}[30s])", 1497 "format": "time_series", 1498 "intervalFactor": 2, 1499 "legendFormat": "alloc rate all {{instance}}", 1500 "legendLink": null, 1501 "step": 10 1502 }, 1503 { 1504 "expr": "rate(go_memstats_heap_alloc_bytes{job=~\"$job\"}[30s])", 1505 "format": "time_series", 1506 "intervalFactor": 2, 1507 "legendFormat": "alloc rate heap {{instance}}", 1508 "legendLink": null, 1509 "step": 10 1510 }, 1511 { 1512 "expr": "go_memstats_stack_inuse_bytes{job=~\"$job\"}", 1513 "format": "time_series", 1514 "intervalFactor": 2, 1515 "legendFormat": "inuse heap {{instance}}", 1516 "legendLink": null, 1517 "step": 10 1518 }, 1519 { 1520 "expr": "go_memstats_heap_inuse_bytes{job=~\"$job\"}", 1521 "format": "time_series", 1522 "intervalFactor": 2, 1523 "legendFormat": "inuse stack {{instance}}", 1524 "legendLink": null, 1525 "step": 10 1526 } 1527 ], 1528 "thresholds": [ ], 1529 "timeFrom": null, 1530 "timeShift": null, 1531 "title": "Memory Used", 1532 "tooltip": { 1533 "shared": false, 1534 "sort": 0, 1535 "value_type": "individual" 1536 }, 1537 "type": "graph", 1538 "xaxis": { 1539 "buckets": null, 1540 "mode": "time", 1541 "name": null, 1542 "show": true, 1543 "values": [ ] 1544 }, 1545 "yaxes": [ 1546 { 1547 "format": "bytes", 1548 "label": null, 1549 "logBase": 1, 1550 "max": null, 1551 "min": 0, 1552 "show": true 1553 }, 1554 { 1555 "format": "short", 1556 "label": null, 1557 "logBase": 1, 1558 "max": null, 1559 "min": null, 1560 "show": false 1561 } 1562 ] 1563 }, 1564 { 1565 "aliasColors": { }, 1566 "bars": false, 1567 "dashLength": 10, 1568 "dashes": false, 1569 "datasource": "$datasource", 1570 "fill": 1, 1571 "id": 18, 1572 "legend": { 1573 "avg": false, 1574 "current": false, 1575 "max": false, 1576 "min": false, 1577 "show": true, 1578 "total": false, 1579 "values": false 1580 }, 1581 "lines": true, 1582 "linewidth": 1, 1583 "links": [ ], 1584 "nullPointMode": "null as zero", 1585 "percentage": false, 1586 "pointradius": 5, 1587 "points": false, 1588 "renderer": "flot", 1589 "seriesOverrides": [ ], 1590 "spaceLength": 10, 1591 "span": 4, 1592 "stack": false, 1593 "steppedLine": false, 1594 "targets": [ 1595 { 1596 "expr": "go_goroutines{job=~\"$job\"}", 1597 "format": "time_series", 1598 "intervalFactor": 2, 1599 "legendFormat": "{{instance}}", 1600 "legendLink": null, 1601 "step": 10 1602 } 1603 ], 1604 "thresholds": [ ], 1605 "timeFrom": null, 1606 "timeShift": null, 1607 "title": "Goroutines", 1608 "tooltip": { 1609 "shared": false, 1610 "sort": 0, 1611 "value_type": "individual" 1612 }, 1613 "type": "graph", 1614 "xaxis": { 1615 "buckets": null, 1616 "mode": "time", 1617 "name": null, 1618 "show": true, 1619 "values": [ ] 1620 }, 1621 "yaxes": [ 1622 { 1623 "format": "short", 1624 "label": null, 1625 "logBase": 1, 1626 "max": null, 1627 "min": 0, 1628 "show": true 1629 }, 1630 { 1631 "format": "short", 1632 "label": null, 1633 "logBase": 1, 1634 "max": null, 1635 "min": null, 1636 "show": false 1637 } 1638 ] 1639 }, 1640 { 1641 "aliasColors": { }, 1642 "bars": false, 1643 "dashLength": 10, 1644 "dashes": false, 1645 "datasource": "$datasource", 1646 "fill": 1, 1647 "id": 19, 1648 "legend": { 1649 "avg": false, 1650 "current": false, 1651 "max": false, 1652 "min": false, 1653 "show": true, 1654 "total": false, 1655 "values": false 1656 }, 1657 "lines": true, 1658 "linewidth": 1, 1659 "links": [ ], 1660 "nullPointMode": "null as zero", 1661 "percentage": false, 1662 "pointradius": 5, 1663 "points": false, 1664 "renderer": "flot", 1665 "seriesOverrides": [ ], 1666 "spaceLength": 10, 1667 "span": 4, 1668 "stack": false, 1669 "steppedLine": false, 1670 "targets": [ 1671 { 1672 "expr": "go_gc_duration_seconds{job=~\"$job\"}", 1673 "format": "time_series", 1674 "intervalFactor": 2, 1675 "legendFormat": "{{quantile}} {{instance}}", 1676 "legendLink": null, 1677 "step": 10 1678 } 1679 ], 1680 "thresholds": [ ], 1681 "timeFrom": null, 1682 "timeShift": null, 1683 "title": "GC Time Quantiles", 1684 "tooltip": { 1685 "shared": false, 1686 "sort": 0, 1687 "value_type": "individual" 1688 }, 1689 "type": "graph", 1690 "xaxis": { 1691 "buckets": null, 1692 "mode": "time", 1693 "name": null, 1694 "show": true, 1695 "values": [ ] 1696 }, 1697 "yaxes": [ 1698 { 1699 "format": "short", 1700 "label": null, 1701 "logBase": 1, 1702 "max": null, 1703 "min": 0, 1704 "show": true 1705 }, 1706 { 1707 "format": "short", 1708 "label": null, 1709 "logBase": 1, 1710 "max": null, 1711 "min": null, 1712 "show": false 1713 } 1714 ] 1715 } 1716 ], 1717 "repeat": null, 1718 "repeatIteration": null, 1719 "repeatRowId": null, 1720 "showTitle": true, 1721 "title": "Resources", 1722 "titleSize": "h6" 1723 } 1724 ], 1725 "schemaVersion": 14, 1726 "style": "dark", 1727 "tags": [ 1728 "thanos-mixin" 1729 ], 1730 "templating": { 1731 "list": [ 1732 { 1733 "current": { 1734 "text": "default", 1735 "value": "default" 1736 }, 1737 "hide": 0, 1738 "label": null, 1739 "name": "datasource", 1740 "options": [ ], 1741 "query": "prometheus", 1742 "refresh": 1, 1743 "regex": "", 1744 "type": "datasource" 1745 }, 1746 { 1747 "auto": true, 1748 "auto_count": 300, 1749 "auto_min": "10s", 1750 "current": { 1751 "text": "5m", 1752 "value": "5m" 1753 }, 1754 "hide": 0, 1755 "label": "interval", 1756 "name": "interval", 1757 "query": "5m,10m,30m,1h,6h,12h", 1758 "refresh": 2, 1759 "type": "interval" 1760 }, 1761 { 1762 "allValue": null, 1763 "current": { 1764 "text": "all", 1765 "value": "$__all" 1766 }, 1767 "datasource": "$datasource", 1768 "hide": 0, 1769 "includeAll": true, 1770 "label": "job", 1771 "multi": false, 1772 "name": "job", 1773 "options": [ ], 1774 "query": "label_values(up{job=~\".*thanos-compact.*\"}, job)", 1775 "refresh": 1, 1776 "regex": "", 1777 "sort": 2, 1778 "tagValuesQuery": "", 1779 "tags": [ ], 1780 "tagsQuery": "", 1781 "type": "query", 1782 "useTags": false 1783 } 1784 ] 1785 }, 1786 "time": { 1787 "from": "now-1h", 1788 "to": "now" 1789 }, 1790 "timepicker": { 1791 "refresh_intervals": [ 1792 "5s", 1793 "10s", 1794 "30s", 1795 "1m", 1796 "5m", 1797 "15m", 1798 "30m", 1799 "1h", 1800 "2h", 1801 "1d" 1802 ], 1803 "time_options": [ 1804 "5m", 1805 "15m", 1806 "1h", 1807 "6h", 1808 "12h", 1809 "24h", 1810 "2d", 1811 "7d", 1812 "30d" 1813 ] 1814 }, 1815 "timezone": "UTC", 1816 "title": "Thanos / Compact", 1817 "uid": "651943d05a8123e32867b4673963f42b", 1818 "version": 0 1819 }