github.com/1aal/kubeblocks@v0.0.0-20231107070852-e1c03e598921/deploy/pulsar/dashboards/pulsar-bookie.json (about) 1 { 2 "annotations": { 3 "list": [ 4 { 5 "$$hashKey": "object:2836", 6 "builtIn": 1, 7 "datasource": { 8 "type": "datasource", 9 "uid": "grafana" 10 }, 11 "enable": true, 12 "hide": true, 13 "iconColor": "rgba(0, 211, 255, 1)", 14 "name": "Annotations & Alerts", 15 "target": { 16 "limit": 100, 17 "matchAny": false, 18 "tags": [], 19 "type": "dashboard" 20 }, 21 "type": "dashboard" 22 } 23 ] 24 }, 25 "description": "Metrics of bookkeeper cluster", 26 "editable": true, 27 "fiscalYearStartMonth": 0, 28 "graphTooltip": 0, 29 "id": 28, 30 "links": [], 31 "liveNow": false, 32 "panels": [ 33 { 34 "collapsed": false, 35 "datasource": { 36 "type": "prometheus", 37 "uid": "${prometheus}" 38 }, 39 "gridPos": { 40 "h": 1, 41 "w": 24, 42 "x": 0, 43 "y": 0 44 }, 45 "id": 25, 46 "panels": [], 47 "targets": [ 48 { 49 "datasource": { 50 "type": "prometheus", 51 "uid": "${prometheus}" 52 }, 53 "refId": "A" 54 } 55 ], 56 "title": "Healthy", 57 "type": "row" 58 }, 59 { 60 "datasource": { 61 "type": "prometheus", 62 "uid": "${prometheus}" 63 }, 64 "fieldConfig": { 65 "defaults": { 66 "color": { 67 "mode": "thresholds" 68 }, 69 "mappings": [ 70 { 71 "options": { 72 "match": "null", 73 "result": { 74 "text": "N/A" 75 } 76 }, 77 "type": "special" 78 } 79 ], 80 "thresholds": { 81 "mode": "absolute", 82 "steps": [ 83 { 84 "color": "green", 85 "value": null 86 }, 87 { 88 "color": "red", 89 "value": 80 90 } 91 ] 92 }, 93 "unit": "none" 94 }, 95 "overrides": [] 96 }, 97 "gridPos": { 98 "h": 7, 99 "w": 3, 100 "x": 0, 101 "y": 1 102 }, 103 "id": 27, 104 "links": [], 105 "maxDataPoints": 100, 106 "options": { 107 "colorMode": "none", 108 "graphMode": "none", 109 "justifyMode": "auto", 110 "orientation": "horizontal", 111 "reduceOptions": { 112 "calcs": [ 113 "mean" 114 ], 115 "fields": "", 116 "values": false 117 }, 118 "textMode": "auto" 119 }, 120 "pluginVersion": "9.2.4", 121 "targets": [ 122 { 123 "datasource": { 124 "type": "prometheus", 125 "uid": "${prometheus}" 126 }, 127 "expr": "count(bookie_SERVER_STATUS{app_kubernetes_io_instance=~\"$cluster\"} == 1)\n ", 128 "format": "time_series", 129 "interval": "", 130 "intervalFactor": 1, 131 "legendFormat": "", 132 "refId": "A" 133 } 134 ], 135 "title": "Writable Bookies", 136 "type": "stat" 137 }, 138 { 139 "datasource": { 140 "type": "prometheus", 141 "uid": "${prometheus}" 142 }, 143 "fieldConfig": { 144 "defaults": { 145 "color": { 146 "mode": "thresholds" 147 }, 148 "mappings": [ 149 { 150 "options": { 151 "match": "null", 152 "result": { 153 "text": "N/A" 154 } 155 }, 156 "type": "special" 157 } 158 ], 159 "thresholds": { 160 "mode": "absolute", 161 "steps": [ 162 { 163 "color": "green", 164 "value": null 165 }, 166 { 167 "color": "red", 168 "value": 80 169 } 170 ] 171 }, 172 "unit": "none" 173 }, 174 "overrides": [] 175 }, 176 "gridPos": { 177 "h": 7, 178 "w": 3, 179 "x": 3, 180 "y": 1 181 }, 182 "id": 31, 183 "links": [], 184 "maxDataPoints": 100, 185 "options": { 186 "colorMode": "none", 187 "graphMode": "none", 188 "justifyMode": "auto", 189 "orientation": "horizontal", 190 "reduceOptions": { 191 "calcs": [ 192 "mean" 193 ], 194 "fields": "", 195 "values": false 196 }, 197 "textMode": "auto" 198 }, 199 "pluginVersion": "9.2.4", 200 "targets": [ 201 { 202 "datasource": { 203 "uid": "${prometheus}" 204 }, 205 "editorMode": "code", 206 "expr": "count(bookie_SERVER_STATUS{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"} == 0) or vector(0)\n", 207 "format": "time_series", 208 "interval": "", 209 "intervalFactor": 1, 210 "legendFormat": "", 211 "range": true, 212 "refId": "A" 213 } 214 ], 215 "title": "ReadOnly Bookies", 216 "type": "stat" 217 }, 218 { 219 "aliasColors": {}, 220 "bars": false, 221 "dashLength": 10, 222 "dashes": false, 223 "datasource": { 224 "type": "prometheus", 225 "uid": "${prometheus}" 226 }, 227 "decimals": 1, 228 "description": "success rate for different types of requests", 229 "fieldConfig": { 230 "defaults": { 231 "links": [] 232 }, 233 "overrides": [] 234 }, 235 "fill": 0, 236 "fillGradient": 0, 237 "gridPos": { 238 "h": 7, 239 "w": 9, 240 "x": 6, 241 "y": 1 242 }, 243 "hiddenSeries": false, 244 "id": 1, 245 "legend": { 246 "avg": false, 247 "current": false, 248 "max": false, 249 "min": false, 250 "show": true, 251 "total": false, 252 "values": false 253 }, 254 "lines": true, 255 "linewidth": 1, 256 "links": [], 257 "nullPointMode": "null", 258 "options": { 259 "alertThreshold": true 260 }, 261 "percentage": false, 262 "pluginVersion": "9.2.4", 263 "pointradius": 5, 264 "points": false, 265 "renderer": "flot", 266 "seriesOverrides": [], 267 "spaceLength": 10, 268 "stack": false, 269 "steppedLine": false, 270 "targets": [ 271 { 272 "datasource": { 273 "uid": "${prometheus}" 274 }, 275 "editorMode": "code", 276 "expr": "100 * sum(irate(bookkeeper_server_ADD_ENTRY_REQUEST_count{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\"}[30s])) / sum(irate(bookkeeper_server_ADD_ENTRY_REQUEST_count{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}[30s]))", 277 "format": "time_series", 278 "interval": "", 279 "intervalFactor": 2, 280 "legendFormat": "add_entry", 281 "range": true, 282 "refId": "A", 283 "step": 2 284 }, 285 { 286 "datasource": { 287 "uid": "${prometheus}" 288 }, 289 "expr": "100 * sum(irate(bookkeeper_server_READ_ENTRY_count{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\"}[30s])) / sum(irate(bookkeeper_server_READ_ENTRY_count{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}[30s]))", 290 "format": "time_series", 291 "interval": "", 292 "intervalFactor": 1, 293 "legendFormat": "read_entry", 294 "refId": "B" 295 } 296 ], 297 "thresholds": [], 298 "timeRegions": [], 299 "title": "Success Rate", 300 "tooltip": { 301 "shared": true, 302 "sort": 0, 303 "value_type": "individual" 304 }, 305 "type": "graph", 306 "xaxis": { 307 "mode": "time", 308 "show": true, 309 "values": [] 310 }, 311 "yaxes": [ 312 { 313 "format": "short", 314 "logBase": 1, 315 "min": "0", 316 "show": true 317 }, 318 { 319 "format": "short", 320 "logBase": 1, 321 "show": true 322 } 323 ], 324 "yaxis": { 325 "align": false 326 } 327 }, 328 { 329 "aliasColors": {}, 330 "bars": false, 331 "dashLength": 10, 332 "dashes": false, 333 "datasource": { 334 "type": "prometheus", 335 "uid": "${prometheus}" 336 }, 337 "decimals": 1, 338 "fieldConfig": { 339 "defaults": { 340 "links": [] 341 }, 342 "overrides": [] 343 }, 344 "fill": 0, 345 "fillGradient": 0, 346 "gridPos": { 347 "h": 7, 348 "w": 9, 349 "x": 15, 350 "y": 1 351 }, 352 "hiddenSeries": false, 353 "id": 13, 354 "legend": { 355 "avg": false, 356 "current": false, 357 "max": false, 358 "min": false, 359 "show": true, 360 "total": false, 361 "values": false 362 }, 363 "lines": true, 364 "linewidth": 1, 365 "links": [], 366 "nullPointMode": "null", 367 "options": { 368 "alertThreshold": true 369 }, 370 "percentage": false, 371 "pluginVersion": "9.2.4", 372 "pointradius": 5, 373 "points": false, 374 "renderer": "flot", 375 "seriesOverrides": [], 376 "spaceLength": 10, 377 "stack": false, 378 "steppedLine": false, 379 "targets": [ 380 { 381 "datasource": { 382 "uid": "${prometheus}" 383 }, 384 "editorMode": "code", 385 "expr": "avg(bookkeeper_server_ADD_ENTRY_REQUEST{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\", quantile=\"0.99\"} > 0) or vector(0)", 386 "format": "time_series", 387 "interval": "", 388 "intervalFactor": 2, 389 "legendFormat": "add (p99)", 390 "range": true, 391 "refId": "A", 392 "step": 2 393 }, 394 { 395 "datasource": { 396 "uid": "${prometheus}" 397 }, 398 "expr": "avg(bookkeeper_server_ADD_ENTRY_REQUEST{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\", quantile=\"0.999\"} > 0)", 399 "format": "time_series", 400 "interval": "", 401 "intervalFactor": 1, 402 "legendFormat": "add (p999)", 403 "refId": "B" 404 }, 405 { 406 "datasource": { 407 "uid": "${prometheus}" 408 }, 409 "expr": "avg(bookkeeper_server_ADD_ENTRY_REQUEST{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\", quantile=\"0.5\"} > 0)", 410 "format": "time_series", 411 "interval": "", 412 "intervalFactor": 1, 413 "legendFormat": "add (p50)", 414 "refId": "C" 415 }, 416 { 417 "datasource": { 418 "uid": "${prometheus}" 419 }, 420 "expr": "avg(bookkeeper_server_READ_ENTRY_REQUEST{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\", quantile=\"0.99\"} > 0)", 421 "format": "time_series", 422 "interval": "", 423 "intervalFactor": 1, 424 "legendFormat": "read (p99)", 425 "refId": "D" 426 }, 427 { 428 "datasource": { 429 "uid": "${prometheus}" 430 }, 431 "expr": "avg(bookkeeper_server_READ_ENTRY_REQUEST{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\", quantile=\"0.999\"} > 0)", 432 "format": "time_series", 433 "interval": "", 434 "intervalFactor": 1, 435 "legendFormat": "read (p999)", 436 "refId": "E" 437 }, 438 { 439 "datasource": { 440 "uid": "${prometheus}" 441 }, 442 "expr": "avg(bookkeeper_server_READ_ENTRY_REQUEST{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\", quantile=\"0.5\"} > 0)", 443 "format": "time_series", 444 "interval": "", 445 "intervalFactor": 1, 446 "legendFormat": "read (p50)", 447 "refId": "F" 448 } 449 ], 450 "thresholds": [], 451 "timeRegions": [], 452 "title": "Latency", 453 "tooltip": { 454 "shared": true, 455 "sort": 0, 456 "value_type": "individual" 457 }, 458 "type": "graph", 459 "xaxis": { 460 "mode": "time", 461 "show": true, 462 "values": [] 463 }, 464 "yaxes": [ 465 { 466 "format": "ms", 467 "logBase": 1, 468 "min": "0", 469 "show": true 470 }, 471 { 472 "format": "short", 473 "logBase": 1, 474 "show": true 475 } 476 ], 477 "yaxis": { 478 "align": false 479 } 480 }, 481 { 482 "datasource": { 483 "uid": "${prometheus}" 484 }, 485 "fieldConfig": { 486 "defaults": { 487 "color": { 488 "mode": "thresholds" 489 }, 490 "mappings": [ 491 { 492 "options": { 493 "match": "null", 494 "result": { 495 "text": "N/A" 496 } 497 }, 498 "type": "special" 499 } 500 ], 501 "max": 100, 502 "min": 0, 503 "thresholds": { 504 "mode": "absolute", 505 "steps": [ 506 { 507 "color": "#d44a3a", 508 "value": null 509 }, 510 { 511 "color": "rgba(237, 129, 40, 0.89)", 512 "value": 80 513 }, 514 { 515 "color": "#299c46", 516 "value": 90 517 } 518 ] 519 }, 520 "unit": "percent" 521 }, 522 "overrides": [] 523 }, 524 "gridPos": { 525 "h": 7, 526 "w": 6, 527 "x": 0, 528 "y": 8 529 }, 530 "id": 33, 531 "links": [], 532 "maxDataPoints": 100, 533 "options": { 534 "orientation": "horizontal", 535 "reduceOptions": { 536 "calcs": [ 537 "lastNotNull" 538 ], 539 "fields": "", 540 "values": false 541 }, 542 "showThresholdLabels": false, 543 "showThresholdMarkers": false 544 }, 545 "pluginVersion": "9.2.4", 546 "targets": [ 547 { 548 "datasource": { 549 "uid": "${prometheus}" 550 }, 551 "expr": "100 \n* \ncount(bookie_SERVER_STATUS{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"} == 1)\n/\ncount(bookie_SERVER_STATUS{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"})\n ", 552 "format": "time_series", 553 "interval": "", 554 "intervalFactor": 1, 555 "legendFormat": "", 556 "refId": "A" 557 } 558 ], 559 "title": "writable bookies (percentage)", 560 "type": "gauge" 561 }, 562 { 563 "aliasColors": {}, 564 "bars": false, 565 "dashLength": 10, 566 "dashes": false, 567 "datasource": { 568 "uid": "${prometheus}" 569 }, 570 "decimals": 1, 571 "fieldConfig": { 572 "defaults": { 573 "links": [] 574 }, 575 "overrides": [] 576 }, 577 "fill": 0, 578 "fillGradient": 0, 579 "gridPos": { 580 "h": 7, 581 "w": 9, 582 "x": 6, 583 "y": 8 584 }, 585 "hiddenSeries": false, 586 "id": 12, 587 "legend": { 588 "avg": false, 589 "current": false, 590 "max": false, 591 "min": false, 592 "show": true, 593 "total": false, 594 "values": false 595 }, 596 "lines": true, 597 "linewidth": 1, 598 "links": [], 599 "nullPointMode": "null", 600 "options": { 601 "alertThreshold": true 602 }, 603 "percentage": false, 604 "pluginVersion": "9.2.4", 605 "pointradius": 5, 606 "points": false, 607 "renderer": "flot", 608 "seriesOverrides": [], 609 "spaceLength": 10, 610 "stack": false, 611 "steppedLine": false, 612 "targets": [ 613 { 614 "datasource": { 615 "uid": "${prometheus}" 616 }, 617 "expr": "sum(rate(bookie_WRITE_BYTES{app_kubernetes_io_instance=~\"$cluster\"}[30s]))", 618 "format": "time_series", 619 "interval": "", 620 "intervalFactor": 2, 621 "legendFormat": "add entry", 622 "refId": "A", 623 "step": 2 624 }, 625 { 626 "datasource": { 627 "uid": "${prometheus}" 628 }, 629 "expr": "sum(rate(bookie_READ_BYTES{app_kubernetes_io_instance=~\"$cluster\"}[30s]))", 630 "format": "time_series", 631 "interval": "", 632 "intervalFactor": 1, 633 "legendFormat": "read entry", 634 "refId": "B" 635 } 636 ], 637 "thresholds": [], 638 "timeRegions": [], 639 "title": "Throughput (MB / s)", 640 "tooltip": { 641 "shared": true, 642 "sort": 0, 643 "value_type": "individual" 644 }, 645 "type": "graph", 646 "xaxis": { 647 "mode": "time", 648 "show": true, 649 "values": [] 650 }, 651 "yaxes": [ 652 { 653 "format": "Bps", 654 "logBase": 1, 655 "min": "0", 656 "show": true 657 }, 658 { 659 "format": "short", 660 "logBase": 1, 661 "show": true 662 } 663 ], 664 "yaxis": { 665 "align": false 666 } 667 }, 668 { 669 "aliasColors": {}, 670 "bars": false, 671 "dashLength": 10, 672 "dashes": false, 673 "datasource": { 674 "type": "prometheus", 675 "uid": "${prometheus}" 676 }, 677 "decimals": 1, 678 "fieldConfig": { 679 "defaults": { 680 "links": [] 681 }, 682 "overrides": [] 683 }, 684 "fill": 0, 685 "fillGradient": 0, 686 "gridPos": { 687 "h": 7, 688 "w": 9, 689 "x": 15, 690 "y": 8 691 }, 692 "hiddenSeries": false, 693 "id": 2, 694 "legend": { 695 "avg": false, 696 "current": false, 697 "max": false, 698 "min": false, 699 "show": true, 700 "total": false, 701 "values": false 702 }, 703 "lines": true, 704 "linewidth": 1, 705 "links": [], 706 "nullPointMode": "null", 707 "options": { 708 "alertThreshold": true 709 }, 710 "percentage": false, 711 "pluginVersion": "9.2.4", 712 "pointradius": 5, 713 "points": false, 714 "renderer": "flot", 715 "seriesOverrides": [], 716 "spaceLength": 10, 717 "stack": false, 718 "steppedLine": false, 719 "targets": [ 720 { 721 "datasource": { 722 "uid": "${prometheus}" 723 }, 724 "editorMode": "code", 725 "expr": "sum(irate(bookkeeper_server_READ_ENTRY_REQUEST_count{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\"}[30s]))", 726 "format": "time_series", 727 "interval": "", 728 "intervalFactor": 2, 729 "legendFormat": "read entry", 730 "range": true, 731 "refId": "A", 732 "step": 2 733 }, 734 { 735 "datasource": { 736 "uid": "${prometheus}" 737 }, 738 "expr": "sum(irate(bookkeeper_server_ADD_ENTRY_count{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\"}[30s]))", 739 "format": "time_series", 740 "interval": "", 741 "intervalFactor": 1, 742 "legendFormat": "add entry", 743 "refId": "B" 744 } 745 ], 746 "thresholds": [], 747 "timeRegions": [], 748 "title": "Entry Rate", 749 "tooltip": { 750 "shared": true, 751 "sort": 0, 752 "value_type": "individual" 753 }, 754 "type": "graph", 755 "xaxis": { 756 "mode": "time", 757 "show": true, 758 "values": [] 759 }, 760 "yaxes": [ 761 { 762 "format": "short", 763 "logBase": 1, 764 "min": "0", 765 "show": true 766 }, 767 { 768 "format": "short", 769 "logBase": 1, 770 "show": true 771 } 772 ], 773 "yaxis": { 774 "align": false 775 } 776 }, 777 { 778 "aliasColors": {}, 779 "bars": false, 780 "dashLength": 10, 781 "dashes": false, 782 "datasource": { 783 "uid": "${prometheus}" 784 }, 785 "decimals": 1, 786 "fieldConfig": { 787 "defaults": { 788 "links": [] 789 }, 790 "overrides": [] 791 }, 792 "fill": 0, 793 "fillGradient": 0, 794 "gridPos": { 795 "h": 7, 796 "w": 24, 797 "x": 0, 798 "y": 15 799 }, 800 "hiddenSeries": false, 801 "id": 64, 802 "legend": { 803 "avg": false, 804 "current": false, 805 "max": false, 806 "min": false, 807 "show": true, 808 "total": false, 809 "values": false 810 }, 811 "lines": true, 812 "linewidth": 1, 813 "links": [], 814 "nullPointMode": "null", 815 "options": { 816 "alertThreshold": true 817 }, 818 "percentage": false, 819 "pluginVersion": "9.2.4", 820 "pointradius": 5, 821 "points": false, 822 "renderer": "flot", 823 "seriesOverrides": [], 824 "spaceLength": 10, 825 "stack": false, 826 "steppedLine": false, 827 "targets": [ 828 { 829 "datasource": { 830 "uid": "${prometheus}" 831 }, 832 "expr": "sum(bookie_ledger_dir__pulsar_data_bookkeeper_ledgers_usage{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}) by (kubernetes_pod_name)", 833 "format": "time_series", 834 "interval": "", 835 "intervalFactor": 2, 836 "legendFormat": "{{ kubernetes_pod_name }}", 837 "refId": "A", 838 "step": 2 839 } 840 ], 841 "thresholds": [], 842 "timeRegions": [], 843 "title": "Ledgers Disk Usage (Bookie Metric)", 844 "tooltip": { 845 "shared": true, 846 "sort": 0, 847 "value_type": "individual" 848 }, 849 "type": "graph", 850 "xaxis": { 851 "mode": "time", 852 "show": true, 853 "values": [] 854 }, 855 "yaxes": [ 856 { 857 "$$hashKey": "object:3323", 858 "format": "percent", 859 "logBase": 1, 860 "min": "0", 861 "show": true 862 }, 863 { 864 "$$hashKey": "object:3324", 865 "format": "short", 866 "logBase": 1, 867 "show": true 868 } 869 ], 870 "yaxis": { 871 "align": false 872 } 873 }, 874 { 875 "collapsed": false, 876 "datasource": { 877 "type": "prometheus", 878 "uid": "prometheus" 879 }, 880 "gridPos": { 881 "h": 1, 882 "w": 24, 883 "x": 0, 884 "y": 22 885 }, 886 "id": 23, 887 "panels": [], 888 "targets": [ 889 { 890 "datasource": { 891 "type": "prometheus", 892 "uid": "prometheus" 893 }, 894 "refId": "A" 895 } 896 ], 897 "title": "Server", 898 "type": "row" 899 }, 900 { 901 "aliasColors": {}, 902 "bars": false, 903 "dashLength": 10, 904 "dashes": false, 905 "datasource": { 906 "uid": "${prometheus}" 907 }, 908 "decimals": 1, 909 "fieldConfig": { 910 "defaults": { 911 "links": [] 912 }, 913 "overrides": [] 914 }, 915 "fill": 0, 916 "fillGradient": 0, 917 "gridPos": { 918 "h": 7, 919 "w": 12, 920 "x": 0, 921 "y": 23 922 }, 923 "hiddenSeries": false, 924 "id": 3, 925 "legend": { 926 "avg": false, 927 "current": false, 928 "max": false, 929 "min": false, 930 "show": true, 931 "total": false, 932 "values": false 933 }, 934 "lines": true, 935 "linewidth": 1, 936 "links": [], 937 "nullPointMode": "null", 938 "options": { 939 "alertThreshold": true 940 }, 941 "percentage": false, 942 "pluginVersion": "9.2.4", 943 "pointradius": 5, 944 "points": false, 945 "renderer": "flot", 946 "seriesOverrides": [], 947 "spaceLength": 10, 948 "stack": false, 949 "steppedLine": false, 950 "targets": [ 951 { 952 "datasource": { 953 "uid": "${prometheus}" 954 }, 955 "expr": "sum(irate(bookie_WRITE_BYTES{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}[30s])) by (kubernetes_pod_name)", 956 "format": "time_series", 957 "interval": "", 958 "intervalFactor": 2, 959 "legendFormat": "{{ kubernetes_pod_name }}", 960 "refId": "A", 961 "step": 2 962 } 963 ], 964 "thresholds": [], 965 "timeRegions": [], 966 "title": "Write Bytes", 967 "tooltip": { 968 "shared": true, 969 "sort": 0, 970 "value_type": "individual" 971 }, 972 "type": "graph", 973 "xaxis": { 974 "mode": "time", 975 "show": true, 976 "values": [] 977 }, 978 "yaxes": [ 979 { 980 "format": "Bps", 981 "logBase": 1, 982 "min": "0", 983 "show": true 984 }, 985 { 986 "format": "short", 987 "logBase": 1, 988 "show": true 989 } 990 ], 991 "yaxis": { 992 "align": false 993 } 994 }, 995 { 996 "aliasColors": {}, 997 "bars": false, 998 "dashLength": 10, 999 "dashes": false, 1000 "datasource": { 1001 "uid": "${prometheus}" 1002 }, 1003 "decimals": 1, 1004 "fieldConfig": { 1005 "defaults": { 1006 "links": [] 1007 }, 1008 "overrides": [] 1009 }, 1010 "fill": 0, 1011 "fillGradient": 0, 1012 "gridPos": { 1013 "h": 7, 1014 "w": 12, 1015 "x": 12, 1016 "y": 23 1017 }, 1018 "hiddenSeries": false, 1019 "id": 51, 1020 "legend": { 1021 "avg": false, 1022 "current": false, 1023 "max": false, 1024 "min": false, 1025 "show": true, 1026 "total": false, 1027 "values": false 1028 }, 1029 "lines": true, 1030 "linewidth": 1, 1031 "links": [], 1032 "nullPointMode": "null", 1033 "options": { 1034 "alertThreshold": true 1035 }, 1036 "percentage": false, 1037 "pluginVersion": "9.2.4", 1038 "pointradius": 5, 1039 "points": false, 1040 "renderer": "flot", 1041 "seriesOverrides": [], 1042 "spaceLength": 10, 1043 "stack": false, 1044 "steppedLine": false, 1045 "targets": [ 1046 { 1047 "datasource": { 1048 "uid": "${prometheus}" 1049 }, 1050 "expr": "sum(irate(bookie_READ_BYTES{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}[30s])) by (kubernetes_pod_name)", 1051 "format": "time_series", 1052 "interval": "", 1053 "intervalFactor": 2, 1054 "legendFormat": "{{ kubernetes_pod_name }}", 1055 "refId": "A", 1056 "step": 2 1057 } 1058 ], 1059 "thresholds": [], 1060 "timeRegions": [], 1061 "title": "Read Bytes (bytes/s)", 1062 "tooltip": { 1063 "shared": true, 1064 "sort": 0, 1065 "value_type": "individual" 1066 }, 1067 "type": "graph", 1068 "xaxis": { 1069 "mode": "time", 1070 "show": true, 1071 "values": [] 1072 }, 1073 "yaxes": [ 1074 { 1075 "format": "Bps", 1076 "logBase": 1, 1077 "min": "0", 1078 "show": true 1079 }, 1080 { 1081 "format": "short", 1082 "logBase": 1, 1083 "show": true 1084 } 1085 ], 1086 "yaxis": { 1087 "align": false 1088 } 1089 }, 1090 { 1091 "aliasColors": {}, 1092 "bars": false, 1093 "dashLength": 10, 1094 "dashes": false, 1095 "datasource": { 1096 "uid": "${prometheus}" 1097 }, 1098 "decimals": 1, 1099 "fieldConfig": { 1100 "defaults": { 1101 "links": [] 1102 }, 1103 "overrides": [] 1104 }, 1105 "fill": 0, 1106 "fillGradient": 0, 1107 "gridPos": { 1108 "h": 7, 1109 "w": 12, 1110 "x": 0, 1111 "y": 30 1112 }, 1113 "hiddenSeries": false, 1114 "id": 52, 1115 "legend": { 1116 "avg": false, 1117 "current": false, 1118 "max": false, 1119 "min": false, 1120 "show": true, 1121 "total": false, 1122 "values": false 1123 }, 1124 "lines": true, 1125 "linewidth": 1, 1126 "links": [], 1127 "nullPointMode": "null", 1128 "options": { 1129 "alertThreshold": true 1130 }, 1131 "percentage": false, 1132 "pluginVersion": "9.2.4", 1133 "pointradius": 5, 1134 "points": false, 1135 "renderer": "flot", 1136 "seriesOverrides": [], 1137 "spaceLength": 10, 1138 "stack": false, 1139 "steppedLine": false, 1140 "targets": [ 1141 { 1142 "datasource": { 1143 "uid": "${prometheus}" 1144 }, 1145 "expr": "sum(irate(bookkeeper_server_ADD_ENTRY_REQUEST_count{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}[30s])) by (kubernetes_pod_name)", 1146 "format": "time_series", 1147 "interval": "", 1148 "intervalFactor": 2, 1149 "legendFormat": "{{ kubernetes_pod_name }}", 1150 "refId": "A", 1151 "step": 2 1152 } 1153 ], 1154 "thresholds": [], 1155 "timeRegions": [], 1156 "title": "Add Entry Rate", 1157 "tooltip": { 1158 "shared": true, 1159 "sort": 0, 1160 "value_type": "individual" 1161 }, 1162 "type": "graph", 1163 "xaxis": { 1164 "mode": "time", 1165 "show": true, 1166 "values": [] 1167 }, 1168 "yaxes": [ 1169 { 1170 "format": "Bps", 1171 "logBase": 1, 1172 "min": "0", 1173 "show": true 1174 }, 1175 { 1176 "format": "short", 1177 "logBase": 1, 1178 "show": true 1179 } 1180 ], 1181 "yaxis": { 1182 "align": false 1183 } 1184 }, 1185 { 1186 "aliasColors": {}, 1187 "bars": false, 1188 "dashLength": 10, 1189 "dashes": false, 1190 "datasource": { 1191 "uid": "${prometheus}" 1192 }, 1193 "decimals": 1, 1194 "fieldConfig": { 1195 "defaults": { 1196 "links": [] 1197 }, 1198 "overrides": [] 1199 }, 1200 "fill": 0, 1201 "fillGradient": 0, 1202 "gridPos": { 1203 "h": 7, 1204 "w": 12, 1205 "x": 12, 1206 "y": 30 1207 }, 1208 "hiddenSeries": false, 1209 "id": 53, 1210 "legend": { 1211 "avg": false, 1212 "current": false, 1213 "max": false, 1214 "min": false, 1215 "show": true, 1216 "total": false, 1217 "values": false 1218 }, 1219 "lines": true, 1220 "linewidth": 1, 1221 "links": [], 1222 "nullPointMode": "null", 1223 "options": { 1224 "alertThreshold": true 1225 }, 1226 "percentage": false, 1227 "pluginVersion": "9.2.4", 1228 "pointradius": 5, 1229 "points": false, 1230 "renderer": "flot", 1231 "seriesOverrides": [], 1232 "spaceLength": 10, 1233 "stack": false, 1234 "steppedLine": false, 1235 "targets": [ 1236 { 1237 "datasource": { 1238 "uid": "${prometheus}" 1239 }, 1240 "expr": "sum(irate(bookkeeper_server_READ_ENTRY_REQUEST_count{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}[30s])) by (kubernetes_pod_name)", 1241 "format": "time_series", 1242 "interval": "", 1243 "intervalFactor": 2, 1244 "legendFormat": "{{ kubernetes_pod_name }}", 1245 "refId": "A", 1246 "step": 2 1247 } 1248 ], 1249 "thresholds": [], 1250 "timeRegions": [], 1251 "title": "Read Entry Rate", 1252 "tooltip": { 1253 "shared": true, 1254 "sort": 0, 1255 "value_type": "individual" 1256 }, 1257 "type": "graph", 1258 "xaxis": { 1259 "mode": "time", 1260 "show": true, 1261 "values": [] 1262 }, 1263 "yaxes": [ 1264 { 1265 "format": "Bps", 1266 "logBase": 1, 1267 "min": "0", 1268 "show": true 1269 }, 1270 { 1271 "format": "short", 1272 "logBase": 1, 1273 "show": true 1274 } 1275 ], 1276 "yaxis": { 1277 "align": false 1278 } 1279 }, 1280 { 1281 "aliasColors": {}, 1282 "bars": false, 1283 "dashLength": 10, 1284 "dashes": false, 1285 "datasource": { 1286 "uid": "${prometheus}" 1287 }, 1288 "decimals": 1, 1289 "fieldConfig": { 1290 "defaults": { 1291 "links": [] 1292 }, 1293 "overrides": [] 1294 }, 1295 "fill": 0, 1296 "fillGradient": 0, 1297 "gridPos": { 1298 "h": 7, 1299 "w": 12, 1300 "x": 0, 1301 "y": 37 1302 }, 1303 "hiddenSeries": false, 1304 "id": 59, 1305 "legend": { 1306 "avg": false, 1307 "current": false, 1308 "max": false, 1309 "min": false, 1310 "show": true, 1311 "total": false, 1312 "values": false 1313 }, 1314 "lines": true, 1315 "linewidth": 1, 1316 "links": [], 1317 "nullPointMode": "null", 1318 "options": { 1319 "alertThreshold": true 1320 }, 1321 "percentage": false, 1322 "pluginVersion": "9.2.4", 1323 "pointradius": 5, 1324 "points": false, 1325 "renderer": "flot", 1326 "seriesOverrides": [], 1327 "spaceLength": 10, 1328 "stack": false, 1329 "steppedLine": false, 1330 "targets": [ 1331 { 1332 "datasource": { 1333 "uid": "${prometheus}" 1334 }, 1335 "expr": "sum(bookkeeper_server_ADD_ENTRY_IN_PROGRESS{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}) by (kubernetes_pod_name)", 1336 "format": "time_series", 1337 "interval": "", 1338 "intervalFactor": 2, 1339 "legendFormat": "{{ kubernetes_pod_name }}", 1340 "refId": "A", 1341 "step": 2 1342 } 1343 ], 1344 "thresholds": [], 1345 "timeRegions": [], 1346 "title": "ADD_ENTRY (Inprogress)", 1347 "tooltip": { 1348 "shared": true, 1349 "sort": 0, 1350 "value_type": "individual" 1351 }, 1352 "type": "graph", 1353 "xaxis": { 1354 "mode": "time", 1355 "show": true, 1356 "values": [] 1357 }, 1358 "yaxes": [ 1359 { 1360 "$$hashKey": "object:3323", 1361 "format": "short", 1362 "logBase": 1, 1363 "min": "0", 1364 "show": true 1365 }, 1366 { 1367 "$$hashKey": "object:3324", 1368 "format": "short", 1369 "logBase": 1, 1370 "show": true 1371 } 1372 ], 1373 "yaxis": { 1374 "align": false 1375 } 1376 }, 1377 { 1378 "aliasColors": {}, 1379 "bars": false, 1380 "dashLength": 10, 1381 "dashes": false, 1382 "datasource": { 1383 "uid": "${prometheus}" 1384 }, 1385 "decimals": 1, 1386 "fieldConfig": { 1387 "defaults": { 1388 "links": [] 1389 }, 1390 "overrides": [] 1391 }, 1392 "fill": 0, 1393 "fillGradient": 0, 1394 "gridPos": { 1395 "h": 7, 1396 "w": 12, 1397 "x": 12, 1398 "y": 37 1399 }, 1400 "hiddenSeries": false, 1401 "id": 60, 1402 "legend": { 1403 "avg": false, 1404 "current": false, 1405 "max": false, 1406 "min": false, 1407 "show": true, 1408 "total": false, 1409 "values": false 1410 }, 1411 "lines": true, 1412 "linewidth": 1, 1413 "links": [], 1414 "nullPointMode": "null", 1415 "options": { 1416 "alertThreshold": true 1417 }, 1418 "percentage": false, 1419 "pluginVersion": "9.2.4", 1420 "pointradius": 5, 1421 "points": false, 1422 "renderer": "flot", 1423 "seriesOverrides": [], 1424 "spaceLength": 10, 1425 "stack": false, 1426 "steppedLine": false, 1427 "targets": [ 1428 { 1429 "datasource": { 1430 "uid": "${prometheus}" 1431 }, 1432 "expr": "sum(bookkeeper_server_READ_ENTRY_IN_PROGRESS{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}) by (kubernetes_pod_name)", 1433 "format": "time_series", 1434 "interval": "", 1435 "intervalFactor": 2, 1436 "legendFormat": "{{ kubernetes_pod_name }}", 1437 "refId": "A", 1438 "step": 2 1439 } 1440 ], 1441 "thresholds": [], 1442 "timeRegions": [], 1443 "title": "READ_ENTRY (Inprogress)", 1444 "tooltip": { 1445 "shared": true, 1446 "sort": 0, 1447 "value_type": "individual" 1448 }, 1449 "type": "graph", 1450 "xaxis": { 1451 "mode": "time", 1452 "show": true, 1453 "values": [] 1454 }, 1455 "yaxes": [ 1456 { 1457 "$$hashKey": "object:3323", 1458 "format": "short", 1459 "logBase": 1, 1460 "min": "0", 1461 "show": true 1462 }, 1463 { 1464 "$$hashKey": "object:3324", 1465 "format": "short", 1466 "logBase": 1, 1467 "show": true 1468 } 1469 ], 1470 "yaxis": { 1471 "align": false 1472 } 1473 }, 1474 { 1475 "aliasColors": {}, 1476 "bars": false, 1477 "dashLength": 10, 1478 "dashes": false, 1479 "datasource": { 1480 "uid": "${prometheus}" 1481 }, 1482 "decimals": 1, 1483 "fieldConfig": { 1484 "defaults": { 1485 "links": [] 1486 }, 1487 "overrides": [] 1488 }, 1489 "fill": 0, 1490 "fillGradient": 0, 1491 "gridPos": { 1492 "h": 7, 1493 "w": 12, 1494 "x": 0, 1495 "y": 44 1496 }, 1497 "hiddenSeries": false, 1498 "id": 61, 1499 "legend": { 1500 "avg": false, 1501 "current": false, 1502 "max": false, 1503 "min": false, 1504 "show": true, 1505 "total": false, 1506 "values": false 1507 }, 1508 "lines": true, 1509 "linewidth": 1, 1510 "links": [], 1511 "nullPointMode": "null", 1512 "options": { 1513 "alertThreshold": true 1514 }, 1515 "percentage": false, 1516 "pluginVersion": "9.2.4", 1517 "pointradius": 5, 1518 "points": false, 1519 "renderer": "flot", 1520 "seriesOverrides": [], 1521 "spaceLength": 10, 1522 "stack": false, 1523 "steppedLine": false, 1524 "targets": [ 1525 { 1526 "datasource": { 1527 "uid": "${prometheus}" 1528 }, 1529 "expr": "sum(bookkeeper_server_ADD_ENTRY_BLOCKED{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}) by (kubernetes_pod_name)", 1530 "format": "time_series", 1531 "interval": "", 1532 "intervalFactor": 2, 1533 "legendFormat": "{{ kubernetes_pod_name }}", 1534 "refId": "A", 1535 "step": 2 1536 } 1537 ], 1538 "thresholds": [], 1539 "timeRegions": [], 1540 "title": "ADD_ENTRY (Blocked)", 1541 "tooltip": { 1542 "shared": true, 1543 "sort": 0, 1544 "value_type": "individual" 1545 }, 1546 "type": "graph", 1547 "xaxis": { 1548 "mode": "time", 1549 "show": true, 1550 "values": [] 1551 }, 1552 "yaxes": [ 1553 { 1554 "$$hashKey": "object:3323", 1555 "format": "short", 1556 "logBase": 1, 1557 "min": "0", 1558 "show": true 1559 }, 1560 { 1561 "$$hashKey": "object:3324", 1562 "format": "short", 1563 "logBase": 1, 1564 "show": true 1565 } 1566 ], 1567 "yaxis": { 1568 "align": false 1569 } 1570 }, 1571 { 1572 "aliasColors": {}, 1573 "bars": false, 1574 "dashLength": 10, 1575 "dashes": false, 1576 "datasource": { 1577 "uid": "${prometheus}" 1578 }, 1579 "decimals": 1, 1580 "fieldConfig": { 1581 "defaults": { 1582 "links": [] 1583 }, 1584 "overrides": [] 1585 }, 1586 "fill": 0, 1587 "fillGradient": 0, 1588 "gridPos": { 1589 "h": 7, 1590 "w": 12, 1591 "x": 12, 1592 "y": 44 1593 }, 1594 "hiddenSeries": false, 1595 "id": 62, 1596 "legend": { 1597 "avg": false, 1598 "current": false, 1599 "max": false, 1600 "min": false, 1601 "show": true, 1602 "total": false, 1603 "values": false 1604 }, 1605 "lines": true, 1606 "linewidth": 1, 1607 "links": [], 1608 "nullPointMode": "null", 1609 "options": { 1610 "alertThreshold": true 1611 }, 1612 "percentage": false, 1613 "pluginVersion": "9.2.4", 1614 "pointradius": 5, 1615 "points": false, 1616 "renderer": "flot", 1617 "seriesOverrides": [], 1618 "spaceLength": 10, 1619 "stack": false, 1620 "steppedLine": false, 1621 "targets": [ 1622 { 1623 "datasource": { 1624 "uid": "${prometheus}" 1625 }, 1626 "expr": "sum(bookkeeper_server_READ_ENTRY_BLOCKED{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}) by (kubernetes_pod_name)", 1627 "format": "time_series", 1628 "interval": "", 1629 "intervalFactor": 2, 1630 "legendFormat": "{{ kubernetes_pod_name }}", 1631 "refId": "A", 1632 "step": 2 1633 } 1634 ], 1635 "thresholds": [], 1636 "timeRegions": [], 1637 "title": "READ_ENTRY (Blocked)", 1638 "tooltip": { 1639 "shared": true, 1640 "sort": 0, 1641 "value_type": "individual" 1642 }, 1643 "type": "graph", 1644 "xaxis": { 1645 "mode": "time", 1646 "show": true, 1647 "values": [] 1648 }, 1649 "yaxes": [ 1650 { 1651 "$$hashKey": "object:3323", 1652 "format": "short", 1653 "logBase": 1, 1654 "min": "0", 1655 "show": true 1656 }, 1657 { 1658 "$$hashKey": "object:3324", 1659 "format": "short", 1660 "logBase": 1, 1661 "show": true 1662 } 1663 ], 1664 "yaxis": { 1665 "align": false 1666 } 1667 }, 1668 { 1669 "aliasColors": {}, 1670 "bars": false, 1671 "dashLength": 10, 1672 "dashes": false, 1673 "datasource": { 1674 "uid": "${prometheus}" 1675 }, 1676 "decimals": 1, 1677 "fieldConfig": { 1678 "defaults": { 1679 "links": [] 1680 }, 1681 "overrides": [] 1682 }, 1683 "fill": 0, 1684 "fillGradient": 0, 1685 "gridPos": { 1686 "h": 7, 1687 "w": 8, 1688 "x": 0, 1689 "y": 51 1690 }, 1691 "hiddenSeries": false, 1692 "id": 48, 1693 "legend": { 1694 "avg": false, 1695 "current": false, 1696 "max": false, 1697 "min": false, 1698 "show": true, 1699 "total": false, 1700 "values": false 1701 }, 1702 "lines": true, 1703 "linewidth": 1, 1704 "links": [], 1705 "nullPointMode": "null", 1706 "options": { 1707 "alertThreshold": true 1708 }, 1709 "percentage": false, 1710 "pluginVersion": "9.2.4", 1711 "pointradius": 5, 1712 "points": false, 1713 "renderer": "flot", 1714 "seriesOverrides": [], 1715 "spaceLength": 10, 1716 "stack": false, 1717 "steppedLine": false, 1718 "targets": [ 1719 { 1720 "datasource": { 1721 "uid": "${prometheus}" 1722 }, 1723 "expr": "sum(bookkeeper_server_ADD_ENTRY_REQUEST{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\", quantile=\"0.5\"}) by (kubernetes_pod_name)", 1724 "format": "time_series", 1725 "interval": "", 1726 "intervalFactor": 2, 1727 "legendFormat": "{{ kubernetes_pod_name }}", 1728 "refId": "A", 1729 "step": 2 1730 } 1731 ], 1732 "thresholds": [], 1733 "timeRegions": [], 1734 "title": "Add entry latency (p50)", 1735 "tooltip": { 1736 "shared": true, 1737 "sort": 0, 1738 "value_type": "individual" 1739 }, 1740 "type": "graph", 1741 "xaxis": { 1742 "mode": "time", 1743 "show": true, 1744 "values": [] 1745 }, 1746 "yaxes": [ 1747 { 1748 "format": "ms", 1749 "logBase": 1, 1750 "min": "0", 1751 "show": true 1752 }, 1753 { 1754 "format": "short", 1755 "logBase": 1, 1756 "show": true 1757 } 1758 ], 1759 "yaxis": { 1760 "align": false 1761 } 1762 }, 1763 { 1764 "aliasColors": {}, 1765 "bars": false, 1766 "dashLength": 10, 1767 "dashes": false, 1768 "datasource": { 1769 "uid": "${prometheus}" 1770 }, 1771 "decimals": 1, 1772 "fieldConfig": { 1773 "defaults": { 1774 "links": [] 1775 }, 1776 "overrides": [] 1777 }, 1778 "fill": 0, 1779 "fillGradient": 0, 1780 "gridPos": { 1781 "h": 7, 1782 "w": 8, 1783 "x": 8, 1784 "y": 51 1785 }, 1786 "hiddenSeries": false, 1787 "id": 5, 1788 "legend": { 1789 "avg": false, 1790 "current": false, 1791 "max": false, 1792 "min": false, 1793 "show": true, 1794 "total": false, 1795 "values": false 1796 }, 1797 "lines": true, 1798 "linewidth": 1, 1799 "links": [], 1800 "nullPointMode": "null", 1801 "options": { 1802 "alertThreshold": true 1803 }, 1804 "percentage": false, 1805 "pluginVersion": "9.2.4", 1806 "pointradius": 5, 1807 "points": false, 1808 "renderer": "flot", 1809 "seriesOverrides": [], 1810 "spaceLength": 10, 1811 "stack": false, 1812 "steppedLine": false, 1813 "targets": [ 1814 { 1815 "datasource": { 1816 "uid": "${prometheus}" 1817 }, 1818 "expr": "sum(bookkeeper_server_ADD_ENTRY_REQUEST{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\", quantile=\"0.99\"}) by (kubernetes_pod_name)", 1819 "format": "time_series", 1820 "interval": "", 1821 "intervalFactor": 2, 1822 "legendFormat": "{{kubernetes_pod_name}}", 1823 "refId": "A", 1824 "step": 2 1825 } 1826 ], 1827 "thresholds": [], 1828 "timeRegions": [], 1829 "title": "Add entry latency (99pct)", 1830 "tooltip": { 1831 "shared": true, 1832 "sort": 0, 1833 "value_type": "individual" 1834 }, 1835 "type": "graph", 1836 "xaxis": { 1837 "mode": "time", 1838 "show": true, 1839 "values": [] 1840 }, 1841 "yaxes": [ 1842 { 1843 "format": "ms", 1844 "logBase": 1, 1845 "min": "0", 1846 "show": true 1847 }, 1848 { 1849 "format": "short", 1850 "logBase": 1, 1851 "show": true 1852 } 1853 ], 1854 "yaxis": { 1855 "align": false 1856 } 1857 }, 1858 { 1859 "aliasColors": {}, 1860 "bars": false, 1861 "dashLength": 10, 1862 "dashes": false, 1863 "datasource": { 1864 "uid": "${prometheus}" 1865 }, 1866 "decimals": 1, 1867 "fieldConfig": { 1868 "defaults": { 1869 "links": [] 1870 }, 1871 "overrides": [] 1872 }, 1873 "fill": 0, 1874 "fillGradient": 0, 1875 "gridPos": { 1876 "h": 7, 1877 "w": 8, 1878 "x": 16, 1879 "y": 51 1880 }, 1881 "hiddenSeries": false, 1882 "id": 49, 1883 "legend": { 1884 "avg": false, 1885 "current": false, 1886 "max": false, 1887 "min": false, 1888 "show": true, 1889 "total": false, 1890 "values": false 1891 }, 1892 "lines": true, 1893 "linewidth": 1, 1894 "links": [], 1895 "nullPointMode": "null", 1896 "options": { 1897 "alertThreshold": true 1898 }, 1899 "percentage": false, 1900 "pluginVersion": "9.2.4", 1901 "pointradius": 5, 1902 "points": false, 1903 "renderer": "flot", 1904 "seriesOverrides": [], 1905 "spaceLength": 10, 1906 "stack": false, 1907 "steppedLine": false, 1908 "targets": [ 1909 { 1910 "datasource": { 1911 "uid": "${prometheus}" 1912 }, 1913 "expr": "sum(bookkeeper_server_ADD_ENTRY_REQUEST{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\", quantile=\"0.999\"}) by (kubernetes_pod_name)", 1914 "format": "time_series", 1915 "interval": "", 1916 "intervalFactor": 2, 1917 "legendFormat": "{{ kubernetes_pod_name }}", 1918 "refId": "A", 1919 "step": 2 1920 } 1921 ], 1922 "thresholds": [], 1923 "timeRegions": [], 1924 "title": "Add entry latency (p999)", 1925 "tooltip": { 1926 "shared": true, 1927 "sort": 0, 1928 "value_type": "individual" 1929 }, 1930 "type": "graph", 1931 "xaxis": { 1932 "mode": "time", 1933 "show": true, 1934 "values": [] 1935 }, 1936 "yaxes": [ 1937 { 1938 "format": "ms", 1939 "logBase": 1, 1940 "min": "0", 1941 "show": true 1942 }, 1943 { 1944 "format": "short", 1945 "logBase": 1, 1946 "show": true 1947 } 1948 ], 1949 "yaxis": { 1950 "align": false 1951 } 1952 }, 1953 { 1954 "aliasColors": {}, 1955 "bars": false, 1956 "dashLength": 10, 1957 "dashes": false, 1958 "datasource": { 1959 "uid": "${prometheus}" 1960 }, 1961 "decimals": 1, 1962 "fieldConfig": { 1963 "defaults": { 1964 "links": [] 1965 }, 1966 "overrides": [] 1967 }, 1968 "fill": 0, 1969 "fillGradient": 0, 1970 "gridPos": { 1971 "h": 7, 1972 "w": 8, 1973 "x": 0, 1974 "y": 58 1975 }, 1976 "hiddenSeries": false, 1977 "id": 54, 1978 "legend": { 1979 "avg": false, 1980 "current": false, 1981 "max": false, 1982 "min": false, 1983 "show": true, 1984 "total": false, 1985 "values": false 1986 }, 1987 "lines": true, 1988 "linewidth": 1, 1989 "links": [], 1990 "nullPointMode": "null", 1991 "options": { 1992 "alertThreshold": true 1993 }, 1994 "percentage": false, 1995 "pluginVersion": "9.2.4", 1996 "pointradius": 5, 1997 "points": false, 1998 "renderer": "flot", 1999 "seriesOverrides": [], 2000 "spaceLength": 10, 2001 "stack": false, 2002 "steppedLine": false, 2003 "targets": [ 2004 { 2005 "datasource": { 2006 "uid": "${prometheus}" 2007 }, 2008 "expr": "sum(bookkeeper_server_READ_ENTRY_REQUEST{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\", quantile=\"0.5\"}) by (kubernetes_pod_name)", 2009 "format": "time_series", 2010 "interval": "", 2011 "intervalFactor": 2, 2012 "legendFormat": "{{ kubernetes_pod_name }}", 2013 "refId": "A", 2014 "step": 2 2015 } 2016 ], 2017 "thresholds": [], 2018 "timeRegions": [], 2019 "title": "Read entry latency (p50)", 2020 "tooltip": { 2021 "shared": true, 2022 "sort": 0, 2023 "value_type": "individual" 2024 }, 2025 "type": "graph", 2026 "xaxis": { 2027 "mode": "time", 2028 "show": true, 2029 "values": [] 2030 }, 2031 "yaxes": [ 2032 { 2033 "format": "ms", 2034 "logBase": 1, 2035 "min": "0", 2036 "show": true 2037 }, 2038 { 2039 "format": "short", 2040 "logBase": 1, 2041 "show": true 2042 } 2043 ], 2044 "yaxis": { 2045 "align": false 2046 } 2047 }, 2048 { 2049 "aliasColors": {}, 2050 "bars": false, 2051 "dashLength": 10, 2052 "dashes": false, 2053 "datasource": { 2054 "uid": "${prometheus}" 2055 }, 2056 "decimals": 1, 2057 "fieldConfig": { 2058 "defaults": { 2059 "links": [] 2060 }, 2061 "overrides": [] 2062 }, 2063 "fill": 0, 2064 "fillGradient": 0, 2065 "gridPos": { 2066 "h": 7, 2067 "w": 8, 2068 "x": 8, 2069 "y": 58 2070 }, 2071 "hiddenSeries": false, 2072 "id": 55, 2073 "legend": { 2074 "avg": false, 2075 "current": false, 2076 "max": false, 2077 "min": false, 2078 "show": true, 2079 "total": false, 2080 "values": false 2081 }, 2082 "lines": true, 2083 "linewidth": 1, 2084 "links": [], 2085 "nullPointMode": "null", 2086 "options": { 2087 "alertThreshold": true 2088 }, 2089 "percentage": false, 2090 "pluginVersion": "9.2.4", 2091 "pointradius": 5, 2092 "points": false, 2093 "renderer": "flot", 2094 "seriesOverrides": [], 2095 "spaceLength": 10, 2096 "stack": false, 2097 "steppedLine": false, 2098 "targets": [ 2099 { 2100 "datasource": { 2101 "uid": "${prometheus}" 2102 }, 2103 "expr": "sum(bookkeeper_server_READ_ENTRY_REQUEST{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\", quantile=\"0.99\"}) by (kubernetes_pod_name)", 2104 "format": "time_series", 2105 "interval": "", 2106 "intervalFactor": 2, 2107 "legendFormat": "{{ kubernetes_pod_name }}", 2108 "refId": "A", 2109 "step": 2 2110 } 2111 ], 2112 "thresholds": [], 2113 "timeRegions": [], 2114 "title": "Read entry latency (p99)", 2115 "tooltip": { 2116 "shared": true, 2117 "sort": 0, 2118 "value_type": "individual" 2119 }, 2120 "type": "graph", 2121 "xaxis": { 2122 "mode": "time", 2123 "show": true, 2124 "values": [] 2125 }, 2126 "yaxes": [ 2127 { 2128 "format": "ms", 2129 "logBase": 1, 2130 "min": "0", 2131 "show": true 2132 }, 2133 { 2134 "format": "short", 2135 "logBase": 1, 2136 "show": true 2137 } 2138 ], 2139 "yaxis": { 2140 "align": false 2141 } 2142 }, 2143 { 2144 "aliasColors": {}, 2145 "bars": false, 2146 "dashLength": 10, 2147 "dashes": false, 2148 "datasource": { 2149 "uid": "${prometheus}" 2150 }, 2151 "decimals": 1, 2152 "fieldConfig": { 2153 "defaults": { 2154 "links": [] 2155 }, 2156 "overrides": [] 2157 }, 2158 "fill": 0, 2159 "fillGradient": 0, 2160 "gridPos": { 2161 "h": 7, 2162 "w": 8, 2163 "x": 16, 2164 "y": 58 2165 }, 2166 "hiddenSeries": false, 2167 "id": 56, 2168 "legend": { 2169 "avg": false, 2170 "current": false, 2171 "max": false, 2172 "min": false, 2173 "show": true, 2174 "total": false, 2175 "values": false 2176 }, 2177 "lines": true, 2178 "linewidth": 1, 2179 "links": [], 2180 "nullPointMode": "null", 2181 "options": { 2182 "alertThreshold": true 2183 }, 2184 "percentage": false, 2185 "pluginVersion": "9.2.4", 2186 "pointradius": 5, 2187 "points": false, 2188 "renderer": "flot", 2189 "seriesOverrides": [], 2190 "spaceLength": 10, 2191 "stack": false, 2192 "steppedLine": false, 2193 "targets": [ 2194 { 2195 "datasource": { 2196 "uid": "${prometheus}" 2197 }, 2198 "expr": "sum(bookkeeper_server_READ_ENTRY_REQUEST{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\", quantile=\"0.999\"}) by (kubernetes_pod_name)", 2199 "format": "time_series", 2200 "interval": "", 2201 "intervalFactor": 2, 2202 "legendFormat": "{{ kubernetes_pod_name }}", 2203 "refId": "A", 2204 "step": 2 2205 } 2206 ], 2207 "thresholds": [], 2208 "timeRegions": [], 2209 "title": "Read entry latency (p999)", 2210 "tooltip": { 2211 "shared": true, 2212 "sort": 0, 2213 "value_type": "individual" 2214 }, 2215 "type": "graph", 2216 "xaxis": { 2217 "mode": "time", 2218 "show": true, 2219 "values": [] 2220 }, 2221 "yaxes": [ 2222 { 2223 "format": "ms", 2224 "logBase": 1, 2225 "min": "0", 2226 "show": true 2227 }, 2228 { 2229 "format": "short", 2230 "logBase": 1, 2231 "show": true 2232 } 2233 ], 2234 "yaxis": { 2235 "align": false 2236 } 2237 }, 2238 { 2239 "collapsed": false, 2240 "datasource": { 2241 "type": "prometheus", 2242 "uid": "prometheus" 2243 }, 2244 "gridPos": { 2245 "h": 1, 2246 "w": 24, 2247 "x": 0, 2248 "y": 65 2249 }, 2250 "id": 35, 2251 "panels": [], 2252 "targets": [ 2253 { 2254 "datasource": { 2255 "type": "prometheus", 2256 "uid": "prometheus" 2257 }, 2258 "refId": "A" 2259 } 2260 ], 2261 "title": "Journal", 2262 "type": "row" 2263 }, 2264 { 2265 "aliasColors": {}, 2266 "bars": false, 2267 "dashLength": 10, 2268 "dashes": false, 2269 "datasource": { 2270 "uid": "${prometheus}" 2271 }, 2272 "decimals": 1, 2273 "fieldConfig": { 2274 "defaults": { 2275 "links": [] 2276 }, 2277 "overrides": [] 2278 }, 2279 "fill": 0, 2280 "fillGradient": 0, 2281 "gridPos": { 2282 "h": 7, 2283 "w": 12, 2284 "x": 0, 2285 "y": 66 2286 }, 2287 "hiddenSeries": false, 2288 "id": 14, 2289 "legend": { 2290 "avg": false, 2291 "current": false, 2292 "max": false, 2293 "min": false, 2294 "show": true, 2295 "total": false, 2296 "values": false 2297 }, 2298 "lines": true, 2299 "linewidth": 1, 2300 "links": [], 2301 "nullPointMode": "null", 2302 "options": { 2303 "alertThreshold": true 2304 }, 2305 "percentage": false, 2306 "pluginVersion": "9.2.4", 2307 "pointradius": 5, 2308 "points": false, 2309 "renderer": "flot", 2310 "seriesOverrides": [], 2311 "spaceLength": 10, 2312 "stack": false, 2313 "steppedLine": false, 2314 "targets": [ 2315 { 2316 "datasource": { 2317 "uid": "${prometheus}" 2318 }, 2319 "expr": "sum(irate(bookie_journal_JOURNAL_SYNC_count{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\"}[30s])) by (kubernetes_pod_name)", 2320 "format": "time_series", 2321 "interval": "", 2322 "intervalFactor": 2, 2323 "legendFormat": "{{kubernetes_pod_name}}", 2324 "refId": "A", 2325 "step": 4 2326 } 2327 ], 2328 "thresholds": [], 2329 "timeRegions": [], 2330 "title": "Journal Sync Rate", 2331 "tooltip": { 2332 "shared": true, 2333 "sort": 0, 2334 "value_type": "individual" 2335 }, 2336 "type": "graph", 2337 "xaxis": { 2338 "mode": "time", 2339 "show": true, 2340 "values": [] 2341 }, 2342 "yaxes": [ 2343 { 2344 "format": "short", 2345 "logBase": 1, 2346 "min": "0", 2347 "show": true 2348 }, 2349 { 2350 "format": "short", 2351 "logBase": 1, 2352 "show": true 2353 } 2354 ], 2355 "yaxis": { 2356 "align": false 2357 } 2358 }, 2359 { 2360 "aliasColors": {}, 2361 "bars": false, 2362 "dashLength": 10, 2363 "dashes": false, 2364 "datasource": { 2365 "uid": "${prometheus}" 2366 }, 2367 "decimals": 1, 2368 "fieldConfig": { 2369 "defaults": { 2370 "links": [] 2371 }, 2372 "overrides": [] 2373 }, 2374 "fill": 0, 2375 "fillGradient": 0, 2376 "gridPos": { 2377 "h": 7, 2378 "w": 12, 2379 "x": 12, 2380 "y": 66 2381 }, 2382 "hiddenSeries": false, 2383 "id": 40, 2384 "legend": { 2385 "avg": false, 2386 "current": false, 2387 "max": false, 2388 "min": false, 2389 "show": true, 2390 "total": false, 2391 "values": false 2392 }, 2393 "lines": true, 2394 "linewidth": 1, 2395 "links": [], 2396 "nullPointMode": "null", 2397 "options": { 2398 "alertThreshold": true 2399 }, 2400 "percentage": false, 2401 "pluginVersion": "9.2.4", 2402 "pointradius": 5, 2403 "points": false, 2404 "renderer": "flot", 2405 "seriesOverrides": [], 2406 "spaceLength": 10, 2407 "stack": false, 2408 "steppedLine": false, 2409 "targets": [ 2410 { 2411 "datasource": { 2412 "uid": "${prometheus}" 2413 }, 2414 "expr": "sum(irate(bookie_journal_JOURNAL_ADD_ENTRY_count{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\"}[30s])) by (kubernetes_pod_name)", 2415 "format": "time_series", 2416 "interval": "", 2417 "intervalFactor": 2, 2418 "legendFormat": "{{kubernetes_pod_name}}", 2419 "refId": "A", 2420 "step": 4 2421 } 2422 ], 2423 "thresholds": [], 2424 "timeRegions": [], 2425 "title": "Journal Add Rate", 2426 "tooltip": { 2427 "shared": true, 2428 "sort": 0, 2429 "value_type": "individual" 2430 }, 2431 "type": "graph", 2432 "xaxis": { 2433 "mode": "time", 2434 "show": true, 2435 "values": [] 2436 }, 2437 "yaxes": [ 2438 { 2439 "format": "short", 2440 "logBase": 1, 2441 "min": "0", 2442 "show": true 2443 }, 2444 { 2445 "format": "short", 2446 "logBase": 1, 2447 "show": true 2448 } 2449 ], 2450 "yaxis": { 2451 "align": false 2452 } 2453 }, 2454 { 2455 "aliasColors": {}, 2456 "bars": false, 2457 "dashLength": 10, 2458 "dashes": false, 2459 "datasource": { 2460 "uid": "${prometheus}" 2461 }, 2462 "decimals": 1, 2463 "fieldConfig": { 2464 "defaults": { 2465 "links": [] 2466 }, 2467 "overrides": [] 2468 }, 2469 "fill": 0, 2470 "fillGradient": 0, 2471 "gridPos": { 2472 "h": 7, 2473 "w": 8, 2474 "x": 0, 2475 "y": 73 2476 }, 2477 "hiddenSeries": false, 2478 "id": 11, 2479 "legend": { 2480 "avg": false, 2481 "current": false, 2482 "max": false, 2483 "min": false, 2484 "show": true, 2485 "total": false, 2486 "values": false 2487 }, 2488 "lines": true, 2489 "linewidth": 1, 2490 "links": [], 2491 "nullPointMode": "null", 2492 "options": { 2493 "alertThreshold": true 2494 }, 2495 "percentage": false, 2496 "pluginVersion": "9.2.4", 2497 "pointradius": 5, 2498 "points": false, 2499 "renderer": "flot", 2500 "seriesOverrides": [], 2501 "spaceLength": 10, 2502 "stack": false, 2503 "steppedLine": false, 2504 "targets": [ 2505 { 2506 "datasource": { 2507 "uid": "${prometheus}" 2508 }, 2509 "expr": "sum(bookie_journal_JOURNAL_QUEUE_SIZE{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}) by (kubernetes_pod_name)", 2510 "format": "time_series", 2511 "interval": "", 2512 "intervalFactor": 2, 2513 "legendFormat": "{{kubernetes_pod_name}}", 2514 "refId": "A", 2515 "step": 2 2516 } 2517 ], 2518 "thresholds": [], 2519 "timeRegions": [], 2520 "title": "Journal Queue length", 2521 "tooltip": { 2522 "shared": true, 2523 "sort": 0, 2524 "value_type": "individual" 2525 }, 2526 "type": "graph", 2527 "xaxis": { 2528 "mode": "time", 2529 "show": true, 2530 "values": [] 2531 }, 2532 "yaxes": [ 2533 { 2534 "format": "short", 2535 "logBase": 1, 2536 "show": true 2537 }, 2538 { 2539 "format": "short", 2540 "logBase": 1, 2541 "show": true 2542 } 2543 ], 2544 "yaxis": { 2545 "align": false 2546 } 2547 }, 2548 { 2549 "aliasColors": {}, 2550 "bars": false, 2551 "dashLength": 10, 2552 "dashes": false, 2553 "datasource": { 2554 "uid": "${prometheus}" 2555 }, 2556 "decimals": 1, 2557 "fieldConfig": { 2558 "defaults": { 2559 "links": [] 2560 }, 2561 "overrides": [] 2562 }, 2563 "fill": 0, 2564 "fillGradient": 0, 2565 "gridPos": { 2566 "h": 7, 2567 "w": 8, 2568 "x": 8, 2569 "y": 73 2570 }, 2571 "hiddenSeries": false, 2572 "id": 36, 2573 "legend": { 2574 "avg": false, 2575 "current": false, 2576 "max": false, 2577 "min": false, 2578 "show": true, 2579 "total": false, 2580 "values": false 2581 }, 2582 "lines": true, 2583 "linewidth": 1, 2584 "links": [], 2585 "nullPointMode": "null", 2586 "options": { 2587 "alertThreshold": true 2588 }, 2589 "percentage": false, 2590 "pluginVersion": "9.2.4", 2591 "pointradius": 5, 2592 "points": false, 2593 "renderer": "flot", 2594 "seriesOverrides": [], 2595 "spaceLength": 10, 2596 "stack": false, 2597 "steppedLine": false, 2598 "targets": [ 2599 { 2600 "datasource": { 2601 "uid": "${prometheus}" 2602 }, 2603 "expr": "sum(bookie_journal_JOURNAL_FORCE_WRITE_QUEUE_SIZE{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}) by (kubernetes_pod_name)", 2604 "format": "time_series", 2605 "interval": "", 2606 "intervalFactor": 2, 2607 "legendFormat": "{{kubernetes_pod_name}}", 2608 "refId": "A", 2609 "step": 2 2610 } 2611 ], 2612 "thresholds": [], 2613 "timeRegions": [], 2614 "title": "Journal Force Write Queue length", 2615 "tooltip": { 2616 "shared": true, 2617 "sort": 0, 2618 "value_type": "individual" 2619 }, 2620 "type": "graph", 2621 "xaxis": { 2622 "mode": "time", 2623 "show": true, 2624 "values": [] 2625 }, 2626 "yaxes": [ 2627 { 2628 "format": "short", 2629 "logBase": 1, 2630 "show": true 2631 }, 2632 { 2633 "format": "short", 2634 "logBase": 1, 2635 "show": true 2636 } 2637 ], 2638 "yaxis": { 2639 "align": false 2640 } 2641 }, 2642 { 2643 "aliasColors": {}, 2644 "bars": false, 2645 "dashLength": 10, 2646 "dashes": false, 2647 "datasource": { 2648 "uid": "${prometheus}" 2649 }, 2650 "decimals": 1, 2651 "fieldConfig": { 2652 "defaults": { 2653 "links": [] 2654 }, 2655 "overrides": [] 2656 }, 2657 "fill": 0, 2658 "fillGradient": 0, 2659 "gridPos": { 2660 "h": 7, 2661 "w": 8, 2662 "x": 16, 2663 "y": 73 2664 }, 2665 "hiddenSeries": false, 2666 "id": 37, 2667 "legend": { 2668 "avg": false, 2669 "current": false, 2670 "max": false, 2671 "min": false, 2672 "show": true, 2673 "total": false, 2674 "values": false 2675 }, 2676 "lines": true, 2677 "linewidth": 1, 2678 "links": [], 2679 "nullPointMode": "null", 2680 "options": { 2681 "alertThreshold": true 2682 }, 2683 "percentage": false, 2684 "pluginVersion": "9.2.4", 2685 "pointradius": 5, 2686 "points": false, 2687 "renderer": "flot", 2688 "seriesOverrides": [], 2689 "spaceLength": 10, 2690 "stack": false, 2691 "steppedLine": false, 2692 "targets": [ 2693 { 2694 "datasource": { 2695 "uid": "${prometheus}" 2696 }, 2697 "expr": "sum(irate(bookie_journal_JOURNAL_CB_QUEUE_SIZE{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"} [30s])) by (kubernetes_pod_name)", 2698 "format": "time_series", 2699 "interval": "", 2700 "intervalFactor": 2, 2701 "legendFormat": "{{kubernetes_pod_name}}", 2702 "refId": "A", 2703 "step": 2 2704 } 2705 ], 2706 "thresholds": [], 2707 "timeRegions": [], 2708 "title": "Journal Callback Queue length", 2709 "tooltip": { 2710 "shared": true, 2711 "sort": 0, 2712 "value_type": "individual" 2713 }, 2714 "type": "graph", 2715 "xaxis": { 2716 "mode": "time", 2717 "show": true, 2718 "values": [] 2719 }, 2720 "yaxes": [ 2721 { 2722 "format": "short", 2723 "logBase": 1, 2724 "show": true 2725 }, 2726 { 2727 "format": "short", 2728 "logBase": 1, 2729 "show": true 2730 } 2731 ], 2732 "yaxis": { 2733 "align": false 2734 } 2735 }, 2736 { 2737 "aliasColors": {}, 2738 "bars": false, 2739 "dashLength": 10, 2740 "dashes": false, 2741 "datasource": { 2742 "uid": "${prometheus}" 2743 }, 2744 "decimals": 1, 2745 "fieldConfig": { 2746 "defaults": { 2747 "links": [] 2748 }, 2749 "overrides": [] 2750 }, 2751 "fill": 0, 2752 "fillGradient": 0, 2753 "gridPos": { 2754 "h": 7, 2755 "w": 8, 2756 "x": 0, 2757 "y": 80 2758 }, 2759 "hiddenSeries": false, 2760 "id": 10, 2761 "legend": { 2762 "avg": false, 2763 "current": false, 2764 "max": false, 2765 "min": false, 2766 "show": true, 2767 "total": false, 2768 "values": false 2769 }, 2770 "lines": true, 2771 "linewidth": 1, 2772 "links": [], 2773 "nullPointMode": "null", 2774 "options": { 2775 "alertThreshold": true 2776 }, 2777 "percentage": false, 2778 "pluginVersion": "9.2.4", 2779 "pointradius": 5, 2780 "points": false, 2781 "renderer": "flot", 2782 "seriesOverrides": [], 2783 "spaceLength": 10, 2784 "stack": false, 2785 "steppedLine": false, 2786 "targets": [ 2787 { 2788 "datasource": { 2789 "uid": "${prometheus}" 2790 }, 2791 "expr": "sum(bookie_journal_JOURNAL_ADD_ENTRY{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\", quantile=\"0.5\"}) by (kubernetes_pod_name)", 2792 "format": "time_series", 2793 "interval": "", 2794 "intervalFactor": 2, 2795 "legendFormat": "{{kubernetes_pod_name}}", 2796 "refId": "A", 2797 "step": 2 2798 } 2799 ], 2800 "thresholds": [], 2801 "timeRegions": [], 2802 "title": "Journal Add latency (p50)", 2803 "tooltip": { 2804 "shared": true, 2805 "sort": 0, 2806 "value_type": "individual" 2807 }, 2808 "type": "graph", 2809 "xaxis": { 2810 "mode": "time", 2811 "show": true, 2812 "values": [] 2813 }, 2814 "yaxes": [ 2815 { 2816 "format": "ms", 2817 "logBase": 1, 2818 "min": "0", 2819 "show": true 2820 }, 2821 { 2822 "format": "short", 2823 "logBase": 1, 2824 "show": true 2825 } 2826 ], 2827 "yaxis": { 2828 "align": false 2829 } 2830 }, 2831 { 2832 "aliasColors": {}, 2833 "bars": false, 2834 "dashLength": 10, 2835 "dashes": false, 2836 "datasource": { 2837 "uid": "${prometheus}" 2838 }, 2839 "decimals": 1, 2840 "fieldConfig": { 2841 "defaults": { 2842 "links": [] 2843 }, 2844 "overrides": [] 2845 }, 2846 "fill": 0, 2847 "fillGradient": 0, 2848 "gridPos": { 2849 "h": 7, 2850 "w": 8, 2851 "x": 8, 2852 "y": 80 2853 }, 2854 "hiddenSeries": false, 2855 "id": 38, 2856 "legend": { 2857 "avg": false, 2858 "current": false, 2859 "max": false, 2860 "min": false, 2861 "show": true, 2862 "total": false, 2863 "values": false 2864 }, 2865 "lines": true, 2866 "linewidth": 1, 2867 "links": [], 2868 "nullPointMode": "null", 2869 "options": { 2870 "alertThreshold": true 2871 }, 2872 "percentage": false, 2873 "pluginVersion": "9.2.4", 2874 "pointradius": 5, 2875 "points": false, 2876 "renderer": "flot", 2877 "seriesOverrides": [], 2878 "spaceLength": 10, 2879 "stack": false, 2880 "steppedLine": false, 2881 "targets": [ 2882 { 2883 "datasource": { 2884 "uid": "${prometheus}" 2885 }, 2886 "expr": "sum(bookie_journal_JOURNAL_ADD_ENTRY{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\", quantile=\"0.99\"}) by (kubernetes_pod_name)", 2887 "format": "time_series", 2888 "interval": "", 2889 "intervalFactor": 2, 2890 "legendFormat": "{{kubernetes_pod_name}}", 2891 "refId": "A", 2892 "step": 2 2893 } 2894 ], 2895 "thresholds": [], 2896 "timeRegions": [], 2897 "title": "Journal Add latency (p99)", 2898 "tooltip": { 2899 "shared": true, 2900 "sort": 0, 2901 "value_type": "individual" 2902 }, 2903 "type": "graph", 2904 "xaxis": { 2905 "mode": "time", 2906 "show": true, 2907 "values": [] 2908 }, 2909 "yaxes": [ 2910 { 2911 "format": "ms", 2912 "logBase": 1, 2913 "min": "0", 2914 "show": true 2915 }, 2916 { 2917 "format": "short", 2918 "logBase": 1, 2919 "show": true 2920 } 2921 ], 2922 "yaxis": { 2923 "align": false 2924 } 2925 }, 2926 { 2927 "aliasColors": {}, 2928 "bars": false, 2929 "dashLength": 10, 2930 "dashes": false, 2931 "datasource": { 2932 "type": "prometheus", 2933 "uid": "${prometheus}" 2934 }, 2935 "decimals": 1, 2936 "fieldConfig": { 2937 "defaults": { 2938 "links": [] 2939 }, 2940 "overrides": [] 2941 }, 2942 "fill": 0, 2943 "fillGradient": 0, 2944 "gridPos": { 2945 "h": 7, 2946 "w": 8, 2947 "x": 16, 2948 "y": 80 2949 }, 2950 "hiddenSeries": false, 2951 "id": 39, 2952 "legend": { 2953 "avg": false, 2954 "current": false, 2955 "max": false, 2956 "min": false, 2957 "show": true, 2958 "total": false, 2959 "values": false 2960 }, 2961 "lines": true, 2962 "linewidth": 1, 2963 "links": [], 2964 "nullPointMode": "null", 2965 "options": { 2966 "alertThreshold": true 2967 }, 2968 "percentage": false, 2969 "pluginVersion": "9.2.4", 2970 "pointradius": 5, 2971 "points": false, 2972 "renderer": "flot", 2973 "seriesOverrides": [], 2974 "spaceLength": 10, 2975 "stack": false, 2976 "steppedLine": false, 2977 "targets": [ 2978 { 2979 "datasource": { 2980 "uid": "${prometheus}" 2981 }, 2982 "editorMode": "code", 2983 "expr": "sum(bookie_journal_JOURNAL_ADD_ENTRY{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\", quantile=\"1\"}) by (kubernetes_pod_name)", 2984 "format": "time_series", 2985 "interval": "", 2986 "intervalFactor": 2, 2987 "legendFormat": "{{kubernetes_pod_name}}", 2988 "range": true, 2989 "refId": "A", 2990 "step": 2 2991 } 2992 ], 2993 "thresholds": [], 2994 "timeRegions": [], 2995 "title": "Journal Add latency (Max)", 2996 "tooltip": { 2997 "shared": true, 2998 "sort": 0, 2999 "value_type": "individual" 3000 }, 3001 "type": "graph", 3002 "xaxis": { 3003 "mode": "time", 3004 "show": true, 3005 "values": [] 3006 }, 3007 "yaxes": [ 3008 { 3009 "format": "ms", 3010 "logBase": 1, 3011 "min": "0", 3012 "show": true 3013 }, 3014 { 3015 "format": "short", 3016 "logBase": 1, 3017 "show": true 3018 } 3019 ], 3020 "yaxis": { 3021 "align": false 3022 } 3023 }, 3024 { 3025 "aliasColors": {}, 3026 "bars": false, 3027 "dashLength": 10, 3028 "dashes": false, 3029 "datasource": { 3030 "uid": "${prometheus}" 3031 }, 3032 "decimals": 1, 3033 "fieldConfig": { 3034 "defaults": { 3035 "links": [] 3036 }, 3037 "overrides": [] 3038 }, 3039 "fill": 0, 3040 "fillGradient": 0, 3041 "gridPos": { 3042 "h": 7, 3043 "w": 8, 3044 "x": 0, 3045 "y": 87 3046 }, 3047 "hiddenSeries": false, 3048 "id": 15, 3049 "legend": { 3050 "avg": false, 3051 "current": false, 3052 "max": false, 3053 "min": false, 3054 "show": true, 3055 "total": false, 3056 "values": false 3057 }, 3058 "lines": true, 3059 "linewidth": 1, 3060 "links": [], 3061 "nullPointMode": "null", 3062 "options": { 3063 "alertThreshold": true 3064 }, 3065 "percentage": false, 3066 "pluginVersion": "9.2.4", 3067 "pointradius": 5, 3068 "points": false, 3069 "renderer": "flot", 3070 "seriesOverrides": [], 3071 "spaceLength": 10, 3072 "stack": false, 3073 "steppedLine": false, 3074 "targets": [ 3075 { 3076 "datasource": { 3077 "uid": "${prometheus}" 3078 }, 3079 "expr": "sum(bookie_journal_JOURNAL_SYNC{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\", quantile=\"0.5\"}) by (kubernetes_pod_name)", 3080 "format": "time_series", 3081 "interval": "", 3082 "intervalFactor": 2, 3083 "legendFormat": "{{kubernetes_pod_name}}", 3084 "refId": "A", 3085 "step": 4 3086 } 3087 ], 3088 "thresholds": [], 3089 "timeRegions": [], 3090 "title": "Journal Sync Latency (p50)", 3091 "tooltip": { 3092 "shared": true, 3093 "sort": 0, 3094 "value_type": "individual" 3095 }, 3096 "type": "graph", 3097 "xaxis": { 3098 "mode": "time", 3099 "show": true, 3100 "values": [] 3101 }, 3102 "yaxes": [ 3103 { 3104 "format": "ms", 3105 "logBase": 1, 3106 "min": "0", 3107 "show": true 3108 }, 3109 { 3110 "format": "short", 3111 "logBase": 1, 3112 "show": true 3113 } 3114 ], 3115 "yaxis": { 3116 "align": false 3117 } 3118 }, 3119 { 3120 "aliasColors": {}, 3121 "bars": false, 3122 "dashLength": 10, 3123 "dashes": false, 3124 "datasource": { 3125 "uid": "${prometheus}" 3126 }, 3127 "decimals": 1, 3128 "fieldConfig": { 3129 "defaults": { 3130 "links": [] 3131 }, 3132 "overrides": [] 3133 }, 3134 "fill": 0, 3135 "fillGradient": 0, 3136 "gridPos": { 3137 "h": 7, 3138 "w": 8, 3139 "x": 8, 3140 "y": 87 3141 }, 3142 "hiddenSeries": false, 3143 "id": 41, 3144 "legend": { 3145 "avg": false, 3146 "current": false, 3147 "max": false, 3148 "min": false, 3149 "show": true, 3150 "total": false, 3151 "values": false 3152 }, 3153 "lines": true, 3154 "linewidth": 1, 3155 "links": [], 3156 "nullPointMode": "null", 3157 "options": { 3158 "alertThreshold": true 3159 }, 3160 "percentage": false, 3161 "pluginVersion": "9.2.4", 3162 "pointradius": 5, 3163 "points": false, 3164 "renderer": "flot", 3165 "seriesOverrides": [], 3166 "spaceLength": 10, 3167 "stack": false, 3168 "steppedLine": false, 3169 "targets": [ 3170 { 3171 "datasource": { 3172 "uid": "${prometheus}" 3173 }, 3174 "expr": "sum(bookie_journal_JOURNAL_SYNC{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\", quantile=\"0.99\"}) by (kubernetes_pod_name)", 3175 "format": "time_series", 3176 "interval": "", 3177 "intervalFactor": 2, 3178 "legendFormat": "{{kubernetes_pod_name}}", 3179 "refId": "A", 3180 "step": 4 3181 } 3182 ], 3183 "thresholds": [], 3184 "timeRegions": [], 3185 "title": "Journal Sync Latency (p99)", 3186 "tooltip": { 3187 "shared": true, 3188 "sort": 0, 3189 "value_type": "individual" 3190 }, 3191 "type": "graph", 3192 "xaxis": { 3193 "mode": "time", 3194 "show": true, 3195 "values": [] 3196 }, 3197 "yaxes": [ 3198 { 3199 "format": "ms", 3200 "logBase": 1, 3201 "min": "0", 3202 "show": true 3203 }, 3204 { 3205 "format": "short", 3206 "logBase": 1, 3207 "show": true 3208 } 3209 ], 3210 "yaxis": { 3211 "align": false 3212 } 3213 }, 3214 { 3215 "aliasColors": {}, 3216 "bars": false, 3217 "dashLength": 10, 3218 "dashes": false, 3219 "datasource": { 3220 "type": "prometheus", 3221 "uid": "${prometheus}" 3222 }, 3223 "decimals": 1, 3224 "fieldConfig": { 3225 "defaults": { 3226 "links": [] 3227 }, 3228 "overrides": [] 3229 }, 3230 "fill": 0, 3231 "fillGradient": 0, 3232 "gridPos": { 3233 "h": 7, 3234 "w": 8, 3235 "x": 16, 3236 "y": 87 3237 }, 3238 "hiddenSeries": false, 3239 "id": 16, 3240 "legend": { 3241 "avg": false, 3242 "current": false, 3243 "max": false, 3244 "min": false, 3245 "show": true, 3246 "total": false, 3247 "values": false 3248 }, 3249 "lines": true, 3250 "linewidth": 1, 3251 "links": [], 3252 "nullPointMode": "null", 3253 "options": { 3254 "alertThreshold": true 3255 }, 3256 "percentage": false, 3257 "pluginVersion": "9.2.4", 3258 "pointradius": 5, 3259 "points": false, 3260 "renderer": "flot", 3261 "seriesOverrides": [], 3262 "spaceLength": 10, 3263 "stack": false, 3264 "steppedLine": false, 3265 "targets": [ 3266 { 3267 "datasource": { 3268 "uid": "${prometheus}" 3269 }, 3270 "editorMode": "code", 3271 "expr": "sum(bookie_journal_JOURNAL_SYNC{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\", success=\"true\", quantile=\"1\"}) by (kubernetes_pod_name)", 3272 "format": "time_series", 3273 "interval": "", 3274 "intervalFactor": 2, 3275 "legendFormat": "{{kubernetes_pod_name}}", 3276 "range": true, 3277 "refId": "A", 3278 "step": 4 3279 } 3280 ], 3281 "thresholds": [], 3282 "timeRegions": [], 3283 "title": "Journal Sync Latency (Max)", 3284 "tooltip": { 3285 "shared": true, 3286 "sort": 0, 3287 "value_type": "individual" 3288 }, 3289 "type": "graph", 3290 "xaxis": { 3291 "mode": "time", 3292 "show": true, 3293 "values": [] 3294 }, 3295 "yaxes": [ 3296 { 3297 "format": "ms", 3298 "logBase": 1, 3299 "min": "0", 3300 "show": true 3301 }, 3302 { 3303 "format": "short", 3304 "logBase": 1, 3305 "show": true 3306 } 3307 ], 3308 "yaxis": { 3309 "align": false 3310 } 3311 }, 3312 { 3313 "collapsed": false, 3314 "datasource": { 3315 "type": "prometheus", 3316 "uid": "prometheus" 3317 }, 3318 "gridPos": { 3319 "h": 1, 3320 "w": 24, 3321 "x": 0, 3322 "y": 94 3323 }, 3324 "id": 43, 3325 "panels": [], 3326 "targets": [ 3327 { 3328 "datasource": { 3329 "type": "prometheus", 3330 "uid": "prometheus" 3331 }, 3332 "refId": "A" 3333 } 3334 ], 3335 "title": "Storage", 3336 "type": "row" 3337 }, 3338 { 3339 "aliasColors": {}, 3340 "bars": false, 3341 "dashLength": 10, 3342 "dashes": false, 3343 "datasource": { 3344 "uid": "${prometheus}" 3345 }, 3346 "decimals": 1, 3347 "fieldConfig": { 3348 "defaults": { 3349 "links": [] 3350 }, 3351 "overrides": [] 3352 }, 3353 "fill": 0, 3354 "fillGradient": 0, 3355 "gridPos": { 3356 "h": 7, 3357 "w": 12, 3358 "x": 0, 3359 "y": 95 3360 }, 3361 "hiddenSeries": false, 3362 "id": 8, 3363 "legend": { 3364 "avg": false, 3365 "current": false, 3366 "max": false, 3367 "min": false, 3368 "show": true, 3369 "total": false, 3370 "values": false 3371 }, 3372 "lines": true, 3373 "linewidth": 1, 3374 "links": [], 3375 "nullPointMode": "null", 3376 "options": { 3377 "alertThreshold": true 3378 }, 3379 "percentage": false, 3380 "pluginVersion": "9.2.4", 3381 "pointradius": 5, 3382 "points": false, 3383 "renderer": "flot", 3384 "seriesOverrides": [], 3385 "spaceLength": 10, 3386 "stack": false, 3387 "steppedLine": false, 3388 "targets": [ 3389 { 3390 "datasource": { 3391 "uid": "${prometheus}" 3392 }, 3393 "expr": "sum(bookie_ledgers_count{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}) by (kubernetes_pod_name)", 3394 "format": "time_series", 3395 "interval": "", 3396 "intervalFactor": 2, 3397 "legendFormat": "{{kubernetes_pod_name}}", 3398 "refId": "A", 3399 "step": 2 3400 } 3401 ], 3402 "thresholds": [], 3403 "timeRegions": [], 3404 "title": "Ledgers count", 3405 "tooltip": { 3406 "shared": true, 3407 "sort": 0, 3408 "value_type": "individual" 3409 }, 3410 "type": "graph", 3411 "xaxis": { 3412 "mode": "time", 3413 "show": true, 3414 "values": [] 3415 }, 3416 "yaxes": [ 3417 { 3418 "format": "short", 3419 "logBase": 1, 3420 "min": "0", 3421 "show": true 3422 }, 3423 { 3424 "format": "short", 3425 "logBase": 1, 3426 "show": true 3427 } 3428 ], 3429 "yaxis": { 3430 "align": false 3431 } 3432 }, 3433 { 3434 "aliasColors": {}, 3435 "bars": false, 3436 "dashLength": 10, 3437 "dashes": false, 3438 "datasource": { 3439 "uid": "${prometheus}" 3440 }, 3441 "decimals": 1, 3442 "fieldConfig": { 3443 "defaults": { 3444 "links": [] 3445 }, 3446 "overrides": [] 3447 }, 3448 "fill": 0, 3449 "fillGradient": 0, 3450 "gridPos": { 3451 "h": 7, 3452 "w": 12, 3453 "x": 12, 3454 "y": 95 3455 }, 3456 "hiddenSeries": false, 3457 "id": 9, 3458 "legend": { 3459 "avg": false, 3460 "current": false, 3461 "max": false, 3462 "min": false, 3463 "show": true, 3464 "total": false, 3465 "values": false 3466 }, 3467 "lines": true, 3468 "linewidth": 1, 3469 "links": [], 3470 "nullPointMode": "null", 3471 "options": { 3472 "alertThreshold": true 3473 }, 3474 "percentage": false, 3475 "pluginVersion": "9.2.4", 3476 "pointradius": 5, 3477 "points": false, 3478 "renderer": "flot", 3479 "seriesOverrides": [], 3480 "spaceLength": 10, 3481 "stack": false, 3482 "steppedLine": false, 3483 "targets": [ 3484 { 3485 "datasource": { 3486 "uid": "${prometheus}" 3487 }, 3488 "expr": "sum(bookie_entries_count{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}) by (kubernetes_pod_name)", 3489 "format": "time_series", 3490 "interval": "", 3491 "intervalFactor": 2, 3492 "legendFormat": "{{kubernetes_pod_name}}", 3493 "refId": "A", 3494 "step": 2 3495 } 3496 ], 3497 "thresholds": [], 3498 "timeRegions": [], 3499 "title": "Entries count", 3500 "tooltip": { 3501 "shared": true, 3502 "sort": 0, 3503 "value_type": "individual" 3504 }, 3505 "type": "graph", 3506 "xaxis": { 3507 "mode": "time", 3508 "show": true, 3509 "values": [] 3510 }, 3511 "yaxes": [ 3512 { 3513 "format": "short", 3514 "logBase": 1, 3515 "min": "0", 3516 "show": true 3517 }, 3518 { 3519 "format": "short", 3520 "logBase": 1, 3521 "show": true 3522 } 3523 ], 3524 "yaxis": { 3525 "align": false 3526 } 3527 }, 3528 { 3529 "aliasColors": {}, 3530 "bars": false, 3531 "dashLength": 10, 3532 "dashes": false, 3533 "datasource": { 3534 "uid": "${prometheus}" 3535 }, 3536 "decimals": 1, 3537 "fieldConfig": { 3538 "defaults": { 3539 "links": [] 3540 }, 3541 "overrides": [] 3542 }, 3543 "fill": 0, 3544 "fillGradient": 0, 3545 "gridPos": { 3546 "h": 7, 3547 "w": 12, 3548 "x": 0, 3549 "y": 102 3550 }, 3551 "hiddenSeries": false, 3552 "id": 18, 3553 "legend": { 3554 "avg": false, 3555 "current": false, 3556 "max": false, 3557 "min": false, 3558 "show": true, 3559 "total": false, 3560 "values": false 3561 }, 3562 "lines": true, 3563 "linewidth": 1, 3564 "links": [], 3565 "nullPointMode": "null", 3566 "options": { 3567 "alertThreshold": true 3568 }, 3569 "percentage": false, 3570 "pluginVersion": "9.2.4", 3571 "pointradius": 5, 3572 "points": false, 3573 "renderer": "flot", 3574 "seriesOverrides": [], 3575 "spaceLength": 10, 3576 "stack": false, 3577 "steppedLine": false, 3578 "targets": [ 3579 { 3580 "datasource": { 3581 "uid": "${prometheus}" 3582 }, 3583 "expr": "sum(bookie_write_cache_size{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}) by (kubernetes_pod_name)", 3584 "format": "time_series", 3585 "interval": "", 3586 "intervalFactor": 2, 3587 "legendFormat": "{{kubernetes_pod_name}}", 3588 "refId": "A", 3589 "step": 2 3590 } 3591 ], 3592 "thresholds": [], 3593 "timeRegions": [], 3594 "title": "Write Cache Size", 3595 "tooltip": { 3596 "shared": true, 3597 "sort": 0, 3598 "value_type": "individual" 3599 }, 3600 "type": "graph", 3601 "xaxis": { 3602 "mode": "time", 3603 "show": true, 3604 "values": [] 3605 }, 3606 "yaxes": [ 3607 { 3608 "format": "decbytes", 3609 "logBase": 1, 3610 "min": "0", 3611 "show": true 3612 }, 3613 { 3614 "format": "short", 3615 "logBase": 1, 3616 "show": true 3617 } 3618 ], 3619 "yaxis": { 3620 "align": false 3621 } 3622 }, 3623 { 3624 "aliasColors": {}, 3625 "bars": false, 3626 "dashLength": 10, 3627 "dashes": false, 3628 "datasource": { 3629 "uid": "${prometheus}" 3630 }, 3631 "decimals": 1, 3632 "fieldConfig": { 3633 "defaults": { 3634 "links": [] 3635 }, 3636 "overrides": [] 3637 }, 3638 "fill": 0, 3639 "fillGradient": 0, 3640 "gridPos": { 3641 "h": 7, 3642 "w": 12, 3643 "x": 12, 3644 "y": 102 3645 }, 3646 "hiddenSeries": false, 3647 "id": 19, 3648 "legend": { 3649 "avg": false, 3650 "current": false, 3651 "max": false, 3652 "min": false, 3653 "show": true, 3654 "total": false, 3655 "values": false 3656 }, 3657 "lines": true, 3658 "linewidth": 1, 3659 "links": [], 3660 "nullPointMode": "null", 3661 "options": { 3662 "alertThreshold": true 3663 }, 3664 "percentage": false, 3665 "pluginVersion": "9.2.4", 3666 "pointradius": 5, 3667 "points": false, 3668 "renderer": "flot", 3669 "seriesOverrides": [], 3670 "spaceLength": 10, 3671 "stack": false, 3672 "steppedLine": false, 3673 "targets": [ 3674 { 3675 "datasource": { 3676 "uid": "${prometheus}" 3677 }, 3678 "expr": "sum(bookie_read_cache_size{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}) by (kubernetes_pod_name)", 3679 "format": "time_series", 3680 "interval": "", 3681 "intervalFactor": 2, 3682 "legendFormat": "{{kubernetes_pod_name}}", 3683 "refId": "A", 3684 "step": 2 3685 } 3686 ], 3687 "thresholds": [], 3688 "timeRegions": [], 3689 "title": "Read Cache Size", 3690 "tooltip": { 3691 "shared": true, 3692 "sort": 0, 3693 "value_type": "individual" 3694 }, 3695 "type": "graph", 3696 "xaxis": { 3697 "mode": "time", 3698 "show": true, 3699 "values": [] 3700 }, 3701 "yaxes": [ 3702 { 3703 "format": "decbytes", 3704 "logBase": 1, 3705 "min": "0", 3706 "show": true 3707 }, 3708 { 3709 "format": "short", 3710 "logBase": 1, 3711 "show": true 3712 } 3713 ], 3714 "yaxis": { 3715 "align": false 3716 } 3717 }, 3718 { 3719 "aliasColors": {}, 3720 "bars": false, 3721 "dashLength": 10, 3722 "dashes": false, 3723 "datasource": { 3724 "type": "prometheus", 3725 "uid": "${prometheus}" 3726 }, 3727 "decimals": 1, 3728 "fieldConfig": { 3729 "defaults": { 3730 "links": [] 3731 }, 3732 "overrides": [] 3733 }, 3734 "fill": 0, 3735 "fillGradient": 0, 3736 "gridPos": { 3737 "h": 7, 3738 "w": 12, 3739 "x": 0, 3740 "y": 109 3741 }, 3742 "hiddenSeries": false, 3743 "id": 66, 3744 "legend": { 3745 "avg": false, 3746 "current": false, 3747 "max": false, 3748 "min": false, 3749 "show": true, 3750 "total": false, 3751 "values": false 3752 }, 3753 "lines": true, 3754 "linewidth": 1, 3755 "links": [], 3756 "nullPointMode": "null", 3757 "options": { 3758 "alertThreshold": true 3759 }, 3760 "percentage": false, 3761 "pluginVersion": "9.2.4", 3762 "pointradius": 5, 3763 "points": false, 3764 "renderer": "flot", 3765 "seriesOverrides": [], 3766 "spaceLength": 10, 3767 "stack": false, 3768 "steppedLine": false, 3769 "targets": [ 3770 { 3771 "datasource": { 3772 "uid": "${prometheus}" 3773 }, 3774 "editorMode": "code", 3775 "expr": "sum(irate(bookie_read_cache_hits{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}[1m])) by (kubernetes_pod_name)", 3776 "format": "time_series", 3777 "interval": "", 3778 "intervalFactor": 2, 3779 "legendFormat": "{{kubernetes_pod_name}}", 3780 "range": true, 3781 "refId": "A", 3782 "step": 2 3783 } 3784 ], 3785 "thresholds": [], 3786 "timeRegions": [], 3787 "title": "Read entry rate from read cache", 3788 "tooltip": { 3789 "shared": true, 3790 "sort": 0, 3791 "value_type": "individual" 3792 }, 3793 "type": "graph", 3794 "xaxis": { 3795 "mode": "time", 3796 "show": true, 3797 "values": [] 3798 }, 3799 "yaxes": [ 3800 { 3801 "$$hashKey": "object:1471", 3802 "format": "decbytes", 3803 "logBase": 1, 3804 "min": "0", 3805 "show": true 3806 }, 3807 { 3808 "$$hashKey": "object:1472", 3809 "format": "short", 3810 "logBase": 1, 3811 "show": true 3812 } 3813 ], 3814 "yaxis": { 3815 "align": false 3816 } 3817 }, 3818 { 3819 "aliasColors": {}, 3820 "bars": false, 3821 "dashLength": 10, 3822 "dashes": false, 3823 "datasource": { 3824 "type": "prometheus", 3825 "uid": "${prometheus}" 3826 }, 3827 "decimals": 1, 3828 "fieldConfig": { 3829 "defaults": { 3830 "links": [] 3831 }, 3832 "overrides": [] 3833 }, 3834 "fill": 0, 3835 "fillGradient": 0, 3836 "gridPos": { 3837 "h": 7, 3838 "w": 12, 3839 "x": 12, 3840 "y": 109 3841 }, 3842 "hiddenSeries": false, 3843 "id": 67, 3844 "legend": { 3845 "avg": false, 3846 "current": false, 3847 "max": false, 3848 "min": false, 3849 "show": true, 3850 "total": false, 3851 "values": false 3852 }, 3853 "lines": true, 3854 "linewidth": 1, 3855 "links": [], 3856 "nullPointMode": "null", 3857 "options": { 3858 "alertThreshold": true 3859 }, 3860 "percentage": false, 3861 "pluginVersion": "9.2.4", 3862 "pointradius": 5, 3863 "points": false, 3864 "renderer": "flot", 3865 "seriesOverrides": [], 3866 "spaceLength": 10, 3867 "stack": false, 3868 "steppedLine": false, 3869 "targets": [ 3870 { 3871 "datasource": { 3872 "uid": "${prometheus}" 3873 }, 3874 "editorMode": "code", 3875 "expr": "sum(irate(bookie_read_cache_misses{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}[1m])) by (kubernetes_pod_name)", 3876 "format": "time_series", 3877 "interval": "", 3878 "intervalFactor": 2, 3879 "legendFormat": "{{kubernetes_pod_name}}", 3880 "range": true, 3881 "refId": "A", 3882 "step": 2 3883 } 3884 ], 3885 "thresholds": [], 3886 "timeRegions": [], 3887 "title": "Read entry rate cache misses", 3888 "tooltip": { 3889 "shared": true, 3890 "sort": 0, 3891 "value_type": "individual" 3892 }, 3893 "type": "graph", 3894 "xaxis": { 3895 "mode": "time", 3896 "show": true, 3897 "values": [] 3898 }, 3899 "yaxes": [ 3900 { 3901 "format": "decbytes", 3902 "logBase": 1, 3903 "min": "0", 3904 "show": true 3905 }, 3906 { 3907 "format": "short", 3908 "logBase": 1, 3909 "show": true 3910 } 3911 ], 3912 "yaxis": { 3913 "align": false 3914 } 3915 }, 3916 { 3917 "aliasColors": {}, 3918 "bars": false, 3919 "dashLength": 10, 3920 "dashes": false, 3921 "datasource": { 3922 "uid": "${prometheus}" 3923 }, 3924 "fieldConfig": { 3925 "defaults": { 3926 "links": [] 3927 }, 3928 "overrides": [] 3929 }, 3930 "fill": 1, 3931 "fillGradient": 0, 3932 "gridPos": { 3933 "h": 9, 3934 "w": 12, 3935 "x": 0, 3936 "y": 116 3937 }, 3938 "hiddenSeries": false, 3939 "id": 45, 3940 "legend": { 3941 "avg": false, 3942 "current": false, 3943 "max": false, 3944 "min": false, 3945 "show": true, 3946 "total": false, 3947 "values": false 3948 }, 3949 "lines": true, 3950 "linewidth": 1, 3951 "links": [], 3952 "nullPointMode": "null", 3953 "options": { 3954 "alertThreshold": true 3955 }, 3956 "percentage": false, 3957 "pluginVersion": "9.2.4", 3958 "pointradius": 5, 3959 "points": false, 3960 "renderer": "flot", 3961 "seriesOverrides": [], 3962 "spaceLength": 10, 3963 "stack": false, 3964 "steppedLine": false, 3965 "targets": [ 3966 { 3967 "datasource": { 3968 "uid": "${prometheus}" 3969 }, 3970 "expr": "sum(bookie_DELETED_LEDGER_COUNT{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}) by (kubernetes_pod_name)", 3971 "format": "time_series", 3972 "interval": "", 3973 "intervalFactor": 1, 3974 "legendFormat": "{{kubernetes_pod_name}}", 3975 "refId": "A" 3976 } 3977 ], 3978 "thresholds": [], 3979 "timeRegions": [], 3980 "title": "Deleted Ledger Count", 3981 "tooltip": { 3982 "shared": true, 3983 "sort": 0, 3984 "value_type": "individual" 3985 }, 3986 "type": "graph", 3987 "xaxis": { 3988 "mode": "time", 3989 "show": true, 3990 "values": [] 3991 }, 3992 "yaxes": [ 3993 { 3994 "format": "short", 3995 "logBase": 1, 3996 "show": true 3997 }, 3998 { 3999 "format": "short", 4000 "logBase": 1, 4001 "show": true 4002 } 4003 ], 4004 "yaxis": { 4005 "align": false 4006 } 4007 }, 4008 { 4009 "aliasColors": {}, 4010 "bars": false, 4011 "dashLength": 10, 4012 "dashes": false, 4013 "datasource": { 4014 "uid": "${prometheus}" 4015 }, 4016 "fieldConfig": { 4017 "defaults": { 4018 "links": [] 4019 }, 4020 "overrides": [] 4021 }, 4022 "fill": 0, 4023 "fillGradient": 0, 4024 "gridPos": { 4025 "h": 9, 4026 "w": 12, 4027 "x": 12, 4028 "y": 116 4029 }, 4030 "hiddenSeries": false, 4031 "id": 46, 4032 "legend": { 4033 "avg": false, 4034 "current": false, 4035 "max": false, 4036 "min": false, 4037 "show": true, 4038 "total": false, 4039 "values": false 4040 }, 4041 "lines": true, 4042 "linewidth": 1, 4043 "links": [], 4044 "nullPointMode": "null", 4045 "options": { 4046 "alertThreshold": true 4047 }, 4048 "percentage": false, 4049 "pluginVersion": "9.2.4", 4050 "pointradius": 5, 4051 "points": false, 4052 "renderer": "flot", 4053 "seriesOverrides": [], 4054 "spaceLength": 10, 4055 "stack": false, 4056 "steppedLine": false, 4057 "targets": [ 4058 { 4059 "datasource": { 4060 "uid": "${prometheus}" 4061 }, 4062 "expr": "sum(bookie_ledger_writable_dirs{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}) by (kubernetes_pod_name)", 4063 "format": "time_series", 4064 "interval": "", 4065 "intervalFactor": 1, 4066 "legendFormat": "{{kubernetes_pod_name}}", 4067 "refId": "A" 4068 } 4069 ], 4070 "thresholds": [], 4071 "timeRegions": [], 4072 "title": "Writable Ledger Directory Count", 4073 "tooltip": { 4074 "shared": true, 4075 "sort": 0, 4076 "value_type": "individual" 4077 }, 4078 "type": "graph", 4079 "xaxis": { 4080 "mode": "time", 4081 "show": true, 4082 "values": [] 4083 }, 4084 "yaxes": [ 4085 { 4086 "format": "short", 4087 "logBase": 1, 4088 "show": true 4089 }, 4090 { 4091 "format": "short", 4092 "logBase": 1, 4093 "show": true 4094 } 4095 ], 4096 "yaxis": { 4097 "align": false 4098 } 4099 }, 4100 { 4101 "aliasColors": {}, 4102 "bars": false, 4103 "dashLength": 10, 4104 "dashes": false, 4105 "datasource": { 4106 "uid": "${prometheus}" 4107 }, 4108 "fieldConfig": { 4109 "defaults": { 4110 "links": [] 4111 }, 4112 "overrides": [] 4113 }, 4114 "fill": 1, 4115 "fillGradient": 0, 4116 "gridPos": { 4117 "h": 9, 4118 "w": 12, 4119 "x": 0, 4120 "y": 125 4121 }, 4122 "hiddenSeries": false, 4123 "id": 57, 4124 "legend": { 4125 "avg": false, 4126 "current": false, 4127 "max": false, 4128 "min": false, 4129 "show": true, 4130 "total": false, 4131 "values": false 4132 }, 4133 "lines": true, 4134 "linewidth": 1, 4135 "links": [], 4136 "nullPointMode": "null", 4137 "options": { 4138 "alertThreshold": true 4139 }, 4140 "percentage": false, 4141 "pluginVersion": "9.2.4", 4142 "pointradius": 5, 4143 "points": false, 4144 "renderer": "flot", 4145 "seriesOverrides": [], 4146 "spaceLength": 10, 4147 "stack": false, 4148 "steppedLine": false, 4149 "targets": [ 4150 { 4151 "datasource": { 4152 "uid": "${prometheus}" 4153 }, 4154 "expr": "sum(bookie_MAJOR_COMPACTION_COUNT{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}) by (kubernetes_pod_name)", 4155 "format": "time_series", 4156 "interval": "", 4157 "intervalFactor": 1, 4158 "legendFormat": "major compaction - {{kubernetes_pod_name}}", 4159 "refId": "A" 4160 }, 4161 { 4162 "datasource": { 4163 "uid": "${prometheus}" 4164 }, 4165 "expr": "sum(bookie_MINOR_COMPACTION_COUNT{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}) by (kubernetes_pod_name)", 4166 "format": "time_series", 4167 "interval": "", 4168 "intervalFactor": 1, 4169 "legendFormat": "minor compaction - {{kubernetes_pod_name}}", 4170 "refId": "B" 4171 } 4172 ], 4173 "thresholds": [], 4174 "timeRegions": [], 4175 "title": "Compaction", 4176 "tooltip": { 4177 "shared": true, 4178 "sort": 0, 4179 "value_type": "individual" 4180 }, 4181 "type": "graph", 4182 "xaxis": { 4183 "mode": "time", 4184 "show": true, 4185 "values": [] 4186 }, 4187 "yaxes": [ 4188 { 4189 "$$hashKey": "object:3218", 4190 "format": "short", 4191 "logBase": 1, 4192 "show": true 4193 }, 4194 { 4195 "$$hashKey": "object:3219", 4196 "format": "short", 4197 "logBase": 1, 4198 "show": true 4199 } 4200 ], 4201 "yaxis": { 4202 "align": false 4203 } 4204 }, 4205 { 4206 "aliasColors": {}, 4207 "bars": false, 4208 "dashLength": 10, 4209 "dashes": false, 4210 "datasource": { 4211 "uid": "${prometheus}" 4212 }, 4213 "fieldConfig": { 4214 "defaults": { 4215 "links": [] 4216 }, 4217 "overrides": [] 4218 }, 4219 "fill": 1, 4220 "fillGradient": 0, 4221 "gridPos": { 4222 "h": 9, 4223 "w": 12, 4224 "x": 12, 4225 "y": 125 4226 }, 4227 "hiddenSeries": false, 4228 "id": 63, 4229 "legend": { 4230 "avg": false, 4231 "current": false, 4232 "max": false, 4233 "min": false, 4234 "show": true, 4235 "total": false, 4236 "values": false 4237 }, 4238 "lines": true, 4239 "linewidth": 1, 4240 "links": [], 4241 "nullPointMode": "null", 4242 "options": { 4243 "alertThreshold": true 4244 }, 4245 "percentage": false, 4246 "pluginVersion": "9.2.4", 4247 "pointradius": 5, 4248 "points": false, 4249 "renderer": "flot", 4250 "seriesOverrides": [], 4251 "spaceLength": 10, 4252 "stack": false, 4253 "steppedLine": false, 4254 "targets": [ 4255 { 4256 "datasource": { 4257 "uid": "${prometheus}" 4258 }, 4259 "expr": "sum(rate(bookie_RECLAIMED_COMPACTION_SPACE_BYTES{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}[30s])) by (kubernetes_pod_name)", 4260 "format": "time_series", 4261 "interval": "", 4262 "intervalFactor": 1, 4263 "legendFormat": "compaction - {{kubernetes_pod_name}}", 4264 "refId": "A" 4265 }, 4266 { 4267 "datasource": { 4268 "uid": "${prometheus}" 4269 }, 4270 "expr": "sum(rate(bookie_RECLAIMED_DELETION_SPACE_BYTES{app_kubernetes_io_instance=~\"$cluster\", kubernetes_pod_name=~\"$kubernetes_pod_name\"}[30s])) by (kubernetes_pod_name)", 4271 "format": "time_series", 4272 "interval": "", 4273 "intervalFactor": 1, 4274 "legendFormat": "deletion - {{kubernetes_pod_name}}", 4275 "refId": "B" 4276 } 4277 ], 4278 "thresholds": [], 4279 "timeRegions": [], 4280 "title": "GC Reclaim Disk (bytes)", 4281 "tooltip": { 4282 "shared": true, 4283 "sort": 0, 4284 "value_type": "individual" 4285 }, 4286 "type": "graph", 4287 "xaxis": { 4288 "mode": "time", 4289 "show": true, 4290 "values": [] 4291 }, 4292 "yaxes": [ 4293 { 4294 "$$hashKey": "object:3218", 4295 "format": "decbytes", 4296 "logBase": 1, 4297 "show": true 4298 }, 4299 { 4300 "$$hashKey": "object:3219", 4301 "format": "short", 4302 "logBase": 1, 4303 "show": true 4304 } 4305 ], 4306 "yaxis": { 4307 "align": false 4308 } 4309 } 4310 ], 4311 "refresh": "", 4312 "schemaVersion": 37, 4313 "style": "dark", 4314 "tags": [ 4315 "pulsar", 4316 "bookie" 4317 ], 4318 "templating": { 4319 "list": [ 4320 { 4321 "current": { 4322 "selected": false, 4323 "text": "default", 4324 "value": "default" 4325 }, 4326 "hide": 0, 4327 "includeAll": false, 4328 "label": "data source", 4329 "multi": false, 4330 "name": "prometheus", 4331 "options": [], 4332 "query": "prometheus", 4333 "queryValue": "", 4334 "refresh": 1, 4335 "regex": "", 4336 "skipUrlSync": false, 4337 "type": "datasource" 4338 }, 4339 { 4340 "current": { 4341 "selected": false, 4342 "text": "pulsar-cluster", 4343 "value": "pulsar-cluster" 4344 }, 4345 "datasource": { 4346 "type": "prometheus", 4347 "uid": "${prometheus}" 4348 }, 4349 "definition": "label_values(bookkeeper_server_ADD_ENTRY_BLOCKED{job=\"bookie\"}, app_kubernetes_io_instance)", 4350 "hide": 0, 4351 "includeAll": false, 4352 "label": "Cluster", 4353 "multi": false, 4354 "name": "cluster", 4355 "options": [], 4356 "query": { 4357 "query": "label_values(bookkeeper_server_ADD_ENTRY_BLOCKED{job=\"bookie\"}, app_kubernetes_io_instance)", 4358 "refId": "StandardVariableQuery" 4359 }, 4360 "refresh": 1, 4361 "regex": "", 4362 "skipUrlSync": false, 4363 "sort": 5, 4364 "tagValuesQuery": "", 4365 "tagsQuery": "", 4366 "type": "query", 4367 "useTags": false 4368 }, 4369 { 4370 "allValue": ".+", 4371 "current": { 4372 "selected": true, 4373 "text": [ 4374 "All" 4375 ], 4376 "value": [ 4377 "$__all" 4378 ] 4379 }, 4380 "datasource": { 4381 "type": "prometheus", 4382 "uid": "${prometheus}" 4383 }, 4384 "definition": "label_values(bookkeeper_server_ADD_ENTRY_BLOCKED{app_kubernetes_io_instance=\"$cluster\", job=\"bookie\"}, kubernetes_pod_name)\n", 4385 "hide": 0, 4386 "includeAll": true, 4387 "label": "Bookie", 4388 "multi": true, 4389 "name": "kubernetes_pod_name", 4390 "options": [], 4391 "query": { 4392 "query": "label_values(bookkeeper_server_ADD_ENTRY_BLOCKED{app_kubernetes_io_instance=\"$cluster\", job=\"bookie\"}, kubernetes_pod_name)\n", 4393 "refId": "StandardVariableQuery" 4394 }, 4395 "refresh": 2, 4396 "regex": "", 4397 "skipUrlSync": false, 4398 "sort": 5, 4399 "tagValuesQuery": "", 4400 "tagsQuery": "", 4401 "type": "query", 4402 "useTags": false 4403 } 4404 ] 4405 }, 4406 "time": { 4407 "from": "now-30m", 4408 "to": "now" 4409 }, 4410 "timepicker": { 4411 "refresh_intervals": [ 4412 "5s", 4413 "10s", 4414 "30s", 4415 "1m", 4416 "5m", 4417 "15m", 4418 "30m", 4419 "1h", 4420 "2h", 4421 "1d" 4422 ], 4423 "time_options": [ 4424 "5m", 4425 "15m", 4426 "1h", 4427 "6h", 4428 "12h", 4429 "24h", 4430 "2d", 4431 "7d", 4432 "30d" 4433 ] 4434 }, 4435 "timezone": "", 4436 "title": "Pulsar / Bookie Metrics", 4437 "uid": "pulsar-bookie", 4438 "version": 1, 4439 "weekStart": "" 4440 }