github.com/grafana/pyroscope@v1.18.0/operations/monitoring/dashboards/v2-metastore.json (about)

     1  {
     2    "annotations": {
     3      "list": [
     4        {
     5          "builtIn": 1,
     6          "datasource": {
     7            "type": "grafana",
     8            "uid": "-- Grafana --"
     9          },
    10          "enable": true,
    11          "hide": true,
    12          "iconColor": "rgba(0, 211, 255, 1)",
    13          "name": "Annotations \u0026 Alerts",
    14          "type": "dashboard"
    15        },
    16        {
    17          "datasource": {
    18            "type": "loki",
    19            "uid": "${loki_datasource}"
    20          },
    21          "enable": true,
    22          "expr": "{cluster=\"$cluster\", container=\"kube-diff-logger\"} | json | namespace_extracted=\"$namespace\"",
    23          "hide": false,
    24          "iconColor": "yellow",
    25          "instant": false,
    26          "name": "K8s Changes",
    27          "tagKeys": "type,name_extracted,verb",
    28          "textFormat": "{{notes}}",
    29          "titleFormat": ""
    30        },
    31        {
    32          "datasource": {
    33            "type": "loki",
    34            "uid": "${loki_datasource}"
    35          },
    36          "enable": false,
    37          "expr": "{namespace=\"$namespace\",container=\"eventrouter\"} != \"Preemption is not helpful\" != \"Failed deploy model\" | logfmt | type=\"Warning\"",
    38          "hide": false,
    39          "iconColor": "orange",
    40          "instant": false,
    41          "name": "K8s Warnings",
    42          "tagKeys": "type,object_kind,object_name",
    43          "textFormat": "{{message}}",
    44          "titleFormat": ""
    45        },
    46        {
    47          "datasource": {
    48            "type": "loki",
    49            "uid": "${loki_datasource}"
    50          },
    51          "enable": true,
    52          "expr": "{namespace=\"$namespace\",container=\"eventrouter\"} | logfmt | type!=\"Normal\" | type!=\"Warning\"",
    53          "hide": false,
    54          "iconColor": "red",
    55          "instant": false,
    56          "name": "K8s Errors",
    57          "tagKeys": "type,object_kind,object_name",
    58          "textFormat": "{{message}}"
    59        },
    60        {
    61          "datasource": {
    62            "type": "loki",
    63            "uid": "${loki_datasource}"
    64          },
    65          "enable": false,
    66          "expr": "{namespace=\"$namespace\",container=\"eventrouter\"} | logfmt",
    67          "hide": false,
    68          "iconColor": "blue",
    69          "instant": false,
    70          "name": "K8s All Events",
    71          "tagKeys": "type,object_kind,object_name",
    72          "textFormat": "{{message}}"
    73        }
    74      ]
    75    },
    76    "editable": true,
    77    "fiscalYearStartMonth": 0,
    78    "graphTooltip": 1,
    79    "id": 12198,
    80    "links": [
    81      {
    82        "asDropdown": true,
    83        "icon": "external link",
    84        "includeVars": true,
    85        "keepTime": true,
    86        "tags": [
    87          "pyroscope"
    88        ],
    89        "targetBlank": false,
    90        "title": "Pyroscope Dashboards",
    91        "type": "dashboards"
    92      }
    93    ],
    94    "panels": [
    95      {
    96        "collapsed": false,
    97        "gridPos": {
    98          "h": 1,
    99          "w": 24,
   100          "x": 0,
   101          "y": 0
   102        },
   103        "id": 71,
   104        "panels": [],
   105        "title": "Raft",
   106        "type": "row"
   107      },
   108      {
   109        "datasource": {
   110          "type": "prometheus",
   111          "uid": "${datasource}"
   112        },
   113        "description": "Raft node roles observed by each node. The view is not meant to be 100% consistent, and only serves monitoring purposes",
   114        "fieldConfig": {
   115          "defaults": {
   116            "color": {
   117              "mode": "continuous-GrYlRd"
   118            },
   119            "custom": {
   120              "axisPlacement": "auto",
   121              "fillOpacity": 15,
   122              "hideFrom": {
   123                "legend": false,
   124                "tooltip": false,
   125                "viz": false
   126              },
   127              "insertNulls": false,
   128              "lineWidth": 1,
   129              "spanNulls": false
   130            },
   131            "fieldMinMax": false,
   132            "mappings": [
   133              {
   134                "options": {
   135                  "from": 0,
   136                  "result": {
   137                    "color": "green",
   138                    "index": 0,
   139                    "text": "Follower"
   140                  },
   141                  "to": 99
   142                },
   143                "type": "range"
   144              },
   145              {
   146                "options": {
   147                  "from": 100,
   148                  "result": {
   149                    "color": "dark-red",
   150                    "index": 1,
   151                    "text": "Leader"
   152                  },
   153                  "to": 1000000
   154                },
   155                "type": "range"
   156              }
   157            ],
   158            "thresholds": {
   159              "mode": "absolute",
   160              "steps": [
   161                {
   162                  "color": "dark-blue",
   163                  "value": 0
   164                },
   165                {
   166                  "color": "light-red",
   167                  "value": 100
   168                }
   169              ]
   170            },
   171            "unit": "short"
   172          },
   173          "overrides": []
   174        },
   175        "gridPos": {
   176          "h": 7,
   177          "w": 8,
   178          "x": 0,
   179          "y": 1
   180        },
   181        "id": 81,
   182        "options": {
   183          "alignValue": "center",
   184          "legend": {
   185            "displayMode": "list",
   186            "placement": "bottom",
   187            "showLegend": false
   188          },
   189          "mergeValues": true,
   190          "rowHeight": 0.8,
   191          "showValue": "never",
   192          "tooltip": {
   193            "hideZeros": false,
   194            "mode": "multi",
   195            "sort": "asc"
   196          }
   197        },
   198        "pluginVersion": "12.1.0-90017",
   199        "targets": [
   200          {
   201            "datasource": {
   202              "type": "prometheus",
   203              "uid": "${datasource}"
   204            },
   205            "editorMode": "code",
   206            "exemplar": false,
   207            "expr": "sum by (pod) (\n  pyroscope_metastore_raft_state{namespace=\"$namespace\",state=\"Follower\"} or\n  pyroscope_metastore_raft_state{namespace=\"$namespace\",state=\"Leader\"} * 100\n)",
   208            "format": "time_series",
   209            "hide": false,
   210            "instant": false,
   211            "legendFormat": "{{label_name}}",
   212            "range": true,
   213            "refId": "D"
   214          }
   215        ],
   216        "title": "Raft Nodes",
   217        "transparent": true,
   218        "type": "state-timeline"
   219      },
   220      {
   221        "datasource": {
   222          "type": "prometheus",
   223          "uid": "${datasource}"
   224        },
   225        "description": "The latency of raft commands observed by the metastore leader",
   226        "fieldConfig": {
   227          "defaults": {
   228            "color": {
   229              "mode": "palette-classic"
   230            },
   231            "custom": {
   232              "axisBorderShow": false,
   233              "axisCenteredZero": false,
   234              "axisColorMode": "text",
   235              "axisLabel": "",
   236              "axisPlacement": "auto",
   237              "barAlignment": 0,
   238              "barWidthFactor": 0.6,
   239              "drawStyle": "line",
   240              "fillOpacity": 0,
   241              "gradientMode": "none",
   242              "hideFrom": {
   243                "legend": false,
   244                "tooltip": false,
   245                "viz": false
   246              },
   247              "insertNulls": false,
   248              "lineInterpolation": "linear",
   249              "lineWidth": 1,
   250              "pointSize": 5,
   251              "scaleDistribution": {
   252                "type": "linear"
   253              },
   254              "showPoints": "auto",
   255              "spanNulls": false,
   256              "stacking": {
   257                "group": "A",
   258                "mode": "none"
   259              },
   260              "thresholdsStyle": {
   261                "mode": "off"
   262              }
   263            },
   264            "mappings": [],
   265            "thresholds": {
   266              "mode": "absolute",
   267              "steps": [
   268                {
   269                  "color": "green",
   270                  "value": 0
   271                },
   272                {
   273                  "color": "red",
   274                  "value": 80
   275                }
   276              ]
   277            },
   278            "unit": "s"
   279          },
   280          "overrides": [
   281            {
   282              "matcher": {
   283                "id": "byFrameRefID",
   284                "options": "B"
   285              },
   286              "properties": [
   287                {
   288                  "id": "custom.drawStyle",
   289                  "value": "points"
   290                },
   291                {
   292                  "id": "color",
   293                  "value": {
   294                    "fixedColor": "red",
   295                    "mode": "fixed"
   296                  }
   297                },
   298                {
   299                  "id": "custom.pointSize",
   300                  "value": 11
   301                }
   302              ]
   303            }
   304          ]
   305        },
   306        "gridPos": {
   307          "h": 8,
   308          "w": 8,
   309          "x": 8,
   310          "y": 1
   311        },
   312        "id": 68,
   313        "options": {
   314          "legend": {
   315            "calcs": [],
   316            "displayMode": "list",
   317            "placement": "bottom",
   318            "showLegend": true
   319          },
   320          "tooltip": {
   321            "hideZeros": false,
   322            "mode": "single",
   323            "sort": "none"
   324          }
   325        },
   326        "pluginVersion": "12.1.0-90017",
   327        "targets": [
   328          {
   329            "datasource": {
   330              "type": "prometheus",
   331              "uid": "${datasource}"
   332            },
   333            "editorMode": "code",
   334            "expr": "histogram_quantile(0.995, sum(rate(pyroscope_metastore_raft_apply_command_duration_seconds{\n  namespace=\"$namespace\",\n}[$__rate_interval])) by (le))",
   335            "hide": false,
   336            "instant": false,
   337            "legendFormat": "p99",
   338            "range": true,
   339            "refId": "D"
   340          }
   341        ],
   342        "title": "Command Latency",
   343        "transparent": true,
   344        "type": "timeseries"
   345      },
   346      {
   347        "datasource": {
   348          "type": "prometheus",
   349          "uid": "${datasource}"
   350        },
   351        "description": "The latency of raft read index observed by the metastore nodes",
   352        "fieldConfig": {
   353          "defaults": {
   354            "color": {
   355              "mode": "palette-classic"
   356            },
   357            "custom": {
   358              "axisBorderShow": false,
   359              "axisCenteredZero": false,
   360              "axisColorMode": "text",
   361              "axisLabel": "",
   362              "axisPlacement": "auto",
   363              "barAlignment": 0,
   364              "barWidthFactor": 0.6,
   365              "drawStyle": "line",
   366              "fillOpacity": 0,
   367              "gradientMode": "none",
   368              "hideFrom": {
   369                "legend": false,
   370                "tooltip": false,
   371                "viz": false
   372              },
   373              "insertNulls": false,
   374              "lineInterpolation": "linear",
   375              "lineWidth": 1,
   376              "pointSize": 5,
   377              "scaleDistribution": {
   378                "type": "linear"
   379              },
   380              "showPoints": "auto",
   381              "spanNulls": false,
   382              "stacking": {
   383                "group": "A",
   384                "mode": "none"
   385              },
   386              "thresholdsStyle": {
   387                "mode": "off"
   388              }
   389            },
   390            "mappings": [],
   391            "thresholds": {
   392              "mode": "absolute",
   393              "steps": [
   394                {
   395                  "color": "green",
   396                  "value": 0
   397                },
   398                {
   399                  "color": "red",
   400                  "value": 80
   401                }
   402              ]
   403            },
   404            "unit": "s"
   405          },
   406          "overrides": [
   407            {
   408              "matcher": {
   409                "id": "byFrameRefID",
   410                "options": "B"
   411              },
   412              "properties": [
   413                {
   414                  "id": "custom.drawStyle",
   415                  "value": "points"
   416                },
   417                {
   418                  "id": "color",
   419                  "value": {
   420                    "fixedColor": "red",
   421                    "mode": "fixed"
   422                  }
   423                },
   424                {
   425                  "id": "custom.pointSize",
   426                  "value": 11
   427                }
   428              ]
   429            }
   430          ]
   431        },
   432        "gridPos": {
   433          "h": 8,
   434          "w": 8,
   435          "x": 16,
   436          "y": 1
   437        },
   438        "id": 72,
   439        "options": {
   440          "legend": {
   441            "calcs": [],
   442            "displayMode": "list",
   443            "placement": "bottom",
   444            "showLegend": true
   445          },
   446          "tooltip": {
   447            "hideZeros": false,
   448            "mode": "single",
   449            "sort": "none"
   450          }
   451        },
   452        "pluginVersion": "12.1.0-90017",
   453        "targets": [
   454          {
   455            "datasource": {
   456              "type": "prometheus",
   457              "uid": "${datasource}"
   458            },
   459            "editorMode": "code",
   460            "expr": "histogram_quantile(0.99, sum(rate(pyroscope_metastore_raft_read_index_wait_duration_seconds{\n  namespace=\"$namespace\",\n}[$__rate_interval])) by (le))",
   461            "hide": false,
   462            "instant": false,
   463            "legendFormat": "p99",
   464            "range": true,
   465            "refId": "D"
   466          }
   467        ],
   468        "title": "ReadIndex Latency",
   469        "transparent": true,
   470        "type": "timeseries"
   471      },
   472      {
   473        "datasource": {
   474          "type": "prometheus",
   475          "uid": "${datasource}"
   476        },
   477        "description": "The time takes to write a snapshot",
   478        "fieldConfig": {
   479          "defaults": {
   480            "color": {
   481              "mode": "palette-classic"
   482            },
   483            "custom": {
   484              "axisBorderShow": false,
   485              "axisCenteredZero": false,
   486              "axisColorMode": "text",
   487              "axisLabel": "",
   488              "axisPlacement": "auto",
   489              "barAlignment": 0,
   490              "barWidthFactor": 0.6,
   491              "drawStyle": "line",
   492              "fillOpacity": 0,
   493              "gradientMode": "none",
   494              "hideFrom": {
   495                "legend": false,
   496                "tooltip": false,
   497                "viz": false
   498              },
   499              "insertNulls": false,
   500              "lineInterpolation": "linear",
   501              "lineWidth": 1,
   502              "pointSize": 5,
   503              "scaleDistribution": {
   504                "type": "linear"
   505              },
   506              "showPoints": "auto",
   507              "spanNulls": false,
   508              "stacking": {
   509                "group": "A",
   510                "mode": "none"
   511              },
   512              "thresholdsStyle": {
   513                "mode": "off"
   514              }
   515            },
   516            "mappings": [],
   517            "thresholds": {
   518              "mode": "absolute",
   519              "steps": [
   520                {
   521                  "color": "green",
   522                  "value": 0
   523                },
   524                {
   525                  "color": "red",
   526                  "value": 80
   527                }
   528              ]
   529            },
   530            "unit": "s"
   531          },
   532          "overrides": [
   533            {
   534              "matcher": {
   535                "id": "byFrameRefID",
   536                "options": "B"
   537              },
   538              "properties": [
   539                {
   540                  "id": "custom.drawStyle",
   541                  "value": "points"
   542                },
   543                {
   544                  "id": "color",
   545                  "value": {
   546                    "fixedColor": "red",
   547                    "mode": "fixed"
   548                  }
   549                },
   550                {
   551                  "id": "custom.pointSize",
   552                  "value": 11
   553                }
   554              ]
   555            }
   556          ]
   557        },
   558        "gridPos": {
   559          "h": 8,
   560          "w": 8,
   561          "x": 0,
   562          "y": 8
   563        },
   564        "id": 78,
   565        "options": {
   566          "legend": {
   567            "calcs": [],
   568            "displayMode": "list",
   569            "placement": "bottom",
   570            "showLegend": false
   571          },
   572          "tooltip": {
   573            "hideZeros": false,
   574            "mode": "single",
   575            "sort": "none"
   576          }
   577        },
   578        "pluginVersion": "12.1.0-90017",
   579        "targets": [
   580          {
   581            "datasource": {
   582              "type": "prometheus",
   583              "uid": "${datasource}"
   584            },
   585            "editorMode": "code",
   586            "expr": "(\n  histogram_quantile(0.99,(sum by (le, pod) (\n    irate(pyroscope_metastore_boltdb_persist_snapshot_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])))\n  ) * on(pod) group_left() (pyroscope_metastore_raft_state{namespace=\"$namespace\", state=\"Leader\"} \u003e= 1) \n)",
   587            "hide": false,
   588            "instant": false,
   589            "legendFormat": "{{pod}} Leader",
   590            "range": true,
   591            "refId": "B"
   592          },
   593          {
   594            "datasource": {
   595              "type": "prometheus",
   596              "uid": "${datasource}"
   597            },
   598            "editorMode": "code",
   599            "expr": "(\n  histogram_quantile(0.99,(sum by (le, pod) (\n    irate(pyroscope_metastore_boltdb_persist_snapshot_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])))\n  ) * on(pod) group_left() (pyroscope_metastore_raft_state{namespace=\"$namespace\", state!=\"Leader\"} \u003e= 1) \n)",
   600            "hide": false,
   601            "instant": false,
   602            "legendFormat": "{{pod}} Follower",
   603            "range": true,
   604            "refId": "A"
   605          }
   606        ],
   607        "title": "Snapshot Latency",
   608        "transparent": true,
   609        "type": "timeseries"
   610      },
   611      {
   612        "datasource": {
   613          "type": "prometheus",
   614          "uid": "${datasource}"
   615        },
   616        "description": "The size of BoltDB snapshot taken by FSM",
   617        "fieldConfig": {
   618          "defaults": {
   619            "color": {
   620              "mode": "palette-classic"
   621            },
   622            "custom": {
   623              "axisBorderShow": false,
   624              "axisCenteredZero": false,
   625              "axisColorMode": "text",
   626              "axisLabel": "",
   627              "axisPlacement": "auto",
   628              "barAlignment": 0,
   629              "barWidthFactor": 0.6,
   630              "drawStyle": "line",
   631              "fillOpacity": 0,
   632              "gradientMode": "none",
   633              "hideFrom": {
   634                "legend": false,
   635                "tooltip": false,
   636                "viz": false
   637              },
   638              "insertNulls": false,
   639              "lineInterpolation": "linear",
   640              "lineWidth": 1,
   641              "pointSize": 5,
   642              "scaleDistribution": {
   643                "type": "linear"
   644              },
   645              "showPoints": "auto",
   646              "spanNulls": false,
   647              "stacking": {
   648                "group": "A",
   649                "mode": "none"
   650              },
   651              "thresholdsStyle": {
   652                "mode": "off"
   653              }
   654            },
   655            "mappings": [],
   656            "thresholds": {
   657              "mode": "absolute",
   658              "steps": [
   659                {
   660                  "color": "green",
   661                  "value": 0
   662                },
   663                {
   664                  "color": "red",
   665                  "value": 80
   666                }
   667              ]
   668            },
   669            "unit": "decbytes"
   670          },
   671          "overrides": [
   672            {
   673              "matcher": {
   674                "id": "byFrameRefID",
   675                "options": "B"
   676              },
   677              "properties": [
   678                {
   679                  "id": "custom.drawStyle",
   680                  "value": "points"
   681                },
   682                {
   683                  "id": "color",
   684                  "value": {
   685                    "fixedColor": "red",
   686                    "mode": "fixed"
   687                  }
   688                },
   689                {
   690                  "id": "custom.pointSize",
   691                  "value": 11
   692                }
   693              ]
   694            }
   695          ]
   696        },
   697        "gridPos": {
   698          "h": 8,
   699          "w": 8,
   700          "x": 8,
   701          "y": 9
   702        },
   703        "id": 77,
   704        "options": {
   705          "legend": {
   706            "calcs": [],
   707            "displayMode": "list",
   708            "placement": "bottom",
   709            "showLegend": false
   710          },
   711          "tooltip": {
   712            "hideZeros": false,
   713            "mode": "single",
   714            "sort": "none"
   715          }
   716        },
   717        "pluginVersion": "12.1.0-90017",
   718        "targets": [
   719          {
   720            "datasource": {
   721              "type": "prometheus",
   722              "uid": "${datasource}"
   723            },
   724            "editorMode": "code",
   725            "expr": "pyroscope_metastore_boltdb_persist_snapshot_size_bytes{namespace=\"$namespace\"}",
   726            "hide": false,
   727            "instant": false,
   728            "legendFormat": "{{pod}}",
   729            "range": true,
   730            "refId": "D"
   731          }
   732        ],
   733        "title": "Snapshot Size",
   734        "transparent": true,
   735        "type": "timeseries"
   736      },
   737      {
   738        "datasource": {
   739          "type": "prometheus",
   740          "uid": "${datasource}"
   741        },
   742        "description": "The time takes to apply the command onto FSM",
   743        "fieldConfig": {
   744          "defaults": {
   745            "color": {
   746              "mode": "palette-classic"
   747            },
   748            "custom": {
   749              "axisBorderShow": false,
   750              "axisCenteredZero": false,
   751              "axisColorMode": "text",
   752              "axisLabel": "",
   753              "axisPlacement": "auto",
   754              "barAlignment": 0,
   755              "barWidthFactor": 0.6,
   756              "drawStyle": "line",
   757              "fillOpacity": 0,
   758              "gradientMode": "none",
   759              "hideFrom": {
   760                "legend": false,
   761                "tooltip": false,
   762                "viz": false
   763              },
   764              "insertNulls": false,
   765              "lineInterpolation": "linear",
   766              "lineWidth": 1,
   767              "pointSize": 5,
   768              "scaleDistribution": {
   769                "type": "linear"
   770              },
   771              "showPoints": "auto",
   772              "spanNulls": false,
   773              "stacking": {
   774                "group": "A",
   775                "mode": "none"
   776              },
   777              "thresholdsStyle": {
   778                "mode": "off"
   779              }
   780            },
   781            "mappings": [],
   782            "thresholds": {
   783              "mode": "absolute",
   784              "steps": [
   785                {
   786                  "color": "green",
   787                  "value": 0
   788                },
   789                {
   790                  "color": "red",
   791                  "value": 80
   792                }
   793              ]
   794            },
   795            "unit": "s"
   796          },
   797          "overrides": []
   798        },
   799        "gridPos": {
   800          "h": 8,
   801          "w": 8,
   802          "x": 16,
   803          "y": 9
   804        },
   805        "id": 79,
   806        "options": {
   807          "legend": {
   808            "calcs": [],
   809            "displayMode": "list",
   810            "placement": "bottom",
   811            "showLegend": false
   812          },
   813          "tooltip": {
   814            "hideZeros": false,
   815            "mode": "single",
   816            "sort": "none"
   817          }
   818        },
   819        "pluginVersion": "12.1.0-90017",
   820        "targets": [
   821          {
   822            "datasource": {
   823              "type": "prometheus",
   824              "uid": "${datasource}"
   825            },
   826            "editorMode": "code",
   827            "expr": "histogram_quantile(0.99, sum(rate(pyroscope_metastore_fsm_apply_command_duration_seconds{namespace=\"$namespace\"}[$__rate_interval])))",
   828            "hide": false,
   829            "instant": false,
   830            "legendFormat": "p99",
   831            "range": true,
   832            "refId": "B"
   833          },
   834          {
   835            "datasource": {
   836              "type": "prometheus",
   837              "uid": "${datasource}"
   838            },
   839            "editorMode": "code",
   840            "expr": "histogram_quantile(0.95, sum(rate(pyroscope_metastore_fsm_apply_command_duration_seconds{namespace=\"$namespace\"}[$__rate_interval])))",
   841            "hide": false,
   842            "instant": false,
   843            "legendFormat": "p95",
   844            "range": true,
   845            "refId": "A"
   846          }
   847        ],
   848        "title": "FSM Latency",
   849        "transparent": true,
   850        "type": "timeseries"
   851      },
   852      {
   853        "collapsed": false,
   854        "gridPos": {
   855          "h": 1,
   856          "w": 24,
   857          "x": 0,
   858          "y": 17
   859        },
   860        "id": 51,
   861        "panels": [],
   862        "title": "Metadata Index Service",
   863        "type": "row"
   864      },
   865      {
   866        "datasource": {
   867          "type": "prometheus",
   868          "uid": "${datasource}"
   869        },
   870        "description": "The latency of AddBlock endpoint observed by metastore nodes\n\nThe only user is the segment-writer service.",
   871        "fieldConfig": {
   872          "defaults": {
   873            "color": {
   874              "mode": "palette-classic"
   875            },
   876            "custom": {
   877              "axisBorderShow": false,
   878              "axisCenteredZero": false,
   879              "axisColorMode": "text",
   880              "axisLabel": "",
   881              "axisPlacement": "auto",
   882              "barAlignment": 0,
   883              "barWidthFactor": 0.6,
   884              "drawStyle": "line",
   885              "fillOpacity": 0,
   886              "gradientMode": "none",
   887              "hideFrom": {
   888                "legend": false,
   889                "tooltip": false,
   890                "viz": false
   891              },
   892              "insertNulls": false,
   893              "lineInterpolation": "linear",
   894              "lineWidth": 1,
   895              "pointSize": 5,
   896              "scaleDistribution": {
   897                "type": "linear"
   898              },
   899              "showPoints": "auto",
   900              "spanNulls": false,
   901              "stacking": {
   902                "group": "A",
   903                "mode": "none"
   904              },
   905              "thresholdsStyle": {
   906                "mode": "off"
   907              }
   908            },
   909            "mappings": [],
   910            "thresholds": {
   911              "mode": "absolute",
   912              "steps": [
   913                {
   914                  "color": "green",
   915                  "value": 0
   916                },
   917                {
   918                  "color": "red",
   919                  "value": 80
   920                }
   921              ]
   922            },
   923            "unit": "s"
   924          },
   925          "overrides": [
   926            {
   927              "matcher": {
   928                "id": "byFrameRefID",
   929                "options": "B"
   930              },
   931              "properties": [
   932                {
   933                  "id": "custom.drawStyle",
   934                  "value": "points"
   935                },
   936                {
   937                  "id": "color",
   938                  "value": {
   939                    "fixedColor": "red",
   940                    "mode": "fixed"
   941                  }
   942                },
   943                {
   944                  "id": "custom.pointSize",
   945                  "value": 11
   946                },
   947                {
   948                  "id": "custom.axisPlacement",
   949                  "value": "right"
   950                }
   951              ]
   952            }
   953          ]
   954        },
   955        "gridPos": {
   956          "h": 8,
   957          "w": 8,
   958          "x": 0,
   959          "y": 18
   960        },
   961        "id": 49,
   962        "options": {
   963          "legend": {
   964            "calcs": [],
   965            "displayMode": "list",
   966            "placement": "bottom",
   967            "showLegend": false
   968          },
   969          "tooltip": {
   970            "hideZeros": false,
   971            "mode": "single",
   972            "sort": "none"
   973          }
   974        },
   975        "pluginVersion": "12.1.0-90017",
   976        "targets": [
   977          {
   978            "datasource": {
   979              "type": "prometheus",
   980              "uid": "${datasource}"
   981            },
   982            "editorMode": "code",
   983            "expr": "(\n  histogram_quantile(1,(sum by (le, pod) (\n    irate(pyroscope_metastore_boltdb_persist_snapshot_duration_seconds_bucket{namespace=\"$namespace\"}[$__rate_interval])))\n  ) * on(pod) group_left() (pyroscope_metastore_raft_state{namespace=\"$namespace\", state=\"Leader\"} \u003e= 1) \n)",
   984            "hide": false,
   985            "instant": false,
   986            "legendFormat": "Metastore leader snapshot ({{pod}})",
   987            "range": true,
   988            "refId": "B"
   989          },
   990          {
   991            "datasource": {
   992              "type": "prometheus",
   993              "uid": "${datasource}"
   994            },
   995            "editorMode": "code",
   996            "expr": "histogram_quantile(0.99, sum(rate(pyroscope_request_duration_seconds_bucket{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.IndexService/AddBlock\",\n}[$__rate_interval])) by (le,pod))",
   997            "hide": false,
   998            "instant": false,
   999            "legendFormat": "p99 {{pod}}",
  1000            "range": true,
  1001            "refId": "D"
  1002          },
  1003          {
  1004            "datasource": {
  1005              "type": "prometheus",
  1006              "uid": "${datasource}"
  1007            },
  1008            "editorMode": "code",
  1009            "expr": "histogram_quantile(0.95, sum(rate(pyroscope_request_duration_seconds_bucket{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.IndexService/AddBlock\",\n}[$__rate_interval])) by (le))",
  1010            "hide": false,
  1011            "instant": false,
  1012            "legendFormat": "p95",
  1013            "range": true,
  1014            "refId": "A"
  1015          }
  1016        ],
  1017        "title": "AddBlock Latency",
  1018        "transparent": true,
  1019        "type": "timeseries"
  1020      },
  1021      {
  1022        "datasource": {
  1023          "type": "datasource",
  1024          "uid": "-- Mixed --"
  1025        },
  1026        "fieldConfig": {
  1027          "defaults": {
  1028            "custom": {
  1029              "hideFrom": {
  1030                "legend": false,
  1031                "tooltip": false,
  1032                "viz": false
  1033              },
  1034              "scaleDistribution": {
  1035                "type": "linear"
  1036              }
  1037            }
  1038          },
  1039          "overrides": []
  1040        },
  1041        "gridPos": {
  1042          "h": 8,
  1043          "w": 8,
  1044          "x": 8,
  1045          "y": 18
  1046        },
  1047        "id": 95,
  1048        "options": {
  1049          "calculate": false,
  1050          "calculation": {
  1051            "xBuckets": {
  1052              "mode": "count",
  1053              "value": ""
  1054            },
  1055            "yBuckets": {
  1056              "mode": "count",
  1057              "scale": {
  1058                "type": "linear"
  1059              }
  1060            }
  1061          },
  1062          "cellGap": 0,
  1063          "cellValues": {
  1064            "unit": "requests"
  1065          },
  1066          "color": {
  1067            "exponent": 0.2,
  1068            "fill": "super-light-red",
  1069            "mode": "scheme",
  1070            "reverse": true,
  1071            "scale": "exponential",
  1072            "scheme": "Turbo",
  1073            "steps": 128
  1074          },
  1075          "exemplars": {
  1076            "color": "orange"
  1077          },
  1078          "filterValues": {
  1079            "le": 1e-9
  1080          },
  1081          "legend": {
  1082            "show": false
  1083          },
  1084          "rowsFrame": {
  1085            "layout": "auto"
  1086          },
  1087          "tooltip": {
  1088            "mode": "single",
  1089            "showColorScale": true,
  1090            "yHistogram": true
  1091          },
  1092          "yAxis": {
  1093            "axisPlacement": "left",
  1094            "reverse": false,
  1095            "unit": "s"
  1096          }
  1097        },
  1098        "pluginVersion": "12.1.0-90017",
  1099        "targets": [
  1100          {
  1101            "datasource": {
  1102              "type": "prometheus",
  1103              "uid": "${datasource}"
  1104            },
  1105            "editorMode": "code",
  1106            "expr": "sum(rate(pyroscope_request_duration_seconds{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.IndexService/AddBlock\",\n}[$__rate_interval]))",
  1107            "hide": false,
  1108            "legendFormat": "__auto",
  1109            "range": true,
  1110            "refId": "B"
  1111          }
  1112        ],
  1113        "title": "Latency Heatmap",
  1114        "transparent": true,
  1115        "type": "heatmap"
  1116      },
  1117      {
  1118        "datasource": {
  1119          "type": "prometheus",
  1120          "uid": "${datasource}"
  1121        },
  1122        "fieldConfig": {
  1123          "defaults": {
  1124            "color": {
  1125              "mode": "palette-classic"
  1126            },
  1127            "custom": {
  1128              "axisBorderShow": false,
  1129              "axisCenteredZero": false,
  1130              "axisColorMode": "text",
  1131              "axisLabel": "",
  1132              "axisPlacement": "auto",
  1133              "barAlignment": 0,
  1134              "barWidthFactor": 0.6,
  1135              "drawStyle": "line",
  1136              "fillOpacity": 0,
  1137              "gradientMode": "none",
  1138              "hideFrom": {
  1139                "legend": false,
  1140                "tooltip": false,
  1141                "viz": false
  1142              },
  1143              "insertNulls": false,
  1144              "lineInterpolation": "linear",
  1145              "lineWidth": 1,
  1146              "pointSize": 5,
  1147              "scaleDistribution": {
  1148                "type": "linear"
  1149              },
  1150              "showPoints": "auto",
  1151              "spanNulls": false,
  1152              "stacking": {
  1153                "group": "A",
  1154                "mode": "none"
  1155              },
  1156              "thresholdsStyle": {
  1157                "mode": "off"
  1158              }
  1159            },
  1160            "mappings": [],
  1161            "thresholds": {
  1162              "mode": "absolute",
  1163              "steps": [
  1164                {
  1165                  "color": "green",
  1166                  "value": 0
  1167                },
  1168                {
  1169                  "color": "red",
  1170                  "value": 80
  1171                }
  1172              ]
  1173            },
  1174            "unit": "ops"
  1175          },
  1176          "overrides": [
  1177            {
  1178              "matcher": {
  1179                "id": "byName",
  1180                "options": "error"
  1181              },
  1182              "properties": [
  1183                {
  1184                  "id": "color",
  1185                  "value": {
  1186                    "fixedColor": "red",
  1187                    "mode": "fixed"
  1188                  }
  1189                }
  1190              ]
  1191            },
  1192            {
  1193              "matcher": {
  1194                "id": "byName",
  1195                "options": "success"
  1196              },
  1197              "properties": [
  1198                {
  1199                  "id": "color",
  1200                  "value": {
  1201                    "fixedColor": "green",
  1202                    "mode": "fixed"
  1203                  }
  1204                }
  1205              ]
  1206            }
  1207          ]
  1208        },
  1209        "gridPos": {
  1210          "h": 8,
  1211          "w": 8,
  1212          "x": 16,
  1213          "y": 18
  1214        },
  1215        "id": 50,
  1216        "options": {
  1217          "legend": {
  1218            "calcs": [],
  1219            "displayMode": "list",
  1220            "placement": "bottom",
  1221            "showLegend": true
  1222          },
  1223          "tooltip": {
  1224            "hideZeros": false,
  1225            "mode": "single",
  1226            "sort": "none"
  1227          }
  1228        },
  1229        "pluginVersion": "12.1.0-90017",
  1230        "targets": [
  1231          {
  1232            "datasource": {
  1233              "type": "prometheus",
  1234              "uid": "${datasource}"
  1235            },
  1236            "editorMode": "code",
  1237            "expr": "sum(irate(pyroscope_request_duration_seconds_count{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.IndexService/AddBlock\",\n}[$__rate_interval])) by (status_code)",
  1238            "hide": false,
  1239            "instant": false,
  1240            "legendFormat": "__auto",
  1241            "range": true,
  1242            "refId": "D"
  1243          }
  1244        ],
  1245        "title": "",
  1246        "transparent": true,
  1247        "type": "timeseries"
  1248      },
  1249      {
  1250        "datasource": {
  1251          "type": "prometheus",
  1252          "uid": "${datasource}"
  1253        },
  1254        "description": "The latency of GetBlockMetadata endpoint observed by metastore nodes.\n\nThe only user is the compaction-worker service.",
  1255        "fieldConfig": {
  1256          "defaults": {
  1257            "color": {
  1258              "mode": "palette-classic"
  1259            },
  1260            "custom": {
  1261              "axisBorderShow": false,
  1262              "axisCenteredZero": false,
  1263              "axisColorMode": "text",
  1264              "axisLabel": "",
  1265              "axisPlacement": "auto",
  1266              "barAlignment": 0,
  1267              "barWidthFactor": 0.6,
  1268              "drawStyle": "line",
  1269              "fillOpacity": 0,
  1270              "gradientMode": "none",
  1271              "hideFrom": {
  1272                "legend": false,
  1273                "tooltip": false,
  1274                "viz": false
  1275              },
  1276              "insertNulls": false,
  1277              "lineInterpolation": "linear",
  1278              "lineWidth": 1,
  1279              "pointSize": 5,
  1280              "scaleDistribution": {
  1281                "type": "linear"
  1282              },
  1283              "showPoints": "auto",
  1284              "spanNulls": false,
  1285              "stacking": {
  1286                "group": "A",
  1287                "mode": "none"
  1288              },
  1289              "thresholdsStyle": {
  1290                "mode": "off"
  1291              }
  1292            },
  1293            "mappings": [],
  1294            "thresholds": {
  1295              "mode": "absolute",
  1296              "steps": [
  1297                {
  1298                  "color": "green",
  1299                  "value": 0
  1300                },
  1301                {
  1302                  "color": "red",
  1303                  "value": 80
  1304                }
  1305              ]
  1306            },
  1307            "unit": "s"
  1308          },
  1309          "overrides": [
  1310            {
  1311              "matcher": {
  1312                "id": "byFrameRefID",
  1313                "options": "B"
  1314              },
  1315              "properties": [
  1316                {
  1317                  "id": "custom.drawStyle",
  1318                  "value": "points"
  1319                },
  1320                {
  1321                  "id": "color",
  1322                  "value": {
  1323                    "fixedColor": "red",
  1324                    "mode": "fixed"
  1325                  }
  1326                },
  1327                {
  1328                  "id": "custom.pointSize",
  1329                  "value": 11
  1330                }
  1331              ]
  1332            }
  1333          ]
  1334        },
  1335        "gridPos": {
  1336          "h": 8,
  1337          "w": 8,
  1338          "x": 0,
  1339          "y": 26
  1340        },
  1341        "id": 75,
  1342        "options": {
  1343          "legend": {
  1344            "calcs": [],
  1345            "displayMode": "list",
  1346            "placement": "bottom",
  1347            "showLegend": true
  1348          },
  1349          "tooltip": {
  1350            "hideZeros": false,
  1351            "mode": "single",
  1352            "sort": "none"
  1353          }
  1354        },
  1355        "pluginVersion": "12.1.0-90017",
  1356        "targets": [
  1357          {
  1358            "datasource": {
  1359              "type": "prometheus",
  1360              "uid": "${datasource}"
  1361            },
  1362            "editorMode": "code",
  1363            "expr": "histogram_quantile(0.99, sum(rate(pyroscope_request_duration_seconds_bucket{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.IndexService/GetBlockMetadata\",\n}[$__rate_interval])) by (le,pod))",
  1364            "hide": false,
  1365            "instant": false,
  1366            "legendFormat": "p99 {{pod}}",
  1367            "range": true,
  1368            "refId": "D"
  1369          },
  1370          {
  1371            "datasource": {
  1372              "type": "prometheus",
  1373              "uid": "${datasource}"
  1374            },
  1375            "editorMode": "code",
  1376            "expr": "histogram_quantile(0.95, sum(rate(pyroscope_request_duration_seconds_bucket{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.IndexService/GetBlockMetadata\",\n}[$__rate_interval])) by (le))",
  1377            "hide": true,
  1378            "instant": false,
  1379            "legendFormat": "p95",
  1380            "range": true,
  1381            "refId": "A"
  1382          }
  1383        ],
  1384        "title": "GetBlockMetadata Latency",
  1385        "transparent": true,
  1386        "type": "timeseries"
  1387      },
  1388      {
  1389        "datasource": {
  1390          "type": "datasource",
  1391          "uid": "-- Mixed --"
  1392        },
  1393        "fieldConfig": {
  1394          "defaults": {
  1395            "custom": {
  1396              "hideFrom": {
  1397                "legend": false,
  1398                "tooltip": false,
  1399                "viz": false
  1400              },
  1401              "scaleDistribution": {
  1402                "type": "linear"
  1403              }
  1404            }
  1405          },
  1406          "overrides": []
  1407        },
  1408        "gridPos": {
  1409          "h": 8,
  1410          "w": 8,
  1411          "x": 8,
  1412          "y": 26
  1413        },
  1414        "id": 99,
  1415        "options": {
  1416          "calculate": false,
  1417          "calculation": {
  1418            "xBuckets": {
  1419              "mode": "count",
  1420              "value": ""
  1421            },
  1422            "yBuckets": {
  1423              "mode": "count",
  1424              "scale": {
  1425                "type": "linear"
  1426              }
  1427            }
  1428          },
  1429          "cellGap": 0,
  1430          "cellValues": {
  1431            "unit": "requests"
  1432          },
  1433          "color": {
  1434            "exponent": 0.2,
  1435            "fill": "super-light-red",
  1436            "mode": "scheme",
  1437            "reverse": true,
  1438            "scale": "exponential",
  1439            "scheme": "Turbo",
  1440            "steps": 128
  1441          },
  1442          "exemplars": {
  1443            "color": "orange"
  1444          },
  1445          "filterValues": {
  1446            "le": 1e-9
  1447          },
  1448          "legend": {
  1449            "show": false
  1450          },
  1451          "rowsFrame": {
  1452            "layout": "auto"
  1453          },
  1454          "tooltip": {
  1455            "mode": "single",
  1456            "showColorScale": true,
  1457            "yHistogram": true
  1458          },
  1459          "yAxis": {
  1460            "axisPlacement": "left",
  1461            "reverse": false,
  1462            "unit": "s"
  1463          }
  1464        },
  1465        "pluginVersion": "12.1.0-90017",
  1466        "targets": [
  1467          {
  1468            "datasource": {
  1469              "type": "prometheus",
  1470              "uid": "${datasource}"
  1471            },
  1472            "editorMode": "code",
  1473            "expr": "sum(rate(pyroscope_request_duration_seconds{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.IndexService/GetBlockMetadata\"\n}[$__rate_interval]))",
  1474            "hide": false,
  1475            "legendFormat": "__auto",
  1476            "range": true,
  1477            "refId": "B"
  1478          }
  1479        ],
  1480        "title": "Latency Heatmap",
  1481        "transparent": true,
  1482        "type": "heatmap"
  1483      },
  1484      {
  1485        "datasource": {
  1486          "type": "prometheus",
  1487          "uid": "${datasource}"
  1488        },
  1489        "fieldConfig": {
  1490          "defaults": {
  1491            "color": {
  1492              "mode": "palette-classic"
  1493            },
  1494            "custom": {
  1495              "axisBorderShow": false,
  1496              "axisCenteredZero": false,
  1497              "axisColorMode": "text",
  1498              "axisLabel": "",
  1499              "axisPlacement": "auto",
  1500              "barAlignment": 0,
  1501              "barWidthFactor": 0.6,
  1502              "drawStyle": "line",
  1503              "fillOpacity": 0,
  1504              "gradientMode": "none",
  1505              "hideFrom": {
  1506                "legend": false,
  1507                "tooltip": false,
  1508                "viz": false
  1509              },
  1510              "insertNulls": false,
  1511              "lineInterpolation": "linear",
  1512              "lineWidth": 1,
  1513              "pointSize": 5,
  1514              "scaleDistribution": {
  1515                "type": "linear"
  1516              },
  1517              "showPoints": "auto",
  1518              "spanNulls": false,
  1519              "stacking": {
  1520                "group": "A",
  1521                "mode": "none"
  1522              },
  1523              "thresholdsStyle": {
  1524                "mode": "off"
  1525              }
  1526            },
  1527            "mappings": [],
  1528            "thresholds": {
  1529              "mode": "absolute",
  1530              "steps": [
  1531                {
  1532                  "color": "green",
  1533                  "value": 0
  1534                },
  1535                {
  1536                  "color": "red",
  1537                  "value": 80
  1538                }
  1539              ]
  1540            },
  1541            "unit": "ops"
  1542          },
  1543          "overrides": [
  1544            {
  1545              "matcher": {
  1546                "id": "byName",
  1547                "options": "error"
  1548              },
  1549              "properties": [
  1550                {
  1551                  "id": "color",
  1552                  "value": {
  1553                    "fixedColor": "red",
  1554                    "mode": "fixed"
  1555                  }
  1556                }
  1557              ]
  1558            },
  1559            {
  1560              "matcher": {
  1561                "id": "byName",
  1562                "options": "success"
  1563              },
  1564              "properties": [
  1565                {
  1566                  "id": "color",
  1567                  "value": {
  1568                    "fixedColor": "green",
  1569                    "mode": "fixed"
  1570                  }
  1571                }
  1572              ]
  1573            }
  1574          ]
  1575        },
  1576        "gridPos": {
  1577          "h": 8,
  1578          "w": 8,
  1579          "x": 16,
  1580          "y": 26
  1581        },
  1582        "id": 76,
  1583        "options": {
  1584          "legend": {
  1585            "calcs": [],
  1586            "displayMode": "list",
  1587            "placement": "bottom",
  1588            "showLegend": true
  1589          },
  1590          "tooltip": {
  1591            "hideZeros": false,
  1592            "mode": "single",
  1593            "sort": "none"
  1594          }
  1595        },
  1596        "pluginVersion": "12.1.0-90017",
  1597        "targets": [
  1598          {
  1599            "datasource": {
  1600              "type": "prometheus",
  1601              "uid": "${datasource}"
  1602            },
  1603            "editorMode": "code",
  1604            "expr": "sum(irate(pyroscope_request_duration_seconds_count{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.IndexService/GetBlockMetadata\",\n}[$__rate_interval])) by (status_code)",
  1605            "hide": false,
  1606            "instant": false,
  1607            "legendFormat": "__auto",
  1608            "range": true,
  1609            "refId": "D"
  1610          }
  1611        ],
  1612        "title": "",
  1613        "transparent": true,
  1614        "type": "timeseries"
  1615      },
  1616      {
  1617        "collapsed": false,
  1618        "gridPos": {
  1619          "h": 1,
  1620          "w": 24,
  1621          "x": 0,
  1622          "y": 34
  1623        },
  1624        "id": 89,
  1625        "panels": [],
  1626        "title": "Metadata Query Service",
  1627        "type": "row"
  1628      },
  1629      {
  1630        "datasource": {
  1631          "type": "prometheus",
  1632          "uid": "${datasource}"
  1633        },
  1634        "description": "The latency of QueryMetadata endpoint observed by metastore nodes.\n\nThe only user is the query-frontend service.",
  1635        "fieldConfig": {
  1636          "defaults": {
  1637            "color": {
  1638              "mode": "palette-classic"
  1639            },
  1640            "custom": {
  1641              "axisBorderShow": false,
  1642              "axisCenteredZero": false,
  1643              "axisColorMode": "text",
  1644              "axisLabel": "",
  1645              "axisPlacement": "auto",
  1646              "barAlignment": 0,
  1647              "barWidthFactor": 0.6,
  1648              "drawStyle": "line",
  1649              "fillOpacity": 0,
  1650              "gradientMode": "none",
  1651              "hideFrom": {
  1652                "legend": false,
  1653                "tooltip": false,
  1654                "viz": false
  1655              },
  1656              "insertNulls": false,
  1657              "lineInterpolation": "linear",
  1658              "lineWidth": 1,
  1659              "pointSize": 5,
  1660              "scaleDistribution": {
  1661                "type": "linear"
  1662              },
  1663              "showPoints": "auto",
  1664              "spanNulls": false,
  1665              "stacking": {
  1666                "group": "A",
  1667                "mode": "none"
  1668              },
  1669              "thresholdsStyle": {
  1670                "mode": "off"
  1671              }
  1672            },
  1673            "mappings": [],
  1674            "thresholds": {
  1675              "mode": "absolute",
  1676              "steps": [
  1677                {
  1678                  "color": "green",
  1679                  "value": 0
  1680                },
  1681                {
  1682                  "color": "red",
  1683                  "value": 80
  1684                }
  1685              ]
  1686            },
  1687            "unit": "s"
  1688          },
  1689          "overrides": [
  1690            {
  1691              "matcher": {
  1692                "id": "byFrameRefID",
  1693                "options": "B"
  1694              },
  1695              "properties": [
  1696                {
  1697                  "id": "custom.drawStyle",
  1698                  "value": "points"
  1699                },
  1700                {
  1701                  "id": "color",
  1702                  "value": {
  1703                    "fixedColor": "red",
  1704                    "mode": "fixed"
  1705                  }
  1706                },
  1707                {
  1708                  "id": "custom.pointSize",
  1709                  "value": 11
  1710                }
  1711              ]
  1712            }
  1713          ]
  1714        },
  1715        "gridPos": {
  1716          "h": 8,
  1717          "w": 8,
  1718          "x": 0,
  1719          "y": 35
  1720        },
  1721        "id": 90,
  1722        "options": {
  1723          "legend": {
  1724            "calcs": [],
  1725            "displayMode": "list",
  1726            "placement": "bottom",
  1727            "showLegend": true
  1728          },
  1729          "tooltip": {
  1730            "hideZeros": false,
  1731            "mode": "single",
  1732            "sort": "none"
  1733          }
  1734        },
  1735        "pluginVersion": "12.1.0-90017",
  1736        "targets": [
  1737          {
  1738            "datasource": {
  1739              "type": "prometheus",
  1740              "uid": "${datasource}"
  1741            },
  1742            "editorMode": "code",
  1743            "expr": "histogram_quantile(0.99, sum(rate(pyroscope_request_duration_seconds_bucket{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.MetadataQueryService/QueryMetadata\",\n}[$__rate_interval])) by (le,pod))",
  1744            "hide": false,
  1745            "instant": false,
  1746            "legendFormat": "p99 {{pod}}",
  1747            "range": true,
  1748            "refId": "D"
  1749          },
  1750          {
  1751            "datasource": {
  1752              "type": "prometheus",
  1753              "uid": "${datasource}"
  1754            },
  1755            "editorMode": "code",
  1756            "expr": "histogram_quantile(0.95, sum(rate(pyroscope_request_duration_seconds_bucket{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.MetadataQueryService/QueryMetadata\",\n}[$__rate_interval])) by (le))",
  1757            "hide": true,
  1758            "instant": false,
  1759            "legendFormat": "p95",
  1760            "range": true,
  1761            "refId": "A"
  1762          }
  1763        ],
  1764        "title": "QueryMetadata Latency",
  1765        "transparent": true,
  1766        "type": "timeseries"
  1767      },
  1768      {
  1769        "datasource": {
  1770          "type": "datasource",
  1771          "uid": "-- Mixed --"
  1772        },
  1773        "fieldConfig": {
  1774          "defaults": {
  1775            "custom": {
  1776              "hideFrom": {
  1777                "legend": false,
  1778                "tooltip": false,
  1779                "viz": false
  1780              },
  1781              "scaleDistribution": {
  1782                "type": "linear"
  1783              }
  1784            }
  1785          },
  1786          "overrides": []
  1787        },
  1788        "gridPos": {
  1789          "h": 8,
  1790          "w": 8,
  1791          "x": 8,
  1792          "y": 35
  1793        },
  1794        "id": 98,
  1795        "options": {
  1796          "calculate": false,
  1797          "calculation": {
  1798            "xBuckets": {
  1799              "mode": "count",
  1800              "value": ""
  1801            },
  1802            "yBuckets": {
  1803              "mode": "count",
  1804              "scale": {
  1805                "type": "linear"
  1806              }
  1807            }
  1808          },
  1809          "cellGap": 0,
  1810          "cellValues": {
  1811            "unit": "requests"
  1812          },
  1813          "color": {
  1814            "exponent": 0.2,
  1815            "fill": "super-light-red",
  1816            "mode": "scheme",
  1817            "reverse": true,
  1818            "scale": "exponential",
  1819            "scheme": "Turbo",
  1820            "steps": 128
  1821          },
  1822          "exemplars": {
  1823            "color": "orange"
  1824          },
  1825          "filterValues": {
  1826            "le": 1e-9
  1827          },
  1828          "legend": {
  1829            "show": false
  1830          },
  1831          "rowsFrame": {
  1832            "layout": "auto"
  1833          },
  1834          "tooltip": {
  1835            "mode": "single",
  1836            "showColorScale": true,
  1837            "yHistogram": true
  1838          },
  1839          "yAxis": {
  1840            "axisPlacement": "left",
  1841            "reverse": false,
  1842            "unit": "s"
  1843          }
  1844        },
  1845        "pluginVersion": "12.1.0-90017",
  1846        "targets": [
  1847          {
  1848            "datasource": {
  1849              "type": "prometheus",
  1850              "uid": "${datasource}"
  1851            },
  1852            "editorMode": "code",
  1853            "expr": "sum(rate(pyroscope_request_duration_seconds{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.MetadataQueryService/QueryMetadata\",\n}[$__rate_interval]))",
  1854            "hide": false,
  1855            "legendFormat": "__auto",
  1856            "range": true,
  1857            "refId": "B"
  1858          }
  1859        ],
  1860        "title": "Latency Heatmap",
  1861        "transparent": true,
  1862        "type": "heatmap"
  1863      },
  1864      {
  1865        "datasource": {
  1866          "type": "prometheus",
  1867          "uid": "${datasource}"
  1868        },
  1869        "fieldConfig": {
  1870          "defaults": {
  1871            "color": {
  1872              "mode": "palette-classic"
  1873            },
  1874            "custom": {
  1875              "axisBorderShow": false,
  1876              "axisCenteredZero": false,
  1877              "axisColorMode": "text",
  1878              "axisLabel": "",
  1879              "axisPlacement": "auto",
  1880              "barAlignment": 0,
  1881              "barWidthFactor": 0.6,
  1882              "drawStyle": "line",
  1883              "fillOpacity": 0,
  1884              "gradientMode": "none",
  1885              "hideFrom": {
  1886                "legend": false,
  1887                "tooltip": false,
  1888                "viz": false
  1889              },
  1890              "insertNulls": false,
  1891              "lineInterpolation": "linear",
  1892              "lineWidth": 1,
  1893              "pointSize": 5,
  1894              "scaleDistribution": {
  1895                "type": "linear"
  1896              },
  1897              "showPoints": "auto",
  1898              "spanNulls": false,
  1899              "stacking": {
  1900                "group": "A",
  1901                "mode": "none"
  1902              },
  1903              "thresholdsStyle": {
  1904                "mode": "off"
  1905              }
  1906            },
  1907            "mappings": [],
  1908            "thresholds": {
  1909              "mode": "absolute",
  1910              "steps": [
  1911                {
  1912                  "color": "green",
  1913                  "value": 0
  1914                },
  1915                {
  1916                  "color": "red",
  1917                  "value": 80
  1918                }
  1919              ]
  1920            },
  1921            "unit": "ops"
  1922          },
  1923          "overrides": [
  1924            {
  1925              "matcher": {
  1926                "id": "byName",
  1927                "options": "error"
  1928              },
  1929              "properties": [
  1930                {
  1931                  "id": "color",
  1932                  "value": {
  1933                    "fixedColor": "red",
  1934                    "mode": "fixed"
  1935                  }
  1936                }
  1937              ]
  1938            },
  1939            {
  1940              "matcher": {
  1941                "id": "byName",
  1942                "options": "success"
  1943              },
  1944              "properties": [
  1945                {
  1946                  "id": "color",
  1947                  "value": {
  1948                    "fixedColor": "green",
  1949                    "mode": "fixed"
  1950                  }
  1951                }
  1952              ]
  1953            }
  1954          ]
  1955        },
  1956        "gridPos": {
  1957          "h": 8,
  1958          "w": 8,
  1959          "x": 16,
  1960          "y": 35
  1961        },
  1962        "id": 92,
  1963        "options": {
  1964          "legend": {
  1965            "calcs": [],
  1966            "displayMode": "list",
  1967            "placement": "bottom",
  1968            "showLegend": true
  1969          },
  1970          "tooltip": {
  1971            "hideZeros": false,
  1972            "mode": "single",
  1973            "sort": "none"
  1974          }
  1975        },
  1976        "pluginVersion": "12.1.0-90017",
  1977        "targets": [
  1978          {
  1979            "datasource": {
  1980              "type": "prometheus",
  1981              "uid": "${datasource}"
  1982            },
  1983            "editorMode": "code",
  1984            "expr": "sum(irate(pyroscope_request_duration_seconds_count{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.MetadataQueryService/QueryMetadata\",\n}[$__rate_interval])) by (status_code)",
  1985            "hide": false,
  1986            "instant": false,
  1987            "legendFormat": "__auto",
  1988            "range": true,
  1989            "refId": "D"
  1990          }
  1991        ],
  1992        "title": "",
  1993        "transparent": true,
  1994        "type": "timeseries"
  1995      },
  1996      {
  1997        "datasource": {
  1998          "type": "prometheus",
  1999          "uid": "${datasource}"
  2000        },
  2001        "description": "The latency of QueryMetadataLabels endpoint observed by metastore nodes.\n\nThe only user is the query-frontend service.",
  2002        "fieldConfig": {
  2003          "defaults": {
  2004            "color": {
  2005              "mode": "palette-classic"
  2006            },
  2007            "custom": {
  2008              "axisBorderShow": false,
  2009              "axisCenteredZero": false,
  2010              "axisColorMode": "text",
  2011              "axisLabel": "",
  2012              "axisPlacement": "auto",
  2013              "barAlignment": 0,
  2014              "barWidthFactor": 0.6,
  2015              "drawStyle": "line",
  2016              "fillOpacity": 0,
  2017              "gradientMode": "none",
  2018              "hideFrom": {
  2019                "legend": false,
  2020                "tooltip": false,
  2021                "viz": false
  2022              },
  2023              "insertNulls": false,
  2024              "lineInterpolation": "linear",
  2025              "lineWidth": 1,
  2026              "pointSize": 5,
  2027              "scaleDistribution": {
  2028                "type": "linear"
  2029              },
  2030              "showPoints": "auto",
  2031              "spanNulls": false,
  2032              "stacking": {
  2033                "group": "A",
  2034                "mode": "none"
  2035              },
  2036              "thresholdsStyle": {
  2037                "mode": "off"
  2038              }
  2039            },
  2040            "mappings": [],
  2041            "thresholds": {
  2042              "mode": "absolute",
  2043              "steps": [
  2044                {
  2045                  "color": "green",
  2046                  "value": 0
  2047                },
  2048                {
  2049                  "color": "red",
  2050                  "value": 80
  2051                }
  2052              ]
  2053            },
  2054            "unit": "s"
  2055          },
  2056          "overrides": [
  2057            {
  2058              "matcher": {
  2059                "id": "byFrameRefID",
  2060                "options": "B"
  2061              },
  2062              "properties": [
  2063                {
  2064                  "id": "custom.drawStyle",
  2065                  "value": "points"
  2066                },
  2067                {
  2068                  "id": "color",
  2069                  "value": {
  2070                    "fixedColor": "red",
  2071                    "mode": "fixed"
  2072                  }
  2073                },
  2074                {
  2075                  "id": "custom.pointSize",
  2076                  "value": 11
  2077                }
  2078              ]
  2079            }
  2080          ]
  2081        },
  2082        "gridPos": {
  2083          "h": 8,
  2084          "w": 8,
  2085          "x": 0,
  2086          "y": 43
  2087        },
  2088        "id": 91,
  2089        "options": {
  2090          "legend": {
  2091            "calcs": [],
  2092            "displayMode": "list",
  2093            "placement": "bottom",
  2094            "showLegend": true
  2095          },
  2096          "tooltip": {
  2097            "hideZeros": false,
  2098            "mode": "single",
  2099            "sort": "none"
  2100          }
  2101        },
  2102        "pluginVersion": "12.1.0-90017",
  2103        "targets": [
  2104          {
  2105            "datasource": {
  2106              "type": "prometheus",
  2107              "uid": "${datasource}"
  2108            },
  2109            "editorMode": "code",
  2110            "expr": "histogram_quantile(0.99, sum(rate(pyroscope_request_duration_seconds_bucket{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.MetadataQueryService/QueryMetadataLabels\",\n}[$__rate_interval])) by (le,pod))",
  2111            "hide": false,
  2112            "instant": false,
  2113            "legendFormat": "p99 {{pod}}",
  2114            "range": true,
  2115            "refId": "D"
  2116          },
  2117          {
  2118            "datasource": {
  2119              "type": "prometheus",
  2120              "uid": "${datasource}"
  2121            },
  2122            "editorMode": "code",
  2123            "expr": "histogram_quantile(0.95, sum(rate(pyroscope_request_duration_seconds_bucket{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.MetadataQueryService/QueryMetadataLabels\",\n}[$__rate_interval])) by (le))",
  2124            "hide": true,
  2125            "instant": false,
  2126            "legendFormat": "p95",
  2127            "range": true,
  2128            "refId": "A"
  2129          }
  2130        ],
  2131        "title": "QueryMetadataLabels Latency",
  2132        "transparent": true,
  2133        "type": "timeseries"
  2134      },
  2135      {
  2136        "datasource": {
  2137          "type": "datasource",
  2138          "uid": "-- Mixed --"
  2139        },
  2140        "fieldConfig": {
  2141          "defaults": {
  2142            "custom": {
  2143              "hideFrom": {
  2144                "legend": false,
  2145                "tooltip": false,
  2146                "viz": false
  2147              },
  2148              "scaleDistribution": {
  2149                "type": "linear"
  2150              }
  2151            }
  2152          },
  2153          "overrides": []
  2154        },
  2155        "gridPos": {
  2156          "h": 8,
  2157          "w": 8,
  2158          "x": 8,
  2159          "y": 43
  2160        },
  2161        "id": 96,
  2162        "options": {
  2163          "calculate": false,
  2164          "calculation": {
  2165            "xBuckets": {
  2166              "mode": "count",
  2167              "value": ""
  2168            },
  2169            "yBuckets": {
  2170              "mode": "count",
  2171              "scale": {
  2172                "type": "linear"
  2173              }
  2174            }
  2175          },
  2176          "cellGap": 0,
  2177          "cellValues": {
  2178            "unit": "requests"
  2179          },
  2180          "color": {
  2181            "exponent": 0.2,
  2182            "fill": "super-light-red",
  2183            "mode": "scheme",
  2184            "reverse": true,
  2185            "scale": "exponential",
  2186            "scheme": "Turbo",
  2187            "steps": 128
  2188          },
  2189          "exemplars": {
  2190            "color": "orange"
  2191          },
  2192          "filterValues": {
  2193            "le": 1e-9
  2194          },
  2195          "legend": {
  2196            "show": false
  2197          },
  2198          "rowsFrame": {
  2199            "layout": "auto"
  2200          },
  2201          "tooltip": {
  2202            "mode": "single",
  2203            "showColorScale": true,
  2204            "yHistogram": true
  2205          },
  2206          "yAxis": {
  2207            "axisPlacement": "left",
  2208            "reverse": false,
  2209            "unit": "s"
  2210          }
  2211        },
  2212        "pluginVersion": "12.1.0-90017",
  2213        "targets": [
  2214          {
  2215            "datasource": {
  2216              "type": "prometheus",
  2217              "uid": "${datasource}"
  2218            },
  2219            "editorMode": "code",
  2220            "expr": "sum(rate(pyroscope_request_duration_seconds{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.MetadataQueryService/QueryMetadataLabels\",\n}[$__rate_interval]))",
  2221            "hide": false,
  2222            "legendFormat": "__auto",
  2223            "range": true,
  2224            "refId": "B"
  2225          }
  2226        ],
  2227        "title": "Latency Heatmap",
  2228        "transparent": true,
  2229        "type": "heatmap"
  2230      },
  2231      {
  2232        "datasource": {
  2233          "type": "prometheus",
  2234          "uid": "${datasource}"
  2235        },
  2236        "fieldConfig": {
  2237          "defaults": {
  2238            "color": {
  2239              "mode": "palette-classic"
  2240            },
  2241            "custom": {
  2242              "axisBorderShow": false,
  2243              "axisCenteredZero": false,
  2244              "axisColorMode": "text",
  2245              "axisLabel": "",
  2246              "axisPlacement": "auto",
  2247              "barAlignment": 0,
  2248              "barWidthFactor": 0.6,
  2249              "drawStyle": "line",
  2250              "fillOpacity": 0,
  2251              "gradientMode": "none",
  2252              "hideFrom": {
  2253                "legend": false,
  2254                "tooltip": false,
  2255                "viz": false
  2256              },
  2257              "insertNulls": false,
  2258              "lineInterpolation": "linear",
  2259              "lineWidth": 1,
  2260              "pointSize": 5,
  2261              "scaleDistribution": {
  2262                "type": "linear"
  2263              },
  2264              "showPoints": "auto",
  2265              "spanNulls": false,
  2266              "stacking": {
  2267                "group": "A",
  2268                "mode": "none"
  2269              },
  2270              "thresholdsStyle": {
  2271                "mode": "off"
  2272              }
  2273            },
  2274            "mappings": [],
  2275            "thresholds": {
  2276              "mode": "absolute",
  2277              "steps": [
  2278                {
  2279                  "color": "green",
  2280                  "value": 0
  2281                },
  2282                {
  2283                  "color": "red",
  2284                  "value": 80
  2285                }
  2286              ]
  2287            },
  2288            "unit": "ops"
  2289          },
  2290          "overrides": [
  2291            {
  2292              "matcher": {
  2293                "id": "byName",
  2294                "options": "error"
  2295              },
  2296              "properties": [
  2297                {
  2298                  "id": "color",
  2299                  "value": {
  2300                    "fixedColor": "red",
  2301                    "mode": "fixed"
  2302                  }
  2303                }
  2304              ]
  2305            },
  2306            {
  2307              "matcher": {
  2308                "id": "byName",
  2309                "options": "success"
  2310              },
  2311              "properties": [
  2312                {
  2313                  "id": "color",
  2314                  "value": {
  2315                    "fixedColor": "green",
  2316                    "mode": "fixed"
  2317                  }
  2318                }
  2319              ]
  2320            }
  2321          ]
  2322        },
  2323        "gridPos": {
  2324          "h": 8,
  2325          "w": 8,
  2326          "x": 16,
  2327          "y": 43
  2328        },
  2329        "id": 93,
  2330        "options": {
  2331          "legend": {
  2332            "calcs": [],
  2333            "displayMode": "list",
  2334            "placement": "bottom",
  2335            "showLegend": true
  2336          },
  2337          "tooltip": {
  2338            "hideZeros": false,
  2339            "mode": "single",
  2340            "sort": "none"
  2341          }
  2342        },
  2343        "pluginVersion": "12.1.0-90017",
  2344        "targets": [
  2345          {
  2346            "datasource": {
  2347              "type": "prometheus",
  2348              "uid": "${datasource}"
  2349            },
  2350            "editorMode": "code",
  2351            "expr": "sum(irate(pyroscope_request_duration_seconds_count{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.MetadataQueryService/QueryMetadataLabels\",\n}[$__rate_interval])) by (status_code)",
  2352            "hide": false,
  2353            "instant": false,
  2354            "legendFormat": "__auto",
  2355            "range": true,
  2356            "refId": "D"
  2357          }
  2358        ],
  2359        "title": "",
  2360        "transparent": true,
  2361        "type": "timeseries"
  2362      },
  2363      {
  2364        "collapsed": false,
  2365        "gridPos": {
  2366          "h": 1,
  2367          "w": 24,
  2368          "x": 0,
  2369          "y": 51
  2370        },
  2371        "id": 86,
  2372        "panels": [],
  2373        "title": "Compaction Service",
  2374        "type": "row"
  2375      },
  2376      {
  2377        "datasource": {
  2378          "type": "prometheus",
  2379          "uid": "${datasource}"
  2380        },
  2381        "description": "The latency of PollCompactionJobs endpoint observed by metastore nodes.\n\nThe only user is the compaction-worker service.",
  2382        "fieldConfig": {
  2383          "defaults": {
  2384            "color": {
  2385              "mode": "palette-classic"
  2386            },
  2387            "custom": {
  2388              "axisBorderShow": false,
  2389              "axisCenteredZero": false,
  2390              "axisColorMode": "text",
  2391              "axisLabel": "",
  2392              "axisPlacement": "auto",
  2393              "barAlignment": 0,
  2394              "barWidthFactor": 0.6,
  2395              "drawStyle": "line",
  2396              "fillOpacity": 0,
  2397              "gradientMode": "none",
  2398              "hideFrom": {
  2399                "legend": false,
  2400                "tooltip": false,
  2401                "viz": false
  2402              },
  2403              "insertNulls": false,
  2404              "lineInterpolation": "linear",
  2405              "lineWidth": 1,
  2406              "pointSize": 5,
  2407              "scaleDistribution": {
  2408                "type": "linear"
  2409              },
  2410              "showPoints": "auto",
  2411              "spanNulls": false,
  2412              "stacking": {
  2413                "group": "A",
  2414                "mode": "none"
  2415              },
  2416              "thresholdsStyle": {
  2417                "mode": "off"
  2418              }
  2419            },
  2420            "mappings": [],
  2421            "thresholds": {
  2422              "mode": "absolute",
  2423              "steps": [
  2424                {
  2425                  "color": "green",
  2426                  "value": 0
  2427                },
  2428                {
  2429                  "color": "red",
  2430                  "value": 80
  2431                }
  2432              ]
  2433            },
  2434            "unit": "s"
  2435          },
  2436          "overrides": [
  2437            {
  2438              "matcher": {
  2439                "id": "byFrameRefID",
  2440                "options": "B"
  2441              },
  2442              "properties": [
  2443                {
  2444                  "id": "custom.drawStyle",
  2445                  "value": "points"
  2446                },
  2447                {
  2448                  "id": "color",
  2449                  "value": {
  2450                    "fixedColor": "red",
  2451                    "mode": "fixed"
  2452                  }
  2453                },
  2454                {
  2455                  "id": "custom.pointSize",
  2456                  "value": 11
  2457                }
  2458              ]
  2459            }
  2460          ]
  2461        },
  2462        "gridPos": {
  2463          "h": 8,
  2464          "w": 8,
  2465          "x": 0,
  2466          "y": 52
  2467        },
  2468        "id": 87,
  2469        "options": {
  2470          "legend": {
  2471            "calcs": [],
  2472            "displayMode": "list",
  2473            "placement": "bottom",
  2474            "showLegend": true
  2475          },
  2476          "tooltip": {
  2477            "hideZeros": false,
  2478            "mode": "single",
  2479            "sort": "none"
  2480          }
  2481        },
  2482        "pluginVersion": "12.1.0-90017",
  2483        "targets": [
  2484          {
  2485            "datasource": {
  2486              "type": "prometheus",
  2487              "uid": "${datasource}"
  2488            },
  2489            "editorMode": "code",
  2490            "expr": "histogram_quantile(0.99, sum(rate(pyroscope_request_duration_seconds_bucket{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.CompactionService/PollCompactionJobs\",\n}[$__rate_interval])) by (le,pod))",
  2491            "hide": false,
  2492            "instant": false,
  2493            "legendFormat": "p99 {{pod}}",
  2494            "range": true,
  2495            "refId": "D"
  2496          },
  2497          {
  2498            "datasource": {
  2499              "type": "prometheus",
  2500              "uid": "${datasource}"
  2501            },
  2502            "editorMode": "code",
  2503            "expr": "histogram_quantile(0.95, sum(rate(pyroscope_request_duration_seconds_bucket{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.CompactionService/PollCompactionJobs\",\n}[$__rate_interval])) by (le))",
  2504            "hide": true,
  2505            "instant": false,
  2506            "legendFormat": "p95",
  2507            "range": true,
  2508            "refId": "A"
  2509          }
  2510        ],
  2511        "title": "PollCompactionJobs Latency",
  2512        "transparent": true,
  2513        "type": "timeseries"
  2514      },
  2515      {
  2516        "datasource": {
  2517          "type": "datasource",
  2518          "uid": "-- Mixed --"
  2519        },
  2520        "fieldConfig": {
  2521          "defaults": {
  2522            "custom": {
  2523              "hideFrom": {
  2524                "legend": false,
  2525                "tooltip": false,
  2526                "viz": false
  2527              },
  2528              "scaleDistribution": {
  2529                "type": "linear"
  2530              }
  2531            }
  2532          },
  2533          "overrides": []
  2534        },
  2535        "gridPos": {
  2536          "h": 8,
  2537          "w": 8,
  2538          "x": 8,
  2539          "y": 52
  2540        },
  2541        "id": 97,
  2542        "options": {
  2543          "calculate": false,
  2544          "calculation": {
  2545            "xBuckets": {
  2546              "mode": "count",
  2547              "value": ""
  2548            },
  2549            "yBuckets": {
  2550              "mode": "count",
  2551              "scale": {
  2552                "type": "linear"
  2553              }
  2554            }
  2555          },
  2556          "cellGap": 0,
  2557          "cellValues": {
  2558            "unit": "requests"
  2559          },
  2560          "color": {
  2561            "exponent": 0.2,
  2562            "fill": "super-light-red",
  2563            "mode": "scheme",
  2564            "reverse": true,
  2565            "scale": "exponential",
  2566            "scheme": "Turbo",
  2567            "steps": 128
  2568          },
  2569          "exemplars": {
  2570            "color": "orange"
  2571          },
  2572          "filterValues": {
  2573            "le": 1e-9
  2574          },
  2575          "legend": {
  2576            "show": false
  2577          },
  2578          "rowsFrame": {
  2579            "layout": "auto"
  2580          },
  2581          "tooltip": {
  2582            "mode": "single",
  2583            "showColorScale": true,
  2584            "yHistogram": true
  2585          },
  2586          "yAxis": {
  2587            "axisPlacement": "left",
  2588            "reverse": false,
  2589            "unit": "s"
  2590          }
  2591        },
  2592        "pluginVersion": "12.1.0-90017",
  2593        "targets": [
  2594          {
  2595            "datasource": {
  2596              "type": "prometheus",
  2597              "uid": "${datasource}"
  2598            },
  2599            "editorMode": "code",
  2600            "expr": "sum(rate(pyroscope_request_duration_seconds{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.CompactionService/PollCompactionJobs\",\n}[$__rate_interval]))",
  2601            "hide": false,
  2602            "legendFormat": "__auto",
  2603            "range": true,
  2604            "refId": "B"
  2605          }
  2606        ],
  2607        "title": "Latency Heatmap",
  2608        "transparent": true,
  2609        "type": "heatmap"
  2610      },
  2611      {
  2612        "datasource": {
  2613          "type": "prometheus",
  2614          "uid": "${datasource}"
  2615        },
  2616        "fieldConfig": {
  2617          "defaults": {
  2618            "color": {
  2619              "mode": "palette-classic"
  2620            },
  2621            "custom": {
  2622              "axisBorderShow": false,
  2623              "axisCenteredZero": false,
  2624              "axisColorMode": "text",
  2625              "axisLabel": "",
  2626              "axisPlacement": "auto",
  2627              "barAlignment": 0,
  2628              "barWidthFactor": 0.6,
  2629              "drawStyle": "line",
  2630              "fillOpacity": 0,
  2631              "gradientMode": "none",
  2632              "hideFrom": {
  2633                "legend": false,
  2634                "tooltip": false,
  2635                "viz": false
  2636              },
  2637              "insertNulls": false,
  2638              "lineInterpolation": "linear",
  2639              "lineWidth": 1,
  2640              "pointSize": 5,
  2641              "scaleDistribution": {
  2642                "type": "linear"
  2643              },
  2644              "showPoints": "auto",
  2645              "spanNulls": false,
  2646              "stacking": {
  2647                "group": "A",
  2648                "mode": "none"
  2649              },
  2650              "thresholdsStyle": {
  2651                "mode": "off"
  2652              }
  2653            },
  2654            "mappings": [],
  2655            "thresholds": {
  2656              "mode": "absolute",
  2657              "steps": [
  2658                {
  2659                  "color": "green",
  2660                  "value": 0
  2661                },
  2662                {
  2663                  "color": "red",
  2664                  "value": 80
  2665                }
  2666              ]
  2667            },
  2668            "unit": "ops"
  2669          },
  2670          "overrides": [
  2671            {
  2672              "matcher": {
  2673                "id": "byName",
  2674                "options": "error"
  2675              },
  2676              "properties": [
  2677                {
  2678                  "id": "color",
  2679                  "value": {
  2680                    "fixedColor": "red",
  2681                    "mode": "fixed"
  2682                  }
  2683                }
  2684              ]
  2685            },
  2686            {
  2687              "matcher": {
  2688                "id": "byName",
  2689                "options": "success"
  2690              },
  2691              "properties": [
  2692                {
  2693                  "id": "color",
  2694                  "value": {
  2695                    "fixedColor": "green",
  2696                    "mode": "fixed"
  2697                  }
  2698                }
  2699              ]
  2700            }
  2701          ]
  2702        },
  2703        "gridPos": {
  2704          "h": 8,
  2705          "w": 8,
  2706          "x": 16,
  2707          "y": 52
  2708        },
  2709        "id": 88,
  2710        "options": {
  2711          "legend": {
  2712            "calcs": [],
  2713            "displayMode": "list",
  2714            "placement": "bottom",
  2715            "showLegend": true
  2716          },
  2717          "tooltip": {
  2718            "hideZeros": false,
  2719            "mode": "single",
  2720            "sort": "none"
  2721          }
  2722        },
  2723        "pluginVersion": "12.1.0-90017",
  2724        "targets": [
  2725          {
  2726            "datasource": {
  2727              "type": "prometheus",
  2728              "uid": "${datasource}"
  2729            },
  2730            "editorMode": "code",
  2731            "expr": "sum(irate(pyroscope_request_duration_seconds_count{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.CompactionService/PollCompactionJobs\",\n}[$__rate_interval])) by (status_code)",
  2732            "hide": false,
  2733            "instant": false,
  2734            "legendFormat": "__auto",
  2735            "range": true,
  2736            "refId": "D"
  2737          }
  2738        ],
  2739        "title": "",
  2740        "transparent": true,
  2741        "type": "timeseries"
  2742      },
  2743      {
  2744        "collapsed": false,
  2745        "gridPos": {
  2746          "h": 1,
  2747          "w": 24,
  2748          "x": 0,
  2749          "y": 60
  2750        },
  2751        "id": 67,
  2752        "panels": [],
  2753        "title": "Resource Usage",
  2754        "type": "row"
  2755      },
  2756      {
  2757        "datasource": {
  2758          "type": "datasource",
  2759          "uid": "-- Mixed --"
  2760        },
  2761        "fieldConfig": {
  2762          "defaults": {
  2763            "color": {
  2764              "mode": "palette-classic"
  2765            },
  2766            "custom": {
  2767              "axisBorderShow": false,
  2768              "axisCenteredZero": false,
  2769              "axisColorMode": "text",
  2770              "axisLabel": "",
  2771              "axisPlacement": "auto",
  2772              "axisSoftMin": 0,
  2773              "barAlignment": 1,
  2774              "barWidthFactor": 1,
  2775              "drawStyle": "line",
  2776              "fillOpacity": 0,
  2777              "gradientMode": "none",
  2778              "hideFrom": {
  2779                "legend": false,
  2780                "tooltip": false,
  2781                "viz": false
  2782              },
  2783              "insertNulls": false,
  2784              "lineInterpolation": "linear",
  2785              "lineStyle": {
  2786                "fill": "solid"
  2787              },
  2788              "lineWidth": 1,
  2789              "pointSize": 5,
  2790              "scaleDistribution": {
  2791                "type": "linear"
  2792              },
  2793              "showPoints": "auto",
  2794              "spanNulls": false,
  2795              "stacking": {
  2796                "group": "A",
  2797                "mode": "none"
  2798              },
  2799              "thresholdsStyle": {
  2800                "mode": "off"
  2801              }
  2802            },
  2803            "fieldMinMax": false,
  2804            "mappings": [],
  2805            "min": 0,
  2806            "thresholds": {
  2807              "mode": "absolute",
  2808              "steps": [
  2809                {
  2810                  "color": "green",
  2811                  "value": 0
  2812                }
  2813              ]
  2814            }
  2815          },
  2816          "overrides": [
  2817            {
  2818              "matcher": {
  2819                "id": "byName",
  2820                "options": "Limit"
  2821              },
  2822              "properties": [
  2823                {
  2824                  "id": "custom.lineStyle",
  2825                  "value": {
  2826                    "dash": [
  2827                      10,
  2828                      10
  2829                    ],
  2830                    "fill": "dash"
  2831                  }
  2832                }
  2833              ]
  2834            }
  2835          ]
  2836        },
  2837        "gridPos": {
  2838          "h": 8,
  2839          "w": 8,
  2840          "x": 0,
  2841          "y": 61
  2842        },
  2843        "id": 65,
  2844        "options": {
  2845          "legend": {
  2846            "calcs": [
  2847              "last",
  2848              "max"
  2849            ],
  2850            "displayMode": "table",
  2851            "placement": "bottom",
  2852            "showLegend": false,
  2853            "sortBy": "Max",
  2854            "sortDesc": true
  2855          },
  2856          "tooltip": {
  2857            "hideZeros": false,
  2858            "mode": "multi",
  2859            "sort": "desc"
  2860          }
  2861        },
  2862        "pluginVersion": "12.1.0-90017",
  2863        "targets": [
  2864          {
  2865            "datasource": {
  2866              "type": "prometheus",
  2867              "uid": "${datasource}"
  2868            },
  2869            "editorMode": "code",
  2870            "expr": "sum by (pod) (node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\",container=\"metastore\"})",
  2871            "legendFormat": "__auto",
  2872            "range": true,
  2873            "refId": "by pod"
  2874          },
  2875          {
  2876            "datasource": {
  2877              "type": "prometheus",
  2878              "uid": "${datasource}"
  2879            },
  2880            "editorMode": "code",
  2881            "expr": "max(\n    kube_pod_container_resource_requests{job=~\"(.*/)?kube-state-metrics\", namespace=\"$namespace\", resource=\"cpu\"}\n  * on(namespace,pod)\n    group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".*-metastore\"}\n)",
  2882            "hide": true,
  2883            "legendFormat": "Request",
  2884            "range": true,
  2885            "refId": "A"
  2886          },
  2887          {
  2888            "datasource": {
  2889              "type": "prometheus",
  2890              "uid": "${datasource}"
  2891            },
  2892            "editorMode": "code",
  2893            "expr": "max(\n    kube_pod_container_resource_limits{job=~\"(.*/)?kube-state-metrics\", namespace=\"$namespace\", resource=\"cpu\"}\n  * on(namespace,pod)\n    group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".*-metastore\"}\n)",
  2894            "hide": true,
  2895            "legendFormat": "Limit",
  2896            "range": true,
  2897            "refId": "B"
  2898          }
  2899        ],
  2900        "title": "CPU Usage",
  2901        "transparent": true,
  2902        "type": "timeseries"
  2903      },
  2904      {
  2905        "datasource": {
  2906          "type": "datasource",
  2907          "uid": "-- Mixed --"
  2908        },
  2909        "fieldConfig": {
  2910          "defaults": {
  2911            "color": {
  2912              "mode": "palette-classic"
  2913            },
  2914            "custom": {
  2915              "axisBorderShow": false,
  2916              "axisCenteredZero": false,
  2917              "axisColorMode": "text",
  2918              "axisLabel": "",
  2919              "axisPlacement": "auto",
  2920              "axisSoftMin": 0,
  2921              "barAlignment": 0,
  2922              "barWidthFactor": 0.6,
  2923              "drawStyle": "line",
  2924              "fillOpacity": 0,
  2925              "gradientMode": "none",
  2926              "hideFrom": {
  2927                "legend": false,
  2928                "tooltip": false,
  2929                "viz": false
  2930              },
  2931              "insertNulls": false,
  2932              "lineInterpolation": "linear",
  2933              "lineStyle": {
  2934                "fill": "solid"
  2935              },
  2936              "lineWidth": 1,
  2937              "pointSize": 5,
  2938              "scaleDistribution": {
  2939                "type": "linear"
  2940              },
  2941              "showPoints": "auto",
  2942              "spanNulls": false,
  2943              "stacking": {
  2944                "group": "A",
  2945                "mode": "none"
  2946              },
  2947              "thresholdsStyle": {
  2948                "mode": "dashed"
  2949              }
  2950            },
  2951            "mappings": [],
  2952            "thresholds": {
  2953              "mode": "absolute",
  2954              "steps": [
  2955                {
  2956                  "color": "green",
  2957                  "value": 0
  2958                }
  2959              ]
  2960            },
  2961            "unit": "bytes"
  2962          },
  2963          "overrides": [
  2964            {
  2965              "matcher": {
  2966                "id": "byName",
  2967                "options": "Limit"
  2968              },
  2969              "properties": [
  2970                {
  2971                  "id": "custom.lineStyle",
  2972                  "value": {
  2973                    "dash": [
  2974                      10,
  2975                      10
  2976                    ],
  2977                    "fill": "dash"
  2978                  }
  2979                }
  2980              ]
  2981            }
  2982          ]
  2983        },
  2984        "gridPos": {
  2985          "h": 8,
  2986          "w": 8,
  2987          "x": 8,
  2988          "y": 61
  2989        },
  2990        "id": 66,
  2991        "options": {
  2992          "legend": {
  2993            "calcs": [
  2994              "last",
  2995              "max"
  2996            ],
  2997            "displayMode": "table",
  2998            "placement": "bottom",
  2999            "showLegend": false,
  3000            "sortBy": "Max",
  3001            "sortDesc": true
  3002          },
  3003          "tooltip": {
  3004            "hideZeros": false,
  3005            "mode": "multi",
  3006            "sort": "desc"
  3007          }
  3008        },
  3009        "pluginVersion": "12.1.0-90017",
  3010        "targets": [
  3011          {
  3012            "datasource": {
  3013              "type": "prometheus",
  3014              "uid": "${datasource}"
  3015            },
  3016            "editorMode": "code",
  3017            "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", container=\"metastore\", image!=\"\"}) by (pod)",
  3018            "hide": false,
  3019            "legendFormat": "__auto",
  3020            "range": true,
  3021            "refId": "A"
  3022          },
  3023          {
  3024            "datasource": {
  3025              "type": "prometheus",
  3026              "uid": "${datasource}"
  3027            },
  3028            "editorMode": "code",
  3029            "expr": "max(\n    kube_pod_container_resource_requests{job=~\"(.*/)?kube-state-metrics\", namespace=\"$namespace\", resource=\"memory\"}\n  * on(namespace,pod)\n    group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".*-metastore\"}\n)",
  3030            "hide": true,
  3031            "legendFormat": "Requests",
  3032            "range": true,
  3033            "refId": "B"
  3034          },
  3035          {
  3036            "datasource": {
  3037              "type": "prometheus",
  3038              "uid": "${datasource}"
  3039            },
  3040            "editorMode": "code",
  3041            "expr": "max(\n    kube_pod_container_resource_limits{job=~\"(.*/)?kube-state-metrics\", namespace=\"$namespace\", resource=\"memory\"}\n  * on(namespace,pod)\n    group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=\".*-metastore\"}\n)",
  3042            "hide": true,
  3043            "legendFormat": "Limit",
  3044            "range": true,
  3045            "refId": "C"
  3046          }
  3047        ],
  3048        "title": "Memory Usage",
  3049        "transparent": true,
  3050        "type": "timeseries"
  3051      },
  3052      {
  3053        "datasource": {
  3054          "type": "datasource",
  3055          "uid": "-- Mixed --"
  3056        },
  3057        "fieldConfig": {
  3058          "defaults": {
  3059            "color": {
  3060              "mode": "palette-classic"
  3061            },
  3062            "custom": {
  3063              "axisBorderShow": false,
  3064              "axisCenteredZero": false,
  3065              "axisColorMode": "text",
  3066              "axisLabel": "",
  3067              "axisPlacement": "auto",
  3068              "barAlignment": 1,
  3069              "barWidthFactor": 1,
  3070              "drawStyle": "line",
  3071              "fillOpacity": 0,
  3072              "gradientMode": "none",
  3073              "hideFrom": {
  3074                "legend": false,
  3075                "tooltip": false,
  3076                "viz": false
  3077              },
  3078              "insertNulls": false,
  3079              "lineInterpolation": "linear",
  3080              "lineStyle": {
  3081                "fill": "solid"
  3082              },
  3083              "lineWidth": 1,
  3084              "pointSize": 5,
  3085              "scaleDistribution": {
  3086                "type": "linear"
  3087              },
  3088              "showPoints": "auto",
  3089              "spanNulls": false,
  3090              "stacking": {
  3091                "group": "A",
  3092                "mode": "none"
  3093              },
  3094              "thresholdsStyle": {
  3095                "mode": "off"
  3096              }
  3097            },
  3098            "fieldMinMax": false,
  3099            "mappings": [],
  3100            "thresholds": {
  3101              "mode": "absolute",
  3102              "steps": [
  3103                {
  3104                  "color": "green",
  3105                  "value": 0
  3106                }
  3107              ]
  3108            },
  3109            "unit": "percentunit"
  3110          },
  3111          "overrides": []
  3112        },
  3113        "gridPos": {
  3114          "h": 8,
  3115          "w": 8,
  3116          "x": 16,
  3117          "y": 61
  3118        },
  3119        "id": 84,
  3120        "options": {
  3121          "legend": {
  3122            "calcs": [
  3123              "last",
  3124              "max"
  3125            ],
  3126            "displayMode": "table",
  3127            "placement": "bottom",
  3128            "showLegend": false,
  3129            "sortBy": "Max",
  3130            "sortDesc": true
  3131          },
  3132          "tooltip": {
  3133            "hideZeros": false,
  3134            "mode": "multi",
  3135            "sort": "desc"
  3136          }
  3137        },
  3138        "pluginVersion": "12.1.0-90017",
  3139        "targets": [
  3140          {
  3141            "datasource": {
  3142              "type": "prometheus",
  3143              "uid": "${datasource}"
  3144            },
  3145            "editorMode": "code",
  3146            "expr": "kubelet_volume_stats_used_bytes{namespace=\"$namespace\",persistentvolumeclaim=~\".*metastore.*\"} /\nkubelet_volume_stats_capacity_bytes{namespace=\"$namespace\",persistentvolumeclaim=~\".*metastore.*\"}",
  3147            "hide": false,
  3148            "legendFormat": "{{persistentvolumeclaim}}",
  3149            "range": true,
  3150            "refId": "C"
  3151          }
  3152        ],
  3153        "title": "Disk Usage",
  3154        "transparent": true,
  3155        "type": "timeseries"
  3156      },
  3157      {
  3158        "datasource": {
  3159          "type": "datasource",
  3160          "uid": "-- Mixed --"
  3161        },
  3162        "fieldConfig": {
  3163          "defaults": {
  3164            "color": {
  3165              "mode": "palette-classic"
  3166            },
  3167            "custom": {
  3168              "axisBorderShow": false,
  3169              "axisCenteredZero": false,
  3170              "axisColorMode": "text",
  3171              "axisLabel": "",
  3172              "axisPlacement": "auto",
  3173              "barAlignment": 0,
  3174              "barWidthFactor": 0.6,
  3175              "drawStyle": "line",
  3176              "fillOpacity": 0,
  3177              "gradientMode": "none",
  3178              "hideFrom": {
  3179                "legend": false,
  3180                "tooltip": false,
  3181                "viz": false
  3182              },
  3183              "insertNulls": false,
  3184              "lineInterpolation": "linear",
  3185              "lineWidth": 1,
  3186              "pointSize": 5,
  3187              "scaleDistribution": {
  3188                "type": "linear"
  3189              },
  3190              "showPoints": "auto",
  3191              "spanNulls": false,
  3192              "stacking": {
  3193                "group": "A",
  3194                "mode": "none"
  3195              },
  3196              "thresholdsStyle": {
  3197                "mode": "off"
  3198              }
  3199            },
  3200            "mappings": [],
  3201            "min": 0,
  3202            "thresholds": {
  3203              "mode": "absolute",
  3204              "steps": [
  3205                {
  3206                  "color": "green",
  3207                  "value": 0
  3208                }
  3209              ]
  3210            },
  3211            "unit": "ops"
  3212          },
  3213          "overrides": []
  3214        },
  3215        "gridPos": {
  3216          "h": 9,
  3217          "w": 6,
  3218          "x": 0,
  3219          "y": 69
  3220        },
  3221        "id": 85,
  3222        "options": {
  3223          "legend": {
  3224            "calcs": [
  3225              "mean",
  3226              "max",
  3227              "last"
  3228            ],
  3229            "displayMode": "table",
  3230            "placement": "bottom",
  3231            "showLegend": false
  3232          },
  3233          "tooltip": {
  3234            "hideZeros": false,
  3235            "mode": "multi",
  3236            "sort": "none"
  3237          }
  3238        },
  3239        "pluginVersion": "12.1.0-90017",
  3240        "targets": [
  3241          {
  3242            "datasource": {
  3243              "type": "prometheus",
  3244              "uid": "${datasource}"
  3245            },
  3246            "editorMode": "code",
  3247            "expr": "sum by(operation) (rate(objstore_bucket_operations_total{namespace=\"$namespace\",container=\"metastore\"}[$__rate_interval]))",
  3248            "legendFormat": "__auto",
  3249            "range": true,
  3250            "refId": "A"
  3251          }
  3252        ],
  3253        "title": "Storage I/O",
  3254        "transparent": true,
  3255        "type": "timeseries"
  3256      },
  3257      {
  3258        "datasource": {
  3259          "type": "datasource",
  3260          "uid": "-- Mixed --"
  3261        },
  3262        "fieldConfig": {
  3263          "defaults": {
  3264            "color": {
  3265              "mode": "palette-classic"
  3266            },
  3267            "custom": {
  3268              "axisBorderShow": false,
  3269              "axisCenteredZero": false,
  3270              "axisColorMode": "text",
  3271              "axisLabel": "",
  3272              "axisPlacement": "auto",
  3273              "barAlignment": 1,
  3274              "barWidthFactor": 1,
  3275              "drawStyle": "line",
  3276              "fillOpacity": 10,
  3277              "gradientMode": "none",
  3278              "hideFrom": {
  3279                "legend": false,
  3280                "tooltip": false,
  3281                "viz": false
  3282              },
  3283              "insertNulls": false,
  3284              "lineInterpolation": "linear",
  3285              "lineStyle": {
  3286                "fill": "solid"
  3287              },
  3288              "lineWidth": 1,
  3289              "pointSize": 5,
  3290              "scaleDistribution": {
  3291                "type": "linear"
  3292              },
  3293              "showPoints": "auto",
  3294              "spanNulls": false,
  3295              "stacking": {
  3296                "group": "A",
  3297                "mode": "none"
  3298              },
  3299              "thresholdsStyle": {
  3300                "mode": "off"
  3301              }
  3302            },
  3303            "fieldMinMax": false,
  3304            "mappings": [],
  3305            "thresholds": {
  3306              "mode": "absolute",
  3307              "steps": [
  3308                {
  3309                  "color": "green",
  3310                  "value": 0
  3311                }
  3312              ]
  3313            },
  3314            "unit": "Bps"
  3315          },
  3316          "overrides": []
  3317        },
  3318        "gridPos": {
  3319          "h": 9,
  3320          "w": 6,
  3321          "x": 6,
  3322          "y": 69
  3323        },
  3324        "id": 82,
  3325        "options": {
  3326          "legend": {
  3327            "calcs": [
  3328              "last",
  3329              "max"
  3330            ],
  3331            "displayMode": "table",
  3332            "placement": "bottom",
  3333            "showLegend": false,
  3334            "sortBy": "Max",
  3335            "sortDesc": true
  3336          },
  3337          "tooltip": {
  3338            "hideZeros": false,
  3339            "mode": "multi",
  3340            "sort": "desc"
  3341          }
  3342        },
  3343        "pluginVersion": "12.1.0-90017",
  3344        "targets": [
  3345          {
  3346            "datasource": {
  3347              "type": "prometheus",
  3348              "uid": "${datasource}"
  3349            },
  3350            "editorMode": "code",
  3351            "expr": "(sum(irate(container_network_transmit_bytes_total{job=~\"(.*/)?cadvisor\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]) * on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*metastore.*\"}) by (namespace,pod)) * -1",
  3352            "hide": false,
  3353            "legendFormat": "tx {{pod}}",
  3354            "range": true,
  3355            "refId": "A"
  3356          },
  3357          {
  3358            "datasource": {
  3359              "type": "prometheus",
  3360              "uid": "${datasource}"
  3361            },
  3362            "editorMode": "code",
  3363            "expr": "(sum(irate(container_network_receive_bytes_total{job=~\"(.*/)?cadvisor\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]) * on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*metastore.*\"}) by (namespace,pod))",
  3364            "hide": false,
  3365            "legendFormat": "rx {{pod}}",
  3366            "range": true,
  3367            "refId": "B"
  3368          }
  3369        ],
  3370        "title": "Network",
  3371        "transparent": true,
  3372        "type": "timeseries"
  3373      },
  3374      {
  3375        "datasource": {
  3376          "type": "datasource",
  3377          "uid": "-- Mixed --"
  3378        },
  3379        "fieldConfig": {
  3380          "defaults": {
  3381            "color": {
  3382              "mode": "palette-classic"
  3383            },
  3384            "custom": {
  3385              "axisBorderShow": false,
  3386              "axisCenteredZero": false,
  3387              "axisColorMode": "text",
  3388              "axisLabel": "",
  3389              "axisPlacement": "auto",
  3390              "barAlignment": 1,
  3391              "barWidthFactor": 1,
  3392              "drawStyle": "line",
  3393              "fillOpacity": 0,
  3394              "gradientMode": "none",
  3395              "hideFrom": {
  3396                "legend": false,
  3397                "tooltip": false,
  3398                "viz": false
  3399              },
  3400              "insertNulls": false,
  3401              "lineInterpolation": "linear",
  3402              "lineStyle": {
  3403                "fill": "solid"
  3404              },
  3405              "lineWidth": 1,
  3406              "pointSize": 5,
  3407              "scaleDistribution": {
  3408                "type": "linear"
  3409              },
  3410              "showPoints": "auto",
  3411              "spanNulls": false,
  3412              "stacking": {
  3413                "group": "A",
  3414                "mode": "none"
  3415              },
  3416              "thresholdsStyle": {
  3417                "mode": "off"
  3418              }
  3419            },
  3420            "mappings": [],
  3421            "thresholds": {
  3422              "mode": "absolute",
  3423              "steps": [
  3424                {
  3425                  "color": "green",
  3426                  "value": 0
  3427                },
  3428                {
  3429                  "color": "red",
  3430                  "value": 80
  3431                }
  3432              ]
  3433            }
  3434          },
  3435          "overrides": []
  3436        },
  3437        "gridPos": {
  3438          "h": 9,
  3439          "w": 6,
  3440          "x": 12,
  3441          "y": 69
  3442        },
  3443        "id": 83,
  3444        "options": {
  3445          "legend": {
  3446            "calcs": [
  3447              "last",
  3448              "max"
  3449            ],
  3450            "displayMode": "table",
  3451            "placement": "bottom",
  3452            "showLegend": false,
  3453            "sortBy": "Max",
  3454            "sortDesc": true
  3455          },
  3456          "tooltip": {
  3457            "hideZeros": false,
  3458            "mode": "multi",
  3459            "sort": "desc"
  3460          }
  3461        },
  3462        "pluginVersion": "12.1.0-90017",
  3463        "targets": [
  3464          {
  3465            "datasource": {
  3466              "type": "prometheus",
  3467              "uid": "${datasource}"
  3468            },
  3469            "editorMode": "code",
  3470            "expr": "sum by (pod) (irate(container_fs_writes_total{namespace=\"$namespace\",container=\"metastore\"}[$__rate_interval])) * -1",
  3471            "format": "time_series",
  3472            "hide": false,
  3473            "legendFormat": "write {{pod}}",
  3474            "range": true,
  3475            "refId": "A"
  3476          },
  3477          {
  3478            "datasource": {
  3479              "type": "prometheus",
  3480              "uid": "${datasource}"
  3481            },
  3482            "editorMode": "code",
  3483            "expr": "sum by (pod) (irate(container_fs_reads_total{namespace=\"$namespace\",container=\"metastore\"}[$__rate_interval]))",
  3484            "hide": false,
  3485            "legendFormat": "read {{pod}}",
  3486            "range": true,
  3487            "refId": "B"
  3488          }
  3489        ],
  3490        "title": "Disk I/O",
  3491        "transparent": true,
  3492        "type": "timeseries"
  3493      },
  3494      {
  3495        "datasource": {
  3496          "type": "datasource",
  3497          "uid": "-- Mixed --"
  3498        },
  3499        "fieldConfig": {
  3500          "defaults": {
  3501            "color": {
  3502              "mode": "palette-classic"
  3503            },
  3504            "custom": {
  3505              "axisBorderShow": false,
  3506              "axisCenteredZero": false,
  3507              "axisColorMode": "text",
  3508              "axisLabel": "",
  3509              "axisPlacement": "auto",
  3510              "barAlignment": 1,
  3511              "barWidthFactor": 1,
  3512              "drawStyle": "line",
  3513              "fillOpacity": 0,
  3514              "gradientMode": "none",
  3515              "hideFrom": {
  3516                "legend": false,
  3517                "tooltip": false,
  3518                "viz": false
  3519              },
  3520              "insertNulls": false,
  3521              "lineInterpolation": "linear",
  3522              "lineStyle": {
  3523                "fill": "solid"
  3524              },
  3525              "lineWidth": 1,
  3526              "pointSize": 5,
  3527              "scaleDistribution": {
  3528                "type": "linear"
  3529              },
  3530              "showPoints": "auto",
  3531              "spanNulls": false,
  3532              "stacking": {
  3533                "group": "A",
  3534                "mode": "none"
  3535              },
  3536              "thresholdsStyle": {
  3537                "mode": "off"
  3538              }
  3539            },
  3540            "mappings": [],
  3541            "thresholds": {
  3542              "mode": "absolute",
  3543              "steps": [
  3544                {
  3545                  "color": "green",
  3546                  "value": 0
  3547                },
  3548                {
  3549                  "color": "red",
  3550                  "value": 80
  3551                }
  3552              ]
  3553            }
  3554          },
  3555          "overrides": []
  3556        },
  3557        "gridPos": {
  3558          "h": 9,
  3559          "w": 6,
  3560          "x": 18,
  3561          "y": 69
  3562        },
  3563        "id": 94,
  3564        "options": {
  3565          "legend": {
  3566            "calcs": [
  3567              "last",
  3568              "max"
  3569            ],
  3570            "displayMode": "table",
  3571            "placement": "bottom",
  3572            "showLegend": true,
  3573            "sortBy": "Max",
  3574            "sortDesc": true
  3575          },
  3576          "tooltip": {
  3577            "hideZeros": false,
  3578            "mode": "multi",
  3579            "sort": "desc"
  3580          }
  3581        },
  3582        "pluginVersion": "12.1.0-90017",
  3583        "targets": [
  3584          {
  3585            "datasource": {
  3586              "type": "prometheus",
  3587              "uid": "${datasource}"
  3588            },
  3589            "editorMode": "code",
  3590            "expr": "irate(node_disk_io_time_seconds_total{cluster=\"$cluster\"}[$__rate_interval])\n+ on(instance, device) group_left(pod)\ntopk by(instance, device) (\n  1,\n  0 * label_replace(\n    container_blkio_device_usage_total{\n      cluster=\"$cluster\",\n      namespace=\"$namespace\",\n      container=\"metastore\",\n      operation=\"Write\",\n    },\n    \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n  )\n)\n",
  3591            "hide": false,
  3592            "legendFormat": "{{pod}}/{{device}}",
  3593            "range": true,
  3594            "refId": "A"
  3595          }
  3596        ],
  3597        "title": "Disk I/O Wait",
  3598        "transparent": true,
  3599        "type": "timeseries"
  3600      },
  3601      {
  3602        "collapsed": true,
  3603        "gridPos": {
  3604          "h": 1,
  3605          "w": 24,
  3606          "x": 0,
  3607          "y": 78
  3608        },
  3609        "id": 60,
  3610        "panels": [
  3611          {
  3612            "datasource": {
  3613              "type": "datasource",
  3614              "uid": "-- Mixed --"
  3615            },
  3616            "description": "Number of OS threads created",
  3617            "fieldConfig": {
  3618              "defaults": {
  3619                "color": {
  3620                  "mode": "palette-classic"
  3621                },
  3622                "custom": {
  3623                  "axisBorderShow": false,
  3624                  "axisCenteredZero": false,
  3625                  "axisColorMode": "text",
  3626                  "axisLabel": "",
  3627                  "axisPlacement": "auto",
  3628                  "axisSoftMin": 0,
  3629                  "barAlignment": 0,
  3630                  "barWidthFactor": 0.6,
  3631                  "drawStyle": "line",
  3632                  "fillOpacity": 0,
  3633                  "gradientMode": "none",
  3634                  "hideFrom": {
  3635                    "legend": false,
  3636                    "tooltip": false,
  3637                    "viz": false
  3638                  },
  3639                  "insertNulls": false,
  3640                  "lineInterpolation": "linear",
  3641                  "lineStyle": {
  3642                    "fill": "solid"
  3643                  },
  3644                  "lineWidth": 1,
  3645                  "pointSize": 5,
  3646                  "scaleDistribution": {
  3647                    "type": "linear"
  3648                  },
  3649                  "showPoints": "auto",
  3650                  "spanNulls": false,
  3651                  "stacking": {
  3652                    "group": "A",
  3653                    "mode": "none"
  3654                  },
  3655                  "thresholdsStyle": {
  3656                    "mode": "dashed"
  3657                  }
  3658                },
  3659                "mappings": [],
  3660                "thresholds": {
  3661                  "mode": "absolute",
  3662                  "steps": [
  3663                    {
  3664                      "color": "green"
  3665                    }
  3666                  ]
  3667                },
  3668                "unit": "short"
  3669              },
  3670              "overrides": [
  3671                {
  3672                  "matcher": {
  3673                    "id": "byName",
  3674                    "options": "Limit"
  3675                  },
  3676                  "properties": [
  3677                    {
  3678                      "id": "custom.lineStyle",
  3679                      "value": {
  3680                        "dash": [
  3681                          10,
  3682                          10
  3683                        ],
  3684                        "fill": "dash"
  3685                      }
  3686                    }
  3687                  ]
  3688                }
  3689              ]
  3690            },
  3691            "gridPos": {
  3692              "h": 8,
  3693              "w": 6,
  3694              "x": 0,
  3695              "y": 113
  3696            },
  3697            "id": 61,
  3698            "options": {
  3699              "legend": {
  3700                "calcs": [
  3701                  "last",
  3702                  "max"
  3703                ],
  3704                "displayMode": "table",
  3705                "placement": "bottom",
  3706                "showLegend": false,
  3707                "sortBy": "Max",
  3708                "sortDesc": true
  3709              },
  3710              "tooltip": {
  3711                "hideZeros": false,
  3712                "mode": "multi",
  3713                "sort": "desc"
  3714              }
  3715            },
  3716            "pluginVersion": "12.0.0-85518.patch7-85777",
  3717            "targets": [
  3718              {
  3719                "datasource": {
  3720                  "type": "prometheus",
  3721                  "uid": "${datasource}"
  3722                },
  3723                "editorMode": "code",
  3724                "expr": "go_threads{namespace=\"$namespace\",container=\"metastore\"}",
  3725                "hide": false,
  3726                "legendFormat": "{{pod}}",
  3727                "range": true,
  3728                "refId": "D"
  3729              }
  3730            ],
  3731            "title": "Go Threads",
  3732            "transparent": true,
  3733            "type": "timeseries"
  3734          },
  3735          {
  3736            "datasource": {
  3737              "type": "datasource",
  3738              "uid": "-- Mixed --"
  3739            },
  3740            "fieldConfig": {
  3741              "defaults": {
  3742                "color": {
  3743                  "mode": "palette-classic"
  3744                },
  3745                "custom": {
  3746                  "axisBorderShow": false,
  3747                  "axisCenteredZero": false,
  3748                  "axisColorMode": "text",
  3749                  "axisLabel": "",
  3750                  "axisPlacement": "auto",
  3751                  "axisSoftMin": 0,
  3752                  "barAlignment": 0,
  3753                  "barWidthFactor": 0.6,
  3754                  "drawStyle": "line",
  3755                  "fillOpacity": 0,
  3756                  "gradientMode": "none",
  3757                  "hideFrom": {
  3758                    "legend": false,
  3759                    "tooltip": false,
  3760                    "viz": false
  3761                  },
  3762                  "insertNulls": false,
  3763                  "lineInterpolation": "linear",
  3764                  "lineStyle": {
  3765                    "fill": "solid"
  3766                  },
  3767                  "lineWidth": 1,
  3768                  "pointSize": 5,
  3769                  "scaleDistribution": {
  3770                    "type": "linear"
  3771                  },
  3772                  "showPoints": "auto",
  3773                  "spanNulls": false,
  3774                  "stacking": {
  3775                    "group": "A",
  3776                    "mode": "none"
  3777                  },
  3778                  "thresholdsStyle": {
  3779                    "mode": "dashed"
  3780                  }
  3781                },
  3782                "mappings": [],
  3783                "thresholds": {
  3784                  "mode": "absolute",
  3785                  "steps": [
  3786                    {
  3787                      "color": "green"
  3788                    }
  3789                  ]
  3790                },
  3791                "unit": "short"
  3792              },
  3793              "overrides": [
  3794                {
  3795                  "matcher": {
  3796                    "id": "byName",
  3797                    "options": "Limit"
  3798                  },
  3799                  "properties": [
  3800                    {
  3801                      "id": "custom.lineStyle",
  3802                      "value": {
  3803                        "dash": [
  3804                          10,
  3805                          10
  3806                        ],
  3807                        "fill": "dash"
  3808                      }
  3809                    }
  3810                  ]
  3811                }
  3812              ]
  3813            },
  3814            "gridPos": {
  3815              "h": 8,
  3816              "w": 6,
  3817              "x": 6,
  3818              "y": 113
  3819            },
  3820            "id": 62,
  3821            "options": {
  3822              "legend": {
  3823                "calcs": [
  3824                  "last",
  3825                  "max"
  3826                ],
  3827                "displayMode": "table",
  3828                "placement": "bottom",
  3829                "showLegend": false,
  3830                "sortBy": "Max",
  3831                "sortDesc": true
  3832              },
  3833              "tooltip": {
  3834                "hideZeros": false,
  3835                "mode": "multi",
  3836                "sort": "desc"
  3837              }
  3838            },
  3839            "pluginVersion": "12.0.0-85518.patch7-85777",
  3840            "targets": [
  3841              {
  3842                "datasource": {
  3843                  "type": "prometheus",
  3844                  "uid": "${datasource}"
  3845                },
  3846                "editorMode": "code",
  3847                "expr": "sum by (pod) (go_goroutines{namespace=\"$namespace\",container=\"metastore\"})",
  3848                "hide": false,
  3849                "legendFormat": "__auto",
  3850                "range": true,
  3851                "refId": "D"
  3852              }
  3853            ],
  3854            "title": "Goroutines",
  3855            "transparent": true,
  3856            "type": "timeseries"
  3857          },
  3858          {
  3859            "datasource": {
  3860              "type": "datasource",
  3861              "uid": "-- Mixed --"
  3862            },
  3863            "fieldConfig": {
  3864              "defaults": {
  3865                "color": {
  3866                  "mode": "palette-classic"
  3867                },
  3868                "custom": {
  3869                  "axisBorderShow": false,
  3870                  "axisCenteredZero": false,
  3871                  "axisColorMode": "text",
  3872                  "axisLabel": "",
  3873                  "axisPlacement": "auto",
  3874                  "axisSoftMin": 0,
  3875                  "barAlignment": 0,
  3876                  "barWidthFactor": 0.6,
  3877                  "drawStyle": "line",
  3878                  "fillOpacity": 0,
  3879                  "gradientMode": "none",
  3880                  "hideFrom": {
  3881                    "legend": false,
  3882                    "tooltip": false,
  3883                    "viz": false
  3884                  },
  3885                  "insertNulls": false,
  3886                  "lineInterpolation": "linear",
  3887                  "lineStyle": {
  3888                    "fill": "solid"
  3889                  },
  3890                  "lineWidth": 1,
  3891                  "pointSize": 5,
  3892                  "scaleDistribution": {
  3893                    "type": "linear"
  3894                  },
  3895                  "showPoints": "auto",
  3896                  "spanNulls": false,
  3897                  "stacking": {
  3898                    "group": "A",
  3899                    "mode": "none"
  3900                  },
  3901                  "thresholdsStyle": {
  3902                    "mode": "dashed"
  3903                  }
  3904                },
  3905                "mappings": [],
  3906                "thresholds": {
  3907                  "mode": "absolute",
  3908                  "steps": [
  3909                    {
  3910                      "color": "green"
  3911                    }
  3912                  ]
  3913                },
  3914                "unit": "bytes"
  3915              },
  3916              "overrides": [
  3917                {
  3918                  "matcher": {
  3919                    "id": "byName",
  3920                    "options": "Limit"
  3921                  },
  3922                  "properties": [
  3923                    {
  3924                      "id": "custom.lineStyle",
  3925                      "value": {
  3926                        "dash": [
  3927                          10,
  3928                          10
  3929                        ],
  3930                        "fill": "dash"
  3931                      }
  3932                    }
  3933                  ]
  3934                }
  3935              ]
  3936            },
  3937            "gridPos": {
  3938              "h": 8,
  3939              "w": 6,
  3940              "x": 12,
  3941              "y": 113
  3942            },
  3943            "id": 63,
  3944            "options": {
  3945              "legend": {
  3946                "calcs": [
  3947                  "last",
  3948                  "max"
  3949                ],
  3950                "displayMode": "table",
  3951                "placement": "bottom",
  3952                "showLegend": false,
  3953                "sortBy": "Max",
  3954                "sortDesc": true
  3955              },
  3956              "tooltip": {
  3957                "hideZeros": false,
  3958                "mode": "multi",
  3959                "sort": "desc"
  3960              }
  3961            },
  3962            "pluginVersion": "12.0.0-85518.patch7-85777",
  3963            "targets": [
  3964              {
  3965                "datasource": {
  3966                  "type": "prometheus",
  3967                  "uid": "${datasource}"
  3968                },
  3969                "editorMode": "code",
  3970                "expr": "sum by (pod) (go_memstats_heap_inuse_bytes{namespace=\"$namespace\",container=\"metastore\"})",
  3971                "hide": false,
  3972                "legendFormat": "__auto",
  3973                "range": true,
  3974                "refId": "D"
  3975              }
  3976            ],
  3977            "title": "Heap In-Use",
  3978            "transparent": true,
  3979            "type": "timeseries"
  3980          },
  3981          {
  3982            "datasource": {
  3983              "type": "datasource",
  3984              "uid": "-- Mixed --"
  3985            },
  3986            "fieldConfig": {
  3987              "defaults": {
  3988                "color": {
  3989                  "mode": "palette-classic"
  3990                },
  3991                "custom": {
  3992                  "axisBorderShow": false,
  3993                  "axisCenteredZero": false,
  3994                  "axisColorMode": "text",
  3995                  "axisLabel": "",
  3996                  "axisPlacement": "auto",
  3997                  "axisSoftMin": 0,
  3998                  "barAlignment": 1,
  3999                  "barWidthFactor": 1,
  4000                  "drawStyle": "line",
  4001                  "fillOpacity": 0,
  4002                  "gradientMode": "none",
  4003                  "hideFrom": {
  4004                    "legend": false,
  4005                    "tooltip": false,
  4006                    "viz": false
  4007                  },
  4008                  "insertNulls": false,
  4009                  "lineInterpolation": "linear",
  4010                  "lineStyle": {
  4011                    "fill": "solid"
  4012                  },
  4013                  "lineWidth": 1,
  4014                  "pointSize": 5,
  4015                  "scaleDistribution": {
  4016                    "type": "linear"
  4017                  },
  4018                  "showPoints": "auto",
  4019                  "spanNulls": false,
  4020                  "stacking": {
  4021                    "group": "A",
  4022                    "mode": "none"
  4023                  },
  4024                  "thresholdsStyle": {
  4025                    "mode": "off"
  4026                  }
  4027                },
  4028                "fieldMinMax": false,
  4029                "mappings": [],
  4030                "min": 0,
  4031                "thresholds": {
  4032                  "mode": "absolute",
  4033                  "steps": [
  4034                    {
  4035                      "color": "green"
  4036                    }
  4037                  ]
  4038                }
  4039              },
  4040              "overrides": [
  4041                {
  4042                  "matcher": {
  4043                    "id": "byName",
  4044                    "options": "Limit"
  4045                  },
  4046                  "properties": [
  4047                    {
  4048                      "id": "custom.lineStyle",
  4049                      "value": {
  4050                        "dash": [
  4051                          10,
  4052                          10
  4053                        ],
  4054                        "fill": "dash"
  4055                      }
  4056                    }
  4057                  ]
  4058                }
  4059              ]
  4060            },
  4061            "gridPos": {
  4062              "h": 8,
  4063              "w": 6,
  4064              "x": 18,
  4065              "y": 113
  4066            },
  4067            "id": 64,
  4068            "options": {
  4069              "legend": {
  4070                "calcs": [
  4071                  "last",
  4072                  "max"
  4073                ],
  4074                "displayMode": "table",
  4075                "placement": "bottom",
  4076                "showLegend": false,
  4077                "sortBy": "Max",
  4078                "sortDesc": true
  4079              },
  4080              "tooltip": {
  4081                "hideZeros": false,
  4082                "mode": "multi",
  4083                "sort": "desc"
  4084              }
  4085            },
  4086            "pluginVersion": "12.0.0-85518.patch7-85777",
  4087            "targets": [
  4088              {
  4089                "datasource": {
  4090                  "type": "prometheus",
  4091                  "uid": "${datasource}"
  4092                },
  4093                "editorMode": "code",
  4094                "expr": "sum by (pod) (container_file_descriptors{namespace=\"$namespace\",container=\"metastore\"})",
  4095                "hide": false,
  4096                "legendFormat": "__auto",
  4097                "range": true,
  4098                "refId": "A"
  4099              }
  4100            ],
  4101            "title": "File Descriptors",
  4102            "transparent": true,
  4103            "type": "timeseries"
  4104          }
  4105        ],
  4106        "title": "Runtime",
  4107        "type": "row"
  4108      }
  4109    ],
  4110    "preload": false,
  4111    "schemaVersion": 41,
  4112    "tags": [
  4113      "pyroscope",
  4114      "pyroscope-v2"
  4115    ],
  4116    "templating": {
  4117      "list": [
  4118        {
  4119          "current": {
  4120            "text": "ops-cortex",
  4121            "value": "000000134"
  4122          },
  4123          "includeAll": false,
  4124          "name": "datasource",
  4125          "options": [],
  4126          "query": "prometheus",
  4127          "refresh": 1,
  4128          "regex": "",
  4129          "type": "datasource"
  4130        },
  4131        {
  4132          "allowCustomValue": false,
  4133          "current": {
  4134            "text": "Grafana Logging",
  4135            "value": "000000193"
  4136          },
  4137          "name": "loki_datasource",
  4138          "options": [],
  4139          "query": "loki",
  4140          "refresh": 1,
  4141          "regex": "",
  4142          "type": "datasource"
  4143        },
  4144        {
  4145          "allowCustomValue": false,
  4146          "current": {
  4147            "text": "pyroscope-dev",
  4148            "value": "pyroscope-dev"
  4149          },
  4150          "datasource": {
  4151            "type": "prometheus",
  4152            "uid": "${datasource}"
  4153          },
  4154          "definition": "label_values(pyroscope_build_info{namespace=\"$namespace\"},cluster)",
  4155          "label": "cluster",
  4156          "name": "cluster",
  4157          "options": [],
  4158          "query": {
  4159            "qryType": 1,
  4160            "query": "label_values(pyroscope_build_info{namespace=\"$namespace\"},cluster)",
  4161            "refId": "PrometheusVariableQueryEditor-VariableQuery"
  4162          },
  4163          "refresh": 1,
  4164          "regex": "",
  4165          "sort": 1,
  4166          "type": "query"
  4167        },
  4168        {
  4169          "allowCustomValue": false,
  4170          "current": {
  4171            "text": "default",
  4172            "value": "default"
  4173          },
  4174          "datasource": {
  4175            "type": "prometheus",
  4176            "uid": "${datasource}"
  4177          },
  4178          "definition": "label_values(pyroscope_build_info,namespace)",
  4179          "label": "namespace",
  4180          "name": "namespace",
  4181          "options": [],
  4182          "query": {
  4183            "qryType": 1,
  4184            "query": "label_values(pyroscope_build_info,namespace)",
  4185            "refId": "PrometheusVariableQueryEditor-VariableQuery"
  4186          },
  4187          "refresh": 1,
  4188          "regex": ".*",
  4189          "type": "query"
  4190        }
  4191      ]
  4192    },
  4193    "time": {
  4194      "from": "now-6h",
  4195      "to": "now"
  4196    },
  4197    "timepicker": {},
  4198    "timezone": "utc",
  4199    "title": "Pyroscope v2 / Metastore"
  4200  }