github.com/grafana/pyroscope@v1.18.0/operations/monitoring/dashboards/v2-read-path.json (about)

     1  {
     2    "annotations": {
     3      "list": [
     4        {
     5          "builtIn": 1,
     6          "datasource": {
     7            "type": "grafana",
     8            "uid": "-- Grafana --"
     9          },
    10          "enable": true,
    11          "hide": true,
    12          "iconColor": "rgba(0, 211, 255, 1)",
    13          "name": "Annotations \u0026 Alerts",
    14          "type": "dashboard"
    15        },
    16        {
    17          "datasource": {
    18            "type": "loki",
    19            "uid": "${loki_datasource}"
    20          },
    21          "enable": false,
    22          "expr": "{cluster=\"$cluster\", container=\"kube-diff-logger\"} | json | namespace_extracted=\"$namespace\"",
    23          "hide": false,
    24          "iconColor": "yellow",
    25          "instant": false,
    26          "name": "K8s Changes",
    27          "tagKeys": "type,name_extracted,verb",
    28          "textFormat": "{{notes}}",
    29          "titleFormat": ""
    30        },
    31        {
    32          "datasource": {
    33            "type": "loki",
    34            "uid": "${loki_datasource}"
    35          },
    36          "enable": false,
    37          "expr": "{namespace=\"$namespace\",container=\"eventrouter\"} != \"Preemption is not helpful\" != \"Failed deploy model\" | logfmt | type=\"Warning\"",
    38          "hide": false,
    39          "iconColor": "orange",
    40          "instant": false,
    41          "name": "K8s Warnings",
    42          "tagKeys": "type,object_kind,object_name",
    43          "textFormat": "{{message}}",
    44          "titleFormat": ""
    45        },
    46        {
    47          "datasource": {
    48            "type": "loki",
    49            "uid": "${loki_datasource}"
    50          },
    51          "enable": true,
    52          "expr": "{namespace=\"$namespace\",container=\"eventrouter\"} | logfmt | type!=\"Normal\" | type!=\"Warning\"",
    53          "hide": false,
    54          "iconColor": "red",
    55          "instant": false,
    56          "name": "K8s Errors",
    57          "tagKeys": "type,object_kind,object_name",
    58          "textFormat": "{{message}}"
    59        },
    60        {
    61          "datasource": {
    62            "type": "loki",
    63            "uid": "${loki_datasource}"
    64          },
    65          "enable": false,
    66          "expr": "{namespace=\"$namespace\",container=\"eventrouter\"} | logfmt",
    67          "hide": false,
    68          "iconColor": "blue",
    69          "instant": false,
    70          "name": "K8s All events",
    71          "tagKeys": "type,object_kind,object_name",
    72          "textFormat": "{{message}}"
    73        }
    74      ]
    75    },
    76    "editable": true,
    77    "fiscalYearStartMonth": 0,
    78    "graphTooltip": 1,
    79    "id": 12593,
    80    "links": [
    81      {
    82        "asDropdown": true,
    83        "icon": "external link",
    84        "includeVars": true,
    85        "keepTime": true,
    86        "tags": [
    87          "pyroscope"
    88        ],
    89        "targetBlank": false,
    90        "title": "Pyroscope Dashboards",
    91        "type": "dashboards"
    92      }
    93    ],
    94    "panels": [
    95      {
    96        "collapsed": true,
    97        "gridPos": {
    98          "h": 1,
    99          "w": 24,
   100          "x": 0,
   101          "y": 0
   102        },
   103        "id": 68,
   104        "panels": [
   105          {
   106            "datasource": {
   107              "type": "loki",
   108              "uid": "${loki_datasource}"
   109            },
   110            "description": "SelectMergeProfile filtered out by default as the API is used by Canary Exporter",
   111            "fieldConfig": {
   112              "defaults": {
   113                "color": {
   114                  "mode": "thresholds"
   115                },
   116                "custom": {
   117                  "align": "auto",
   118                  "cellOptions": {
   119                    "type": "auto"
   120                  },
   121                  "filterable": true,
   122                  "inspect": false
   123                },
   124                "mappings": [],
   125                "thresholds": {
   126                  "mode": "absolute",
   127                  "steps": [
   128                    {
   129                      "color": "green",
   130                      "value": 0
   131                    },
   132                    {
   133                      "color": "red",
   134                      "value": 80
   135                    }
   136                  ]
   137                }
   138              },
   139              "overrides": [
   140                {
   141                  "matcher": {
   142                    "id": "byName",
   143                    "options": "pod"
   144                  },
   145                  "properties": [
   146                    {
   147                      "id": "custom.width",
   148                      "value": 280
   149                    }
   150                  ]
   151                },
   152                {
   153                  "matcher": {
   154                    "id": "byName",
   155                    "options": "orgID"
   156                  },
   157                  "properties": [
   158                    {
   159                      "id": "custom.width",
   160                      "value": 86
   161                    }
   162                  ]
   163                },
   164                {
   165                  "matcher": {
   166                    "id": "byName",
   167                    "options": "uri"
   168                  },
   169                  "properties": [
   170                    {
   171                      "id": "custom.width",
   172                      "value": 312
   173                    }
   174                  ]
   175                },
   176                {
   177                  "matcher": {
   178                    "id": "byName",
   179                    "options": "Time"
   180                  },
   181                  "properties": [
   182                    {
   183                      "id": "custom.width",
   184                      "value": 179
   185                    }
   186                  ]
   187                },
   188                {
   189                  "matcher": {
   190                    "id": "byName",
   191                    "options": "Trace ID"
   192                  },
   193                  "properties": [
   194                    {
   195                      "id": "custom.width",
   196                      "value": 318
   197                    }
   198                  ]
   199                },
   200                {
   201                  "matcher": {
   202                    "id": "byName",
   203                    "options": "Endpoint"
   204                  },
   205                  "properties": [
   206                    {
   207                      "id": "custom.width",
   208                      "value": 420
   209                    }
   210                  ]
   211                },
   212                {
   213                  "matcher": {
   214                    "id": "byName",
   215                    "options": "Pod"
   216                  },
   217                  "properties": [
   218                    {
   219                      "id": "custom.width",
   220                      "value": 379
   221                    }
   222                  ]
   223                },
   224                {
   225                  "matcher": {
   226                    "id": "byName",
   227                    "options": "route"
   228                  },
   229                  "properties": [
   230                    {
   231                      "id": "custom.width",
   232                      "value": 347
   233                    }
   234                  ]
   235                },
   236                {
   237                  "matcher": {
   238                    "id": "byName",
   239                    "options": "Route"
   240                  },
   241                  "properties": [
   242                    {
   243                      "id": "custom.width",
   244                      "value": 363
   245                    }
   246                  ]
   247                }
   248              ]
   249            },
   250            "gridPos": {
   251              "h": 8,
   252              "w": 24,
   253              "x": 0,
   254              "y": 1
   255            },
   256            "id": 69,
   257            "options": {
   258              "cellHeight": "sm",
   259              "footer": {
   260                "countRows": false,
   261                "fields": "",
   262                "reducer": [
   263                  "sum"
   264                ],
   265                "show": false
   266              },
   267              "showHeader": true,
   268              "sortBy": [
   269                {
   270                  "desc": true,
   271                  "displayName": "Time"
   272                }
   273              ]
   274            },
   275            "pluginVersion": "12.1.0-90017",
   276            "targets": [
   277              {
   278                "datasource": {
   279                  "type": "loki",
   280                  "uid": "${loki_datasource}"
   281                },
   282                "direction": "backward",
   283                "editorMode": "code",
   284                "expr": "{namespace=~\"$namespace\",cluster=~\"$cluster\",container=\"query-frontend\"}\n|= \"route=/querier.v1.\"\n| logfmt | line_format \"tenant={{.orgID}} status={{.status}}  duration={{.duration}} traceID={{.traceID}} {{.uri}}\"",
   285                "queryType": "range",
   286                "refId": "A"
   287              }
   288            ],
   289            "title": "All Queries",
   290            "transformations": [
   291              {
   292                "id": "extractFields",
   293                "options": {
   294                  "delimiter": ",",
   295                  "format": "json",
   296                  "keepTime": false,
   297                  "replace": false,
   298                  "source": "labels"
   299                }
   300              },
   301              {
   302                "id": "organize",
   303                "options": {
   304                  "excludeByName": {
   305                    "Drone build for Grafana": true,
   306                    "Drone build for Grafana Enterprise": true,
   307                    "Line": true,
   308                    "Okta Logs": true,
   309                    "Profile TraceID": true,
   310                    "TraceID (json)": true,
   311                    "__adaptive_logs_sampled__": true,
   312                    "app_kubernetes_io_component": true,
   313                    "app_kubernetes_io_instance": true,
   314                    "app_kubernetes_io_name": true,
   315                    "caller": true,
   316                    "cluster": true,
   317                    "container": true,
   318                    "debug": true,
   319                    "detected_level": true,
   320                    "id": true,
   321                    "job": true,
   322                    "labelTypes": true,
   323                    "labels": true,
   324                    "level": true,
   325                    "method": true,
   326                    "msg": true,
   327                    "name": true,
   328                    "pod_template_hash": true,
   329                    "request_body_size": true,
   330                    "service_name": true,
   331                    "status": false,
   332                    "stream": true,
   333                    "test_run_id": true,
   334                    "traceID": true,
   335                    "traceID (label)": true,
   336                    "traceID 1": true,
   337                    "trace_id (labels)": true,
   338                    "ts": true,
   339                    "tsNs": true
   340                  },
   341                  "includeByName": {},
   342                  "indexByName": {
   343                    "Drone build for Grafana": 30,
   344                    "Drone build for Grafana Enterprise": 31,
   345                    "Line": 4,
   346                    "Time": 1,
   347                    "TraceID (json)": 8,
   348                    "TraceID (log line)": 29,
   349                    "app_kubernetes_io_component": 11,
   350                    "app_kubernetes_io_instance": 12,
   351                    "app_kubernetes_io_name": 13,
   352                    "caller": 14,
   353                    "cluster": 15,
   354                    "container": 16,
   355                    "detected_level": 17,
   356                    "duration": 18,
   357                    "id": 7,
   358                    "job": 19,
   359                    "labelTypes": 6,
   360                    "labels": 0,
   361                    "level": 20,
   362                    "msg": 21,
   363                    "name": 22,
   364                    "namespace": 24,
   365                    "pod": 23,
   366                    "pod_template_hash": 25,
   367                    "route": 3,
   368                    "service_name": 26,
   369                    "stream": 27,
   370                    "tenant": 2,
   371                    "test_run_id": 10,
   372                    "traceID": 9,
   373                    "traceID (label)": 32,
   374                    "trace_id (labels)": 33,
   375                    "ts": 28,
   376                    "tsNs": 5
   377                  },
   378                  "renameByName": {
   379                    "Line": "",
   380                    "Time": "",
   381                    "TraceID": "Trace ID",
   382                    "duration": "Duration",
   383                    "labels": "",
   384                    "namespace": "Namespace",
   385                    "orgID": "Tenant",
   386                    "pod": "Pod",
   387                    "pod_template_hash": "",
   388                    "route": "Route",
   389                    "status": "Status",
   390                    "tenant": "Tenant",
   391                    "uri": "Endpoint"
   392                  }
   393                }
   394              }
   395            ],
   396            "transparent": true,
   397            "type": "table"
   398          },
   399          {
   400            "datasource": {
   401              "type": "tempo",
   402              "uid": "${tempo_datasource}"
   403            },
   404            "fieldConfig": {
   405              "defaults": {
   406                "color": {
   407                  "mode": "thresholds"
   408                },
   409                "custom": {
   410                  "align": "auto",
   411                  "cellOptions": {
   412                    "type": "auto",
   413                    "wrapText": false
   414                  },
   415                  "filterable": true,
   416                  "inspect": false
   417                },
   418                "mappings": [],
   419                "thresholds": {
   420                  "mode": "absolute",
   421                  "steps": [
   422                    {
   423                      "color": "green",
   424                      "value": 0
   425                    },
   426                    {
   427                      "color": "red",
   428                      "value": 80
   429                    }
   430                  ]
   431                }
   432              },
   433              "overrides": [
   434                {
   435                  "matcher": {
   436                    "id": "byName",
   437                    "options": "Trace Name"
   438                  },
   439                  "properties": [
   440                    {
   441                      "id": "custom.width",
   442                      "value": 455
   443                    }
   444                  ]
   445                },
   446                {
   447                  "matcher": {
   448                    "id": "byName",
   449                    "options": "Trace ID"
   450                  },
   451                  "properties": [
   452                    {
   453                      "id": "custom.width",
   454                      "value": 311
   455                    }
   456                  ]
   457                }
   458              ]
   459            },
   460            "gridPos": {
   461              "h": 10,
   462              "w": 24,
   463              "x": 0,
   464              "y": 9
   465            },
   466            "id": 70,
   467            "options": {
   468              "cellHeight": "sm",
   469              "footer": {
   470                "countRows": false,
   471                "enablePagination": false,
   472                "fields": "",
   473                "reducer": [
   474                  "sum"
   475                ],
   476                "show": false
   477              },
   478              "frameIndex": 0,
   479              "showHeader": true,
   480              "sortBy": [
   481                {
   482                  "desc": true,
   483                  "displayName": "Start time"
   484                }
   485              ]
   486            },
   487            "pluginVersion": "12.1.0-90017",
   488            "targets": [
   489              {
   490                "datasource": {
   491                  "type": "tempo",
   492                  "uid": "${tempo_datasource}"
   493                },
   494                "filters": [
   495                  {
   496                    "id": "d5be7410",
   497                    "operator": "=",
   498                    "scope": "resource",
   499                    "tag": "k8s.namespace.name",
   500                    "value": [
   501                      "$namespace"
   502                    ],
   503                    "valueType": "string"
   504                  },
   505                  {
   506                    "id": "service-name",
   507                    "operator": "=",
   508                    "scope": "resource",
   509                    "tag": "service.name",
   510                    "value": [
   511                      "pyroscope-query-frontend",
   512                      "pyroscope-query-backend"
   513                    ],
   514                    "valueType": "string"
   515                  },
   516                  {
   517                    "id": "min-duration",
   518                    "operator": "\u003e",
   519                    "tag": "duration",
   520                    "value": "100ms",
   521                    "valueType": "duration"
   522                  },
   523                  {
   524                    "id": "span-name",
   525                    "operator": "!=",
   526                    "scope": "span",
   527                    "tag": "name",
   528                    "value": [
   529                      "HTTP GET - debug_pprof"
   530                    ],
   531                    "valueType": "string"
   532                  }
   533                ],
   534                "limit": 25,
   535                "metricsQueryType": "instant",
   536                "queryType": "traceqlSearch",
   537                "refId": "A",
   538                "spss": 5,
   539                "tableType": "traces"
   540              }
   541            ],
   542            "title": "Traces",
   543            "transparent": true,
   544            "type": "table"
   545          }
   546        ],
   547        "title": "Query Log",
   548        "type": "row"
   549      },
   550      {
   551        "collapsed": false,
   552        "gridPos": {
   553          "h": 1,
   554          "w": 24,
   555          "x": 0,
   556          "y": 1
   557        },
   558        "id": 57,
   559        "panels": [],
   560        "title": "Metadata Query Service (metastore)",
   561        "type": "row"
   562      },
   563      {
   564        "datasource": {
   565          "type": "prometheus",
   566          "uid": "${datasource}"
   567        },
   568        "description": "Raft node roles observed by each node. The view is not meant to be 100% consistent, and only serves monitoring purposes",
   569        "fieldConfig": {
   570          "defaults": {
   571            "color": {
   572              "mode": "continuous-GrYlRd"
   573            },
   574            "custom": {
   575              "axisPlacement": "auto",
   576              "fillOpacity": 15,
   577              "hideFrom": {
   578                "legend": false,
   579                "tooltip": false,
   580                "viz": false
   581              },
   582              "insertNulls": false,
   583              "lineWidth": 1,
   584              "spanNulls": false
   585            },
   586            "fieldMinMax": false,
   587            "mappings": [
   588              {
   589                "options": {
   590                  "from": 0,
   591                  "result": {
   592                    "color": "green",
   593                    "index": 0,
   594                    "text": "Follower"
   595                  },
   596                  "to": 99
   597                },
   598                "type": "range"
   599              },
   600              {
   601                "options": {
   602                  "from": 100,
   603                  "result": {
   604                    "color": "dark-red",
   605                    "index": 1,
   606                    "text": "Leader"
   607                  },
   608                  "to": 1000000
   609                },
   610                "type": "range"
   611              }
   612            ],
   613            "thresholds": {
   614              "mode": "absolute",
   615              "steps": [
   616                {
   617                  "color": "dark-blue",
   618                  "value": 0
   619                },
   620                {
   621                  "color": "light-red",
   622                  "value": 100
   623                }
   624              ]
   625            },
   626            "unit": "short"
   627          },
   628          "overrides": []
   629        },
   630        "gridPos": {
   631          "h": 7,
   632          "w": 12,
   633          "x": 0,
   634          "y": 2
   635        },
   636        "id": 54,
   637        "options": {
   638          "alignValue": "center",
   639          "legend": {
   640            "displayMode": "list",
   641            "placement": "bottom",
   642            "showLegend": false
   643          },
   644          "mergeValues": true,
   645          "rowHeight": 0.8,
   646          "showValue": "never",
   647          "tooltip": {
   648            "hideZeros": false,
   649            "mode": "multi",
   650            "sort": "asc"
   651          }
   652        },
   653        "pluginVersion": "12.1.0-90017",
   654        "targets": [
   655          {
   656            "datasource": {
   657              "type": "prometheus",
   658              "uid": "${datasource}"
   659            },
   660            "editorMode": "code",
   661            "exemplar": false,
   662            "expr": "sum by (pod) (\n  pyroscope_metastore_raft_state{namespace=\"$namespace\",state=\"Follower\"} or\n  pyroscope_metastore_raft_state{namespace=\"$namespace\",state=\"Leader\"} * 100\n)",
   663            "format": "time_series",
   664            "hide": false,
   665            "instant": false,
   666            "legendFormat": "{{label_name}}",
   667            "range": true,
   668            "refId": "D"
   669          }
   670        ],
   671        "title": "Raft Nodes",
   672        "transparent": true,
   673        "type": "state-timeline"
   674      },
   675      {
   676        "datasource": {
   677          "type": "prometheus",
   678          "uid": "${datasource}"
   679        },
   680        "description": "The latency of QueryMetadata endpoint observed by metastore nodes.\n\nThe only user is the query-frontend service.",
   681        "fieldConfig": {
   682          "defaults": {
   683            "color": {
   684              "mode": "palette-classic"
   685            },
   686            "custom": {
   687              "axisBorderShow": false,
   688              "axisCenteredZero": false,
   689              "axisColorMode": "text",
   690              "axisLabel": "",
   691              "axisPlacement": "auto",
   692              "barAlignment": 0,
   693              "barWidthFactor": 0.6,
   694              "drawStyle": "line",
   695              "fillOpacity": 0,
   696              "gradientMode": "none",
   697              "hideFrom": {
   698                "legend": false,
   699                "tooltip": false,
   700                "viz": false
   701              },
   702              "insertNulls": false,
   703              "lineInterpolation": "linear",
   704              "lineWidth": 1,
   705              "pointSize": 5,
   706              "scaleDistribution": {
   707                "type": "linear"
   708              },
   709              "showPoints": "auto",
   710              "spanNulls": false,
   711              "stacking": {
   712                "group": "A",
   713                "mode": "none"
   714              },
   715              "thresholdsStyle": {
   716                "mode": "off"
   717              }
   718            },
   719            "mappings": [],
   720            "thresholds": {
   721              "mode": "absolute",
   722              "steps": [
   723                {
   724                  "color": "green",
   725                  "value": 0
   726                },
   727                {
   728                  "color": "red",
   729                  "value": 80
   730                }
   731              ]
   732            },
   733            "unit": "s"
   734          },
   735          "overrides": [
   736            {
   737              "matcher": {
   738                "id": "byFrameRefID",
   739                "options": "B"
   740              },
   741              "properties": [
   742                {
   743                  "id": "custom.drawStyle",
   744                  "value": "points"
   745                },
   746                {
   747                  "id": "color",
   748                  "value": {
   749                    "fixedColor": "red",
   750                    "mode": "fixed"
   751                  }
   752                },
   753                {
   754                  "id": "custom.pointSize",
   755                  "value": 11
   756                }
   757              ]
   758            }
   759          ]
   760        },
   761        "gridPos": {
   762          "h": 7,
   763          "w": 6,
   764          "x": 12,
   765          "y": 2
   766        },
   767        "id": 56,
   768        "options": {
   769          "legend": {
   770            "calcs": [],
   771            "displayMode": "list",
   772            "placement": "bottom",
   773            "showLegend": true
   774          },
   775          "tooltip": {
   776            "hideZeros": false,
   777            "mode": "single",
   778            "sort": "none"
   779          }
   780        },
   781        "pluginVersion": "12.1.0-90017",
   782        "targets": [
   783          {
   784            "datasource": {
   785              "type": "prometheus",
   786              "uid": "${datasource}"
   787            },
   788            "editorMode": "code",
   789            "expr": "histogram_quantile(0.99, sum(rate(pyroscope_request_duration_seconds_bucket{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.MetadataQueryService/QueryMetadata\",\n}[$__rate_interval])) by (le))",
   790            "hide": false,
   791            "instant": false,
   792            "legendFormat": "p99",
   793            "range": true,
   794            "refId": "D"
   795          },
   796          {
   797            "datasource": {
   798              "type": "prometheus",
   799              "uid": "${datasource}"
   800            },
   801            "editorMode": "code",
   802            "expr": "histogram_quantile(0.95, sum(rate(pyroscope_request_duration_seconds_bucket{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.MetadataQueryService/QueryMetadata\",\n}[$__rate_interval])) by (le))",
   803            "hide": false,
   804            "instant": false,
   805            "legendFormat": "p95",
   806            "range": true,
   807            "refId": "A"
   808          }
   809        ],
   810        "title": "QueryMetadata Latency",
   811        "transparent": true,
   812        "type": "timeseries"
   813      },
   814      {
   815        "datasource": {
   816          "type": "prometheus",
   817          "uid": "${datasource}"
   818        },
   819        "description": "The latency of QueryMetadataLabels endpoint observed by metastore nodes.\n\nThe only user is the query-frontend service.",
   820        "fieldConfig": {
   821          "defaults": {
   822            "color": {
   823              "mode": "palette-classic"
   824            },
   825            "custom": {
   826              "axisBorderShow": false,
   827              "axisCenteredZero": false,
   828              "axisColorMode": "text",
   829              "axisLabel": "",
   830              "axisPlacement": "auto",
   831              "barAlignment": 0,
   832              "barWidthFactor": 0.6,
   833              "drawStyle": "line",
   834              "fillOpacity": 0,
   835              "gradientMode": "none",
   836              "hideFrom": {
   837                "legend": false,
   838                "tooltip": false,
   839                "viz": false
   840              },
   841              "insertNulls": false,
   842              "lineInterpolation": "linear",
   843              "lineWidth": 1,
   844              "pointSize": 5,
   845              "scaleDistribution": {
   846                "type": "linear"
   847              },
   848              "showPoints": "auto",
   849              "spanNulls": false,
   850              "stacking": {
   851                "group": "A",
   852                "mode": "none"
   853              },
   854              "thresholdsStyle": {
   855                "mode": "off"
   856              }
   857            },
   858            "mappings": [],
   859            "thresholds": {
   860              "mode": "absolute",
   861              "steps": [
   862                {
   863                  "color": "green",
   864                  "value": 0
   865                },
   866                {
   867                  "color": "red",
   868                  "value": 80
   869                }
   870              ]
   871            },
   872            "unit": "s"
   873          },
   874          "overrides": [
   875            {
   876              "matcher": {
   877                "id": "byFrameRefID",
   878                "options": "B"
   879              },
   880              "properties": [
   881                {
   882                  "id": "custom.drawStyle",
   883                  "value": "points"
   884                },
   885                {
   886                  "id": "color",
   887                  "value": {
   888                    "fixedColor": "red",
   889                    "mode": "fixed"
   890                  }
   891                },
   892                {
   893                  "id": "custom.pointSize",
   894                  "value": 11
   895                }
   896              ]
   897            }
   898          ]
   899        },
   900        "gridPos": {
   901          "h": 7,
   902          "w": 6,
   903          "x": 18,
   904          "y": 2
   905        },
   906        "id": 55,
   907        "options": {
   908          "legend": {
   909            "calcs": [],
   910            "displayMode": "list",
   911            "placement": "bottom",
   912            "showLegend": false
   913          },
   914          "tooltip": {
   915            "hideZeros": false,
   916            "mode": "single",
   917            "sort": "none"
   918          }
   919        },
   920        "pluginVersion": "12.1.0-90017",
   921        "targets": [
   922          {
   923            "datasource": {
   924              "type": "prometheus",
   925              "uid": "${datasource}"
   926            },
   927            "editorMode": "code",
   928            "expr": "histogram_quantile(0.99, sum(rate(pyroscope_request_duration_seconds_bucket{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.MetadataQueryService/QueryMetadataLabels\",\n}[$__rate_interval])) by (le))",
   929            "hide": false,
   930            "instant": false,
   931            "legendFormat": "p99",
   932            "range": true,
   933            "refId": "D"
   934          },
   935          {
   936            "datasource": {
   937              "type": "prometheus",
   938              "uid": "${datasource}"
   939            },
   940            "editorMode": "code",
   941            "expr": "histogram_quantile(0.95, sum(rate(pyroscope_request_duration_seconds_bucket{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.MetadataQueryService/QueryMetadataLabels\",\n}[$__rate_interval])) by (le))",
   942            "hide": false,
   943            "instant": false,
   944            "legendFormat": "p95",
   945            "range": true,
   946            "refId": "A"
   947          }
   948        ],
   949        "title": "QueryMetadataLabels Latency",
   950        "transparent": true,
   951        "type": "timeseries"
   952      },
   953      {
   954        "datasource": {
   955          "type": "prometheus",
   956          "uid": "${datasource}"
   957        },
   958        "description": "The size of BoltDB snapshot taken by FSM",
   959        "fieldConfig": {
   960          "defaults": {
   961            "color": {
   962              "mode": "palette-classic"
   963            },
   964            "custom": {
   965              "axisBorderShow": false,
   966              "axisCenteredZero": false,
   967              "axisColorMode": "text",
   968              "axisLabel": "",
   969              "axisPlacement": "auto",
   970              "barAlignment": 0,
   971              "barWidthFactor": 0.6,
   972              "drawStyle": "line",
   973              "fillOpacity": 0,
   974              "gradientMode": "none",
   975              "hideFrom": {
   976                "legend": false,
   977                "tooltip": false,
   978                "viz": false
   979              },
   980              "insertNulls": false,
   981              "lineInterpolation": "linear",
   982              "lineWidth": 1,
   983              "pointSize": 5,
   984              "scaleDistribution": {
   985                "type": "linear"
   986              },
   987              "showPoints": "auto",
   988              "spanNulls": false,
   989              "stacking": {
   990                "group": "A",
   991                "mode": "none"
   992              },
   993              "thresholdsStyle": {
   994                "mode": "off"
   995              }
   996            },
   997            "mappings": [],
   998            "thresholds": {
   999              "mode": "absolute",
  1000              "steps": [
  1001                {
  1002                  "color": "green",
  1003                  "value": 0
  1004                },
  1005                {
  1006                  "color": "red",
  1007                  "value": 80
  1008                }
  1009              ]
  1010            },
  1011            "unit": "decbytes"
  1012          },
  1013          "overrides": [
  1014            {
  1015              "matcher": {
  1016                "id": "byFrameRefID",
  1017                "options": "B"
  1018              },
  1019              "properties": [
  1020                {
  1021                  "id": "custom.drawStyle",
  1022                  "value": "points"
  1023                },
  1024                {
  1025                  "id": "color",
  1026                  "value": {
  1027                    "fixedColor": "red",
  1028                    "mode": "fixed"
  1029                  }
  1030                },
  1031                {
  1032                  "id": "custom.pointSize",
  1033                  "value": 11
  1034                }
  1035              ]
  1036            }
  1037          ]
  1038        },
  1039        "gridPos": {
  1040          "h": 7,
  1041          "w": 12,
  1042          "x": 0,
  1043          "y": 9
  1044        },
  1045        "id": 60,
  1046        "options": {
  1047          "legend": {
  1048            "calcs": [],
  1049            "displayMode": "list",
  1050            "placement": "bottom",
  1051            "showLegend": false
  1052          },
  1053          "tooltip": {
  1054            "hideZeros": false,
  1055            "mode": "single",
  1056            "sort": "none"
  1057          }
  1058        },
  1059        "pluginVersion": "12.1.0-90017",
  1060        "targets": [
  1061          {
  1062            "datasource": {
  1063              "type": "prometheus",
  1064              "uid": "${datasource}"
  1065            },
  1066            "editorMode": "code",
  1067            "expr": "pyroscope_metastore_boltdb_persist_snapshot_size_bytes{namespace=\"$namespace\"}",
  1068            "hide": false,
  1069            "instant": false,
  1070            "legendFormat": "{{pod}}",
  1071            "range": true,
  1072            "refId": "D"
  1073          }
  1074        ],
  1075        "title": "Snapshot Size",
  1076        "transparent": true,
  1077        "type": "timeseries"
  1078      },
  1079      {
  1080        "datasource": {
  1081          "type": "prometheus",
  1082          "uid": "${datasource}"
  1083        },
  1084        "fieldConfig": {
  1085          "defaults": {
  1086            "color": {
  1087              "mode": "palette-classic"
  1088            },
  1089            "custom": {
  1090              "axisBorderShow": false,
  1091              "axisCenteredZero": false,
  1092              "axisColorMode": "text",
  1093              "axisLabel": "",
  1094              "axisPlacement": "auto",
  1095              "barAlignment": 0,
  1096              "barWidthFactor": 0.6,
  1097              "drawStyle": "line",
  1098              "fillOpacity": 0,
  1099              "gradientMode": "none",
  1100              "hideFrom": {
  1101                "legend": false,
  1102                "tooltip": false,
  1103                "viz": false
  1104              },
  1105              "insertNulls": false,
  1106              "lineInterpolation": "linear",
  1107              "lineWidth": 1,
  1108              "pointSize": 5,
  1109              "scaleDistribution": {
  1110                "type": "linear"
  1111              },
  1112              "showPoints": "auto",
  1113              "spanNulls": false,
  1114              "stacking": {
  1115                "group": "A",
  1116                "mode": "none"
  1117              },
  1118              "thresholdsStyle": {
  1119                "mode": "off"
  1120              }
  1121            },
  1122            "mappings": [],
  1123            "thresholds": {
  1124              "mode": "absolute",
  1125              "steps": [
  1126                {
  1127                  "color": "green",
  1128                  "value": 0
  1129                },
  1130                {
  1131                  "color": "red",
  1132                  "value": 80
  1133                }
  1134              ]
  1135            },
  1136            "unit": "ops"
  1137          },
  1138          "overrides": [
  1139            {
  1140              "matcher": {
  1141                "id": "byName",
  1142                "options": "error"
  1143              },
  1144              "properties": [
  1145                {
  1146                  "id": "color",
  1147                  "value": {
  1148                    "fixedColor": "red",
  1149                    "mode": "fixed"
  1150                  }
  1151                }
  1152              ]
  1153            },
  1154            {
  1155              "matcher": {
  1156                "id": "byName",
  1157                "options": "success"
  1158              },
  1159              "properties": [
  1160                {
  1161                  "id": "color",
  1162                  "value": {
  1163                    "fixedColor": "green",
  1164                    "mode": "fixed"
  1165                  }
  1166                }
  1167              ]
  1168            }
  1169          ]
  1170        },
  1171        "gridPos": {
  1172          "h": 7,
  1173          "w": 6,
  1174          "x": 12,
  1175          "y": 9
  1176        },
  1177        "id": 58,
  1178        "options": {
  1179          "legend": {
  1180            "calcs": [],
  1181            "displayMode": "list",
  1182            "placement": "bottom",
  1183            "showLegend": false
  1184          },
  1185          "tooltip": {
  1186            "hideZeros": false,
  1187            "mode": "single",
  1188            "sort": "none"
  1189          }
  1190        },
  1191        "pluginVersion": "12.1.0-90017",
  1192        "targets": [
  1193          {
  1194            "datasource": {
  1195              "type": "prometheus",
  1196              "uid": "${datasource}"
  1197            },
  1198            "editorMode": "code",
  1199            "expr": "sum(irate(pyroscope_request_duration_seconds_count{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.MetadataQueryService/QueryMetadata\",\n}[$__rate_interval])) by (status_code)",
  1200            "hide": false,
  1201            "instant": false,
  1202            "legendFormat": "__auto",
  1203            "range": true,
  1204            "refId": "D"
  1205          }
  1206        ],
  1207        "title": "",
  1208        "transparent": true,
  1209        "type": "timeseries"
  1210      },
  1211      {
  1212        "datasource": {
  1213          "type": "prometheus",
  1214          "uid": "${datasource}"
  1215        },
  1216        "fieldConfig": {
  1217          "defaults": {
  1218            "color": {
  1219              "mode": "palette-classic"
  1220            },
  1221            "custom": {
  1222              "axisBorderShow": false,
  1223              "axisCenteredZero": false,
  1224              "axisColorMode": "text",
  1225              "axisLabel": "",
  1226              "axisPlacement": "auto",
  1227              "barAlignment": 0,
  1228              "barWidthFactor": 0.6,
  1229              "drawStyle": "line",
  1230              "fillOpacity": 0,
  1231              "gradientMode": "none",
  1232              "hideFrom": {
  1233                "legend": false,
  1234                "tooltip": false,
  1235                "viz": false
  1236              },
  1237              "insertNulls": false,
  1238              "lineInterpolation": "linear",
  1239              "lineWidth": 1,
  1240              "pointSize": 5,
  1241              "scaleDistribution": {
  1242                "type": "linear"
  1243              },
  1244              "showPoints": "auto",
  1245              "spanNulls": false,
  1246              "stacking": {
  1247                "group": "A",
  1248                "mode": "none"
  1249              },
  1250              "thresholdsStyle": {
  1251                "mode": "off"
  1252              }
  1253            },
  1254            "mappings": [],
  1255            "thresholds": {
  1256              "mode": "absolute",
  1257              "steps": [
  1258                {
  1259                  "color": "green",
  1260                  "value": 0
  1261                },
  1262                {
  1263                  "color": "red",
  1264                  "value": 80
  1265                }
  1266              ]
  1267            },
  1268            "unit": "ops"
  1269          },
  1270          "overrides": [
  1271            {
  1272              "matcher": {
  1273                "id": "byName",
  1274                "options": "error"
  1275              },
  1276              "properties": [
  1277                {
  1278                  "id": "color",
  1279                  "value": {
  1280                    "fixedColor": "red",
  1281                    "mode": "fixed"
  1282                  }
  1283                }
  1284              ]
  1285            },
  1286            {
  1287              "matcher": {
  1288                "id": "byName",
  1289                "options": "success"
  1290              },
  1291              "properties": [
  1292                {
  1293                  "id": "color",
  1294                  "value": {
  1295                    "fixedColor": "green",
  1296                    "mode": "fixed"
  1297                  }
  1298                }
  1299              ]
  1300            }
  1301          ]
  1302        },
  1303        "gridPos": {
  1304          "h": 7,
  1305          "w": 6,
  1306          "x": 18,
  1307          "y": 9
  1308        },
  1309        "id": 59,
  1310        "options": {
  1311          "legend": {
  1312            "calcs": [],
  1313            "displayMode": "list",
  1314            "placement": "bottom",
  1315            "showLegend": false
  1316          },
  1317          "tooltip": {
  1318            "hideZeros": false,
  1319            "mode": "single",
  1320            "sort": "none"
  1321          }
  1322        },
  1323        "pluginVersion": "12.1.0-90017",
  1324        "targets": [
  1325          {
  1326            "datasource": {
  1327              "type": "prometheus",
  1328              "uid": "${datasource}"
  1329            },
  1330            "editorMode": "code",
  1331            "expr": "sum(irate(pyroscope_request_duration_seconds_count{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/metastore.v1.MetadataQueryService/QueryMetadataLabels\",\n}[$__rate_interval])) by (status_code)",
  1332            "hide": false,
  1333            "instant": false,
  1334            "legendFormat": "__auto",
  1335            "range": true,
  1336            "refId": "D"
  1337          }
  1338        ],
  1339        "title": "",
  1340        "transparent": true,
  1341        "type": "timeseries"
  1342      },
  1343      {
  1344        "collapsed": false,
  1345        "gridPos": {
  1346          "h": 1,
  1347          "w": 24,
  1348          "x": 0,
  1349          "y": 16
  1350        },
  1351        "id": 63,
  1352        "panels": [],
  1353        "title": "Query Frontend",
  1354        "type": "row"
  1355      },
  1356      {
  1357        "datasource": {
  1358          "type": "prometheus",
  1359          "uid": "$datasource"
  1360        },
  1361        "description": "Query rate observed by cortex-gw",
  1362        "fieldConfig": {
  1363          "defaults": {
  1364            "color": {
  1365              "mode": "palette-classic"
  1366            },
  1367            "custom": {
  1368              "axisBorderShow": false,
  1369              "axisCenteredZero": true,
  1370              "axisColorMode": "text",
  1371              "axisLabel": "",
  1372              "axisPlacement": "auto",
  1373              "axisSoftMin": 0,
  1374              "barAlignment": 0,
  1375              "barWidthFactor": 0.6,
  1376              "drawStyle": "line",
  1377              "fillOpacity": 0,
  1378              "gradientMode": "none",
  1379              "hideFrom": {
  1380                "legend": false,
  1381                "tooltip": false,
  1382                "viz": false
  1383              },
  1384              "insertNulls": false,
  1385              "lineInterpolation": "linear",
  1386              "lineStyle": {
  1387                "fill": "solid"
  1388              },
  1389              "lineWidth": 1,
  1390              "pointSize": 1,
  1391              "scaleDistribution": {
  1392                "type": "linear"
  1393              },
  1394              "showPoints": "never",
  1395              "spanNulls": false,
  1396              "stacking": {
  1397                "group": "A",
  1398                "mode": "none"
  1399              },
  1400              "thresholdsStyle": {
  1401                "mode": "off"
  1402              }
  1403            },
  1404            "mappings": [],
  1405            "min": 0,
  1406            "thresholds": {
  1407              "mode": "absolute",
  1408              "steps": [
  1409                {
  1410                  "color": "green",
  1411                  "value": 0
  1412                },
  1413                {
  1414                  "color": "red",
  1415                  "value": 80
  1416                }
  1417              ]
  1418            },
  1419            "unit": "reqps"
  1420          },
  1421          "overrides": []
  1422        },
  1423        "gridPos": {
  1424          "h": 9,
  1425          "w": 12,
  1426          "x": 0,
  1427          "y": 17
  1428        },
  1429        "id": 67,
  1430        "options": {
  1431          "dataLinks": [],
  1432          "legend": {
  1433            "calcs": [
  1434              "max",
  1435              "mean"
  1436            ],
  1437            "displayMode": "table",
  1438            "placement": "right",
  1439            "showLegend": true
  1440          },
  1441          "tooltip": {
  1442            "hideZeros": true,
  1443            "mode": "multi",
  1444            "sort": "none"
  1445          }
  1446        },
  1447        "pluginVersion": "12.1.0-90017",
  1448        "targets": [
  1449          {
  1450            "datasource": {
  1451              "uid": "$datasource"
  1452            },
  1453            "editorMode": "code",
  1454            "expr": "sum by (status_code) (rate(pyroscope_request_duration_seconds_count{cluster=~\"$cluster\", container=~\"pyroscope|distributor|query-frontend\", namespace=~\"$namespace\", route=~\".*pyroscope_render.*|.*pyroscope_label.*|.*querierservice.*\"}[5m]))",
  1455            "legendFormat": "{{status_code}} ",
  1456            "range": true,
  1457            "refId": "A"
  1458          }
  1459        ],
  1460        "title": "Query rate",
  1461        "transparent": true,
  1462        "type": "timeseries"
  1463      },
  1464      {
  1465        "datasource": {
  1466          "type": "prometheus",
  1467          "uid": "${datasource}"
  1468        },
  1469        "description": "Query duration observed by cortex-gw",
  1470        "fieldConfig": {
  1471          "defaults": {
  1472            "color": {
  1473              "mode": "palette-classic"
  1474            },
  1475            "custom": {
  1476              "axisBorderShow": false,
  1477              "axisCenteredZero": false,
  1478              "axisColorMode": "text",
  1479              "axisLabel": "",
  1480              "axisPlacement": "auto",
  1481              "barAlignment": 0,
  1482              "barWidthFactor": 0.6,
  1483              "drawStyle": "line",
  1484              "fillOpacity": 0,
  1485              "gradientMode": "none",
  1486              "hideFrom": {
  1487                "legend": false,
  1488                "tooltip": false,
  1489                "viz": false
  1490              },
  1491              "insertNulls": false,
  1492              "lineInterpolation": "linear",
  1493              "lineWidth": 1,
  1494              "pointSize": 5,
  1495              "scaleDistribution": {
  1496                "type": "linear"
  1497              },
  1498              "showPoints": "auto",
  1499              "spanNulls": false,
  1500              "stacking": {
  1501                "group": "A",
  1502                "mode": "none"
  1503              },
  1504              "thresholdsStyle": {
  1505                "mode": "off"
  1506              }
  1507            },
  1508            "mappings": [],
  1509            "thresholds": {
  1510              "mode": "absolute",
  1511              "steps": [
  1512                {
  1513                  "color": "green",
  1514                  "value": 0
  1515                },
  1516                {
  1517                  "color": "red",
  1518                  "value": 80
  1519                }
  1520              ]
  1521            },
  1522            "unit": "s"
  1523          },
  1524          "overrides": [
  1525            {
  1526              "matcher": {
  1527                "id": "byFrameRefID",
  1528                "options": "B"
  1529              },
  1530              "properties": [
  1531                {
  1532                  "id": "custom.drawStyle",
  1533                  "value": "points"
  1534                },
  1535                {
  1536                  "id": "color",
  1537                  "value": {
  1538                    "fixedColor": "red",
  1539                    "mode": "fixed"
  1540                  }
  1541                },
  1542                {
  1543                  "id": "custom.pointSize",
  1544                  "value": 11
  1545                }
  1546              ]
  1547            }
  1548          ]
  1549        },
  1550        "gridPos": {
  1551          "h": 9,
  1552          "w": 12,
  1553          "x": 12,
  1554          "y": 17
  1555        },
  1556        "id": 66,
  1557        "options": {
  1558          "legend": {
  1559            "calcs": [
  1560              "max"
  1561            ],
  1562            "displayMode": "table",
  1563            "placement": "right",
  1564            "showLegend": true,
  1565            "sortBy": "Name",
  1566            "sortDesc": true
  1567          },
  1568          "tooltip": {
  1569            "hideZeros": false,
  1570            "mode": "single",
  1571            "sort": "none"
  1572          }
  1573        },
  1574        "pluginVersion": "12.1.0-90017",
  1575        "targets": [
  1576          {
  1577            "datasource": {
  1578              "type": "prometheus",
  1579              "uid": "${datasource}"
  1580            },
  1581            "editorMode": "code",
  1582            "expr": "histogram_quantile(0.99, sum by (le,route) ( rate(pyroscope_request_duration_seconds_bucket{container=~\"pyroscope|distributor|query-frontend\", namespace=~\"$namespace\", cluster=~\"$cluster\",route=~\".*pyroscope_render.*|.*pyroscope_label.*|.*querierservice.*\"}[$__rate_interval])))",
  1583            "hide": false,
  1584            "instant": false,
  1585            "legendFormat": "__auto",
  1586            "range": true,
  1587            "refId": "D"
  1588          }
  1589        ],
  1590        "title": "Query Duartion",
  1591        "transparent": true,
  1592        "type": "timeseries"
  1593      },
  1594      {
  1595        "collapsed": false,
  1596        "gridPos": {
  1597          "h": 1,
  1598          "w": 24,
  1599          "x": 0,
  1600          "y": 26
  1601        },
  1602        "id": 61,
  1603        "panels": [],
  1604        "title": "Query Backend",
  1605        "type": "row"
  1606      },
  1607      {
  1608        "datasource": {
  1609          "type": "prometheus",
  1610          "uid": "${datasource}"
  1611        },
  1612        "description": "Number of query-backend invocations.\n\nErrors are expected: usually those indicate that the query is throttled with the concurrency limiter.\n\nIf errors are presents in the query-frontend/gateway, the query is failed.\n",
  1613        "fieldConfig": {
  1614          "defaults": {
  1615            "color": {
  1616              "mode": "palette-classic"
  1617            },
  1618            "custom": {
  1619              "axisBorderShow": false,
  1620              "axisCenteredZero": false,
  1621              "axisColorMode": "text",
  1622              "axisLabel": "",
  1623              "axisPlacement": "auto",
  1624              "barAlignment": 0,
  1625              "barWidthFactor": 0.6,
  1626              "drawStyle": "line",
  1627              "fillOpacity": 0,
  1628              "gradientMode": "none",
  1629              "hideFrom": {
  1630                "legend": false,
  1631                "tooltip": false,
  1632                "viz": false
  1633              },
  1634              "insertNulls": false,
  1635              "lineInterpolation": "linear",
  1636              "lineWidth": 1,
  1637              "pointSize": 5,
  1638              "scaleDistribution": {
  1639                "type": "linear"
  1640              },
  1641              "showPoints": "auto",
  1642              "spanNulls": false,
  1643              "stacking": {
  1644                "group": "A",
  1645                "mode": "none"
  1646              },
  1647              "thresholdsStyle": {
  1648                "mode": "off"
  1649              }
  1650            },
  1651            "mappings": [],
  1652            "thresholds": {
  1653              "mode": "absolute",
  1654              "steps": [
  1655                {
  1656                  "color": "green",
  1657                  "value": 0
  1658                },
  1659                {
  1660                  "color": "red",
  1661                  "value": 80
  1662                }
  1663              ]
  1664            },
  1665            "unit": "ops"
  1666          },
  1667          "overrides": [
  1668            {
  1669              "matcher": {
  1670                "id": "byName",
  1671                "options": "error"
  1672              },
  1673              "properties": [
  1674                {
  1675                  "id": "color",
  1676                  "value": {
  1677                    "fixedColor": "red",
  1678                    "mode": "fixed"
  1679                  }
  1680                }
  1681              ]
  1682            },
  1683            {
  1684              "matcher": {
  1685                "id": "byName",
  1686                "options": "success"
  1687              },
  1688              "properties": [
  1689                {
  1690                  "id": "color",
  1691                  "value": {
  1692                    "fixedColor": "green",
  1693                    "mode": "fixed"
  1694                  }
  1695                }
  1696              ]
  1697            }
  1698          ]
  1699        },
  1700        "gridPos": {
  1701          "h": 8,
  1702          "w": 6,
  1703          "x": 0,
  1704          "y": 27
  1705        },
  1706        "id": 64,
  1707        "options": {
  1708          "legend": {
  1709            "calcs": [],
  1710            "displayMode": "list",
  1711            "placement": "bottom",
  1712            "showLegend": false
  1713          },
  1714          "tooltip": {
  1715            "hideZeros": false,
  1716            "mode": "single",
  1717            "sort": "none"
  1718          }
  1719        },
  1720        "pluginVersion": "12.1.0-90017",
  1721        "targets": [
  1722          {
  1723            "datasource": {
  1724              "type": "prometheus",
  1725              "uid": "${datasource}"
  1726            },
  1727            "editorMode": "code",
  1728            "expr": "sum(irate(pyroscope_request_duration_seconds_count{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/query.v1.QueryBackendService/Invoke\",\n}[$__rate_interval])) by (status_code)",
  1729            "hide": false,
  1730            "instant": false,
  1731            "legendFormat": "__auto",
  1732            "range": true,
  1733            "refId": "D"
  1734          }
  1735        ],
  1736        "title": "Invocations",
  1737        "transparent": true,
  1738        "type": "timeseries"
  1739      },
  1740      {
  1741        "datasource": {
  1742          "type": "prometheus",
  1743          "uid": "${datasource}"
  1744        },
  1745        "description": "QueryBackend.Invoke latency observed by query-backend instances",
  1746        "fieldConfig": {
  1747          "defaults": {
  1748            "color": {
  1749              "mode": "palette-classic"
  1750            },
  1751            "custom": {
  1752              "axisBorderShow": false,
  1753              "axisCenteredZero": false,
  1754              "axisColorMode": "text",
  1755              "axisLabel": "",
  1756              "axisPlacement": "auto",
  1757              "barAlignment": 0,
  1758              "barWidthFactor": 0.6,
  1759              "drawStyle": "line",
  1760              "fillOpacity": 0,
  1761              "gradientMode": "none",
  1762              "hideFrom": {
  1763                "legend": false,
  1764                "tooltip": false,
  1765                "viz": false
  1766              },
  1767              "insertNulls": false,
  1768              "lineInterpolation": "linear",
  1769              "lineWidth": 1,
  1770              "pointSize": 5,
  1771              "scaleDistribution": {
  1772                "type": "linear"
  1773              },
  1774              "showPoints": "auto",
  1775              "spanNulls": false,
  1776              "stacking": {
  1777                "group": "A",
  1778                "mode": "none"
  1779              },
  1780              "thresholdsStyle": {
  1781                "mode": "off"
  1782              }
  1783            },
  1784            "mappings": [],
  1785            "thresholds": {
  1786              "mode": "absolute",
  1787              "steps": [
  1788                {
  1789                  "color": "green",
  1790                  "value": 0
  1791                },
  1792                {
  1793                  "color": "red",
  1794                  "value": 80
  1795                }
  1796              ]
  1797            },
  1798            "unit": "s"
  1799          },
  1800          "overrides": [
  1801            {
  1802              "matcher": {
  1803                "id": "byFrameRefID",
  1804                "options": "B"
  1805              },
  1806              "properties": [
  1807                {
  1808                  "id": "custom.drawStyle",
  1809                  "value": "points"
  1810                },
  1811                {
  1812                  "id": "color",
  1813                  "value": {
  1814                    "fixedColor": "red",
  1815                    "mode": "fixed"
  1816                  }
  1817                },
  1818                {
  1819                  "id": "custom.pointSize",
  1820                  "value": 11
  1821                }
  1822              ]
  1823            }
  1824          ]
  1825        },
  1826        "gridPos": {
  1827          "h": 8,
  1828          "w": 6,
  1829          "x": 6,
  1830          "y": 27
  1831        },
  1832        "id": 62,
  1833        "options": {
  1834          "legend": {
  1835            "calcs": [],
  1836            "displayMode": "list",
  1837            "placement": "bottom",
  1838            "showLegend": false
  1839          },
  1840          "tooltip": {
  1841            "hideZeros": false,
  1842            "mode": "single",
  1843            "sort": "none"
  1844          }
  1845        },
  1846        "pluginVersion": "12.1.0-90017",
  1847        "targets": [
  1848          {
  1849            "datasource": {
  1850              "type": "prometheus",
  1851              "uid": "${datasource}"
  1852            },
  1853            "editorMode": "code",
  1854            "expr": "histogram_quantile(0.99, sum(rate(pyroscope_request_duration_seconds_bucket{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/query.v1.QueryBackendService/Invoke\",\n}[$__rate_interval])) by (le))",
  1855            "hide": false,
  1856            "instant": false,
  1857            "legendFormat": "p99",
  1858            "range": true,
  1859            "refId": "D"
  1860          },
  1861          {
  1862            "datasource": {
  1863              "type": "prometheus",
  1864              "uid": "${datasource}"
  1865            },
  1866            "editorMode": "code",
  1867            "expr": "histogram_quantile(0.95, sum(rate(pyroscope_request_duration_seconds_bucket{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/query.v1.QueryBackendService/Invoke\",\n}[$__rate_interval])) by (le))",
  1868            "hide": false,
  1869            "instant": false,
  1870            "legendFormat": "p95",
  1871            "range": true,
  1872            "refId": "A"
  1873          }
  1874        ],
  1875        "title": "Invocation Duration",
  1876        "transparent": true,
  1877        "type": "timeseries"
  1878      },
  1879      {
  1880        "datasource": {
  1881          "type": "prometheus",
  1882          "uid": "${datasource}"
  1883        },
  1884        "description": "QueryBackend.Invokeduration observed by query-backend instances",
  1885        "fieldConfig": {
  1886          "defaults": {
  1887            "custom": {
  1888              "hideFrom": {
  1889                "legend": false,
  1890                "tooltip": false,
  1891                "viz": false
  1892              },
  1893              "scaleDistribution": {
  1894                "type": "linear"
  1895              }
  1896            }
  1897          },
  1898          "overrides": [
  1899            {
  1900              "matcher": {
  1901                "id": "byFrameRefID",
  1902                "options": "B"
  1903              },
  1904              "properties": []
  1905            }
  1906          ]
  1907        },
  1908        "gridPos": {
  1909          "h": 8,
  1910          "w": 6,
  1911          "x": 12,
  1912          "y": 27
  1913        },
  1914        "id": 71,
  1915        "options": {
  1916          "calculate": false,
  1917          "cellGap": 0,
  1918          "cellValues": {},
  1919          "color": {
  1920            "exponent": 0.5,
  1921            "fill": "dark-orange",
  1922            "mode": "scheme",
  1923            "reverse": false,
  1924            "scale": "exponential",
  1925            "scheme": "Rainbow",
  1926            "steps": 64
  1927          },
  1928          "exemplars": {
  1929            "color": "rgba(255,0,255,0.7)"
  1930          },
  1931          "filterValues": {
  1932            "le": 1e-9
  1933          },
  1934          "legend": {
  1935            "show": true
  1936          },
  1937          "rowsFrame": {
  1938            "layout": "auto"
  1939          },
  1940          "tooltip": {
  1941            "mode": "single",
  1942            "showColorScale": false,
  1943            "yHistogram": false
  1944          },
  1945          "yAxis": {
  1946            "axisPlacement": "left",
  1947            "reverse": false,
  1948            "unit": "s"
  1949          }
  1950        },
  1951        "pluginVersion": "12.1.0-90017",
  1952        "targets": [
  1953          {
  1954            "datasource": {
  1955              "type": "prometheus",
  1956              "uid": "${datasource}"
  1957            },
  1958            "editorMode": "code",
  1959            "expr": "sum (rate(pyroscope_request_duration_seconds{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/query.v1.QueryBackendService/Invoke\",\n  status_code=\"success\"\n}[$__rate_interval]))",
  1960            "hide": false,
  1961            "instant": false,
  1962            "legendFormat": "p99",
  1963            "range": true,
  1964            "refId": "D"
  1965          }
  1966        ],
  1967        "title": "Heatmap",
  1968        "transparent": true,
  1969        "type": "heatmap"
  1970      },
  1971      {
  1972        "datasource": {
  1973          "type": "prometheus",
  1974          "uid": "${datasource}"
  1975        },
  1976        "description": "",
  1977        "fieldConfig": {
  1978          "defaults": {
  1979            "color": {
  1980              "mode": "palette-classic"
  1981            },
  1982            "custom": {
  1983              "axisBorderShow": false,
  1984              "axisCenteredZero": false,
  1985              "axisColorMode": "text",
  1986              "axisLabel": "",
  1987              "axisPlacement": "auto",
  1988              "barAlignment": 0,
  1989              "barWidthFactor": 0.6,
  1990              "drawStyle": "line",
  1991              "fillOpacity": 0,
  1992              "gradientMode": "none",
  1993              "hideFrom": {
  1994                "legend": false,
  1995                "tooltip": false,
  1996                "viz": false
  1997              },
  1998              "insertNulls": false,
  1999              "lineInterpolation": "linear",
  2000              "lineWidth": 1,
  2001              "pointSize": 5,
  2002              "scaleDistribution": {
  2003                "type": "linear"
  2004              },
  2005              "showPoints": "auto",
  2006              "spanNulls": false,
  2007              "stacking": {
  2008                "group": "A",
  2009                "mode": "none"
  2010              },
  2011              "thresholdsStyle": {
  2012                "mode": "off"
  2013              }
  2014            },
  2015            "mappings": [],
  2016            "thresholds": {
  2017              "mode": "absolute",
  2018              "steps": [
  2019                {
  2020                  "color": "green",
  2021                  "value": 0
  2022                },
  2023                {
  2024                  "color": "red",
  2025                  "value": 80
  2026                }
  2027              ]
  2028            },
  2029            "unit": "s"
  2030          },
  2031          "overrides": [
  2032            {
  2033              "matcher": {
  2034                "id": "byFrameRefID",
  2035                "options": "B"
  2036              },
  2037              "properties": [
  2038                {
  2039                  "id": "custom.drawStyle",
  2040                  "value": "points"
  2041                },
  2042                {
  2043                  "id": "color",
  2044                  "value": {
  2045                    "fixedColor": "red",
  2046                    "mode": "fixed"
  2047                  }
  2048                },
  2049                {
  2050                  "id": "custom.pointSize",
  2051                  "value": 11
  2052                }
  2053              ]
  2054            }
  2055          ]
  2056        },
  2057        "gridPos": {
  2058          "h": 8,
  2059          "w": 6,
  2060          "x": 18,
  2061          "y": 27
  2062        },
  2063        "id": 65,
  2064        "options": {
  2065          "legend": {
  2066            "calcs": [],
  2067            "displayMode": "list",
  2068            "placement": "bottom",
  2069            "showLegend": false
  2070          },
  2071          "tooltip": {
  2072            "hideZeros": false,
  2073            "mode": "single",
  2074            "sort": "none"
  2075          }
  2076        },
  2077        "pluginVersion": "12.1.0-90017",
  2078        "targets": [
  2079          {
  2080            "datasource": {
  2081              "type": "prometheus",
  2082              "uid": "${datasource}"
  2083            },
  2084            "editorMode": "code",
  2085            "expr": "histogram_quantile(0.99, sum(rate(pyroscope_request_duration_seconds_bucket{\n  namespace=\"$namespace\",\n  method=\"gRPC\",\n  route=\"/query.v1.QueryBackendService/Invoke\",\n}[$__rate_interval])) by (le,pod))",
  2086            "hide": false,
  2087            "instant": false,
  2088            "legendFormat": "{{pod}}",
  2089            "range": true,
  2090            "refId": "D"
  2091          }
  2092        ],
  2093        "title": "Breakdown",
  2094        "transparent": true,
  2095        "type": "timeseries"
  2096      },
  2097      {
  2098        "collapsed": false,
  2099        "gridPos": {
  2100          "h": 1,
  2101          "w": 24,
  2102          "x": 0,
  2103          "y": 35
  2104        },
  2105        "id": 32,
  2106        "panels": [],
  2107        "title": "Resource Usage",
  2108        "type": "row"
  2109      },
  2110      {
  2111        "datasource": {
  2112          "type": "datasource",
  2113          "uid": "-- Mixed --"
  2114        },
  2115        "fieldConfig": {
  2116          "defaults": {
  2117            "color": {
  2118              "mode": "palette-classic"
  2119            },
  2120            "custom": {
  2121              "axisBorderShow": false,
  2122              "axisCenteredZero": false,
  2123              "axisColorMode": "text",
  2124              "axisLabel": "",
  2125              "axisPlacement": "auto",
  2126              "axisSoftMin": 0,
  2127              "barAlignment": 1,
  2128              "barWidthFactor": 1,
  2129              "drawStyle": "line",
  2130              "fillOpacity": 0,
  2131              "gradientMode": "none",
  2132              "hideFrom": {
  2133                "legend": false,
  2134                "tooltip": false,
  2135                "viz": false
  2136              },
  2137              "insertNulls": false,
  2138              "lineInterpolation": "linear",
  2139              "lineStyle": {
  2140                "fill": "solid"
  2141              },
  2142              "lineWidth": 1,
  2143              "pointSize": 5,
  2144              "scaleDistribution": {
  2145                "type": "linear"
  2146              },
  2147              "showPoints": "auto",
  2148              "spanNulls": false,
  2149              "stacking": {
  2150                "group": "A",
  2151                "mode": "none"
  2152              },
  2153              "thresholdsStyle": {
  2154                "mode": "off"
  2155              }
  2156            },
  2157            "fieldMinMax": false,
  2158            "mappings": [],
  2159            "min": 0,
  2160            "thresholds": {
  2161              "mode": "absolute",
  2162              "steps": [
  2163                {
  2164                  "color": "green",
  2165                  "value": 0
  2166                }
  2167              ]
  2168            }
  2169          },
  2170          "overrides": [
  2171            {
  2172              "matcher": {
  2173                "id": "byFrameRefID",
  2174                "options": "A"
  2175              },
  2176              "properties": [
  2177                {
  2178                  "id": "custom.axisColorMode",
  2179                  "value": "text"
  2180                },
  2181                {
  2182                  "id": "color",
  2183                  "value": {
  2184                    "fixedColor": "orange",
  2185                    "mode": "fixed"
  2186                  }
  2187                },
  2188                {
  2189                  "id": "custom.lineWidth",
  2190                  "value": 1
  2191                },
  2192                {
  2193                  "id": "custom.lineStyle",
  2194                  "value": {
  2195                    "dash": [
  2196                      10,
  2197                      10
  2198                    ],
  2199                    "fill": "dash"
  2200                  }
  2201                },
  2202                {
  2203                  "id": "custom.stacking",
  2204                  "value": {
  2205                    "group": "A",
  2206                    "mode": "none"
  2207                  }
  2208                }
  2209              ]
  2210            }
  2211          ]
  2212        },
  2213        "gridPos": {
  2214          "h": 8,
  2215          "w": 6,
  2216          "x": 0,
  2217          "y": 36
  2218        },
  2219        "id": 29,
  2220        "options": {
  2221          "legend": {
  2222            "calcs": [
  2223              "last",
  2224              "max"
  2225            ],
  2226            "displayMode": "table",
  2227            "placement": "bottom",
  2228            "showLegend": false,
  2229            "sortBy": "Max",
  2230            "sortDesc": true
  2231          },
  2232          "tooltip": {
  2233            "hideZeros": false,
  2234            "mode": "multi",
  2235            "sort": "desc"
  2236          }
  2237        },
  2238        "pluginVersion": "12.1.0-90017",
  2239        "targets": [
  2240          {
  2241            "datasource": {
  2242              "type": "prometheus",
  2243              "uid": "${datasource}"
  2244            },
  2245            "editorMode": "code",
  2246            "expr": "sum by (pod) (irate(container_cpu_usage_seconds_total{namespace=\"$namespace\",container=\"query-backend\"}[$__rate_interval]))",
  2247            "legendFormat": "__auto",
  2248            "range": true,
  2249            "refId": "by pod"
  2250          },
  2251          {
  2252            "datasource": {
  2253              "type": "prometheus",
  2254              "uid": "${datasource}"
  2255            },
  2256            "editorMode": "code",
  2257            "expr": "max(\n    kube_pod_container_resource_requests{job=~\"(.*/)?kube-state-metrics\", namespace=\"$namespace\", resource=\"cpu\"}\n  * on(namespace,pod)\n    group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".*-query-backend\"}\n)",
  2258            "hide": false,
  2259            "legendFormat": "Request",
  2260            "range": true,
  2261            "refId": "A"
  2262          },
  2263          {
  2264            "datasource": {
  2265              "type": "prometheus",
  2266              "uid": "${datasource}"
  2267            },
  2268            "editorMode": "code",
  2269            "expr": "max(\n    kube_pod_container_resource_limits{job=~\"(.*/)?kube-state-metrics\", namespace=\"$namespace\", resource=\"cpu\"}\n  * on(namespace,pod)\n    group_left(workload) namespace_workload_pod:kube_pod_owner:relabel{namespace=\"$namespace\", workload=~\".*-query-backend\"}\n)",
  2270            "hide": false,
  2271            "legendFormat": "Limit",
  2272            "range": true,
  2273            "refId": "B"
  2274          }
  2275        ],
  2276        "title": "CPU Usage",
  2277        "transparent": true,
  2278        "type": "timeseries"
  2279      },
  2280      {
  2281        "datasource": {
  2282          "type": "datasource",
  2283          "uid": "-- Mixed --"
  2284        },
  2285        "fieldConfig": {
  2286          "defaults": {
  2287            "color": {
  2288              "mode": "palette-classic"
  2289            },
  2290            "custom": {
  2291              "axisBorderShow": false,
  2292              "axisCenteredZero": false,
  2293              "axisColorMode": "text",
  2294              "axisLabel": "",
  2295              "axisPlacement": "auto",
  2296              "axisSoftMin": 0,
  2297              "barAlignment": 1,
  2298              "barWidthFactor": 1,
  2299              "drawStyle": "line",
  2300              "fillOpacity": 0,
  2301              "gradientMode": "none",
  2302              "hideFrom": {
  2303                "legend": false,
  2304                "tooltip": false,
  2305                "viz": false
  2306              },
  2307              "insertNulls": false,
  2308              "lineInterpolation": "linear",
  2309              "lineStyle": {
  2310                "fill": "solid"
  2311              },
  2312              "lineWidth": 1,
  2313              "pointSize": 5,
  2314              "scaleDistribution": {
  2315                "type": "linear"
  2316              },
  2317              "showPoints": "auto",
  2318              "spanNulls": false,
  2319              "stacking": {
  2320                "group": "A",
  2321                "mode": "normal"
  2322              },
  2323              "thresholdsStyle": {
  2324                "mode": "off"
  2325              }
  2326            },
  2327            "fieldMinMax": false,
  2328            "mappings": [],
  2329            "min": 0,
  2330            "thresholds": {
  2331              "mode": "absolute",
  2332              "steps": [
  2333                {
  2334                  "color": "green",
  2335                  "value": 0
  2336                }
  2337              ]
  2338            }
  2339          },
  2340          "overrides": [
  2341            {
  2342              "matcher": {
  2343                "id": "byFrameRefID",
  2344                "options": "A"
  2345              },
  2346              "properties": [
  2347                {
  2348                  "id": "custom.axisColorMode",
  2349                  "value": "text"
  2350                },
  2351                {
  2352                  "id": "color",
  2353                  "value": {
  2354                    "fixedColor": "red",
  2355                    "mode": "fixed"
  2356                  }
  2357                },
  2358                {
  2359                  "id": "custom.stacking",
  2360                  "value": {
  2361                    "group": "A",
  2362                    "mode": "none"
  2363                  }
  2364                },
  2365                {
  2366                  "id": "custom.lineWidth",
  2367                  "value": 2
  2368                }
  2369              ]
  2370            },
  2371            {
  2372              "matcher": {
  2373                "id": "byName",
  2374                "options": "Request"
  2375              },
  2376              "properties": [
  2377                {
  2378                  "id": "custom.stacking",
  2379                  "value": {
  2380                    "group": "A",
  2381                    "mode": "none"
  2382                  }
  2383                },
  2384                {
  2385                  "id": "color",
  2386                  "value": {
  2387                    "fixedColor": "yellow",
  2388                    "mode": "fixed"
  2389                  }
  2390                }
  2391              ]
  2392            }
  2393          ]
  2394        },
  2395        "gridPos": {
  2396          "h": 8,
  2397          "w": 6,
  2398          "x": 6,
  2399          "y": 36
  2400        },
  2401        "id": 72,
  2402        "options": {
  2403          "legend": {
  2404            "calcs": [
  2405              "last",
  2406              "max"
  2407            ],
  2408            "displayMode": "table",
  2409            "placement": "right",
  2410            "showLegend": false,
  2411            "sortBy": "Max",
  2412            "sortDesc": true
  2413          },
  2414          "tooltip": {
  2415            "hideZeros": false,
  2416            "mode": "multi",
  2417            "sort": "desc"
  2418          }
  2419        },
  2420        "pluginVersion": "12.1.0-90017",
  2421        "targets": [
  2422          {
  2423            "datasource": {
  2424              "type": "prometheus",
  2425              "uid": "${datasource}"
  2426            },
  2427            "editorMode": "code",
  2428            "expr": "sum by (pod) (irate(container_cpu_usage_seconds_total{namespace=\"$namespace\",container=\"query-backend\"}[$__rate_interval]))",
  2429            "legendFormat": "__auto",
  2430            "range": true,
  2431            "refId": "by pod"
  2432          },
  2433          {
  2434            "datasource": {
  2435              "type": "prometheus",
  2436              "uid": "${datasource}"
  2437            },
  2438            "editorMode": "code",
  2439            "expr": "sum (irate(container_cpu_usage_seconds_total{namespace=\"$namespace\",container=\"query-backend\"}[$__rate_interval]))",
  2440            "hide": false,
  2441            "legendFormat": "Total",
  2442            "range": true,
  2443            "refId": "A"
  2444          },
  2445          {
  2446            "datasource": {
  2447              "type": "prometheus",
  2448              "uid": "${datasource}"
  2449            },
  2450            "editorMode": "code",
  2451            "expr": "sum(kube_pod_container_resource_requests{namespace=\"$namespace\", resource=\"cpu\",container=\"query-backend\"})",
  2452            "hide": false,
  2453            "legendFormat": "Request",
  2454            "range": true,
  2455            "refId": "B"
  2456          }
  2457        ],
  2458        "title": "CPU Usage (stack)",
  2459        "transparent": true,
  2460        "type": "timeseries"
  2461      },
  2462      {
  2463        "datasource": {
  2464          "type": "datasource",
  2465          "uid": "-- Mixed --"
  2466        },
  2467        "fieldConfig": {
  2468          "defaults": {
  2469            "color": {
  2470              "mode": "palette-classic"
  2471            },
  2472            "custom": {
  2473              "axisBorderShow": false,
  2474              "axisCenteredZero": false,
  2475              "axisColorMode": "text",
  2476              "axisLabel": "",
  2477              "axisPlacement": "auto",
  2478              "axisSoftMin": 0,
  2479              "barAlignment": 0,
  2480              "barWidthFactor": 0.6,
  2481              "drawStyle": "line",
  2482              "fillOpacity": 0,
  2483              "gradientMode": "none",
  2484              "hideFrom": {
  2485                "legend": false,
  2486                "tooltip": false,
  2487                "viz": false
  2488              },
  2489              "insertNulls": false,
  2490              "lineInterpolation": "linear",
  2491              "lineStyle": {
  2492                "fill": "solid"
  2493              },
  2494              "lineWidth": 1,
  2495              "pointSize": 5,
  2496              "scaleDistribution": {
  2497                "type": "linear"
  2498              },
  2499              "showPoints": "auto",
  2500              "spanNulls": false,
  2501              "stacking": {
  2502                "group": "A",
  2503                "mode": "none"
  2504              },
  2505              "thresholdsStyle": {
  2506                "mode": "dashed"
  2507              }
  2508            },
  2509            "mappings": [],
  2510            "thresholds": {
  2511              "mode": "absolute",
  2512              "steps": [
  2513                {
  2514                  "color": "green",
  2515                  "value": 0
  2516                }
  2517              ]
  2518            },
  2519            "unit": "bytes"
  2520          },
  2521          "overrides": [
  2522            {
  2523              "matcher": {
  2524                "id": "byName",
  2525                "options": "Request"
  2526              },
  2527              "properties": [
  2528                {
  2529                  "id": "color",
  2530                  "value": {
  2531                    "fixedColor": "yellow",
  2532                    "mode": "fixed"
  2533                  }
  2534                }
  2535              ]
  2536            },
  2537            {
  2538              "matcher": {
  2539                "id": "byName",
  2540                "options": "Limit"
  2541              },
  2542              "properties": [
  2543                {
  2544                  "id": "color",
  2545                  "value": {
  2546                    "fixedColor": "red",
  2547                    "mode": "fixed"
  2548                  }
  2549                }
  2550              ]
  2551            }
  2552          ]
  2553        },
  2554        "gridPos": {
  2555          "h": 8,
  2556          "w": 12,
  2557          "x": 12,
  2558          "y": 36
  2559        },
  2560        "id": 30,
  2561        "options": {
  2562          "legend": {
  2563            "calcs": [
  2564              "last",
  2565              "max"
  2566            ],
  2567            "displayMode": "table",
  2568            "placement": "bottom",
  2569            "showLegend": false,
  2570            "sortBy": "Max",
  2571            "sortDesc": true
  2572          },
  2573          "tooltip": {
  2574            "hideZeros": false,
  2575            "mode": "multi",
  2576            "sort": "desc"
  2577          }
  2578        },
  2579        "pluginVersion": "12.1.0-90017",
  2580        "targets": [
  2581          {
  2582            "datasource": {
  2583              "type": "prometheus",
  2584              "uid": "${datasource}"
  2585            },
  2586            "editorMode": "code",
  2587            "expr": "sum(container_memory_working_set_bytes{namespace=\"$namespace\", container=\"query-backend\", image!=\"\"}) by (pod)",
  2588            "hide": false,
  2589            "legendFormat": "__auto",
  2590            "range": true,
  2591            "refId": "A"
  2592          },
  2593          {
  2594            "datasource": {
  2595              "type": "prometheus",
  2596              "uid": "${datasource}"
  2597            },
  2598            "editorMode": "code",
  2599            "expr": "max(kube_pod_container_resource_requests{namespace=\"$namespace\", resource=\"memory\",container=\"query-backend\"})",
  2600            "hide": false,
  2601            "legendFormat": "Request",
  2602            "range": true,
  2603            "refId": "B"
  2604          },
  2605          {
  2606            "datasource": {
  2607              "type": "prometheus",
  2608              "uid": "${datasource}"
  2609            },
  2610            "editorMode": "code",
  2611            "expr": "max (kube_pod_container_resource_limits{namespace=\"$namespace\", resource=\"memory\",container=\"query-backend\"})",
  2612            "hide": false,
  2613            "legendFormat": "Limit",
  2614            "range": true,
  2615            "refId": "C"
  2616          }
  2617        ],
  2618        "title": "Memory Usage",
  2619        "transparent": true,
  2620        "type": "timeseries"
  2621      },
  2622      {
  2623        "datasource": {
  2624          "type": "datasource",
  2625          "uid": "-- Mixed --"
  2626        },
  2627        "fieldConfig": {
  2628          "defaults": {
  2629            "color": {
  2630              "mode": "palette-classic"
  2631            },
  2632            "custom": {
  2633              "axisBorderShow": false,
  2634              "axisCenteredZero": false,
  2635              "axisColorMode": "text",
  2636              "axisLabel": "",
  2637              "axisPlacement": "auto",
  2638              "barAlignment": 0,
  2639              "barWidthFactor": 0.6,
  2640              "drawStyle": "line",
  2641              "fillOpacity": 0,
  2642              "gradientMode": "none",
  2643              "hideFrom": {
  2644                "legend": false,
  2645                "tooltip": false,
  2646                "viz": false
  2647              },
  2648              "insertNulls": false,
  2649              "lineInterpolation": "linear",
  2650              "lineWidth": 1,
  2651              "pointSize": 5,
  2652              "scaleDistribution": {
  2653                "type": "linear"
  2654              },
  2655              "showPoints": "auto",
  2656              "spanNulls": false,
  2657              "stacking": {
  2658                "group": "A",
  2659                "mode": "none"
  2660              },
  2661              "thresholdsStyle": {
  2662                "mode": "off"
  2663              }
  2664            },
  2665            "mappings": [],
  2666            "min": 0,
  2667            "thresholds": {
  2668              "mode": "absolute",
  2669              "steps": [
  2670                {
  2671                  "color": "green",
  2672                  "value": 0
  2673                }
  2674              ]
  2675            },
  2676            "unit": "ops"
  2677          },
  2678          "overrides": []
  2679        },
  2680        "gridPos": {
  2681          "h": 8,
  2682          "w": 12,
  2683          "x": 0,
  2684          "y": 44
  2685        },
  2686        "id": 31,
  2687        "options": {
  2688          "legend": {
  2689            "calcs": [
  2690              "mean",
  2691              "max",
  2692              "last"
  2693            ],
  2694            "displayMode": "table",
  2695            "placement": "bottom",
  2696            "showLegend": false
  2697          },
  2698          "tooltip": {
  2699            "hideZeros": false,
  2700            "mode": "multi",
  2701            "sort": "none"
  2702          }
  2703        },
  2704        "pluginVersion": "12.1.0-90017",
  2705        "targets": [
  2706          {
  2707            "datasource": {
  2708              "type": "prometheus",
  2709              "uid": "${datasource}"
  2710            },
  2711            "editorMode": "code",
  2712            "expr": "sum by(operation) (irate(objstore_bucket_operations_total{namespace=\"$namespace\",container=\"query-backend\"}[$__rate_interval]))",
  2713            "legendFormat": "__auto",
  2714            "range": true,
  2715            "refId": "A"
  2716          }
  2717        ],
  2718        "title": "Storage I/O",
  2719        "transparent": true,
  2720        "type": "timeseries"
  2721      },
  2722      {
  2723        "datasource": {
  2724          "type": "datasource",
  2725          "uid": "-- Mixed --"
  2726        },
  2727        "fieldConfig": {
  2728          "defaults": {
  2729            "color": {
  2730              "mode": "palette-classic"
  2731            },
  2732            "custom": {
  2733              "axisBorderShow": false,
  2734              "axisCenteredZero": false,
  2735              "axisColorMode": "text",
  2736              "axisLabel": "",
  2737              "axisPlacement": "auto",
  2738              "barAlignment": 1,
  2739              "barWidthFactor": 1,
  2740              "drawStyle": "line",
  2741              "fillOpacity": 10,
  2742              "gradientMode": "none",
  2743              "hideFrom": {
  2744                "legend": false,
  2745                "tooltip": false,
  2746                "viz": false
  2747              },
  2748              "insertNulls": false,
  2749              "lineInterpolation": "linear",
  2750              "lineStyle": {
  2751                "fill": "solid"
  2752              },
  2753              "lineWidth": 1,
  2754              "pointSize": 5,
  2755              "scaleDistribution": {
  2756                "type": "linear"
  2757              },
  2758              "showPoints": "auto",
  2759              "spanNulls": false,
  2760              "stacking": {
  2761                "group": "A",
  2762                "mode": "normal"
  2763              },
  2764              "thresholdsStyle": {
  2765                "mode": "off"
  2766              }
  2767            },
  2768            "fieldMinMax": false,
  2769            "mappings": [],
  2770            "thresholds": {
  2771              "mode": "absolute",
  2772              "steps": [
  2773                {
  2774                  "color": "green",
  2775                  "value": 0
  2776                }
  2777              ]
  2778            },
  2779            "unit": "Bps"
  2780          },
  2781          "overrides": []
  2782        },
  2783        "gridPos": {
  2784          "h": 8,
  2785          "w": 12,
  2786          "x": 12,
  2787          "y": 44
  2788        },
  2789        "id": 53,
  2790        "options": {
  2791          "legend": {
  2792            "calcs": [
  2793              "last",
  2794              "max"
  2795            ],
  2796            "displayMode": "table",
  2797            "placement": "bottom",
  2798            "showLegend": false,
  2799            "sortBy": "Max",
  2800            "sortDesc": true
  2801          },
  2802          "tooltip": {
  2803            "hideZeros": false,
  2804            "mode": "multi",
  2805            "sort": "desc"
  2806          }
  2807        },
  2808        "pluginVersion": "12.1.0-90017",
  2809        "targets": [
  2810          {
  2811            "datasource": {
  2812              "type": "prometheus",
  2813              "uid": "${datasource}"
  2814            },
  2815            "editorMode": "code",
  2816            "expr": "(sum(irate(container_network_transmit_bytes_total{job=~\"(.*/)?cadvisor\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]) * on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*query-backend.*\"}) by (namespace,pod)) * -1",
  2817            "hide": false,
  2818            "legendFormat": "tx {{pod}}",
  2819            "range": true,
  2820            "refId": "A"
  2821          },
  2822          {
  2823            "datasource": {
  2824              "type": "prometheus",
  2825              "uid": "${datasource}"
  2826            },
  2827            "editorMode": "code",
  2828            "expr": "(sum(irate(container_network_receive_bytes_total{job=~\"(.*/)?cadvisor\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]) * on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=~\"$cluster\", namespace=~\"$namespace\", workload=~\".*query-backend.*\"}) by (namespace,pod))",
  2829            "hide": false,
  2830            "legendFormat": "rx {{pod}}",
  2831            "range": true,
  2832            "refId": "B"
  2833          }
  2834        ],
  2835        "title": "Network",
  2836        "transparent": true,
  2837        "type": "timeseries"
  2838      },
  2839      {
  2840        "collapsed": true,
  2841        "gridPos": {
  2842          "h": 1,
  2843          "w": 24,
  2844          "x": 0,
  2845          "y": 52
  2846        },
  2847        "id": 43,
  2848        "panels": [
  2849          {
  2850            "datasource": {
  2851              "type": "datasource",
  2852              "uid": "-- Mixed --"
  2853            },
  2854            "description": "Number of OS threads created",
  2855            "fieldConfig": {
  2856              "defaults": {
  2857                "color": {
  2858                  "mode": "palette-classic"
  2859                },
  2860                "custom": {
  2861                  "axisBorderShow": false,
  2862                  "axisCenteredZero": false,
  2863                  "axisColorMode": "text",
  2864                  "axisLabel": "",
  2865                  "axisPlacement": "auto",
  2866                  "axisSoftMin": 0,
  2867                  "barAlignment": 0,
  2868                  "barWidthFactor": 0.6,
  2869                  "drawStyle": "line",
  2870                  "fillOpacity": 0,
  2871                  "gradientMode": "none",
  2872                  "hideFrom": {
  2873                    "legend": false,
  2874                    "tooltip": false,
  2875                    "viz": false
  2876                  },
  2877                  "insertNulls": false,
  2878                  "lineInterpolation": "linear",
  2879                  "lineStyle": {
  2880                    "fill": "solid"
  2881                  },
  2882                  "lineWidth": 1,
  2883                  "pointSize": 5,
  2884                  "scaleDistribution": {
  2885                    "type": "linear"
  2886                  },
  2887                  "showPoints": "auto",
  2888                  "spanNulls": false,
  2889                  "stacking": {
  2890                    "group": "A",
  2891                    "mode": "none"
  2892                  },
  2893                  "thresholdsStyle": {
  2894                    "mode": "dashed"
  2895                  }
  2896                },
  2897                "mappings": [],
  2898                "thresholds": {
  2899                  "mode": "absolute",
  2900                  "steps": [
  2901                    {
  2902                      "color": "green"
  2903                    }
  2904                  ]
  2905                },
  2906                "unit": "short"
  2907              },
  2908              "overrides": [
  2909                {
  2910                  "matcher": {
  2911                    "id": "byName",
  2912                    "options": "Limit"
  2913                  },
  2914                  "properties": [
  2915                    {
  2916                      "id": "custom.lineStyle",
  2917                      "value": {
  2918                        "dash": [
  2919                          10,
  2920                          10
  2921                        ],
  2922                        "fill": "dash"
  2923                      }
  2924                    }
  2925                  ]
  2926                }
  2927              ]
  2928            },
  2929            "gridPos": {
  2930              "h": 8,
  2931              "w": 6,
  2932              "x": 0,
  2933              "y": 155
  2934            },
  2935            "id": 48,
  2936            "options": {
  2937              "legend": {
  2938                "calcs": [
  2939                  "last",
  2940                  "max"
  2941                ],
  2942                "displayMode": "table",
  2943                "placement": "bottom",
  2944                "showLegend": false,
  2945                "sortBy": "Max",
  2946                "sortDesc": true
  2947              },
  2948              "tooltip": {
  2949                "hideZeros": false,
  2950                "mode": "multi",
  2951                "sort": "desc"
  2952              }
  2953            },
  2954            "pluginVersion": "12.0.0-85113",
  2955            "targets": [
  2956              {
  2957                "datasource": {
  2958                  "type": "prometheus",
  2959                  "uid": "${datasource}"
  2960                },
  2961                "editorMode": "code",
  2962                "expr": "go_threads{namespace=\"$namespace\",container=\"query-backend\"}",
  2963                "hide": false,
  2964                "legendFormat": "{{pod}}",
  2965                "range": true,
  2966                "refId": "D"
  2967              }
  2968            ],
  2969            "title": "Go Threads",
  2970            "transparent": true,
  2971            "type": "timeseries"
  2972          },
  2973          {
  2974            "datasource": {
  2975              "type": "datasource",
  2976              "uid": "-- Mixed --"
  2977            },
  2978            "fieldConfig": {
  2979              "defaults": {
  2980                "color": {
  2981                  "mode": "palette-classic"
  2982                },
  2983                "custom": {
  2984                  "axisBorderShow": false,
  2985                  "axisCenteredZero": false,
  2986                  "axisColorMode": "text",
  2987                  "axisLabel": "",
  2988                  "axisPlacement": "auto",
  2989                  "axisSoftMin": 0,
  2990                  "barAlignment": 0,
  2991                  "barWidthFactor": 0.6,
  2992                  "drawStyle": "line",
  2993                  "fillOpacity": 0,
  2994                  "gradientMode": "none",
  2995                  "hideFrom": {
  2996                    "legend": false,
  2997                    "tooltip": false,
  2998                    "viz": false
  2999                  },
  3000                  "insertNulls": false,
  3001                  "lineInterpolation": "linear",
  3002                  "lineStyle": {
  3003                    "fill": "solid"
  3004                  },
  3005                  "lineWidth": 1,
  3006                  "pointSize": 5,
  3007                  "scaleDistribution": {
  3008                    "type": "linear"
  3009                  },
  3010                  "showPoints": "auto",
  3011                  "spanNulls": false,
  3012                  "stacking": {
  3013                    "group": "A",
  3014                    "mode": "none"
  3015                  },
  3016                  "thresholdsStyle": {
  3017                    "mode": "dashed"
  3018                  }
  3019                },
  3020                "mappings": [],
  3021                "thresholds": {
  3022                  "mode": "absolute",
  3023                  "steps": [
  3024                    {
  3025                      "color": "green"
  3026                    }
  3027                  ]
  3028                },
  3029                "unit": "short"
  3030              },
  3031              "overrides": [
  3032                {
  3033                  "matcher": {
  3034                    "id": "byName",
  3035                    "options": "Limit"
  3036                  },
  3037                  "properties": [
  3038                    {
  3039                      "id": "custom.lineStyle",
  3040                      "value": {
  3041                        "dash": [
  3042                          10,
  3043                          10
  3044                        ],
  3045                        "fill": "dash"
  3046                      }
  3047                    }
  3048                  ]
  3049                }
  3050              ]
  3051            },
  3052            "gridPos": {
  3053              "h": 8,
  3054              "w": 6,
  3055              "x": 6,
  3056              "y": 155
  3057            },
  3058            "id": 46,
  3059            "options": {
  3060              "legend": {
  3061                "calcs": [
  3062                  "last",
  3063                  "max"
  3064                ],
  3065                "displayMode": "table",
  3066                "placement": "bottom",
  3067                "showLegend": false,
  3068                "sortBy": "Max",
  3069                "sortDesc": true
  3070              },
  3071              "tooltip": {
  3072                "hideZeros": false,
  3073                "mode": "multi",
  3074                "sort": "desc"
  3075              }
  3076            },
  3077            "pluginVersion": "12.0.0-85113",
  3078            "targets": [
  3079              {
  3080                "datasource": {
  3081                  "type": "prometheus",
  3082                  "uid": "${datasource}"
  3083                },
  3084                "editorMode": "code",
  3085                "expr": "sum by (pod) (go_goroutines{namespace=\"$namespace\",container=\"query-backend\"})",
  3086                "hide": false,
  3087                "legendFormat": "__auto",
  3088                "range": true,
  3089                "refId": "D"
  3090              }
  3091            ],
  3092            "title": "Goroutines",
  3093            "transparent": true,
  3094            "type": "timeseries"
  3095          },
  3096          {
  3097            "datasource": {
  3098              "type": "datasource",
  3099              "uid": "-- Mixed --"
  3100            },
  3101            "fieldConfig": {
  3102              "defaults": {
  3103                "color": {
  3104                  "mode": "palette-classic"
  3105                },
  3106                "custom": {
  3107                  "axisBorderShow": false,
  3108                  "axisCenteredZero": false,
  3109                  "axisColorMode": "text",
  3110                  "axisLabel": "",
  3111                  "axisPlacement": "auto",
  3112                  "axisSoftMin": 0,
  3113                  "barAlignment": 0,
  3114                  "barWidthFactor": 0.6,
  3115                  "drawStyle": "line",
  3116                  "fillOpacity": 0,
  3117                  "gradientMode": "none",
  3118                  "hideFrom": {
  3119                    "legend": false,
  3120                    "tooltip": false,
  3121                    "viz": false
  3122                  },
  3123                  "insertNulls": false,
  3124                  "lineInterpolation": "linear",
  3125                  "lineStyle": {
  3126                    "fill": "solid"
  3127                  },
  3128                  "lineWidth": 1,
  3129                  "pointSize": 5,
  3130                  "scaleDistribution": {
  3131                    "type": "linear"
  3132                  },
  3133                  "showPoints": "auto",
  3134                  "spanNulls": false,
  3135                  "stacking": {
  3136                    "group": "A",
  3137                    "mode": "none"
  3138                  },
  3139                  "thresholdsStyle": {
  3140                    "mode": "dashed"
  3141                  }
  3142                },
  3143                "mappings": [],
  3144                "thresholds": {
  3145                  "mode": "absolute",
  3146                  "steps": [
  3147                    {
  3148                      "color": "green"
  3149                    }
  3150                  ]
  3151                },
  3152                "unit": "bytes"
  3153              },
  3154              "overrides": [
  3155                {
  3156                  "matcher": {
  3157                    "id": "byName",
  3158                    "options": "Limit"
  3159                  },
  3160                  "properties": [
  3161                    {
  3162                      "id": "custom.lineStyle",
  3163                      "value": {
  3164                        "dash": [
  3165                          10,
  3166                          10
  3167                        ],
  3168                        "fill": "dash"
  3169                      }
  3170                    }
  3171                  ]
  3172                }
  3173              ]
  3174            },
  3175            "gridPos": {
  3176              "h": 8,
  3177              "w": 6,
  3178              "x": 12,
  3179              "y": 155
  3180            },
  3181            "id": 44,
  3182            "options": {
  3183              "legend": {
  3184                "calcs": [
  3185                  "last",
  3186                  "max"
  3187                ],
  3188                "displayMode": "table",
  3189                "placement": "bottom",
  3190                "showLegend": false,
  3191                "sortBy": "Max",
  3192                "sortDesc": true
  3193              },
  3194              "tooltip": {
  3195                "hideZeros": false,
  3196                "mode": "multi",
  3197                "sort": "desc"
  3198              }
  3199            },
  3200            "pluginVersion": "12.0.0-85113",
  3201            "targets": [
  3202              {
  3203                "datasource": {
  3204                  "type": "prometheus",
  3205                  "uid": "${datasource}"
  3206                },
  3207                "editorMode": "code",
  3208                "expr": "sum by (pod) (go_memstats_heap_inuse_bytes{namespace=\"$namespace\",container=\"query-backend\"})",
  3209                "hide": false,
  3210                "legendFormat": "__auto",
  3211                "range": true,
  3212                "refId": "D"
  3213              }
  3214            ],
  3215            "title": "Heap In-Use",
  3216            "transparent": true,
  3217            "type": "timeseries"
  3218          },
  3219          {
  3220            "datasource": {
  3221              "type": "datasource",
  3222              "uid": "-- Mixed --"
  3223            },
  3224            "fieldConfig": {
  3225              "defaults": {
  3226                "color": {
  3227                  "mode": "palette-classic"
  3228                },
  3229                "custom": {
  3230                  "axisBorderShow": false,
  3231                  "axisCenteredZero": false,
  3232                  "axisColorMode": "text",
  3233                  "axisLabel": "",
  3234                  "axisPlacement": "auto",
  3235                  "axisSoftMin": 0,
  3236                  "barAlignment": 1,
  3237                  "barWidthFactor": 1,
  3238                  "drawStyle": "line",
  3239                  "fillOpacity": 0,
  3240                  "gradientMode": "none",
  3241                  "hideFrom": {
  3242                    "legend": false,
  3243                    "tooltip": false,
  3244                    "viz": false
  3245                  },
  3246                  "insertNulls": false,
  3247                  "lineInterpolation": "linear",
  3248                  "lineStyle": {
  3249                    "fill": "solid"
  3250                  },
  3251                  "lineWidth": 1,
  3252                  "pointSize": 5,
  3253                  "scaleDistribution": {
  3254                    "type": "linear"
  3255                  },
  3256                  "showPoints": "auto",
  3257                  "spanNulls": false,
  3258                  "stacking": {
  3259                    "group": "A",
  3260                    "mode": "none"
  3261                  },
  3262                  "thresholdsStyle": {
  3263                    "mode": "off"
  3264                  }
  3265                },
  3266                "fieldMinMax": false,
  3267                "mappings": [],
  3268                "min": 0,
  3269                "thresholds": {
  3270                  "mode": "absolute",
  3271                  "steps": [
  3272                    {
  3273                      "color": "green"
  3274                    }
  3275                  ]
  3276                }
  3277              },
  3278              "overrides": [
  3279                {
  3280                  "matcher": {
  3281                    "id": "byName",
  3282                    "options": "Limit"
  3283                  },
  3284                  "properties": [
  3285                    {
  3286                      "id": "custom.lineStyle",
  3287                      "value": {
  3288                        "dash": [
  3289                          10,
  3290                          10
  3291                        ],
  3292                        "fill": "dash"
  3293                      }
  3294                    }
  3295                  ]
  3296                }
  3297              ]
  3298            },
  3299            "gridPos": {
  3300              "h": 8,
  3301              "w": 6,
  3302              "x": 18,
  3303              "y": 155
  3304            },
  3305            "id": 45,
  3306            "options": {
  3307              "legend": {
  3308                "calcs": [
  3309                  "last",
  3310                  "max"
  3311                ],
  3312                "displayMode": "table",
  3313                "placement": "bottom",
  3314                "showLegend": false,
  3315                "sortBy": "Max",
  3316                "sortDesc": true
  3317              },
  3318              "tooltip": {
  3319                "hideZeros": false,
  3320                "mode": "multi",
  3321                "sort": "desc"
  3322              }
  3323            },
  3324            "pluginVersion": "12.0.0-85113",
  3325            "targets": [
  3326              {
  3327                "datasource": {
  3328                  "type": "prometheus",
  3329                  "uid": "${datasource}"
  3330                },
  3331                "editorMode": "code",
  3332                "expr": "sum by (pod) (container_file_descriptors{namespace=\"$namespace\",container=\"query-backend\"})",
  3333                "hide": false,
  3334                "legendFormat": "__auto",
  3335                "range": true,
  3336                "refId": "A"
  3337              }
  3338            ],
  3339            "title": "File Descriptors",
  3340            "transparent": true,
  3341            "type": "timeseries"
  3342          }
  3343        ],
  3344        "title": "Runtime",
  3345        "type": "row"
  3346      },
  3347      {
  3348        "collapsed": true,
  3349        "gridPos": {
  3350          "h": 1,
  3351          "w": 24,
  3352          "x": 0,
  3353          "y": 53
  3354        },
  3355        "id": 42,
  3356        "panels": [
  3357          {
  3358            "datasource": {
  3359              "type": "loki",
  3360              "uid": "OP27Xzxnk"
  3361            },
  3362            "fieldConfig": {
  3363              "defaults": {},
  3364              "overrides": []
  3365            },
  3366            "gridPos": {
  3367              "h": 8,
  3368              "w": 24,
  3369              "x": 0,
  3370              "y": 156
  3371            },
  3372            "id": 41,
  3373            "options": {
  3374              "dedupStrategy": "none",
  3375              "enableInfiniteScrolling": false,
  3376              "enableLogDetails": true,
  3377              "prettifyLogMessage": false,
  3378              "showCommonLabels": false,
  3379              "showLabels": false,
  3380              "showTime": false,
  3381              "sortOrder": "Descending",
  3382              "wrapLogMessage": false
  3383            },
  3384            "pluginVersion": "12.0.0-85113",
  3385            "targets": [
  3386              {
  3387                "datasource": {
  3388                  "type": "loki",
  3389                  "uid": "OP27Xzxnk"
  3390                },
  3391                "direction": "backward",
  3392                "editorMode": "code",
  3393                "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"query-backend\"} |= \"level=err\" or \"level=warn\" or \"panic\"",
  3394                "queryType": "range",
  3395                "refId": "A"
  3396              }
  3397            ],
  3398            "title": "Worker log",
  3399            "transparent": true,
  3400            "type": "logs"
  3401          }
  3402        ],
  3403        "title": "Errors and warnings",
  3404        "type": "row"
  3405      }
  3406    ],
  3407    "preload": false,
  3408    "schemaVersion": 41,
  3409    "tags": [
  3410      "pyroscope",
  3411      "pyroscope-v2"
  3412    ],
  3413    "templating": {
  3414      "list": [
  3415        {
  3416          "allowCustomValue": true,
  3417          "current": {
  3418            "text": "ops-cortex",
  3419            "value": "000000134"
  3420          },
  3421          "label": "metrics",
  3422          "name": "datasource",
  3423          "options": [],
  3424          "query": "prometheus",
  3425          "refresh": 1,
  3426          "regex": "",
  3427          "type": "datasource"
  3428        },
  3429        {
  3430          "allowCustomValue": false,
  3431          "current": {
  3432            "text": "Grafana Logging",
  3433            "value": "000000193"
  3434          },
  3435          "description": "",
  3436          "label": "logs",
  3437          "name": "loki_datasource",
  3438          "options": [],
  3439          "query": "loki",
  3440          "refresh": 1,
  3441          "regex": "",
  3442          "type": "datasource"
  3443        },
  3444        {
  3445          "allowCustomValue": false,
  3446          "current": {
  3447            "text": "Tempo Ops (tempo-ops-01)",
  3448            "value": "fds8vtxx3ao74b"
  3449          },
  3450          "description": "",
  3451          "label": "traces",
  3452          "name": "tempo_datasource",
  3453          "options": [],
  3454          "query": "tempo",
  3455          "refresh": 1,
  3456          "regex": "",
  3457          "type": "datasource"
  3458        },
  3459        {
  3460          "allowCustomValue": false,
  3461          "current": {
  3462            "text": "pyroscope-dev",
  3463            "value": "pyroscope-dev"
  3464          },
  3465          "datasource": {
  3466            "type": "prometheus",
  3467            "uid": "${datasource}"
  3468          },
  3469          "definition": "label_values(pyroscope_build_info{namespace=\"$namespace\"},cluster)",
  3470          "label": "cluster",
  3471          "name": "cluster",
  3472          "options": [],
  3473          "query": {
  3474            "qryType": 1,
  3475            "query": "label_values(pyroscope_build_info{namespace=\"$namespace\"},cluster)",
  3476            "refId": "PrometheusVariableQueryEditor-VariableQuery"
  3477          },
  3478          "refresh": 1,
  3479          "regex": "",
  3480          "sort": 1,
  3481          "type": "query"
  3482        },
  3483        {
  3484          "allowCustomValue": false,
  3485          "current": {
  3486            "text": "default",
  3487            "value": "default"
  3488          },
  3489          "datasource": {
  3490            "type": "prometheus",
  3491            "uid": "${datasource}"
  3492          },
  3493          "definition": "label_values(pyroscope_build_info,namespace)",
  3494          "label": "namespace",
  3495          "name": "namespace",
  3496          "options": [],
  3497          "query": {
  3498            "qryType": 1,
  3499            "query": "label_values(pyroscope_build_info,namespace)",
  3500            "refId": "PrometheusVariableQueryEditor-VariableQuery"
  3501          },
  3502          "refresh": 1,
  3503          "regex": ".*",
  3504          "type": "query"
  3505        }
  3506      ]
  3507    },
  3508    "time": {
  3509      "from": "now-6h",
  3510      "to": "now"
  3511    },
  3512    "timepicker": {},
  3513    "timezone": "utc",
  3514    "title": "Pyroscope v2 / Read Path"
  3515  }