github.com/smartcontractkit/chainlink-testing-framework/libs@v0.0.0-20240227141906-ec710b4eb1a3/k8s/grafana/cl_insights.json (about)

     1  {
     2    "annotations": {
     3      "list": [
     4        {
     5          "builtIn": 1,
     6          "datasource": {
     7            "type": "datasource",
     8            "uid": "grafana"
     9          },
    10          "enable": true,
    11          "hide": true,
    12          "iconColor": "rgba(0, 211, 255, 1)",
    13          "name": "Annotations & Alerts",
    14          "target": {
    15            "limit": 100,
    16            "matchAny": false,
    17            "tags": [],
    18            "type": "dashboard"
    19          },
    20          "type": "dashboard"
    21        }
    22      ]
    23    },
    24    "description": "A look at the components of Chainlink Integration tests",
    25    "editable": true,
    26    "fiscalYearStartMonth": 0,
    27    "graphTooltip": 0,
    28    "iteration": 1662412920215,
    29    "links": [],
    30    "liveNow": false,
    31    "panels": [
    32      {
    33        "collapsed": true,
    34        "gridPos": {
    35          "h": 1,
    36          "w": 24,
    37          "x": 0,
    38          "y": 0
    39        },
    40        "id": 161,
    41        "panels": [
    42          {
    43            "datasource": {
    44              "type": "prometheus",
    45              "uid": "PBFA97CFB590B2093"
    46            },
    47            "description": "Components health",
    48            "fieldConfig": {
    49              "defaults": {
    50                "color": {
    51                  "mode": "thresholds"
    52                },
    53                "mappings": [],
    54                "thresholds": {
    55                  "mode": "absolute",
    56                  "steps": [
    57                    {
    58                      "color": "green",
    59                      "value": null
    60                    },
    61                    {
    62                      "color": "red",
    63                      "value": 80
    64                    }
    65                  ]
    66                }
    67              },
    68              "overrides": []
    69            },
    70            "gridPos": {
    71              "h": 20,
    72              "w": 24,
    73              "x": 0,
    74              "y": 1
    75            },
    76            "id": 163,
    77            "options": {
    78              "colorMode": "value",
    79              "graphMode": "area",
    80              "justifyMode": "auto",
    81              "orientation": "auto",
    82              "reduceOptions": {
    83                "calcs": ["last"],
    84                "fields": "",
    85                "values": false
    86              },
    87              "text": {
    88                "titleSize": 14,
    89                "valueSize": 14
    90              },
    91              "textMode": "name"
    92            },
    93            "pluginVersion": "8.3.5",
    94            "targets": [
    95              {
    96                "datasource": {
    97                  "type": "prometheus",
    98                  "uid": "PBFA97CFB590B2093"
    99                },
   100                "exemplar": true,
   101                "expr": "health{namespace=\"$namespace\", pod=\"$cl_node\"}",
   102                "interval": "",
   103                "legendFormat": "service: {{service_id}}",
   104                "refId": "A"
   105              }
   106            ],
   107            "title": "Components health",
   108            "transparent": true,
   109            "type": "stat"
   110          }
   111        ],
   112        "title": "Components health",
   113        "type": "row"
   114      },
   115      {
   116        "collapsed": true,
   117        "gridPos": {
   118          "h": 1,
   119          "w": 24,
   120          "x": 0,
   121          "y": 1
   122        },
   123        "id": 181,
   124        "panels": [
   125          {
   126            "datasource": {
   127              "type": "prometheus",
   128              "uid": "PBFA97CFB590B2093"
   129            },
   130            "description": "Overall log count by type",
   131            "fieldConfig": {
   132              "defaults": {
   133                "color": {
   134                  "mode": "palette-classic"
   135                },
   136                "custom": {
   137                  "axisLabel": "",
   138                  "axisPlacement": "auto",
   139                  "barAlignment": 0,
   140                  "drawStyle": "line",
   141                  "fillOpacity": 0,
   142                  "gradientMode": "none",
   143                  "hideFrom": {
   144                    "legend": false,
   145                    "tooltip": false,
   146                    "viz": false
   147                  },
   148                  "lineInterpolation": "linear",
   149                  "lineWidth": 1,
   150                  "pointSize": 5,
   151                  "scaleDistribution": {
   152                    "type": "linear"
   153                  },
   154                  "showPoints": "auto",
   155                  "spanNulls": false,
   156                  "stacking": {
   157                    "group": "A",
   158                    "mode": "none"
   159                  },
   160                  "thresholdsStyle": {
   161                    "mode": "off"
   162                  }
   163                },
   164                "mappings": [],
   165                "thresholds": {
   166                  "mode": "absolute",
   167                  "steps": [
   168                    {
   169                      "color": "green",
   170                      "value": null
   171                    },
   172                    {
   173                      "color": "red",
   174                      "value": 80
   175                    }
   176                  ]
   177                }
   178              },
   179              "overrides": []
   180            },
   181            "gridPos": {
   182              "h": 8,
   183              "w": 12,
   184              "x": 0,
   185              "y": 2
   186            },
   187            "id": 179,
   188            "options": {
   189              "legend": {
   190                "calcs": [],
   191                "displayMode": "list",
   192                "placement": "right"
   193              },
   194              "tooltip": {
   195                "mode": "single"
   196              }
   197            },
   198            "targets": [
   199              {
   200                "datasource": {
   201                  "type": "prometheus",
   202                  "uid": "PBFA97CFB590B2093"
   203                },
   204                "exemplar": true,
   205                "expr": "log_error_count{namespace=\"$namespace\"}",
   206                "interval": "",
   207                "legendFormat": "WARN, pod: {{pod}}",
   208                "refId": "A"
   209              },
   210              {
   211                "datasource": {
   212                  "type": "prometheus",
   213                  "uid": "PBFA97CFB590B2093"
   214                },
   215                "exemplar": true,
   216                "expr": "log_warn_count{namespace=\"$namespace\"}",
   217                "hide": false,
   218                "interval": "",
   219                "legendFormat": "ERROR, pod: {{pod}}",
   220                "refId": "B"
   221              },
   222              {
   223                "datasource": {
   224                  "type": "prometheus",
   225                  "uid": "PBFA97CFB590B2093"
   226                },
   227                "exemplar": true,
   228                "expr": "log_panic_count{namespace=\"$namespace\"}",
   229                "hide": false,
   230                "interval": "",
   231                "legendFormat": "PANIC, pod: {{pod}}",
   232                "refId": "C"
   233              },
   234              {
   235                "datasource": {
   236                  "type": "prometheus",
   237                  "uid": "PBFA97CFB590B2093"
   238                },
   239                "exemplar": true,
   240                "expr": "log_critical_count{namespace=\"$namespace\"}",
   241                "hide": false,
   242                "interval": "",
   243                "legendFormat": "CRITICAL, pod: {{pod}}",
   244                "refId": "D"
   245              },
   246              {
   247                "datasource": {
   248                  "type": "prometheus",
   249                  "uid": "PBFA97CFB590B2093"
   250                },
   251                "exemplar": true,
   252                "expr": "log_fatal_count{namespace=\"$namespace\"}",
   253                "hide": false,
   254                "interval": "",
   255                "legendFormat": "FATAL, pod: {{pod}}",
   256                "refId": "E"
   257              }
   258            ],
   259            "title": "Overall log count by type",
   260            "transparent": true,
   261            "type": "timeseries"
   262          }
   263        ],
   264        "title": "Overall metrics across all CL nodes",
   265        "type": "row"
   266      },
   267      {
   268        "collapsed": true,
   269        "gridPos": {
   270          "h": 1,
   271          "w": 24,
   272          "x": 0,
   273          "y": 2
   274        },
   275        "id": 169,
   276        "panels": [
   277          {
   278            "datasource": {
   279              "type": "prometheus",
   280              "uid": "PBFA97CFB590B2093"
   281            },
   282            "fieldConfig": {
   283              "defaults": {
   284                "color": {
   285                  "mode": "thresholds"
   286                },
   287                "mappings": [],
   288                "thresholds": {
   289                  "mode": "absolute",
   290                  "steps": [
   291                    {
   292                      "color": "green",
   293                      "value": null
   294                    },
   295                    {
   296                      "color": "red",
   297                      "value": 80
   298                    }
   299                  ]
   300                }
   301              },
   302              "overrides": []
   303            },
   304            "gridPos": {
   305              "h": 4,
   306              "w": 24,
   307              "x": 0,
   308              "y": 3
   309            },
   310            "id": 167,
   311            "options": {
   312              "colorMode": "value",
   313              "graphMode": "area",
   314              "justifyMode": "auto",
   315              "orientation": "auto",
   316              "reduceOptions": {
   317                "calcs": ["lastNotNull"],
   318                "fields": "",
   319                "values": false
   320              },
   321              "textMode": "auto"
   322            },
   323            "pluginVersion": "8.3.5",
   324            "targets": [
   325              {
   326                "datasource": {
   327                  "type": "prometheus",
   328                  "uid": "PBFA97CFB590B2093"
   329                },
   330                "exemplar": true,
   331                "expr": "pipeline_tasks_total_finished{namespace=\"$namespace\", pod=\"$cl_node\"}",
   332                "interval": "",
   333                "legendFormat": "type: {{task_type}}, jobID: {{job_id}}, status: {{status}}",
   334                "refId": "A"
   335              }
   336            ],
   337            "title": "Pipeline tasks",
   338            "transparent": true,
   339            "type": "stat"
   340          },
   341          {
   342            "datasource": {
   343              "type": "prometheus",
   344              "uid": "PBFA97CFB590B2093"
   345            },
   346            "fieldConfig": {
   347              "defaults": {
   348                "color": {
   349                  "mode": "palette-classic"
   350                },
   351                "custom": {
   352                  "axisLabel": "",
   353                  "axisPlacement": "auto",
   354                  "barAlignment": 0,
   355                  "drawStyle": "line",
   356                  "fillOpacity": 0,
   357                  "gradientMode": "none",
   358                  "hideFrom": {
   359                    "legend": false,
   360                    "tooltip": false,
   361                    "viz": false
   362                  },
   363                  "lineInterpolation": "linear",
   364                  "lineWidth": 1,
   365                  "pointSize": 5,
   366                  "scaleDistribution": {
   367                    "type": "linear"
   368                  },
   369                  "showPoints": "auto",
   370                  "spanNulls": false,
   371                  "stacking": {
   372                    "group": "A",
   373                    "mode": "none"
   374                  },
   375                  "thresholdsStyle": {
   376                    "mode": "off"
   377                  }
   378                },
   379                "mappings": [],
   380                "thresholds": {
   381                  "mode": "absolute",
   382                  "steps": [
   383                    {
   384                      "color": "green",
   385                      "value": null
   386                    },
   387                    {
   388                      "color": "red",
   389                      "value": 80
   390                    }
   391                  ]
   392                }
   393              },
   394              "overrides": []
   395            },
   396            "gridPos": {
   397              "h": 8,
   398              "w": 12,
   399              "x": 0,
   400              "y": 7
   401            },
   402            "id": 171,
   403            "options": {
   404              "legend": {
   405                "calcs": [],
   406                "displayMode": "list",
   407                "placement": "bottom"
   408              },
   409              "tooltip": {
   410                "mode": "single"
   411              }
   412            },
   413            "targets": [
   414              {
   415                "datasource": {
   416                  "type": "prometheus",
   417                  "uid": "PBFA97CFB590B2093"
   418                },
   419                "exemplar": true,
   420                "expr": "pipeline_task_runs_queued{namespace=\"$namespace\", pod=\"$cl_node\"}",
   421                "interval": "",
   422                "legendFormat": "Tasks",
   423                "refId": "A"
   424              },
   425              {
   426                "datasource": {
   427                  "type": "prometheus",
   428                  "uid": "PBFA97CFB590B2093"
   429                },
   430                "exemplar": true,
   431                "expr": "pipeline_runs_queued{namespace=\"$namespace\", pod=\"$cl_node\"}",
   432                "hide": false,
   433                "interval": "",
   434                "legendFormat": "Pipelines",
   435                "refId": "B"
   436              }
   437            ],
   438            "title": "Queued tasks and pipelines",
   439            "transparent": true,
   440            "type": "timeseries"
   441          },
   442          {
   443            "datasource": {
   444              "type": "prometheus",
   445              "uid": "PBFA97CFB590B2093"
   446            },
   447            "description": "Pipeline avg time to completion",
   448            "fieldConfig": {
   449              "defaults": {
   450                "color": {
   451                  "mode": "palette-classic"
   452                },
   453                "custom": {
   454                  "axisLabel": "Duration, ms",
   455                  "axisPlacement": "left",
   456                  "barAlignment": 0,
   457                  "drawStyle": "line",
   458                  "fillOpacity": 0,
   459                  "gradientMode": "none",
   460                  "hideFrom": {
   461                    "legend": false,
   462                    "tooltip": false,
   463                    "viz": false
   464                  },
   465                  "lineInterpolation": "linear",
   466                  "lineWidth": 1,
   467                  "pointSize": 5,
   468                  "scaleDistribution": {
   469                    "type": "linear"
   470                  },
   471                  "showPoints": "auto",
   472                  "spanNulls": false,
   473                  "stacking": {
   474                    "group": "A",
   475                    "mode": "none"
   476                  },
   477                  "thresholdsStyle": {
   478                    "mode": "off"
   479                  }
   480                },
   481                "mappings": [],
   482                "thresholds": {
   483                  "mode": "absolute",
   484                  "steps": [
   485                    {
   486                      "color": "green",
   487                      "value": null
   488                    },
   489                    {
   490                      "color": "red",
   491                      "value": 80
   492                    }
   493                  ]
   494                }
   495              },
   496              "overrides": []
   497            },
   498            "gridPos": {
   499              "h": 8,
   500              "w": 12,
   501              "x": 12,
   502              "y": 7
   503            },
   504            "id": 173,
   505            "options": {
   506              "legend": {
   507                "calcs": [],
   508                "displayMode": "list",
   509                "placement": "bottom"
   510              },
   511              "tooltip": {
   512                "mode": "single"
   513              }
   514            },
   515            "targets": [
   516              {
   517                "datasource": {
   518                  "type": "prometheus",
   519                  "uid": "PBFA97CFB590B2093"
   520                },
   521                "exemplar": true,
   522                "expr": "pipeline_run_total_time_to_completion{namespace=\"$namespace\", pod=\"$cl_node\"} * 1e-6",
   523                "interval": "",
   524                "legendFormat": "jobID: {{job_id}}",
   525                "refId": "A"
   526              }
   527            ],
   528            "title": "Pipeline avg time to completion",
   529            "transparent": true,
   530            "type": "timeseries"
   531          },
   532          {
   533            "datasource": {
   534              "type": "prometheus",
   535              "uid": "PBFA97CFB590B2093"
   536            },
   537            "description": "Pipeline fetch avg bytes",
   538            "fieldConfig": {
   539              "defaults": {
   540                "color": {
   541                  "mode": "palette-classic"
   542                },
   543                "custom": {
   544                  "axisLabel": "Size, Bytes",
   545                  "axisPlacement": "left",
   546                  "barAlignment": 0,
   547                  "drawStyle": "line",
   548                  "fillOpacity": 0,
   549                  "gradientMode": "none",
   550                  "hideFrom": {
   551                    "legend": false,
   552                    "tooltip": false,
   553                    "viz": false
   554                  },
   555                  "lineInterpolation": "linear",
   556                  "lineWidth": 1,
   557                  "pointSize": 5,
   558                  "scaleDistribution": {
   559                    "type": "linear"
   560                  },
   561                  "showPoints": "auto",
   562                  "spanNulls": false,
   563                  "stacking": {
   564                    "group": "A",
   565                    "mode": "none"
   566                  },
   567                  "thresholdsStyle": {
   568                    "mode": "off"
   569                  }
   570                },
   571                "mappings": [],
   572                "thresholds": {
   573                  "mode": "absolute",
   574                  "steps": [
   575                    {
   576                      "color": "green",
   577                      "value": null
   578                    },
   579                    {
   580                      "color": "red",
   581                      "value": 80
   582                    }
   583                  ]
   584                }
   585              },
   586              "overrides": []
   587            },
   588            "gridPos": {
   589              "h": 8,
   590              "w": 12,
   591              "x": 0,
   592              "y": 15
   593            },
   594            "id": 177,
   595            "options": {
   596              "legend": {
   597                "calcs": [],
   598                "displayMode": "list",
   599                "placement": "bottom"
   600              },
   601              "tooltip": {
   602                "mode": "single"
   603              }
   604            },
   605            "targets": [
   606              {
   607                "datasource": {
   608                  "type": "prometheus",
   609                  "uid": "PBFA97CFB590B2093"
   610                },
   611                "exemplar": true,
   612                "expr": "pipeline_task_http_response_body_size{namespace=\"$namespace\", pod=\"$cl_node\"}",
   613                "interval": "",
   614                "legendFormat": "Task type: {{pipeline_task_spec_id}}",
   615                "refId": "A"
   616              }
   617            ],
   618            "title": "Pipeline fetch avg bytes",
   619            "transparent": true,
   620            "type": "timeseries"
   621          },
   622          {
   623            "datasource": {
   624              "type": "prometheus",
   625              "uid": "PBFA97CFB590B2093"
   626            },
   627            "description": "Task execution time by type",
   628            "fieldConfig": {
   629              "defaults": {
   630                "color": {
   631                  "mode": "palette-classic"
   632                },
   633                "custom": {
   634                  "axisLabel": "Duration, ms",
   635                  "axisPlacement": "left",
   636                  "barAlignment": 0,
   637                  "drawStyle": "line",
   638                  "fillOpacity": 0,
   639                  "gradientMode": "none",
   640                  "hideFrom": {
   641                    "legend": false,
   642                    "tooltip": false,
   643                    "viz": false
   644                  },
   645                  "lineInterpolation": "linear",
   646                  "lineWidth": 1,
   647                  "pointSize": 5,
   648                  "scaleDistribution": {
   649                    "type": "linear"
   650                  },
   651                  "showPoints": "auto",
   652                  "spanNulls": false,
   653                  "stacking": {
   654                    "group": "A",
   655                    "mode": "none"
   656                  },
   657                  "thresholdsStyle": {
   658                    "mode": "off"
   659                  }
   660                },
   661                "mappings": [],
   662                "thresholds": {
   663                  "mode": "absolute",
   664                  "steps": [
   665                    {
   666                      "color": "green",
   667                      "value": null
   668                    },
   669                    {
   670                      "color": "red",
   671                      "value": 80
   672                    }
   673                  ]
   674                }
   675              },
   676              "overrides": []
   677            },
   678            "gridPos": {
   679              "h": 8,
   680              "w": 12,
   681              "x": 12,
   682              "y": 15
   683            },
   684            "id": 175,
   685            "options": {
   686              "legend": {
   687                "calcs": [],
   688                "displayMode": "list",
   689                "placement": "bottom"
   690              },
   691              "tooltip": {
   692                "mode": "single"
   693              }
   694            },
   695            "targets": [
   696              {
   697                "datasource": {
   698                  "type": "prometheus",
   699                  "uid": "PBFA97CFB590B2093"
   700                },
   701                "exemplar": true,
   702                "expr": "pipeline_task_execution_time{namespace=\"$namespace\", pod=\"$cl_node\"} * 1e-6",
   703                "interval": "",
   704                "legendFormat": "type: {{task_type}}",
   705                "refId": "A"
   706              }
   707            ],
   708            "title": "Task execution time by type",
   709            "transparent": true,
   710            "type": "timeseries"
   711          }
   712        ],
   713        "title": "Pipeline tasks ($cl_node)",
   714        "type": "row"
   715      },
   716      {
   717        "collapsed": true,
   718        "gridPos": {
   719          "h": 1,
   720          "w": 24,
   721          "x": 0,
   722          "y": 3
   723        },
   724        "id": 147,
   725        "panels": [
   726          {
   727            "datasource": {
   728              "type": "prometheus",
   729              "uid": "PBFA97CFB590B2093"
   730            },
   731            "fieldConfig": {
   732              "defaults": {
   733                "color": {
   734                  "mode": "palette-classic"
   735                },
   736                "custom": {
   737                  "axisLabel": "",
   738                  "axisPlacement": "auto",
   739                  "barAlignment": 0,
   740                  "drawStyle": "line",
   741                  "fillOpacity": 0,
   742                  "gradientMode": "none",
   743                  "hideFrom": {
   744                    "legend": false,
   745                    "tooltip": false,
   746                    "viz": false
   747                  },
   748                  "lineInterpolation": "linear",
   749                  "lineWidth": 1,
   750                  "pointSize": 5,
   751                  "scaleDistribution": {
   752                    "type": "linear"
   753                  },
   754                  "showPoints": "auto",
   755                  "spanNulls": false,
   756                  "stacking": {
   757                    "group": "A",
   758                    "mode": "none"
   759                  },
   760                  "thresholdsStyle": {
   761                    "mode": "off"
   762                  }
   763                },
   764                "mappings": [],
   765                "thresholds": {
   766                  "mode": "absolute",
   767                  "steps": [
   768                    {
   769                      "color": "green",
   770                      "value": null
   771                    },
   772                    {
   773                      "color": "red",
   774                      "value": 80
   775                    }
   776                  ]
   777                }
   778              },
   779              "overrides": []
   780            },
   781            "gridPos": {
   782              "h": 8,
   783              "w": 12,
   784              "x": 0,
   785              "y": 4
   786            },
   787            "id": 159,
   788            "options": {
   789              "legend": {
   790                "calcs": [],
   791                "displayMode": "list",
   792                "placement": "bottom"
   793              },
   794              "tooltip": {
   795                "mode": "single"
   796              }
   797            },
   798            "targets": [
   799              {
   800                "datasource": {
   801                  "type": "prometheus",
   802                  "uid": "PBFA97CFB590B2093"
   803                },
   804                "exemplar": true,
   805                "expr": "rate(evm_pool_rpc_node_polls_success{namespace=\"$namespace\", pod=\"$cl_node\"} [2m])",
   806                "interval": "",
   807                "legendFormat": "",
   808                "refId": "A"
   809              }
   810            ],
   811            "title": "EVM node polls",
   812            "transparent": true,
   813            "type": "timeseries"
   814          },
   815          {
   816            "datasource": {
   817              "type": "prometheus",
   818              "uid": "PBFA97CFB590B2093"
   819            },
   820            "description": "RPC calls hist",
   821            "fieldConfig": {
   822              "defaults": {
   823                "color": {
   824                  "mode": "palette-classic"
   825                },
   826                "custom": {
   827                  "axisLabel": "",
   828                  "axisPlacement": "auto",
   829                  "barAlignment": 0,
   830                  "drawStyle": "line",
   831                  "fillOpacity": 0,
   832                  "gradientMode": "none",
   833                  "hideFrom": {
   834                    "legend": false,
   835                    "tooltip": false,
   836                    "viz": false
   837                  },
   838                  "lineInterpolation": "linear",
   839                  "lineWidth": 1,
   840                  "pointSize": 5,
   841                  "scaleDistribution": {
   842                    "type": "linear"
   843                  },
   844                  "showPoints": "auto",
   845                  "spanNulls": false,
   846                  "stacking": {
   847                    "group": "A",
   848                    "mode": "none"
   849                  },
   850                  "thresholdsStyle": {
   851                    "mode": "off"
   852                  }
   853                },
   854                "mappings": [],
   855                "thresholds": {
   856                  "mode": "absolute",
   857                  "steps": [
   858                    {
   859                      "color": "green",
   860                      "value": null
   861                    },
   862                    {
   863                      "color": "red",
   864                      "value": 80
   865                    }
   866                  ]
   867                }
   868              },
   869              "overrides": []
   870            },
   871            "gridPos": {
   872              "h": 8,
   873              "w": 12,
   874              "x": 12,
   875              "y": 4
   876            },
   877            "id": 157,
   878            "options": {
   879              "legend": {
   880                "calcs": [],
   881                "displayMode": "list",
   882                "placement": "bottom"
   883              },
   884              "tooltip": {
   885                "mode": "single"
   886              }
   887            },
   888            "targets": [
   889              {
   890                "datasource": {
   891                  "type": "prometheus",
   892                  "uid": "PBFA97CFB590B2093"
   893                },
   894                "exemplar": true,
   895                "expr": "histogram_quantile(0.95, sum(rate(evm_pool_rpc_node_rpc_call_time_bucket{pod=\"$cl_node\", rpcCallName=\"CallContext\"}[2m])) by (le)) * 1e-6",
   896                "interval": "",
   897                "legendFormat": "CallContext - 95",
   898                "refId": "A"
   899              },
   900              {
   901                "datasource": {
   902                  "type": "prometheus",
   903                  "uid": "PBFA97CFB590B2093"
   904                },
   905                "exemplar": true,
   906                "expr": "histogram_quantile(0.5, sum(rate(evm_pool_rpc_node_rpc_call_time_bucket{pod=\"$cl_node\", rpcCallName=\"CallContext\"}[2m])) by (le)) * 1e-6",
   907                "hide": false,
   908                "interval": "",
   909                "legendFormat": "CallContext - 50",
   910                "refId": "B"
   911              },
   912              {
   913                "datasource": {
   914                  "type": "prometheus",
   915                  "uid": "PBFA97CFB590B2093"
   916                },
   917                "exemplar": true,
   918                "expr": "histogram_quantile(0.95, sum(rate(evm_pool_rpc_node_rpc_call_time_bucket{rpcCallName=\"BatchCallContext\"}[2m])) by (le)) * 1e-6",
   919                "hide": false,
   920                "interval": "",
   921                "legendFormat": "BatchCallContext - 95",
   922                "refId": "C"
   923              },
   924              {
   925                "datasource": {
   926                  "type": "prometheus",
   927                  "uid": "PBFA97CFB590B2093"
   928                },
   929                "exemplar": true,
   930                "expr": "histogram_quantile(0.5, sum(rate(evm_pool_rpc_node_rpc_call_time_bucket{rpcCallName=\"BatchCallContext\"}[2m])) by (le)) * 1e-6",
   931                "hide": false,
   932                "interval": "",
   933                "legendFormat": "BatchCallContext - 50",
   934                "refId": "D"
   935              },
   936              {
   937                "datasource": {
   938                  "type": "prometheus",
   939                  "uid": "PBFA97CFB590B2093"
   940                },
   941                "exemplar": true,
   942                "expr": "histogram_quantile(0.95, sum(rate(evm_pool_rpc_node_rpc_call_time_bucket{rpcCallName=\"BalanceAt\"}[2m])) by (le)) * 1e-6",
   943                "hide": false,
   944                "interval": "",
   945                "legendFormat": "BalanceAt - 95",
   946                "refId": "E"
   947              },
   948              {
   949                "datasource": {
   950                  "type": "prometheus",
   951                  "uid": "PBFA97CFB590B2093"
   952                },
   953                "exemplar": true,
   954                "expr": "histogram_quantile(0.50, sum(rate(evm_pool_rpc_node_rpc_call_time_bucket{rpcCallName=\"BalanceAt\"}[2m])) by (le)) * 1e-6",
   955                "hide": false,
   956                "interval": "",
   957                "legendFormat": "BalanceAt - 50",
   958                "refId": "F"
   959              },
   960              {
   961                "datasource": {
   962                  "type": "prometheus",
   963                  "uid": "PBFA97CFB590B2093"
   964                },
   965                "exemplar": true,
   966                "expr": "histogram_quantile(0.95, sum(rate(evm_pool_rpc_node_rpc_call_time_bucket{rpcCallName=\"PendingNonceAt\"}[2m])) by (le)) * 1e-6",
   967                "hide": false,
   968                "interval": "",
   969                "legendFormat": "PendingNonceAt - 95",
   970                "refId": "G"
   971              },
   972              {
   973                "datasource": {
   974                  "type": "prometheus",
   975                  "uid": "PBFA97CFB590B2093"
   976                },
   977                "exemplar": true,
   978                "expr": "histogram_quantile(0.50, sum(rate(evm_pool_rpc_node_rpc_call_time_bucket{rpcCallName=\"PendingNonceAt\"}[2m])) by (le)) * 1e-6",
   979                "hide": false,
   980                "interval": "",
   981                "legendFormat": "PendingNonceAt - 50",
   982                "refId": "H"
   983              }
   984            ],
   985            "title": "RPC calls hist",
   986            "transparent": true,
   987            "type": "timeseries"
   988          },
   989          {
   990            "datasource": {
   991              "type": "prometheus",
   992              "uid": "PBFA97CFB590B2093"
   993            },
   994            "description": "RPC node states",
   995            "fieldConfig": {
   996              "defaults": {
   997                "color": {
   998                  "mode": "palette-classic"
   999                },
  1000                "custom": {
  1001                  "axisLabel": "",
  1002                  "axisPlacement": "auto",
  1003                  "barAlignment": 0,
  1004                  "drawStyle": "line",
  1005                  "fillOpacity": 0,
  1006                  "gradientMode": "none",
  1007                  "hideFrom": {
  1008                    "legend": false,
  1009                    "tooltip": false,
  1010                    "viz": false
  1011                  },
  1012                  "lineInterpolation": "linear",
  1013                  "lineWidth": 1,
  1014                  "pointSize": 5,
  1015                  "scaleDistribution": {
  1016                    "type": "linear"
  1017                  },
  1018                  "showPoints": "auto",
  1019                  "spanNulls": false,
  1020                  "stacking": {
  1021                    "group": "A",
  1022                    "mode": "none"
  1023                  },
  1024                  "thresholdsStyle": {
  1025                    "mode": "off"
  1026                  }
  1027                },
  1028                "mappings": [],
  1029                "thresholds": {
  1030                  "mode": "absolute",
  1031                  "steps": [
  1032                    {
  1033                      "color": "green",
  1034                      "value": null
  1035                    },
  1036                    {
  1037                      "color": "red",
  1038                      "value": 80
  1039                    }
  1040                  ]
  1041                }
  1042              },
  1043              "overrides": []
  1044            },
  1045            "gridPos": {
  1046              "h": 8,
  1047              "w": 12,
  1048              "x": 0,
  1049              "y": 12
  1050            },
  1051            "id": 153,
  1052            "options": {
  1053              "legend": {
  1054                "calcs": [],
  1055                "displayMode": "list",
  1056                "placement": "bottom"
  1057              },
  1058              "tooltip": {
  1059                "mode": "single"
  1060              }
  1061            },
  1062            "targets": [
  1063              {
  1064                "datasource": {
  1065                  "type": "prometheus",
  1066                  "uid": "PBFA97CFB590B2093"
  1067                },
  1068                "exemplar": true,
  1069                "expr": "evm_pool_rpc_node_states{namespace=\"$namespace\", pod=\"$cl_node\"}",
  1070                "interval": "",
  1071                "legendFormat": "chainId: {{evmChainID}} pod: {{pod}} state: {{state}}",
  1072                "refId": "A"
  1073              }
  1074            ],
  1075            "title": "RPC node states",
  1076            "transparent": true,
  1077            "type": "timeseries"
  1078          },
  1079          {
  1080            "datasource": {
  1081              "type": "prometheus",
  1082              "uid": "PBFA97CFB590B2093"
  1083            },
  1084            "fieldConfig": {
  1085              "defaults": {
  1086                "color": {
  1087                  "mode": "palette-classic"
  1088                },
  1089                "custom": {
  1090                  "axisLabel": "",
  1091                  "axisPlacement": "auto",
  1092                  "barAlignment": 0,
  1093                  "drawStyle": "line",
  1094                  "fillOpacity": 0,
  1095                  "gradientMode": "none",
  1096                  "hideFrom": {
  1097                    "legend": false,
  1098                    "tooltip": false,
  1099                    "viz": false
  1100                  },
  1101                  "lineInterpolation": "linear",
  1102                  "lineWidth": 1,
  1103                  "pointSize": 5,
  1104                  "scaleDistribution": {
  1105                    "type": "linear"
  1106                  },
  1107                  "showPoints": "auto",
  1108                  "spanNulls": false,
  1109                  "stacking": {
  1110                    "group": "A",
  1111                    "mode": "none"
  1112                  },
  1113                  "thresholdsStyle": {
  1114                    "mode": "off"
  1115                  }
  1116                },
  1117                "mappings": [],
  1118                "thresholds": {
  1119                  "mode": "absolute",
  1120                  "steps": [
  1121                    {
  1122                      "color": "green",
  1123                      "value": null
  1124                    },
  1125                    {
  1126                      "color": "red",
  1127                      "value": 80
  1128                    }
  1129                  ]
  1130                }
  1131              },
  1132              "overrides": []
  1133            },
  1134            "gridPos": {
  1135              "h": 8,
  1136              "w": 12,
  1137              "x": 12,
  1138              "y": 12
  1139            },
  1140            "id": 151,
  1141            "options": {
  1142              "legend": {
  1143                "calcs": [],
  1144                "displayMode": "list",
  1145                "placement": "bottom"
  1146              },
  1147              "tooltip": {
  1148                "mode": "single"
  1149              }
  1150            },
  1151            "pluginVersion": "8.3.5",
  1152            "targets": [
  1153              {
  1154                "datasource": {
  1155                  "type": "prometheus",
  1156                  "uid": "PBFA97CFB590B2093"
  1157                },
  1158                "exemplar": true,
  1159                "expr": "evm_pool_rpc_node_num_transitions_to_alive{namespace=\"$namespace\", pod=\"$cl_node\"}",
  1160                "interval": "",
  1161                "legendFormat": "chainId: {{evmChainID}} pod: {{pod}}",
  1162                "refId": "A"
  1163              }
  1164            ],
  1165            "title": "RPC node alive statuses",
  1166            "transparent": true,
  1167            "type": "timeseries"
  1168          },
  1169          {
  1170            "datasource": {
  1171              "type": "prometheus",
  1172              "uid": "PBFA97CFB590B2093"
  1173            },
  1174            "description": "RPC calls total (rate)",
  1175            "fieldConfig": {
  1176              "defaults": {
  1177                "color": {
  1178                  "mode": "palette-classic"
  1179                },
  1180                "custom": {
  1181                  "axisLabel": "",
  1182                  "axisPlacement": "auto",
  1183                  "barAlignment": 0,
  1184                  "drawStyle": "line",
  1185                  "fillOpacity": 0,
  1186                  "gradientMode": "none",
  1187                  "hideFrom": {
  1188                    "legend": false,
  1189                    "tooltip": false,
  1190                    "viz": false
  1191                  },
  1192                  "lineInterpolation": "linear",
  1193                  "lineWidth": 1,
  1194                  "pointSize": 5,
  1195                  "scaleDistribution": {
  1196                    "type": "linear"
  1197                  },
  1198                  "showPoints": "auto",
  1199                  "spanNulls": false,
  1200                  "stacking": {
  1201                    "group": "A",
  1202                    "mode": "none"
  1203                  },
  1204                  "thresholdsStyle": {
  1205                    "mode": "off"
  1206                  }
  1207                },
  1208                "mappings": [],
  1209                "thresholds": {
  1210                  "mode": "absolute",
  1211                  "steps": [
  1212                    {
  1213                      "color": "green",
  1214                      "value": null
  1215                    },
  1216                    {
  1217                      "color": "red",
  1218                      "value": 80
  1219                    }
  1220                  ]
  1221                }
  1222              },
  1223              "overrides": []
  1224            },
  1225            "gridPos": {
  1226              "h": 8,
  1227              "w": 12,
  1228              "x": 0,
  1229              "y": 20
  1230            },
  1231            "id": 145,
  1232            "options": {
  1233              "legend": {
  1234                "calcs": [],
  1235                "displayMode": "list",
  1236                "placement": "bottom"
  1237              },
  1238              "tooltip": {
  1239                "mode": "single"
  1240              }
  1241            },
  1242            "targets": [
  1243              {
  1244                "datasource": {
  1245                  "type": "prometheus",
  1246                  "uid": "PBFA97CFB590B2093"
  1247                },
  1248                "exemplar": true,
  1249                "expr": "rate(evm_pool_rpc_node_calls_total{namespace=\"$namespace\", pod=\"$cl_node\"}[2m])",
  1250                "interval": "",
  1251                "legendFormat": "",
  1252                "refId": "A"
  1253              }
  1254            ],
  1255            "title": "RPC calls total (rate)",
  1256            "transparent": true,
  1257            "type": "timeseries"
  1258          },
  1259          {
  1260            "datasource": {
  1261              "type": "prometheus",
  1262              "uid": "PBFA97CFB590B2093"
  1263            },
  1264            "fieldConfig": {
  1265              "defaults": {
  1266                "color": {
  1267                  "mode": "palette-classic"
  1268                },
  1269                "custom": {
  1270                  "axisLabel": "",
  1271                  "axisPlacement": "auto",
  1272                  "barAlignment": 0,
  1273                  "drawStyle": "line",
  1274                  "fillOpacity": 0,
  1275                  "gradientMode": "none",
  1276                  "hideFrom": {
  1277                    "legend": false,
  1278                    "tooltip": false,
  1279                    "viz": false
  1280                  },
  1281                  "lineInterpolation": "linear",
  1282                  "lineWidth": 1,
  1283                  "pointSize": 5,
  1284                  "scaleDistribution": {
  1285                    "type": "linear"
  1286                  },
  1287                  "showPoints": "auto",
  1288                  "spanNulls": false,
  1289                  "stacking": {
  1290                    "group": "A",
  1291                    "mode": "none"
  1292                  },
  1293                  "thresholdsStyle": {
  1294                    "mode": "off"
  1295                  }
  1296                },
  1297                "mappings": [],
  1298                "thresholds": {
  1299                  "mode": "absolute",
  1300                  "steps": [
  1301                    {
  1302                      "color": "green",
  1303                      "value": null
  1304                    },
  1305                    {
  1306                      "color": "red",
  1307                      "value": 80
  1308                    }
  1309                  ]
  1310                }
  1311              },
  1312              "overrides": []
  1313            },
  1314            "gridPos": {
  1315              "h": 8,
  1316              "w": 12,
  1317              "x": 12,
  1318              "y": 20
  1319            },
  1320            "id": 155,
  1321            "options": {
  1322              "legend": {
  1323                "calcs": [],
  1324                "displayMode": "list",
  1325                "placement": "bottom"
  1326              },
  1327              "tooltip": {
  1328                "mode": "single"
  1329              }
  1330            },
  1331            "targets": [
  1332              {
  1333                "datasource": {
  1334                  "type": "prometheus",
  1335                  "uid": "PBFA97CFB590B2093"
  1336                },
  1337                "exemplar": true,
  1338                "expr": "rate(evm_pool_rpc_node_verifies{namespace=\"$namespace\", pod=\"$cl_node\"} [2m])",
  1339                "interval": "",
  1340                "legendFormat": "Success, chainId: {{evmChainID}} nodeName: {{nodeName }} instance: {{instance}}",
  1341                "refId": "A"
  1342              },
  1343              {
  1344                "datasource": {
  1345                  "type": "prometheus",
  1346                  "uid": "PBFA97CFB590B2093"
  1347                },
  1348                "exemplar": true,
  1349                "expr": "rate(evm_pool_rpc_node_verifies_success{namespace=\"$namespace\", pod=\"$cl_node\"} [2m])",
  1350                "hide": false,
  1351                "interval": "",
  1352                "legendFormat": "Total, chainId: {{evmChainID}} nodeName: {{nodeName }} instance: {{instance}}",
  1353                "refId": "B"
  1354              }
  1355            ],
  1356            "title": "Total verifications (success/errors)",
  1357            "transparent": true,
  1358            "type": "timeseries"
  1359          },
  1360          {
  1361            "datasource": {
  1362              "type": "prometheus",
  1363              "uid": "PBFA97CFB590B2093"
  1364            },
  1365            "description": "ETH account value",
  1366            "fieldConfig": {
  1367              "defaults": {
  1368                "color": {
  1369                  "mode": "palette-classic"
  1370                },
  1371                "custom": {
  1372                  "axisLabel": "",
  1373                  "axisPlacement": "left",
  1374                  "barAlignment": 0,
  1375                  "drawStyle": "line",
  1376                  "fillOpacity": 0,
  1377                  "gradientMode": "none",
  1378                  "hideFrom": {
  1379                    "legend": false,
  1380                    "tooltip": false,
  1381                    "viz": false
  1382                  },
  1383                  "lineInterpolation": "linear",
  1384                  "lineStyle": {
  1385                    "fill": "solid"
  1386                  },
  1387                  "lineWidth": 1,
  1388                  "pointSize": 5,
  1389                  "scaleDistribution": {
  1390                    "type": "linear"
  1391                  },
  1392                  "showPoints": "auto",
  1393                  "spanNulls": false,
  1394                  "stacking": {
  1395                    "group": "A",
  1396                    "mode": "none"
  1397                  },
  1398                  "thresholdsStyle": {
  1399                    "mode": "off"
  1400                  }
  1401                },
  1402                "mappings": [],
  1403                "thresholds": {
  1404                  "mode": "absolute",
  1405                  "steps": [
  1406                    {
  1407                      "color": "green",
  1408                      "value": null
  1409                    },
  1410                    {
  1411                      "color": "red",
  1412                      "value": 80
  1413                    }
  1414                  ]
  1415                }
  1416              },
  1417              "overrides": []
  1418            },
  1419            "gridPos": {
  1420              "h": 8,
  1421              "w": 12,
  1422              "x": 0,
  1423              "y": 28
  1424            },
  1425            "id": 165,
  1426            "options": {
  1427              "legend": {
  1428                "calcs": [],
  1429                "displayMode": "list",
  1430                "placement": "bottom"
  1431              },
  1432              "tooltip": {
  1433                "mode": "single"
  1434              }
  1435            },
  1436            "targets": [
  1437              {
  1438                "datasource": {
  1439                  "type": "prometheus",
  1440                  "uid": "PBFA97CFB590B2093"
  1441                },
  1442                "exemplar": false,
  1443                "expr": "eth_balance{namespace=\"$namespace\", pod=\"$cl_node\"}",
  1444                "interval": "",
  1445                "intervalFactor": 1,
  1446                "legendFormat": "account: {{account}}",
  1447                "refId": "A"
  1448              }
  1449            ],
  1450            "title": "ETH account value",
  1451            "transparent": true,
  1452            "type": "timeseries"
  1453          }
  1454        ],
  1455        "title": "EVM CL node metrics ($cl_node)",
  1456        "type": "row"
  1457      },
  1458      {
  1459        "collapsed": true,
  1460        "gridPos": {
  1461          "h": 1,
  1462          "w": 24,
  1463          "x": 0,
  1464          "y": 4
  1465        },
  1466        "id": 183,
  1467        "panels": [
  1468          {
  1469            "datasource": {
  1470              "type": "prometheus",
  1471              "uid": "PBFA97CFB590B2093"
  1472            },
  1473            "description": "TXns count (confirmed/unconfirmed)",
  1474            "fieldConfig": {
  1475              "defaults": {
  1476                "color": {
  1477                  "mode": "palette-classic"
  1478                },
  1479                "custom": {
  1480                  "axisLabel": "",
  1481                  "axisPlacement": "auto",
  1482                  "barAlignment": 0,
  1483                  "drawStyle": "line",
  1484                  "fillOpacity": 0,
  1485                  "gradientMode": "none",
  1486                  "hideFrom": {
  1487                    "legend": false,
  1488                    "tooltip": false,
  1489                    "viz": false
  1490                  },
  1491                  "lineInterpolation": "linear",
  1492                  "lineWidth": 1,
  1493                  "pointSize": 5,
  1494                  "scaleDistribution": {
  1495                    "type": "linear"
  1496                  },
  1497                  "showPoints": "auto",
  1498                  "spanNulls": false,
  1499                  "stacking": {
  1500                    "group": "A",
  1501                    "mode": "none"
  1502                  },
  1503                  "thresholdsStyle": {
  1504                    "mode": "off"
  1505                  }
  1506                },
  1507                "mappings": [],
  1508                "thresholds": {
  1509                  "mode": "absolute",
  1510                  "steps": [
  1511                    {
  1512                      "color": "green",
  1513                      "value": null
  1514                    },
  1515                    {
  1516                      "color": "red",
  1517                      "value": 80
  1518                    }
  1519                  ]
  1520                }
  1521              },
  1522              "overrides": []
  1523            },
  1524            "gridPos": {
  1525              "h": 8,
  1526              "w": 12,
  1527              "x": 0,
  1528              "y": 5
  1529            },
  1530            "id": 185,
  1531            "options": {
  1532              "legend": {
  1533                "calcs": [],
  1534                "displayMode": "list",
  1535                "placement": "bottom"
  1536              },
  1537              "tooltip": {
  1538                "mode": "single"
  1539              }
  1540            },
  1541            "targets": [
  1542              {
  1543                "datasource": {
  1544                  "type": "prometheus",
  1545                  "uid": "PBFA97CFB590B2093"
  1546                },
  1547                "exemplar": true,
  1548                "expr": "unconfirmed_transactions{namespace=\"$namespace\", pod=\"$cl_node\"}",
  1549                "interval": "",
  1550                "legendFormat": "Unconfirmed, Pod: {{pod}}",
  1551                "refId": "A"
  1552              },
  1553              {
  1554                "datasource": {
  1555                  "type": "prometheus",
  1556                  "uid": "PBFA97CFB590B2093"
  1557                },
  1558                "exemplar": true,
  1559                "expr": "tx_manager_num_confirmed_transactions{namespace=\"$namespace\", pod=\"$cl_node\"}",
  1560                "hide": false,
  1561                "interval": "",
  1562                "legendFormat": "Confirmed, Pod: {{pod}}",
  1563                "refId": "B"
  1564              },
  1565              {
  1566                "datasource": {
  1567                  "type": "prometheus",
  1568                  "uid": "PBFA97CFB590B2093"
  1569                },
  1570                "exemplar": true,
  1571                "expr": "tx_manager_num_successful_transactions{namespace=\"$namespace\", pod=\"$cl_node\"}",
  1572                "hide": false,
  1573                "interval": "",
  1574                "legendFormat": "Successful, Pod: {{pod}}",
  1575                "refId": "C"
  1576              }
  1577            ],
  1578            "title": "TXns count (confirmed/unconfirmed)",
  1579            "transparent": true,
  1580            "type": "timeseries"
  1581          },
  1582          {
  1583            "datasource": {
  1584              "type": "prometheus",
  1585              "uid": "PBFA97CFB590B2093"
  1586            },
  1587            "fieldConfig": {
  1588              "defaults": {
  1589                "color": {
  1590                  "mode": "palette-classic"
  1591                },
  1592                "custom": {
  1593                  "axisLabel": "Duration, ms",
  1594                  "axisPlacement": "left",
  1595                  "barAlignment": 0,
  1596                  "drawStyle": "line",
  1597                  "fillOpacity": 0,
  1598                  "gradientMode": "none",
  1599                  "hideFrom": {
  1600                    "legend": false,
  1601                    "tooltip": false,
  1602                    "viz": false
  1603                  },
  1604                  "lineInterpolation": "linear",
  1605                  "lineWidth": 1,
  1606                  "pointSize": 5,
  1607                  "scaleDistribution": {
  1608                    "type": "linear"
  1609                  },
  1610                  "showPoints": "auto",
  1611                  "spanNulls": false,
  1612                  "stacking": {
  1613                    "group": "A",
  1614                    "mode": "none"
  1615                  },
  1616                  "thresholdsStyle": {
  1617                    "mode": "off"
  1618                  }
  1619                },
  1620                "mappings": [],
  1621                "thresholds": {
  1622                  "mode": "absolute",
  1623                  "steps": [
  1624                    {
  1625                      "color": "green",
  1626                      "value": null
  1627                    },
  1628                    {
  1629                      "color": "red",
  1630                      "value": 80
  1631                    }
  1632                  ]
  1633                }
  1634              },
  1635              "overrides": []
  1636            },
  1637            "gridPos": {
  1638              "h": 8,
  1639              "w": 12,
  1640              "x": 12,
  1641              "y": 5
  1642            },
  1643            "id": 187,
  1644            "options": {
  1645              "legend": {
  1646                "calcs": [],
  1647                "displayMode": "list",
  1648                "placement": "bottom"
  1649              },
  1650              "tooltip": {
  1651                "mode": "single"
  1652              }
  1653            },
  1654            "targets": [
  1655              {
  1656                "datasource": {
  1657                  "type": "prometheus",
  1658                  "uid": "PBFA97CFB590B2093"
  1659                },
  1660                "exemplar": true,
  1661                "expr": "histogram_quantile(0.95, sum(rate(tx_manager_time_until_tx_confirmed_bucket [2m])) by (le)) * 1e-6",
  1662                "instant": false,
  1663                "interval": "",
  1664                "legendFormat": "TxConfirmed - 95",
  1665                "refId": "A"
  1666              },
  1667              {
  1668                "datasource": {
  1669                  "type": "prometheus",
  1670                  "uid": "PBFA97CFB590B2093"
  1671                },
  1672                "exemplar": true,
  1673                "expr": "histogram_quantile(0.50, sum(rate(tx_manager_time_until_tx_confirmed_bucket [2m])) by (le)) * 1e-6",
  1674                "hide": false,
  1675                "interval": "",
  1676                "legendFormat": "TxConfirmed - 50",
  1677                "refId": "B"
  1678              },
  1679              {
  1680                "datasource": {
  1681                  "type": "prometheus",
  1682                  "uid": "PBFA97CFB590B2093"
  1683                },
  1684                "exemplar": true,
  1685                "expr": "histogram_quantile(0.95, sum(rate(tx_manager_time_until_tx_broadcast_bucket [2m])) by (le)) * 1e-6",
  1686                "hide": false,
  1687                "interval": "",
  1688                "legendFormat": "TxBroadcast -95",
  1689                "refId": "C"
  1690              },
  1691              {
  1692                "datasource": {
  1693                  "type": "prometheus",
  1694                  "uid": "PBFA97CFB590B2093"
  1695                },
  1696                "exemplar": true,
  1697                "expr": "histogram_quantile(0.50, sum(rate(tx_manager_time_until_tx_broadcast_bucket [2m])) by (le)) * 1e-6",
  1698                "hide": false,
  1699                "interval": "",
  1700                "legendFormat": "TxBroadcast -50",
  1701                "refId": "D"
  1702              }
  1703            ],
  1704            "title": "TXns times",
  1705            "transparent": true,
  1706            "type": "timeseries"
  1707          }
  1708        ],
  1709        "title": "TX manager ($cl_node)",
  1710        "type": "row"
  1711      },
  1712      {
  1713        "collapsed": true,
  1714        "gridPos": {
  1715          "h": 1,
  1716          "w": 24,
  1717          "x": 0,
  1718          "y": 5
  1719        },
  1720        "id": 35,
  1721        "panels": [
  1722          {
  1723            "description": "All logs of the Chainlink node",
  1724            "gridPos": {
  1725              "h": 10,
  1726              "w": 12,
  1727              "x": 0,
  1728              "y": 6
  1729            },
  1730            "id": 33,
  1731            "options": {
  1732              "dedupStrategy": "none",
  1733              "enableLogDetails": true,
  1734              "prettifyLogMessage": false,
  1735              "showCommonLabels": false,
  1736              "showLabels": false,
  1737              "showTime": false,
  1738              "sortOrder": "Descending",
  1739              "wrapLogMessage": false
  1740            },
  1741            "pluginVersion": "8.5.0",
  1742            "targets": [
  1743              {
  1744                "datasource": {
  1745                  "type": "loki",
  1746                  "uid": "grafanacloud-logs"
  1747                },
  1748                "expr": "{namespace=\"$namespace\", container=\"node\", pod=\"$cl_node\"}",
  1749                "queryType": "range",
  1750                "refId": "A"
  1751              }
  1752            ],
  1753            "title": "Chainlink Node Logs",
  1754            "transparent": true,
  1755            "type": "logs"
  1756          },
  1757          {
  1758            "description": "All [ERROR] logs from the Chainlink node.",
  1759            "gridPos": {
  1760              "h": 10,
  1761              "w": 12,
  1762              "x": 12,
  1763              "y": 6
  1764            },
  1765            "id": 38,
  1766            "options": {
  1767              "dedupStrategy": "none",
  1768              "enableLogDetails": true,
  1769              "prettifyLogMessage": false,
  1770              "showCommonLabels": false,
  1771              "showLabels": false,
  1772              "showTime": false,
  1773              "sortOrder": "Descending",
  1774              "wrapLogMessage": false
  1775            },
  1776            "pluginVersion": "8.5.0",
  1777            "targets": [
  1778              {
  1779                "datasource": {
  1780                  "type": "loki",
  1781                  "uid": "grafanacloud-logs"
  1782                },
  1783                "expr": "{namespace=\"$namespace\", container=\"node\", pod=\"$cl_node\"} | json | level=~\"error|crit\"",
  1784                "queryType": "range",
  1785                "refId": "A"
  1786              }
  1787            ],
  1788            "title": "Chainlink Node Errors",
  1789            "transparent": true,
  1790            "type": "logs"
  1791          },
  1792          {
  1793            "datasource": {
  1794              "type": "prometheus",
  1795              "uid": "PBFA97CFB590B2093"
  1796            },
  1797            "description": "Chainlink CPU usage and limit",
  1798            "fieldConfig": {
  1799              "defaults": {
  1800                "color": {
  1801                  "mode": "palette-classic"
  1802                },
  1803                "custom": {
  1804                  "axisLabel": "CPU Power",
  1805                  "axisPlacement": "left",
  1806                  "barAlignment": 0,
  1807                  "drawStyle": "line",
  1808                  "fillOpacity": 5,
  1809                  "gradientMode": "opacity",
  1810                  "hideFrom": {
  1811                    "legend": false,
  1812                    "tooltip": false,
  1813                    "viz": false
  1814                  },
  1815                  "lineInterpolation": "linear",
  1816                  "lineStyle": {
  1817                    "fill": "solid"
  1818                  },
  1819                  "lineWidth": 2,
  1820                  "pointSize": 5,
  1821                  "scaleDistribution": {
  1822                    "type": "linear"
  1823                  },
  1824                  "showPoints": "auto",
  1825                  "spanNulls": false,
  1826                  "stacking": {
  1827                    "group": "A",
  1828                    "mode": "none"
  1829                  },
  1830                  "thresholdsStyle": {
  1831                    "mode": "off"
  1832                  }
  1833                },
  1834                "mappings": [],
  1835                "min": 0,
  1836                "thresholds": {
  1837                  "mode": "absolute",
  1838                  "steps": [
  1839                    {
  1840                      "color": "green",
  1841                      "value": null
  1842                    },
  1843                    {
  1844                      "color": "red",
  1845                      "value": 80
  1846                    }
  1847                  ]
  1848                }
  1849              },
  1850              "overrides": []
  1851            },
  1852            "gridPos": {
  1853              "h": 8,
  1854              "w": 6,
  1855              "x": 0,
  1856              "y": 16
  1857            },
  1858            "id": 41,
  1859            "options": {
  1860              "legend": {
  1861                "calcs": [],
  1862                "displayMode": "list",
  1863                "placement": "bottom",
  1864                "showLegend": true
  1865              },
  1866              "tooltip": {
  1867                "mode": "single",
  1868                "sort": "none"
  1869              }
  1870            },
  1871            "targets": [
  1872              {
  1873                "datasource": {
  1874                  "type": "prometheus",
  1875                  "uid": "PBFA97CFB590B2093"
  1876                },
  1877                "exemplar": true,
  1878                "expr": "kube_pod_container_resource_limits{container=\"node\", pod=\"$cl_node\", resource=\"cpu\"}",
  1879                "interval": "",
  1880                "legendFormat": "",
  1881                "refId": "A"
  1882              },
  1883              {
  1884                "datasource": {
  1885                  "type": "prometheus",
  1886                  "uid": "PBFA97CFB590B2093"
  1887                },
  1888                "exemplar": true,
  1889                "expr": "container_cpu_load_average_10s{namespace=\"$namespace\", container=\"node\", pod=\"$cl_node\"}",
  1890                "hide": false,
  1891                "interval": "",
  1892                "legendFormat": "",
  1893                "refId": "B"
  1894              }
  1895            ],
  1896            "title": "CPU",
  1897            "transparent": true,
  1898            "type": "timeseries"
  1899          },
  1900          {
  1901            "datasource": {
  1902              "type": "prometheus",
  1903              "uid": "PBFA97CFB590B2093"
  1904            },
  1905            "description": "Chainlink RAM usage and limit",
  1906            "fieldConfig": {
  1907              "defaults": {
  1908                "color": {
  1909                  "mode": "palette-classic"
  1910                },
  1911                "custom": {
  1912                  "axisLabel": "MiB",
  1913                  "axisPlacement": "auto",
  1914                  "barAlignment": 0,
  1915                  "drawStyle": "line",
  1916                  "fillOpacity": 5,
  1917                  "gradientMode": "opacity",
  1918                  "hideFrom": {
  1919                    "legend": false,
  1920                    "tooltip": false,
  1921                    "viz": false
  1922                  },
  1923                  "lineInterpolation": "linear",
  1924                  "lineWidth": 2,
  1925                  "pointSize": 5,
  1926                  "scaleDistribution": {
  1927                    "type": "linear"
  1928                  },
  1929                  "showPoints": "auto",
  1930                  "spanNulls": false,
  1931                  "stacking": {
  1932                    "group": "A",
  1933                    "mode": "none"
  1934                  },
  1935                  "thresholdsStyle": {
  1936                    "mode": "off"
  1937                  }
  1938                },
  1939                "mappings": [],
  1940                "min": 0,
  1941                "thresholds": {
  1942                  "mode": "absolute",
  1943                  "steps": [
  1944                    {
  1945                      "color": "green",
  1946                      "value": null
  1947                    },
  1948                    {
  1949                      "color": "red",
  1950                      "value": 80
  1951                    }
  1952                  ]
  1953                }
  1954              },
  1955              "overrides": []
  1956            },
  1957            "gridPos": {
  1958              "h": 8,
  1959              "w": 6,
  1960              "x": 6,
  1961              "y": 16
  1962            },
  1963            "id": 42,
  1964            "options": {
  1965              "legend": {
  1966                "calcs": [],
  1967                "displayMode": "list",
  1968                "placement": "bottom",
  1969                "showLegend": true
  1970              },
  1971              "tooltip": {
  1972                "mode": "single",
  1973                "sort": "none"
  1974              }
  1975            },
  1976            "targets": [
  1977              {
  1978                "datasource": {
  1979                  "type": "prometheus",
  1980                  "uid": "PBFA97CFB590B2093"
  1981                },
  1982                "exemplar": true,
  1983                "expr": "kube_pod_container_resource_limits{container=\"node\", pod=\"$cl_node\", resource=\"memory\"} / 1024 / 1024",
  1984                "interval": "",
  1985                "legendFormat": "Limits, container: {{container}} pod: {{pod}}",
  1986                "refId": "A"
  1987              },
  1988              {
  1989                "datasource": {
  1990                  "type": "prometheus",
  1991                  "uid": "PBFA97CFB590B2093"
  1992                },
  1993                "exemplar": true,
  1994                "expr": "container_memory_usage_bytes{container=\"node\", pod=\"$cl_node\"} / 1024 / 1024",
  1995                "hide": false,
  1996                "interval": "",
  1997                "legendFormat": "Used, container: {{container}} pod: {{pod}}",
  1998                "refId": "B"
  1999              }
  2000            ],
  2001            "title": "RAM",
  2002            "transparent": true,
  2003            "type": "timeseries"
  2004          },
  2005          {
  2006            "description": "All [WARN] logs from the Chainlink node.",
  2007            "gridPos": {
  2008              "h": 8,
  2009              "w": 12,
  2010              "x": 12,
  2011              "y": 16
  2012            },
  2013            "id": 102,
  2014            "options": {
  2015              "dedupStrategy": "none",
  2016              "enableLogDetails": true,
  2017              "prettifyLogMessage": false,
  2018              "showCommonLabels": false,
  2019              "showLabels": false,
  2020              "showTime": false,
  2021              "sortOrder": "Descending",
  2022              "wrapLogMessage": false
  2023            },
  2024            "pluginVersion": "8.5.0",
  2025            "targets": [
  2026              {
  2027                "datasource": {
  2028                  "type": "loki",
  2029                  "uid": "grafanacloud-logs"
  2030                },
  2031                "expr": "{namespace=\"$namespace\", container=\"node\", pod=\"$cl_node\"}  | json | level=~\"warn\" | msg!=\"Disk space is not enough to log into disk any longer, required disk space\"",
  2032                "queryType": "range",
  2033                "refId": "A"
  2034              }
  2035            ],
  2036            "title": "Chainlink Node Warnings",
  2037            "transparent": true,
  2038            "type": "logs"
  2039          }
  2040        ],
  2041        "repeat": "cl_node",
  2042        "title": "CL node metrics ($cl_node)",
  2043        "type": "row"
  2044      },
  2045      {
  2046        "collapsed": true,
  2047        "gridPos": {
  2048          "h": 1,
  2049          "w": 24,
  2050          "x": 0,
  2051          "y": 6
  2052        },
  2053        "id": 104,
  2054        "panels": [
  2055          {
  2056            "gridPos": {
  2057              "h": 12,
  2058              "w": 18,
  2059              "x": 0,
  2060              "y": 7
  2061            },
  2062            "id": 106,
  2063            "options": {
  2064              "dedupStrategy": "none",
  2065              "enableLogDetails": true,
  2066              "prettifyLogMessage": false,
  2067              "showCommonLabels": false,
  2068              "showLabels": false,
  2069              "showTime": false,
  2070              "sortOrder": "Descending",
  2071              "wrapLogMessage": false
  2072            },
  2073            "pluginVersion": "8.5.5",
  2074            "targets": [
  2075              {
  2076                "datasource": {
  2077                  "type": "loki",
  2078                  "uid": "grafanacloud-logs"
  2079                },
  2080                "expr": "{namespace=\"$namespace\", container=~\"geth-network|starknetdev\", pod=\"$geth_node\"}",
  2081                "refId": "A"
  2082              }
  2083            ],
  2084            "title": "Geth Logs",
  2085            "transparent": true,
  2086            "type": "logs"
  2087          },
  2088          {
  2089            "datasource": {
  2090              "type": "prometheus",
  2091              "uid": "PBFA97CFB590B2093"
  2092            },
  2093            "description": "Geth CPU usage and limit",
  2094            "fieldConfig": {
  2095              "defaults": {
  2096                "color": {
  2097                  "mode": "palette-classic"
  2098                },
  2099                "custom": {
  2100                  "axisLabel": "CPU Power",
  2101                  "axisPlacement": "left",
  2102                  "barAlignment": 0,
  2103                  "drawStyle": "line",
  2104                  "fillOpacity": 5,
  2105                  "gradientMode": "opacity",
  2106                  "hideFrom": {
  2107                    "legend": false,
  2108                    "tooltip": false,
  2109                    "viz": false
  2110                  },
  2111                  "lineInterpolation": "linear",
  2112                  "lineStyle": {
  2113                    "fill": "solid"
  2114                  },
  2115                  "lineWidth": 2,
  2116                  "pointSize": 5,
  2117                  "scaleDistribution": {
  2118                    "type": "linear"
  2119                  },
  2120                  "showPoints": "auto",
  2121                  "spanNulls": false,
  2122                  "stacking": {
  2123                    "group": "A",
  2124                    "mode": "none"
  2125                  },
  2126                  "thresholdsStyle": {
  2127                    "mode": "off"
  2128                  }
  2129                },
  2130                "mappings": [],
  2131                "min": 0,
  2132                "thresholds": {
  2133                  "mode": "absolute",
  2134                  "steps": [
  2135                    {
  2136                      "color": "green",
  2137                      "value": null
  2138                    },
  2139                    {
  2140                      "color": "red",
  2141                      "value": 80
  2142                    }
  2143                  ]
  2144                }
  2145              },
  2146              "overrides": []
  2147            },
  2148            "gridPos": {
  2149              "h": 6,
  2150              "w": 6,
  2151              "x": 18,
  2152              "y": 7
  2153            },
  2154            "id": 107,
  2155            "options": {
  2156              "legend": {
  2157                "calcs": [],
  2158                "displayMode": "list",
  2159                "placement": "bottom",
  2160                "showLegend": true
  2161              },
  2162              "tooltip": {
  2163                "mode": "single",
  2164                "sort": "none"
  2165              }
  2166            },
  2167            "targets": [
  2168              {
  2169                "datasource": {
  2170                  "type": "prometheus",
  2171                  "uid": "PBFA97CFB590B2093"
  2172                },
  2173                "exemplar": true,
  2174                "expr": "kube_pod_container_resource_limits{exported_namespace=\"$namespace\", container=\"geth-network\", resource=\"cpu\"}",
  2175                "interval": "",
  2176                "legendFormat": "",
  2177                "refId": "A"
  2178              },
  2179              {
  2180                "datasource": {
  2181                  "type": "prometheus",
  2182                  "uid": "PBFA97CFB590B2093"
  2183                },
  2184                "exemplar": true,
  2185                "expr": "container_cpu_load_average_10s{namespace=\"$namespace\", container=~\"geth-network|starknetdev\"}",
  2186                "hide": false,
  2187                "interval": "",
  2188                "legendFormat": "",
  2189                "refId": "B"
  2190              }
  2191            ],
  2192            "title": "Geth CPU",
  2193            "transparent": true,
  2194            "type": "timeseries"
  2195          },
  2196          {
  2197            "datasource": {
  2198              "type": "prometheus",
  2199              "uid": "PBFA97CFB590B2093"
  2200            },
  2201            "description": "Geth RAM usage and limit",
  2202            "fieldConfig": {
  2203              "defaults": {
  2204                "color": {
  2205                  "mode": "palette-classic"
  2206                },
  2207                "custom": {
  2208                  "axisLabel": "MiB",
  2209                  "axisPlacement": "auto",
  2210                  "barAlignment": 0,
  2211                  "drawStyle": "line",
  2212                  "fillOpacity": 5,
  2213                  "gradientMode": "opacity",
  2214                  "hideFrom": {
  2215                    "legend": false,
  2216                    "tooltip": false,
  2217                    "viz": false
  2218                  },
  2219                  "lineInterpolation": "linear",
  2220                  "lineWidth": 2,
  2221                  "pointSize": 5,
  2222                  "scaleDistribution": {
  2223                    "type": "linear"
  2224                  },
  2225                  "showPoints": "auto",
  2226                  "spanNulls": false,
  2227                  "stacking": {
  2228                    "group": "A",
  2229                    "mode": "none"
  2230                  },
  2231                  "thresholdsStyle": {
  2232                    "mode": "off"
  2233                  }
  2234                },
  2235                "mappings": [],
  2236                "min": 0,
  2237                "thresholds": {
  2238                  "mode": "absolute",
  2239                  "steps": [
  2240                    {
  2241                      "color": "green",
  2242                      "value": null
  2243                    },
  2244                    {
  2245                      "color": "red",
  2246                      "value": 80
  2247                    }
  2248                  ]
  2249                }
  2250              },
  2251              "overrides": []
  2252            },
  2253            "gridPos": {
  2254              "h": 6,
  2255              "w": 6,
  2256              "x": 18,
  2257              "y": 13
  2258            },
  2259            "id": 108,
  2260            "options": {
  2261              "legend": {
  2262                "calcs": [],
  2263                "displayMode": "list",
  2264                "placement": "bottom",
  2265                "showLegend": true
  2266              },
  2267              "tooltip": {
  2268                "mode": "single",
  2269                "sort": "none"
  2270              }
  2271            },
  2272            "targets": [
  2273              {
  2274                "datasource": {
  2275                  "type": "prometheus",
  2276                  "uid": "PBFA97CFB590B2093"
  2277                },
  2278                "exemplar": true,
  2279                "expr": "kube_pod_container_resource_limits{exported_namespace=\"$namespace\", container=~\"geth-network|starknetdev\", resource=\"memory\"} / 1024 / 1024",
  2280                "interval": "",
  2281                "legendFormat": "",
  2282                "refId": "A"
  2283              },
  2284              {
  2285                "datasource": {
  2286                  "type": "prometheus",
  2287                  "uid": "PBFA97CFB590B2093"
  2288                },
  2289                "exemplar": true,
  2290                "expr": "container_memory_usage_bytes{namespace=\"$namespace\", container=~\"geth-network|starknetdev\"} / 1024 / 1024",
  2291                "hide": false,
  2292                "interval": "",
  2293                "legendFormat": "",
  2294                "refId": "B"
  2295              }
  2296            ],
  2297            "title": "Geth RAM",
  2298            "transparent": true,
  2299            "type": "timeseries"
  2300          }
  2301        ],
  2302        "repeat": "geth_node",
  2303        "title": "Simulated network metrics ($geth_node)",
  2304        "type": "row"
  2305      },
  2306      {
  2307        "collapsed": true,
  2308        "gridPos": {
  2309          "h": 1,
  2310          "w": 24,
  2311          "x": 0,
  2312          "y": 7
  2313        },
  2314        "id": 76,
  2315        "panels": [
  2316          {
  2317            "description": "Logs of the mockserver",
  2318            "gridPos": {
  2319              "h": 12,
  2320              "w": 18,
  2321              "x": 0,
  2322              "y": 8
  2323            },
  2324            "id": 70,
  2325            "options": {
  2326              "dedupStrategy": "none",
  2327              "enableLogDetails": true,
  2328              "prettifyLogMessage": false,
  2329              "showCommonLabels": false,
  2330              "showLabels": false,
  2331              "showTime": true,
  2332              "sortOrder": "Descending",
  2333              "wrapLogMessage": false
  2334            },
  2335            "pluginVersion": "8.5.0",
  2336            "targets": [
  2337              {
  2338                "datasource": {
  2339                  "type": "loki",
  2340                  "uid": "grafanacloud-logs"
  2341                },
  2342                "expr": "{namespace=\"$namespace\", container=\"mockserver\"}",
  2343                "refId": "A"
  2344              }
  2345            ],
  2346            "title": "Mock Server Logs",
  2347            "transparent": true,
  2348            "type": "logs"
  2349          },
  2350          {
  2351            "datasource": {
  2352              "type": "prometheus",
  2353              "uid": "PBFA97CFB590B2093"
  2354            },
  2355            "description": "Mock Server CPU usage and limit",
  2356            "fieldConfig": {
  2357              "defaults": {
  2358                "color": {
  2359                  "mode": "palette-classic"
  2360                },
  2361                "custom": {
  2362                  "axisLabel": "CPU Power",
  2363                  "axisPlacement": "left",
  2364                  "barAlignment": 0,
  2365                  "drawStyle": "line",
  2366                  "fillOpacity": 5,
  2367                  "gradientMode": "opacity",
  2368                  "hideFrom": {
  2369                    "legend": false,
  2370                    "tooltip": false,
  2371                    "viz": false
  2372                  },
  2373                  "lineInterpolation": "linear",
  2374                  "lineStyle": {
  2375                    "fill": "solid"
  2376                  },
  2377                  "lineWidth": 2,
  2378                  "pointSize": 5,
  2379                  "scaleDistribution": {
  2380                    "type": "linear"
  2381                  },
  2382                  "showPoints": "auto",
  2383                  "spanNulls": false,
  2384                  "stacking": {
  2385                    "group": "A",
  2386                    "mode": "none"
  2387                  },
  2388                  "thresholdsStyle": {
  2389                    "mode": "off"
  2390                  }
  2391                },
  2392                "mappings": [],
  2393                "min": 0,
  2394                "thresholds": {
  2395                  "mode": "absolute",
  2396                  "steps": [
  2397                    {
  2398                      "color": "green",
  2399                      "value": null
  2400                    },
  2401                    {
  2402                      "color": "red",
  2403                      "value": 80
  2404                    }
  2405                  ]
  2406                }
  2407              },
  2408              "overrides": []
  2409            },
  2410            "gridPos": {
  2411              "h": 6,
  2412              "w": 6,
  2413              "x": 18,
  2414              "y": 8
  2415            },
  2416            "id": 71,
  2417            "options": {
  2418              "legend": {
  2419                "calcs": [],
  2420                "displayMode": "list",
  2421                "placement": "bottom",
  2422                "showLegend": true
  2423              },
  2424              "tooltip": {
  2425                "mode": "single",
  2426                "sort": "none"
  2427              }
  2428            },
  2429            "targets": [
  2430              {
  2431                "datasource": {
  2432                  "type": "prometheus",
  2433                  "uid": "PBFA97CFB590B2093"
  2434                },
  2435                "exemplar": true,
  2436                "expr": "kube_pod_container_resource_limits{exported_namespace=\"$namespace\", container=\"mockserver\", resource=\"cpu\"}",
  2437                "interval": "",
  2438                "legendFormat": "",
  2439                "refId": "A"
  2440              },
  2441              {
  2442                "datasource": {
  2443                  "type": "prometheus",
  2444                  "uid": "PBFA97CFB590B2093"
  2445                },
  2446                "exemplar": true,
  2447                "expr": "container_cpu_load_average_10s{namespace=\"$namespace\", container=\"mockserver\"}",
  2448                "hide": false,
  2449                "interval": "",
  2450                "legendFormat": "",
  2451                "refId": "B"
  2452              }
  2453            ],
  2454            "title": "Mock Server CPU",
  2455            "transparent": true,
  2456            "type": "timeseries"
  2457          },
  2458          {
  2459            "datasource": {
  2460              "type": "prometheus",
  2461              "uid": "PBFA97CFB590B2093"
  2462            },
  2463            "description": "Mock Server RAM usage and limit",
  2464            "fieldConfig": {
  2465              "defaults": {
  2466                "color": {
  2467                  "mode": "palette-classic"
  2468                },
  2469                "custom": {
  2470                  "axisLabel": "MiB",
  2471                  "axisPlacement": "auto",
  2472                  "barAlignment": 0,
  2473                  "drawStyle": "line",
  2474                  "fillOpacity": 5,
  2475                  "gradientMode": "opacity",
  2476                  "hideFrom": {
  2477                    "legend": false,
  2478                    "tooltip": false,
  2479                    "viz": false
  2480                  },
  2481                  "lineInterpolation": "linear",
  2482                  "lineWidth": 2,
  2483                  "pointSize": 5,
  2484                  "scaleDistribution": {
  2485                    "type": "linear"
  2486                  },
  2487                  "showPoints": "auto",
  2488                  "spanNulls": false,
  2489                  "stacking": {
  2490                    "group": "A",
  2491                    "mode": "none"
  2492                  },
  2493                  "thresholdsStyle": {
  2494                    "mode": "off"
  2495                  }
  2496                },
  2497                "mappings": [],
  2498                "thresholds": {
  2499                  "mode": "absolute",
  2500                  "steps": [
  2501                    {
  2502                      "color": "green",
  2503                      "value": null
  2504                    },
  2505                    {
  2506                      "color": "red",
  2507                      "value": 80
  2508                    }
  2509                  ]
  2510                }
  2511              },
  2512              "overrides": []
  2513            },
  2514            "gridPos": {
  2515              "h": 6,
  2516              "w": 6,
  2517              "x": 18,
  2518              "y": 14
  2519            },
  2520            "id": 72,
  2521            "options": {
  2522              "legend": {
  2523                "calcs": [],
  2524                "displayMode": "list",
  2525                "placement": "bottom",
  2526                "showLegend": true
  2527              },
  2528              "tooltip": {
  2529                "mode": "single",
  2530                "sort": "none"
  2531              }
  2532            },
  2533            "targets": [
  2534              {
  2535                "datasource": {
  2536                  "type": "prometheus",
  2537                  "uid": "PBFA97CFB590B2093"
  2538                },
  2539                "exemplar": true,
  2540                "expr": "kube_pod_container_resource_limits{exported_namespace=\"$namespace\", container=\"mockserver\", resource=\"memory\"} / 1024 / 1024",
  2541                "interval": "",
  2542                "legendFormat": "",
  2543                "refId": "A"
  2544              },
  2545              {
  2546                "datasource": {
  2547                  "type": "prometheus",
  2548                  "uid": "PBFA97CFB590B2093"
  2549                },
  2550                "exemplar": true,
  2551                "expr": "container_memory_usage_bytes{namespace=\"$namespace\", container=\"mockserver\"} / 1024 / 1024",
  2552                "hide": false,
  2553                "interval": "",
  2554                "legendFormat": "",
  2555                "refId": "B"
  2556              }
  2557            ],
  2558            "title": "Mock Server RAM",
  2559            "transparent": true,
  2560            "type": "timeseries"
  2561          }
  2562        ],
  2563        "title": "Mock Server",
  2564        "type": "row"
  2565      },
  2566      {
  2567        "collapsed": true,
  2568        "gridPos": {
  2569          "h": 1,
  2570          "w": 24,
  2571          "x": 0,
  2572          "y": 8
  2573        },
  2574        "id": 74,
  2575        "panels": [
  2576          {
  2577            "description": "Logs of the instance actually running the tests",
  2578            "gridPos": {
  2579              "h": 12,
  2580              "w": 18,
  2581              "x": 0,
  2582              "y": 9
  2583            },
  2584            "id": 36,
  2585            "options": {
  2586              "dedupStrategy": "none",
  2587              "enableLogDetails": true,
  2588              "prettifyLogMessage": false,
  2589              "showCommonLabels": false,
  2590              "showLabels": false,
  2591              "showTime": false,
  2592              "sortOrder": "Descending",
  2593              "wrapLogMessage": false
  2594            },
  2595            "pluginVersion": "8.5.0",
  2596            "targets": [
  2597              {
  2598                "datasource": {
  2599                  "type": "loki",
  2600                  "uid": "grafanacloud-logs"
  2601                },
  2602                "expr": "{namespace=\"$namespace\", container=\"remote-test-runner\"}",
  2603                "refId": "A"
  2604              }
  2605            ],
  2606            "title": "Test Runner Logs",
  2607            "transparent": true,
  2608            "type": "logs"
  2609          },
  2610          {
  2611            "datasource": {
  2612              "type": "prometheus",
  2613              "uid": "PBFA97CFB590B2093"
  2614            },
  2615            "description": "Test Runner CPU usage and limit",
  2616            "fieldConfig": {
  2617              "defaults": {
  2618                "color": {
  2619                  "mode": "palette-classic"
  2620                },
  2621                "custom": {
  2622                  "axisLabel": "CPU Power",
  2623                  "axisPlacement": "left",
  2624                  "barAlignment": 0,
  2625                  "drawStyle": "line",
  2626                  "fillOpacity": 5,
  2627                  "gradientMode": "opacity",
  2628                  "hideFrom": {
  2629                    "legend": false,
  2630                    "tooltip": false,
  2631                    "viz": false
  2632                  },
  2633                  "lineInterpolation": "linear",
  2634                  "lineStyle": {
  2635                    "fill": "solid"
  2636                  },
  2637                  "lineWidth": 2,
  2638                  "pointSize": 5,
  2639                  "scaleDistribution": {
  2640                    "type": "linear"
  2641                  },
  2642                  "showPoints": "auto",
  2643                  "spanNulls": false,
  2644                  "stacking": {
  2645                    "group": "A",
  2646                    "mode": "none"
  2647                  },
  2648                  "thresholdsStyle": {
  2649                    "mode": "off"
  2650                  }
  2651                },
  2652                "mappings": [],
  2653                "min": 0,
  2654                "thresholds": {
  2655                  "mode": "absolute",
  2656                  "steps": [
  2657                    {
  2658                      "color": "green",
  2659                      "value": null
  2660                    },
  2661                    {
  2662                      "color": "red",
  2663                      "value": 80
  2664                    }
  2665                  ]
  2666                }
  2667              },
  2668              "overrides": []
  2669            },
  2670            "gridPos": {
  2671              "h": 6,
  2672              "w": 6,
  2673              "x": 18,
  2674              "y": 9
  2675            },
  2676            "id": 43,
  2677            "options": {
  2678              "legend": {
  2679                "calcs": [],
  2680                "displayMode": "list",
  2681                "placement": "bottom",
  2682                "showLegend": true
  2683              },
  2684              "tooltip": {
  2685                "mode": "single",
  2686                "sort": "none"
  2687              }
  2688            },
  2689            "targets": [
  2690              {
  2691                "datasource": {
  2692                  "type": "prometheus",
  2693                  "uid": "PBFA97CFB590B2093"
  2694                },
  2695                "exemplar": true,
  2696                "expr": "kube_pod_container_resource_limits{exported_namespace=\"$namespace\", container=\"remote-test-runner\", resource=\"cpu\"}",
  2697                "interval": "",
  2698                "legendFormat": "",
  2699                "refId": "A"
  2700              },
  2701              {
  2702                "datasource": {
  2703                  "type": "prometheus",
  2704                  "uid": "PBFA97CFB590B2093"
  2705                },
  2706                "exemplar": true,
  2707                "expr": "container_cpu_load_average_10s{namespace=\"$namespace\", container=\"remote-test-runner\"}",
  2708                "hide": false,
  2709                "interval": "",
  2710                "legendFormat": "",
  2711                "refId": "B"
  2712              }
  2713            ],
  2714            "title": "Runner CPU",
  2715            "transparent": true,
  2716            "type": "timeseries"
  2717          },
  2718          {
  2719            "datasource": {
  2720              "type": "prometheus",
  2721              "uid": "PBFA97CFB590B2093"
  2722            },
  2723            "description": "Test Runner RAM usage and limit",
  2724            "fieldConfig": {
  2725              "defaults": {
  2726                "color": {
  2727                  "mode": "palette-classic"
  2728                },
  2729                "custom": {
  2730                  "axisLabel": "MiB",
  2731                  "axisPlacement": "auto",
  2732                  "barAlignment": 0,
  2733                  "drawStyle": "line",
  2734                  "fillOpacity": 5,
  2735                  "gradientMode": "opacity",
  2736                  "hideFrom": {
  2737                    "legend": false,
  2738                    "tooltip": false,
  2739                    "viz": false
  2740                  },
  2741                  "lineInterpolation": "linear",
  2742                  "lineWidth": 2,
  2743                  "pointSize": 5,
  2744                  "scaleDistribution": {
  2745                    "type": "linear"
  2746                  },
  2747                  "showPoints": "auto",
  2748                  "spanNulls": false,
  2749                  "stacking": {
  2750                    "group": "A",
  2751                    "mode": "none"
  2752                  },
  2753                  "thresholdsStyle": {
  2754                    "mode": "off"
  2755                  }
  2756                },
  2757                "mappings": [],
  2758                "min": 0,
  2759                "thresholds": {
  2760                  "mode": "absolute",
  2761                  "steps": [
  2762                    {
  2763                      "color": "green",
  2764                      "value": null
  2765                    },
  2766                    {
  2767                      "color": "red",
  2768                      "value": 80
  2769                    }
  2770                  ]
  2771                }
  2772              },
  2773              "overrides": []
  2774            },
  2775            "gridPos": {
  2776              "h": 6,
  2777              "w": 6,
  2778              "x": 18,
  2779              "y": 15
  2780            },
  2781            "id": 44,
  2782            "options": {
  2783              "legend": {
  2784                "calcs": [],
  2785                "displayMode": "list",
  2786                "placement": "bottom",
  2787                "showLegend": true
  2788              },
  2789              "tooltip": {
  2790                "mode": "single",
  2791                "sort": "none"
  2792              }
  2793            },
  2794            "targets": [
  2795              {
  2796                "datasource": {
  2797                  "type": "prometheus",
  2798                  "uid": "PBFA97CFB590B2093"
  2799                },
  2800                "exemplar": true,
  2801                "expr": "kube_pod_container_resource_limits{exported_namespace=\"$namespace\", container=\"remote-test-runner\", resource=\"memory\"} / 1024 / 1024",
  2802                "interval": "",
  2803                "legendFormat": "",
  2804                "refId": "A"
  2805              },
  2806              {
  2807                "datasource": {
  2808                  "type": "prometheus",
  2809                  "uid": "PBFA97CFB590B2093"
  2810                },
  2811                "exemplar": true,
  2812                "expr": "container_memory_usage_bytes{namespace=\"$namespace\", container=\"remote-test-runner\"} / 1024 / 1024",
  2813                "hide": false,
  2814                "interval": "",
  2815                "legendFormat": "",
  2816                "refId": "B"
  2817              }
  2818            ],
  2819            "title": "Runner RAM",
  2820            "transparent": true,
  2821            "type": "timeseries"
  2822          },
  2823          {
  2824            "description": "Errors and Warnings found in Test Runner",
  2825            "gridPos": {
  2826              "h": 5,
  2827              "w": 8,
  2828              "x": 0,
  2829              "y": 21
  2830            },
  2831            "id": 139,
  2832            "options": {
  2833              "dedupStrategy": "none",
  2834              "enableLogDetails": true,
  2835              "prettifyLogMessage": false,
  2836              "showCommonLabels": false,
  2837              "showLabels": false,
  2838              "showTime": false,
  2839              "sortOrder": "Descending",
  2840              "wrapLogMessage": false
  2841            },
  2842            "pluginVersion": "8.5.0",
  2843            "targets": [
  2844              {
  2845                "datasource": {
  2846                  "type": "loki",
  2847                  "uid": "grafanacloud-logs"
  2848                },
  2849                "expr": "{namespace=\"$namespace\", container=\"remote-test-runner\"} |~ \"WRN|ERR\"",
  2850                "refId": "A"
  2851              }
  2852            ],
  2853            "title": "Test Runner Errors and Warnings",
  2854            "transparent": true,
  2855            "type": "logs"
  2856          },
  2857          {
  2858            "description": "Info logs",
  2859            "gridPos": {
  2860              "h": 5,
  2861              "w": 8,
  2862              "x": 8,
  2863              "y": 21
  2864            },
  2865            "id": 140,
  2866            "options": {
  2867              "dedupStrategy": "none",
  2868              "enableLogDetails": true,
  2869              "prettifyLogMessage": false,
  2870              "showCommonLabels": false,
  2871              "showLabels": false,
  2872              "showTime": false,
  2873              "sortOrder": "Descending",
  2874              "wrapLogMessage": false
  2875            },
  2876            "pluginVersion": "8.5.0",
  2877            "targets": [
  2878              {
  2879                "datasource": {
  2880                  "type": "loki",
  2881                  "uid": "grafanacloud-logs"
  2882                },
  2883                "expr": "{namespace=\"$namespace\", container=\"remote-test-runner\"} |= \"INF\"",
  2884                "refId": "A"
  2885              }
  2886            ],
  2887            "title": "Test Runner Info",
  2888            "transparent": true,
  2889            "type": "logs"
  2890          },
  2891          {
  2892            "description": "Debugs",
  2893            "gridPos": {
  2894              "h": 5,
  2895              "w": 8,
  2896              "x": 16,
  2897              "y": 21
  2898            },
  2899            "id": 141,
  2900            "options": {
  2901              "dedupStrategy": "none",
  2902              "enableLogDetails": true,
  2903              "prettifyLogMessage": false,
  2904              "showCommonLabels": false,
  2905              "showLabels": false,
  2906              "showTime": false,
  2907              "sortOrder": "Descending",
  2908              "wrapLogMessage": false
  2909            },
  2910            "pluginVersion": "8.5.0",
  2911            "targets": [
  2912              {
  2913                "datasource": {
  2914                  "type": "loki",
  2915                  "uid": "grafanacloud-logs"
  2916                },
  2917                "expr": "{namespace=\"$namespace\", container=\"remote-test-runner\"} |~ \"DBG\"",
  2918                "refId": "A"
  2919              }
  2920            ],
  2921            "title": "Test Runner Debugs",
  2922            "transparent": true,
  2923            "type": "logs"
  2924          }
  2925        ],
  2926        "title": "Test runner",
  2927        "type": "row"
  2928      }
  2929    ],
  2930    "refresh": "5s",
  2931    "schemaVersion": 34,
  2932    "style": "dark",
  2933    "tags": ["tests"],
  2934    "templating": {
  2935      "list": [
  2936        {
  2937          "current": {
  2938            "selected": false,
  2939            "text": "chainlink-test-env-3735f",
  2940            "value": "chainlink-test-env-3735f"
  2941          },
  2942          "definition": "label_values(namespace)",
  2943          "description": "Namespace",
  2944          "hide": 0,
  2945          "includeAll": false,
  2946          "label": "Namespace",
  2947          "multi": false,
  2948          "name": "namespace",
  2949          "options": [],
  2950          "query": "label_values(namespace)",
  2951          "refresh": 1,
  2952          "regex": "chainlink-.*|.*-soak-.*|.*performance-.*|.*-chaos-.*",
  2953          "skipUrlSync": false,
  2954          "sort": 0,
  2955          "type": "query"
  2956        },
  2957        {
  2958          "current": {
  2959            "selected": true,
  2960            "text": ["chainlink-0-98744888d-r9n6r"],
  2961            "value": ["chainlink-0-98744888d-r9n6r"]
  2962          },
  2963          "definition": "label_values({namespace=\"$namespace\"}, pod)",
  2964          "description": "The specific Chainlink node to get insight into",
  2965          "hide": 0,
  2966          "includeAll": false,
  2967          "label": "Chainlink Node",
  2968          "multi": true,
  2969          "name": "cl_node",
  2970          "options": [],
  2971          "query": "label_values({namespace=\"$namespace\"}, pod)",
  2972          "refresh": 1,
  2973          "regex": "chainlink-.*",
  2974          "skipUrlSync": false,
  2975          "sort": 0,
  2976          "type": "query"
  2977        },
  2978        {
  2979          "current": {
  2980            "selected": false,
  2981            "text": "geth-6469b5b5f8-smd2z",
  2982            "value": "geth-6469b5b5f8-smd2z"
  2983          },
  2984          "definition": "label_values({namespace=\"$namespace\"}, pod)",
  2985          "description": "If launched, a simulated instance of Geth",
  2986          "hide": 0,
  2987          "includeAll": false,
  2988          "label": "Simulated Geth Node",
  2989          "multi": false,
  2990          "name": "geth_node",
  2991          "options": [],
  2992          "query": "label_values({namespace=\"$namespace\"}, pod)",
  2993          "refresh": 1,
  2994          "regex": "geth-|starknet.*",
  2995          "skipUrlSync": false,
  2996          "sort": 0,
  2997          "type": "query"
  2998        }
  2999      ]
  3000    },
  3001    "time": {
  3002      "from": "now-30m",
  3003      "to": "now"
  3004    },
  3005    "timepicker": {},
  3006    "timezone": "",
  3007    "title": "Chainlink Testing Insights",
  3008    "uid": "AFATC9A7k",
  3009    "version": 1,
  3010    "weekStart": ""
  3011  }