github.com/redhat-appstudio/e2e-tests@v0.0.0-20240520140907-9709f6f59323/tests/load-tests/cluster_read_config.yaml

- name: measurements.tekton_pipelines_controller_running_pipelineruns_count
  monitoring_query: sum(tekton_pipelines_controller_running_pipelineruns_count)
  monitoring_step: 15

- name: measurements.storage_count_attachable_volumes_in_use
  monitoring_query: sum(storage_count_attachable_volumes_in_use)
  monitoring_step: 15

- name: measurements.cluster_cpu_usage_seconds_total_rate
  monitoring_query: sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=""})
  monitoring_step: 15

- name: measurements.cluster_memory_usage_rss_total
  monitoring_query: sum(container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", cluster="", container!=""})
  monitoring_step: 15

- name: measurements.cluster_disk_throughput_total
  monitoring_query: sum (rate(container_fs_reads_bytes_total{id!="", device=~"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+", cluster=""}[5m]) + rate(container_fs_writes_bytes_total{id!="", device=~"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+", cluster=""}[5m]))
  monitoring_step: 15

- name: measurements.token_pool_rate_primary
  monitoring_query: sum(rate(token_pool_gauge{rateLimited="primary"}[5m]))
  monitoring_step: 15

- name: measurements.token_pool_rate_secondary
  monitoring_query: sum(rate(token_pool_gauge{rateLimited="secondary"}[5m]))
  monitoring_step: 15

- name: measurements.cluster_nodes_worker_count
  monitoring_query: count(kube_node_role{role="worker"})
  monitoring_step: 15

- name: measurements.cluster_pods_count
  monitoring_query: count(kube_pod_info)
  monitoring_step: 15

- name: measurements.cluster_running_pods_on_workers_count
  monitoring_query: count(kube_pod_info * on(node) group_left(role) kube_node_role{role="worker"} and on(pod, namespace) (kube_pod_status_phase{job="kube-state-metrics", phase="Running"} > 0))
  monitoring_step: 15

- name: measurements.scheduler_pending_pods_count
  monitoring_query: sum(scheduler_pending_pods)
  monitoring_step: 15

- name: measurements.tekton_tekton_pipelines_controller_workqueue_depth
  monitoring_query: sum(tekton_pipelines_controller_workqueue_depth)
  monitoring_step: 15

- name: measurements.pipelinerun_duration_scheduled_seconds
  monitoring_query: sum(pipelinerun_duration_scheduled_seconds_sum / pipelinerun_duration_scheduled_seconds_count)
  monitoring_step: 15

- name: measurements.tekton_pipelines_controller_running_taskruns_throttled_by_node
  monitoring_query: sum(tekton_pipelines_controller_running_taskruns_throttled_by_node_count)
  monitoring_step: 15

- name: measurements.tekton_pipelines_controller_running_taskruns_throttled_by_quota
  monitoring_query: sum(tekton_pipelines_controller_running_taskruns_throttled_by_quota_count)
  monitoring_step: 15

- name: measurements.etcd_request_duration_seconds_average
  monitoring_query: sum(rate(etcd_request_duration_seconds_sum{}[5m])) / sum(rate(etcd_request_duration_seconds_count[5m]))
  monitoring_step: 15

- name: measurements.cluster_network_bytes_total
  monitoring_query: sum(irate(container_network_receive_bytes_total{cluster="",namespace=~".*"}[5m])) + sum(irate(container_network_transmit_bytes_total{cluster="",namespace=~".*"}[5m]))
  monitoring_step: 15

- name: measurements.cluster_network_receive_bytes_total
  monitoring_query: sum(irate(container_network_receive_bytes_total{cluster="",namespace=~".*"}[5m]))
  monitoring_step: 15

- name: measurements.cluster_network_transmit_bytes_total
  monitoring_query: sum(irate(container_network_transmit_bytes_total{cluster="",namespace=~".*"}[5m]))
  monitoring_step: 15

- name: measurements.node_disk_io_time_seconds_total
  monitoring_query: sum(irate(node_disk_io_time_seconds_total{cluster="",namespace=~".*"}[5m]))
  monitoring_step: 15

# redhat-appstudio metrics
# Availability of GitHub app
- name: measurements.redhat_appstudio_buildservice_global_github_app_available
  monitoring_query: sum(redhat_appstudio_buildservice_global_github_app_available)
  monitoring_step: 15

# Component creation til simple build pipeline submision or PaC provision in seconds
- name: measurements.redhat_appstudio_buildservice_component_onboarding_time_sum
  monitoring_query: sum(redhat_appstudio_buildservice_component_onboarding_time_sum)
  monitoring_step: 15

# Image repository provision to ready to use in seconds
- name: measurements.redhat_appstudio_imagecontroller_image_repository_provision_time_sum
  monitoring_query: sum(redhat_appstudio_imagecontroller_image_repository_provision_time_sum)
  monitoring_step: 15

# Interesting CI environment variables
{% for var in [
  'BUILD_ID',
  'HOSTNAME',
  'JOB_NAME',
  'OPENSHIFT_API',
  'PROW_JOB_ID',
  'PULL_BASE_REF',
  'PULL_BASE_SHA',
  'PULL_HEAD_REF',
  'PULL_NUMBER',
  'PULL_PULL_SHA',
  'PULL_REFS',
  'REPO_NAME',
  'REPO_OWNER',
  'SCENARIO',
] %}
- name: metadata.env.{{ var }}
  env_variable: {{ var }}
{% endfor %}

# Git info
{% macro git_info(dir, path) -%}
- name: metadata.git.{{ path }}.commit.hash
  command: cd "{{ dir }}" && git log -1 --pretty=format:"%H"
- name: metadata.git.{{ path }}.commit.abbreviated_hash
  command: cd "{{ dir }}" && git log -1 --pretty=format:"%h"
- name: metadata.git.{{ path }}.commit.author_date
  command: cd "{{ dir }}" && git log -1 --pretty=format:"%aI"
- name: metadata.git.{{ path }}.commit.committer_date
  command: cd "{{ dir }}" && git log -1 --pretty=format:"%cI"
- name: metadata.git.{{ path }}.commit.subject
  command: cd "{{ dir }}" && git log -1 --pretty=format:"%s"
- name: metadata.git.{{ path }}.commit.author_name
  command: cd "{{ dir }}" && git log -1 --pretty=format:"%aN"
- name: metadata.git.{{ path }}.commit.author_email
  command: cd "{{ dir }}" && git log -1 --pretty=format:"%aE"
{%- endmacro %}
{{ git_info('.', 'redhat_appstudio.e2e_tests') }}
{{ git_info('tmp/infra-deployments', 'redhat_appstudio.infra_deployments') }}

# Cluster version
- name: metadata.cluster.versions
  command: oc version -o json
  output: json

# Cluster nodes info
- name: metadata.cluster.control-plane.count
  command: oc get nodes -l node-role.kubernetes.io/master -o name | wc -l

- name: metadata.cluster.control-plane.flavor
  command: oc get nodes -l node-role.kubernetes.io/master -o json | jq --raw-output '.items | map(.metadata.labels."beta.kubernetes.io/instance-type") | unique | sort | join(",")'

- name: metadata.cluster.control-plane.nodes
  command: oc get nodes -l node-role.kubernetes.io/master -o json | jq '.items | map(.metadata.name)'
  output: json

- name: metadata.cluster.compute-nodes.count
  command: oc get nodes -l node-role.kubernetes.io/worker -o name | wc -l

- name: metadata.cluster.compute-nodes.flavor
  command: oc get nodes -l node-role.kubernetes.io/worker -o json | jq --raw-output '.items | map(.metadata.labels."beta.kubernetes.io/instance-type") | unique | sort | join(",")'

- name: metadata.cluster.compute-nodes.nodes
  command: oc get nodes -l node-role.kubernetes.io/worker -o json | jq '.items | map(.metadata.name)'
  output: json

- name: metadata.scenario
  command: if [ -r /usr/local/ci-secrets/redhat-appstudio-load-test/load-test-scenario.${SCENARIO} ]; then cat /usr/local/ci-secrets/redhat-appstudio-load-test/load-test-scenario.${SCENARIO} | sed 's/\\ /,/g' | sed 's/[^ ]* \([^= ]*\)=\([^= ]*\)/"\1":"\2",/g' | sed 's/\(.*\),$/{\1}/g'; else echo '{}'; fi
  output: json

{% macro monitor_pod(namespace, pod, step=15, pod_suffix_regex='-[0-9a-f]+-.*') -%}
# Gather monitoring data about the pod
- name: measurements.{{ pod }}.cpu
  monitoring_query: sum(pod:container_cpu_usage:sum{namespace='{{ namespace }}', pod=~'{{ pod }}{{ pod_suffix_regex }}'})
  monitoring_step: {{ step }}
- name: measurements.{{ pod }}.memory
  monitoring_query: sum(container_memory_usage_bytes{namespace='{{ namespace }}', pod=~'{{ pod }}{{ pod_suffix_regex }}', container!='POD', container!=''})
  monitoring_step: {{ step }}
- name: measurements.{{ pod }}.network_throughput
  monitoring_query: sum( rate(container_network_transmit_bytes_total{namespace='{{ namespace }}', pod=~'{{ pod }}{{ pod_suffix_regex }}'}[{{ step * 4 }}s]) + rate(container_network_receive_bytes_total{namespace='{{ namespace }}', pod=~'{{ pod }}{{ pod_suffix_regex }}'}[{{ step * 4 }}s]) )
  monitoring_step: {{ step * 4 }}
- name: measurements.{{ pod }}.network_drop
  monitoring_query: sum( rate(container_network_transmit_packets_dropped_total{namespace='{{ namespace }}', pod=~'{{ pod }}{{ pod_suffix_regex }}'}[{{ step * 4 }}s]) + rate(container_network_receive_packets_dropped_total{namespace='{{ namespace }}', pod=~'{{ pod }}{{ pod_suffix_regex }}'}[{{ step * 4 }}s]) )
  monitoring_step: {{ step * 4 }}
- name: measurements.{{ pod }}.disk_throughput
  monitoring_query: sum( sum(rate(container_fs_reads_bytes_total{namespace='{{ namespace }}', pod=~'{{ pod }}{{ pod_suffix_regex }}', device!='/dev/dm-0'}[{{ step * 4 }}s])) + sum(rate(container_fs_writes_bytes_total{namespace='{{ namespace }}', pod=~'{{ pod }}{{ pod_suffix_regex }}', device!='/dev/dm-0'}[{{ step * 4 }}s])) )
  monitoring_step: {{ step * 4 }}
- name: measurements.{{ pod }}.restarts
  monitoring_query: sum(kube_pod_container_status_restarts_total{namespace='{{ namespace }}', pod=~'{{ pod }}{{ pod_suffix_regex }}'})
  monitoring_step: {{ step }}
- name: measurements.{{ pod }}.count_ready
  monitoring_query: sum( kube_pod_status_ready{namespace='{{ namespace }}', pod=~'{{ pod }}{{ pod_suffix_regex }}'} )
  monitoring_step: {{ step }}
{%- endmacro %}

{% macro monitor_pod_container(namespace, pod, container, step=15, pod_suffix_regex='-[0-9a-f]+-.*') -%}
# Gather monitoring data about the pod's container
- name: measurements.{{ pod }}.container[{{ container }}].memory
  monitoring_query: sum(container_memory_usage_bytes{namespace='{{ namespace }}', pod=~'{{ pod }}{{ pod_suffix_regex }}', container='{{container}}'})
  monitoring_step: {{ step }}
{%- endmacro %}

{{ monitor_pod('openshift-pipelines', 'tekton-pipelines-controller', 15) }}
{{ monitor_pod('tekton-results', 'tekton-results-watcher', 1, '-.*') }}
{{ monitor_pod_container('tekton-results', 'tekton-results-watcher', 'watcher', 1, '-.*') }}