github.com/percona/percona-xtradb-cluster-operator@v1.14.0/e2e-tests/monitoring-2-0/run (about)

     1  #!/bin/bash
     2  
     3  set -o errexit
     4  
     5  test_dir=$(realpath $(dirname $0))
     6  . ${test_dir}/../functions
     7  
     8  set_debug
     9  
    10  get_mgmnt_service_list() {
    11  	local api_key=$1
    12  	local endpoint=$2
    13  	local namespace=$3
    14  
    15  	curl -s -k -H "Authorization: Bearer ${api_key}" -X POST "https://$endpoint/v1/management/Service/List" \
    16  		| jq 'walk(if type=="object" then with_entries(select(.key | test("service_id|node_id|agent_id|created_at|updated_at") | not)) else . end)' \
    17  		| jq 'walk(if type == "array" then sort_by(.agent_type) else . end)' \
    18  			>${tmp_dir}/active_pmm_agents.json
    19  
    20  	# Pmm server is serving one namespace only, so stripping the namespace name out
    21  	$sed -i 's/'${namespace}'-//g' ${tmp_dir}/active_pmm_agents.json
    22  	cat ${tmp_dir}/active_pmm_agents.json | jq '.services | sort_by(.node_name)' >${tmp_dir}/active_pmm_agents_sorted.json
    23  
    24  	echo "${tmp_dir}/active_pmm_agents_sorted.json"
    25  }
    26  
    27  get_node_id_from_pmm() {
    28  	local -a nodeList=()
    29  	for instance in $(kubectl_bin get pods --no-headers -l app.kubernetes.io/component=pxc --output=custom-columns='NAME:.metadata.name'); do
    30  		nodeList+=($(kubectl_bin exec -n "$namespace" $instance -c pmm-client -- pmm-admin status --json | jq -r '.pmm_agent_status.node_id'))
    31  	done
    32  
    33  	echo "${nodeList[@]}"
    34  }
    35  
    36  does_node_id_exists() {
    37  	local -a nodeList=("$@")
    38  	local -a nodeList_from_pmm=()
    39  	for node_id in "${nodeList[@]}"; do
    40  		nodeList_from_pmm+=($(kubectl_bin exec -n "${namespace}" monitoring-0 -- pmm-admin --server-url=https://admin:admin@$(get_service_ip monitoring-service)/ --server-insecure-tls inventory list nodes --node-type=CONTAINER_NODE | grep $node_id | awk '{print $4}'))
    41  	done
    42  
    43  	echo "${nodeList_from_pmm[@]}"
    44  }
    45  
    46  cluster="monitoring"
    47  
    48  create_infra $namespace
    49  deploy_helm $namespace
    50  
    51  desc 'install PMM Server'
    52  deploy_pmm_server
    53  kubectl_bin wait --for=condition=Ready pod/${cluster}-0 --timeout=120s
    54  until kubectl_bin exec monitoring-0 -- bash -c "ls -l /proc/*/exe 2>/dev/null| grep postgres >/dev/null"; do
    55  	echo "Retry $retry"
    56  	sleep 5
    57  	let retry+=1
    58  	if [ $retry -ge 20 ]; then
    59  		echo "Max retry count $retry reached. Pmm-server can't start"
    60  		exit 1
    61  	fi
    62  done
    63  ADMIN_PASSWORD=$(kubectl_bin exec monitoring-0 -- bash -c "printenv | grep ADMIN_PASSWORD | cut -d '=' -f2")
    64  sleep 5
    65  kubectl_bin exec monitoring-0 -- bash -c "grafana-cli --homepath=/usr/share/grafana --config=/etc/grafana/grafana.ini admin reset-admin-password $ADMIN_PASSWORD"
    66  
    67  desc 'create PXC cluster'
    68  spinup_pxc "$cluster" "$test_dir/conf/$cluster.yml" 3 120 "$test_dir/conf/secrets.yaml"
    69  
    70  desc 'add PMM API key to secret'
    71  API_KEY=$(curl --insecure -X POST -H "Content-Type: application/json" -d '{"name":"operator", "role": "Admin"}' "https://admin:$ADMIN_PASSWORD@$(get_service_endpoint monitoring-service)/graph/api/auth/keys" | jq .key)
    72  kubectl_bin patch secret my-cluster-secrets --type merge --patch '{"stringData": {"pmmserverkey": '"$API_KEY"'}}'
    73  
    74  wait_for_generation "sts/$cluster-pxc" 2
    75  wait_for_generation "sts/$cluster-haproxy" 2
    76  sleep 10
    77  kubectl wait pod -l 'app.kubernetes.io/managed-by=percona-xtradb-cluster-operator' --for=condition=ready --timeout=600s
    78  wait_cluster_consistency ${cluster} 3 2
    79  
    80  compare_kubectl statefulset/$cluster-pxc "-no-prefix"
    81  compare_kubectl statefulset/$cluster-haproxy "-no-prefix"
    82  
    83  desc 'apply my-env-var-secrets to add PMM_PREFIX'
    84  kubectl_bin apply -f "$test_dir/conf/envsecrets.yaml"
    85  
    86  desc 'add new PMM API key to secret'
    87  API_KEY_NEW=$(curl --insecure -X POST -H "Content-Type: application/json" -d '{"name":"operator-new", "role": "Admin"}' "https://admin:$ADMIN_PASSWORD@$(get_service_endpoint monitoring-service)/graph/api/auth/keys" | jq .key)
    88  kubectl_bin patch secret my-cluster-secrets --type merge --patch '{"stringData": {"pmmserverkey": '"$API_KEY_NEW"'}}'
    89  
    90  desc 'delete old PMM key'
    91  ID_API_KEY_OLD=$(curl --insecure -X GET "https://admin:$ADMIN_PASSWORD@$(get_service_endpoint monitoring-service)/graph/api/auth/keys" | jq '.[] | select( .name == "operator").id')
    92  curl --insecure -X DELETE "https://admin:$ADMIN_PASSWORD@$(get_service_endpoint monitoring-service)/graph/api/auth/keys/$ID_API_KEY_OLD"
    93  
    94  wait_for_generation "sts/$cluster-pxc" 3
    95  wait_for_generation "sts/$cluster-haproxy" 3
    96  sleep 10
    97  kubectl wait pod -l 'app.kubernetes.io/managed-by=percona-xtradb-cluster-operator' --for=condition=ready --timeout=600s
    98  
    99  desc 'check if pmm-client container enabled'
   100  compare_kubectl statefulset/$cluster-pxc
   101  compare_kubectl statefulset/$cluster-haproxy
   102  
   103  desc 'verify clients agents statuses'
   104  # wait for proper agents statuses
   105  sleep 300
   106  API_KEY=$(getSecretData "my-cluster-secrets" "pmmserverkey")
   107  diff -u ${test_dir}/compare/agents-list.json $(get_mgmnt_service_list ${API_KEY} $(get_service_endpoint monitoring-service) ${namespace})
   108  
   109  desc 'check mysql metrics'
   110  get_metric_values node_boot_time_seconds pxc-prefix-$namespace-$cluster-pxc-0 admin:admin
   111  get_metric_values mysql_global_status_uptime pxc-prefix-$namespace-$cluster-pxc-0 admin:admin
   112  
   113  desc 'check haproxy metrics'
   114  get_metric_values haproxy_backend_status pxc-prefix-$namespace-$cluster-haproxy-0 admin:admin
   115  get_metric_values haproxy_backend_active_servers pxc-prefix-$namespace-$cluster-haproxy-0 admin:admin
   116  
   117  desc 'check QAN data'
   118  get_qan20_values $cluster-pxc-0 admin:admin
   119  
   120  nodeList=($(get_node_id_from_pmm))
   121  nodeList_from_pmm=($(does_node_id_exists "${nodeList[@]}"))
   122  for node_id in "${nodeList_from_pmm[@]}"; do
   123  	if [ -z "$node_id" ]; then
   124  		echo "Can't get $node_id node_id from PMM server"
   125  		exit 1
   126  	fi
   127  done
   128  
   129  kubectl_bin patch pxc ${cluster} --type json -p='[{"op":"add","path":"/spec/pause","value":true}]'
   130  wait_for_delete "pod/${cluster}-pxc-0"
   131  
   132  does_node_id_exists_in_pmm=($(does_node_id_exists "${nodeList[@]}"))
   133  for instance in "${does_node_id_exists_in_pmm[@]}"; do
   134  	if [ -n "$instance" ]; then
   135  		echo "The $instance pod was not deleted from server inventory"
   136  		exit 1
   137  	fi
   138  done
   139  
   140  if [[ -n ${OPENSHIFT} ]]; then
   141  	oc adm policy remove-scc-from-user privileged -z pmm-server
   142  	if [ -n "$OPERATOR_NS" ]; then
   143  		oc delete clusterrolebinding pmm-pxc-operator-cluster-wide
   144  	else
   145  		oc delete rolebinding pmm-pxc-operator-namespace-only
   146  	fi
   147  fi
   148  
   149  desc 'check for passwords leak'
   150  check_passwords_leak
   151  
   152  helm uninstall monitoring
   153  destroy $namespace
   154  desc "test passed"