github.com/redhat-appstudio/e2e-tests@v0.0.0-20230619105049-9a422b2094d7/tests/load-tests/ci-scripts/collect-results.sh (about) 1 #!/bin/bash 2 3 set -o nounset 4 set -o errexit 5 set -o pipefail 6 7 # shellcheck disable=SC1090 8 source "/usr/local/ci-secrets/redhat-appstudio-load-test/load-test-scenario.${1:-concurrent}" 9 10 pushd "${2:-.}" 11 12 echo "Collecting load test results" 13 load_test_log=$ARTIFACT_DIR/load-tests.log 14 cp -vf ./tests/load-tests/load-tests.log "$load_test_log" 15 cp -vf ./tests/load-tests/load-tests.json "$ARTIFACT_DIR" 16 cp -vf ./tests/load-tests/gh-rate-limits-remaining.csv "$ARTIFACT_DIR" 17 18 application_timestamps=$ARTIFACT_DIR/applications.appstudio.redhat.com_timestamps 19 application_timestamps_csv=${application_timestamps}.csv 20 application_timestamps_txt=${application_timestamps}.txt 21 componentdetectionquery_timestamps=$ARTIFACT_DIR/componentdetectionqueries.appstudio.redhat.com_timestamps.csv 22 component_timestamps=$ARTIFACT_DIR/components.appstudio.redhat.com_timestamps.csv 23 pipelinerun_timestamps=$ARTIFACT_DIR/pipelineruns.tekton.dev_timestamps.csv 24 application_service_log=$ARTIFACT_DIR/application-service.log 25 application_service_log_segments=$ARTIFACT_DIR/application-service-log-segments 26 monitoring_collection_log=$ARTIFACT_DIR/monitoring-collection.log 27 monitoring_collection_data=$ARTIFACT_DIR/load-tests.json 28 csv_delim=";" 29 csv_delim_quoted="\"$csv_delim\"" 30 dt_format='"%Y-%m-%dT%H:%M:%SZ"' 31 ## Application timestamps 32 echo "Collecting Application timestamps..." 33 echo "Application${csv_delim}StatusSucceeded${csv_delim}StatusMessage${csv_delim}CreatedTimestamp${csv_delim}SucceededTimestamp${csv_delim}Duration" >"$application_timestamps_csv" 34 jq_cmd=".items[] | (.metadata.name) \ 35 + $csv_delim_quoted + (.status.conditions[0].status) \ 36 + $csv_delim_quoted + (.status.conditions[0].message) \ 37 + $csv_delim_quoted + (.metadata.creationTimestamp) \ 38 + $csv_delim_quoted + (.status.conditions[0].lastTransitionTime) \ 39 + $csv_delim_quoted + ((.status.conditions[0].lastTransitionTime | strptime($dt_format) | mktime) - (.metadata.creationTimestamp | strptime($dt_format) | mktime) | tostring)" 40 oc get applications.appstudio.redhat.com -A -o json | jq -rc "$jq_cmd" | sed -e 's,Z,,g' >>"$application_timestamps_csv" 41 oc get applications.appstudio.redhat.com -A -o 'custom-columns=NAME:.metadata.name,CREATED:.metadata.creationTimestamp,LAST_UPDATED:.status.conditions[0].lastTransitionTime,STATUS:.status.conditions[0].reason,MESSAGE:.status.conditions[0].message' >"$application_timestamps_txt" 42 43 ## ComponentDetectionQuery timestamps 44 echo "Collecting ComponentDetectionQuery timestamps..." 45 echo "ComponentDetectionQuery${csv_delim}Namespace${csv_delim}CreationTimestamp${csv_delim}Completed${csv_delim}Completed.Reason${csv_delim}Completed.Mesasge${csv_delim}Duration" >"$componentdetectionquery_timestamps" 46 jq_cmd=".items[] | (.metadata.name) \ 47 + $csv_delim_quoted + (.metadata.namespace) \ 48 + $csv_delim_quoted + (.metadata.creationTimestamp) \ 49 + $csv_delim_quoted + (if ((.status.conditions[] | select(.type == \"Completed\")) // false) then (.status.conditions[] | select(.type == \"Completed\") | .lastTransitionTime + $csv_delim_quoted + .reason + $csv_delim_quoted + .message) else \"$csv_delim$csv_delim\" end)\ 50 + $csv_delim_quoted + (if ((.status.conditions[] | select(.type == \"Completed\")) // false) then ((.status.conditions[] | select(.type == \"Completed\") | .lastTransitionTime | strptime($dt_format) | mktime) - (.metadata.creationTimestamp | strptime($dt_format) | mktime) | tostring) else \"\" end)" 51 oc get componentdetectionqueries.appstudio.redhat.com -A -o json | jq -rc "$jq_cmd" | sed -e 's,Z,,g' >>"$componentdetectionquery_timestamps" 52 53 ## Component timestamps 54 echo "Collecting Component timestamps..." 55 echo "Component${csv_delim}Namespace${csv_delim}CreationTimestamp${csv_delim}Created${csv_delim}Created.Reason${csv_delim}Create.Mesasge${csv_delim}GitOpsResourcesGenerated${csv_delim}GitOpsResourcesGenerated.Reason${csv_delim}GitOpsResourcesGenerated.Message${csv_delim}Updated${csv_delim}Updated.Reason${csv_delim}Updated.Message${csv_delim}CreationTimestamp->Created${csv_delim}Created->GitOpsResourcesGenerated${csv_delim}GitOpsResourcesGenerated->Updated${csv_delim}Duration" >"$component_timestamps" 56 jq_cmd=".items[] | (.metadata.name) \ 57 + $csv_delim_quoted + (.metadata.namespace) \ 58 + $csv_delim_quoted + (.metadata.creationTimestamp) \ 59 + $csv_delim_quoted + (if ((.status.conditions[] | select(.type == \"Created\")) // false) then (.status.conditions[] | select(.type == \"Created\") | .lastTransitionTime + $csv_delim_quoted + .reason + $csv_delim_quoted + (.message|split($csv_delim_quoted)|join(\"_\"))) else \"$csv_delim$csv_delim\" end) \ 60 + $csv_delim_quoted + (if ((.status.conditions[] | select(.type == \"GitOpsResourcesGenerated\")) // false) then (.status.conditions[] | select(.type == \"GitOpsResourcesGenerated\") | .lastTransitionTime + $csv_delim_quoted + .reason + $csv_delim_quoted + (.message|split($csv_delim_quoted)|join(\"_\"))) else \"$csv_delim$csv_delim\" end) \ 61 + $csv_delim_quoted + (if ((.status.conditions[] | select(.type == \"Updated\")) // false) then (.status.conditions[] | select(.type == \"Updated\") | .lastTransitionTime + $csv_delim_quoted + .reason + $csv_delim_quoted + (.message|split($csv_delim_quoted)|join(\"_\"))) else \"$csv_delim$csv_delim\" end) \ 62 + $csv_delim_quoted + (if ((.status.conditions[] | select(.type == \"Created\")) // false) then ((.status.conditions[] | select(.type == \"Created\") | .lastTransitionTime | strptime($dt_format) | mktime) - (.metadata.creationTimestamp | strptime($dt_format) | mktime) | tostring) else \"\" end)\ 63 + $csv_delim_quoted + (if ((.status.conditions[] | select(.type == \"GitOpsResourcesGenerated\")) // false) and ((.status.conditions[] | select(.type == \"Created\")) // false) then ((.status.conditions[] | select(.type == \"GitOpsResourcesGenerated\") | .lastTransitionTime | strptime($dt_format) | mktime) - (.status.conditions[] | select(.type == \"Created\") | .lastTransitionTime | strptime($dt_format) | mktime) | tostring) else \"\" end) \ 64 + $csv_delim_quoted + (if ((.status.conditions[] | select(.type == \"Updated\")) // false) and ((.status.conditions[] | select(.type == \"GitOpsResourcesGenerated\")) // false) then ((.status.conditions[] | select(.type == \"Updated\") | .lastTransitionTime | strptime($dt_format) | mktime) - (.status.conditions[] | select(.type == \"GitOpsResourcesGenerated\") | .lastTransitionTime | strptime($dt_format) | mktime) | tostring) else \"\" end) \ 65 + $csv_delim_quoted + (if ((.status.conditions[] | select(.type == \"Updated\")) // false) then ((.status.conditions[] | select(.type == \"Updated\") | .lastTransitionTime | strptime($dt_format) | mktime) - (.metadata.creationTimestamp | strptime($dt_format) | mktime) | tostring) else \"\" end)" 66 oc get components.appstudio.redhat.com -A -o json | jq -rc "$jq_cmd" | sed -e 's,Z,,g' >>"$component_timestamps" 67 68 ## PipelineRun timestamps 69 echo "Collecting PipelineRun timestamps..." 70 echo "PipelineRun${csv_delim}Namespace${csv_delim}Succeeded${csv_delim}Reason${csv_delim}Message${csv_delim}Created${csv_delim}Started${csv_delim}FinallyStarted${csv_delim}Completed${csv_delim}Created->Started${csv_delim}Started->FinallyStarted${csv_delim}FinallyStarted->Completed${csv_delim}SucceededDuration${csv_delim}FailedDuration" >"$pipelinerun_timestamps" 71 jq_cmd=".items[] | (.metadata.name) \ 72 + $csv_delim_quoted + (.metadata.namespace) \ 73 + $csv_delim_quoted + (.status.conditions[0].status) \ 74 + $csv_delim_quoted + (.status.conditions[0].reason) \ 75 + $csv_delim_quoted + (.status.conditions[0].message|split($csv_delim_quoted)|join(\"_\")) \ 76 + $csv_delim_quoted + (.metadata.creationTimestamp) \ 77 + $csv_delim_quoted + (.status.startTime) \ 78 + $csv_delim_quoted + (.status.finallyStartTime) \ 79 + $csv_delim_quoted + (.status.completionTime) \ 80 + $csv_delim_quoted + (if .status.startTime != null and .metadata.creationTimestamp != null then ((.status.startTime | strptime($dt_format) | mktime) - (.metadata.creationTimestamp | strptime($dt_format) | mktime) | tostring) else \"\" end) \ 81 + $csv_delim_quoted + (if .status.finallyStartTime != null and .status.startTime != null then ((.status.finallyStartTime | strptime($dt_format) | mktime) - (.status.startTime | strptime($dt_format) | mktime) | tostring) else \"\" end) \ 82 + $csv_delim_quoted + (if .status.completionTime != null and .status.finallyStartTime != null then ((.status.completionTime | strptime($dt_format) | mktime) - (.status.finallyStartTime | strptime($dt_format) | mktime) | tostring) else \"\" end) \ 83 + $csv_delim_quoted + (if .status.conditions[0].status == \"True\" and .status.completionTime != null and .metadata.creationTimestamp != null then ((.status.completionTime | strptime($dt_format) | mktime) - (.metadata.creationTimestamp | strptime($dt_format) | mktime) | tostring) else \"\" end) \ 84 + $csv_delim_quoted + (if .status.conditions[0].status == \"False\" and .status.completionTime != null and .metadata.creationTimestamp != null then ((.status.completionTime | strptime($dt_format) | mktime) - (.metadata.creationTimestamp | strptime($dt_format) | mktime) | tostring) else \"\" end)" 85 oc get pipelineruns.tekton.dev -A -o json | jq "$jq_cmd" | sed -e "s/\n//g" -e "s/^\"//g" -e "s/\"$//g" -e "s/Z;/;/g" >>"$pipelinerun_timestamps" 86 87 ## Application service log segments per user app 88 echo "Collecting application service log segments per user app..." 89 oc logs -l "control-plane=controller-manager" --tail=-1 -n application-service >"$application_service_log" 90 mkdir -p "$application_service_log_segments" 91 for i in $(grep -Eo "${USER_PREFIX}-....-app" "$application_service_log" | sort | uniq); do grep "$i" "$application_service_log" >"$application_service_log_segments/$i.log"; done 92 ## Error summary 93 echo "Error summary:" 94 if [ -f "$load_test_log" ]; then 95 grep -Eo "Error #[0-9]+" "$load_test_log" | sort | uniq | while read -r i; do 96 echo -n " - $i: " 97 grep -c "$i" "$load_test_log" 98 done | sort -V || : 99 else 100 echo "File $load_test_log does not exist!" 101 exit 1 102 fi 103 104 ## Monitoring data 105 echo "Setting up tool to collect monitoring data..." 106 python3 -m venv venv 107 set +u; source venv/bin/activate; set -u 108 python3 -m pip install -U pip 109 python3 -m pip install -U pip 110 python3 -m pip install -e "git+https://github.com/redhat-performance/opl.git#egg=opl-rhcloud-perf-team-core&subdirectory=core" 111 112 echo "Collecting monitoring data..." 113 mstart=$( date --utc --date "$( status_data.py --status-data-file "$monitoring_collection_data" --get timestamp )" --iso-8601=seconds ) 114 mend=$( date --utc --date "$( status_data.py --status-data-file "$monitoring_collection_data" --get endTimestamp )" --iso-8601=seconds ) 115 mhost=$( oc -n openshift-monitoring get route -l app.kubernetes.io/name=thanos-query -o json | jq --raw-output '.items[0].spec.host' ) 116 status_data.py \ 117 --status-data-file "$monitoring_collection_data" \ 118 --additional ./tests/load-tests/cluster_read_config.yaml \ 119 --monitoring-start "$mstart" \ 120 --monitoring-end "$mend" \ 121 --prometheus-host "https://$mhost" \ 122 --prometheus-port 443 \ 123 --prometheus-token "$( oc whoami -t )" \ 124 -d &>$monitoring_collection_log 125 set +u; deactivate; set -u 126 127 popd