k8s.io/kubernetes@v1.31.0-alpha.0.0.20240520171757-56147500dadc/hack/verify-e2e-test-ownership.sh (about) 1 #!/usr/bin/env bash 2 3 # Copyright 2014 The Kubernetes Authors. 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 # This script verifies the following e2e test ownership policies 18 # - tests MUST start with [sig-foo] 19 # - tests SHOULD NOT have multiple [sig-foo] tags 20 # TODO: these two can be dropped if KubeDescribe is gone from codebase 21 # - tests MUST NOT have [k8s.io] in test names 22 # - tests MUST NOT use KubeDescribe 23 24 set -o errexit 25 set -o nounset 26 set -o pipefail 27 28 # This will canonicalize the path 29 KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd -P) 30 source "${KUBE_ROOT}/hack/lib/init.sh" 31 32 # Set REUSE_BUILD_OUTPUT=y to skip rebuilding dependencies if present 33 REUSE_BUILD_OUTPUT=${REUSE_BUILD_OUTPUT:-n} 34 # set VERBOSE_OUTPUT=y to output .jq files and shell commands 35 VERBOSE_OUTPUT=${VERBOSE_OUTPUT:-n} 36 37 if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then 38 set -x 39 fi 40 41 pushd "${KUBE_ROOT}" > /dev/null 42 43 # Setup a tmpdir to hold generated scripts and results 44 tmpdir=$(mktemp -d -t verify-e2e-test-ownership.XXXX) 45 readonly tmpdir 46 trap 'rm -rf ${tmpdir}' EXIT 47 48 # input 49 spec_summaries="${KUBE_ROOT}/_output/specsummaries.json" 50 # output 51 results_json="${tmpdir}/results.json" 52 summary_json="${tmpdir}/summary.json" 53 failures_json="${tmpdir}/failures.json" 54 55 # rebuild dependencies if necessary 56 function ensure_dependencies() { 57 local -r ginkgo="${KUBE_ROOT}/_output/bin/ginkgo" 58 local -r e2e_test="${KUBE_ROOT}/_output/bin/e2e.test" 59 if ! { [ -f "${ginkgo}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then 60 make ginkgo 61 fi 62 if ! { [ -f "${e2e_test}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then 63 hack/make-rules/build.sh test/e2e/e2e.test 64 fi 65 if ! { [ -f "${spec_summaries}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then 66 "${ginkgo}" --dry-run=true "${e2e_test}" -- --spec-dump "${spec_summaries}" > /dev/null 67 fi 68 } 69 70 # evaluate ginkgo spec summaries against e2e test ownership polices 71 # output to ${results_json} 72 function generate_results_json() { 73 readonly results_jq=${tmpdir}/results.jq 74 cat >"${results_jq}" <<EOS 75 [.[] | select( .LeafNodeType == "It") | . as { ContainerHierarchyTexts: \$text, ContainerHierarchyLocations: \$code, LeafNodeText: \$leafText, LeafNodeLocation: \$leafCode} | { 76 calls: ([ \$text | range(0;length) as \$i | { 77 sig: ((\$text[\$i] | match("\\\[(sig-[^\\\]]+)\\\]") | .captures[0].string) // "unknown"), 78 text: \$text[\$i], 79 # unused, but if we ever wanted to have policies based on other tags... 80 # tags: \$text[\$i] | [match("(\\\[[^\\\]]+\\\])"; "g").string], 81 line: \$code[\$i] | "\(.FileName):\(.LineNumber)" 82 }] + [{ 83 sig: ((\$leafText | match("\\\[(sig-[^\\\]]+)\\\]") | .captures[0].string) // "unknown"), 84 text: \$leafText, 85 # unused, but if we ever wanted to have policies based on other tags... 86 # tags: \$leafText | [match("(\\\[[^\\\]]+\\\])"; "g").string], 87 line: \$leafCode | "\(.FileName):\(.LineNumber)" 88 }]), 89 } | { 90 owner: .calls[0].sig, 91 calls: .calls, 92 testname: .calls | map(.text) | join(" "), 93 policies: [( 94 .calls[0] | 95 { 96 fail: (.sig == "unknown"), 97 level: "FAIL", 98 category: "unowned_test", 99 reason: "must start with [sig-foo]", 100 found: ., 101 } 102 ), ( 103 .calls[1:] | 104 (map(select(.sig != "unknown")) // [] | { 105 fail: . | any, 106 level: "WARN", 107 category: "too_many_sigs", 108 reason: "should not have multiple [sig-foo] tags", 109 found: ., 110 }) 111 ) 112 ] 113 }] 114 EOS 115 if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then 116 echo "about to ${results_jq}..." 117 cat -n "${results_jq}" 118 echo 119 fi 120 <"${spec_summaries}" jq --slurp --from-file "${results_jq}" > "${results_json}" 121 } 122 123 # summarize e2e test policy results 124 # output to ${summary_json} 125 function generate_summary_json() { 126 summary_jq=${tmpdir}/summary.jq 127 cat >"${summary_jq}" <<EOS 128 . as \$results | 129 # for each policy category 130 reduce \$results[0].policies[] as \$p ({}; . + { 131 # add a convenience .policy field containing that policy's result 132 (\$p.category): \$results | map(. + {policy: .policies[] | select(.category == \$p.category)}) | { 133 level: \$p.level, 134 reason: \$p.reason, 135 passing: map(select(.policy.fail | not)) | length, 136 failing: map(select(.policy.fail)) | length, 137 testnames: map(select(.policy.fail) | .testname), 138 } 139 }) 140 # add a meta policy based on whether any policy failed 141 + { 142 all_policies: \$results | { 143 level: "WARN", 144 reason: "should pass all policies", 145 passing: map(select(.policies | map(.fail) | any | not)) | length, 146 failing: map(select(.policies | map(.fail) | any)) | length, 147 testnames: map(select(.policies | map(.fail) | any) | .testname), 148 } 149 } 150 # if a policy has no failing tests, change its log output to PASS 151 | with_entries(.value += { log: (if (.value.failing == 0) then "PASS" else .value.level end) }) 152 # sort by policies with the most failing tests first 153 | to_entries | sort_by(.value.failing) | reverse | from_entries 154 EOS 155 if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then 156 echo "about to run ${results_jq}..." 157 cat -n "${summary_jq}" 158 echo 159 fi 160 <"${results_json}" jq --from-file "${summary_jq}" > "${summary_json}" 161 } 162 163 # filter e2e policy tests results to tests that failed, with the policies they failed 164 # output to ${failures_json} 165 function generate_failures_json() { 166 local -r failures_jq="${tmpdir}/failures.jq" 167 cat >"${failures_jq}" <<EOS 168 . 169 # for each test 170 | map( 171 # filter down to failing policies; trim category, .reason is more verbose 172 .policies |= map(select(.fail) | del(.category)) 173 # trim the full callstack, .found will contain the relevant call 174 | del(.calls) 175 ) 176 # filter down to tests that have failed policies 177 | map(select(.policies | map (.fail) | any)) 178 EOS 179 if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then 180 echo "about to run ${failures_jq}..." 181 cat -n "${failures_jq}" 182 echo 183 fi 184 <"${results_json}" jq --from-file "${failures_jq}" > "${failures_json}" 185 } 186 187 function output_results_and_exit_if_failed() { 188 local -r total_tests=$(<"${spec_summaries}" wc -l | awk '{print $1}') 189 190 # output results to console 191 ( 192 echo "run at datetime: $(date -u +%Y-%m-%dT%H:%M:%SZ)" 193 echo "based on commit: $(git log -n1 --date=iso-strict --pretty='%h - %cd - %s')" 194 echo 195 <"${failures_json}" cat 196 printf "%4s: e2e tests %-40s: %-4d\n" "INFO" "in total" "${total_tests}" 197 <"${summary_json}" jq -r 'to_entries[].value | 198 "printf \"%4s: ..failing %-40s: %-4d\\n\" \"\(.log)\" \"\(.reason)\" \"\(.failing)\""' | sh 199 ) | tee "${tmpdir}/output.txt" 200 # if we said "FAIL" in that output, we should fail 201 if <"${tmpdir}/output.txt" grep -q "^FAIL"; then 202 echo "FAIL" 203 exit 1 204 fi 205 } 206 207 ensure_dependencies 208 generate_results_json 209 generate_failures_json 210 generate_summary_json 211 output_results_and_exit_if_failed 212 echo "PASS"