k8s.io/kubernetes@v1.29.3/test/e2e/framework/internal/output/output.go (about) 1 /* 2 Copyright 2022 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package output 18 19 import ( 20 "encoding/xml" 21 "os" 22 "path" 23 "regexp" 24 "testing" 25 26 "github.com/google/go-cmp/cmp" 27 "github.com/google/go-cmp/cmp/cmpopts" 28 "github.com/onsi/ginkgo/v2" 29 "github.com/onsi/ginkgo/v2/reporters" 30 "github.com/onsi/gomega" 31 "github.com/stretchr/testify/require" 32 33 "k8s.io/kubernetes/test/e2e/framework" 34 "k8s.io/kubernetes/test/e2e/framework/internal/junit" 35 ) 36 37 // TestGinkgoOutput runs the current suite and verifies that the generated 38 // JUnit file matches the expected result. 39 // 40 // The Ginkgo output on the console (aka the test suite log) does not get 41 // checked. It is usually less important for the CI and more relevant when 42 // using test suite interactively. To see what that Ginkgo output looks like, 43 // run tests with "go test -v". 44 func TestGinkgoOutput(t *testing.T, expected TestResult, runSpecsArgs ...interface{}) { 45 tmpdir := t.TempDir() 46 junitFile := path.Join(tmpdir, "junit.xml") 47 gomega.RegisterFailHandler(framework.Fail) 48 ginkgo.ReportAfterSuite("write JUnit file", func(report ginkgo.Report) { 49 junit.WriteJUnitReport(report, junitFile) 50 }) 51 fakeT := &testing.T{} 52 ginkgo.RunSpecs(fakeT, "Logging Suite", runSpecsArgs...) 53 54 var actual reporters.JUnitTestSuites 55 data, err := os.ReadFile(junitFile) 56 require.NoError(t, err) 57 err = xml.Unmarshal(data, &actual) 58 require.NoError(t, err) 59 60 if len(actual.TestSuites) != 1 { 61 t.Fatalf("expected one test suite, got %d, JUnit content:\n%s", len(actual.TestSuites), string(data)) 62 } 63 diff := cmp.Diff(expected.Suite, actual.TestSuites[0], 64 // Time varies. 65 // Name and Classname are "Logging Suite". 66 // Package includes a varying path, not interesting. 67 // Properties also too complicated to compare. 68 cmpopts.IgnoreFields(reporters.JUnitTestSuite{}, "Time", "Timestamp", "Name", "Package", "Properties"), 69 cmpopts.IgnoreFields(reporters.JUnitTestCase{}, "Time", "Classname"), 70 cmpopts.SortSlices(func(tc1, tc2 reporters.JUnitTestCase) bool { 71 return tc1.Name < tc2.Name 72 }), 73 cmpopts.AcyclicTransformer("simplify", func(in string) any { 74 out := simplify(in, expected) 75 // Sometimes cmp.Diff does not print the full string when it is long. 76 // Uncommenting this here may help debug differences. 77 // if len(out) > 100 { 78 // t.Logf("%s\n---------------------------------------\n%s\n", in, out) 79 // } 80 81 // Same idea as in 82 // https://github.com/google/go-cmp/issues/192#issuecomment-605346277: 83 // it forces cmp.Diff to diff strings line-by-line, 84 // even when it normally wouldn't. The downside is 85 // that the output is harder to turn back into the 86 // expected reference string. 87 // if len(out) > 50 { 88 // return strings.Split(out, "\n") 89 // } 90 91 return out 92 }), 93 ) 94 if diff != "" { 95 t.Fatalf("Simplified JUnit report not as expected (-want, +got):\n%s\n\nFull XML:\n%s", diff, string(data)) 96 } 97 } 98 99 // TestResult is the expected outcome of the suite, with additional parameters that 100 // determine equality. 101 type TestResult struct { 102 // Called to normalize all output strings before comparison if non-nil. 103 NormalizeOutput func(string) string 104 105 // All test cases and overall suite results. 106 Suite reporters.JUnitTestSuite 107 } 108 109 func simplify(in string, expected TestResult) string { 110 out := normalizeLocation(in) 111 out = stripTimes(out) 112 out = stripAddresses(out) 113 out = normalizeInitFunctions(out) 114 if expected.NormalizeOutput != nil { 115 out = expected.NormalizeOutput(out) 116 } 117 return out 118 } 119 120 // timePrefix matches "Jul 17 08:08:25.950: " at the beginning of each line. 121 var timePrefix = regexp.MustCompile(`(?m)^[[:alpha:]]{3} +[[:digit:]]{1,2} +[[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2}.[[:digit:]]{3}: `) 122 123 // elapsedSuffix matches "Elapsed: 16.189µs" 124 var elapsedSuffix = regexp.MustCompile(`Elapsed: [[:digit:]]+(\.[[:digit:]]+)?(µs|ns|ms|s|m)`) 125 126 // afterSuffix matches "after 5.001s." 127 var afterSuffix = regexp.MustCompile(`after [[:digit:]]+(\.[[:digit:]]+)?(µs|ns|ms|s|m).`) 128 129 // timeSuffix matches "@ 09/06/22 15:36:43.44 (5.001s)" as printed by Ginkgo v2 for log output, with the duration being optional. 130 var timeSuffix = regexp.MustCompile(`(?m)@[[:space:]][[:digit:]]{2}/[[:digit:]]{2}/[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2}(\.[[:digit:]]{1,3})?( \([[:digit:]]+(\.[[:digit:]]+)?(µs|ns|ms|s|m)\))?$`) 131 132 func stripTimes(in string) string { 133 out := timePrefix.ReplaceAllString(in, "") 134 out = elapsedSuffix.ReplaceAllString(out, "Elapsed: <elapsed>") 135 out = timeSuffix.ReplaceAllString(out, "<time>") 136 out = afterSuffix.ReplaceAllString(out, "after <after>.") 137 return out 138 } 139 140 // instanceAddr matches " | 0xc0003dec60>" 141 var instanceAddr = regexp.MustCompile(` \| 0x[0-9a-fA-F]+>`) 142 143 func stripAddresses(in string) string { 144 return instanceAddr.ReplaceAllString(in, ">") 145 } 146 147 // stackLocation matches "<some path>/<file>.go:75 +0x1f1" after a slash (built 148 // locally) or one of a few relative paths (built in the Kubernetes CI). 149 var stackLocation = regexp.MustCompile(`(?:/|vendor/|test/|GOROOT/).*/([[:^space:]]+.go:[[:digit:]]+)( \+0x[0-9a-fA-F]+)?`) 150 151 // functionArgs matches "<function name>(...)" where <function name> may be an anonymous function (e.g. "pod_test.glob..func1.1") 152 var functionArgs = regexp.MustCompile(`([[:alpha:][:digit:].]+)\(.*\)`) 153 154 // klogPrefix matches "I0822 16:10:39.343790 989127 " 155 var klogPrefix = regexp.MustCompile(`(?m)^[IEF][[:digit:]]{4} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2}\.[[:digit:]]{6}[[:space:]]+[[:digit:]]+ `) 156 157 // testFailureOutput matches TestFailureOutput() and its source followed by additional stack entries: 158 // 159 // k8s.io/kubernetes/test/e2e/framework/pod/pod_test.TestFailureOutput(0xc000558800) 160 // 161 // /nvme/gopath/src/k8s.io/kubernetes/test/e2e/framework/pod/wait_test.go:73 +0x1c9 162 // 163 // testing.tRunner(0xc000558800, 0x1af2848) 164 // 165 // /nvme/gopath/go/src/testing/testing.go:865 +0xc0 166 // 167 // created by testing.(*T).Run 168 // 169 // /nvme/gopath/go/src/testing/testing.go:916 +0x35a 170 var testFailureOutput = regexp.MustCompile(`(?m)^k8s.io/kubernetes/test/e2e/framework/internal/output\.TestGinkgoOutput\(.*\n\t.*(\n.*\n\t.*)*`) 171 172 // normalizeLocation removes path prefix and function parameters and certain stack entries 173 // that we don't care about. 174 func normalizeLocation(in string) string { 175 out := in 176 out = stackLocation.ReplaceAllString(out, "$1") 177 out = functionArgs.ReplaceAllString(out, "$1()") 178 out = testFailureOutput.ReplaceAllString(out, "") 179 out = klogPrefix.ReplaceAllString(out, "<klog> ") 180 return out 181 } 182 183 var initFunc = regexp.MustCompile(`(init\.+func|glob\.+func)`) 184 185 // normalizeInitFunctions maps both init.func (used by Go >= 1.22) and 186 // glob..func (used by Go < 1.22) to <init.func>. 187 func normalizeInitFunctions(in string) string { 188 out := initFunc.ReplaceAllString(in, "<init.func>") 189 return out 190 }