github.com/onsi/ginkgo@v1.16.6-0.20211118180735-4e1925ba4c95/integration/run_test.go (about) 1 package integration_test 2 3 import ( 4 "fmt" 5 "os" 6 "regexp" 7 "runtime" 8 "strings" 9 10 . "github.com/onsi/ginkgo" 11 . "github.com/onsi/ginkgo/internal/test_helpers" 12 "github.com/onsi/ginkgo/types" 13 . "github.com/onsi/gomega" 14 "github.com/onsi/gomega/gbytes" 15 "github.com/onsi/gomega/gexec" 16 ) 17 18 var _ = Describe("Running Specs", func() { 19 denoter := "•" 20 if runtime.GOOS == "windows" { 21 denoter = "+" 22 } 23 24 Context("when pointed at the current directory", func() { 25 BeforeEach(func() { 26 fm.MountFixture("passing_ginkgo_tests") 27 }) 28 29 It("should run the tests in the working directory", func() { 30 session := startGinkgo(fm.PathTo("passing_ginkgo_tests"), "--no-color") 31 Eventually(session).Should(gexec.Exit(0)) 32 output := string(session.Out.Contents()) 33 34 Ω(output).Should(ContainSubstring("Running Suite: Passing_ginkgo_tests Suite")) 35 Ω(output).Should(ContainSubstring(strings.Repeat(denoter, 4))) 36 Ω(output).Should(ContainSubstring("SUCCESS! -- 4 Passed")) 37 Ω(output).Should(ContainSubstring("Test Suite Passed")) 38 }) 39 }) 40 41 Context("when passed an explicit package to run", func() { 42 BeforeEach(func() { 43 fm.MountFixture("passing_ginkgo_tests") 44 }) 45 46 It("should run the ginkgo style tests", func() { 47 session := startGinkgo(fm.TmpDir, "--no-color", "passing_ginkgo_tests") 48 Eventually(session).Should(gexec.Exit(0)) 49 output := string(session.Out.Contents()) 50 51 Ω(output).Should(ContainSubstring("Running Suite: Passing_ginkgo_tests Suite")) 52 Ω(output).Should(ContainSubstring(strings.Repeat(denoter, 4))) 53 Ω(output).Should(ContainSubstring("SUCCESS! -- 4 Passed")) 54 Ω(output).Should(ContainSubstring("Test Suite Passed")) 55 }) 56 }) 57 58 Context("when passed a number of packages to run", func() { 59 BeforeEach(func() { 60 fm.MountFixture("passing_ginkgo_tests") 61 fm.MountFixture("more_ginkgo_tests") 62 }) 63 64 It("should run the ginkgo style tests", func() { 65 session := startGinkgo(fm.TmpDir, "--no-color", "--succinct=false", "passing_ginkgo_tests", "more_ginkgo_tests") 66 Eventually(session).Should(gexec.Exit(0)) 67 output := string(session.Out.Contents()) 68 69 Ω(output).Should(ContainSubstring("Running Suite: Passing_ginkgo_tests Suite")) 70 Ω(output).Should(ContainSubstring("Running Suite: More_ginkgo_tests Suite")) 71 Ω(output).Should(ContainSubstring("Test Suite Passed")) 72 }) 73 }) 74 75 Context("when passed a number of packages to run, some of which have focused tests", func() { 76 BeforeEach(func() { 77 fm.MountFixture("passing_ginkgo_tests") 78 fm.MountFixture("more_ginkgo_tests") 79 fm.MountFixture("focused") 80 }) 81 82 It("should exit with a status code of 2 and explain why", func() { 83 session := startGinkgo(fm.TmpDir, "--no-color", "--succinct=false", "-r") 84 Eventually(session).Should(gexec.Exit(types.GINKGO_FOCUS_EXIT_CODE)) 85 output := string(session.Out.Contents()) 86 87 Ω(output).Should(ContainSubstring("Running Suite: Passing_ginkgo_tests Suite")) 88 Ω(output).Should(ContainSubstring("Running Suite: More_ginkgo_tests Suite")) 89 Ω(output).Should(ContainSubstring("Test Suite Passed")) 90 Ω(output).Should(ContainSubstring("Detected Programmatic Focus - setting exit status to %d", types.GINKGO_FOCUS_EXIT_CODE)) 91 }) 92 93 Context("when the GINKGO_EDITOR_INTEGRATION environment variable is set", func() { 94 BeforeEach(func() { 95 os.Setenv("GINKGO_EDITOR_INTEGRATION", "true") 96 }) 97 AfterEach(func() { 98 os.Setenv("GINKGO_EDITOR_INTEGRATION", "") 99 }) 100 It("should exit with a status code of 0 to allow a coverage file to be generated", func() { 101 session := startGinkgo(fm.TmpDir, "--no-color", "--succinct=false", "-r") 102 Eventually(session).Should(gexec.Exit(0)) 103 output := string(session.Out.Contents()) 104 105 Ω(output).Should(ContainSubstring("Running Suite: Passing_ginkgo_tests Suite")) 106 Ω(output).Should(ContainSubstring("Running Suite: More_ginkgo_tests Suite")) 107 Ω(output).Should(ContainSubstring("Test Suite Passed")) 108 }) 109 }) 110 }) 111 112 Context("when told to skipPackages", func() { 113 BeforeEach(func() { 114 fm.MountFixture("passing_ginkgo_tests") 115 fm.MountFixture("more_ginkgo_tests") 116 fm.MountFixture("focused") 117 }) 118 119 It("should skip packages that match the list", func() { 120 session := startGinkgo(fm.TmpDir, "--no-color", "--skip-package=more_ginkgo_tests,focused", "-r") 121 Eventually(session).Should(gexec.Exit(0)) 122 output := string(session.Out.Contents()) 123 124 Ω(output).Should(ContainSubstring("Passing_ginkgo_tests Suite")) 125 Ω(output).ShouldNot(ContainSubstring("More_ginkgo_tests Suite")) 126 Ω(output).ShouldNot(ContainSubstring("Focused_fixture Suite")) 127 Ω(output).Should(ContainSubstring("Test Suite Passed")) 128 }) 129 130 Context("when all packages are skipped", func() { 131 It("should not run anything, but still exit 0", func() { 132 session := startGinkgo(fm.TmpDir, "--no-color", "--skip-package=passing_ginkgo_tests,more_ginkgo_tests,focused", "-r") 133 Eventually(session).Should(gexec.Exit(0)) 134 output := string(session.Out.Contents()) 135 errOutput := string(session.Err.Contents()) 136 137 Ω(errOutput).Should(ContainSubstring("All tests skipped!")) 138 Ω(output).ShouldNot(ContainSubstring("Passing_ginkgo_tests Suite")) 139 Ω(output).ShouldNot(ContainSubstring("More_ginkgo_tests Suite")) 140 Ω(output).ShouldNot(ContainSubstring("Focused_fixture Suite")) 141 Ω(output).ShouldNot(ContainSubstring("Test Suite Passed")) 142 }) 143 }) 144 }) 145 146 Context("when there are no tests to run", func() { 147 It("should exit 1", func() { 148 session := startGinkgo(fm.TmpDir, "--no-color", "-r") 149 Eventually(session).Should(gexec.Exit(1)) 150 output := string(session.Err.Contents()) 151 152 Ω(output).Should(ContainSubstring("Found no test suites")) 153 }) 154 }) 155 156 Context("when there are test files but `go test` reports there are no tests to run", func() { 157 BeforeEach(func() { 158 fm.MountFixture("no_test_fn") 159 }) 160 161 It("suggests running ginkgo bootstrap", func() { 162 session := startGinkgo(fm.TmpDir, "--no-color", "-r") 163 Eventually(session).Should(gexec.Exit(0)) 164 output := string(session.Err.Contents()) 165 166 Ω(output).Should(ContainSubstring(`Found no test suites, did you forget to run "ginkgo bootstrap"?`)) 167 }) 168 169 It("fails if told to requireSuite", func() { 170 session := startGinkgo(fm.TmpDir, "--no-color", "-r", "-require-suite") 171 Eventually(session).Should(gexec.Exit(1)) 172 output := string(session.Err.Contents()) 173 174 Ω(output).Should(ContainSubstring(`Found no test suites, did you forget to run "ginkgo bootstrap"?`)) 175 }) 176 }) 177 178 Context("when told to randomize-suites", func() { 179 BeforeEach(func() { 180 fm.MountFixture("passing_ginkgo_tests") 181 fm.MountFixture("more_ginkgo_tests") 182 }) 183 184 It("should mix up the order of the test suites", Label("slow"), func() { 185 session := startGinkgo(fm.TmpDir, "--no-color", "--randomize-suites", "-r", "--seed=1") 186 Eventually(session).Should(gexec.Exit(0)) 187 188 Ω(session).Should(gbytes.Say("More_ginkgo_tests Suite")) 189 Ω(session).Should(gbytes.Say("Passing_ginkgo_tests Suite")) 190 191 session = startGinkgo(fm.TmpDir, "--no-color", "--randomize-suites", "-r", "--seed=4") 192 Eventually(session).Should(gexec.Exit(0)) 193 194 Ω(session).Should(gbytes.Say("Passing_ginkgo_tests Suite")) 195 Ω(session).Should(gbytes.Say("More_ginkgo_tests Suite")) 196 }) 197 }) 198 199 Context("when pointed at a package with xunit style tests", func() { 200 BeforeEach(func() { 201 fm.MountFixture("xunit") 202 }) 203 204 It("should run the xunit style tests, always setting -test.v and passing in supported go test flags", func() { 205 session := startGinkgo(fm.PathTo("xunit"), "-blockprofile=block-profile.out") 206 Eventually(session).Should(gexec.Exit(0)) 207 output := string(session.Out.Contents()) 208 209 Ω(output).Should(ContainSubstring("--- PASS: TestAlwaysTrue")) 210 Ω(output).Should(ContainSubstring("Test Suite Passed")) 211 212 Ω(fm.PathTo("xunit", "block-profile.out")).Should(BeAnExistingFile()) 213 }) 214 }) 215 216 Context("when pointed at a package with no tests", func() { 217 BeforeEach(func() { 218 fm.MountFixture("no_tests") 219 }) 220 221 It("should fail", func() { 222 session := startGinkgo(fm.PathTo("no_tests"), "--no-color") 223 Eventually(session).Should(gexec.Exit(1)) 224 225 Ω(session.Err.Contents()).Should(ContainSubstring("Found no test suites")) 226 }) 227 }) 228 229 Context("when pointed at a package with tests, but the tests have been excluded via go tags", func() { 230 BeforeEach(func() { 231 fm.MountFixture("no_tagged_tests") 232 }) 233 234 It("should exit 0, not run anything, and generate a json report if asked", func() { 235 session := startGinkgo(fm.PathTo("no_tagged_tests"), "--no-color", "--json-report=report.json") 236 Eventually(session).Should(gexec.Exit(0)) 237 Ω(session).Should(gbytes.Say("no test files")) 238 239 report := fm.LoadJSONReports("no_tagged_tests", "report.json")[0] 240 Ω(report.SuiteSucceeded).Should(BeTrue()) 241 Ω(report.SpecialSuiteFailureReasons).Should(ConsistOf("Suite did not run go test reported that no test files were found")) 242 }) 243 }) 244 245 Context("when pointed at a package that fails to compile", func() { 246 BeforeEach(func() { 247 fm.MountFixture("does_not_compile") 248 }) 249 250 It("should fail", func() { 251 session := startGinkgo(fm.PathTo("does_not_compile"), "--no-color") 252 Eventually(session).Should(gexec.Exit(1)) 253 output := string(session.Out.Contents()) 254 255 Ω(output).Should(ContainSubstring("Failed to compile")) 256 }) 257 }) 258 259 Context("when running in parallel", func() { 260 BeforeEach(func() { 261 fm.MountFixture("passing_ginkgo_tests") 262 }) 263 264 Context("with a specific number of -procs", func() { 265 It("should use the specified number of processes", func() { 266 session := startGinkgo(fm.PathTo("passing_ginkgo_tests"), "--no-color", "-succinct", "--procs=2") 267 Eventually(session).Should(gexec.Exit(0)) 268 output := string(session.Out.Contents()) 269 270 Ω(output).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4/4 specs - 2 procs [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]s`, regexp.QuoteMeta(denoter))) 271 Ω(output).Should(ContainSubstring("Test Suite Passed")) 272 }) 273 }) 274 275 Context("with -p", func() { 276 It("it should autocompute the number of nodes", func() { 277 session := startGinkgo(fm.PathTo("passing_ginkgo_tests"), "--no-color", "-succinct", "-p") 278 Eventually(session).Should(gexec.Exit(0)) 279 output := string(session.Out.Contents()) 280 281 nodes := runtime.NumCPU() 282 if nodes == 1 { 283 Skip("Can't test parallel testings with 1 CPU") 284 } 285 if nodes > 4 { 286 nodes = nodes - 1 287 } 288 Ω(output).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4/4 specs - %d procs [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]?s`, nodes, regexp.QuoteMeta(denoter))) 289 Ω(output).Should(ContainSubstring("Test Suite Passed")) 290 }) 291 }) 292 }) 293 294 Context("when running in parallel and there are specs marked Serial", Label("slow"), func() { 295 BeforeEach(func() { 296 fm.MountFixture("serial") 297 }) 298 299 It("runs the serial specs after all the parallel specs have finished", func() { 300 By("running a carefully crafted test without the serial decorator") 301 session := startGinkgo(fm.PathTo("serial"), "--no-color", "--procs=2", "--randomize-all", "--fail-fast", "--", "--no-serial") 302 Eventually(session).Should(gexec.Exit(1)) 303 304 By("running a carefully crafted test with the serial decorator") 305 session = startGinkgo(fm.PathTo("serial"), "--no-color", "--procs=2", "--randomize-all", "--fail-fast") 306 Eventually(session).Should(gexec.Exit(0)) 307 }) 308 }) 309 310 Context("when running with ordered specs", func() { 311 BeforeEach(func() { 312 fm.MountFixture("ordered") 313 }) 314 315 It("always preserve spec order within ordered contexts", func() { 316 By("running a carefully crafted test without the ordered decorator") 317 session := startGinkgo(fm.PathTo("ordered"), "--no-color", "--procs=2", "-v", "--randomize-all", "--fail-fast", "--", "--no-ordered") 318 Eventually(session).Should(gexec.Exit(1)) 319 320 By("running a carefully crafted test with the ordered decorator") 321 session = startGinkgo(fm.PathTo("ordered"), "--no-color", "--procs=2", "-v", "--randomize-all", "--fail-fast") 322 Eventually(session).Should(gexec.Exit(0)) 323 }) 324 }) 325 326 Context("when running multiple tests", func() { 327 BeforeEach(func() { 328 fm.MountFixture("passing_ginkgo_tests") 329 fm.MountFixture("more_ginkgo_tests") 330 fm.MountFixture("no_tagged_tests") 331 }) 332 333 Context("when all the tests pass", func() { 334 Context("with the -r flag", func() { 335 It("should run all the tests (in succinct mode) and succeed", func() { 336 session := startGinkgo(fm.TmpDir, "--no-color", "-r", ".") 337 Eventually(session).Should(gexec.Exit(0)) 338 output := string(session.Out.Contents()) 339 340 outputLines := strings.Split(output, "\n") 341 Ω(outputLines[0]).Should(MatchRegexp(`\[\d+\] More_ginkgo_tests Suite - 2/2 specs [%s]{2} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) 342 Ω(outputLines[1]).Should(ContainSubstring("Skipping ./no_tagged_tests (no test files)")) 343 Ω(outputLines[2]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4/4 specs [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) 344 Ω(output).Should(ContainSubstring("Test Suite Passed")) 345 }) 346 }) 347 Context("with a trailing /...", func() { 348 It("should run all the tests (in succinct mode) and succeed", func() { 349 session := startGinkgo(fm.TmpDir, "--no-color", "./...") 350 Eventually(session).Should(gexec.Exit(0)) 351 output := string(session.Out.Contents()) 352 353 outputLines := strings.Split(output, "\n") 354 Ω(outputLines[0]).Should(MatchRegexp(`\[\d+\] More_ginkgo_tests Suite - 2/2 specs [%s]{2} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) 355 Ω(outputLines[1]).Should(ContainSubstring("Skipping ./no_tagged_tests (no test files)")) 356 Ω(outputLines[2]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4/4 specs [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) 357 Ω(output).Should(ContainSubstring("Test Suite Passed")) 358 }) 359 }) 360 }) 361 362 Context("when one of the packages has a failing tests", func() { 363 BeforeEach(func() { 364 fm.MountFixture("failing_ginkgo_tests") 365 }) 366 367 It("should fail and stop running tests", func() { 368 session := startGinkgo(fm.TmpDir, "--no-color", "no_tagged_tests", "passing_ginkgo_tests", "failing_ginkgo_tests", "more_ginkgo_tests") 369 Eventually(session).Should(gexec.Exit(1)) 370 output := string(session.Out.Contents()) 371 372 outputLines := strings.Split(output, "\n") 373 Ω(outputLines[0]).Should(ContainSubstring("Skipping ./no_tagged_tests (no test files)")) 374 Ω(outputLines[1]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4/4 specs [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) 375 Ω(outputLines[2]).Should(MatchRegexp(`\[\d+\] Failing_ginkgo_tests Suite - 2/2 specs`)) 376 Ω(output).Should(ContainSubstring(fmt.Sprintf("%s [FAILED]", denoter))) 377 Ω(output).ShouldNot(ContainSubstring("More_ginkgo_tests Suite")) 378 Ω(output).Should(ContainSubstring("Test Suite Failed")) 379 }) 380 }) 381 382 Context("when one of the packages fails to compile", func() { 383 BeforeEach(func() { 384 fm.MountFixture("does_not_compile") 385 }) 386 387 It("should fail and stop running tests", func() { 388 session := startGinkgo(fm.TmpDir, "--no-color", "no_tagged_tests", "passing_ginkgo_tests", "does_not_compile", "more_ginkgo_tests") 389 Eventually(session).Should(gexec.Exit(1)) 390 output := string(session.Out.Contents()) 391 392 outputLines := strings.Split(output, "\n") 393 Ω(outputLines[0]).Should(ContainSubstring("Skipping ./no_tagged_tests (no test files)")) 394 Ω(outputLines[1]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4/4 specs [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) 395 Ω(outputLines[2]).Should(ContainSubstring("Failed to compile does_not_compile:")) 396 Ω(output).ShouldNot(ContainSubstring("More_ginkgo_tests Suite")) 397 Ω(output).Should(ContainSubstring("Test Suite Failed")) 398 }) 399 }) 400 401 Context("when either is the case, but the keep-going flag is set", func() { 402 BeforeEach(func() { 403 fm.MountFixture("does_not_compile") 404 fm.MountFixture("failing_ginkgo_tests") 405 }) 406 407 It("should soldier on", func() { 408 session := startGinkgo(fm.TmpDir, "--no-color", "-keep-going", "no_tagged_tests", "passing_ginkgo_tests", "does_not_compile", "failing_ginkgo_tests", "more_ginkgo_tests") 409 Eventually(session).Should(gexec.Exit(1)) 410 output := string(session.Out.Contents()) 411 412 outputLines := strings.Split(output, "\n") 413 Ω(outputLines[0]).Should(ContainSubstring("Skipping ./no_tagged_tests (no test files)")) 414 Ω(outputLines[1]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4/4 specs [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) 415 Ω(outputLines[2]).Should(ContainSubstring("Failed to compile does_not_compile:")) 416 Ω(output).Should(MatchRegexp(`\[\d+\] Failing_ginkgo_tests Suite - 2/2 specs`)) 417 Ω(output).Should(ContainSubstring(fmt.Sprintf("%s [FAILED]", denoter))) 418 Ω(output).Should(MatchRegexp(`\[\d+\] More_ginkgo_tests Suite - 2/2 specs [%s]{2} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) 419 Ω(output).Should(ContainSubstring("Test Suite Failed")) 420 }) 421 }) 422 }) 423 424 Context("when running large suites in parallel", Label("slow"), func() { 425 BeforeEach(func() { 426 fm.MountFixture("large") 427 }) 428 429 It("doesn't miss any tests (a sanity test)", func() { 430 session := startGinkgo(fm.PathTo("large"), "--no-color", "--procs=3", "--json-report=report.json") 431 Eventually(session).Should(gexec.Exit(0)) 432 report := Reports(fm.LoadJSONReports("large", "report.json")[0].SpecReports) 433 434 expectedNames := []string{} 435 for i := 0; i < 2048; i++ { 436 expectedNames = append(expectedNames, fmt.Sprintf("%d", i)) 437 } 438 Ω(report.Names()).Should(ConsistOf(expectedNames)) 439 440 }) 441 }) 442 })