gotest.tools/gotestsum@v1.11.0/cmd/main_test.go (about) 1 package cmd 2 3 import ( 4 "bytes" 5 "encoding/json" 6 "os" 7 "os/exec" 8 "path/filepath" 9 "runtime" 10 "strings" 11 "testing" 12 13 "github.com/fatih/color" 14 "gotest.tools/gotestsum/testjson" 15 "gotest.tools/v3/assert" 16 "gotest.tools/v3/assert/cmp" 17 "gotest.tools/v3/env" 18 "gotest.tools/v3/golden" 19 "gotest.tools/v3/skip" 20 ) 21 22 func TestUsage_WithFlagsFromSetupFlags(t *testing.T) { 23 env.PatchAll(t, nil) 24 patchNoColor(t, false) 25 26 name := "gotestsum" 27 flags, _ := setupFlags(name) 28 buf := new(bytes.Buffer) 29 usage(buf, name, flags) 30 31 golden.Assert(t, buf.String(), "gotestsum-help-text") 32 } 33 34 func patchNoColor(t *testing.T, value bool) { 35 orig := color.NoColor 36 color.NoColor = value 37 t.Cleanup(func() { 38 color.NoColor = orig 39 }) 40 } 41 42 func TestOptions_Validate_FromFlags(t *testing.T) { 43 type testCase struct { 44 name string 45 args []string 46 expected string 47 } 48 fn := func(t *testing.T, tc testCase) { 49 flags, opts := setupFlags("gotestsum") 50 err := flags.Parse(tc.args) 51 assert.NilError(t, err) 52 opts.args = flags.Args() 53 54 err = opts.Validate() 55 if tc.expected == "" { 56 assert.NilError(t, err) 57 return 58 } 59 assert.ErrorContains(t, err, tc.expected, "opts: %#v", opts) 60 } 61 var testCases = []testCase{ 62 { 63 name: "no flags", 64 }, 65 { 66 name: "rerun flag, raw command", 67 args: []string{"--rerun-fails", "--raw-command", "--", "./test-all"}, 68 }, 69 { 70 name: "rerun flag, no go-test args", 71 args: []string{"--rerun-fails", "--"}, 72 }, 73 { 74 name: "rerun flag, go-test args, no packages flag", 75 args: []string{"--rerun-fails", "--", "./..."}, 76 expected: "the list of packages to test must be specified by the --packages flag", 77 }, 78 { 79 name: "rerun flag, go-test args, with packages flag", 80 args: []string{"--rerun-fails", "--packages", "./...", "--", "--foo"}, 81 }, 82 { 83 name: "rerun flag, no go-test args, with packages flag", 84 args: []string{"--rerun-fails", "--packages", "./..."}, 85 }, 86 { 87 name: "rerun-fails with failfast", 88 args: []string{"--rerun-fails", "--packages=./...", "--", "-failfast"}, 89 expected: "-failfast can not be used with --rerun-fails", 90 }, 91 } 92 for _, tc := range testCases { 93 t.Run(tc.name, func(t *testing.T) { 94 fn(t, tc) 95 }) 96 } 97 } 98 99 func TestGoTestCmdArgs(t *testing.T) { 100 type testCase struct { 101 opts *options 102 rerunOpts rerunOpts 103 env []string 104 expected []string 105 } 106 107 run := func(t *testing.T, name string, tc testCase) { 108 t.Helper() 109 runCase(t, name, func(t *testing.T) { 110 env.PatchAll(t, env.ToMap(tc.env)) 111 actual := goTestCmdArgs(tc.opts, tc.rerunOpts) 112 assert.DeepEqual(t, actual, tc.expected) 113 }) 114 } 115 116 run(t, "raw command", testCase{ 117 opts: &options{ 118 rawCommand: true, 119 args: []string{"./script", "-test.timeout=20m"}, 120 }, 121 expected: []string{"./script", "-test.timeout=20m"}, 122 }) 123 run(t, "no args", testCase{ 124 opts: &options{}, 125 expected: []string{"go", "test", "-json", "./..."}, 126 }) 127 run(t, "no args, with rerunPackageList arg", testCase{ 128 opts: &options{ 129 packages: []string{"./pkg"}, 130 }, 131 expected: []string{"go", "test", "-json", "./pkg"}, 132 }) 133 run(t, "TEST_DIRECTORY env var no args", testCase{ 134 opts: &options{}, 135 env: []string{"TEST_DIRECTORY=testdir"}, 136 expected: []string{"go", "test", "-json", "testdir"}, 137 }) 138 run(t, "TEST_DIRECTORY env var with args", testCase{ 139 opts: &options{ 140 args: []string{"-tags=integration"}, 141 }, 142 env: []string{"TEST_DIRECTORY=testdir"}, 143 expected: []string{"go", "test", "-json", "-tags=integration", "testdir"}, 144 }) 145 run(t, "no -json arg", testCase{ 146 opts: &options{ 147 args: []string{"-timeout=2m", "./pkg"}, 148 }, 149 expected: []string{"go", "test", "-json", "-timeout=2m", "./pkg"}, 150 }) 151 run(t, "with -json arg", testCase{ 152 opts: &options{ 153 args: []string{"-json", "-timeout=2m", "./pkg"}, 154 }, 155 expected: []string{"go", "test", "-json", "-timeout=2m", "./pkg"}, 156 }) 157 run(t, "raw command, with rerunOpts", testCase{ 158 opts: &options{ 159 rawCommand: true, 160 args: []string{"./script", "-test.timeout=20m"}, 161 }, 162 rerunOpts: rerunOpts{ 163 runFlag: "-run=TestOne|TestTwo", 164 pkg: "./fails", 165 }, 166 expected: []string{"./script", "-test.timeout=20m", "-run=TestOne|TestTwo", "./fails"}, 167 }) 168 run(t, "no args, with rerunOpts", testCase{ 169 opts: &options{}, 170 rerunOpts: rerunOpts{ 171 runFlag: "-run=TestOne|TestTwo", 172 pkg: "./fails", 173 }, 174 expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "./fails"}, 175 }) 176 run(t, "TEST_DIRECTORY env var, no args, with rerunOpts", testCase{ 177 opts: &options{}, 178 rerunOpts: rerunOpts{ 179 runFlag: "-run=TestOne|TestTwo", 180 pkg: "./fails", 181 }, 182 env: []string{"TEST_DIRECTORY=testdir"}, 183 // TEST_DIRECTORY should be overridden by rerun opts 184 expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "./fails"}, 185 }) 186 run(t, "TEST_DIRECTORY env var, with args, with rerunOpts", testCase{ 187 opts: &options{ 188 args: []string{"-tags=integration"}, 189 }, 190 rerunOpts: rerunOpts{ 191 runFlag: "-run=TestOne|TestTwo", 192 pkg: "./fails", 193 }, 194 env: []string{"TEST_DIRECTORY=testdir"}, 195 expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "-tags=integration", "./fails"}, 196 }) 197 run(t, "no -json arg, with rerunOpts", testCase{ 198 opts: &options{ 199 args: []string{"-timeout=2m"}, 200 packages: []string{"./pkg"}, 201 }, 202 rerunOpts: rerunOpts{ 203 runFlag: "-run=TestOne|TestTwo", 204 pkg: "./fails", 205 }, 206 expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "-timeout=2m", "./fails"}, 207 }) 208 run(t, "with -json arg, with rerunOpts", testCase{ 209 opts: &options{ 210 args: []string{"-json", "-timeout=2m"}, 211 packages: []string{"./pkg"}, 212 }, 213 rerunOpts: rerunOpts{ 214 runFlag: "-run=TestOne|TestTwo", 215 pkg: "./fails", 216 }, 217 expected: []string{"go", "test", "-run=TestOne|TestTwo", "-json", "-timeout=2m", "./fails"}, 218 }) 219 run(t, "with args, with reunFailsPackageList args, with rerunOpts", testCase{ 220 opts: &options{ 221 args: []string{"-timeout=2m"}, 222 packages: []string{"./pkg1", "./pkg2", "./pkg3"}, 223 }, 224 rerunOpts: rerunOpts{ 225 runFlag: "-run=TestOne|TestTwo", 226 pkg: "./fails", 227 }, 228 expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "-timeout=2m", "./fails"}, 229 }) 230 run(t, "with args, with reunFailsPackageList", testCase{ 231 opts: &options{ 232 args: []string{"-timeout=2m"}, 233 packages: []string{"./pkg1", "./pkg2", "./pkg3"}, 234 }, 235 expected: []string{"go", "test", "-json", "-timeout=2m", "./pkg1", "./pkg2", "./pkg3"}, 236 }) 237 run(t, "reunFailsPackageList args, with rerunOpts ", testCase{ 238 opts: &options{ 239 packages: []string{"./pkg1", "./pkg2", "./pkg3"}, 240 }, 241 rerunOpts: rerunOpts{ 242 runFlag: "-run=TestOne|TestTwo", 243 pkg: "./fails", 244 }, 245 expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "./fails"}, 246 }) 247 run(t, "reunFailsPackageList args, with rerunOpts, with -args ", testCase{ 248 opts: &options{ 249 args: []string{"before", "-args", "after"}, 250 packages: []string{"./pkg1"}, 251 }, 252 rerunOpts: rerunOpts{ 253 runFlag: "-run=TestOne|TestTwo", 254 pkg: "./fails", 255 }, 256 expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "before", "./fails", "-args", "after"}, 257 }) 258 run(t, "reunFailsPackageList args, with rerunOpts, with -args at end", testCase{ 259 opts: &options{ 260 args: []string{"before", "-args"}, 261 packages: []string{"./pkg1"}, 262 }, 263 rerunOpts: rerunOpts{ 264 runFlag: "-run=TestOne|TestTwo", 265 pkg: "./fails", 266 }, 267 expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "before", "./fails", "-args"}, 268 }) 269 run(t, "reunFailsPackageList args, with -args at start", testCase{ 270 opts: &options{ 271 args: []string{"-args", "after"}, 272 packages: []string{"./pkg1"}, 273 }, 274 expected: []string{"go", "test", "-json", "./pkg1", "-args", "after"}, 275 }) 276 run(t, "-run arg at start, with rerunOpts ", testCase{ 277 opts: &options{ 278 args: []string{"-run=TestFoo", "-args"}, 279 packages: []string{"./pkg"}, 280 }, 281 rerunOpts: rerunOpts{ 282 runFlag: "-run=TestOne|TestTwo", 283 pkg: "./fails", 284 }, 285 expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "./fails", "-args"}, 286 }) 287 run(t, "-run arg in middle, with rerunOpts ", testCase{ 288 opts: &options{ 289 args: []string{"-count", "1", "--run", "TestFoo", "-args"}, 290 packages: []string{"./pkg"}, 291 }, 292 rerunOpts: rerunOpts{ 293 runFlag: "-run=TestOne|TestTwo", 294 pkg: "./fails", 295 }, 296 expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "-count", "1", "./fails", "-args"}, 297 }) 298 run(t, "-run arg at end with missing value, with rerunOpts ", testCase{ 299 opts: &options{ 300 args: []string{"-count", "1", "-run"}, 301 packages: []string{"./pkg"}, 302 }, 303 rerunOpts: rerunOpts{ 304 runFlag: "-run=TestOne|TestTwo", 305 pkg: "./fails", 306 }, 307 expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "-count", "1", "-run", "./fails"}, 308 }) 309 } 310 311 func runCase(t *testing.T, name string, fn func(t *testing.T)) { 312 t.Helper() 313 t.Run(name, func(t *testing.T) { 314 t.Helper() 315 t.Log("case:", name) 316 fn(t) 317 }) 318 } 319 320 func TestRun_RerunFails_WithTooManyInitialFailures(t *testing.T) { 321 jsonFailed := `{"Package": "pkg", "Action": "run"} 322 {"Package": "pkg", "Test": "TestOne", "Action": "run"} 323 {"Package": "pkg", "Test": "TestOne", "Action": "fail"} 324 {"Package": "pkg", "Test": "TestTwo", "Action": "run"} 325 {"Package": "pkg", "Test": "TestTwo", "Action": "fail"} 326 {"Package": "pkg", "Action": "fail"} 327 ` 328 329 fn := func(args []string) *proc { 330 return &proc{ 331 cmd: fakeWaiter{result: newExitCode("failed", 1)}, 332 stdout: strings.NewReader(jsonFailed), 333 stderr: bytes.NewReader(nil), 334 } 335 } 336 reset := patchStartGoTestFn(fn) 337 defer reset() 338 339 out := new(bytes.Buffer) 340 opts := &options{ 341 rawCommand: true, 342 args: []string{"./test.test"}, 343 format: "testname", 344 rerunFailsMaxAttempts: 3, 345 rerunFailsMaxInitialFailures: 1, 346 stdout: out, 347 stderr: os.Stderr, 348 hideSummary: newHideSummaryValue(), 349 } 350 err := run(opts) 351 assert.ErrorContains(t, err, "number of test failures (2) exceeds maximum (1)", out.String()) 352 } 353 354 func TestRun_RerunFails_BuildErrorPreventsRerun(t *testing.T) { 355 jsonFailed := `{"Package": "pkg", "Action": "run"} 356 {"Package": "pkg", "Test": "TestOne", "Action": "run"} 357 {"Package": "pkg", "Test": "TestOne", "Action": "fail"} 358 {"Package": "pkg", "Test": "TestTwo", "Action": "run"} 359 {"Package": "pkg", "Test": "TestTwo", "Action": "fail"} 360 {"Package": "pkg", "Action": "fail"} 361 ` 362 363 fn := func(args []string) *proc { 364 return &proc{ 365 cmd: fakeWaiter{result: newExitCode("failed", 1)}, 366 stdout: strings.NewReader(jsonFailed), 367 stderr: strings.NewReader("anything here is an error\n"), 368 } 369 } 370 reset := patchStartGoTestFn(fn) 371 defer reset() 372 373 out := new(bytes.Buffer) 374 opts := &options{ 375 rawCommand: true, 376 args: []string{"./test.test"}, 377 format: "testname", 378 rerunFailsMaxAttempts: 3, 379 rerunFailsMaxInitialFailures: 1, 380 stdout: out, 381 stderr: os.Stderr, 382 hideSummary: newHideSummaryValue(), 383 } 384 err := run(opts) 385 assert.ErrorContains(t, err, "rerun aborted because previous run had errors", out.String()) 386 } 387 388 // type checking of os/exec.ExitError is done in a test file so that users 389 // installing from source can continue to use versions prior to go1.12. 390 var _ exitCoder = &exec.ExitError{} 391 392 func TestRun_RerunFails_PanicPreventsRerun(t *testing.T) { 393 jsonFailed := `{"Package": "pkg", "Action": "run"} 394 {"Package": "pkg", "Test": "TestOne", "Action": "run"} 395 {"Package": "pkg", "Test": "TestOne", "Action": "output","Output":"panic: something went wrong\n"} 396 {"Package": "pkg", "Action": "fail"} 397 ` 398 399 fn := func(args []string) *proc { 400 return &proc{ 401 cmd: fakeWaiter{result: newExitCode("failed", 1)}, 402 stdout: strings.NewReader(jsonFailed), 403 stderr: bytes.NewReader(nil), 404 } 405 } 406 reset := patchStartGoTestFn(fn) 407 defer reset() 408 409 out := new(bytes.Buffer) 410 opts := &options{ 411 rawCommand: true, 412 args: []string{"./test.test"}, 413 format: "testname", 414 rerunFailsMaxAttempts: 3, 415 rerunFailsMaxInitialFailures: 1, 416 stdout: out, 417 stderr: os.Stderr, 418 hideSummary: newHideSummaryValue(), 419 } 420 err := run(opts) 421 assert.ErrorContains(t, err, "rerun aborted because previous run had a suspected panic", out.String()) 422 } 423 424 func TestRun_InputFromStdin(t *testing.T) { 425 stdin := os.Stdin 426 t.Cleanup(func() { os.Stdin = stdin }) 427 428 r, w, err := os.Pipe() 429 assert.NilError(t, err) 430 t.Cleanup(func() { _ = r.Close() }) 431 432 os.Stdin = r 433 434 go func() { 435 defer func() { _ = w.Close() }() 436 437 e := json.NewEncoder(w) 438 for _, event := range []testjson.TestEvent{ 439 {Action: "run", Package: "pkg"}, 440 {Action: "run", Package: "pkg", Test: "TestOne"}, 441 {Action: "fail", Package: "pkg", Test: "TestOne"}, 442 {Action: "fail", Package: "pkg"}, 443 } { 444 assert.Check(t, e.Encode(event)) 445 } 446 }() 447 448 stdout := new(bytes.Buffer) 449 err = run(&options{ 450 args: []string{"cat"}, 451 format: "testname", 452 hideSummary: newHideSummaryValue(), 453 rawCommand: true, 454 455 stdout: stdout, 456 stderr: os.Stderr, 457 }) 458 assert.NilError(t, err) 459 assert.Assert(t, cmp.Contains(stdout.String(), "DONE 1")) 460 } 461 462 func TestRun_JsonFileIsSyncedBeforePostRunCommand(t *testing.T) { 463 skip.If(t, runtime.GOOS == "windows") 464 465 input := golden.Get(t, "../../testjson/testdata/input/go-test-json.out") 466 467 fn := func(args []string) *proc { 468 return &proc{ 469 cmd: fakeWaiter{}, 470 stdout: bytes.NewReader(input), 471 stderr: bytes.NewReader(nil), 472 } 473 } 474 reset := patchStartGoTestFn(fn) 475 defer reset() 476 477 tmp := t.TempDir() 478 jsonFile := filepath.Join(tmp, "json.log") 479 480 out := new(bytes.Buffer) 481 opts := &options{ 482 rawCommand: true, 483 args: []string{"./test.test"}, 484 format: "none", 485 stdout: out, 486 stderr: os.Stderr, 487 hideSummary: &hideSummaryValue{value: testjson.SummarizeNone}, 488 jsonFile: jsonFile, 489 postRunHookCmd: &commandValue{ 490 command: []string{"cat", jsonFile}, 491 }, 492 } 493 err := run(opts) 494 assert.NilError(t, err) 495 expected := string(input) 496 _, actual, _ := strings.Cut(out.String(), "s\n") // remove the DONE line 497 assert.Equal(t, actual, expected) 498 } 499 500 func TestRun_JsonFileTimingEvents(t *testing.T) { 501 input := golden.Get(t, "../../testjson/testdata/input/go-test-json.out") 502 503 fn := func(args []string) *proc { 504 return &proc{ 505 cmd: fakeWaiter{}, 506 stdout: bytes.NewReader(input), 507 stderr: bytes.NewReader(nil), 508 } 509 } 510 reset := patchStartGoTestFn(fn) 511 defer reset() 512 513 tmp := t.TempDir() 514 jsonFileTiming := filepath.Join(tmp, "json.log") 515 516 out := new(bytes.Buffer) 517 opts := &options{ 518 rawCommand: true, 519 args: []string{"./test.test"}, 520 format: "none", 521 stdout: out, 522 stderr: os.Stderr, 523 hideSummary: &hideSummaryValue{value: testjson.SummarizeNone}, 524 jsonFileTimingEvents: jsonFileTiming, 525 } 526 err := run(opts) 527 assert.NilError(t, err) 528 529 raw, err := os.ReadFile(jsonFileTiming) 530 assert.NilError(t, err) 531 golden.Assert(t, string(raw), "expected-jsonfile-timing-events") 532 }