gotest.tools/gotestsum@v1.11.0/cmd/main_test.go (about)

     1  package cmd
     2  
     3  import (
     4  	"bytes"
     5  	"encoding/json"
     6  	"os"
     7  	"os/exec"
     8  	"path/filepath"
     9  	"runtime"
    10  	"strings"
    11  	"testing"
    12  
    13  	"github.com/fatih/color"
    14  	"gotest.tools/gotestsum/testjson"
    15  	"gotest.tools/v3/assert"
    16  	"gotest.tools/v3/assert/cmp"
    17  	"gotest.tools/v3/env"
    18  	"gotest.tools/v3/golden"
    19  	"gotest.tools/v3/skip"
    20  )
    21  
    22  func TestUsage_WithFlagsFromSetupFlags(t *testing.T) {
    23  	env.PatchAll(t, nil)
    24  	patchNoColor(t, false)
    25  
    26  	name := "gotestsum"
    27  	flags, _ := setupFlags(name)
    28  	buf := new(bytes.Buffer)
    29  	usage(buf, name, flags)
    30  
    31  	golden.Assert(t, buf.String(), "gotestsum-help-text")
    32  }
    33  
    34  func patchNoColor(t *testing.T, value bool) {
    35  	orig := color.NoColor
    36  	color.NoColor = value
    37  	t.Cleanup(func() {
    38  		color.NoColor = orig
    39  	})
    40  }
    41  
    42  func TestOptions_Validate_FromFlags(t *testing.T) {
    43  	type testCase struct {
    44  		name     string
    45  		args     []string
    46  		expected string
    47  	}
    48  	fn := func(t *testing.T, tc testCase) {
    49  		flags, opts := setupFlags("gotestsum")
    50  		err := flags.Parse(tc.args)
    51  		assert.NilError(t, err)
    52  		opts.args = flags.Args()
    53  
    54  		err = opts.Validate()
    55  		if tc.expected == "" {
    56  			assert.NilError(t, err)
    57  			return
    58  		}
    59  		assert.ErrorContains(t, err, tc.expected, "opts: %#v", opts)
    60  	}
    61  	var testCases = []testCase{
    62  		{
    63  			name: "no flags",
    64  		},
    65  		{
    66  			name: "rerun flag, raw command",
    67  			args: []string{"--rerun-fails", "--raw-command", "--", "./test-all"},
    68  		},
    69  		{
    70  			name: "rerun flag, no go-test args",
    71  			args: []string{"--rerun-fails", "--"},
    72  		},
    73  		{
    74  			name:     "rerun flag, go-test args, no packages flag",
    75  			args:     []string{"--rerun-fails", "--", "./..."},
    76  			expected: "the list of packages to test must be specified by the --packages flag",
    77  		},
    78  		{
    79  			name: "rerun flag, go-test args, with packages flag",
    80  			args: []string{"--rerun-fails", "--packages", "./...", "--", "--foo"},
    81  		},
    82  		{
    83  			name: "rerun flag, no go-test args, with packages flag",
    84  			args: []string{"--rerun-fails", "--packages", "./..."},
    85  		},
    86  		{
    87  			name:     "rerun-fails with failfast",
    88  			args:     []string{"--rerun-fails", "--packages=./...", "--", "-failfast"},
    89  			expected: "-failfast can not be used with --rerun-fails",
    90  		},
    91  	}
    92  	for _, tc := range testCases {
    93  		t.Run(tc.name, func(t *testing.T) {
    94  			fn(t, tc)
    95  		})
    96  	}
    97  }
    98  
    99  func TestGoTestCmdArgs(t *testing.T) {
   100  	type testCase struct {
   101  		opts      *options
   102  		rerunOpts rerunOpts
   103  		env       []string
   104  		expected  []string
   105  	}
   106  
   107  	run := func(t *testing.T, name string, tc testCase) {
   108  		t.Helper()
   109  		runCase(t, name, func(t *testing.T) {
   110  			env.PatchAll(t, env.ToMap(tc.env))
   111  			actual := goTestCmdArgs(tc.opts, tc.rerunOpts)
   112  			assert.DeepEqual(t, actual, tc.expected)
   113  		})
   114  	}
   115  
   116  	run(t, "raw command", testCase{
   117  		opts: &options{
   118  			rawCommand: true,
   119  			args:       []string{"./script", "-test.timeout=20m"},
   120  		},
   121  		expected: []string{"./script", "-test.timeout=20m"},
   122  	})
   123  	run(t, "no args", testCase{
   124  		opts:     &options{},
   125  		expected: []string{"go", "test", "-json", "./..."},
   126  	})
   127  	run(t, "no args, with rerunPackageList arg", testCase{
   128  		opts: &options{
   129  			packages: []string{"./pkg"},
   130  		},
   131  		expected: []string{"go", "test", "-json", "./pkg"},
   132  	})
   133  	run(t, "TEST_DIRECTORY env var no args", testCase{
   134  		opts:     &options{},
   135  		env:      []string{"TEST_DIRECTORY=testdir"},
   136  		expected: []string{"go", "test", "-json", "testdir"},
   137  	})
   138  	run(t, "TEST_DIRECTORY env var with args", testCase{
   139  		opts: &options{
   140  			args: []string{"-tags=integration"},
   141  		},
   142  		env:      []string{"TEST_DIRECTORY=testdir"},
   143  		expected: []string{"go", "test", "-json", "-tags=integration", "testdir"},
   144  	})
   145  	run(t, "no -json arg", testCase{
   146  		opts: &options{
   147  			args: []string{"-timeout=2m", "./pkg"},
   148  		},
   149  		expected: []string{"go", "test", "-json", "-timeout=2m", "./pkg"},
   150  	})
   151  	run(t, "with -json arg", testCase{
   152  		opts: &options{
   153  			args: []string{"-json", "-timeout=2m", "./pkg"},
   154  		},
   155  		expected: []string{"go", "test", "-json", "-timeout=2m", "./pkg"},
   156  	})
   157  	run(t, "raw command, with rerunOpts", testCase{
   158  		opts: &options{
   159  			rawCommand: true,
   160  			args:       []string{"./script", "-test.timeout=20m"},
   161  		},
   162  		rerunOpts: rerunOpts{
   163  			runFlag: "-run=TestOne|TestTwo",
   164  			pkg:     "./fails",
   165  		},
   166  		expected: []string{"./script", "-test.timeout=20m", "-run=TestOne|TestTwo", "./fails"},
   167  	})
   168  	run(t, "no args, with rerunOpts", testCase{
   169  		opts: &options{},
   170  		rerunOpts: rerunOpts{
   171  			runFlag: "-run=TestOne|TestTwo",
   172  			pkg:     "./fails",
   173  		},
   174  		expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "./fails"},
   175  	})
   176  	run(t, "TEST_DIRECTORY env var, no args, with rerunOpts", testCase{
   177  		opts: &options{},
   178  		rerunOpts: rerunOpts{
   179  			runFlag: "-run=TestOne|TestTwo",
   180  			pkg:     "./fails",
   181  		},
   182  		env: []string{"TEST_DIRECTORY=testdir"},
   183  		// TEST_DIRECTORY should be overridden by rerun opts
   184  		expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "./fails"},
   185  	})
   186  	run(t, "TEST_DIRECTORY env var, with args, with rerunOpts", testCase{
   187  		opts: &options{
   188  			args: []string{"-tags=integration"},
   189  		},
   190  		rerunOpts: rerunOpts{
   191  			runFlag: "-run=TestOne|TestTwo",
   192  			pkg:     "./fails",
   193  		},
   194  		env:      []string{"TEST_DIRECTORY=testdir"},
   195  		expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "-tags=integration", "./fails"},
   196  	})
   197  	run(t, "no -json arg, with rerunOpts", testCase{
   198  		opts: &options{
   199  			args:     []string{"-timeout=2m"},
   200  			packages: []string{"./pkg"},
   201  		},
   202  		rerunOpts: rerunOpts{
   203  			runFlag: "-run=TestOne|TestTwo",
   204  			pkg:     "./fails",
   205  		},
   206  		expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "-timeout=2m", "./fails"},
   207  	})
   208  	run(t, "with -json arg, with rerunOpts", testCase{
   209  		opts: &options{
   210  			args:     []string{"-json", "-timeout=2m"},
   211  			packages: []string{"./pkg"},
   212  		},
   213  		rerunOpts: rerunOpts{
   214  			runFlag: "-run=TestOne|TestTwo",
   215  			pkg:     "./fails",
   216  		},
   217  		expected: []string{"go", "test", "-run=TestOne|TestTwo", "-json", "-timeout=2m", "./fails"},
   218  	})
   219  	run(t, "with args, with reunFailsPackageList args, with rerunOpts", testCase{
   220  		opts: &options{
   221  			args:     []string{"-timeout=2m"},
   222  			packages: []string{"./pkg1", "./pkg2", "./pkg3"},
   223  		},
   224  		rerunOpts: rerunOpts{
   225  			runFlag: "-run=TestOne|TestTwo",
   226  			pkg:     "./fails",
   227  		},
   228  		expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "-timeout=2m", "./fails"},
   229  	})
   230  	run(t, "with args, with reunFailsPackageList", testCase{
   231  		opts: &options{
   232  			args:     []string{"-timeout=2m"},
   233  			packages: []string{"./pkg1", "./pkg2", "./pkg3"},
   234  		},
   235  		expected: []string{"go", "test", "-json", "-timeout=2m", "./pkg1", "./pkg2", "./pkg3"},
   236  	})
   237  	run(t, "reunFailsPackageList args, with rerunOpts ", testCase{
   238  		opts: &options{
   239  			packages: []string{"./pkg1", "./pkg2", "./pkg3"},
   240  		},
   241  		rerunOpts: rerunOpts{
   242  			runFlag: "-run=TestOne|TestTwo",
   243  			pkg:     "./fails",
   244  		},
   245  		expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "./fails"},
   246  	})
   247  	run(t, "reunFailsPackageList args, with rerunOpts, with -args ", testCase{
   248  		opts: &options{
   249  			args:     []string{"before", "-args", "after"},
   250  			packages: []string{"./pkg1"},
   251  		},
   252  		rerunOpts: rerunOpts{
   253  			runFlag: "-run=TestOne|TestTwo",
   254  			pkg:     "./fails",
   255  		},
   256  		expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "before", "./fails", "-args", "after"},
   257  	})
   258  	run(t, "reunFailsPackageList args, with rerunOpts, with -args at end", testCase{
   259  		opts: &options{
   260  			args:     []string{"before", "-args"},
   261  			packages: []string{"./pkg1"},
   262  		},
   263  		rerunOpts: rerunOpts{
   264  			runFlag: "-run=TestOne|TestTwo",
   265  			pkg:     "./fails",
   266  		},
   267  		expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "before", "./fails", "-args"},
   268  	})
   269  	run(t, "reunFailsPackageList args, with -args at start", testCase{
   270  		opts: &options{
   271  			args:     []string{"-args", "after"},
   272  			packages: []string{"./pkg1"},
   273  		},
   274  		expected: []string{"go", "test", "-json", "./pkg1", "-args", "after"},
   275  	})
   276  	run(t, "-run arg at start, with rerunOpts ", testCase{
   277  		opts: &options{
   278  			args:     []string{"-run=TestFoo", "-args"},
   279  			packages: []string{"./pkg"},
   280  		},
   281  		rerunOpts: rerunOpts{
   282  			runFlag: "-run=TestOne|TestTwo",
   283  			pkg:     "./fails",
   284  		},
   285  		expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "./fails", "-args"},
   286  	})
   287  	run(t, "-run arg in middle, with rerunOpts ", testCase{
   288  		opts: &options{
   289  			args:     []string{"-count", "1", "--run", "TestFoo", "-args"},
   290  			packages: []string{"./pkg"},
   291  		},
   292  		rerunOpts: rerunOpts{
   293  			runFlag: "-run=TestOne|TestTwo",
   294  			pkg:     "./fails",
   295  		},
   296  		expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "-count", "1", "./fails", "-args"},
   297  	})
   298  	run(t, "-run arg at end with missing value, with rerunOpts ", testCase{
   299  		opts: &options{
   300  			args:     []string{"-count", "1", "-run"},
   301  			packages: []string{"./pkg"},
   302  		},
   303  		rerunOpts: rerunOpts{
   304  			runFlag: "-run=TestOne|TestTwo",
   305  			pkg:     "./fails",
   306  		},
   307  		expected: []string{"go", "test", "-json", "-run=TestOne|TestTwo", "-count", "1", "-run", "./fails"},
   308  	})
   309  }
   310  
   311  func runCase(t *testing.T, name string, fn func(t *testing.T)) {
   312  	t.Helper()
   313  	t.Run(name, func(t *testing.T) {
   314  		t.Helper()
   315  		t.Log("case:", name)
   316  		fn(t)
   317  	})
   318  }
   319  
   320  func TestRun_RerunFails_WithTooManyInitialFailures(t *testing.T) {
   321  	jsonFailed := `{"Package": "pkg", "Action": "run"}
   322  {"Package": "pkg", "Test": "TestOne", "Action": "run"}
   323  {"Package": "pkg", "Test": "TestOne", "Action": "fail"}
   324  {"Package": "pkg", "Test": "TestTwo", "Action": "run"}
   325  {"Package": "pkg", "Test": "TestTwo", "Action": "fail"}
   326  {"Package": "pkg", "Action": "fail"}
   327  `
   328  
   329  	fn := func(args []string) *proc {
   330  		return &proc{
   331  			cmd:    fakeWaiter{result: newExitCode("failed", 1)},
   332  			stdout: strings.NewReader(jsonFailed),
   333  			stderr: bytes.NewReader(nil),
   334  		}
   335  	}
   336  	reset := patchStartGoTestFn(fn)
   337  	defer reset()
   338  
   339  	out := new(bytes.Buffer)
   340  	opts := &options{
   341  		rawCommand:                   true,
   342  		args:                         []string{"./test.test"},
   343  		format:                       "testname",
   344  		rerunFailsMaxAttempts:        3,
   345  		rerunFailsMaxInitialFailures: 1,
   346  		stdout:                       out,
   347  		stderr:                       os.Stderr,
   348  		hideSummary:                  newHideSummaryValue(),
   349  	}
   350  	err := run(opts)
   351  	assert.ErrorContains(t, err, "number of test failures (2) exceeds maximum (1)", out.String())
   352  }
   353  
   354  func TestRun_RerunFails_BuildErrorPreventsRerun(t *testing.T) {
   355  	jsonFailed := `{"Package": "pkg", "Action": "run"}
   356  {"Package": "pkg", "Test": "TestOne", "Action": "run"}
   357  {"Package": "pkg", "Test": "TestOne", "Action": "fail"}
   358  {"Package": "pkg", "Test": "TestTwo", "Action": "run"}
   359  {"Package": "pkg", "Test": "TestTwo", "Action": "fail"}
   360  {"Package": "pkg", "Action": "fail"}
   361  `
   362  
   363  	fn := func(args []string) *proc {
   364  		return &proc{
   365  			cmd:    fakeWaiter{result: newExitCode("failed", 1)},
   366  			stdout: strings.NewReader(jsonFailed),
   367  			stderr: strings.NewReader("anything here is an error\n"),
   368  		}
   369  	}
   370  	reset := patchStartGoTestFn(fn)
   371  	defer reset()
   372  
   373  	out := new(bytes.Buffer)
   374  	opts := &options{
   375  		rawCommand:                   true,
   376  		args:                         []string{"./test.test"},
   377  		format:                       "testname",
   378  		rerunFailsMaxAttempts:        3,
   379  		rerunFailsMaxInitialFailures: 1,
   380  		stdout:                       out,
   381  		stderr:                       os.Stderr,
   382  		hideSummary:                  newHideSummaryValue(),
   383  	}
   384  	err := run(opts)
   385  	assert.ErrorContains(t, err, "rerun aborted because previous run had errors", out.String())
   386  }
   387  
   388  // type checking of os/exec.ExitError is done in a test file so that users
   389  // installing from source can continue to use versions prior to go1.12.
   390  var _ exitCoder = &exec.ExitError{}
   391  
   392  func TestRun_RerunFails_PanicPreventsRerun(t *testing.T) {
   393  	jsonFailed := `{"Package": "pkg", "Action": "run"}
   394  {"Package": "pkg", "Test": "TestOne", "Action": "run"}
   395  {"Package": "pkg", "Test": "TestOne", "Action": "output","Output":"panic: something went wrong\n"}
   396  {"Package": "pkg", "Action": "fail"}
   397  `
   398  
   399  	fn := func(args []string) *proc {
   400  		return &proc{
   401  			cmd:    fakeWaiter{result: newExitCode("failed", 1)},
   402  			stdout: strings.NewReader(jsonFailed),
   403  			stderr: bytes.NewReader(nil),
   404  		}
   405  	}
   406  	reset := patchStartGoTestFn(fn)
   407  	defer reset()
   408  
   409  	out := new(bytes.Buffer)
   410  	opts := &options{
   411  		rawCommand:                   true,
   412  		args:                         []string{"./test.test"},
   413  		format:                       "testname",
   414  		rerunFailsMaxAttempts:        3,
   415  		rerunFailsMaxInitialFailures: 1,
   416  		stdout:                       out,
   417  		stderr:                       os.Stderr,
   418  		hideSummary:                  newHideSummaryValue(),
   419  	}
   420  	err := run(opts)
   421  	assert.ErrorContains(t, err, "rerun aborted because previous run had a suspected panic", out.String())
   422  }
   423  
   424  func TestRun_InputFromStdin(t *testing.T) {
   425  	stdin := os.Stdin
   426  	t.Cleanup(func() { os.Stdin = stdin })
   427  
   428  	r, w, err := os.Pipe()
   429  	assert.NilError(t, err)
   430  	t.Cleanup(func() { _ = r.Close() })
   431  
   432  	os.Stdin = r
   433  
   434  	go func() {
   435  		defer func() { _ = w.Close() }()
   436  
   437  		e := json.NewEncoder(w)
   438  		for _, event := range []testjson.TestEvent{
   439  			{Action: "run", Package: "pkg"},
   440  			{Action: "run", Package: "pkg", Test: "TestOne"},
   441  			{Action: "fail", Package: "pkg", Test: "TestOne"},
   442  			{Action: "fail", Package: "pkg"},
   443  		} {
   444  			assert.Check(t, e.Encode(event))
   445  		}
   446  	}()
   447  
   448  	stdout := new(bytes.Buffer)
   449  	err = run(&options{
   450  		args:        []string{"cat"},
   451  		format:      "testname",
   452  		hideSummary: newHideSummaryValue(),
   453  		rawCommand:  true,
   454  
   455  		stdout: stdout,
   456  		stderr: os.Stderr,
   457  	})
   458  	assert.NilError(t, err)
   459  	assert.Assert(t, cmp.Contains(stdout.String(), "DONE 1"))
   460  }
   461  
   462  func TestRun_JsonFileIsSyncedBeforePostRunCommand(t *testing.T) {
   463  	skip.If(t, runtime.GOOS == "windows")
   464  
   465  	input := golden.Get(t, "../../testjson/testdata/input/go-test-json.out")
   466  
   467  	fn := func(args []string) *proc {
   468  		return &proc{
   469  			cmd:    fakeWaiter{},
   470  			stdout: bytes.NewReader(input),
   471  			stderr: bytes.NewReader(nil),
   472  		}
   473  	}
   474  	reset := patchStartGoTestFn(fn)
   475  	defer reset()
   476  
   477  	tmp := t.TempDir()
   478  	jsonFile := filepath.Join(tmp, "json.log")
   479  
   480  	out := new(bytes.Buffer)
   481  	opts := &options{
   482  		rawCommand:  true,
   483  		args:        []string{"./test.test"},
   484  		format:      "none",
   485  		stdout:      out,
   486  		stderr:      os.Stderr,
   487  		hideSummary: &hideSummaryValue{value: testjson.SummarizeNone},
   488  		jsonFile:    jsonFile,
   489  		postRunHookCmd: &commandValue{
   490  			command: []string{"cat", jsonFile},
   491  		},
   492  	}
   493  	err := run(opts)
   494  	assert.NilError(t, err)
   495  	expected := string(input)
   496  	_, actual, _ := strings.Cut(out.String(), "s\n") // remove the DONE line
   497  	assert.Equal(t, actual, expected)
   498  }
   499  
   500  func TestRun_JsonFileTimingEvents(t *testing.T) {
   501  	input := golden.Get(t, "../../testjson/testdata/input/go-test-json.out")
   502  
   503  	fn := func(args []string) *proc {
   504  		return &proc{
   505  			cmd:    fakeWaiter{},
   506  			stdout: bytes.NewReader(input),
   507  			stderr: bytes.NewReader(nil),
   508  		}
   509  	}
   510  	reset := patchStartGoTestFn(fn)
   511  	defer reset()
   512  
   513  	tmp := t.TempDir()
   514  	jsonFileTiming := filepath.Join(tmp, "json.log")
   515  
   516  	out := new(bytes.Buffer)
   517  	opts := &options{
   518  		rawCommand:           true,
   519  		args:                 []string{"./test.test"},
   520  		format:               "none",
   521  		stdout:               out,
   522  		stderr:               os.Stderr,
   523  		hideSummary:          &hideSummaryValue{value: testjson.SummarizeNone},
   524  		jsonFileTimingEvents: jsonFileTiming,
   525  	}
   526  	err := run(opts)
   527  	assert.NilError(t, err)
   528  
   529  	raw, err := os.ReadFile(jsonFileTiming)
   530  	assert.NilError(t, err)
   531  	golden.Assert(t, string(raw), "expected-jsonfile-timing-events")
   532  }