github.com/onsi/ginkgo@v1.16.6-0.20211118180735-4e1925ba4c95/types/config.go (about) 1 /* 2 Ginkgo accepts a number of configuration options. 3 These are documented [here](http://onsi.github.io/ginkgo/#the-ginkgo-cli) 4 */ 5 6 package types 7 8 import ( 9 "flag" 10 "os" 11 "runtime" 12 "strconv" 13 "strings" 14 "time" 15 ) 16 17 // Configuration controlling how an individual test suite is run 18 type SuiteConfig struct { 19 RandomSeed int64 20 RandomizeAllSpecs bool 21 FocusStrings []string 22 SkipStrings []string 23 FocusFiles []string 24 SkipFiles []string 25 LabelFilter string 26 FailOnPending bool 27 FailFast bool 28 FlakeAttempts int 29 EmitSpecProgress bool 30 DryRun bool 31 Timeout time.Duration 32 OutputInterceptorMode string 33 34 ParallelProcess int 35 ParallelTotal int 36 ParallelHost string 37 } 38 39 func NewDefaultSuiteConfig() SuiteConfig { 40 return SuiteConfig{ 41 RandomSeed: time.Now().Unix(), 42 Timeout: time.Hour, 43 ParallelProcess: 1, 44 ParallelTotal: 1, 45 } 46 } 47 48 type VerbosityLevel uint 49 50 const ( 51 VerbosityLevelSuccinct VerbosityLevel = iota 52 VerbosityLevelNormal 53 VerbosityLevelVerbose 54 VerbosityLevelVeryVerbose 55 ) 56 57 func (vl VerbosityLevel) GT(comp VerbosityLevel) bool { 58 return vl > comp 59 } 60 61 func (vl VerbosityLevel) GTE(comp VerbosityLevel) bool { 62 return vl >= comp 63 } 64 65 func (vl VerbosityLevel) Is(comp VerbosityLevel) bool { 66 return vl == comp 67 } 68 69 func (vl VerbosityLevel) LTE(comp VerbosityLevel) bool { 70 return vl <= comp 71 } 72 73 func (vl VerbosityLevel) LT(comp VerbosityLevel) bool { 74 return vl < comp 75 } 76 77 // Configuration for Ginkgo's reporter 78 type ReporterConfig struct { 79 NoColor bool 80 SlowSpecThreshold time.Duration 81 Succinct bool 82 Verbose bool 83 VeryVerbose bool 84 FullTrace bool 85 AlwaysEmitGinkgoWriter bool 86 87 JSONReport string 88 JUnitReport string 89 TeamcityReport string 90 } 91 92 func (rc ReporterConfig) Verbosity() VerbosityLevel { 93 if rc.Succinct { 94 return VerbosityLevelSuccinct 95 } else if rc.Verbose { 96 return VerbosityLevelVerbose 97 } else if rc.VeryVerbose { 98 return VerbosityLevelVeryVerbose 99 } 100 return VerbosityLevelNormal 101 } 102 103 func (rc ReporterConfig) WillGenerateReport() bool { 104 return rc.JSONReport != "" || rc.JUnitReport != "" || rc.TeamcityReport != "" 105 } 106 107 func NewDefaultReporterConfig() ReporterConfig { 108 return ReporterConfig{ 109 SlowSpecThreshold: 5 * time.Second, 110 } 111 } 112 113 // Configuration for the Ginkgo CLI 114 type CLIConfig struct { 115 //for build, run, and watch 116 Recurse bool 117 SkipPackage string 118 RequireSuite bool 119 NumCompilers int 120 121 //for run and watch only 122 Procs int 123 Parallel bool 124 AfterRunHook string 125 OutputDir string 126 KeepSeparateCoverprofiles bool 127 KeepSeparateReports bool 128 129 //for run only 130 KeepGoing bool 131 UntilItFails bool 132 Repeat int 133 RandomizeSuites bool 134 135 //for watch only 136 Depth int 137 WatchRegExp string 138 } 139 140 func NewDefaultCLIConfig() CLIConfig { 141 return CLIConfig{ 142 Depth: 1, 143 WatchRegExp: `\.go$`, 144 } 145 } 146 147 func (g CLIConfig) ComputedProcs() int { 148 if g.Procs > 0 { 149 return g.Procs 150 } 151 152 n := 1 153 if g.Parallel { 154 n = runtime.NumCPU() 155 if n > 4 { 156 n = n - 1 157 } 158 } 159 return n 160 } 161 162 func (g CLIConfig) ComputedNumCompilers() int { 163 if g.NumCompilers > 0 { 164 return g.NumCompilers 165 } 166 167 return runtime.NumCPU() 168 } 169 170 // Configuration for the Ginkgo CLI capturing available go flags 171 // A subset of Go flags are exposed by Ginkgo. Some are avaiable at compile time (e.g. ginkgo build) and others only at run time (e.g. ginkgo run - which has both build and run time flags). 172 // More details can be found at: 173 // https://docs.google.com/spreadsheets/d/1zkp-DS4hU4sAJl5eHh1UmgwxCPQhf3s5a8fbiOI8tJU/ 174 type GoFlagsConfig struct { 175 //build-time flags for code-and-performance analysis 176 Race bool 177 Cover bool 178 CoverMode string 179 CoverPkg string 180 Vet string 181 182 //run-time flags for code-and-performance analysis 183 BlockProfile string 184 BlockProfileRate int 185 CoverProfile string 186 CPUProfile string 187 MemProfile string 188 MemProfileRate int 189 MutexProfile string 190 MutexProfileFraction int 191 Trace string 192 193 //build-time flags for building 194 A bool 195 ASMFlags string 196 BuildMode string 197 Compiler string 198 GCCGoFlags string 199 GCFlags string 200 InstallSuffix string 201 LDFlags string 202 LinkShared bool 203 Mod string 204 N bool 205 ModFile string 206 ModCacheRW bool 207 MSan bool 208 PkgDir string 209 Tags string 210 TrimPath bool 211 ToolExec string 212 Work bool 213 X bool 214 } 215 216 func NewDefaultGoFlagsConfig() GoFlagsConfig { 217 return GoFlagsConfig{} 218 } 219 220 func (g GoFlagsConfig) BinaryMustBePreserved() bool { 221 return g.BlockProfile != "" || g.CPUProfile != "" || g.MemProfile != "" || g.MutexProfile != "" 222 } 223 224 // Configuration that were deprecated in 2.0 225 type deprecatedConfig struct { 226 DebugParallel bool 227 NoisySkippings bool 228 NoisyPendings bool 229 RegexScansFilePath bool 230 SlowSpecThresholdWithFLoatUnits float64 231 Stream bool 232 Notify bool 233 } 234 235 // Flags 236 237 // Flags sections used by both the CLI and the Ginkgo test process 238 var FlagSections = GinkgoFlagSections{ 239 {Key: "multiple-suites", Style: "{{dark-green}}", Heading: "Running Multiple Test Suites"}, 240 {Key: "order", Style: "{{green}}", Heading: "Controlling Test Order"}, 241 {Key: "parallel", Style: "{{yellow}}", Heading: "Controlling Test Parallelism"}, 242 {Key: "low-level-parallel", Style: "{{yellow}}", Heading: "Controlling Test Parallelism", 243 Description: "These are set by the Ginkgo CLI, {{red}}{{bold}}do not set them manually{{/}} via go test.\nUse ginkgo -p or ginkgo -procs=N instead."}, 244 {Key: "filter", Style: "{{cyan}}", Heading: "Filtering Tests"}, 245 {Key: "failure", Style: "{{red}}", Heading: "Failure Handling"}, 246 {Key: "output", Style: "{{magenta}}", Heading: "Controlling Output Formatting"}, 247 {Key: "code-and-coverage-analysis", Style: "{{orange}}", Heading: "Code and Coverage Analysis"}, 248 {Key: "performance-analysis", Style: "{{coral}}", Heading: "Performance Analysis"}, 249 {Key: "debug", Style: "{{blue}}", Heading: "Debugging Tests", 250 Description: "In addition to these flags, Ginkgo supports a few debugging environment variables. To change the parallel server protocol set {{blue}}GINKGO_PARALLEL_PROTOCOL{{/}} to {{bold}}HTTP{{/}}. To avoid pruning callstacks set {{blue}}GINKGO_PRUNE_STACK{{/}} to {{bold}}FALSE{{/}}."}, 251 {Key: "watch", Style: "{{light-yellow}}", Heading: "Controlling Ginkgo Watch"}, 252 {Key: "misc", Style: "{{light-gray}}", Heading: "Miscellaneous"}, 253 {Key: "go-build", Style: "{{light-gray}}", Heading: "Go Build Flags", Succinct: true, 254 Description: "These flags are inherited from go build. Run {{bold}}ginkgo help build{{/}} for more detailed flag documentation."}, 255 } 256 257 // SuiteConfigFlags provides flags for the Ginkgo test process, and CLI 258 var SuiteConfigFlags = GinkgoFlags{ 259 {KeyPath: "S.RandomSeed", Name: "seed", SectionKey: "order", UsageDefaultValue: "randomly generated by Ginkgo", 260 Usage: "The seed used to randomize the spec suite."}, 261 {KeyPath: "S.RandomizeAllSpecs", Name: "randomize-all", SectionKey: "order", DeprecatedName: "randomizeAllSpecs", DeprecatedDocLink: "changed-command-line-flags", 262 Usage: "If set, ginkgo will randomize all specs together. By default, ginkgo only randomizes the top level Describe, Context and When containers."}, 263 264 {KeyPath: "S.FailOnPending", Name: "fail-on-pending", SectionKey: "failure", DeprecatedName: "failOnPending", DeprecatedDocLink: "changed-command-line-flags", 265 Usage: "If set, ginkgo will mark the test suite as failed if any specs are pending."}, 266 {KeyPath: "S.FailFast", Name: "fail-fast", SectionKey: "failure", DeprecatedName: "failFast", DeprecatedDocLink: "changed-command-line-flags", 267 Usage: "If set, ginkgo will stop running a test suite after a failure occurs."}, 268 {KeyPath: "S.FlakeAttempts", Name: "flake-attempts", SectionKey: "failure", UsageDefaultValue: "0 - failed tests are not retried", DeprecatedName: "flakeAttempts", DeprecatedDocLink: "changed-command-line-flags", 269 Usage: "Make up to this many attempts to run each spec. If any of the attempts succeed, the suite will not be failed."}, 270 271 {KeyPath: "S.DryRun", Name: "dry-run", SectionKey: "debug", DeprecatedName: "dryRun", DeprecatedDocLink: "changed-command-line-flags", 272 Usage: "If set, ginkgo will walk the test hierarchy without actually running anything. Best paired with -v."}, 273 {KeyPath: "S.EmitSpecProgress", Name: "progress", SectionKey: "debug", 274 Usage: "If set, ginkgo will emit progress information as each spec runs to the GinkgoWriter."}, 275 {KeyPath: "S.Timeout", Name: "timeout", SectionKey: "debug", UsageDefaultValue: "1h", 276 Usage: "Test suite fails if it does not complete within the specified timeout."}, 277 {KeyPath: "S.OutputInterceptorMode", Name: "output-interceptor-mode", SectionKey: "debug", UsageArgument: "dup, swap, or none", 278 Usage: "If set, ginkgo will use the specified output interception strategy when running in parallel. Defaults to dup on unix and swap on windows."}, 279 280 {KeyPath: "S.LabelFilter", Name: "label-filter", SectionKey: "filter", UsageArgument: "expression", 281 Usage: "If set, ginkgo will only run specs with labels that match the label-filter. The passed-in expression can include boolean operations (!, &&, ||, ','), groupings via '()', and regular expresions '/regexp/'. e.g. '(cat || dog) && !fruit'"}, 282 {KeyPath: "S.FocusStrings", Name: "focus", SectionKey: "filter", 283 Usage: "If set, ginkgo will only run specs that match this regular expression. Can be specified multiple times, values are ORed."}, 284 {KeyPath: "S.SkipStrings", Name: "skip", SectionKey: "filter", 285 Usage: "If set, ginkgo will only run specs that do not match this regular expression. Can be specified multiple times, values are ORed."}, 286 {KeyPath: "S.FocusFiles", Name: "focus-file", SectionKey: "filter", UsageArgument: "file (regexp) | file:line | file:lineA-lineB | file:line,line,line", 287 Usage: "If set, ginkgo will only run specs in matching files. Can be specified multiple times, values are ORed."}, 288 {KeyPath: "S.SkipFiles", Name: "skip-file", SectionKey: "filter", UsageArgument: "file (regexp) | file:line | file:lineA-lineB | file:line,line,line", 289 Usage: "If set, ginkgo will skip specs in matching files. Can be specified multiple times, values are ORed."}, 290 291 {KeyPath: "D.RegexScansFilePath", DeprecatedName: "regexScansFilePath", DeprecatedDocLink: "removed--regexscansfilepath", DeprecatedVersion: "2.0.0"}, 292 {KeyPath: "D.DebugParallel", DeprecatedName: "debug", DeprecatedDocLink: "removed--debug", DeprecatedVersion: "2.0.0"}, 293 } 294 295 // ParallelConfigFlags provides flags for the Ginkgo test process (not the CLI) 296 var ParallelConfigFlags = GinkgoFlags{ 297 {KeyPath: "S.ParallelProcess", Name: "parallel.process", SectionKey: "low-level-parallel", UsageDefaultValue: "1", 298 Usage: "This worker process's (one-indexed) process number. For running specs in parallel."}, 299 {KeyPath: "S.ParallelTotal", Name: "parallel.total", SectionKey: "low-level-parallel", UsageDefaultValue: "1", 300 Usage: "The total number of worker processes. For running specs in parallel."}, 301 {KeyPath: "S.ParallelHost", Name: "parallel.host", SectionKey: "low-level-parallel", UsageDefaultValue: "set by Ginkgo CLI", 302 Usage: "The address for the server that will synchronize the processes."}, 303 } 304 305 // ReporterConfigFlags provides flags for the Ginkgo test process, and CLI 306 var ReporterConfigFlags = GinkgoFlags{ 307 {KeyPath: "R.NoColor", Name: "no-color", SectionKey: "output", DeprecatedName: "noColor", DeprecatedDocLink: "changed-command-line-flags", 308 Usage: "If set, suppress color output in default reporter."}, 309 {KeyPath: "R.SlowSpecThreshold", Name: "slow-spec-threshold", SectionKey: "output", UsageArgument: "duration", UsageDefaultValue: "5s", 310 Usage: "Specs that take longer to run than this threshold are flagged as slow by the default reporter."}, 311 {KeyPath: "R.Verbose", Name: "v", SectionKey: "output", 312 Usage: "If set, emits more output including GinkgoWriter contents."}, 313 {KeyPath: "R.VeryVerbose", Name: "vv", SectionKey: "output", 314 Usage: "If set, emits with maximal verbosity - includes skipped and pending tests."}, 315 {KeyPath: "R.Succinct", Name: "succinct", SectionKey: "output", 316 Usage: "If set, default reporter prints out a very succinct report"}, 317 {KeyPath: "R.FullTrace", Name: "trace", SectionKey: "output", 318 Usage: "If set, default reporter prints out the full stack trace when a failure occurs"}, 319 {KeyPath: "R.AlwaysEmitGinkgoWriter", Name: "always-emit-ginkgo-writer", SectionKey: "output", DeprecatedName: "reportPassed", DeprecatedDocLink: "renamed--reportpassed", 320 Usage: "If set, default reporter prints out captured output of passed tests."}, 321 322 {KeyPath: "R.JSONReport", Name: "json-report", UsageArgument: "filename.json", SectionKey: "output", 323 Usage: "If set, Ginkgo will generate a JSON-formatted test report at the specified location."}, 324 {KeyPath: "R.JUnitReport", Name: "junit-report", UsageArgument: "filename.xml", SectionKey: "output", DeprecatedName: "reportFile", DeprecatedDocLink: "improved-reporting-infrastructure", 325 Usage: "If set, Ginkgo will generate a conformant junit test report in the specified file."}, 326 {KeyPath: "R.TeamcityReport", Name: "teamcity-report", UsageArgument: "filename", SectionKey: "output", 327 Usage: "If set, Ginkgo will generate a Teamcity-formatted test report at the specified location."}, 328 329 {KeyPath: "D.SlowSpecThresholdWithFLoatUnits", DeprecatedName: "slowSpecThreshold", DeprecatedDocLink: "changed--slowspecthreshold", 330 Usage: "use --slow-spec-threshold instead and pass in a duration string (e.g. '5s', not '5.0')"}, 331 {KeyPath: "D.NoisyPendings", DeprecatedName: "noisyPendings", DeprecatedDocLink: "removed--noisypendings-and--noisyskippings", DeprecatedVersion: "2.0.0"}, 332 {KeyPath: "D.NoisySkippings", DeprecatedName: "noisySkippings", DeprecatedDocLink: "removed--noisypendings-and--noisyskippings", DeprecatedVersion: "2.0.0"}, 333 } 334 335 // BuildTestSuiteFlagSet attaches to the CommandLine flagset and provides flags for the Ginkgo test process 336 func BuildTestSuiteFlagSet(suiteConfig *SuiteConfig, reporterConfig *ReporterConfig) (GinkgoFlagSet, error) { 337 flags := SuiteConfigFlags.CopyAppend(ParallelConfigFlags...).CopyAppend(ReporterConfigFlags...) 338 flags = flags.WithPrefix("ginkgo") 339 bindings := map[string]interface{}{ 340 "S": suiteConfig, 341 "R": reporterConfig, 342 "D": &deprecatedConfig{}, 343 } 344 extraGoFlagsSection := GinkgoFlagSection{Style: "{{gray}}", Heading: "Go test flags"} 345 346 return NewAttachedGinkgoFlagSet(flag.CommandLine, flags, bindings, FlagSections, extraGoFlagsSection) 347 } 348 349 // VetConfig validates that the Ginkgo test process' configuration is sound 350 func VetConfig(flagSet GinkgoFlagSet, suiteConfig SuiteConfig, reporterConfig ReporterConfig) []error { 351 errors := []error{} 352 353 if flagSet.WasSet("count") || flagSet.WasSet("test.count") { 354 flag := flagSet.Lookup("count") 355 if flag == nil { 356 flag = flagSet.Lookup("test.count") 357 } 358 count, err := strconv.Atoi(flag.Value.String()) 359 if err != nil || count != 1 { 360 errors = append(errors, GinkgoErrors.InvalidGoFlagCount()) 361 } 362 } 363 364 if flagSet.WasSet("parallel") || flagSet.WasSet("test.parallel") { 365 errors = append(errors, GinkgoErrors.InvalidGoFlagParallel()) 366 } 367 368 if suiteConfig.ParallelTotal < 1 { 369 errors = append(errors, GinkgoErrors.InvalidParallelTotalConfiguration()) 370 } 371 372 if suiteConfig.ParallelProcess > suiteConfig.ParallelTotal || suiteConfig.ParallelProcess < 1 { 373 errors = append(errors, GinkgoErrors.InvalidParallelProcessConfiguration()) 374 } 375 376 if suiteConfig.ParallelTotal > 1 && suiteConfig.ParallelHost == "" { 377 errors = append(errors, GinkgoErrors.MissingParallelHostConfiguration()) 378 } 379 380 if suiteConfig.DryRun && suiteConfig.ParallelTotal > 1 { 381 errors = append(errors, GinkgoErrors.DryRunInParallelConfiguration()) 382 } 383 384 if len(suiteConfig.FocusFiles) > 0 { 385 _, err := ParseFileFilters(suiteConfig.FocusFiles) 386 if err != nil { 387 errors = append(errors, err) 388 } 389 } 390 391 if len(suiteConfig.SkipFiles) > 0 { 392 _, err := ParseFileFilters(suiteConfig.SkipFiles) 393 if err != nil { 394 errors = append(errors, err) 395 } 396 } 397 398 if suiteConfig.LabelFilter != "" { 399 _, err := ParseLabelFilter(suiteConfig.LabelFilter) 400 if err != nil { 401 errors = append(errors, err) 402 } 403 } 404 405 switch strings.ToLower(suiteConfig.OutputInterceptorMode) { 406 case "", "dup", "swap", "none": 407 default: 408 errors = append(errors, GinkgoErrors.InvalidOutputInterceptorModeConfiguration(suiteConfig.OutputInterceptorMode)) 409 } 410 411 numVerbosity := 0 412 for _, v := range []bool{reporterConfig.Succinct, reporterConfig.Verbose, reporterConfig.VeryVerbose} { 413 if v { 414 numVerbosity++ 415 } 416 } 417 if numVerbosity > 1 { 418 errors = append(errors, GinkgoErrors.ConflictingVerbosityConfiguration()) 419 } 420 421 return errors 422 } 423 424 // GinkgoCLISharedFlags provides flags shared by the Ginkgo CLI's build, watch, and run commands 425 var GinkgoCLISharedFlags = GinkgoFlags{ 426 {KeyPath: "C.Recurse", Name: "r", SectionKey: "multiple-suites", 427 Usage: "If set, ginkgo finds and runs test suites under the current directory recursively."}, 428 {KeyPath: "C.SkipPackage", Name: "skip-package", SectionKey: "multiple-suites", DeprecatedName: "skipPackage", DeprecatedDocLink: "changed-command-line-flags", 429 UsageArgument: "comma-separated list of packages", 430 Usage: "A comma-separated list of package names to be skipped. If any part of the package's path matches, that package is ignored."}, 431 {KeyPath: "C.RequireSuite", Name: "require-suite", SectionKey: "failure", DeprecatedName: "requireSuite", DeprecatedDocLink: "changed-command-line-flags", 432 Usage: "If set, Ginkgo fails if there are ginkgo tests in a directory but no invocation of RunSpecs."}, 433 {KeyPath: "C.NumCompilers", Name: "compilers", SectionKey: "multiple-suites", UsageDefaultValue: "0 (will autodetect)", 434 Usage: "When running multiple packages, the number of concurrent compilations to perform."}, 435 } 436 437 // GinkgoCLIRunAndWatchFlags provides flags shared by the Ginkgo CLI's build and watch commands (but not run) 438 var GinkgoCLIRunAndWatchFlags = GinkgoFlags{ 439 {KeyPath: "C.Procs", Name: "procs", SectionKey: "parallel", UsageDefaultValue: "1 (run in series)", 440 Usage: "The number of parallel test nodes to run."}, 441 {KeyPath: "C.Procs", Name: "nodes", SectionKey: "parallel", UsageDefaultValue: "1 (run in series)", 442 Usage: "--nodes is an alias for --procs"}, 443 {KeyPath: "C.Parallel", Name: "p", SectionKey: "parallel", 444 Usage: "If set, ginkgo will run in parallel with an auto-detected number of nodes."}, 445 {KeyPath: "C.AfterRunHook", Name: "after-run-hook", SectionKey: "misc", DeprecatedName: "afterSuiteHook", DeprecatedDocLink: "changed-command-line-flags", 446 Usage: "Command to run when a test suite completes."}, 447 {KeyPath: "C.OutputDir", Name: "output-dir", SectionKey: "output", UsageArgument: "directory", DeprecatedName: "outputdir", DeprecatedDocLink: "improved-profiling-support", 448 Usage: "A location to place all generated profiles and reports."}, 449 {KeyPath: "C.KeepSeparateCoverprofiles", Name: "keep-separate-coverprofiles", SectionKey: "code-and-coverage-analysis", 450 Usage: "If set, Ginkgo does not merge coverprofiles into one monolithic coverprofile. The coverprofiles will remain in their respective package directories or in -output-dir if set."}, 451 {KeyPath: "C.KeepSeparateReports", Name: "keep-separate-reports", SectionKey: "output", 452 Usage: "If set, Ginkgo does not merge per-suite reports (e.g. -json-report) into one monolithic report for the entire testrun. The reports will remain in their respective package directories or in -output-dir if set."}, 453 454 {KeyPath: "D.Stream", DeprecatedName: "stream", DeprecatedDocLink: "removed--stream", DeprecatedVersion: "2.0.0"}, 455 {KeyPath: "D.Notify", DeprecatedName: "notify", DeprecatedDocLink: "removed--notify", DeprecatedVersion: "2.0.0"}, 456 } 457 458 // GinkgoCLIRunFlags provides flags for Ginkgo CLI's run command that aren't shared by any other commands 459 var GinkgoCLIRunFlags = GinkgoFlags{ 460 {KeyPath: "C.KeepGoing", Name: "keep-going", SectionKey: "multiple-suites", DeprecatedName: "keepGoing", DeprecatedDocLink: "changed-command-line-flags", 461 Usage: "If set, failures from earlier test suites do not prevent later test suites from running."}, 462 {KeyPath: "C.UntilItFails", Name: "until-it-fails", SectionKey: "debug", DeprecatedName: "untilItFails", DeprecatedDocLink: "changed-command-line-flags", 463 Usage: "If set, ginkgo will keep rerunning test suites until a failure occurs."}, 464 {KeyPath: "C.Repeat", Name: "repeat", SectionKey: "debug", UsageArgument: "n", UsageDefaultValue: "0 - i.e. no repetition, run only once", 465 Usage: "The number of times to re-run a test-suite. Useful for debugging flaky tests. If set to N the suite will be run N+1 times and will be required to pass each time."}, 466 {KeyPath: "C.RandomizeSuites", Name: "randomize-suites", SectionKey: "order", DeprecatedName: "randomizeSuites", DeprecatedDocLink: "changed-command-line-flags", 467 Usage: "If set, ginkgo will randomize the order in which test suites run."}, 468 } 469 470 // GinkgoCLIRunFlags provides flags for Ginkgo CLI's watch command that aren't shared by any other commands 471 var GinkgoCLIWatchFlags = GinkgoFlags{ 472 {KeyPath: "C.Depth", Name: "depth", SectionKey: "watch", 473 Usage: "Ginkgo will watch dependencies down to this depth in the dependency tree."}, 474 {KeyPath: "C.WatchRegExp", Name: "watch-regexp", SectionKey: "watch", DeprecatedName: "watchRegExp", DeprecatedDocLink: "changed-command-line-flags", 475 UsageArgument: "Regular Expression", 476 UsageDefaultValue: `\.go$`, 477 Usage: "Only files matching this regular expression will be watched for changes."}, 478 } 479 480 // GoBuildFlags provides flags for the Ginkgo CLI build, run, and watch commands that capture go's build-time flags. These are passed to go test -c by the ginkgo CLI 481 var GoBuildFlags = GinkgoFlags{ 482 {KeyPath: "Go.Race", Name: "race", SectionKey: "code-and-coverage-analysis", 483 Usage: "enable data race detection. Supported only on linux/amd64, freebsd/amd64, darwin/amd64, windows/amd64, linux/ppc64le and linux/arm64 (only for 48-bit VMA)."}, 484 {KeyPath: "Go.Vet", Name: "vet", UsageArgument: "list", SectionKey: "code-and-coverage-analysis", 485 Usage: `Configure the invocation of "go vet" during "go test" to use the comma-separated list of vet checks. If list is empty, "go test" runs "go vet" with a curated list of checks believed to be always worth addressing. If list is "off", "go test" does not run "go vet" at all. Available checks can be found by running 'go doc cmd/vet'`}, 486 {KeyPath: "Go.Cover", Name: "cover", SectionKey: "code-and-coverage-analysis", 487 Usage: "Enable coverage analysis. Note that because coverage works by annotating the source code before compilation, compilation and test failures with coverage enabled may report line numbers that don't correspond to the original sources."}, 488 {KeyPath: "Go.CoverMode", Name: "covermode", UsageArgument: "set,count,atomic", SectionKey: "code-and-coverage-analysis", 489 Usage: `Set the mode for coverage analysis for the package[s] being tested. 'set': does this statement run? 'count': how many times does this statement run? 'atomic': like count, but correct in multithreaded tests and more expensive (must use atomic with -race). Sets -cover`}, 490 {KeyPath: "Go.CoverPkg", Name: "coverpkg", UsageArgument: "pattern1,pattern2,pattern3", SectionKey: "code-and-coverage-analysis", 491 Usage: "Apply coverage analysis in each test to packages matching the patterns. The default is for each test to analyze only the package being tested. See 'go help packages' for a description of package patterns. Sets -cover."}, 492 493 {KeyPath: "Go.A", Name: "a", SectionKey: "go-build", 494 Usage: "force rebuilding of packages that are already up-to-date."}, 495 {KeyPath: "Go.ASMFlags", Name: "asmflags", UsageArgument: "'[pattern=]arg list'", SectionKey: "go-build", 496 Usage: "arguments to pass on each go tool asm invocation."}, 497 {KeyPath: "Go.BuildMode", Name: "buildmode", UsageArgument: "mode", SectionKey: "go-build", 498 Usage: "build mode to use. See 'go help buildmode' for more."}, 499 {KeyPath: "Go.Compiler", Name: "compiler", UsageArgument: "name", SectionKey: "go-build", 500 Usage: "name of compiler to use, as in runtime.Compiler (gccgo or gc)."}, 501 {KeyPath: "Go.GCCGoFlags", Name: "gccgoflags", UsageArgument: "'[pattern=]arg list'", SectionKey: "go-build", 502 Usage: "arguments to pass on each gccgo compiler/linker invocation."}, 503 {KeyPath: "Go.GCFlags", Name: "gcflags", UsageArgument: "'[pattern=]arg list'", SectionKey: "go-build", 504 Usage: "arguments to pass on each go tool compile invocation."}, 505 {KeyPath: "Go.InstallSuffix", Name: "installsuffix", SectionKey: "go-build", 506 Usage: "a suffix to use in the name of the package installation directory, in order to keep output separate from default builds. If using the -race flag, the install suffix is automatically set to raceor, if set explicitly, has _race appended to it. Likewise for the -msan flag. Using a -buildmode option that requires non-default compile flags has a similar effect."}, 507 {KeyPath: "Go.LDFlags", Name: "ldflags", UsageArgument: "'[pattern=]arg list'", SectionKey: "go-build", 508 Usage: "arguments to pass on each go tool link invocation."}, 509 {KeyPath: "Go.LinkShared", Name: "linkshared", SectionKey: "go-build", 510 Usage: "build code that will be linked against shared libraries previously created with -buildmode=shared."}, 511 {KeyPath: "Go.Mod", Name: "mod", UsageArgument: "mode (readonly, vendor, or mod)", SectionKey: "go-build", 512 Usage: "module download mode to use: readonly, vendor, or mod. See 'go help modules' for more."}, 513 {KeyPath: "Go.ModCacheRW", Name: "modcacherw", SectionKey: "go-build", 514 Usage: "leave newly-created directories in the module cache read-write instead of making them read-only."}, 515 {KeyPath: "Go.ModFile", Name: "modfile", UsageArgument: "file", SectionKey: "go-build", 516 Usage: `in module aware mode, read (and possibly write) an alternate go.mod file instead of the one in the module root directory. A file named go.mod must still be present in order to determine the module root directory, but it is not accessed. When -modfile is specified, an alternate go.sum file is also used: its path is derived from the -modfile flag by trimming the ".mod" extension and appending ".sum".`}, 517 {KeyPath: "Go.MSan", Name: "msan", SectionKey: "go-build", 518 Usage: "enable interoperation with memory sanitizer. Supported only on linux/amd64, linux/arm64 and only with Clang/LLVM as the host C compiler. On linux/arm64, pie build mode will be used."}, 519 {KeyPath: "Go.N", Name: "n", SectionKey: "go-build", 520 Usage: "print the commands but do not run them."}, 521 {KeyPath: "Go.PkgDir", Name: "pkgdir", UsageArgument: "dir", SectionKey: "go-build", 522 Usage: "install and load all packages from dir instead of the usual locations. For example, when building with a non-standard configuration, use -pkgdir to keep generated packages in a separate location."}, 523 {KeyPath: "Go.Tags", Name: "tags", UsageArgument: "tag,list", SectionKey: "go-build", 524 Usage: "a comma-separated list of build tags to consider satisfied during the build. For more information about build tags, see the description of build constraints in the documentation for the go/build package. (Earlier versions of Go used a space-separated list, and that form is deprecated but still recognized.)"}, 525 {KeyPath: "Go.TrimPath", Name: "trimpath", SectionKey: "go-build", 526 Usage: `remove all file system paths from the resulting executable. Instead of absolute file system paths, the recorded file names will begin with either "go" (for the standard library), or a module path@version (when using modules), or a plain import path (when using GOPATH).`}, 527 {KeyPath: "Go.ToolExec", Name: "toolexec", UsageArgument: "'cmd args'", SectionKey: "go-build", 528 Usage: "a program to use to invoke toolchain programs like vet and asm. For example, instead of running asm, the go command will run cmd args /path/to/asm <arguments for asm>'."}, 529 {KeyPath: "Go.Work", Name: "work", SectionKey: "go-build", 530 Usage: "print the name of the temporary work directory and do not delete it when exiting."}, 531 {KeyPath: "Go.X", Name: "x", SectionKey: "go-build", 532 Usage: "print the commands."}, 533 } 534 535 // GoRunFlags provides flags for the Ginkgo CLI run, and watch commands that capture go's run-time flags. These are passed to the compiled test binary by the ginkgo CLI 536 var GoRunFlags = GinkgoFlags{ 537 {KeyPath: "Go.CoverProfile", Name: "coverprofile", UsageArgument: "file", SectionKey: "code-and-coverage-analysis", 538 Usage: `Write a coverage profile to the file after all tests have passed. Sets -cover.`}, 539 {KeyPath: "Go.BlockProfile", Name: "blockprofile", UsageArgument: "file", SectionKey: "performance-analysis", 540 Usage: `Write a goroutine blocking profile to the specified file when all tests are complete. Preserves test binary.`}, 541 {KeyPath: "Go.BlockProfileRate", Name: "blockprofilerate", UsageArgument: "rate", SectionKey: "performance-analysis", 542 Usage: `Control the detail provided in goroutine blocking profiles by calling runtime.SetBlockProfileRate with rate. See 'go doc runtime.SetBlockProfileRate'. The profiler aims to sample, on average, one blocking event every n nanoseconds the program spends blocked. By default, if -test.blockprofile is set without this flag, all blocking events are recorded, equivalent to -test.blockprofilerate=1.`}, 543 {KeyPath: "Go.CPUProfile", Name: "cpuprofile", UsageArgument: "file", SectionKey: "performance-analysis", 544 Usage: `Write a CPU profile to the specified file before exiting. Preserves test binary.`}, 545 {KeyPath: "Go.MemProfile", Name: "memprofile", UsageArgument: "file", SectionKey: "performance-analysis", 546 Usage: `Write an allocation profile to the file after all tests have passed. Preserves test binary.`}, 547 {KeyPath: "Go.MemProfileRate", Name: "memprofilerate", UsageArgument: "rate", SectionKey: "performance-analysis", 548 Usage: `Enable more precise (and expensive) memory allocation profiles by setting runtime.MemProfileRate. See 'go doc runtime.MemProfileRate'. To profile all memory allocations, use -test.memprofilerate=1.`}, 549 {KeyPath: "Go.MutexProfile", Name: "mutexprofile", UsageArgument: "file", SectionKey: "performance-analysis", 550 Usage: `Write a mutex contention profile to the specified file when all tests are complete. Preserves test binary.`}, 551 {KeyPath: "Go.MutexProfileFraction", Name: "mutexprofilefraction", UsageArgument: "n", SectionKey: "performance-analysis", 552 Usage: `if >= 0, calls runtime.SetMutexProfileFraction() Sample 1 in n stack traces of goroutines holding a contended mutex.`}, 553 {KeyPath: "Go.Trace", Name: "execution-trace", UsageArgument: "file", ExportAs: "trace", SectionKey: "performance-analysis", 554 Usage: `Write an execution trace to the specified file before exiting.`}, 555 } 556 557 // VetAndInitializeCLIAndGoConfig validates that the Ginkgo CLI's configuration is sound 558 // It returns a potentially mutated copy of the config that rationalizes the configuraiton to ensure consistency for downstream consumers 559 func VetAndInitializeCLIAndGoConfig(cliConfig CLIConfig, goFlagsConfig GoFlagsConfig) (CLIConfig, GoFlagsConfig, []error) { 560 errors := []error{} 561 562 if cliConfig.Repeat > 0 && cliConfig.UntilItFails { 563 errors = append(errors, GinkgoErrors.BothRepeatAndUntilItFails()) 564 } 565 566 //initialize the output directory 567 if cliConfig.OutputDir != "" { 568 err := os.MkdirAll(cliConfig.OutputDir, 0777) 569 if err != nil { 570 errors = append(errors, err) 571 } 572 } 573 574 //ensure cover mode is configured appropriately 575 if goFlagsConfig.CoverMode != "" || goFlagsConfig.CoverPkg != "" || goFlagsConfig.CoverProfile != "" { 576 goFlagsConfig.Cover = true 577 } 578 if goFlagsConfig.Cover && goFlagsConfig.CoverProfile == "" { 579 goFlagsConfig.CoverProfile = "coverprofile.out" 580 } 581 582 return cliConfig, goFlagsConfig, errors 583 } 584 585 // GenerateGoTestCompileArgs is used by the Ginkgo CLI to generate command line arguments to pass to the go test -c command when compiling the test 586 func GenerateGoTestCompileArgs(goFlagsConfig GoFlagsConfig, destination string, packageToBuild string) ([]string, error) { 587 // if the user has set the CoverProfile run-time flag make sure to set the build-time cover flag to make sure 588 // the built test binary can generate a coverprofile 589 if goFlagsConfig.CoverProfile != "" { 590 goFlagsConfig.Cover = true 591 } 592 593 args := []string{"test", "-c", "-o", destination, packageToBuild} 594 goArgs, err := GenerateFlagArgs( 595 GoBuildFlags, 596 map[string]interface{}{ 597 "Go": &goFlagsConfig, 598 }, 599 ) 600 601 if err != nil { 602 return []string{}, err 603 } 604 args = append(args, goArgs...) 605 return args, nil 606 } 607 608 // GenerateGinkgoTestRunArgs is used by the Ginkgo CLI to generate command line arguments to pass to the compiled Ginkgo test binary 609 func GenerateGinkgoTestRunArgs(suiteConfig SuiteConfig, reporterConfig ReporterConfig, goFlagsConfig GoFlagsConfig) ([]string, error) { 610 var flags GinkgoFlags 611 flags = SuiteConfigFlags.WithPrefix("ginkgo") 612 flags = flags.CopyAppend(ParallelConfigFlags.WithPrefix("ginkgo")...) 613 flags = flags.CopyAppend(ReporterConfigFlags.WithPrefix("ginkgo")...) 614 flags = flags.CopyAppend(GoRunFlags.WithPrefix("test")...) 615 bindings := map[string]interface{}{ 616 "S": &suiteConfig, 617 "R": &reporterConfig, 618 "Go": &goFlagsConfig, 619 } 620 621 return GenerateFlagArgs(flags, bindings) 622 } 623 624 // GenerateGoTestRunArgs is used by the Ginkgo CLI to generate command line arguments to pass to the compiled non-Ginkgo test binary 625 func GenerateGoTestRunArgs(goFlagsConfig GoFlagsConfig) ([]string, error) { 626 flags := GoRunFlags.WithPrefix("test") 627 bindings := map[string]interface{}{ 628 "Go": &goFlagsConfig, 629 } 630 631 args, err := GenerateFlagArgs(flags, bindings) 632 if err != nil { 633 return args, err 634 } 635 args = append(args, "--test.v") 636 return args, nil 637 } 638 639 // BuildRunCommandFlagSet builds the FlagSet for the `ginkgo run` command 640 func BuildRunCommandFlagSet(suiteConfig *SuiteConfig, reporterConfig *ReporterConfig, cliConfig *CLIConfig, goFlagsConfig *GoFlagsConfig) (GinkgoFlagSet, error) { 641 flags := SuiteConfigFlags 642 flags = flags.CopyAppend(ReporterConfigFlags...) 643 flags = flags.CopyAppend(GinkgoCLISharedFlags...) 644 flags = flags.CopyAppend(GinkgoCLIRunAndWatchFlags...) 645 flags = flags.CopyAppend(GinkgoCLIRunFlags...) 646 flags = flags.CopyAppend(GoBuildFlags...) 647 flags = flags.CopyAppend(GoRunFlags...) 648 649 bindings := map[string]interface{}{ 650 "S": suiteConfig, 651 "R": reporterConfig, 652 "C": cliConfig, 653 "Go": goFlagsConfig, 654 "D": &deprecatedConfig{}, 655 } 656 657 return NewGinkgoFlagSet(flags, bindings, FlagSections) 658 } 659 660 // BuildWatchCommandFlagSet builds the FlagSet for the `ginkgo watch` command 661 func BuildWatchCommandFlagSet(suiteConfig *SuiteConfig, reporterConfig *ReporterConfig, cliConfig *CLIConfig, goFlagsConfig *GoFlagsConfig) (GinkgoFlagSet, error) { 662 flags := SuiteConfigFlags 663 flags = flags.CopyAppend(ReporterConfigFlags...) 664 flags = flags.CopyAppend(GinkgoCLISharedFlags...) 665 flags = flags.CopyAppend(GinkgoCLIRunAndWatchFlags...) 666 flags = flags.CopyAppend(GinkgoCLIWatchFlags...) 667 flags = flags.CopyAppend(GoBuildFlags...) 668 flags = flags.CopyAppend(GoRunFlags...) 669 670 bindings := map[string]interface{}{ 671 "S": suiteConfig, 672 "R": reporterConfig, 673 "C": cliConfig, 674 "Go": goFlagsConfig, 675 "D": &deprecatedConfig{}, 676 } 677 678 return NewGinkgoFlagSet(flags, bindings, FlagSections) 679 } 680 681 // BuildBuildCommandFlagSet builds the FlagSet for the `ginkgo build` command 682 func BuildBuildCommandFlagSet(cliConfig *CLIConfig, goFlagsConfig *GoFlagsConfig) (GinkgoFlagSet, error) { 683 flags := GinkgoCLISharedFlags 684 flags = flags.CopyAppend(GoBuildFlags...) 685 686 bindings := map[string]interface{}{ 687 "C": cliConfig, 688 "Go": goFlagsConfig, 689 "D": &deprecatedConfig{}, 690 } 691 692 flagSections := make(GinkgoFlagSections, len(FlagSections)) 693 copy(flagSections, FlagSections) 694 for i := range flagSections { 695 if flagSections[i].Key == "multiple-suites" { 696 flagSections[i].Heading = "Building Multiple Suites" 697 } 698 if flagSections[i].Key == "go-build" { 699 flagSections[i] = GinkgoFlagSection{Key: "go-build", Style: "{{/}}", Heading: "Go Build Flags", 700 Description: "These flags are inherited from go build."} 701 } 702 } 703 704 return NewGinkgoFlagSet(flags, bindings, flagSections) 705 } 706 707 func BuildLabelsCommandFlagSet(cliConfig *CLIConfig) (GinkgoFlagSet, error) { 708 flags := GinkgoCLISharedFlags.SubsetWithNames("r", "skip-package") 709 710 bindings := map[string]interface{}{ 711 "C": cliConfig, 712 } 713 714 flagSections := make(GinkgoFlagSections, len(FlagSections)) 715 copy(flagSections, FlagSections) 716 for i := range flagSections { 717 if flagSections[i].Key == "multiple-suites" { 718 flagSections[i].Heading = "Fetching Labels from Multiple Suites" 719 } 720 } 721 722 return NewGinkgoFlagSet(flags, bindings, flagSections) 723 }