github.com/iqoqo/nomad@v0.11.3-0.20200911112621-d7021c74d101/e2e/cli/command/run.go (about) 1 package command 2 3 import ( 4 "fmt" 5 "os" 6 "os/exec" 7 "strings" 8 9 capi "github.com/hashicorp/consul/api" 10 hclog "github.com/hashicorp/go-hclog" 11 vapi "github.com/hashicorp/vault/api" 12 "github.com/mitchellh/cli" 13 ) 14 15 func RunCommandFactory(meta Meta) cli.CommandFactory { 16 return func() (cli.Command, error) { 17 return &Run{Meta: meta}, nil 18 } 19 } 20 21 type Run struct { 22 Meta 23 } 24 25 func (c *Run) Help() string { 26 helpText := ` 27 Usage: nomad-e2e run (<provider>/<name>)... 28 29 Two modes exist when using the run command. 30 31 When no arguments are given to the run command, it will launch 32 the e2e test suite against the Nomad cluster specified by the 33 NOMAD_ADDR environment variable. If this is not set, it defaults 34 to 'http://localhost:4646' 35 36 Multiple arguments may be given to specify one or more environments to 37 provision and run the e2e tests against. These are given in the form of 38 <provider>/<name>. Globs are support, for example 'aws/*' would run tests 39 against all of the environments under the aws provider. When using this mode, 40 all of the provision flags are supported. 41 42 General Options: 43 44 ` + generalOptionsUsage() + ` 45 46 Run Options: 47 48 -run regex 49 Sets a regular expression for what tests to run. Uses '/' as a separator 50 to allow hierarchy between Suite/Case/Test. 51 52 Example '-run MyTestSuite' would only run tests under the MyTestSuite suite. 53 54 -slow 55 If set, will only run test suites marked as slow. 56 ` 57 return strings.TrimSpace(helpText) 58 } 59 60 func (c *Run) Synopsis() string { 61 return "Runs the e2e test suite" 62 } 63 64 func (c *Run) Run(args []string) int { 65 var envPath string 66 var nomadBinary string 67 var tfPath string 68 var slow bool 69 var run string 70 cmdFlags := c.FlagSet("run") 71 cmdFlags.Usage = func() { c.Ui.Output(c.Help()) } 72 cmdFlags.StringVar(&envPath, "env-path", DefaultEnvironmentsPath, "Path to e2e environment terraform configs") 73 cmdFlags.StringVar(&nomadBinary, "nomad-binary", "", "") 74 cmdFlags.StringVar(&tfPath, "tf-path", "", "") 75 cmdFlags.StringVar(&run, "run", "", "Regex to target specific test suites/cases") 76 cmdFlags.BoolVar(&slow, "slow", false, "Toggle slow running suites") 77 78 if err := cmdFlags.Parse(args); err != nil { 79 c.logger.Error("failed to parse flags", "error", err) 80 return 1 81 } 82 if c.verbose { 83 c.logger.SetLevel(hclog.Debug) 84 } 85 86 args = cmdFlags.Args() 87 88 if len(args) == 0 { 89 c.logger.Info("no environments specified, running test suite locally") 90 report, err := c.runTest(&runOpts{ 91 run: run, 92 slow: slow, 93 verbose: c.verbose, 94 }) 95 if err != nil { 96 c.logger.Error("failed to run test suite", "error", err) 97 return 1 98 } 99 if report.TotalFailedTests > 0 { 100 c.Ui.Error("***FAILED***") 101 c.Ui.Error(report.Summary()) 102 return 1 103 } 104 c.Ui.Output("PASSED!") 105 if c.verbose { 106 c.Ui.Output(report.Summary()) 107 } 108 return 0 109 } 110 111 environments := []*environment{} 112 for _, e := range args { 113 if len(strings.Split(e, "/")) != 2 { 114 c.logger.Error("argument should be formated as <provider>/<environment>", "args", e) 115 return 1 116 } 117 envs, err := envsFromGlob(envPath, e, tfPath, c.logger) 118 if err != nil { 119 c.logger.Error("failed to build environment", "environment", e, "error", err) 120 return 1 121 } 122 environments = append(environments, envs...) 123 124 } 125 126 // Use go-getter to fetch the nomad binary 127 nomadPath, err := fetchBinary(nomadBinary) 128 defer os.RemoveAll(nomadPath) 129 if err != nil { 130 c.logger.Error("failed to fetch nomad binary", "error", err) 131 return 1 132 } 133 134 envCount := len(environments) 135 c.logger.Debug("starting tests", "totalEnvironments", envCount) 136 failedEnvs := map[string]*TestReport{} 137 for i, env := range environments { 138 logger := c.logger.With("name", env.name, "provider", env.provider) 139 logger.Debug("provisioning environment") 140 results, err := env.provision(nomadPath) 141 if err != nil { 142 logger.Error("failed to provision environment", "error", err) 143 return 1 144 } 145 146 opts := &runOpts{ 147 provider: env.provider, 148 env: env.name, 149 slow: slow, 150 run: run, 151 verbose: c.verbose, 152 nomadAddr: results.nomadAddr, 153 consulAddr: results.consulAddr, 154 vaultAddr: results.vaultAddr, 155 } 156 157 var report *TestReport 158 if report, err = c.runTest(opts); err != nil { 159 logger.Error("failed to run tests against environment", "error", err) 160 return 1 161 } 162 if report.TotalFailedTests > 0 { 163 c.Ui.Error(fmt.Sprintf("[%d/%d] %s: ***FAILED***", i+1, envCount, env.canonicalName())) 164 c.Ui.Error(fmt.Sprintf("[%d/%d] %s: %s", i+1, envCount, env.canonicalName(), report.Summary())) 165 failedEnvs[env.canonicalName()] = report 166 } 167 168 c.Ui.Output(fmt.Sprintf("[%d/%d] %s: PASSED!", i+1, envCount, env.canonicalName())) 169 if c.verbose { 170 c.Ui.Output(fmt.Sprintf("[%d/%d] %s: %s", i+1, envCount, env.canonicalName(), report.Summary())) 171 } 172 } 173 174 if len(failedEnvs) > 0 { 175 c.Ui.Error(fmt.Sprintf("The following environments ***FAILED***")) 176 for name, report := range failedEnvs { 177 c.Ui.Error(fmt.Sprintf(" [%s]: %d out of %d suite failures", 178 name, report.TotalFailedSuites, report.TotalSuites)) 179 } 180 return 1 181 } 182 c.Ui.Output("All Environments PASSED!") 183 return 0 184 } 185 186 func (c *Run) runTest(opts *runOpts) (*TestReport, error) { 187 goBin, err := exec.LookPath("go") 188 if err != nil { 189 return nil, err 190 } 191 192 cmd := exec.Command(goBin, opts.goArgs()...) 193 cmd.Env = opts.goEnv() 194 out, err := cmd.StdoutPipe() 195 if err != nil { 196 return nil, err 197 } 198 199 err = cmd.Start() 200 if err != nil { 201 return nil, err 202 } 203 204 err = cmd.Wait() 205 if err != nil { 206 // should command fail, log here then proceed to generate test report 207 // to report more informative info about which tests fail 208 c.logger.Error("test command failed", "error", err) 209 } 210 211 dec := NewDecoder(out) 212 report, err := dec.Decode(c.logger.Named("run.gotest")) 213 if err != nil { 214 return nil, err 215 } 216 217 return report, nil 218 219 } 220 221 // runOpts contains fields used to build the arguments and environment variabled 222 // nessicary to run go test and initialize the e2e framework 223 type runOpts struct { 224 nomadAddr string 225 consulAddr string 226 vaultAddr string 227 provider string 228 env string 229 run string 230 local bool 231 slow bool 232 verbose bool 233 } 234 235 // goArgs returns the list of arguments passed to the go command to start the 236 // e2e test framework 237 func (opts *runOpts) goArgs() []string { 238 a := []string{ 239 "test", 240 "-json", 241 } 242 243 if opts.run != "" { 244 a = append(a, "-run=TestE2E/"+opts.run) 245 } 246 247 a = append(a, []string{ 248 "github.com/hashicorp/nomad/e2e", 249 "-env=" + opts.env, 250 "-env.provider=" + opts.provider, 251 }...) 252 253 if opts.slow { 254 a = append(a, "-slow") 255 } 256 257 if opts.local { 258 a = append(a, "-local") 259 } 260 return a 261 } 262 263 // goEnv returns the list of environment variabled passed to the go command to start 264 // the e2e test framework 265 func (opts *runOpts) goEnv() []string { 266 env := append(os.Environ(), "NOMAD_E2E=1") 267 if opts.nomadAddr != "" { 268 env = append(env, "NOMAD_ADDR="+opts.nomadAddr) 269 } 270 if opts.consulAddr != "" { 271 env = append(env, fmt.Sprintf("%s=%s", capi.HTTPAddrEnvName, opts.consulAddr)) 272 } 273 if opts.vaultAddr != "" { 274 env = append(env, fmt.Sprintf("%s=%s", vapi.EnvVaultAddress, opts.consulAddr)) 275 } 276 277 return env 278 }