github.com/iqoqo/nomad@v0.11.3-0.20200911112621-d7021c74d101/e2e/cli/command/run.go (about)

     1  package command
     2  
     3  import (
     4  	"fmt"
     5  	"os"
     6  	"os/exec"
     7  	"strings"
     8  
     9  	capi "github.com/hashicorp/consul/api"
    10  	hclog "github.com/hashicorp/go-hclog"
    11  	vapi "github.com/hashicorp/vault/api"
    12  	"github.com/mitchellh/cli"
    13  )
    14  
    15  func RunCommandFactory(meta Meta) cli.CommandFactory {
    16  	return func() (cli.Command, error) {
    17  		return &Run{Meta: meta}, nil
    18  	}
    19  }
    20  
    21  type Run struct {
    22  	Meta
    23  }
    24  
    25  func (c *Run) Help() string {
    26  	helpText := `
    27  Usage: nomad-e2e run (<provider>/<name>)...
    28  
    29    Two modes exist when using the run command.
    30  
    31    When no arguments are given to the run command, it will launch
    32    the e2e test suite against the Nomad cluster specified by the
    33    NOMAD_ADDR environment variable. If this is not set, it defaults
    34    to 'http://localhost:4646'
    35  
    36    Multiple arguments may be given to specify one or more environments to
    37    provision and run the e2e tests against. These are given in the form of
    38    <provider>/<name>. Globs are support, for example 'aws/*' would run tests
    39    against all of the environments under the aws provider. When using this mode,
    40    all of the provision flags are supported.
    41  
    42  General Options:
    43  
    44  ` + generalOptionsUsage() + `
    45  
    46  Run Options:
    47  
    48    -run regex
    49      Sets a regular expression for what tests to run. Uses '/' as a separator
    50  	to allow hierarchy between Suite/Case/Test.
    51  
    52  	Example '-run MyTestSuite' would only run tests under the MyTestSuite suite.
    53  
    54    -slow
    55      If set, will only run test suites marked as slow.
    56  `
    57  	return strings.TrimSpace(helpText)
    58  }
    59  
    60  func (c *Run) Synopsis() string {
    61  	return "Runs the e2e test suite"
    62  }
    63  
    64  func (c *Run) Run(args []string) int {
    65  	var envPath string
    66  	var nomadBinary string
    67  	var tfPath string
    68  	var slow bool
    69  	var run string
    70  	cmdFlags := c.FlagSet("run")
    71  	cmdFlags.Usage = func() { c.Ui.Output(c.Help()) }
    72  	cmdFlags.StringVar(&envPath, "env-path", DefaultEnvironmentsPath, "Path to e2e environment terraform configs")
    73  	cmdFlags.StringVar(&nomadBinary, "nomad-binary", "", "")
    74  	cmdFlags.StringVar(&tfPath, "tf-path", "", "")
    75  	cmdFlags.StringVar(&run, "run", "", "Regex to target specific test suites/cases")
    76  	cmdFlags.BoolVar(&slow, "slow", false, "Toggle slow running suites")
    77  
    78  	if err := cmdFlags.Parse(args); err != nil {
    79  		c.logger.Error("failed to parse flags", "error", err)
    80  		return 1
    81  	}
    82  	if c.verbose {
    83  		c.logger.SetLevel(hclog.Debug)
    84  	}
    85  
    86  	args = cmdFlags.Args()
    87  
    88  	if len(args) == 0 {
    89  		c.logger.Info("no environments specified, running test suite locally")
    90  		report, err := c.runTest(&runOpts{
    91  			run:     run,
    92  			slow:    slow,
    93  			verbose: c.verbose,
    94  		})
    95  		if err != nil {
    96  			c.logger.Error("failed to run test suite", "error", err)
    97  			return 1
    98  		}
    99  		if report.TotalFailedTests > 0 {
   100  			c.Ui.Error("***FAILED***")
   101  			c.Ui.Error(report.Summary())
   102  			return 1
   103  		}
   104  		c.Ui.Output("PASSED!")
   105  		if c.verbose {
   106  			c.Ui.Output(report.Summary())
   107  		}
   108  		return 0
   109  	}
   110  
   111  	environments := []*environment{}
   112  	for _, e := range args {
   113  		if len(strings.Split(e, "/")) != 2 {
   114  			c.logger.Error("argument should be formated as <provider>/<environment>", "args", e)
   115  			return 1
   116  		}
   117  		envs, err := envsFromGlob(envPath, e, tfPath, c.logger)
   118  		if err != nil {
   119  			c.logger.Error("failed to build environment", "environment", e, "error", err)
   120  			return 1
   121  		}
   122  		environments = append(environments, envs...)
   123  
   124  	}
   125  
   126  	// Use go-getter to fetch the nomad binary
   127  	nomadPath, err := fetchBinary(nomadBinary)
   128  	defer os.RemoveAll(nomadPath)
   129  	if err != nil {
   130  		c.logger.Error("failed to fetch nomad binary", "error", err)
   131  		return 1
   132  	}
   133  
   134  	envCount := len(environments)
   135  	c.logger.Debug("starting tests", "totalEnvironments", envCount)
   136  	failedEnvs := map[string]*TestReport{}
   137  	for i, env := range environments {
   138  		logger := c.logger.With("name", env.name, "provider", env.provider)
   139  		logger.Debug("provisioning environment")
   140  		results, err := env.provision(nomadPath)
   141  		if err != nil {
   142  			logger.Error("failed to provision environment", "error", err)
   143  			return 1
   144  		}
   145  
   146  		opts := &runOpts{
   147  			provider:   env.provider,
   148  			env:        env.name,
   149  			slow:       slow,
   150  			run:        run,
   151  			verbose:    c.verbose,
   152  			nomadAddr:  results.nomadAddr,
   153  			consulAddr: results.consulAddr,
   154  			vaultAddr:  results.vaultAddr,
   155  		}
   156  
   157  		var report *TestReport
   158  		if report, err = c.runTest(opts); err != nil {
   159  			logger.Error("failed to run tests against environment", "error", err)
   160  			return 1
   161  		}
   162  		if report.TotalFailedTests > 0 {
   163  			c.Ui.Error(fmt.Sprintf("[%d/%d] %s: ***FAILED***", i+1, envCount, env.canonicalName()))
   164  			c.Ui.Error(fmt.Sprintf("[%d/%d] %s: %s", i+1, envCount, env.canonicalName(), report.Summary()))
   165  			failedEnvs[env.canonicalName()] = report
   166  		}
   167  
   168  		c.Ui.Output(fmt.Sprintf("[%d/%d] %s: PASSED!", i+1, envCount, env.canonicalName()))
   169  		if c.verbose {
   170  			c.Ui.Output(fmt.Sprintf("[%d/%d] %s: %s", i+1, envCount, env.canonicalName(), report.Summary()))
   171  		}
   172  	}
   173  
   174  	if len(failedEnvs) > 0 {
   175  		c.Ui.Error(fmt.Sprintf("The following environments ***FAILED***"))
   176  		for name, report := range failedEnvs {
   177  			c.Ui.Error(fmt.Sprintf("  [%s]: %d out of %d suite failures",
   178  				name, report.TotalFailedSuites, report.TotalSuites))
   179  		}
   180  		return 1
   181  	}
   182  	c.Ui.Output("All Environments PASSED!")
   183  	return 0
   184  }
   185  
   186  func (c *Run) runTest(opts *runOpts) (*TestReport, error) {
   187  	goBin, err := exec.LookPath("go")
   188  	if err != nil {
   189  		return nil, err
   190  	}
   191  
   192  	cmd := exec.Command(goBin, opts.goArgs()...)
   193  	cmd.Env = opts.goEnv()
   194  	out, err := cmd.StdoutPipe()
   195  	if err != nil {
   196  		return nil, err
   197  	}
   198  
   199  	err = cmd.Start()
   200  	if err != nil {
   201  		return nil, err
   202  	}
   203  
   204  	err = cmd.Wait()
   205  	if err != nil {
   206  		// should command fail, log here then proceed to generate test report
   207  		// to report more informative info about which tests fail
   208  		c.logger.Error("test command failed", "error", err)
   209  	}
   210  
   211  	dec := NewDecoder(out)
   212  	report, err := dec.Decode(c.logger.Named("run.gotest"))
   213  	if err != nil {
   214  		return nil, err
   215  	}
   216  
   217  	return report, nil
   218  
   219  }
   220  
   221  // runOpts contains fields used to build the arguments and environment variabled
   222  // nessicary to run go test and initialize the e2e framework
   223  type runOpts struct {
   224  	nomadAddr  string
   225  	consulAddr string
   226  	vaultAddr  string
   227  	provider   string
   228  	env        string
   229  	run        string
   230  	local      bool
   231  	slow       bool
   232  	verbose    bool
   233  }
   234  
   235  // goArgs returns the list of arguments passed to the go command to start the
   236  // e2e test framework
   237  func (opts *runOpts) goArgs() []string {
   238  	a := []string{
   239  		"test",
   240  		"-json",
   241  	}
   242  
   243  	if opts.run != "" {
   244  		a = append(a, "-run=TestE2E/"+opts.run)
   245  	}
   246  
   247  	a = append(a, []string{
   248  		"github.com/hashicorp/nomad/e2e",
   249  		"-env=" + opts.env,
   250  		"-env.provider=" + opts.provider,
   251  	}...)
   252  
   253  	if opts.slow {
   254  		a = append(a, "-slow")
   255  	}
   256  
   257  	if opts.local {
   258  		a = append(a, "-local")
   259  	}
   260  	return a
   261  }
   262  
   263  // goEnv returns the list of environment variabled passed to the go command to start
   264  // the e2e test framework
   265  func (opts *runOpts) goEnv() []string {
   266  	env := append(os.Environ(), "NOMAD_E2E=1")
   267  	if opts.nomadAddr != "" {
   268  		env = append(env, "NOMAD_ADDR="+opts.nomadAddr)
   269  	}
   270  	if opts.consulAddr != "" {
   271  		env = append(env, fmt.Sprintf("%s=%s", capi.HTTPAddrEnvName, opts.consulAddr))
   272  	}
   273  	if opts.vaultAddr != "" {
   274  		env = append(env, fmt.Sprintf("%s=%s", vapi.EnvVaultAddress, opts.consulAddr))
   275  	}
   276  
   277  	return env
   278  }