github.com/hernad/nomad@v1.6.112/e2e/framework/framework.go (about) 1 // Copyright (c) HashiCorp, Inc. 2 // SPDX-License-Identifier: MPL-2.0 3 4 package framework 5 6 import ( 7 "flag" 8 "fmt" 9 "log" 10 "reflect" 11 "strings" 12 "testing" 13 14 "github.com/hernad/nomad/ci" 15 "github.com/stretchr/testify/require" 16 ) 17 18 const frameworkHelp = ` 19 Usage: go test -v ./e2e [options] 20 21 These flags are coarse overrides for the test environment. 22 23 -forceRun skip all environment checks when filtering test suites 24 -local denotes execution is against a local environment 25 -slow include execution of slow test suites 26 -suite run specified test suite 27 -showHelp shows this help text 28 29 TestSuites can request Constraints on the Framework.Environment so that tests 30 are only run in the appropriate conditions. These environment flags provide 31 the information for those constraints. 32 33 -env=string name of the environment 34 -env.arch=string cpu architecture of the targets 35 -env.os=string operating system of the targets 36 -env.provider=string cloud provider of the environment 37 -env.tags=string comma delimited list of tags for the environment 38 39 ` 40 41 var fHelp = flag.Bool("showHelp", false, "print the help screen") 42 var fLocal = flag.Bool("local", false, 43 "denotes execution is against a local environment") 44 var fSlow = flag.Bool("slow", false, "toggles execution of slow test suites") 45 var fForceRun = flag.Bool("forceRun", false, 46 "if set, skips all environment checks when filtering test suites") 47 var fSuite = flag.String("suite", "", "run specified test suite") 48 49 // Environment flags 50 // TODO: 51 // It would be nice if we could match the target environment against 52 // the tests automatically so that we always avoid running tests that 53 // don't apply, and then have these flags override that behavior. 54 var fEnv = flag.String("env", "", "name of the environment executing against") 55 var fProvider = flag.String("env.provider", "", 56 "cloud provider for which environment is executing against") 57 var fOS = flag.String("env.os", "", 58 "operating system for which the environment is executing against") 59 var fArch = flag.String("env.arch", "", 60 "cpu architecture for which the environment is executing against") 61 var fTags = flag.String("env.tags", "", 62 "comma delimited list of tags associated with the environment") 63 64 // Deprecated: no longer use e2e/framework for new tests; see TestExample for new e2e test structure. 65 type Framework struct { 66 suites []*TestSuite 67 provisioner Provisioner 68 env Environment 69 70 isLocalRun bool 71 slow bool 72 force bool 73 suite string 74 } 75 76 // Environment contains information about the test target environment, used 77 // to constrain the set of tests run. See the environment flags above. 78 // 79 // Deprecated: no longer use e2e/framework for new tests; see TestExample for new e2e test structure. 80 type Environment struct { 81 Name string 82 Provider string 83 OS string 84 Arch string 85 Tags map[string]struct{} 86 } 87 88 // New creates a Framework 89 // 90 // Deprecated: no longer use e2e/framework for new tests; see TestExample for new e2e test structure. 91 func New() *Framework { 92 flag.Parse() 93 if *fHelp { 94 log.Fatal(frameworkHelp) 95 } 96 env := Environment{ 97 Name: *fEnv, 98 Provider: *fProvider, 99 OS: *fOS, 100 Arch: *fArch, 101 Tags: map[string]struct{}{}, 102 } 103 for _, tag := range strings.Split(*fTags, ",") { 104 env.Tags[tag] = struct{}{} 105 } 106 return &Framework{ 107 provisioner: DefaultProvisioner, 108 env: env, 109 isLocalRun: *fLocal, 110 slow: *fSlow, 111 force: *fForceRun, 112 suite: *fSuite, 113 } 114 } 115 116 // AddSuites adds a set of test suites to a Framework 117 // 118 // Deprecated: no longer use e2e/framework for new tests; see TestExample for new e2e test structure. 119 func (f *Framework) AddSuites(s ...*TestSuite) *Framework { 120 f.suites = append(f.suites, s...) 121 return f 122 } 123 124 var pkgSuites []*TestSuite 125 126 // AddSuites adds a set of test suites to the package scoped Framework 127 // 128 // Deprecated: no longer use e2e/framework for new tests; see TestExample for new e2e test structure. 129 func AddSuites(s ...*TestSuite) { 130 pkgSuites = append(pkgSuites, s...) 131 } 132 133 // Run starts the test framework, running each TestSuite 134 func (f *Framework) Run(t *testing.T) { 135 info, err := f.provisioner.SetupTestRun(t, SetupOptions{}) 136 if err != nil { 137 require.NoError(t, err, "could not provision cluster") 138 } 139 defer f.provisioner.TearDownTestRun(t, info.ID) 140 141 for _, s := range f.suites { 142 t.Run(s.Component, func(t *testing.T) { 143 skip, err := f.runSuite(t, s) 144 if skip { 145 t.Skipf("skipping suite '%s': %v", s.Component, err) 146 return 147 } 148 if err != nil { 149 t.Errorf("error starting suite '%s': %v", s.Component, err) 150 } 151 }) 152 } 153 } 154 155 // Run starts the package scoped Framework, running each TestSuite 156 // 157 // Deprecated: no longer use e2e/framework for new tests; see TestExample for new e2e test structure. 158 func Run(t *testing.T) { 159 f := New() 160 f.AddSuites(pkgSuites...) 161 f.Run(t) 162 } 163 164 // runSuite is called from Framework.Run inside of a sub test for each TestSuite. 165 // If skip is returned as true, the test suite is skipped with the error text added 166 // to the Skip reason 167 // If skip is false and an error is returned, the test suite is failed. 168 func (f *Framework) runSuite(t *testing.T, s *TestSuite) (skip bool, err error) { 169 170 // If -forceRun is set, skip all constraint checks 171 if !f.force { 172 // If this is a local run, check that the suite supports running locally 173 if !s.CanRunLocal && f.isLocalRun { 174 return true, fmt.Errorf("local run detected and suite cannot run locally") 175 } 176 177 // Check that constraints are met 178 if err := s.Constraints.matches(f.env); err != nil { 179 return true, fmt.Errorf("constraint failed: %v", err) 180 } 181 182 // Check the slow toggle and if the suite's slow flag is that same 183 if f.slow != s.Slow { 184 return true, fmt.Errorf("framework slow suite configuration is %v but suite is %v", f.slow, s.Slow) 185 } 186 } 187 188 // If -suite is set, skip any suite that is not the one specified. 189 if f.suite != "" && f.suite != s.Component { 190 return true, fmt.Errorf("only running suite %q", f.suite) 191 } 192 193 info, err := f.provisioner.SetupTestSuite(t, SetupOptions{ 194 Name: s.Component, 195 ExpectConsul: s.Consul, 196 ExpectVault: s.Vault, 197 }) 198 require.NoError(t, err, "could not provision cluster") 199 defer f.provisioner.TearDownTestSuite(t, info.ID) 200 201 for _, c := range s.Cases { 202 f.runCase(t, s, c) 203 } 204 205 return false, nil 206 } 207 208 func (f *Framework) runCase(t *testing.T, s *TestSuite, c TestCase) { 209 210 // The test name is set to the name of the implementing type, including package 211 name := fmt.Sprintf("%T", c) 212 213 // The ClusterInfo handle should be used by each TestCase to isolate 214 // job/task state created during the test. 215 info, err := f.provisioner.SetupTestCase(t, SetupOptions{ 216 Name: name, 217 ExpectConsul: s.Consul, 218 ExpectVault: s.Vault, 219 }) 220 if err != nil { 221 t.Errorf("could not provision cluster for case: %v", err) 222 } 223 defer f.provisioner.TearDownTestCase(t, info.ID) 224 c.setClusterInfo(info) 225 226 // Each TestCase runs as a subtest of the TestSuite 227 t.Run(c.Name(), func(t *testing.T) { 228 // If the TestSuite has Parallel set, all cases run in parallel 229 if s.Parallel { 230 ci.Parallel(t) 231 } 232 233 f := newF(t) 234 235 // Check if the case includes a before all function 236 if beforeAllTests, ok := c.(BeforeAllTests); ok { 237 beforeAllTests.BeforeAll(f) 238 } 239 240 // Check if the case includes an after all function at the end 241 defer func() { 242 if afterAllTests, ok := c.(AfterAllTests); ok { 243 afterAllTests.AfterAll(f) 244 } 245 }() 246 247 // Here we need to iterate through the methods of the case to find 248 // ones that are test functions 249 reflectC := reflect.TypeOf(c) 250 for i := 0; i < reflectC.NumMethod(); i++ { 251 method := reflectC.Method(i) 252 if ok := isTestMethod(method.Name); !ok { 253 continue 254 } 255 // Each test is run as its own sub test of the case 256 // Test cases are never parallel 257 t.Run(method.Name, func(t *testing.T) { 258 259 cF := newFFromParent(f, t) 260 if BeforeEachTest, ok := c.(BeforeEachTest); ok { 261 BeforeEachTest.BeforeEach(cF) 262 } 263 defer func() { 264 if afterEachTest, ok := c.(AfterEachTest); ok { 265 afterEachTest.AfterEach(cF) 266 } 267 }() 268 269 //Call the method 270 method.Func.Call([]reflect.Value{reflect.ValueOf(c), reflect.ValueOf(cF)}) 271 }) 272 } 273 }) 274 } 275 276 func isTestMethod(m string) bool { 277 return strings.HasPrefix(m, "Test") 278 }