github.com/jdextraze/terraform@v0.6.17-0.20160511153921-e33847c8a8af/helper/resource/testing.go (about) 1 package resource 2 3 import ( 4 "fmt" 5 "io" 6 "io/ioutil" 7 "log" 8 "os" 9 "path/filepath" 10 "reflect" 11 "regexp" 12 "strings" 13 "testing" 14 15 "github.com/davecgh/go-spew/spew" 16 "github.com/hashicorp/go-getter" 17 "github.com/hashicorp/terraform/config/module" 18 "github.com/hashicorp/terraform/helper/logging" 19 "github.com/hashicorp/terraform/terraform" 20 ) 21 22 const TestEnvVar = "TF_ACC" 23 24 // UnitTestOverride is a value that when set in TestEnvVar indicates that this 25 // is a unit test borrowing the acceptance testing framework. 26 const UnitTestOverride = "UnitTestOverride" 27 28 // TestCheckFunc is the callback type used with acceptance tests to check 29 // the state of a resource. The state passed in is the latest state known, 30 // or in the case of being after a destroy, it is the last known state when 31 // it was created. 32 type TestCheckFunc func(*terraform.State) error 33 34 // TestCase is a single acceptance test case used to test the apply/destroy 35 // lifecycle of a resource in a specific configuration. 36 // 37 // When the destroy plan is executed, the config from the last TestStep 38 // is used to plan it. 39 type TestCase struct { 40 // PreCheck, if non-nil, will be called before any test steps are 41 // executed. It will only be executed in the case that the steps 42 // would run, so it can be used for some validation before running 43 // acceptance tests, such as verifying that keys are setup. 44 PreCheck func() 45 46 // Providers is the ResourceProvider that will be under test. 47 // 48 // Alternately, ProviderFactories can be specified for the providers 49 // that are valid. This takes priority over Providers. 50 // 51 // The end effect of each is the same: specifying the providers that 52 // are used within the tests. 53 Providers map[string]terraform.ResourceProvider 54 ProviderFactories map[string]terraform.ResourceProviderFactory 55 56 // CheckDestroy is called after the resource is finally destroyed 57 // to allow the tester to test that the resource is truly gone. 58 CheckDestroy TestCheckFunc 59 60 // Steps are the apply sequences done within the context of the 61 // same state. Each step can have its own check to verify correctness. 62 Steps []TestStep 63 64 // The settings below control the "ID-only refresh test." This is 65 // an enabled-by-default test that tests that a refresh can be 66 // refreshed with only an ID to result in the same attributes. 67 // This validates completeness of Refresh. 68 // 69 // IDRefreshName is the name of the resource to check. This will 70 // default to the first non-nil primary resource in the state. 71 // 72 // IDRefreshIgnore is a list of configuration keys that will be ignored. 73 IDRefreshName string 74 IDRefreshIgnore []string 75 } 76 77 // TestStep is a single apply sequence of a test, done within the 78 // context of a state. 79 // 80 // Multiple TestSteps can be sequenced in a Test to allow testing 81 // potentially complex update logic. In general, simply create/destroy 82 // tests will only need one step. 83 type TestStep struct { 84 // PreConfig is called before the Config is applied to perform any per-step 85 // setup that needs to happen 86 PreConfig func() 87 88 // Config a string of the configuration to give to Terraform. 89 Config string 90 91 // Check is called after the Config is applied. Use this step to 92 // make your own API calls to check the status of things, and to 93 // inspect the format of the ResourceState itself. 94 // 95 // If an error is returned, the test will fail. In this case, a 96 // destroy plan will still be attempted. 97 // 98 // If this is nil, no check is done on this step. 99 Check TestCheckFunc 100 101 // Destroy will create a destroy plan if set to true. 102 Destroy bool 103 104 // ExpectNonEmptyPlan can be set to true for specific types of tests that are 105 // looking to verify that a diff occurs 106 ExpectNonEmptyPlan bool 107 } 108 109 // Test performs an acceptance test on a resource. 110 // 111 // Tests are not run unless an environmental variable "TF_ACC" is 112 // set to some non-empty value. This is to avoid test cases surprising 113 // a user by creating real resources. 114 // 115 // Tests will fail unless the verbose flag (`go test -v`, or explicitly 116 // the "-test.v" flag) is set. Because some acceptance tests take quite 117 // long, we require the verbose flag so users are able to see progress 118 // output. 119 func Test(t TestT, c TestCase) { 120 // We only run acceptance tests if an env var is set because they're 121 // slow and generally require some outside configuration. 122 if os.Getenv(TestEnvVar) == "" { 123 t.Skip(fmt.Sprintf( 124 "Acceptance tests skipped unless env '%s' set", 125 TestEnvVar)) 126 return 127 } 128 129 isUnitTest := (os.Getenv(TestEnvVar) == UnitTestOverride) 130 131 logWriter, err := logging.LogOutput() 132 if err != nil { 133 t.Error(fmt.Errorf("error setting up logging: %s", err)) 134 } 135 log.SetOutput(logWriter) 136 137 // We require verbose mode so that the user knows what is going on. 138 if !testTesting && !testing.Verbose() && !isUnitTest { 139 t.Fatal("Acceptance tests must be run with the -v flag on tests") 140 return 141 } 142 143 // Run the PreCheck if we have it 144 if c.PreCheck != nil { 145 c.PreCheck() 146 } 147 148 // Build our context options that we can 149 ctxProviders := c.ProviderFactories 150 if ctxProviders == nil { 151 ctxProviders = make(map[string]terraform.ResourceProviderFactory) 152 for k, p := range c.Providers { 153 ctxProviders[k] = terraform.ResourceProviderFactoryFixed(p) 154 } 155 } 156 opts := terraform.ContextOpts{Providers: ctxProviders} 157 158 // A single state variable to track the lifecycle, starting with no state 159 var state *terraform.State 160 161 // Go through each step and run it 162 var idRefreshCheck *terraform.ResourceState 163 idRefresh := c.IDRefreshName != "" 164 errored := false 165 for i, step := range c.Steps { 166 var err error 167 log.Printf("[WARN] Test: Executing step %d", i) 168 state, err = testStep(opts, state, step) 169 if err != nil { 170 errored = true 171 t.Error(fmt.Sprintf( 172 "Step %d error: %s", i, err)) 173 break 174 } 175 176 // If we've never checked an id-only refresh and our state isn't 177 // empty, find the first resource and test it. 178 if idRefresh && idRefreshCheck == nil && !state.Empty() { 179 // Find the first non-nil resource in the state 180 for _, m := range state.Modules { 181 if len(m.Resources) > 0 { 182 if v, ok := m.Resources[c.IDRefreshName]; ok { 183 idRefreshCheck = v 184 } 185 186 break 187 } 188 } 189 190 // If we have an instance to check for refreshes, do it 191 // immediately. We do it in the middle of another test 192 // because it shouldn't affect the overall state (refresh 193 // is read-only semantically) and we want to fail early if 194 // this fails. If refresh isn't read-only, then this will have 195 // caught a different bug. 196 if idRefreshCheck != nil { 197 log.Printf( 198 "[WARN] Test: Running ID-only refresh check on %s", 199 idRefreshCheck.Primary.ID) 200 if err := testIDOnlyRefresh(c, opts, step, idRefreshCheck); err != nil { 201 log.Printf("[ERROR] Test: ID-only test failed: %s", err) 202 t.Error(fmt.Sprintf( 203 "ID-Only refresh test failure: %s", err)) 204 break 205 } 206 } 207 } 208 } 209 210 // If we never checked an id-only refresh, it is a failure. 211 if idRefresh { 212 if !errored && len(c.Steps) > 0 && idRefreshCheck == nil { 213 t.Error("ID-only refresh check never ran.") 214 } 215 } 216 217 // If we have a state, then run the destroy 218 if state != nil { 219 destroyStep := TestStep{ 220 Config: c.Steps[len(c.Steps)-1].Config, 221 Check: c.CheckDestroy, 222 Destroy: true, 223 } 224 225 log.Printf("[WARN] Test: Executing destroy step") 226 state, err := testStep(opts, state, destroyStep) 227 if err != nil { 228 t.Error(fmt.Sprintf( 229 "Error destroying resource! WARNING: Dangling resources\n"+ 230 "may exist. The full state and error is shown below.\n\n"+ 231 "Error: %s\n\nState: %s", 232 err, 233 state)) 234 } 235 } else { 236 log.Printf("[WARN] Skipping destroy test since there is no state.") 237 } 238 } 239 240 // UnitTest is a helper to force the acceptance testing harness to run in the 241 // normal unit test suite. This should only be used for resource that don't 242 // have any external dependencies. 243 func UnitTest(t TestT, c TestCase) { 244 oldEnv := os.Getenv(TestEnvVar) 245 if err := os.Setenv(TestEnvVar, UnitTestOverride); err != nil { 246 t.Fatal(err) 247 } 248 defer func() { 249 if err := os.Setenv(TestEnvVar, oldEnv); err != nil { 250 t.Fatal(err) 251 } 252 }() 253 Test(t, c) 254 } 255 256 func testIDOnlyRefresh(c TestCase, opts terraform.ContextOpts, step TestStep, r *terraform.ResourceState) error { 257 // TODO: We guard by this right now so master doesn't explode. We 258 // need to remove this eventually to make this part of the normal tests. 259 if os.Getenv("TF_ACC_IDONLY") == "" { 260 return nil 261 } 262 263 name := fmt.Sprintf("%s.foo", r.Type) 264 265 // Build the state. The state is just the resource with an ID. There 266 // are no attributes. We only set what is needed to perform a refresh. 267 state := terraform.NewState() 268 state.RootModule().Resources[name] = &terraform.ResourceState{ 269 Type: r.Type, 270 Primary: &terraform.InstanceState{ 271 ID: r.Primary.ID, 272 }, 273 } 274 275 // Create the config module. We use the full config because Refresh 276 // doesn't have access to it and we may need things like provider 277 // configurations. The initial implementation of id-only checks used 278 // an empty config module, but that caused the aforementioned problems. 279 mod, err := testModule(opts, step) 280 if err != nil { 281 return err 282 } 283 284 // Initialize the context 285 opts.Module = mod 286 opts.State = state 287 ctx, err := terraform.NewContext(&opts) 288 if err != nil { 289 return err 290 } 291 if ws, es := ctx.Validate(); len(ws) > 0 || len(es) > 0 { 292 if len(es) > 0 { 293 estrs := make([]string, len(es)) 294 for i, e := range es { 295 estrs[i] = e.Error() 296 } 297 return fmt.Errorf( 298 "Configuration is invalid.\n\nWarnings: %#v\n\nErrors: %#v", 299 ws, estrs) 300 } 301 302 log.Printf("[WARN] Config warnings: %#v", ws) 303 } 304 305 // Refresh! 306 state, err = ctx.Refresh() 307 if err != nil { 308 return fmt.Errorf("Error refreshing: %s", err) 309 } 310 311 // Verify attribute equivalence. 312 actualR := state.RootModule().Resources[name] 313 if actualR == nil { 314 return fmt.Errorf("Resource gone!") 315 } 316 if actualR.Primary == nil { 317 return fmt.Errorf("Resource has no primary instance") 318 } 319 actual := actualR.Primary.Attributes 320 expected := r.Primary.Attributes 321 // Remove fields we're ignoring 322 for _, v := range c.IDRefreshIgnore { 323 for k, _ := range actual { 324 if strings.HasPrefix(k, v) { 325 delete(actual, k) 326 } 327 } 328 for k, _ := range expected { 329 if strings.HasPrefix(k, v) { 330 delete(expected, k) 331 } 332 } 333 } 334 335 if !reflect.DeepEqual(actual, expected) { 336 // Determine only the different attributes 337 for k, v := range expected { 338 if av, ok := actual[k]; ok && v == av { 339 delete(expected, k) 340 delete(actual, k) 341 } 342 } 343 344 spewConf := spew.NewDefaultConfig() 345 spewConf.SortKeys = true 346 return fmt.Errorf( 347 "Attributes not equivalent. Difference is shown below. Top is actual, bottom is expected."+ 348 "\n\n%s\n\n%s", 349 spewConf.Sdump(actual), spewConf.Sdump(expected)) 350 } 351 352 return nil 353 } 354 355 func testStep( 356 opts terraform.ContextOpts, 357 state *terraform.State, 358 step TestStep) (*terraform.State, error) { 359 mod, err := testModule(opts, step) 360 if err != nil { 361 return state, err 362 } 363 364 // Build the context 365 opts.Module = mod 366 opts.State = state 367 opts.Destroy = step.Destroy 368 ctx, err := terraform.NewContext(&opts) 369 if err != nil { 370 return state, fmt.Errorf("Error initializing context: %s", err) 371 } 372 if ws, es := ctx.Validate(); len(ws) > 0 || len(es) > 0 { 373 if len(es) > 0 { 374 estrs := make([]string, len(es)) 375 for i, e := range es { 376 estrs[i] = e.Error() 377 } 378 return state, fmt.Errorf( 379 "Configuration is invalid.\n\nWarnings: %#v\n\nErrors: %#v", 380 ws, estrs) 381 } 382 log.Printf("[WARN] Config warnings: %#v", ws) 383 } 384 385 // Refresh! 386 state, err = ctx.Refresh() 387 if err != nil { 388 return state, fmt.Errorf( 389 "Error refreshing: %s", err) 390 } 391 392 // Plan! 393 if p, err := ctx.Plan(); err != nil { 394 return state, fmt.Errorf( 395 "Error planning: %s", err) 396 } else { 397 log.Printf("[WARN] Test: Step plan: %s", p) 398 } 399 400 // We need to keep a copy of the state prior to destroying 401 // such that destroy steps can verify their behaviour in the check 402 // function 403 stateBeforeApplication := state.DeepCopy() 404 405 // Apply! 406 state, err = ctx.Apply() 407 if err != nil { 408 return state, fmt.Errorf("Error applying: %s", err) 409 } 410 411 // Check! Excitement! 412 if step.Check != nil { 413 if step.Destroy { 414 if err := step.Check(stateBeforeApplication); err != nil { 415 return state, fmt.Errorf("Check failed: %s", err) 416 } 417 } else { 418 if err := step.Check(state); err != nil { 419 return state, fmt.Errorf("Check failed: %s", err) 420 } 421 } 422 } 423 424 // Now, verify that Plan is now empty and we don't have a perpetual diff issue 425 // We do this with TWO plans. One without a refresh. 426 var p *terraform.Plan 427 if p, err = ctx.Plan(); err != nil { 428 return state, fmt.Errorf("Error on follow-up plan: %s", err) 429 } 430 if p.Diff != nil && !p.Diff.Empty() { 431 if step.ExpectNonEmptyPlan { 432 log.Printf("[INFO] Got non-empty plan, as expected:\n\n%s", p) 433 } else { 434 return state, fmt.Errorf( 435 "After applying this step, the plan was not empty:\n\n%s", p) 436 } 437 } 438 439 // And another after a Refresh. 440 state, err = ctx.Refresh() 441 if err != nil { 442 return state, fmt.Errorf( 443 "Error on follow-up refresh: %s", err) 444 } 445 if p, err = ctx.Plan(); err != nil { 446 return state, fmt.Errorf("Error on second follow-up plan: %s", err) 447 } 448 if p.Diff != nil && !p.Diff.Empty() { 449 if step.ExpectNonEmptyPlan { 450 log.Printf("[INFO] Got non-empty plan, as expected:\n\n%s", p) 451 } else { 452 return state, fmt.Errorf( 453 "After applying this step and refreshing, "+ 454 "the plan was not empty:\n\n%s", p) 455 } 456 } 457 458 // Made it here, but expected a non-empty plan, fail! 459 if step.ExpectNonEmptyPlan && (p.Diff == nil || p.Diff.Empty()) { 460 return state, fmt.Errorf("Expected a non-empty plan, but got an empty plan!") 461 } 462 463 // Made it here? Good job test step! 464 return state, nil 465 } 466 467 func testModule( 468 opts terraform.ContextOpts, 469 step TestStep) (*module.Tree, error) { 470 if step.PreConfig != nil { 471 step.PreConfig() 472 } 473 474 cfgPath, err := ioutil.TempDir("", "tf-test") 475 if err != nil { 476 return nil, fmt.Errorf( 477 "Error creating temporary directory for config: %s", err) 478 } 479 defer os.RemoveAll(cfgPath) 480 481 // Write the configuration 482 cfgF, err := os.Create(filepath.Join(cfgPath, "main.tf")) 483 if err != nil { 484 return nil, fmt.Errorf( 485 "Error creating temporary file for config: %s", err) 486 } 487 488 _, err = io.Copy(cfgF, strings.NewReader(step.Config)) 489 cfgF.Close() 490 if err != nil { 491 return nil, fmt.Errorf( 492 "Error creating temporary file for config: %s", err) 493 } 494 495 // Parse the configuration 496 mod, err := module.NewTreeModule("", cfgPath) 497 if err != nil { 498 return nil, fmt.Errorf( 499 "Error loading configuration: %s", err) 500 } 501 502 // Load the modules 503 modStorage := &getter.FolderStorage{ 504 StorageDir: filepath.Join(cfgPath, ".tfmodules"), 505 } 506 err = mod.Load(modStorage, module.GetModeGet) 507 if err != nil { 508 return nil, fmt.Errorf("Error downloading modules: %s", err) 509 } 510 511 return mod, nil 512 } 513 514 // ComposeTestCheckFunc lets you compose multiple TestCheckFuncs into 515 // a single TestCheckFunc. 516 // 517 // As a user testing their provider, this lets you decompose your checks 518 // into smaller pieces more easily. 519 func ComposeTestCheckFunc(fs ...TestCheckFunc) TestCheckFunc { 520 return func(s *terraform.State) error { 521 for i, f := range fs { 522 if err := f(s); err != nil { 523 return fmt.Errorf("Check %d/%d error: %s", i+1, len(fs), err) 524 } 525 } 526 527 return nil 528 } 529 } 530 531 func TestCheckResourceAttr(name, key, value string) TestCheckFunc { 532 return func(s *terraform.State) error { 533 ms := s.RootModule() 534 rs, ok := ms.Resources[name] 535 if !ok { 536 return fmt.Errorf("Not found: %s", name) 537 } 538 539 is := rs.Primary 540 if is == nil { 541 return fmt.Errorf("No primary instance: %s", name) 542 } 543 544 if is.Attributes[key] != value { 545 return fmt.Errorf( 546 "%s: Attribute '%s' expected %#v, got %#v", 547 name, 548 key, 549 value, 550 is.Attributes[key]) 551 } 552 553 return nil 554 } 555 } 556 557 func TestMatchResourceAttr(name, key string, r *regexp.Regexp) TestCheckFunc { 558 return func(s *terraform.State) error { 559 ms := s.RootModule() 560 rs, ok := ms.Resources[name] 561 if !ok { 562 return fmt.Errorf("Not found: %s", name) 563 } 564 565 is := rs.Primary 566 if is == nil { 567 return fmt.Errorf("No primary instance: %s", name) 568 } 569 570 if !r.MatchString(is.Attributes[key]) { 571 return fmt.Errorf( 572 "%s: Attribute '%s' didn't match %q, got %#v", 573 name, 574 key, 575 r.String(), 576 is.Attributes[key]) 577 } 578 579 return nil 580 } 581 } 582 583 // TestCheckResourceAttrPtr is like TestCheckResourceAttr except the 584 // value is a pointer so that it can be updated while the test is running. 585 // It will only be dereferenced at the point this step is run. 586 func TestCheckResourceAttrPtr(name string, key string, value *string) TestCheckFunc { 587 return func(s *terraform.State) error { 588 return TestCheckResourceAttr(name, key, *value)(s) 589 } 590 } 591 592 // TestCheckOutput checks an output in the Terraform configuration 593 func TestCheckOutput(name, value string) TestCheckFunc { 594 return func(s *terraform.State) error { 595 ms := s.RootModule() 596 rs, ok := ms.Outputs[name] 597 if !ok { 598 return fmt.Errorf("Not found: %s", name) 599 } 600 601 if rs != value { 602 return fmt.Errorf( 603 "Output '%s': expected %#v, got %#v", 604 name, 605 value, 606 rs) 607 } 608 609 return nil 610 } 611 } 612 613 // TestT is the interface used to handle the test lifecycle of a test. 614 // 615 // Users should just use a *testing.T object, which implements this. 616 type TestT interface { 617 Error(args ...interface{}) 618 Fatal(args ...interface{}) 619 Skip(args ...interface{}) 620 } 621 622 // This is set to true by unit tests to alter some behavior 623 var testTesting = false