github.com/anfernee/terraform@v0.6.16-0.20160430000239-06e5085a92f2/helper/resource/testing.go (about)

     1  package resource
     2  
     3  import (
     4  	"fmt"
     5  	"io"
     6  	"io/ioutil"
     7  	"log"
     8  	"os"
     9  	"path/filepath"
    10  	"reflect"
    11  	"regexp"
    12  	"strings"
    13  	"testing"
    14  
    15  	"github.com/davecgh/go-spew/spew"
    16  	"github.com/hashicorp/go-getter"
    17  	"github.com/hashicorp/terraform/config/module"
    18  	"github.com/hashicorp/terraform/helper/logging"
    19  	"github.com/hashicorp/terraform/terraform"
    20  )
    21  
    22  const TestEnvVar = "TF_ACC"
    23  
    24  // UnitTestOverride is a value that when set in TestEnvVar indicates that this
    25  // is a unit test borrowing the acceptance testing framework.
    26  const UnitTestOverride = "UnitTestOverride"
    27  
    28  // TestCheckFunc is the callback type used with acceptance tests to check
    29  // the state of a resource. The state passed in is the latest state known,
    30  // or in the case of being after a destroy, it is the last known state when
    31  // it was created.
    32  type TestCheckFunc func(*terraform.State) error
    33  
    34  // TestCase is a single acceptance test case used to test the apply/destroy
    35  // lifecycle of a resource in a specific configuration.
    36  //
    37  // When the destroy plan is executed, the config from the last TestStep
    38  // is used to plan it.
    39  type TestCase struct {
    40  	// PreCheck, if non-nil, will be called before any test steps are
    41  	// executed. It will only be executed in the case that the steps
    42  	// would run, so it can be used for some validation before running
    43  	// acceptance tests, such as verifying that keys are setup.
    44  	PreCheck func()
    45  
    46  	// Providers is the ResourceProvider that will be under test.
    47  	//
    48  	// Alternately, ProviderFactories can be specified for the providers
    49  	// that are valid. This takes priority over Providers.
    50  	//
    51  	// The end effect of each is the same: specifying the providers that
    52  	// are used within the tests.
    53  	Providers         map[string]terraform.ResourceProvider
    54  	ProviderFactories map[string]terraform.ResourceProviderFactory
    55  
    56  	// CheckDestroy is called after the resource is finally destroyed
    57  	// to allow the tester to test that the resource is truly gone.
    58  	CheckDestroy TestCheckFunc
    59  
    60  	// Steps are the apply sequences done within the context of the
    61  	// same state. Each step can have its own check to verify correctness.
    62  	Steps []TestStep
    63  
    64  	// The settings below control the "ID-only refresh test." This is
    65  	// an enabled-by-default test that tests that a refresh can be
    66  	// refreshed with only an ID to result in the same attributes.
    67  	// This validates completeness of Refresh.
    68  	//
    69  	// IDRefreshName is the name of the resource to check. This will
    70  	// default to the first non-nil primary resource in the state.
    71  	//
    72  	// IDRefreshIgnore is a list of configuration keys that will be ignored.
    73  	IDRefreshName   string
    74  	IDRefreshIgnore []string
    75  }
    76  
    77  // TestStep is a single apply sequence of a test, done within the
    78  // context of a state.
    79  //
    80  // Multiple TestSteps can be sequenced in a Test to allow testing
    81  // potentially complex update logic. In general, simply create/destroy
    82  // tests will only need one step.
    83  type TestStep struct {
    84  	// PreConfig is called before the Config is applied to perform any per-step
    85  	// setup that needs to happen
    86  	PreConfig func()
    87  
    88  	// Config a string of the configuration to give to Terraform.
    89  	Config string
    90  
    91  	// Check is called after the Config is applied. Use this step to
    92  	// make your own API calls to check the status of things, and to
    93  	// inspect the format of the ResourceState itself.
    94  	//
    95  	// If an error is returned, the test will fail. In this case, a
    96  	// destroy plan will still be attempted.
    97  	//
    98  	// If this is nil, no check is done on this step.
    99  	Check TestCheckFunc
   100  
   101  	// Destroy will create a destroy plan if set to true.
   102  	Destroy bool
   103  
   104  	// ExpectNonEmptyPlan can be set to true for specific types of tests that are
   105  	// looking to verify that a diff occurs
   106  	ExpectNonEmptyPlan bool
   107  }
   108  
   109  // Test performs an acceptance test on a resource.
   110  //
   111  // Tests are not run unless an environmental variable "TF_ACC" is
   112  // set to some non-empty value. This is to avoid test cases surprising
   113  // a user by creating real resources.
   114  //
   115  // Tests will fail unless the verbose flag (`go test -v`, or explicitly
   116  // the "-test.v" flag) is set. Because some acceptance tests take quite
   117  // long, we require the verbose flag so users are able to see progress
   118  // output.
   119  func Test(t TestT, c TestCase) {
   120  	// We only run acceptance tests if an env var is set because they're
   121  	// slow and generally require some outside configuration.
   122  	if os.Getenv(TestEnvVar) == "" {
   123  		t.Skip(fmt.Sprintf(
   124  			"Acceptance tests skipped unless env '%s' set",
   125  			TestEnvVar))
   126  		return
   127  	}
   128  
   129  	isUnitTest := (os.Getenv(TestEnvVar) == UnitTestOverride)
   130  
   131  	logWriter, err := logging.LogOutput()
   132  	if err != nil {
   133  		t.Error(fmt.Errorf("error setting up logging: %s", err))
   134  	}
   135  	log.SetOutput(logWriter)
   136  
   137  	// We require verbose mode so that the user knows what is going on.
   138  	if !testTesting && !testing.Verbose() && !isUnitTest {
   139  		t.Fatal("Acceptance tests must be run with the -v flag on tests")
   140  		return
   141  	}
   142  
   143  	// Run the PreCheck if we have it
   144  	if c.PreCheck != nil {
   145  		c.PreCheck()
   146  	}
   147  
   148  	// Build our context options that we can
   149  	ctxProviders := c.ProviderFactories
   150  	if ctxProviders == nil {
   151  		ctxProviders = make(map[string]terraform.ResourceProviderFactory)
   152  		for k, p := range c.Providers {
   153  			ctxProviders[k] = terraform.ResourceProviderFactoryFixed(p)
   154  		}
   155  	}
   156  	opts := terraform.ContextOpts{Providers: ctxProviders}
   157  
   158  	// A single state variable to track the lifecycle, starting with no state
   159  	var state *terraform.State
   160  
   161  	// Go through each step and run it
   162  	var idRefreshCheck *terraform.ResourceState
   163  	idRefresh := c.IDRefreshName != ""
   164  	errored := false
   165  	for i, step := range c.Steps {
   166  		var err error
   167  		log.Printf("[WARN] Test: Executing step %d", i)
   168  		state, err = testStep(opts, state, step)
   169  		if err != nil {
   170  			errored = true
   171  			t.Error(fmt.Sprintf(
   172  				"Step %d error: %s", i, err))
   173  			break
   174  		}
   175  
   176  		// If we've never checked an id-only refresh and our state isn't
   177  		// empty, find the first resource and test it.
   178  		if idRefresh && idRefreshCheck == nil && !state.Empty() {
   179  			// Find the first non-nil resource in the state
   180  			for _, m := range state.Modules {
   181  				if len(m.Resources) > 0 {
   182  					if v, ok := m.Resources[c.IDRefreshName]; ok {
   183  						idRefreshCheck = v
   184  					}
   185  
   186  					break
   187  				}
   188  			}
   189  
   190  			// If we have an instance to check for refreshes, do it
   191  			// immediately. We do it in the middle of another test
   192  			// because it shouldn't affect the overall state (refresh
   193  			// is read-only semantically) and we want to fail early if
   194  			// this fails. If refresh isn't read-only, then this will have
   195  			// caught a different bug.
   196  			if idRefreshCheck != nil {
   197  				log.Printf(
   198  					"[WARN] Test: Running ID-only refresh check on %s",
   199  					idRefreshCheck.Primary.ID)
   200  				if err := testIDOnlyRefresh(c, opts, step, idRefreshCheck); err != nil {
   201  					log.Printf("[ERROR] Test: ID-only test failed: %s", err)
   202  					t.Error(fmt.Sprintf(
   203  						"ID-Only refresh test failure: %s", err))
   204  					break
   205  				}
   206  			}
   207  		}
   208  	}
   209  
   210  	// If we never checked an id-only refresh, it is a failure.
   211  	if idRefresh {
   212  		if !errored && len(c.Steps) > 0 && idRefreshCheck == nil {
   213  			t.Error("ID-only refresh check never ran.")
   214  		}
   215  	}
   216  
   217  	// If we have a state, then run the destroy
   218  	if state != nil {
   219  		destroyStep := TestStep{
   220  			Config:  c.Steps[len(c.Steps)-1].Config,
   221  			Check:   c.CheckDestroy,
   222  			Destroy: true,
   223  		}
   224  
   225  		log.Printf("[WARN] Test: Executing destroy step")
   226  		state, err := testStep(opts, state, destroyStep)
   227  		if err != nil {
   228  			t.Error(fmt.Sprintf(
   229  				"Error destroying resource! WARNING: Dangling resources\n"+
   230  					"may exist. The full state and error is shown below.\n\n"+
   231  					"Error: %s\n\nState: %s",
   232  				err,
   233  				state))
   234  		}
   235  	} else {
   236  		log.Printf("[WARN] Skipping destroy test since there is no state.")
   237  	}
   238  }
   239  
   240  // UnitTest is a helper to force the acceptance testing harness to run in the
   241  // normal unit test suite. This should only be used for resource that don't
   242  // have any external dependencies.
   243  func UnitTest(t TestT, c TestCase) {
   244  	oldEnv := os.Getenv(TestEnvVar)
   245  	if err := os.Setenv(TestEnvVar, UnitTestOverride); err != nil {
   246  		t.Fatal(err)
   247  	}
   248  	defer func() {
   249  		if err := os.Setenv(TestEnvVar, oldEnv); err != nil {
   250  			t.Fatal(err)
   251  		}
   252  	}()
   253  	Test(t, c)
   254  }
   255  
   256  func testIDOnlyRefresh(c TestCase, opts terraform.ContextOpts, step TestStep, r *terraform.ResourceState) error {
   257  	// TODO: We guard by this right now so master doesn't explode. We
   258  	// need to remove this eventually to make this part of the normal tests.
   259  	if os.Getenv("TF_ACC_IDONLY") == "" {
   260  		return nil
   261  	}
   262  
   263  	name := fmt.Sprintf("%s.foo", r.Type)
   264  
   265  	// Build the state. The state is just the resource with an ID. There
   266  	// are no attributes. We only set what is needed to perform a refresh.
   267  	state := terraform.NewState()
   268  	state.RootModule().Resources[name] = &terraform.ResourceState{
   269  		Type: r.Type,
   270  		Primary: &terraform.InstanceState{
   271  			ID: r.Primary.ID,
   272  		},
   273  	}
   274  
   275  	// Create the config module. We use the full config because Refresh
   276  	// doesn't have access to it and we may need things like provider
   277  	// configurations. The initial implementation of id-only checks used
   278  	// an empty config module, but that caused the aforementioned problems.
   279  	mod, err := testModule(opts, step)
   280  	if err != nil {
   281  		return err
   282  	}
   283  
   284  	// Initialize the context
   285  	opts.Module = mod
   286  	opts.State = state
   287  	ctx := terraform.NewContext(&opts)
   288  	if ws, es := ctx.Validate(); len(ws) > 0 || len(es) > 0 {
   289  		if len(es) > 0 {
   290  			estrs := make([]string, len(es))
   291  			for i, e := range es {
   292  				estrs[i] = e.Error()
   293  			}
   294  			return fmt.Errorf(
   295  				"Configuration is invalid.\n\nWarnings: %#v\n\nErrors: %#v",
   296  				ws, estrs)
   297  		}
   298  
   299  		log.Printf("[WARN] Config warnings: %#v", ws)
   300  	}
   301  
   302  	// Refresh!
   303  	state, err = ctx.Refresh()
   304  	if err != nil {
   305  		return fmt.Errorf("Error refreshing: %s", err)
   306  	}
   307  
   308  	// Verify attribute equivalence.
   309  	actualR := state.RootModule().Resources[name]
   310  	if actualR == nil {
   311  		return fmt.Errorf("Resource gone!")
   312  	}
   313  	if actualR.Primary == nil {
   314  		return fmt.Errorf("Resource has no primary instance")
   315  	}
   316  	actual := actualR.Primary.Attributes
   317  	expected := r.Primary.Attributes
   318  	// Remove fields we're ignoring
   319  	for _, v := range c.IDRefreshIgnore {
   320  		for k, _ := range actual {
   321  			if strings.HasPrefix(k, v) {
   322  				delete(actual, k)
   323  			}
   324  		}
   325  		for k, _ := range expected {
   326  			if strings.HasPrefix(k, v) {
   327  				delete(expected, k)
   328  			}
   329  		}
   330  	}
   331  
   332  	if !reflect.DeepEqual(actual, expected) {
   333  		// Determine only the different attributes
   334  		for k, v := range expected {
   335  			if av, ok := actual[k]; ok && v == av {
   336  				delete(expected, k)
   337  				delete(actual, k)
   338  			}
   339  		}
   340  
   341  		spewConf := spew.NewDefaultConfig()
   342  		spewConf.SortKeys = true
   343  		return fmt.Errorf(
   344  			"Attributes not equivalent. Difference is shown below. Top is actual, bottom is expected."+
   345  				"\n\n%s\n\n%s",
   346  			spewConf.Sdump(actual), spewConf.Sdump(expected))
   347  	}
   348  
   349  	return nil
   350  }
   351  
   352  func testStep(
   353  	opts terraform.ContextOpts,
   354  	state *terraform.State,
   355  	step TestStep) (*terraform.State, error) {
   356  	mod, err := testModule(opts, step)
   357  	if err != nil {
   358  		return state, err
   359  	}
   360  
   361  	// Build the context
   362  	opts.Module = mod
   363  	opts.State = state
   364  	opts.Destroy = step.Destroy
   365  	ctx := terraform.NewContext(&opts)
   366  	if ws, es := ctx.Validate(); len(ws) > 0 || len(es) > 0 {
   367  		if len(es) > 0 {
   368  			estrs := make([]string, len(es))
   369  			for i, e := range es {
   370  				estrs[i] = e.Error()
   371  			}
   372  			return state, fmt.Errorf(
   373  				"Configuration is invalid.\n\nWarnings: %#v\n\nErrors: %#v",
   374  				ws, estrs)
   375  		}
   376  		log.Printf("[WARN] Config warnings: %#v", ws)
   377  	}
   378  
   379  	// Refresh!
   380  	state, err = ctx.Refresh()
   381  	if err != nil {
   382  		return state, fmt.Errorf(
   383  			"Error refreshing: %s", err)
   384  	}
   385  
   386  	// Plan!
   387  	if p, err := ctx.Plan(); err != nil {
   388  		return state, fmt.Errorf(
   389  			"Error planning: %s", err)
   390  	} else {
   391  		log.Printf("[WARN] Test: Step plan: %s", p)
   392  	}
   393  
   394  	// We need to keep a copy of the state prior to destroying
   395  	// such that destroy steps can verify their behaviour in the check
   396  	// function
   397  	stateBeforeApplication := state.DeepCopy()
   398  
   399  	// Apply!
   400  	state, err = ctx.Apply()
   401  	if err != nil {
   402  		return state, fmt.Errorf("Error applying: %s", err)
   403  	}
   404  
   405  	// Check! Excitement!
   406  	if step.Check != nil {
   407  		if step.Destroy {
   408  			if err := step.Check(stateBeforeApplication); err != nil {
   409  				return state, fmt.Errorf("Check failed: %s", err)
   410  			}
   411  		} else {
   412  			if err := step.Check(state); err != nil {
   413  				return state, fmt.Errorf("Check failed: %s", err)
   414  			}
   415  		}
   416  	}
   417  
   418  	// Now, verify that Plan is now empty and we don't have a perpetual diff issue
   419  	// We do this with TWO plans. One without a refresh.
   420  	var p *terraform.Plan
   421  	if p, err = ctx.Plan(); err != nil {
   422  		return state, fmt.Errorf("Error on follow-up plan: %s", err)
   423  	}
   424  	if p.Diff != nil && !p.Diff.Empty() {
   425  		if step.ExpectNonEmptyPlan {
   426  			log.Printf("[INFO] Got non-empty plan, as expected:\n\n%s", p)
   427  		} else {
   428  			return state, fmt.Errorf(
   429  				"After applying this step, the plan was not empty:\n\n%s", p)
   430  		}
   431  	}
   432  
   433  	// And another after a Refresh.
   434  	state, err = ctx.Refresh()
   435  	if err != nil {
   436  		return state, fmt.Errorf(
   437  			"Error on follow-up refresh: %s", err)
   438  	}
   439  	if p, err = ctx.Plan(); err != nil {
   440  		return state, fmt.Errorf("Error on second follow-up plan: %s", err)
   441  	}
   442  	if p.Diff != nil && !p.Diff.Empty() {
   443  		if step.ExpectNonEmptyPlan {
   444  			log.Printf("[INFO] Got non-empty plan, as expected:\n\n%s", p)
   445  		} else {
   446  			return state, fmt.Errorf(
   447  				"After applying this step and refreshing, "+
   448  					"the plan was not empty:\n\n%s", p)
   449  		}
   450  	}
   451  
   452  	// Made it here, but expected a non-empty plan, fail!
   453  	if step.ExpectNonEmptyPlan && (p.Diff == nil || p.Diff.Empty()) {
   454  		return state, fmt.Errorf("Expected a non-empty plan, but got an empty plan!")
   455  	}
   456  
   457  	// Made it here? Good job test step!
   458  	return state, nil
   459  }
   460  
   461  func testModule(
   462  	opts terraform.ContextOpts,
   463  	step TestStep) (*module.Tree, error) {
   464  	if step.PreConfig != nil {
   465  		step.PreConfig()
   466  	}
   467  
   468  	cfgPath, err := ioutil.TempDir("", "tf-test")
   469  	if err != nil {
   470  		return nil, fmt.Errorf(
   471  			"Error creating temporary directory for config: %s", err)
   472  	}
   473  	defer os.RemoveAll(cfgPath)
   474  
   475  	// Write the configuration
   476  	cfgF, err := os.Create(filepath.Join(cfgPath, "main.tf"))
   477  	if err != nil {
   478  		return nil, fmt.Errorf(
   479  			"Error creating temporary file for config: %s", err)
   480  	}
   481  
   482  	_, err = io.Copy(cfgF, strings.NewReader(step.Config))
   483  	cfgF.Close()
   484  	if err != nil {
   485  		return nil, fmt.Errorf(
   486  			"Error creating temporary file for config: %s", err)
   487  	}
   488  
   489  	// Parse the configuration
   490  	mod, err := module.NewTreeModule("", cfgPath)
   491  	if err != nil {
   492  		return nil, fmt.Errorf(
   493  			"Error loading configuration: %s", err)
   494  	}
   495  
   496  	// Load the modules
   497  	modStorage := &getter.FolderStorage{
   498  		StorageDir: filepath.Join(cfgPath, ".tfmodules"),
   499  	}
   500  	err = mod.Load(modStorage, module.GetModeGet)
   501  	if err != nil {
   502  		return nil, fmt.Errorf("Error downloading modules: %s", err)
   503  	}
   504  
   505  	return mod, nil
   506  }
   507  
   508  // ComposeTestCheckFunc lets you compose multiple TestCheckFuncs into
   509  // a single TestCheckFunc.
   510  //
   511  // As a user testing their provider, this lets you decompose your checks
   512  // into smaller pieces more easily.
   513  func ComposeTestCheckFunc(fs ...TestCheckFunc) TestCheckFunc {
   514  	return func(s *terraform.State) error {
   515  		for i, f := range fs {
   516  			if err := f(s); err != nil {
   517  				return fmt.Errorf("Check %d/%d error: %s", i+1, len(fs), err)
   518  			}
   519  		}
   520  
   521  		return nil
   522  	}
   523  }
   524  
   525  func TestCheckResourceAttr(name, key, value string) TestCheckFunc {
   526  	return func(s *terraform.State) error {
   527  		ms := s.RootModule()
   528  		rs, ok := ms.Resources[name]
   529  		if !ok {
   530  			return fmt.Errorf("Not found: %s", name)
   531  		}
   532  
   533  		is := rs.Primary
   534  		if is == nil {
   535  			return fmt.Errorf("No primary instance: %s", name)
   536  		}
   537  
   538  		if is.Attributes[key] != value {
   539  			return fmt.Errorf(
   540  				"%s: Attribute '%s' expected %#v, got %#v",
   541  				name,
   542  				key,
   543  				value,
   544  				is.Attributes[key])
   545  		}
   546  
   547  		return nil
   548  	}
   549  }
   550  
   551  func TestMatchResourceAttr(name, key string, r *regexp.Regexp) TestCheckFunc {
   552  	return func(s *terraform.State) error {
   553  		ms := s.RootModule()
   554  		rs, ok := ms.Resources[name]
   555  		if !ok {
   556  			return fmt.Errorf("Not found: %s", name)
   557  		}
   558  
   559  		is := rs.Primary
   560  		if is == nil {
   561  			return fmt.Errorf("No primary instance: %s", name)
   562  		}
   563  
   564  		if !r.MatchString(is.Attributes[key]) {
   565  			return fmt.Errorf(
   566  				"%s: Attribute '%s' didn't match %q, got %#v",
   567  				name,
   568  				key,
   569  				r.String(),
   570  				is.Attributes[key])
   571  		}
   572  
   573  		return nil
   574  	}
   575  }
   576  
   577  // TestCheckResourceAttrPtr is like TestCheckResourceAttr except the
   578  // value is a pointer so that it can be updated while the test is running.
   579  // It will only be dereferenced at the point this step is run.
   580  func TestCheckResourceAttrPtr(name string, key string, value *string) TestCheckFunc {
   581  	return func(s *terraform.State) error {
   582  		return TestCheckResourceAttr(name, key, *value)(s)
   583  	}
   584  }
   585  
   586  // TestCheckOutput checks an output in the Terraform configuration
   587  func TestCheckOutput(name, value string) TestCheckFunc {
   588  	return func(s *terraform.State) error {
   589  		ms := s.RootModule()
   590  		rs, ok := ms.Outputs[name]
   591  		if !ok {
   592  			return fmt.Errorf("Not found: %s", name)
   593  		}
   594  
   595  		if rs != value {
   596  			return fmt.Errorf(
   597  				"Output '%s': expected %#v, got %#v",
   598  				name,
   599  				value,
   600  				rs)
   601  		}
   602  
   603  		return nil
   604  	}
   605  }
   606  
   607  // TestT is the interface used to handle the test lifecycle of a test.
   608  //
   609  // Users should just use a *testing.T object, which implements this.
   610  type TestT interface {
   611  	Error(args ...interface{})
   612  	Fatal(args ...interface{})
   613  	Skip(args ...interface{})
   614  }
   615  
   616  // This is set to true by unit tests to alter some behavior
   617  var testTesting = false