github.com/jdextraze/terraform@v0.6.17-0.20160511153921-e33847c8a8af/helper/resource/testing.go (about)

     1  package resource
     2  
     3  import (
     4  	"fmt"
     5  	"io"
     6  	"io/ioutil"
     7  	"log"
     8  	"os"
     9  	"path/filepath"
    10  	"reflect"
    11  	"regexp"
    12  	"strings"
    13  	"testing"
    14  
    15  	"github.com/davecgh/go-spew/spew"
    16  	"github.com/hashicorp/go-getter"
    17  	"github.com/hashicorp/terraform/config/module"
    18  	"github.com/hashicorp/terraform/helper/logging"
    19  	"github.com/hashicorp/terraform/terraform"
    20  )
    21  
    22  const TestEnvVar = "TF_ACC"
    23  
    24  // UnitTestOverride is a value that when set in TestEnvVar indicates that this
    25  // is a unit test borrowing the acceptance testing framework.
    26  const UnitTestOverride = "UnitTestOverride"
    27  
    28  // TestCheckFunc is the callback type used with acceptance tests to check
    29  // the state of a resource. The state passed in is the latest state known,
    30  // or in the case of being after a destroy, it is the last known state when
    31  // it was created.
    32  type TestCheckFunc func(*terraform.State) error
    33  
    34  // TestCase is a single acceptance test case used to test the apply/destroy
    35  // lifecycle of a resource in a specific configuration.
    36  //
    37  // When the destroy plan is executed, the config from the last TestStep
    38  // is used to plan it.
    39  type TestCase struct {
    40  	// PreCheck, if non-nil, will be called before any test steps are
    41  	// executed. It will only be executed in the case that the steps
    42  	// would run, so it can be used for some validation before running
    43  	// acceptance tests, such as verifying that keys are setup.
    44  	PreCheck func()
    45  
    46  	// Providers is the ResourceProvider that will be under test.
    47  	//
    48  	// Alternately, ProviderFactories can be specified for the providers
    49  	// that are valid. This takes priority over Providers.
    50  	//
    51  	// The end effect of each is the same: specifying the providers that
    52  	// are used within the tests.
    53  	Providers         map[string]terraform.ResourceProvider
    54  	ProviderFactories map[string]terraform.ResourceProviderFactory
    55  
    56  	// CheckDestroy is called after the resource is finally destroyed
    57  	// to allow the tester to test that the resource is truly gone.
    58  	CheckDestroy TestCheckFunc
    59  
    60  	// Steps are the apply sequences done within the context of the
    61  	// same state. Each step can have its own check to verify correctness.
    62  	Steps []TestStep
    63  
    64  	// The settings below control the "ID-only refresh test." This is
    65  	// an enabled-by-default test that tests that a refresh can be
    66  	// refreshed with only an ID to result in the same attributes.
    67  	// This validates completeness of Refresh.
    68  	//
    69  	// IDRefreshName is the name of the resource to check. This will
    70  	// default to the first non-nil primary resource in the state.
    71  	//
    72  	// IDRefreshIgnore is a list of configuration keys that will be ignored.
    73  	IDRefreshName   string
    74  	IDRefreshIgnore []string
    75  }
    76  
    77  // TestStep is a single apply sequence of a test, done within the
    78  // context of a state.
    79  //
    80  // Multiple TestSteps can be sequenced in a Test to allow testing
    81  // potentially complex update logic. In general, simply create/destroy
    82  // tests will only need one step.
    83  type TestStep struct {
    84  	// PreConfig is called before the Config is applied to perform any per-step
    85  	// setup that needs to happen
    86  	PreConfig func()
    87  
    88  	// Config a string of the configuration to give to Terraform.
    89  	Config string
    90  
    91  	// Check is called after the Config is applied. Use this step to
    92  	// make your own API calls to check the status of things, and to
    93  	// inspect the format of the ResourceState itself.
    94  	//
    95  	// If an error is returned, the test will fail. In this case, a
    96  	// destroy plan will still be attempted.
    97  	//
    98  	// If this is nil, no check is done on this step.
    99  	Check TestCheckFunc
   100  
   101  	// Destroy will create a destroy plan if set to true.
   102  	Destroy bool
   103  
   104  	// ExpectNonEmptyPlan can be set to true for specific types of tests that are
   105  	// looking to verify that a diff occurs
   106  	ExpectNonEmptyPlan bool
   107  }
   108  
   109  // Test performs an acceptance test on a resource.
   110  //
   111  // Tests are not run unless an environmental variable "TF_ACC" is
   112  // set to some non-empty value. This is to avoid test cases surprising
   113  // a user by creating real resources.
   114  //
   115  // Tests will fail unless the verbose flag (`go test -v`, or explicitly
   116  // the "-test.v" flag) is set. Because some acceptance tests take quite
   117  // long, we require the verbose flag so users are able to see progress
   118  // output.
   119  func Test(t TestT, c TestCase) {
   120  	// We only run acceptance tests if an env var is set because they're
   121  	// slow and generally require some outside configuration.
   122  	if os.Getenv(TestEnvVar) == "" {
   123  		t.Skip(fmt.Sprintf(
   124  			"Acceptance tests skipped unless env '%s' set",
   125  			TestEnvVar))
   126  		return
   127  	}
   128  
   129  	isUnitTest := (os.Getenv(TestEnvVar) == UnitTestOverride)
   130  
   131  	logWriter, err := logging.LogOutput()
   132  	if err != nil {
   133  		t.Error(fmt.Errorf("error setting up logging: %s", err))
   134  	}
   135  	log.SetOutput(logWriter)
   136  
   137  	// We require verbose mode so that the user knows what is going on.
   138  	if !testTesting && !testing.Verbose() && !isUnitTest {
   139  		t.Fatal("Acceptance tests must be run with the -v flag on tests")
   140  		return
   141  	}
   142  
   143  	// Run the PreCheck if we have it
   144  	if c.PreCheck != nil {
   145  		c.PreCheck()
   146  	}
   147  
   148  	// Build our context options that we can
   149  	ctxProviders := c.ProviderFactories
   150  	if ctxProviders == nil {
   151  		ctxProviders = make(map[string]terraform.ResourceProviderFactory)
   152  		for k, p := range c.Providers {
   153  			ctxProviders[k] = terraform.ResourceProviderFactoryFixed(p)
   154  		}
   155  	}
   156  	opts := terraform.ContextOpts{Providers: ctxProviders}
   157  
   158  	// A single state variable to track the lifecycle, starting with no state
   159  	var state *terraform.State
   160  
   161  	// Go through each step and run it
   162  	var idRefreshCheck *terraform.ResourceState
   163  	idRefresh := c.IDRefreshName != ""
   164  	errored := false
   165  	for i, step := range c.Steps {
   166  		var err error
   167  		log.Printf("[WARN] Test: Executing step %d", i)
   168  		state, err = testStep(opts, state, step)
   169  		if err != nil {
   170  			errored = true
   171  			t.Error(fmt.Sprintf(
   172  				"Step %d error: %s", i, err))
   173  			break
   174  		}
   175  
   176  		// If we've never checked an id-only refresh and our state isn't
   177  		// empty, find the first resource and test it.
   178  		if idRefresh && idRefreshCheck == nil && !state.Empty() {
   179  			// Find the first non-nil resource in the state
   180  			for _, m := range state.Modules {
   181  				if len(m.Resources) > 0 {
   182  					if v, ok := m.Resources[c.IDRefreshName]; ok {
   183  						idRefreshCheck = v
   184  					}
   185  
   186  					break
   187  				}
   188  			}
   189  
   190  			// If we have an instance to check for refreshes, do it
   191  			// immediately. We do it in the middle of another test
   192  			// because it shouldn't affect the overall state (refresh
   193  			// is read-only semantically) and we want to fail early if
   194  			// this fails. If refresh isn't read-only, then this will have
   195  			// caught a different bug.
   196  			if idRefreshCheck != nil {
   197  				log.Printf(
   198  					"[WARN] Test: Running ID-only refresh check on %s",
   199  					idRefreshCheck.Primary.ID)
   200  				if err := testIDOnlyRefresh(c, opts, step, idRefreshCheck); err != nil {
   201  					log.Printf("[ERROR] Test: ID-only test failed: %s", err)
   202  					t.Error(fmt.Sprintf(
   203  						"ID-Only refresh test failure: %s", err))
   204  					break
   205  				}
   206  			}
   207  		}
   208  	}
   209  
   210  	// If we never checked an id-only refresh, it is a failure.
   211  	if idRefresh {
   212  		if !errored && len(c.Steps) > 0 && idRefreshCheck == nil {
   213  			t.Error("ID-only refresh check never ran.")
   214  		}
   215  	}
   216  
   217  	// If we have a state, then run the destroy
   218  	if state != nil {
   219  		destroyStep := TestStep{
   220  			Config:  c.Steps[len(c.Steps)-1].Config,
   221  			Check:   c.CheckDestroy,
   222  			Destroy: true,
   223  		}
   224  
   225  		log.Printf("[WARN] Test: Executing destroy step")
   226  		state, err := testStep(opts, state, destroyStep)
   227  		if err != nil {
   228  			t.Error(fmt.Sprintf(
   229  				"Error destroying resource! WARNING: Dangling resources\n"+
   230  					"may exist. The full state and error is shown below.\n\n"+
   231  					"Error: %s\n\nState: %s",
   232  				err,
   233  				state))
   234  		}
   235  	} else {
   236  		log.Printf("[WARN] Skipping destroy test since there is no state.")
   237  	}
   238  }
   239  
   240  // UnitTest is a helper to force the acceptance testing harness to run in the
   241  // normal unit test suite. This should only be used for resource that don't
   242  // have any external dependencies.
   243  func UnitTest(t TestT, c TestCase) {
   244  	oldEnv := os.Getenv(TestEnvVar)
   245  	if err := os.Setenv(TestEnvVar, UnitTestOverride); err != nil {
   246  		t.Fatal(err)
   247  	}
   248  	defer func() {
   249  		if err := os.Setenv(TestEnvVar, oldEnv); err != nil {
   250  			t.Fatal(err)
   251  		}
   252  	}()
   253  	Test(t, c)
   254  }
   255  
   256  func testIDOnlyRefresh(c TestCase, opts terraform.ContextOpts, step TestStep, r *terraform.ResourceState) error {
   257  	// TODO: We guard by this right now so master doesn't explode. We
   258  	// need to remove this eventually to make this part of the normal tests.
   259  	if os.Getenv("TF_ACC_IDONLY") == "" {
   260  		return nil
   261  	}
   262  
   263  	name := fmt.Sprintf("%s.foo", r.Type)
   264  
   265  	// Build the state. The state is just the resource with an ID. There
   266  	// are no attributes. We only set what is needed to perform a refresh.
   267  	state := terraform.NewState()
   268  	state.RootModule().Resources[name] = &terraform.ResourceState{
   269  		Type: r.Type,
   270  		Primary: &terraform.InstanceState{
   271  			ID: r.Primary.ID,
   272  		},
   273  	}
   274  
   275  	// Create the config module. We use the full config because Refresh
   276  	// doesn't have access to it and we may need things like provider
   277  	// configurations. The initial implementation of id-only checks used
   278  	// an empty config module, but that caused the aforementioned problems.
   279  	mod, err := testModule(opts, step)
   280  	if err != nil {
   281  		return err
   282  	}
   283  
   284  	// Initialize the context
   285  	opts.Module = mod
   286  	opts.State = state
   287  	ctx, err := terraform.NewContext(&opts)
   288  	if err != nil {
   289  		return err
   290  	}
   291  	if ws, es := ctx.Validate(); len(ws) > 0 || len(es) > 0 {
   292  		if len(es) > 0 {
   293  			estrs := make([]string, len(es))
   294  			for i, e := range es {
   295  				estrs[i] = e.Error()
   296  			}
   297  			return fmt.Errorf(
   298  				"Configuration is invalid.\n\nWarnings: %#v\n\nErrors: %#v",
   299  				ws, estrs)
   300  		}
   301  
   302  		log.Printf("[WARN] Config warnings: %#v", ws)
   303  	}
   304  
   305  	// Refresh!
   306  	state, err = ctx.Refresh()
   307  	if err != nil {
   308  		return fmt.Errorf("Error refreshing: %s", err)
   309  	}
   310  
   311  	// Verify attribute equivalence.
   312  	actualR := state.RootModule().Resources[name]
   313  	if actualR == nil {
   314  		return fmt.Errorf("Resource gone!")
   315  	}
   316  	if actualR.Primary == nil {
   317  		return fmt.Errorf("Resource has no primary instance")
   318  	}
   319  	actual := actualR.Primary.Attributes
   320  	expected := r.Primary.Attributes
   321  	// Remove fields we're ignoring
   322  	for _, v := range c.IDRefreshIgnore {
   323  		for k, _ := range actual {
   324  			if strings.HasPrefix(k, v) {
   325  				delete(actual, k)
   326  			}
   327  		}
   328  		for k, _ := range expected {
   329  			if strings.HasPrefix(k, v) {
   330  				delete(expected, k)
   331  			}
   332  		}
   333  	}
   334  
   335  	if !reflect.DeepEqual(actual, expected) {
   336  		// Determine only the different attributes
   337  		for k, v := range expected {
   338  			if av, ok := actual[k]; ok && v == av {
   339  				delete(expected, k)
   340  				delete(actual, k)
   341  			}
   342  		}
   343  
   344  		spewConf := spew.NewDefaultConfig()
   345  		spewConf.SortKeys = true
   346  		return fmt.Errorf(
   347  			"Attributes not equivalent. Difference is shown below. Top is actual, bottom is expected."+
   348  				"\n\n%s\n\n%s",
   349  			spewConf.Sdump(actual), spewConf.Sdump(expected))
   350  	}
   351  
   352  	return nil
   353  }
   354  
   355  func testStep(
   356  	opts terraform.ContextOpts,
   357  	state *terraform.State,
   358  	step TestStep) (*terraform.State, error) {
   359  	mod, err := testModule(opts, step)
   360  	if err != nil {
   361  		return state, err
   362  	}
   363  
   364  	// Build the context
   365  	opts.Module = mod
   366  	opts.State = state
   367  	opts.Destroy = step.Destroy
   368  	ctx, err := terraform.NewContext(&opts)
   369  	if err != nil {
   370  		return state, fmt.Errorf("Error initializing context: %s", err)
   371  	}
   372  	if ws, es := ctx.Validate(); len(ws) > 0 || len(es) > 0 {
   373  		if len(es) > 0 {
   374  			estrs := make([]string, len(es))
   375  			for i, e := range es {
   376  				estrs[i] = e.Error()
   377  			}
   378  			return state, fmt.Errorf(
   379  				"Configuration is invalid.\n\nWarnings: %#v\n\nErrors: %#v",
   380  				ws, estrs)
   381  		}
   382  		log.Printf("[WARN] Config warnings: %#v", ws)
   383  	}
   384  
   385  	// Refresh!
   386  	state, err = ctx.Refresh()
   387  	if err != nil {
   388  		return state, fmt.Errorf(
   389  			"Error refreshing: %s", err)
   390  	}
   391  
   392  	// Plan!
   393  	if p, err := ctx.Plan(); err != nil {
   394  		return state, fmt.Errorf(
   395  			"Error planning: %s", err)
   396  	} else {
   397  		log.Printf("[WARN] Test: Step plan: %s", p)
   398  	}
   399  
   400  	// We need to keep a copy of the state prior to destroying
   401  	// such that destroy steps can verify their behaviour in the check
   402  	// function
   403  	stateBeforeApplication := state.DeepCopy()
   404  
   405  	// Apply!
   406  	state, err = ctx.Apply()
   407  	if err != nil {
   408  		return state, fmt.Errorf("Error applying: %s", err)
   409  	}
   410  
   411  	// Check! Excitement!
   412  	if step.Check != nil {
   413  		if step.Destroy {
   414  			if err := step.Check(stateBeforeApplication); err != nil {
   415  				return state, fmt.Errorf("Check failed: %s", err)
   416  			}
   417  		} else {
   418  			if err := step.Check(state); err != nil {
   419  				return state, fmt.Errorf("Check failed: %s", err)
   420  			}
   421  		}
   422  	}
   423  
   424  	// Now, verify that Plan is now empty and we don't have a perpetual diff issue
   425  	// We do this with TWO plans. One without a refresh.
   426  	var p *terraform.Plan
   427  	if p, err = ctx.Plan(); err != nil {
   428  		return state, fmt.Errorf("Error on follow-up plan: %s", err)
   429  	}
   430  	if p.Diff != nil && !p.Diff.Empty() {
   431  		if step.ExpectNonEmptyPlan {
   432  			log.Printf("[INFO] Got non-empty plan, as expected:\n\n%s", p)
   433  		} else {
   434  			return state, fmt.Errorf(
   435  				"After applying this step, the plan was not empty:\n\n%s", p)
   436  		}
   437  	}
   438  
   439  	// And another after a Refresh.
   440  	state, err = ctx.Refresh()
   441  	if err != nil {
   442  		return state, fmt.Errorf(
   443  			"Error on follow-up refresh: %s", err)
   444  	}
   445  	if p, err = ctx.Plan(); err != nil {
   446  		return state, fmt.Errorf("Error on second follow-up plan: %s", err)
   447  	}
   448  	if p.Diff != nil && !p.Diff.Empty() {
   449  		if step.ExpectNonEmptyPlan {
   450  			log.Printf("[INFO] Got non-empty plan, as expected:\n\n%s", p)
   451  		} else {
   452  			return state, fmt.Errorf(
   453  				"After applying this step and refreshing, "+
   454  					"the plan was not empty:\n\n%s", p)
   455  		}
   456  	}
   457  
   458  	// Made it here, but expected a non-empty plan, fail!
   459  	if step.ExpectNonEmptyPlan && (p.Diff == nil || p.Diff.Empty()) {
   460  		return state, fmt.Errorf("Expected a non-empty plan, but got an empty plan!")
   461  	}
   462  
   463  	// Made it here? Good job test step!
   464  	return state, nil
   465  }
   466  
   467  func testModule(
   468  	opts terraform.ContextOpts,
   469  	step TestStep) (*module.Tree, error) {
   470  	if step.PreConfig != nil {
   471  		step.PreConfig()
   472  	}
   473  
   474  	cfgPath, err := ioutil.TempDir("", "tf-test")
   475  	if err != nil {
   476  		return nil, fmt.Errorf(
   477  			"Error creating temporary directory for config: %s", err)
   478  	}
   479  	defer os.RemoveAll(cfgPath)
   480  
   481  	// Write the configuration
   482  	cfgF, err := os.Create(filepath.Join(cfgPath, "main.tf"))
   483  	if err != nil {
   484  		return nil, fmt.Errorf(
   485  			"Error creating temporary file for config: %s", err)
   486  	}
   487  
   488  	_, err = io.Copy(cfgF, strings.NewReader(step.Config))
   489  	cfgF.Close()
   490  	if err != nil {
   491  		return nil, fmt.Errorf(
   492  			"Error creating temporary file for config: %s", err)
   493  	}
   494  
   495  	// Parse the configuration
   496  	mod, err := module.NewTreeModule("", cfgPath)
   497  	if err != nil {
   498  		return nil, fmt.Errorf(
   499  			"Error loading configuration: %s", err)
   500  	}
   501  
   502  	// Load the modules
   503  	modStorage := &getter.FolderStorage{
   504  		StorageDir: filepath.Join(cfgPath, ".tfmodules"),
   505  	}
   506  	err = mod.Load(modStorage, module.GetModeGet)
   507  	if err != nil {
   508  		return nil, fmt.Errorf("Error downloading modules: %s", err)
   509  	}
   510  
   511  	return mod, nil
   512  }
   513  
   514  // ComposeTestCheckFunc lets you compose multiple TestCheckFuncs into
   515  // a single TestCheckFunc.
   516  //
   517  // As a user testing their provider, this lets you decompose your checks
   518  // into smaller pieces more easily.
   519  func ComposeTestCheckFunc(fs ...TestCheckFunc) TestCheckFunc {
   520  	return func(s *terraform.State) error {
   521  		for i, f := range fs {
   522  			if err := f(s); err != nil {
   523  				return fmt.Errorf("Check %d/%d error: %s", i+1, len(fs), err)
   524  			}
   525  		}
   526  
   527  		return nil
   528  	}
   529  }
   530  
   531  func TestCheckResourceAttr(name, key, value string) TestCheckFunc {
   532  	return func(s *terraform.State) error {
   533  		ms := s.RootModule()
   534  		rs, ok := ms.Resources[name]
   535  		if !ok {
   536  			return fmt.Errorf("Not found: %s", name)
   537  		}
   538  
   539  		is := rs.Primary
   540  		if is == nil {
   541  			return fmt.Errorf("No primary instance: %s", name)
   542  		}
   543  
   544  		if is.Attributes[key] != value {
   545  			return fmt.Errorf(
   546  				"%s: Attribute '%s' expected %#v, got %#v",
   547  				name,
   548  				key,
   549  				value,
   550  				is.Attributes[key])
   551  		}
   552  
   553  		return nil
   554  	}
   555  }
   556  
   557  func TestMatchResourceAttr(name, key string, r *regexp.Regexp) TestCheckFunc {
   558  	return func(s *terraform.State) error {
   559  		ms := s.RootModule()
   560  		rs, ok := ms.Resources[name]
   561  		if !ok {
   562  			return fmt.Errorf("Not found: %s", name)
   563  		}
   564  
   565  		is := rs.Primary
   566  		if is == nil {
   567  			return fmt.Errorf("No primary instance: %s", name)
   568  		}
   569  
   570  		if !r.MatchString(is.Attributes[key]) {
   571  			return fmt.Errorf(
   572  				"%s: Attribute '%s' didn't match %q, got %#v",
   573  				name,
   574  				key,
   575  				r.String(),
   576  				is.Attributes[key])
   577  		}
   578  
   579  		return nil
   580  	}
   581  }
   582  
   583  // TestCheckResourceAttrPtr is like TestCheckResourceAttr except the
   584  // value is a pointer so that it can be updated while the test is running.
   585  // It will only be dereferenced at the point this step is run.
   586  func TestCheckResourceAttrPtr(name string, key string, value *string) TestCheckFunc {
   587  	return func(s *terraform.State) error {
   588  		return TestCheckResourceAttr(name, key, *value)(s)
   589  	}
   590  }
   591  
   592  // TestCheckOutput checks an output in the Terraform configuration
   593  func TestCheckOutput(name, value string) TestCheckFunc {
   594  	return func(s *terraform.State) error {
   595  		ms := s.RootModule()
   596  		rs, ok := ms.Outputs[name]
   597  		if !ok {
   598  			return fmt.Errorf("Not found: %s", name)
   599  		}
   600  
   601  		if rs != value {
   602  			return fmt.Errorf(
   603  				"Output '%s': expected %#v, got %#v",
   604  				name,
   605  				value,
   606  				rs)
   607  		}
   608  
   609  		return nil
   610  	}
   611  }
   612  
   613  // TestT is the interface used to handle the test lifecycle of a test.
   614  //
   615  // Users should just use a *testing.T object, which implements this.
   616  type TestT interface {
   617  	Error(args ...interface{})
   618  	Fatal(args ...interface{})
   619  	Skip(args ...interface{})
   620  }
   621  
   622  // This is set to true by unit tests to alter some behavior
   623  var testTesting = false