github.com/sarguru/terraform@v0.6.17-0.20160525232901-8fcdfd7e3dc9/helper/resource/testing.go (about)

     1  package resource
     2  
     3  import (
     4  	"fmt"
     5  	"io"
     6  	"io/ioutil"
     7  	"log"
     8  	"os"
     9  	"path/filepath"
    10  	"reflect"
    11  	"regexp"
    12  	"strings"
    13  	"testing"
    14  
    15  	"github.com/davecgh/go-spew/spew"
    16  	"github.com/hashicorp/go-getter"
    17  	"github.com/hashicorp/terraform/config/module"
    18  	"github.com/hashicorp/terraform/helper/logging"
    19  	"github.com/hashicorp/terraform/terraform"
    20  )
    21  
    22  const TestEnvVar = "TF_ACC"
    23  
    24  // UnitTestOverride is a value that when set in TestEnvVar indicates that this
    25  // is a unit test borrowing the acceptance testing framework.
    26  const UnitTestOverride = "UnitTestOverride"
    27  
    28  // TestCheckFunc is the callback type used with acceptance tests to check
    29  // the state of a resource. The state passed in is the latest state known,
    30  // or in the case of being after a destroy, it is the last known state when
    31  // it was created.
    32  type TestCheckFunc func(*terraform.State) error
    33  
    34  // ImportStateCheckFunc is the check function for ImportState tests
    35  type ImportStateCheckFunc func([]*terraform.InstanceState) error
    36  
    37  // TestCase is a single acceptance test case used to test the apply/destroy
    38  // lifecycle of a resource in a specific configuration.
    39  //
    40  // When the destroy plan is executed, the config from the last TestStep
    41  // is used to plan it.
    42  type TestCase struct {
    43  	// PreCheck, if non-nil, will be called before any test steps are
    44  	// executed. It will only be executed in the case that the steps
    45  	// would run, so it can be used for some validation before running
    46  	// acceptance tests, such as verifying that keys are setup.
    47  	PreCheck func()
    48  
    49  	// Providers is the ResourceProvider that will be under test.
    50  	//
    51  	// Alternately, ProviderFactories can be specified for the providers
    52  	// that are valid. This takes priority over Providers.
    53  	//
    54  	// The end effect of each is the same: specifying the providers that
    55  	// are used within the tests.
    56  	Providers         map[string]terraform.ResourceProvider
    57  	ProviderFactories map[string]terraform.ResourceProviderFactory
    58  
    59  	// CheckDestroy is called after the resource is finally destroyed
    60  	// to allow the tester to test that the resource is truly gone.
    61  	CheckDestroy TestCheckFunc
    62  
    63  	// Steps are the apply sequences done within the context of the
    64  	// same state. Each step can have its own check to verify correctness.
    65  	Steps []TestStep
    66  
    67  	// The settings below control the "ID-only refresh test." This is
    68  	// an enabled-by-default test that tests that a refresh can be
    69  	// refreshed with only an ID to result in the same attributes.
    70  	// This validates completeness of Refresh.
    71  	//
    72  	// IDRefreshName is the name of the resource to check. This will
    73  	// default to the first non-nil primary resource in the state.
    74  	//
    75  	// IDRefreshIgnore is a list of configuration keys that will be ignored.
    76  	IDRefreshName   string
    77  	IDRefreshIgnore []string
    78  }
    79  
    80  // TestStep is a single apply sequence of a test, done within the
    81  // context of a state.
    82  //
    83  // Multiple TestSteps can be sequenced in a Test to allow testing
    84  // potentially complex update logic. In general, simply create/destroy
    85  // tests will only need one step.
    86  type TestStep struct {
    87  	// ResourceName should be set to the name of the resource
    88  	// that is being tested. Example: "aws_instance.foo". Various test
    89  	// modes use this to auto-detect state information.
    90  	//
    91  	// This is only required if the test mode settings below say it is
    92  	// for the mode you're using.
    93  	ResourceName string
    94  
    95  	// PreConfig is called before the Config is applied to perform any per-step
    96  	// setup that needs to happen. This is called regardless of "test mode"
    97  	// below.
    98  	PreConfig func()
    99  
   100  	//---------------------------------------------------------------
   101  	// Test modes. One of the following groups of settings must be
   102  	// set to determine what the test step will do. Ideally we would've
   103  	// used Go interfaces here but there are now hundreds of tests we don't
   104  	// want to re-type so instead we just determine which step logic
   105  	// to run based on what settings below are set.
   106  	//---------------------------------------------------------------
   107  
   108  	//---------------------------------------------------------------
   109  	// Plan, Apply testing
   110  	//---------------------------------------------------------------
   111  
   112  	// Config a string of the configuration to give to Terraform. If this
   113  	// is set, then the TestCase will execute this step with the same logic
   114  	// as a `terraform apply`.
   115  	Config string
   116  
   117  	// Check is called after the Config is applied. Use this step to
   118  	// make your own API calls to check the status of things, and to
   119  	// inspect the format of the ResourceState itself.
   120  	//
   121  	// If an error is returned, the test will fail. In this case, a
   122  	// destroy plan will still be attempted.
   123  	//
   124  	// If this is nil, no check is done on this step.
   125  	Check TestCheckFunc
   126  
   127  	// Destroy will create a destroy plan if set to true.
   128  	Destroy bool
   129  
   130  	// ExpectNonEmptyPlan can be set to true for specific types of tests that are
   131  	// looking to verify that a diff occurs
   132  	ExpectNonEmptyPlan bool
   133  
   134  	//---------------------------------------------------------------
   135  	// ImportState testing
   136  	//---------------------------------------------------------------
   137  
   138  	// ImportState, if true, will test the functionality of ImportState
   139  	// by importing the resource with ResourceName (must be set) and the
   140  	// ID of that resource.
   141  	ImportState bool
   142  
   143  	// ImportStateId is the ID to perform an ImportState operation with.
   144  	// This is optional. If it isn't set, then the resource ID is automatically
   145  	// determined by inspecting the state for ResourceName's ID.
   146  	ImportStateId string
   147  
   148  	// ImportStateCheck checks the results of ImportState. It should be
   149  	// used to verify that the resulting value of ImportState has the
   150  	// proper resources, IDs, and attributes.
   151  	ImportStateCheck ImportStateCheckFunc
   152  
   153  	// ImportStateVerify, if true, will also check that the state values
   154  	// that are finally put into the state after import match for all the
   155  	// IDs returned by the Import.
   156  	//
   157  	// ImportStateVerifyIgnore are fields that should not be verified to
   158  	// be equal. These can be set to ephemeral fields or fields that can't
   159  	// be refreshed and don't matter.
   160  	ImportStateVerify       bool
   161  	ImportStateVerifyIgnore []string
   162  }
   163  
   164  // Test performs an acceptance test on a resource.
   165  //
   166  // Tests are not run unless an environmental variable "TF_ACC" is
   167  // set to some non-empty value. This is to avoid test cases surprising
   168  // a user by creating real resources.
   169  //
   170  // Tests will fail unless the verbose flag (`go test -v`, or explicitly
   171  // the "-test.v" flag) is set. Because some acceptance tests take quite
   172  // long, we require the verbose flag so users are able to see progress
   173  // output.
   174  func Test(t TestT, c TestCase) {
   175  	// We only run acceptance tests if an env var is set because they're
   176  	// slow and generally require some outside configuration.
   177  	if os.Getenv(TestEnvVar) == "" {
   178  		t.Skip(fmt.Sprintf(
   179  			"Acceptance tests skipped unless env '%s' set",
   180  			TestEnvVar))
   181  		return
   182  	}
   183  
   184  	isUnitTest := (os.Getenv(TestEnvVar) == UnitTestOverride)
   185  
   186  	logWriter, err := logging.LogOutput()
   187  	if err != nil {
   188  		t.Error(fmt.Errorf("error setting up logging: %s", err))
   189  	}
   190  	log.SetOutput(logWriter)
   191  
   192  	// We require verbose mode so that the user knows what is going on.
   193  	if !testTesting && !testing.Verbose() && !isUnitTest {
   194  		t.Fatal("Acceptance tests must be run with the -v flag on tests")
   195  		return
   196  	}
   197  
   198  	// Run the PreCheck if we have it
   199  	if c.PreCheck != nil {
   200  		c.PreCheck()
   201  	}
   202  
   203  	// Build our context options that we can
   204  	ctxProviders := c.ProviderFactories
   205  	if ctxProviders == nil {
   206  		ctxProviders = make(map[string]terraform.ResourceProviderFactory)
   207  		for k, p := range c.Providers {
   208  			ctxProviders[k] = terraform.ResourceProviderFactoryFixed(p)
   209  		}
   210  	}
   211  	opts := terraform.ContextOpts{Providers: ctxProviders}
   212  
   213  	// A single state variable to track the lifecycle, starting with no state
   214  	var state *terraform.State
   215  
   216  	// Go through each step and run it
   217  	var idRefreshCheck *terraform.ResourceState
   218  	idRefresh := c.IDRefreshName != ""
   219  	errored := false
   220  	for i, step := range c.Steps {
   221  		var err error
   222  		log.Printf("[WARN] Test: Executing step %d", i)
   223  
   224  		// Determine the test mode to execute
   225  		if step.Config != "" {
   226  			state, err = testStepConfig(opts, state, step)
   227  		} else if step.ImportState {
   228  			state, err = testStepImportState(opts, state, step)
   229  		} else {
   230  			err = fmt.Errorf(
   231  				"unknown test mode for step. Please see TestStep docs\n\n%#v",
   232  				step)
   233  		}
   234  
   235  		// If there was an error, exit
   236  		if err != nil {
   237  			errored = true
   238  			t.Error(fmt.Sprintf(
   239  				"Step %d error: %s", i, err))
   240  			break
   241  		}
   242  
   243  		// If we've never checked an id-only refresh and our state isn't
   244  		// empty, find the first resource and test it.
   245  		if idRefresh && idRefreshCheck == nil && !state.Empty() {
   246  			// Find the first non-nil resource in the state
   247  			for _, m := range state.Modules {
   248  				if len(m.Resources) > 0 {
   249  					if v, ok := m.Resources[c.IDRefreshName]; ok {
   250  						idRefreshCheck = v
   251  					}
   252  
   253  					break
   254  				}
   255  			}
   256  
   257  			// If we have an instance to check for refreshes, do it
   258  			// immediately. We do it in the middle of another test
   259  			// because it shouldn't affect the overall state (refresh
   260  			// is read-only semantically) and we want to fail early if
   261  			// this fails. If refresh isn't read-only, then this will have
   262  			// caught a different bug.
   263  			if idRefreshCheck != nil {
   264  				log.Printf(
   265  					"[WARN] Test: Running ID-only refresh check on %s",
   266  					idRefreshCheck.Primary.ID)
   267  				if err := testIDOnlyRefresh(c, opts, step, idRefreshCheck); err != nil {
   268  					log.Printf("[ERROR] Test: ID-only test failed: %s", err)
   269  					t.Error(fmt.Sprintf(
   270  						"[ERROR] Test: ID-only test failed: %s", err))
   271  					break
   272  				}
   273  			}
   274  		}
   275  	}
   276  
   277  	// If we never checked an id-only refresh, it is a failure.
   278  	if idRefresh {
   279  		if !errored && len(c.Steps) > 0 && idRefreshCheck == nil {
   280  			t.Error("ID-only refresh check never ran.")
   281  		}
   282  	}
   283  
   284  	// If we have a state, then run the destroy
   285  	if state != nil {
   286  		destroyStep := TestStep{
   287  			Config:  c.Steps[len(c.Steps)-1].Config,
   288  			Check:   c.CheckDestroy,
   289  			Destroy: true,
   290  		}
   291  
   292  		log.Printf("[WARN] Test: Executing destroy step")
   293  		state, err := testStep(opts, state, destroyStep)
   294  		if err != nil {
   295  			t.Error(fmt.Sprintf(
   296  				"Error destroying resource! WARNING: Dangling resources\n"+
   297  					"may exist. The full state and error is shown below.\n\n"+
   298  					"Error: %s\n\nState: %s",
   299  				err,
   300  				state))
   301  		}
   302  	} else {
   303  		log.Printf("[WARN] Skipping destroy test since there is no state.")
   304  	}
   305  }
   306  
   307  // UnitTest is a helper to force the acceptance testing harness to run in the
   308  // normal unit test suite. This should only be used for resource that don't
   309  // have any external dependencies.
   310  func UnitTest(t TestT, c TestCase) {
   311  	oldEnv := os.Getenv(TestEnvVar)
   312  	if err := os.Setenv(TestEnvVar, UnitTestOverride); err != nil {
   313  		t.Fatal(err)
   314  	}
   315  	defer func() {
   316  		if err := os.Setenv(TestEnvVar, oldEnv); err != nil {
   317  			t.Fatal(err)
   318  		}
   319  	}()
   320  	Test(t, c)
   321  }
   322  
   323  func testIDOnlyRefresh(c TestCase, opts terraform.ContextOpts, step TestStep, r *terraform.ResourceState) error {
   324  	// TODO: We guard by this right now so master doesn't explode. We
   325  	// need to remove this eventually to make this part of the normal tests.
   326  	if os.Getenv("TF_ACC_IDONLY") == "" {
   327  		return nil
   328  	}
   329  
   330  	name := fmt.Sprintf("%s.foo", r.Type)
   331  
   332  	// Build the state. The state is just the resource with an ID. There
   333  	// are no attributes. We only set what is needed to perform a refresh.
   334  	state := terraform.NewState()
   335  	state.RootModule().Resources[name] = &terraform.ResourceState{
   336  		Type: r.Type,
   337  		Primary: &terraform.InstanceState{
   338  			ID: r.Primary.ID,
   339  		},
   340  	}
   341  
   342  	// Create the config module. We use the full config because Refresh
   343  	// doesn't have access to it and we may need things like provider
   344  	// configurations. The initial implementation of id-only checks used
   345  	// an empty config module, but that caused the aforementioned problems.
   346  	mod, err := testModule(opts, step)
   347  	if err != nil {
   348  		return err
   349  	}
   350  
   351  	// Initialize the context
   352  	opts.Module = mod
   353  	opts.State = state
   354  	ctx, err := terraform.NewContext(&opts)
   355  	if err != nil {
   356  		return err
   357  	}
   358  	if ws, es := ctx.Validate(); len(ws) > 0 || len(es) > 0 {
   359  		if len(es) > 0 {
   360  			estrs := make([]string, len(es))
   361  			for i, e := range es {
   362  				estrs[i] = e.Error()
   363  			}
   364  			return fmt.Errorf(
   365  				"Configuration is invalid.\n\nWarnings: %#v\n\nErrors: %#v",
   366  				ws, estrs)
   367  		}
   368  
   369  		log.Printf("[WARN] Config warnings: %#v", ws)
   370  	}
   371  
   372  	// Refresh!
   373  	state, err = ctx.Refresh()
   374  	if err != nil {
   375  		return fmt.Errorf("Error refreshing: %s", err)
   376  	}
   377  
   378  	// Verify attribute equivalence.
   379  	actualR := state.RootModule().Resources[name]
   380  	if actualR == nil {
   381  		return fmt.Errorf("Resource gone!")
   382  	}
   383  	if actualR.Primary == nil {
   384  		return fmt.Errorf("Resource has no primary instance")
   385  	}
   386  	actual := actualR.Primary.Attributes
   387  	expected := r.Primary.Attributes
   388  	// Remove fields we're ignoring
   389  	for _, v := range c.IDRefreshIgnore {
   390  		for k, _ := range actual {
   391  			if strings.HasPrefix(k, v) {
   392  				delete(actual, k)
   393  			}
   394  		}
   395  		for k, _ := range expected {
   396  			if strings.HasPrefix(k, v) {
   397  				delete(expected, k)
   398  			}
   399  		}
   400  	}
   401  
   402  	if !reflect.DeepEqual(actual, expected) {
   403  		// Determine only the different attributes
   404  		for k, v := range expected {
   405  			if av, ok := actual[k]; ok && v == av {
   406  				delete(expected, k)
   407  				delete(actual, k)
   408  			}
   409  		}
   410  
   411  		spewConf := spew.NewDefaultConfig()
   412  		spewConf.SortKeys = true
   413  		return fmt.Errorf(
   414  			"Attributes not equivalent. Difference is shown below. Top is actual, bottom is expected."+
   415  				"\n\n%s\n\n%s",
   416  			spewConf.Sdump(actual), spewConf.Sdump(expected))
   417  	}
   418  
   419  	return nil
   420  }
   421  
   422  func testModule(
   423  	opts terraform.ContextOpts,
   424  	step TestStep) (*module.Tree, error) {
   425  	if step.PreConfig != nil {
   426  		step.PreConfig()
   427  	}
   428  
   429  	cfgPath, err := ioutil.TempDir("", "tf-test")
   430  	if err != nil {
   431  		return nil, fmt.Errorf(
   432  			"Error creating temporary directory for config: %s", err)
   433  	}
   434  	defer os.RemoveAll(cfgPath)
   435  
   436  	// Write the configuration
   437  	cfgF, err := os.Create(filepath.Join(cfgPath, "main.tf"))
   438  	if err != nil {
   439  		return nil, fmt.Errorf(
   440  			"Error creating temporary file for config: %s", err)
   441  	}
   442  
   443  	_, err = io.Copy(cfgF, strings.NewReader(step.Config))
   444  	cfgF.Close()
   445  	if err != nil {
   446  		return nil, fmt.Errorf(
   447  			"Error creating temporary file for config: %s", err)
   448  	}
   449  
   450  	// Parse the configuration
   451  	mod, err := module.NewTreeModule("", cfgPath)
   452  	if err != nil {
   453  		return nil, fmt.Errorf(
   454  			"Error loading configuration: %s", err)
   455  	}
   456  
   457  	// Load the modules
   458  	modStorage := &getter.FolderStorage{
   459  		StorageDir: filepath.Join(cfgPath, ".tfmodules"),
   460  	}
   461  	err = mod.Load(modStorage, module.GetModeGet)
   462  	if err != nil {
   463  		return nil, fmt.Errorf("Error downloading modules: %s", err)
   464  	}
   465  
   466  	return mod, nil
   467  }
   468  
   469  func testResource(c TestStep, state *terraform.State) (*terraform.ResourceState, error) {
   470  	if c.ResourceName == "" {
   471  		return nil, fmt.Errorf("ResourceName must be set in TestStep")
   472  	}
   473  
   474  	for _, m := range state.Modules {
   475  		if len(m.Resources) > 0 {
   476  			if v, ok := m.Resources[c.ResourceName]; ok {
   477  				return v, nil
   478  			}
   479  		}
   480  	}
   481  
   482  	return nil, fmt.Errorf(
   483  		"Resource specified by ResourceName couldn't be found: %s", c.ResourceName)
   484  }
   485  
   486  // ComposeTestCheckFunc lets you compose multiple TestCheckFuncs into
   487  // a single TestCheckFunc.
   488  //
   489  // As a user testing their provider, this lets you decompose your checks
   490  // into smaller pieces more easily.
   491  func ComposeTestCheckFunc(fs ...TestCheckFunc) TestCheckFunc {
   492  	return func(s *terraform.State) error {
   493  		for i, f := range fs {
   494  			if err := f(s); err != nil {
   495  				return fmt.Errorf("Check %d/%d error: %s", i+1, len(fs), err)
   496  			}
   497  		}
   498  
   499  		return nil
   500  	}
   501  }
   502  
   503  func TestCheckResourceAttr(name, key, value string) TestCheckFunc {
   504  	return func(s *terraform.State) error {
   505  		ms := s.RootModule()
   506  		rs, ok := ms.Resources[name]
   507  		if !ok {
   508  			return fmt.Errorf("Not found: %s", name)
   509  		}
   510  
   511  		is := rs.Primary
   512  		if is == nil {
   513  			return fmt.Errorf("No primary instance: %s", name)
   514  		}
   515  
   516  		if is.Attributes[key] != value {
   517  			return fmt.Errorf(
   518  				"%s: Attribute '%s' expected %#v, got %#v",
   519  				name,
   520  				key,
   521  				value,
   522  				is.Attributes[key])
   523  		}
   524  
   525  		return nil
   526  	}
   527  }
   528  
   529  func TestMatchResourceAttr(name, key string, r *regexp.Regexp) TestCheckFunc {
   530  	return func(s *terraform.State) error {
   531  		ms := s.RootModule()
   532  		rs, ok := ms.Resources[name]
   533  		if !ok {
   534  			return fmt.Errorf("Not found: %s", name)
   535  		}
   536  
   537  		is := rs.Primary
   538  		if is == nil {
   539  			return fmt.Errorf("No primary instance: %s", name)
   540  		}
   541  
   542  		if !r.MatchString(is.Attributes[key]) {
   543  			return fmt.Errorf(
   544  				"%s: Attribute '%s' didn't match %q, got %#v",
   545  				name,
   546  				key,
   547  				r.String(),
   548  				is.Attributes[key])
   549  		}
   550  
   551  		return nil
   552  	}
   553  }
   554  
   555  // TestCheckResourceAttrPtr is like TestCheckResourceAttr except the
   556  // value is a pointer so that it can be updated while the test is running.
   557  // It will only be dereferenced at the point this step is run.
   558  func TestCheckResourceAttrPtr(name string, key string, value *string) TestCheckFunc {
   559  	return func(s *terraform.State) error {
   560  		return TestCheckResourceAttr(name, key, *value)(s)
   561  	}
   562  }
   563  
   564  // TestCheckOutput checks an output in the Terraform configuration
   565  func TestCheckOutput(name, value string) TestCheckFunc {
   566  	return func(s *terraform.State) error {
   567  		ms := s.RootModule()
   568  		rs, ok := ms.Outputs[name]
   569  		if !ok {
   570  			return fmt.Errorf("Not found: %s", name)
   571  		}
   572  
   573  		if rs.Value != value {
   574  			return fmt.Errorf(
   575  				"Output '%s': expected %#v, got %#v",
   576  				name,
   577  				value,
   578  				rs)
   579  		}
   580  
   581  		return nil
   582  	}
   583  }
   584  
   585  // TestT is the interface used to handle the test lifecycle of a test.
   586  //
   587  // Users should just use a *testing.T object, which implements this.
   588  type TestT interface {
   589  	Error(args ...interface{})
   590  	Fatal(args ...interface{})
   591  	Skip(args ...interface{})
   592  }
   593  
   594  // This is set to true by unit tests to alter some behavior
   595  var testTesting = false