github.com/trawler/terraform@v0.10.8-0.20171106022149-4b1c7a1d9b48/config/module/tree.go (about)

     1  package module
     2  
     3  import (
     4  	"bufio"
     5  	"bytes"
     6  	"fmt"
     7  	"log"
     8  	"path/filepath"
     9  	"strings"
    10  	"sync"
    11  
    12  	getter "github.com/hashicorp/go-getter"
    13  	"github.com/hashicorp/terraform/config"
    14  )
    15  
    16  // RootName is the name of the root tree.
    17  const RootName = "root"
    18  
    19  // Tree represents the module import tree of configurations.
    20  //
    21  // This Tree structure can be used to get (download) new modules, load
    22  // all the modules without getting, flatten the tree into something
    23  // Terraform can use, etc.
    24  type Tree struct {
    25  	name     string
    26  	config   *config.Config
    27  	children map[string]*Tree
    28  	path     []string
    29  	lock     sync.RWMutex
    30  
    31  	// version is the final version of the config loaded for the Tree's module
    32  	version string
    33  	// source is the "source" string used to load this module. It's possible
    34  	// for a module source to change, but the path remains the same, preventing
    35  	// it from being reloaded.
    36  	source string
    37  	// parent allows us to walk back up the tree and determine if there are any
    38  	// versioned ancestor modules which may effect the stored location of
    39  	// submodules
    40  	parent *Tree
    41  }
    42  
    43  // NewTree returns a new Tree for the given config structure.
    44  func NewTree(name string, c *config.Config) *Tree {
    45  	return &Tree{config: c, name: name}
    46  }
    47  
    48  // NewEmptyTree returns a new tree that is empty (contains no configuration).
    49  func NewEmptyTree() *Tree {
    50  	t := &Tree{config: &config.Config{}}
    51  
    52  	// We do this dummy load so that the tree is marked as "loaded". It
    53  	// should never fail because this is just about a no-op. If it does fail
    54  	// we panic so we can know its a bug.
    55  	if err := t.Load(&Storage{Mode: GetModeGet}); err != nil {
    56  		panic(err)
    57  	}
    58  
    59  	return t
    60  }
    61  
    62  // NewTreeModule is like NewTree except it parses the configuration in
    63  // the directory and gives it a specific name. Use a blank name "" to specify
    64  // the root module.
    65  func NewTreeModule(name, dir string) (*Tree, error) {
    66  	c, err := config.LoadDir(dir)
    67  	if err != nil {
    68  		return nil, err
    69  	}
    70  
    71  	return NewTree(name, c), nil
    72  }
    73  
    74  // Config returns the configuration for this module.
    75  func (t *Tree) Config() *config.Config {
    76  	return t.config
    77  }
    78  
    79  // Child returns the child with the given path (by name).
    80  func (t *Tree) Child(path []string) *Tree {
    81  	if t == nil {
    82  		return nil
    83  	}
    84  
    85  	if len(path) == 0 {
    86  		return t
    87  	}
    88  
    89  	c := t.Children()[path[0]]
    90  	if c == nil {
    91  		return nil
    92  	}
    93  
    94  	return c.Child(path[1:])
    95  }
    96  
    97  // Children returns the children of this tree (the modules that are
    98  // imported by this root).
    99  //
   100  // This will only return a non-nil value after Load is called.
   101  func (t *Tree) Children() map[string]*Tree {
   102  	t.lock.RLock()
   103  	defer t.lock.RUnlock()
   104  	return t.children
   105  }
   106  
   107  // DeepEach calls the provided callback for the receiver and then all of
   108  // its descendents in the tree, allowing an operation to be performed on
   109  // all modules in the tree.
   110  //
   111  // Parents will be visited before their children but otherwise the order is
   112  // not defined.
   113  func (t *Tree) DeepEach(cb func(*Tree)) {
   114  	t.lock.RLock()
   115  	defer t.lock.RUnlock()
   116  	t.deepEach(cb)
   117  }
   118  
   119  func (t *Tree) deepEach(cb func(*Tree)) {
   120  	cb(t)
   121  	for _, c := range t.children {
   122  		c.deepEach(cb)
   123  	}
   124  }
   125  
   126  // Loaded says whether or not this tree has been loaded or not yet.
   127  func (t *Tree) Loaded() bool {
   128  	t.lock.RLock()
   129  	defer t.lock.RUnlock()
   130  	return t.children != nil
   131  }
   132  
   133  // Modules returns the list of modules that this tree imports.
   134  //
   135  // This is only the imports of _this_ level of the tree. To retrieve the
   136  // full nested imports, you'll have to traverse the tree.
   137  func (t *Tree) Modules() []*Module {
   138  	result := make([]*Module, len(t.config.Modules))
   139  	for i, m := range t.config.Modules {
   140  		result[i] = &Module{
   141  			Name:      m.Name,
   142  			Version:   m.Version,
   143  			Source:    m.Source,
   144  			Providers: m.Providers,
   145  		}
   146  	}
   147  
   148  	return result
   149  }
   150  
   151  // Name returns the name of the tree. This will be "<root>" for the root
   152  // tree and then the module name given for any children.
   153  func (t *Tree) Name() string {
   154  	if t.name == "" {
   155  		return RootName
   156  	}
   157  
   158  	return t.name
   159  }
   160  
   161  // Load loads the configuration of the entire tree.
   162  //
   163  // The parameters are used to tell the tree where to find modules and
   164  // whether it can download/update modules along the way.
   165  //
   166  // Calling this multiple times will reload the tree.
   167  //
   168  // Various semantic-like checks are made along the way of loading since
   169  // module trees inherently require the configuration to be in a reasonably
   170  // sane state: no circular dependencies, proper module sources, etc. A full
   171  // suite of validations can be done by running Validate (after loading).
   172  func (t *Tree) Load(s *Storage) error {
   173  	t.lock.Lock()
   174  	defer t.lock.Unlock()
   175  
   176  	children, err := t.getChildren(s)
   177  	if err != nil {
   178  		return err
   179  	}
   180  
   181  	// Go through all the children and load them.
   182  	for _, c := range children {
   183  		if err := c.Load(s); err != nil {
   184  			return err
   185  		}
   186  	}
   187  
   188  	// Set our tree up
   189  	t.children = children
   190  
   191  	// if we're the root module, we can now set the provider inheritance
   192  	if len(t.path) == 0 {
   193  		t.inheritProviderConfigs(nil)
   194  	}
   195  
   196  	return nil
   197  }
   198  
   199  func (t *Tree) getChildren(s *Storage) (map[string]*Tree, error) {
   200  	children := make(map[string]*Tree)
   201  
   202  	// Go through all the modules and get the directory for them.
   203  	for _, m := range t.Modules() {
   204  		if _, ok := children[m.Name]; ok {
   205  			return nil, fmt.Errorf(
   206  				"module %s: duplicated. module names must be unique", m.Name)
   207  		}
   208  
   209  		// Determine the path to this child
   210  		modPath := make([]string, len(t.path), len(t.path)+1)
   211  		copy(modPath, t.path)
   212  		modPath = append(modPath, m.Name)
   213  
   214  		log.Printf("[TRACE] module source: %q", m.Source)
   215  
   216  		// add the module path to help indicate where modules with relative
   217  		// paths are being loaded from
   218  		s.output(fmt.Sprintf("- module.%s", strings.Join(modPath, ".")))
   219  
   220  		// Lookup the local location of the module.
   221  		// dir is the local directory where the module is stored
   222  		mod, err := s.findRegistryModule(m.Source, m.Version)
   223  		if err != nil {
   224  			return nil, err
   225  		}
   226  
   227  		// The key is the string that will be used to uniquely id the Source in
   228  		// the local storage.  The prefix digit can be incremented to
   229  		// invalidate the local module storage.
   230  		key := "1." + t.versionedPathKey(m)
   231  		if mod.Version != "" {
   232  			key += "." + mod.Version
   233  		}
   234  
   235  		// Check for the exact key if it's not a registry module
   236  		if !mod.registry {
   237  			mod.Dir, err = s.findModule(key)
   238  			if err != nil {
   239  				return nil, err
   240  			}
   241  		}
   242  
   243  		if mod.Dir != "" && s.Mode != GetModeUpdate {
   244  			// We found it locally, but in order to load the Tree we need to
   245  			// find out if there was another subDir stored from detection.
   246  			subDir, err := s.getModuleRoot(mod.Dir)
   247  			if err != nil {
   248  				// If there's a problem with the subdir record, we'll let the
   249  				// recordSubdir method fix it up.  Any other filesystem errors
   250  				// will turn up again below.
   251  				log.Println("[WARN] error reading subdir record:", err)
   252  			}
   253  
   254  			fullDir := filepath.Join(mod.Dir, subDir)
   255  
   256  			child, err := NewTreeModule(m.Name, fullDir)
   257  			if err != nil {
   258  				return nil, fmt.Errorf("module %s: %s", m.Name, err)
   259  			}
   260  			child.path = modPath
   261  			child.parent = t
   262  			child.version = mod.Version
   263  			child.source = m.Source
   264  			children[m.Name] = child
   265  			continue
   266  		}
   267  
   268  		// Split out the subdir if we have one.
   269  		// Terraform keeps the entire requested tree, so that modules can
   270  		// reference sibling modules from the same archive or repo.
   271  		rawSource, subDir := getter.SourceDirSubdir(m.Source)
   272  
   273  		// we haven't found a source, so fallback to the go-getter detectors
   274  		source := mod.url
   275  		if source == "" {
   276  			source, err = getter.Detect(rawSource, t.config.Dir, getter.Detectors)
   277  			if err != nil {
   278  				return nil, fmt.Errorf("module %s: %s", m.Name, err)
   279  			}
   280  		}
   281  
   282  		log.Printf("[TRACE] detected module source %q", source)
   283  
   284  		// Check if the detector introduced something new.
   285  		// For example, the registry always adds a subdir of `//*`,
   286  		// indicating that we need to strip off the first component from the
   287  		// tar archive, though we may not yet know what it is called.
   288  		source, detectedSubDir := getter.SourceDirSubdir(source)
   289  		if detectedSubDir != "" {
   290  			subDir = filepath.Join(detectedSubDir, subDir)
   291  		}
   292  
   293  		output := ""
   294  		switch s.Mode {
   295  		case GetModeUpdate:
   296  			output = fmt.Sprintf("  Updating source %q", m.Source)
   297  		default:
   298  			output = fmt.Sprintf("  Getting source %q", m.Source)
   299  		}
   300  		s.output(output)
   301  
   302  		dir, ok, err := s.getStorage(key, source)
   303  		if err != nil {
   304  			return nil, err
   305  		}
   306  		if !ok {
   307  			return nil, fmt.Errorf("module %s: not found, may need to run 'terraform init'", m.Name)
   308  		}
   309  
   310  		log.Printf("[TRACE] %q stored in %q", source, dir)
   311  
   312  		// expand and record the subDir for later
   313  		fullDir := dir
   314  		if subDir != "" {
   315  			fullDir, err = getter.SubdirGlob(dir, subDir)
   316  			if err != nil {
   317  				return nil, err
   318  			}
   319  
   320  			// +1 to account for the pathsep
   321  			if len(dir)+1 > len(fullDir) {
   322  				return nil, fmt.Errorf("invalid module storage path %q", fullDir)
   323  			}
   324  			subDir = fullDir[len(dir)+1:]
   325  		}
   326  
   327  		// add new info to the module record
   328  		mod.Key = key
   329  		mod.Dir = dir
   330  		mod.Root = subDir
   331  
   332  		// record the module in our manifest
   333  		if err := s.recordModule(mod); err != nil {
   334  			return nil, err
   335  		}
   336  
   337  		child, err := NewTreeModule(m.Name, fullDir)
   338  		if err != nil {
   339  			return nil, fmt.Errorf("module %s: %s", m.Name, err)
   340  		}
   341  		child.path = modPath
   342  		child.parent = t
   343  		child.version = mod.Version
   344  		child.source = m.Source
   345  		children[m.Name] = child
   346  	}
   347  
   348  	return children, nil
   349  }
   350  
   351  // inheritProviderConfig resolves all provider config inheritance after the
   352  // tree is loaded.
   353  //
   354  // If there is a provider block without a config, look in the parent's Module
   355  // block for a provider, and fetch that provider's configuration. If that
   356  // doesn't exist, assume a default empty config. Implicit providers can still
   357  // inherit their config all the way up from the root, so walk up the tree and
   358  // copy the first matching provider into the module.
   359  func (t *Tree) inheritProviderConfigs(stack []*Tree) {
   360  	// the recursive calls only append, so we don't need to worry about copying
   361  	// this slice.
   362  	stack = append(stack, t)
   363  	for _, c := range t.children {
   364  		c.inheritProviderConfigs(stack)
   365  	}
   366  
   367  	if len(stack) == 1 {
   368  		return
   369  	}
   370  
   371  	providers := make(map[string]*config.ProviderConfig)
   372  	missingProviders := make(map[string]bool)
   373  
   374  	for _, p := range t.config.ProviderConfigs {
   375  		providers[p.FullName()] = p
   376  	}
   377  
   378  	for _, r := range t.config.Resources {
   379  		p := r.ProviderFullName()
   380  		if _, ok := providers[p]; !(ok || strings.Contains(p, ".")) {
   381  			missingProviders[p] = true
   382  		}
   383  	}
   384  
   385  	// get our parent's module config block
   386  	parent := stack[len(stack)-2]
   387  	var parentModule *config.Module
   388  	for _, m := range parent.config.Modules {
   389  		if m.Name == t.name {
   390  			parentModule = m
   391  			break
   392  		}
   393  	}
   394  
   395  	if parentModule == nil {
   396  		panic("can't be a module without a parent module config")
   397  	}
   398  
   399  	// now look for providers that need a config
   400  	for p, pc := range providers {
   401  		if len(pc.RawConfig.RawMap()) > 0 {
   402  			log.Printf("[TRACE] provider %q has a config, continuing", p)
   403  			continue
   404  		}
   405  
   406  		// this provider has no config yet, check for one being passed in
   407  		parentProviderName, ok := parentModule.Providers[p]
   408  		if !ok {
   409  			continue
   410  		}
   411  
   412  		var parentProvider *config.ProviderConfig
   413  		// there's a config for us in the parent module
   414  		for _, pp := range parent.config.ProviderConfigs {
   415  			if pp.FullName() == parentProviderName {
   416  				parentProvider = pp
   417  				break
   418  			}
   419  		}
   420  
   421  		if parentProvider == nil {
   422  			// no config found, assume defaults
   423  			continue
   424  		}
   425  
   426  		// Copy it in, but set an interpolation Scope.
   427  		// An interpolation Scope always need to have "root"
   428  		pc.Path = append([]string{RootName}, parent.path...)
   429  		pc.RawConfig = parentProvider.RawConfig
   430  		log.Printf("[TRACE] provider %q inheriting config from %q",
   431  			strings.Join(append(t.Path(), pc.FullName()), "."),
   432  			strings.Join(append(parent.Path(), parentProvider.FullName()), "."),
   433  		)
   434  	}
   435  
   436  }
   437  
   438  // Path is the full path to this tree.
   439  func (t *Tree) Path() []string {
   440  	return t.path
   441  }
   442  
   443  // String gives a nice output to describe the tree.
   444  func (t *Tree) String() string {
   445  	var result bytes.Buffer
   446  	path := strings.Join(t.path, ", ")
   447  	if path != "" {
   448  		path = fmt.Sprintf(" (path: %s)", path)
   449  	}
   450  	result.WriteString(t.Name() + path + "\n")
   451  
   452  	cs := t.Children()
   453  	if cs == nil {
   454  		result.WriteString("  not loaded")
   455  	} else {
   456  		// Go through each child and get its string value, then indent it
   457  		// by two.
   458  		for _, c := range cs {
   459  			r := strings.NewReader(c.String())
   460  			scanner := bufio.NewScanner(r)
   461  			for scanner.Scan() {
   462  				result.WriteString("  ")
   463  				result.WriteString(scanner.Text())
   464  				result.WriteString("\n")
   465  			}
   466  		}
   467  	}
   468  
   469  	return result.String()
   470  }
   471  
   472  // Validate does semantic checks on the entire tree of configurations.
   473  //
   474  // This will call the respective config.Config.Validate() functions as well
   475  // as verifying things such as parameters/outputs between the various modules.
   476  //
   477  // Load must be called prior to calling Validate or an error will be returned.
   478  func (t *Tree) Validate() error {
   479  	if !t.Loaded() {
   480  		return fmt.Errorf("tree must be loaded before calling Validate")
   481  	}
   482  
   483  	// If something goes wrong, here is our error template
   484  	newErr := &treeError{Name: []string{t.Name()}}
   485  
   486  	// Terraform core does not handle root module children named "root".
   487  	// We plan to fix this in the future but this bug was brought up in
   488  	// the middle of a release and we don't want to introduce wide-sweeping
   489  	// changes at that time.
   490  	if len(t.path) == 1 && t.name == "root" {
   491  		return fmt.Errorf("root module cannot contain module named 'root'")
   492  	}
   493  
   494  	// Validate our configuration first.
   495  	if err := t.config.Validate(); err != nil {
   496  		newErr.Add(err)
   497  	}
   498  
   499  	// If we're the root, we do extra validation. This validation usually
   500  	// requires the entire tree (since children don't have parent pointers).
   501  	if len(t.path) == 0 {
   502  		if err := t.validateProviderAlias(); err != nil {
   503  			newErr.Add(err)
   504  		}
   505  	}
   506  
   507  	// Get the child trees
   508  	children := t.Children()
   509  
   510  	// Validate all our children
   511  	for _, c := range children {
   512  		err := c.Validate()
   513  		if err == nil {
   514  			continue
   515  		}
   516  
   517  		verr, ok := err.(*treeError)
   518  		if !ok {
   519  			// Unknown error, just return...
   520  			return err
   521  		}
   522  
   523  		// Append ourselves to the error and then return
   524  		verr.Name = append(verr.Name, t.Name())
   525  		newErr.AddChild(verr)
   526  	}
   527  
   528  	// Go over all the modules and verify that any parameters are valid
   529  	// variables into the module in question.
   530  	for _, m := range t.config.Modules {
   531  		tree, ok := children[m.Name]
   532  		if !ok {
   533  			// This should never happen because Load watches us
   534  			panic("module not found in children: " + m.Name)
   535  		}
   536  
   537  		// Build the variables that the module defines
   538  		requiredMap := make(map[string]struct{})
   539  		varMap := make(map[string]struct{})
   540  		for _, v := range tree.config.Variables {
   541  			varMap[v.Name] = struct{}{}
   542  
   543  			if v.Required() {
   544  				requiredMap[v.Name] = struct{}{}
   545  			}
   546  		}
   547  
   548  		// Compare to the keys in our raw config for the module
   549  		for k, _ := range m.RawConfig.Raw {
   550  			if _, ok := varMap[k]; !ok {
   551  				newErr.Add(fmt.Errorf(
   552  					"module %s: %s is not a valid parameter",
   553  					m.Name, k))
   554  			}
   555  
   556  			// Remove the required
   557  			delete(requiredMap, k)
   558  		}
   559  
   560  		// If we have any required left over, they aren't set.
   561  		for k, _ := range requiredMap {
   562  			newErr.Add(fmt.Errorf(
   563  				"module %s: required variable %q not set",
   564  				m.Name, k))
   565  		}
   566  	}
   567  
   568  	// Go over all the variables used and make sure that any module
   569  	// variables represent outputs properly.
   570  	for source, vs := range t.config.InterpolatedVariables() {
   571  		for _, v := range vs {
   572  			mv, ok := v.(*config.ModuleVariable)
   573  			if !ok {
   574  				continue
   575  			}
   576  
   577  			tree, ok := children[mv.Name]
   578  			if !ok {
   579  				newErr.Add(fmt.Errorf(
   580  					"%s: undefined module referenced %s",
   581  					source, mv.Name))
   582  				continue
   583  			}
   584  
   585  			found := false
   586  			for _, o := range tree.config.Outputs {
   587  				if o.Name == mv.Field {
   588  					found = true
   589  					break
   590  				}
   591  			}
   592  			if !found {
   593  				newErr.Add(fmt.Errorf(
   594  					"%s: %s is not a valid output for module %s",
   595  					source, mv.Field, mv.Name))
   596  			}
   597  		}
   598  	}
   599  
   600  	return newErr.ErrOrNil()
   601  }
   602  
   603  // versionedPathKey returns a path string with every levels full name, version
   604  // and source encoded. This is to provide a unique key for our module storage,
   605  // since submodules need to know which versions of their ancestor modules they
   606  // are loaded from.
   607  // For example, if module A has a subdirectory B, if module A's source or
   608  // version is updated B's storage key must reflect this change in order for the
   609  // correct version of B's source to be loaded.
   610  func (t *Tree) versionedPathKey(m *Module) string {
   611  	path := make([]string, len(t.path)+1)
   612  	path[len(path)-1] = m.Name + ";" + m.Source
   613  	// We're going to load these in order for easier reading and debugging, but
   614  	// in practice they only need to be unique and consistent.
   615  
   616  	p := t
   617  	i := len(path) - 2
   618  	for ; i >= 0; i-- {
   619  		if p == nil {
   620  			break
   621  		}
   622  		// we may have been loaded under a blank Tree, so always check for a name
   623  		// too.
   624  		if p.name == "" {
   625  			break
   626  		}
   627  		seg := p.name
   628  		if p.version != "" {
   629  			seg += "#" + p.version
   630  		}
   631  
   632  		if p.source != "" {
   633  			seg += ";" + p.source
   634  		}
   635  
   636  		path[i] = seg
   637  		p = p.parent
   638  	}
   639  
   640  	key := strings.Join(path, "|")
   641  	return key
   642  }
   643  
   644  // treeError is an error use by Tree.Validate to accumulates all
   645  // validation errors.
   646  type treeError struct {
   647  	Name     []string
   648  	Errs     []error
   649  	Children []*treeError
   650  }
   651  
   652  func (e *treeError) Add(err error) {
   653  	e.Errs = append(e.Errs, err)
   654  }
   655  
   656  func (e *treeError) AddChild(err *treeError) {
   657  	e.Children = append(e.Children, err)
   658  }
   659  
   660  func (e *treeError) ErrOrNil() error {
   661  	if len(e.Errs) > 0 || len(e.Children) > 0 {
   662  		return e
   663  	}
   664  	return nil
   665  }
   666  
   667  func (e *treeError) Error() string {
   668  	name := strings.Join(e.Name, ".")
   669  	var out bytes.Buffer
   670  	fmt.Fprintf(&out, "module %s: ", name)
   671  
   672  	if len(e.Errs) == 1 {
   673  		// single like error
   674  		out.WriteString(e.Errs[0].Error())
   675  	} else {
   676  		// multi-line error
   677  		for _, err := range e.Errs {
   678  			fmt.Fprintf(&out, "\n    %s", err)
   679  		}
   680  	}
   681  
   682  	if len(e.Children) > 0 {
   683  		// start the next error on a new line
   684  		out.WriteString("\n  ")
   685  	}
   686  	for _, child := range e.Children {
   687  		out.WriteString(child.Error())
   688  	}
   689  
   690  	return out.String()
   691  }