github.com/sfdevops1/terrra4orm@v0.11.12-beta1/configs/configload/loader_install.go (about)

     1  package configload
     2  
     3  import (
     4  	"fmt"
     5  	"log"
     6  	"os"
     7  	"path/filepath"
     8  	"strings"
     9  
    10  	version "github.com/hashicorp/go-version"
    11  	"github.com/hashicorp/hcl2/hcl"
    12  	"github.com/hashicorp/terraform/configs"
    13  	"github.com/hashicorp/terraform/registry"
    14  	"github.com/hashicorp/terraform/registry/regsrc"
    15  )
    16  
    17  // InstallModules analyses the root module in the given directory and installs
    18  // all of its direct and transitive dependencies into the loader's modules
    19  // directory, which must already exist.
    20  //
    21  // Since InstallModules makes possibly-time-consuming calls to remote services,
    22  // a hook interface is supported to allow the caller to be notified when
    23  // each module is installed and, for remote modules, when downloading begins.
    24  // LoadConfig guarantees that two hook calls will not happen concurrently but
    25  // it does not guarantee any particular ordering of hook calls. This mechanism
    26  // is for UI feedback only and does not give the caller any control over the
    27  // process.
    28  //
    29  // If modules are already installed in the target directory, they will be
    30  // skipped unless their source address or version have changed or unless
    31  // the upgrade flag is set.
    32  //
    33  // InstallModules never deletes any directory, except in the case where it
    34  // needs to replace a directory that is already present with a newly-extracted
    35  // package.
    36  //
    37  // If the returned diagnostics contains errors then the module installation
    38  // may have wholly or partially completed. Modules must be loaded in order
    39  // to find their dependencies, so this function does many of the same checks
    40  // as LoadConfig as a side-effect.
    41  //
    42  // This function will panic if called on a loader that cannot install modules.
    43  // Use CanInstallModules to determine if a loader can install modules, or
    44  // refer to the documentation for that method for situations where module
    45  // installation capability is guaranteed.
    46  func (l *Loader) InstallModules(rootDir string, upgrade bool, hooks InstallHooks) hcl.Diagnostics {
    47  	if !l.CanInstallModules() {
    48  		panic(fmt.Errorf("InstallModules called on loader that cannot install modules"))
    49  	}
    50  
    51  	rootMod, diags := l.parser.LoadConfigDir(rootDir)
    52  	if rootMod == nil {
    53  		return diags
    54  	}
    55  
    56  	instDiags := l.installDescendentModules(rootMod, rootDir, upgrade, hooks)
    57  	diags = append(diags, instDiags...)
    58  
    59  	return diags
    60  }
    61  
    62  func (l *Loader) installDescendentModules(rootMod *configs.Module, rootDir string, upgrade bool, hooks InstallHooks) hcl.Diagnostics {
    63  	var diags hcl.Diagnostics
    64  
    65  	if hooks == nil {
    66  		// Use our no-op implementation as a placeholder
    67  		hooks = InstallHooksImpl{}
    68  	}
    69  
    70  	// Create a manifest record for the root module. This will be used if
    71  	// there are any relative-pathed modules in the root.
    72  	l.modules.manifest[""] = moduleRecord{
    73  		Key: "",
    74  		Dir: rootDir,
    75  	}
    76  
    77  	_, cDiags := configs.BuildConfig(rootMod, configs.ModuleWalkerFunc(
    78  		func(req *configs.ModuleRequest) (*configs.Module, *version.Version, hcl.Diagnostics) {
    79  
    80  			key := manifestKey(req.Path)
    81  			instPath := l.packageInstallPath(req.Path)
    82  
    83  			log.Printf("[DEBUG] Module installer: begin %s", key)
    84  
    85  			// First we'll check if we need to upgrade/replace an existing
    86  			// installed module, and delete it out of the way if so.
    87  			replace := upgrade
    88  			if !replace {
    89  				record, recorded := l.modules.manifest[key]
    90  				switch {
    91  				case !recorded:
    92  					log.Printf("[TRACE] %s is not yet installed", key)
    93  					replace = true
    94  				case record.SourceAddr != req.SourceAddr:
    95  					log.Printf("[TRACE] %s source address has changed from %q to %q", key, record.SourceAddr, req.SourceAddr)
    96  					replace = true
    97  				case record.Version != nil && !req.VersionConstraint.Required.Check(record.Version):
    98  					log.Printf("[TRACE] %s version %s no longer compatible with constraints %s", key, record.Version, req.VersionConstraint.Required)
    99  					replace = true
   100  				}
   101  			}
   102  
   103  			// If we _are_ planning to replace this module, then we'll remove
   104  			// it now so our installation code below won't conflict with any
   105  			// existing remnants.
   106  			if replace {
   107  				if _, recorded := l.modules.manifest[key]; recorded {
   108  					log.Printf("[TRACE] discarding previous record of %s prior to reinstall", key)
   109  				}
   110  				delete(l.modules.manifest, key)
   111  				// Deleting a module invalidates all of its descendent modules too.
   112  				keyPrefix := key + "."
   113  				for subKey := range l.modules.manifest {
   114  					if strings.HasPrefix(subKey, keyPrefix) {
   115  						if _, recorded := l.modules.manifest[subKey]; recorded {
   116  							log.Printf("[TRACE] also discarding downstream %s", subKey)
   117  						}
   118  						delete(l.modules.manifest, subKey)
   119  					}
   120  				}
   121  			}
   122  
   123  			record, recorded := l.modules.manifest[key]
   124  			if !recorded {
   125  				// Clean up any stale cache directory that might be present.
   126  				// If this is a local (relative) source then the dir will
   127  				// not exist, but we'll ignore that.
   128  				log.Printf("[TRACE] cleaning directory %s prior to install of %s", instPath, key)
   129  				err := l.modules.FS.RemoveAll(instPath)
   130  				if err != nil && !os.IsNotExist(err) {
   131  					log.Printf("[TRACE] failed to remove %s: %s", key, err)
   132  					diags = append(diags, &hcl.Diagnostic{
   133  						Severity: hcl.DiagError,
   134  						Summary:  "Failed to remove local module cache",
   135  						Detail: fmt.Sprintf(
   136  							"Terraform tried to remove %s in order to reinstall this module, but encountered an error: %s",
   137  							instPath, err,
   138  						),
   139  						Subject: &req.CallRange,
   140  					})
   141  					return nil, nil, diags
   142  				}
   143  			} else {
   144  				// If this module is already recorded and its root directory
   145  				// exists then we will just load what's already there and
   146  				// keep our existing record.
   147  				info, err := l.modules.FS.Stat(record.Dir)
   148  				if err == nil && info.IsDir() {
   149  					mod, mDiags := l.parser.LoadConfigDir(record.Dir)
   150  					diags = append(diags, mDiags...)
   151  
   152  					log.Printf("[TRACE] Module installer: %s %s already installed in %s", key, record.Version, record.Dir)
   153  					return mod, record.Version, diags
   154  				}
   155  			}
   156  
   157  			// If we get down here then it's finally time to actually install
   158  			// the module. There are some variants to this process depending
   159  			// on what type of module source address we have.
   160  			switch {
   161  
   162  			case isLocalSourceAddr(req.SourceAddr):
   163  				log.Printf("[TRACE] %s has local path %q", key, req.SourceAddr)
   164  				mod, mDiags := l.installLocalModule(req, key, hooks)
   165  				diags = append(diags, mDiags...)
   166  				return mod, nil, diags
   167  
   168  			case isRegistrySourceAddr(req.SourceAddr):
   169  				addr, err := regsrc.ParseModuleSource(req.SourceAddr)
   170  				if err != nil {
   171  					// Should never happen because isRegistrySourceAddr already validated
   172  					panic(err)
   173  				}
   174  				log.Printf("[TRACE] %s is a registry module at %s", key, addr)
   175  
   176  				mod, v, mDiags := l.installRegistryModule(req, key, instPath, addr, hooks)
   177  				diags = append(diags, mDiags...)
   178  				return mod, v, diags
   179  
   180  			default:
   181  				log.Printf("[TRACE] %s address %q will be handled by go-getter", key, req.SourceAddr)
   182  
   183  				mod, mDiags := l.installGoGetterModule(req, key, instPath, hooks)
   184  				diags = append(diags, mDiags...)
   185  				return mod, nil, diags
   186  			}
   187  
   188  		},
   189  	))
   190  	diags = append(diags, cDiags...)
   191  
   192  	err := l.modules.writeModuleManifestSnapshot()
   193  	if err != nil {
   194  		diags = append(diags, &hcl.Diagnostic{
   195  			Severity: hcl.DiagError,
   196  			Summary:  "Failed to update module manifest",
   197  			Detail:   fmt.Sprintf("Unable to write the module manifest file: %s", err),
   198  		})
   199  	}
   200  
   201  	return diags
   202  }
   203  
   204  // CanInstallModules returns true if InstallModules can be used with this
   205  // loader.
   206  //
   207  // Loaders created with NewLoader can always install modules. Loaders created
   208  // from plan files (where the configuration is embedded in the plan file itself)
   209  // cannot install modules, because the plan file is read-only.
   210  func (l *Loader) CanInstallModules() bool {
   211  	return l.modules.CanInstall
   212  }
   213  
   214  func (l *Loader) installLocalModule(req *configs.ModuleRequest, key string, hooks InstallHooks) (*configs.Module, hcl.Diagnostics) {
   215  	var diags hcl.Diagnostics
   216  
   217  	parentKey := manifestKey(req.Parent.Path)
   218  	parentRecord, recorded := l.modules.manifest[parentKey]
   219  	if !recorded {
   220  		// This is indicative of a bug rather than a user-actionable error
   221  		panic(fmt.Errorf("missing manifest record for parent module %s", parentKey))
   222  	}
   223  
   224  	if len(req.VersionConstraint.Required) != 0 {
   225  		diags = append(diags, &hcl.Diagnostic{
   226  			Severity: hcl.DiagError,
   227  			Summary:  "Invalid version constraint",
   228  			Detail:   "A version constraint cannot be applied to a module at a relative local path.",
   229  			Subject:  &req.VersionConstraint.DeclRange,
   230  		})
   231  	}
   232  
   233  	// For local sources we don't actually need to modify the
   234  	// filesystem at all because the parent already wrote
   235  	// the files we need, and so we just load up what's already here.
   236  	newDir := filepath.Join(parentRecord.Dir, req.SourceAddr)
   237  	log.Printf("[TRACE] %s uses directory from parent: %s", key, newDir)
   238  	mod, mDiags := l.parser.LoadConfigDir(newDir)
   239  	if mod == nil {
   240  		// nil indicates missing or unreadable directory, so we'll
   241  		// discard the returned diags and return a more specific
   242  		// error message here.
   243  		diags = append(diags, &hcl.Diagnostic{
   244  			Severity: hcl.DiagError,
   245  			Summary:  "Unreadable module directory",
   246  			Detail:   fmt.Sprintf("The directory %s could not be read.", newDir),
   247  			Subject:  &req.SourceAddrRange,
   248  		})
   249  	} else {
   250  		diags = append(diags, mDiags...)
   251  	}
   252  
   253  	// Note the local location in our manifest.
   254  	l.modules.manifest[key] = moduleRecord{
   255  		Key:        key,
   256  		Dir:        newDir,
   257  		SourceAddr: req.SourceAddr,
   258  	}
   259  	log.Printf("[DEBUG] Module installer: %s installed at %s", key, newDir)
   260  	hooks.Install(key, nil, newDir)
   261  
   262  	return mod, diags
   263  }
   264  
   265  func (l *Loader) installRegistryModule(req *configs.ModuleRequest, key string, instPath string, addr *regsrc.Module, hooks InstallHooks) (*configs.Module, *version.Version, hcl.Diagnostics) {
   266  	var diags hcl.Diagnostics
   267  
   268  	hostname, err := addr.SvcHost()
   269  	if err != nil {
   270  		// If it looks like the user was trying to use punycode then we'll generate
   271  		// a specialized error for that case. We require the unicode form of
   272  		// hostname so that hostnames are always human-readable in configuration
   273  		// and punycode can't be used to hide a malicious module hostname.
   274  		if strings.HasPrefix(addr.RawHost.Raw, "xn--") {
   275  			diags = append(diags, &hcl.Diagnostic{
   276  				Severity: hcl.DiagError,
   277  				Summary:  "Invalid module registry hostname",
   278  				Detail:   "The hostname portion of this source address is not an acceptable hostname. Internationalized domain names must be given in unicode form rather than ASCII (\"punycode\") form.",
   279  				Subject:  &req.SourceAddrRange,
   280  			})
   281  		} else {
   282  			diags = append(diags, &hcl.Diagnostic{
   283  				Severity: hcl.DiagError,
   284  				Summary:  "Invalid module registry hostname",
   285  				Detail:   "The hostname portion of this source address is not a valid hostname.",
   286  				Subject:  &req.SourceAddrRange,
   287  			})
   288  		}
   289  		return nil, nil, diags
   290  	}
   291  
   292  	reg := l.modules.Registry
   293  
   294  	log.Printf("[DEBUG] %s listing available versions of %s at %s", key, addr, hostname)
   295  	resp, err := reg.Versions(addr)
   296  	if err != nil {
   297  		if registry.IsModuleNotFound(err) {
   298  			diags = append(diags, &hcl.Diagnostic{
   299  				Severity: hcl.DiagError,
   300  				Summary:  "Module not found",
   301  				Detail:   fmt.Sprintf("The specified module could not be found in the module registry at %s.", hostname),
   302  				Subject:  &req.SourceAddrRange,
   303  			})
   304  		} else {
   305  			diags = append(diags, &hcl.Diagnostic{
   306  				Severity: hcl.DiagError,
   307  				Summary:  "Error accessing remote module registry",
   308  				Detail:   fmt.Sprintf("Failed to retrieve available versions for this module from %s: %s.", hostname, err),
   309  				Subject:  &req.SourceAddrRange,
   310  			})
   311  		}
   312  		return nil, nil, diags
   313  	}
   314  
   315  	// The response might contain information about dependencies to allow us
   316  	// to potentially optimize future requests, but we don't currently do that
   317  	// and so for now we'll just take the first item which is guaranteed to
   318  	// be the address we requested.
   319  	if len(resp.Modules) < 1 {
   320  		// Should never happen, but since this is a remote service that may
   321  		// be implemented by third-parties we will handle it gracefully.
   322  		diags = append(diags, &hcl.Diagnostic{
   323  			Severity: hcl.DiagError,
   324  			Summary:  "Invalid response from remote module registry",
   325  			Detail:   fmt.Sprintf("The registry at %s returned an invalid response when Terraform requested available versions for this module.", hostname),
   326  			Subject:  &req.SourceAddrRange,
   327  		})
   328  		return nil, nil, diags
   329  	}
   330  
   331  	modMeta := resp.Modules[0]
   332  
   333  	var latestMatch *version.Version
   334  	var latestVersion *version.Version
   335  	for _, mv := range modMeta.Versions {
   336  		v, err := version.NewVersion(mv.Version)
   337  		if err != nil {
   338  			// Should never happen if the registry server is compliant with
   339  			// the protocol, but we'll warn if not to assist someone who
   340  			// might be developing a module registry server.
   341  			diags = append(diags, &hcl.Diagnostic{
   342  				Severity: hcl.DiagWarning,
   343  				Summary:  "Invalid response from remote module registry",
   344  				Detail:   fmt.Sprintf("The registry at %s returned an invalid version string %q for this module, which Terraform ignored.", hostname, mv.Version),
   345  				Subject:  &req.SourceAddrRange,
   346  			})
   347  			continue
   348  		}
   349  
   350  		// If we've found a pre-release version then we'll ignore it unless
   351  		// it was exactly requested.
   352  		if v.Prerelease() != "" && req.VersionConstraint.Required.String() != v.String() {
   353  			log.Printf("[TRACE] %s ignoring %s because it is a pre-release and was not requested exactly", key, v)
   354  			continue
   355  		}
   356  
   357  		if latestVersion == nil || v.GreaterThan(latestVersion) {
   358  			latestVersion = v
   359  		}
   360  
   361  		if req.VersionConstraint.Required.Check(v) {
   362  			if latestMatch == nil || v.GreaterThan(latestMatch) {
   363  				latestMatch = v
   364  			}
   365  		}
   366  	}
   367  
   368  	if latestVersion == nil {
   369  		diags = append(diags, &hcl.Diagnostic{
   370  			Severity: hcl.DiagError,
   371  			Summary:  "Module has no versions",
   372  			Detail:   fmt.Sprintf("The specified module does not have any available versions."),
   373  			Subject:  &req.SourceAddrRange,
   374  		})
   375  		return nil, nil, diags
   376  	}
   377  
   378  	if latestMatch == nil {
   379  		diags = append(diags, &hcl.Diagnostic{
   380  			Severity: hcl.DiagError,
   381  			Summary:  "Unresolvable module version constraint",
   382  			Detail:   fmt.Sprintf("There is no available version of %q that matches the given version constraint. The newest available version is %s.", addr, latestVersion),
   383  			Subject:  &req.VersionConstraint.DeclRange,
   384  		})
   385  		return nil, nil, diags
   386  	}
   387  
   388  	// Report up to the caller that we're about to start downloading.
   389  	packageAddr, _ := splitAddrSubdir(req.SourceAddr)
   390  	hooks.Download(key, packageAddr, latestMatch)
   391  
   392  	// If we manage to get down here then we've found a suitable version to
   393  	// install, so we need to ask the registry where we should download it from.
   394  	// The response to this is a go-getter-style address string.
   395  	dlAddr, err := reg.Location(addr, latestMatch.String())
   396  	if err != nil {
   397  		log.Printf("[ERROR] %s from %s %s: %s", key, addr, latestMatch, err)
   398  		diags = append(diags, &hcl.Diagnostic{
   399  			Severity: hcl.DiagError,
   400  			Summary:  "Invalid response from remote module registry",
   401  			Detail:   fmt.Sprintf("The remote registry at %s failed to return a download URL for %s %s.", hostname, addr, latestMatch),
   402  			Subject:  &req.VersionConstraint.DeclRange,
   403  		})
   404  		return nil, nil, diags
   405  	}
   406  
   407  	log.Printf("[TRACE] %s %s %s is available at %q", key, addr, latestMatch, dlAddr)
   408  
   409  	modDir, err := getWithGoGetter(instPath, dlAddr)
   410  	if err != nil {
   411  		// Errors returned by go-getter have very inconsistent quality as
   412  		// end-user error messages, but for now we're accepting that because
   413  		// we have no way to recognize any specific errors to improve them
   414  		// and masking the error entirely would hide valuable diagnostic
   415  		// information from the user.
   416  		diags = append(diags, &hcl.Diagnostic{
   417  			Severity: hcl.DiagError,
   418  			Summary:  "Failed to download module",
   419  			Detail:   fmt.Sprintf("Error attempting to download module source code from %q: %s", dlAddr, err),
   420  			Subject:  &req.CallRange,
   421  		})
   422  		return nil, nil, diags
   423  	}
   424  
   425  	log.Printf("[TRACE] %s %q was downloaded to %s", key, dlAddr, modDir)
   426  
   427  	if addr.RawSubmodule != "" {
   428  		// Append the user's requested subdirectory to any subdirectory that
   429  		// was implied by any of the nested layers we expanded within go-getter.
   430  		modDir = filepath.Join(modDir, addr.RawSubmodule)
   431  	}
   432  
   433  	log.Printf("[TRACE] %s should now be at %s", key, modDir)
   434  
   435  	// Finally we are ready to try actually loading the module.
   436  	mod, mDiags := l.parser.LoadConfigDir(modDir)
   437  	if mod == nil {
   438  		// nil indicates missing or unreadable directory, so we'll
   439  		// discard the returned diags and return a more specific
   440  		// error message here. For registry modules this actually
   441  		// indicates a bug in the code above, since it's not the
   442  		// user's responsibility to create the directory in this case.
   443  		diags = append(diags, &hcl.Diagnostic{
   444  			Severity: hcl.DiagError,
   445  			Summary:  "Unreadable module directory",
   446  			Detail:   fmt.Sprintf("The directory %s could not be read. This is a bug in Terraform and should be reported.", modDir),
   447  			Subject:  &req.CallRange,
   448  		})
   449  	} else {
   450  		diags = append(diags, mDiags...)
   451  	}
   452  
   453  	// Note the local location in our manifest.
   454  	l.modules.manifest[key] = moduleRecord{
   455  		Key:        key,
   456  		Version:    latestMatch,
   457  		Dir:        modDir,
   458  		SourceAddr: req.SourceAddr,
   459  	}
   460  	log.Printf("[DEBUG] Module installer: %s installed at %s", key, modDir)
   461  	hooks.Install(key, latestMatch, modDir)
   462  
   463  	return mod, latestMatch, diags
   464  }
   465  
   466  func (l *Loader) installGoGetterModule(req *configs.ModuleRequest, key string, instPath string, hooks InstallHooks) (*configs.Module, hcl.Diagnostics) {
   467  	var diags hcl.Diagnostics
   468  
   469  	// Report up to the caller that we're about to start downloading.
   470  	packageAddr, _ := splitAddrSubdir(req.SourceAddr)
   471  	hooks.Download(key, packageAddr, nil)
   472  
   473  	modDir, err := getWithGoGetter(instPath, req.SourceAddr)
   474  	if err != nil {
   475  		// Errors returned by go-getter have very inconsistent quality as
   476  		// end-user error messages, but for now we're accepting that because
   477  		// we have no way to recognize any specific errors to improve them
   478  		// and masking the error entirely would hide valuable diagnostic
   479  		// information from the user.
   480  		diags = append(diags, &hcl.Diagnostic{
   481  			Severity: hcl.DiagError,
   482  			Summary:  "Failed to download module",
   483  			Detail:   fmt.Sprintf("Error attempting to download module source code from %q: %s", packageAddr, err),
   484  			Subject:  &req.SourceAddrRange,
   485  		})
   486  		return nil, diags
   487  	}
   488  
   489  	log.Printf("[TRACE] %s %q was downloaded to %s", key, req.SourceAddr, modDir)
   490  
   491  	mod, mDiags := l.parser.LoadConfigDir(modDir)
   492  	if mod == nil {
   493  		// nil indicates missing or unreadable directory, so we'll
   494  		// discard the returned diags and return a more specific
   495  		// error message here. For registry modules this actually
   496  		// indicates a bug in the code above, since it's not the
   497  		// user's responsibility to create the directory in this case.
   498  		diags = append(diags, &hcl.Diagnostic{
   499  			Severity: hcl.DiagError,
   500  			Summary:  "Unreadable module directory",
   501  			Detail:   fmt.Sprintf("The directory %s could not be read. This is a bug in Terraform and should be reported.", modDir),
   502  			Subject:  &req.CallRange,
   503  		})
   504  	} else {
   505  		diags = append(diags, mDiags...)
   506  	}
   507  
   508  	// Note the local location in our manifest.
   509  	l.modules.manifest[key] = moduleRecord{
   510  		Key:        key,
   511  		Dir:        modDir,
   512  		SourceAddr: req.SourceAddr,
   513  	}
   514  	log.Printf("[DEBUG] Module installer: %s installed at %s", key, modDir)
   515  	hooks.Install(key, nil, modDir)
   516  
   517  	return mod, diags
   518  }
   519  
   520  func (l *Loader) packageInstallPath(modulePath []string) string {
   521  	return filepath.Join(l.modules.Dir, strings.Join(modulePath, "."))
   522  }