github.com/pgray/terraform@v0.5.4-0.20170822184730-b6a464c5214d/plugin/discovery/get.go (about)

     1  package discovery
     2  
     3  import (
     4  	"errors"
     5  	"fmt"
     6  	"io/ioutil"
     7  	"log"
     8  	"net/http"
     9  	"os"
    10  	"runtime"
    11  	"strconv"
    12  	"strings"
    13  
    14  	"golang.org/x/net/html"
    15  
    16  	cleanhttp "github.com/hashicorp/go-cleanhttp"
    17  	getter "github.com/hashicorp/go-getter"
    18  	multierror "github.com/hashicorp/go-multierror"
    19  	"github.com/mitchellh/cli"
    20  )
    21  
    22  // Releases are located by parsing the html listing from releases.hashicorp.com.
    23  //
    24  // The URL for releases follows the pattern:
    25  //    https://releases.hashicorp.com/terraform-provider-name/<x.y.z>/terraform-provider-name_<x.y.z>_<os>_<arch>.<ext>
    26  //
    27  // The plugin protocol version will be saved with the release and returned in
    28  // the header X-TERRAFORM_PROTOCOL_VERSION.
    29  
    30  const protocolVersionHeader = "x-terraform-protocol-version"
    31  
    32  var releaseHost = "https://releases.hashicorp.com"
    33  
    34  var httpClient = cleanhttp.DefaultClient()
    35  
    36  // An Installer maintains a local cache of plugins by downloading plugins
    37  // from an online repository.
    38  type Installer interface {
    39  	Get(name string, req Constraints) (PluginMeta, error)
    40  	PurgeUnused(used map[string]PluginMeta) (removed PluginMetaSet, err error)
    41  }
    42  
    43  // ProviderInstaller is an Installer implementation that knows how to
    44  // download Terraform providers from the official HashiCorp releases service
    45  // into a local directory. The files downloaded are compliant with the
    46  // naming scheme expected by FindPlugins, so the target directory of a
    47  // provider installer can be used as one of several plugin discovery sources.
    48  type ProviderInstaller struct {
    49  	Dir string
    50  
    51  	PluginProtocolVersion uint
    52  
    53  	// OS and Arch specify the OS and architecture that should be used when
    54  	// installing plugins. These use the same labels as the runtime.GOOS and
    55  	// runtime.GOARCH variables respectively, and indeed the values of these
    56  	// are used as defaults if either of these is the empty string.
    57  	OS   string
    58  	Arch string
    59  
    60  	// Skip checksum and signature verification
    61  	SkipVerify bool
    62  
    63  	Ui cli.Ui // Ui for output
    64  }
    65  
    66  // Get is part of an implementation of type Installer, and attempts to download
    67  // and install a Terraform provider matching the given constraints.
    68  //
    69  // This method may return one of a number of sentinel errors from this
    70  // package to indicate issues that are likely to be resolvable via user action:
    71  //
    72  //     ErrorNoSuchProvider: no provider with the given name exists in the repository.
    73  //     ErrorNoSuitableVersion: the provider exists but no available version matches constraints.
    74  //     ErrorNoVersionCompatible: a plugin was found within the constraints but it is
    75  //                               incompatible with the current Terraform version.
    76  //
    77  // These errors should be recognized and handled as special cases by the caller
    78  // to present a suitable user-oriented error message.
    79  //
    80  // All other errors indicate an internal problem that is likely _not_ solvable
    81  // through user action, or at least not within Terraform's scope. Error messages
    82  // are produced under the assumption that if presented to the user they will
    83  // be presented alongside context about what is being installed, and thus the
    84  // error messages do not redundantly include such information.
    85  func (i *ProviderInstaller) Get(provider string, req Constraints) (PluginMeta, error) {
    86  	versions, err := i.listProviderVersions(provider)
    87  	// TODO: return multiple errors
    88  	if err != nil {
    89  		return PluginMeta{}, err
    90  	}
    91  
    92  	if len(versions) == 0 {
    93  		return PluginMeta{}, ErrorNoSuitableVersion
    94  	}
    95  
    96  	versions = allowedVersions(versions, req)
    97  	if len(versions) == 0 {
    98  		return PluginMeta{}, ErrorNoSuitableVersion
    99  	}
   100  
   101  	// sort them newest to oldest
   102  	Versions(versions).Sort()
   103  
   104  	// take the first matching plugin we find
   105  	for _, v := range versions {
   106  		url := i.providerURL(provider, v.String())
   107  
   108  		if !i.SkipVerify {
   109  			sha256, err := i.getProviderChecksum(provider, v.String())
   110  			if err != nil {
   111  				return PluginMeta{}, err
   112  			}
   113  
   114  			// add the checksum parameter for go-getter to verify the download for us.
   115  			if sha256 != "" {
   116  				url = url + "?checksum=sha256:" + sha256
   117  			}
   118  		}
   119  
   120  		log.Printf("[DEBUG] fetching provider info for %s version %s", provider, v)
   121  		if checkPlugin(url, i.PluginProtocolVersion) {
   122  			i.Ui.Info(fmt.Sprintf("- Downloading plugin for provider %q (%s)...", provider, v.String()))
   123  			log.Printf("[DEBUG] getting provider %q version %q at %s", provider, v, url)
   124  			err := getter.Get(i.Dir, url)
   125  			if err != nil {
   126  				return PluginMeta{}, err
   127  			}
   128  
   129  			// Find what we just installed
   130  			// (This is weird, because go-getter doesn't directly return
   131  			//  information about what was extracted, and we just extracted
   132  			//  the archive directly into a shared dir here.)
   133  			log.Printf("[DEBUG] looking for the %s %s plugin we just installed", provider, v)
   134  			metas := FindPlugins("provider", []string{i.Dir})
   135  			log.Printf("[DEBUG] all plugins found %#v", metas)
   136  			metas, _ = metas.ValidateVersions()
   137  			metas = metas.WithName(provider).WithVersion(v)
   138  			log.Printf("[DEBUG] filtered plugins %#v", metas)
   139  			if metas.Count() == 0 {
   140  				// This should never happen. Suggests that the release archive
   141  				// contains an executable file whose name doesn't match the
   142  				// expected convention.
   143  				return PluginMeta{}, fmt.Errorf(
   144  					"failed to find installed plugin version %s; this is a bug in Terraform and should be reported",
   145  					v,
   146  				)
   147  			}
   148  
   149  			if metas.Count() > 1 {
   150  				// This should also never happen, and suggests that a
   151  				// particular version was re-released with a different
   152  				// executable filename. We consider releases as immutable, so
   153  				// this is an error.
   154  				return PluginMeta{}, fmt.Errorf(
   155  					"multiple plugins installed for version %s; this is a bug in Terraform and should be reported",
   156  					v,
   157  				)
   158  			}
   159  
   160  			// By now we know we have exactly one meta, and so "Newest" will
   161  			// return that one.
   162  			return metas.Newest(), nil
   163  		}
   164  
   165  		log.Printf("[INFO] incompatible ProtocolVersion for %s version %s", provider, v)
   166  	}
   167  
   168  	return PluginMeta{}, ErrorNoVersionCompatible
   169  }
   170  
   171  func (i *ProviderInstaller) PurgeUnused(used map[string]PluginMeta) (PluginMetaSet, error) {
   172  	purge := make(PluginMetaSet)
   173  
   174  	present := FindPlugins("provider", []string{i.Dir})
   175  	for meta := range present {
   176  		chosen, ok := used[meta.Name]
   177  		if !ok {
   178  			purge.Add(meta)
   179  		}
   180  		if chosen.Path != meta.Path {
   181  			purge.Add(meta)
   182  		}
   183  	}
   184  
   185  	removed := make(PluginMetaSet)
   186  	var errs error
   187  	for meta := range purge {
   188  		path := meta.Path
   189  		err := os.Remove(path)
   190  		if err != nil {
   191  			errs = multierror.Append(errs, fmt.Errorf(
   192  				"failed to remove unused provider plugin %s: %s",
   193  				path, err,
   194  			))
   195  		} else {
   196  			removed.Add(meta)
   197  		}
   198  	}
   199  
   200  	return removed, errs
   201  }
   202  
   203  // Plugins are referred to by the short name, but all URLs and files will use
   204  // the full name prefixed with terraform-<plugin_type>-
   205  func (i *ProviderInstaller) providerName(name string) string {
   206  	return "terraform-provider-" + name
   207  }
   208  
   209  func (i *ProviderInstaller) providerFileName(name, version string) string {
   210  	os := i.OS
   211  	arch := i.Arch
   212  	if os == "" {
   213  		os = runtime.GOOS
   214  	}
   215  	if arch == "" {
   216  		arch = runtime.GOARCH
   217  	}
   218  	return fmt.Sprintf("%s_%s_%s_%s.zip", i.providerName(name), version, os, arch)
   219  }
   220  
   221  // providerVersionsURL returns the path to the released versions directory for the provider:
   222  // https://releases.hashicorp.com/terraform-provider-name/
   223  func (i *ProviderInstaller) providerVersionsURL(name string) string {
   224  	return releaseHost + "/" + i.providerName(name) + "/"
   225  }
   226  
   227  // providerURL returns the full path to the provider file, using the current OS
   228  // and ARCH:
   229  // .../terraform-provider-name_<x.y.z>/terraform-provider-name_<x.y.z>_<os>_<arch>.<ext>
   230  func (i *ProviderInstaller) providerURL(name, version string) string {
   231  	return fmt.Sprintf("%s%s/%s", i.providerVersionsURL(name), version, i.providerFileName(name, version))
   232  }
   233  
   234  func (i *ProviderInstaller) providerChecksumURL(name, version string) string {
   235  	fileName := fmt.Sprintf("%s_%s_SHA256SUMS", i.providerName(name), version)
   236  	u := fmt.Sprintf("%s%s/%s", i.providerVersionsURL(name), version, fileName)
   237  	return u
   238  }
   239  
   240  func (i *ProviderInstaller) getProviderChecksum(name, version string) (string, error) {
   241  	checksums, err := getPluginSHA256SUMs(i.providerChecksumURL(name, version))
   242  	if err != nil {
   243  		return "", err
   244  	}
   245  
   246  	return checksumForFile(checksums, i.providerFileName(name, version)), nil
   247  }
   248  
   249  // Return the plugin version by making a HEAD request to the provided url.
   250  // If the header is not present, we assume the latest version will be
   251  // compatible, and leave the check for discovery or execution.
   252  func checkPlugin(url string, pluginProtocolVersion uint) bool {
   253  	resp, err := httpClient.Head(url)
   254  	if err != nil {
   255  		log.Printf("[ERROR] error fetching plugin headers: %s", err)
   256  		return false
   257  	}
   258  
   259  	if resp.StatusCode != http.StatusOK {
   260  		log.Println("[ERROR] non-200 status fetching plugin headers:", resp.Status)
   261  		return false
   262  	}
   263  
   264  	proto := resp.Header.Get(protocolVersionHeader)
   265  	if proto == "" {
   266  		// The header isn't present, but we don't make this error fatal since
   267  		// the latest version will probably work.
   268  		log.Printf("[WARNING] missing %s from: %s", protocolVersionHeader, url)
   269  		return true
   270  	}
   271  
   272  	protoVersion, err := strconv.Atoi(proto)
   273  	if err != nil {
   274  		log.Printf("[ERROR] invalid ProtocolVersion: %s", proto)
   275  		return false
   276  	}
   277  
   278  	return protoVersion == int(pluginProtocolVersion)
   279  }
   280  
   281  // list the version available for the named plugin
   282  func (i *ProviderInstaller) listProviderVersions(name string) ([]Version, error) {
   283  	versions, err := listPluginVersions(i.providerVersionsURL(name))
   284  	if err != nil {
   285  		// listPluginVersions returns a verbose error message indicating
   286  		// what was being accessed and what failed
   287  		return nil, err
   288  	}
   289  	return versions, nil
   290  }
   291  
   292  var errVersionNotFound = errors.New("version not found")
   293  
   294  // take the list of available versions for a plugin, and filter out those that
   295  // don't fit the constraints.
   296  func allowedVersions(available []Version, required Constraints) []Version {
   297  	var allowed []Version
   298  
   299  	for _, v := range available {
   300  		if required.Allows(v) {
   301  			allowed = append(allowed, v)
   302  		}
   303  	}
   304  
   305  	return allowed
   306  }
   307  
   308  // return a list of the plugin versions at the given URL
   309  func listPluginVersions(url string) ([]Version, error) {
   310  	resp, err := httpClient.Get(url)
   311  	if err != nil {
   312  		// http library produces a verbose error message that includes the
   313  		// URL being accessed, etc.
   314  		return nil, err
   315  	}
   316  	defer resp.Body.Close()
   317  
   318  	if resp.StatusCode != http.StatusOK {
   319  		body, _ := ioutil.ReadAll(resp.Body)
   320  		log.Printf("[ERROR] failed to fetch plugin versions from %s\n%s\n%s", url, resp.Status, body)
   321  
   322  		switch resp.StatusCode {
   323  		case http.StatusNotFound, http.StatusForbidden:
   324  			// These are treated as indicative of the given name not being
   325  			// a valid provider name at all.
   326  			return nil, ErrorNoSuchProvider
   327  
   328  		default:
   329  			// All other errors are assumed to be operational problems.
   330  			return nil, fmt.Errorf("error accessing %s: %s", url, resp.Status)
   331  		}
   332  
   333  	}
   334  
   335  	body, err := html.Parse(resp.Body)
   336  	if err != nil {
   337  		log.Fatal(err)
   338  	}
   339  
   340  	names := []string{}
   341  
   342  	// all we need to do is list links on the directory listing page that look like plugins
   343  	var f func(*html.Node)
   344  	f = func(n *html.Node) {
   345  		if n.Type == html.ElementNode && n.Data == "a" {
   346  			c := n.FirstChild
   347  			if c != nil && c.Type == html.TextNode && strings.HasPrefix(c.Data, "terraform-") {
   348  				names = append(names, c.Data)
   349  				return
   350  			}
   351  		}
   352  		for c := n.FirstChild; c != nil; c = c.NextSibling {
   353  			f(c)
   354  		}
   355  	}
   356  	f(body)
   357  
   358  	return versionsFromNames(names), nil
   359  }
   360  
   361  // parse the list of directory names into a sorted list of available versions
   362  func versionsFromNames(names []string) []Version {
   363  	var versions []Version
   364  	for _, name := range names {
   365  		parts := strings.SplitN(name, "_", 2)
   366  		if len(parts) == 2 && parts[1] != "" {
   367  			v, err := VersionStr(parts[1]).Parse()
   368  			if err != nil {
   369  				// filter invalid versions scraped from the page
   370  				log.Printf("[WARN] invalid version found for %q: %s", name, err)
   371  				continue
   372  			}
   373  
   374  			versions = append(versions, v)
   375  		}
   376  	}
   377  
   378  	return versions
   379  }
   380  
   381  func checksumForFile(sums []byte, name string) string {
   382  	for _, line := range strings.Split(string(sums), "\n") {
   383  		parts := strings.Fields(line)
   384  		if len(parts) > 1 && parts[1] == name {
   385  			return parts[0]
   386  		}
   387  	}
   388  	return ""
   389  }
   390  
   391  // fetch the SHA256SUMS file provided, and verify its signature.
   392  func getPluginSHA256SUMs(sumsURL string) ([]byte, error) {
   393  	sigURL := sumsURL + ".sig"
   394  
   395  	sums, err := getFile(sumsURL)
   396  	if err != nil {
   397  		return nil, fmt.Errorf("error fetching checksums: %s", err)
   398  	}
   399  
   400  	sig, err := getFile(sigURL)
   401  	if err != nil {
   402  		return nil, fmt.Errorf("error fetching checksums signature: %s", err)
   403  	}
   404  
   405  	if err := verifySig(sums, sig); err != nil {
   406  		return nil, err
   407  	}
   408  
   409  	return sums, nil
   410  }
   411  
   412  func getFile(url string) ([]byte, error) {
   413  	resp, err := httpClient.Get(url)
   414  	if err != nil {
   415  		return nil, err
   416  	}
   417  	defer resp.Body.Close()
   418  
   419  	if resp.StatusCode != http.StatusOK {
   420  		return nil, fmt.Errorf("%s", resp.Status)
   421  	}
   422  
   423  	data, err := ioutil.ReadAll(resp.Body)
   424  	if err != nil {
   425  		return data, err
   426  	}
   427  	return data, nil
   428  }
   429  
   430  func GetReleaseHost() string {
   431  	return releaseHost
   432  }