github.com/vic3lord/terraform@v0.8.0-rc1.0.20170626102919-16c6dd2cb372/plugin/discovery/get.go (about)

     1  package discovery
     2  
     3  import (
     4  	"errors"
     5  	"fmt"
     6  	"io/ioutil"
     7  	"log"
     8  	"net/http"
     9  	"os"
    10  	"runtime"
    11  	"strconv"
    12  	"strings"
    13  
    14  	"golang.org/x/net/html"
    15  
    16  	cleanhttp "github.com/hashicorp/go-cleanhttp"
    17  	getter "github.com/hashicorp/go-getter"
    18  	multierror "github.com/hashicorp/go-multierror"
    19  )
    20  
    21  // Releases are located by parsing the html listing from releases.hashicorp.com.
    22  //
    23  // The URL for releases follows the pattern:
    24  //    https://releases.hashicorp.com/terraform-provider-name/<x.y.z>/terraform-provider-name_<x.y.z>_<os>_<arch>.<ext>
    25  //
    26  // The plugin protocol version will be saved with the release and returned in
    27  // the header X-TERRAFORM_PROTOCOL_VERSION.
    28  
    29  const protocolVersionHeader = "x-terraform-protocol-version"
    30  
    31  var releaseHost = "https://releases.hashicorp.com"
    32  
    33  var httpClient = cleanhttp.DefaultClient()
    34  
    35  // Plugins are referred to by the short name, but all URLs and files will use
    36  // the full name prefixed with terraform-<plugin_type>-
    37  func providerName(name string) string {
    38  	return "terraform-provider-" + name
    39  }
    40  
    41  func providerFileName(name, version string) string {
    42  	return fmt.Sprintf("%s_%s_%s_%s.zip", providerName(name), version, runtime.GOOS, runtime.GOARCH)
    43  }
    44  
    45  // providerVersionsURL returns the path to the released versions directory for the provider:
    46  // https://releases.hashicorp.com/terraform-provider-name/
    47  func providerVersionsURL(name string) string {
    48  	return releaseHost + "/" + providerName(name) + "/"
    49  }
    50  
    51  // providerURL returns the full path to the provider file, using the current OS
    52  // and ARCH:
    53  // .../terraform-provider-name_<x.y.z>/terraform-provider-name_<x.y.z>_<os>_<arch>.<ext>
    54  func providerURL(name, version string) string {
    55  	return fmt.Sprintf("%s%s/%s", providerVersionsURL(name), version, providerFileName(name, version))
    56  }
    57  
    58  func providerChecksumURL(name, version string) string {
    59  	fileName := fmt.Sprintf("%s_%s_SHA256SUMS", providerName(name), version)
    60  	u := fmt.Sprintf("%s%s/%s", providerVersionsURL(name), version, fileName)
    61  	return u
    62  }
    63  
    64  // An Installer maintains a local cache of plugins by downloading plugins
    65  // from an online repository.
    66  type Installer interface {
    67  	Get(name string, req Constraints) (PluginMeta, error)
    68  	PurgeUnused(used map[string]PluginMeta) (removed PluginMetaSet, err error)
    69  }
    70  
    71  // ProviderInstaller is an Installer implementation that knows how to
    72  // download Terraform providers from the official HashiCorp releases service
    73  // into a local directory. The files downloaded are compliant with the
    74  // naming scheme expected by FindPlugins, so the target directory of a
    75  // provider installer can be used as one of several plugin discovery sources.
    76  type ProviderInstaller struct {
    77  	Dir string
    78  
    79  	PluginProtocolVersion uint
    80  
    81  	// Skip checksum and signature verification
    82  	SkipVerify bool
    83  }
    84  
    85  // Get is part of an implementation of type Installer, and attempts to download
    86  // and install a Terraform provider matching the given constraints.
    87  //
    88  // This method may return one of a number of sentinel errors from this
    89  // package to indicate issues that are likely to be resolvable via user action:
    90  //
    91  //     ErrorNoSuchProvider: no provider with the given name exists in the repository.
    92  //     ErrorNoSuitableVersion: the provider exists but no available version matches constraints.
    93  //     ErrorNoVersionCompatible: a plugin was found within the constraints but it is
    94  //                               incompatible with the current Terraform version.
    95  //
    96  // These errors should be recognized and handled as special cases by the caller
    97  // to present a suitable user-oriented error message.
    98  //
    99  // All other errors indicate an internal problem that is likely _not_ solvable
   100  // through user action, or at least not within Terraform's scope. Error messages
   101  // are produced under the assumption that if presented to the user they will
   102  // be presented alongside context about what is being installed, and thus the
   103  // error messages do not redundantly include such information.
   104  func (i *ProviderInstaller) Get(provider string, req Constraints) (PluginMeta, error) {
   105  	versions, err := listProviderVersions(provider)
   106  	// TODO: return multiple errors
   107  	if err != nil {
   108  		return PluginMeta{}, err
   109  	}
   110  
   111  	if len(versions) == 0 {
   112  		return PluginMeta{}, ErrorNoSuitableVersion
   113  	}
   114  
   115  	versions = allowedVersions(versions, req)
   116  	if len(versions) == 0 {
   117  		return PluginMeta{}, ErrorNoSuitableVersion
   118  	}
   119  
   120  	// sort them newest to oldest
   121  	Versions(versions).Sort()
   122  
   123  	// take the first matching plugin we find
   124  	for _, v := range versions {
   125  		url := providerURL(provider, v.String())
   126  
   127  		if !i.SkipVerify {
   128  			sha256, err := getProviderChecksum(provider, v.String())
   129  			if err != nil {
   130  				return PluginMeta{}, err
   131  			}
   132  
   133  			// add the checksum parameter for go-getter to verify the download for us.
   134  			if sha256 != "" {
   135  				url = url + "?checksum=sha256:" + sha256
   136  			}
   137  		}
   138  
   139  		log.Printf("[DEBUG] fetching provider info for %s version %s", provider, v)
   140  		if checkPlugin(url, i.PluginProtocolVersion) {
   141  			log.Printf("[DEBUG] getting provider %q version %q at %s", provider, v, url)
   142  			err := getter.Get(i.Dir, url)
   143  			if err != nil {
   144  				return PluginMeta{}, err
   145  			}
   146  
   147  			// Find what we just installed
   148  			// (This is weird, because go-getter doesn't directly return
   149  			//  information about what was extracted, and we just extracted
   150  			//  the archive directly into a shared dir here.)
   151  			log.Printf("[DEBUG] looking for the %s %s plugin we just installed", provider, v)
   152  			metas := FindPlugins("provider", []string{i.Dir})
   153  			log.Printf("[DEBUG] all plugins found %#v", metas)
   154  			metas, _ = metas.ValidateVersions()
   155  			metas = metas.WithName(provider).WithVersion(v)
   156  			log.Printf("[DEBUG] filtered plugins %#v", metas)
   157  			if metas.Count() == 0 {
   158  				// This should never happen. Suggests that the release archive
   159  				// contains an executable file whose name doesn't match the
   160  				// expected convention.
   161  				return PluginMeta{}, fmt.Errorf(
   162  					"failed to find installed plugin version %s; this is a bug in Terraform and should be reported",
   163  					v,
   164  				)
   165  			}
   166  
   167  			if metas.Count() > 1 {
   168  				// This should also never happen, and suggests that a
   169  				// particular version was re-released with a different
   170  				// executable filename. We consider releases as immutable, so
   171  				// this is an error.
   172  				return PluginMeta{}, fmt.Errorf(
   173  					"multiple plugins installed for version %s; this is a bug in Terraform and should be reported",
   174  					v,
   175  				)
   176  			}
   177  
   178  			// By now we know we have exactly one meta, and so "Newest" will
   179  			// return that one.
   180  			return metas.Newest(), nil
   181  		}
   182  
   183  		log.Printf("[INFO] incompatible ProtocolVersion for %s version %s", provider, v)
   184  	}
   185  
   186  	return PluginMeta{}, ErrorNoVersionCompatible
   187  }
   188  
   189  func (i *ProviderInstaller) PurgeUnused(used map[string]PluginMeta) (PluginMetaSet, error) {
   190  	purge := make(PluginMetaSet)
   191  
   192  	present := FindPlugins("provider", []string{i.Dir})
   193  	for meta := range present {
   194  		chosen, ok := used[meta.Name]
   195  		if !ok {
   196  			purge.Add(meta)
   197  		}
   198  		if chosen.Path != meta.Path {
   199  			purge.Add(meta)
   200  		}
   201  	}
   202  
   203  	removed := make(PluginMetaSet)
   204  	var errs error
   205  	for meta := range purge {
   206  		path := meta.Path
   207  		err := os.Remove(path)
   208  		if err != nil {
   209  			errs = multierror.Append(errs, fmt.Errorf(
   210  				"failed to remove unused provider plugin %s: %s",
   211  				path, err,
   212  			))
   213  		} else {
   214  			removed.Add(meta)
   215  		}
   216  	}
   217  
   218  	return removed, errs
   219  }
   220  
   221  // Return the plugin version by making a HEAD request to the provided url
   222  func checkPlugin(url string, pluginProtocolVersion uint) bool {
   223  	resp, err := httpClient.Head(url)
   224  	if err != nil {
   225  		log.Printf("[ERROR] error fetching plugin headers: %s", err)
   226  		return false
   227  	}
   228  
   229  	if resp.StatusCode != http.StatusOK {
   230  		log.Println("[ERROR] non-200 status fetching plugin headers:", resp.Status)
   231  		return false
   232  	}
   233  
   234  	proto := resp.Header.Get(protocolVersionHeader)
   235  	if proto == "" {
   236  		log.Printf("[WARNING] missing %s from: %s", protocolVersionHeader, url)
   237  		return false
   238  	}
   239  
   240  	protoVersion, err := strconv.Atoi(proto)
   241  	if err != nil {
   242  		log.Printf("[ERROR] invalid ProtocolVersion: %s", proto)
   243  		return false
   244  	}
   245  
   246  	return protoVersion == int(pluginProtocolVersion)
   247  }
   248  
   249  var errVersionNotFound = errors.New("version not found")
   250  
   251  // take the list of available versions for a plugin, and filter out those that
   252  // don't fit the constraints.
   253  func allowedVersions(available []Version, required Constraints) []Version {
   254  	var allowed []Version
   255  
   256  	for _, v := range available {
   257  		if required.Allows(v) {
   258  			allowed = append(allowed, v)
   259  		}
   260  	}
   261  
   262  	return allowed
   263  }
   264  
   265  // list the version available for the named plugin
   266  func listProviderVersions(name string) ([]Version, error) {
   267  	versions, err := listPluginVersions(providerVersionsURL(name))
   268  	if err != nil {
   269  		// listPluginVersions returns a verbose error message indicating
   270  		// what was being accessed and what failed
   271  		return nil, err
   272  	}
   273  	return versions, nil
   274  }
   275  
   276  // return a list of the plugin versions at the given URL
   277  func listPluginVersions(url string) ([]Version, error) {
   278  	resp, err := httpClient.Get(url)
   279  	if err != nil {
   280  		// http library produces a verbose error message that includes the
   281  		// URL being accessed, etc.
   282  		return nil, err
   283  	}
   284  	defer resp.Body.Close()
   285  
   286  	if resp.StatusCode != http.StatusOK {
   287  		body, _ := ioutil.ReadAll(resp.Body)
   288  		log.Printf("[ERROR] failed to fetch plugin versions from %s\n%s\n%s", url, resp.Status, body)
   289  
   290  		switch resp.StatusCode {
   291  		case http.StatusNotFound, http.StatusForbidden:
   292  			// These are treated as indicative of the given name not being
   293  			// a valid provider name at all.
   294  			return nil, ErrorNoSuchProvider
   295  
   296  		default:
   297  			// All other errors are assumed to be operational problems.
   298  			return nil, fmt.Errorf("error accessing %s: %s", url, resp.Status)
   299  		}
   300  
   301  	}
   302  
   303  	body, err := html.Parse(resp.Body)
   304  	if err != nil {
   305  		log.Fatal(err)
   306  	}
   307  
   308  	names := []string{}
   309  
   310  	// all we need to do is list links on the directory listing page that look like plugins
   311  	var f func(*html.Node)
   312  	f = func(n *html.Node) {
   313  		if n.Type == html.ElementNode && n.Data == "a" {
   314  			c := n.FirstChild
   315  			if c != nil && c.Type == html.TextNode && strings.HasPrefix(c.Data, "terraform-") {
   316  				names = append(names, c.Data)
   317  				return
   318  			}
   319  		}
   320  		for c := n.FirstChild; c != nil; c = c.NextSibling {
   321  			f(c)
   322  		}
   323  	}
   324  	f(body)
   325  
   326  	return versionsFromNames(names), nil
   327  }
   328  
   329  // parse the list of directory names into a sorted list of available versions
   330  func versionsFromNames(names []string) []Version {
   331  	var versions []Version
   332  	for _, name := range names {
   333  		parts := strings.SplitN(name, "_", 2)
   334  		if len(parts) == 2 && parts[1] != "" {
   335  			v, err := VersionStr(parts[1]).Parse()
   336  			if err != nil {
   337  				// filter invalid versions scraped from the page
   338  				log.Printf("[WARN] invalid version found for %q: %s", name, err)
   339  				continue
   340  			}
   341  
   342  			versions = append(versions, v)
   343  		}
   344  	}
   345  
   346  	return versions
   347  }
   348  
   349  func getProviderChecksum(name, version string) (string, error) {
   350  	checksums, err := getPluginSHA256SUMs(providerChecksumURL(name, version))
   351  	if err != nil {
   352  		return "", err
   353  	}
   354  
   355  	return checksumForFile(checksums, providerFileName(name, version)), nil
   356  }
   357  
   358  func checksumForFile(sums []byte, name string) string {
   359  	for _, line := range strings.Split(string(sums), "\n") {
   360  		parts := strings.Fields(line)
   361  		if len(parts) > 1 && parts[1] == name {
   362  			return parts[0]
   363  		}
   364  	}
   365  	return ""
   366  }
   367  
   368  // fetch the SHA256SUMS file provided, and verify its signature.
   369  func getPluginSHA256SUMs(sumsURL string) ([]byte, error) {
   370  	sigURL := sumsURL + ".sig"
   371  
   372  	sums, err := getFile(sumsURL)
   373  	if err != nil {
   374  		return nil, fmt.Errorf("error fetching checksums: %s", err)
   375  	}
   376  
   377  	sig, err := getFile(sigURL)
   378  	if err != nil {
   379  		return nil, fmt.Errorf("error fetching checksums signature: %s", err)
   380  	}
   381  
   382  	if err := verifySig(sums, sig); err != nil {
   383  		return nil, err
   384  	}
   385  
   386  	return sums, nil
   387  }
   388  
   389  func getFile(url string) ([]byte, error) {
   390  	resp, err := httpClient.Get(url)
   391  	if err != nil {
   392  		return nil, err
   393  	}
   394  	defer resp.Body.Close()
   395  
   396  	if resp.StatusCode != http.StatusOK {
   397  		return nil, fmt.Errorf("%s", resp.Status)
   398  	}
   399  
   400  	data, err := ioutil.ReadAll(resp.Body)
   401  	if err != nil {
   402  		return data, err
   403  	}
   404  	return data, nil
   405  }