github.com/spirius/terraform@v0.10.0-beta2.0.20170714185654-87b2c0cf8fea/plugin/discovery/get.go (about) 1 package discovery 2 3 import ( 4 "errors" 5 "fmt" 6 "io/ioutil" 7 "log" 8 "net/http" 9 "os" 10 "runtime" 11 "strconv" 12 "strings" 13 14 "golang.org/x/net/html" 15 16 cleanhttp "github.com/hashicorp/go-cleanhttp" 17 getter "github.com/hashicorp/go-getter" 18 multierror "github.com/hashicorp/go-multierror" 19 ) 20 21 // Releases are located by parsing the html listing from releases.hashicorp.com. 22 // 23 // The URL for releases follows the pattern: 24 // https://releases.hashicorp.com/terraform-provider-name/<x.y.z>/terraform-provider-name_<x.y.z>_<os>_<arch>.<ext> 25 // 26 // The plugin protocol version will be saved with the release and returned in 27 // the header X-TERRAFORM_PROTOCOL_VERSION. 28 29 const protocolVersionHeader = "x-terraform-protocol-version" 30 31 var releaseHost = "https://releases.hashicorp.com" 32 33 var httpClient = cleanhttp.DefaultClient() 34 35 // An Installer maintains a local cache of plugins by downloading plugins 36 // from an online repository. 37 type Installer interface { 38 Get(name string, req Constraints) (PluginMeta, error) 39 PurgeUnused(used map[string]PluginMeta) (removed PluginMetaSet, err error) 40 } 41 42 // ProviderInstaller is an Installer implementation that knows how to 43 // download Terraform providers from the official HashiCorp releases service 44 // into a local directory. The files downloaded are compliant with the 45 // naming scheme expected by FindPlugins, so the target directory of a 46 // provider installer can be used as one of several plugin discovery sources. 47 type ProviderInstaller struct { 48 Dir string 49 50 PluginProtocolVersion uint 51 52 // OS and Arch specify the OS and architecture that should be used when 53 // installing plugins. These use the same labels as the runtime.GOOS and 54 // runtime.GOARCH variables respectively, and indeed the values of these 55 // are used as defaults if either of these is the empty string. 56 OS string 57 Arch string 58 59 // Skip checksum and signature verification 60 SkipVerify bool 61 } 62 63 // Get is part of an implementation of type Installer, and attempts to download 64 // and install a Terraform provider matching the given constraints. 65 // 66 // This method may return one of a number of sentinel errors from this 67 // package to indicate issues that are likely to be resolvable via user action: 68 // 69 // ErrorNoSuchProvider: no provider with the given name exists in the repository. 70 // ErrorNoSuitableVersion: the provider exists but no available version matches constraints. 71 // ErrorNoVersionCompatible: a plugin was found within the constraints but it is 72 // incompatible with the current Terraform version. 73 // 74 // These errors should be recognized and handled as special cases by the caller 75 // to present a suitable user-oriented error message. 76 // 77 // All other errors indicate an internal problem that is likely _not_ solvable 78 // through user action, or at least not within Terraform's scope. Error messages 79 // are produced under the assumption that if presented to the user they will 80 // be presented alongside context about what is being installed, and thus the 81 // error messages do not redundantly include such information. 82 func (i *ProviderInstaller) Get(provider string, req Constraints) (PluginMeta, error) { 83 versions, err := i.listProviderVersions(provider) 84 // TODO: return multiple errors 85 if err != nil { 86 return PluginMeta{}, err 87 } 88 89 if len(versions) == 0 { 90 return PluginMeta{}, ErrorNoSuitableVersion 91 } 92 93 versions = allowedVersions(versions, req) 94 if len(versions) == 0 { 95 return PluginMeta{}, ErrorNoSuitableVersion 96 } 97 98 // sort them newest to oldest 99 Versions(versions).Sort() 100 101 // take the first matching plugin we find 102 for _, v := range versions { 103 url := i.providerURL(provider, v.String()) 104 105 if !i.SkipVerify { 106 sha256, err := i.getProviderChecksum(provider, v.String()) 107 if err != nil { 108 return PluginMeta{}, err 109 } 110 111 // add the checksum parameter for go-getter to verify the download for us. 112 if sha256 != "" { 113 url = url + "?checksum=sha256:" + sha256 114 } 115 } 116 117 log.Printf("[DEBUG] fetching provider info for %s version %s", provider, v) 118 if checkPlugin(url, i.PluginProtocolVersion) { 119 log.Printf("[DEBUG] getting provider %q version %q at %s", provider, v, url) 120 err := getter.Get(i.Dir, url) 121 if err != nil { 122 return PluginMeta{}, err 123 } 124 125 // Find what we just installed 126 // (This is weird, because go-getter doesn't directly return 127 // information about what was extracted, and we just extracted 128 // the archive directly into a shared dir here.) 129 log.Printf("[DEBUG] looking for the %s %s plugin we just installed", provider, v) 130 metas := FindPlugins("provider", []string{i.Dir}) 131 log.Printf("[DEBUG] all plugins found %#v", metas) 132 metas, _ = metas.ValidateVersions() 133 metas = metas.WithName(provider).WithVersion(v) 134 log.Printf("[DEBUG] filtered plugins %#v", metas) 135 if metas.Count() == 0 { 136 // This should never happen. Suggests that the release archive 137 // contains an executable file whose name doesn't match the 138 // expected convention. 139 return PluginMeta{}, fmt.Errorf( 140 "failed to find installed plugin version %s; this is a bug in Terraform and should be reported", 141 v, 142 ) 143 } 144 145 if metas.Count() > 1 { 146 // This should also never happen, and suggests that a 147 // particular version was re-released with a different 148 // executable filename. We consider releases as immutable, so 149 // this is an error. 150 return PluginMeta{}, fmt.Errorf( 151 "multiple plugins installed for version %s; this is a bug in Terraform and should be reported", 152 v, 153 ) 154 } 155 156 // By now we know we have exactly one meta, and so "Newest" will 157 // return that one. 158 return metas.Newest(), nil 159 } 160 161 log.Printf("[INFO] incompatible ProtocolVersion for %s version %s", provider, v) 162 } 163 164 return PluginMeta{}, ErrorNoVersionCompatible 165 } 166 167 func (i *ProviderInstaller) PurgeUnused(used map[string]PluginMeta) (PluginMetaSet, error) { 168 purge := make(PluginMetaSet) 169 170 present := FindPlugins("provider", []string{i.Dir}) 171 for meta := range present { 172 chosen, ok := used[meta.Name] 173 if !ok { 174 purge.Add(meta) 175 } 176 if chosen.Path != meta.Path { 177 purge.Add(meta) 178 } 179 } 180 181 removed := make(PluginMetaSet) 182 var errs error 183 for meta := range purge { 184 path := meta.Path 185 err := os.Remove(path) 186 if err != nil { 187 errs = multierror.Append(errs, fmt.Errorf( 188 "failed to remove unused provider plugin %s: %s", 189 path, err, 190 )) 191 } else { 192 removed.Add(meta) 193 } 194 } 195 196 return removed, errs 197 } 198 199 // Plugins are referred to by the short name, but all URLs and files will use 200 // the full name prefixed with terraform-<plugin_type>- 201 func (i *ProviderInstaller) providerName(name string) string { 202 return "terraform-provider-" + name 203 } 204 205 func (i *ProviderInstaller) providerFileName(name, version string) string { 206 os := i.OS 207 arch := i.Arch 208 if os == "" { 209 os = runtime.GOOS 210 } 211 if arch == "" { 212 arch = runtime.GOARCH 213 } 214 return fmt.Sprintf("%s_%s_%s_%s.zip", i.providerName(name), version, os, arch) 215 } 216 217 // providerVersionsURL returns the path to the released versions directory for the provider: 218 // https://releases.hashicorp.com/terraform-provider-name/ 219 func (i *ProviderInstaller) providerVersionsURL(name string) string { 220 return releaseHost + "/" + i.providerName(name) + "/" 221 } 222 223 // providerURL returns the full path to the provider file, using the current OS 224 // and ARCH: 225 // .../terraform-provider-name_<x.y.z>/terraform-provider-name_<x.y.z>_<os>_<arch>.<ext> 226 func (i *ProviderInstaller) providerURL(name, version string) string { 227 return fmt.Sprintf("%s%s/%s", i.providerVersionsURL(name), version, i.providerFileName(name, version)) 228 } 229 230 func (i *ProviderInstaller) providerChecksumURL(name, version string) string { 231 fileName := fmt.Sprintf("%s_%s_SHA256SUMS", i.providerName(name), version) 232 u := fmt.Sprintf("%s%s/%s", i.providerVersionsURL(name), version, fileName) 233 return u 234 } 235 236 func (i *ProviderInstaller) getProviderChecksum(name, version string) (string, error) { 237 checksums, err := getPluginSHA256SUMs(i.providerChecksumURL(name, version)) 238 if err != nil { 239 return "", err 240 } 241 242 return checksumForFile(checksums, i.providerFileName(name, version)), nil 243 } 244 245 // Return the plugin version by making a HEAD request to the provided url 246 func checkPlugin(url string, pluginProtocolVersion uint) bool { 247 resp, err := httpClient.Head(url) 248 if err != nil { 249 log.Printf("[ERROR] error fetching plugin headers: %s", err) 250 return false 251 } 252 253 if resp.StatusCode != http.StatusOK { 254 log.Println("[ERROR] non-200 status fetching plugin headers:", resp.Status) 255 return false 256 } 257 258 proto := resp.Header.Get(protocolVersionHeader) 259 if proto == "" { 260 log.Printf("[WARNING] missing %s from: %s", protocolVersionHeader, url) 261 return false 262 } 263 264 protoVersion, err := strconv.Atoi(proto) 265 if err != nil { 266 log.Printf("[ERROR] invalid ProtocolVersion: %s", proto) 267 return false 268 } 269 270 return protoVersion == int(pluginProtocolVersion) 271 } 272 273 // list the version available for the named plugin 274 func (i *ProviderInstaller) listProviderVersions(name string) ([]Version, error) { 275 versions, err := listPluginVersions(i.providerVersionsURL(name)) 276 if err != nil { 277 // listPluginVersions returns a verbose error message indicating 278 // what was being accessed and what failed 279 return nil, err 280 } 281 return versions, nil 282 } 283 284 var errVersionNotFound = errors.New("version not found") 285 286 // take the list of available versions for a plugin, and filter out those that 287 // don't fit the constraints. 288 func allowedVersions(available []Version, required Constraints) []Version { 289 var allowed []Version 290 291 for _, v := range available { 292 if required.Allows(v) { 293 allowed = append(allowed, v) 294 } 295 } 296 297 return allowed 298 } 299 300 // return a list of the plugin versions at the given URL 301 func listPluginVersions(url string) ([]Version, error) { 302 resp, err := httpClient.Get(url) 303 if err != nil { 304 // http library produces a verbose error message that includes the 305 // URL being accessed, etc. 306 return nil, err 307 } 308 defer resp.Body.Close() 309 310 if resp.StatusCode != http.StatusOK { 311 body, _ := ioutil.ReadAll(resp.Body) 312 log.Printf("[ERROR] failed to fetch plugin versions from %s\n%s\n%s", url, resp.Status, body) 313 314 switch resp.StatusCode { 315 case http.StatusNotFound, http.StatusForbidden: 316 // These are treated as indicative of the given name not being 317 // a valid provider name at all. 318 return nil, ErrorNoSuchProvider 319 320 default: 321 // All other errors are assumed to be operational problems. 322 return nil, fmt.Errorf("error accessing %s: %s", url, resp.Status) 323 } 324 325 } 326 327 body, err := html.Parse(resp.Body) 328 if err != nil { 329 log.Fatal(err) 330 } 331 332 names := []string{} 333 334 // all we need to do is list links on the directory listing page that look like plugins 335 var f func(*html.Node) 336 f = func(n *html.Node) { 337 if n.Type == html.ElementNode && n.Data == "a" { 338 c := n.FirstChild 339 if c != nil && c.Type == html.TextNode && strings.HasPrefix(c.Data, "terraform-") { 340 names = append(names, c.Data) 341 return 342 } 343 } 344 for c := n.FirstChild; c != nil; c = c.NextSibling { 345 f(c) 346 } 347 } 348 f(body) 349 350 return versionsFromNames(names), nil 351 } 352 353 // parse the list of directory names into a sorted list of available versions 354 func versionsFromNames(names []string) []Version { 355 var versions []Version 356 for _, name := range names { 357 parts := strings.SplitN(name, "_", 2) 358 if len(parts) == 2 && parts[1] != "" { 359 v, err := VersionStr(parts[1]).Parse() 360 if err != nil { 361 // filter invalid versions scraped from the page 362 log.Printf("[WARN] invalid version found for %q: %s", name, err) 363 continue 364 } 365 366 versions = append(versions, v) 367 } 368 } 369 370 return versions 371 } 372 373 func checksumForFile(sums []byte, name string) string { 374 for _, line := range strings.Split(string(sums), "\n") { 375 parts := strings.Fields(line) 376 if len(parts) > 1 && parts[1] == name { 377 return parts[0] 378 } 379 } 380 return "" 381 } 382 383 // fetch the SHA256SUMS file provided, and verify its signature. 384 func getPluginSHA256SUMs(sumsURL string) ([]byte, error) { 385 sigURL := sumsURL + ".sig" 386 387 sums, err := getFile(sumsURL) 388 if err != nil { 389 return nil, fmt.Errorf("error fetching checksums: %s", err) 390 } 391 392 sig, err := getFile(sigURL) 393 if err != nil { 394 return nil, fmt.Errorf("error fetching checksums signature: %s", err) 395 } 396 397 if err := verifySig(sums, sig); err != nil { 398 return nil, err 399 } 400 401 return sums, nil 402 } 403 404 func getFile(url string) ([]byte, error) { 405 resp, err := httpClient.Get(url) 406 if err != nil { 407 return nil, err 408 } 409 defer resp.Body.Close() 410 411 if resp.StatusCode != http.StatusOK { 412 return nil, fmt.Errorf("%s", resp.Status) 413 } 414 415 data, err := ioutil.ReadAll(resp.Body) 416 if err != nil { 417 return data, err 418 } 419 return data, nil 420 }