github.com/pgray/terraform@v0.5.4-0.20170822184730-b6a464c5214d/plugin/discovery/get.go (about) 1 package discovery 2 3 import ( 4 "errors" 5 "fmt" 6 "io/ioutil" 7 "log" 8 "net/http" 9 "os" 10 "runtime" 11 "strconv" 12 "strings" 13 14 "golang.org/x/net/html" 15 16 cleanhttp "github.com/hashicorp/go-cleanhttp" 17 getter "github.com/hashicorp/go-getter" 18 multierror "github.com/hashicorp/go-multierror" 19 "github.com/mitchellh/cli" 20 ) 21 22 // Releases are located by parsing the html listing from releases.hashicorp.com. 23 // 24 // The URL for releases follows the pattern: 25 // https://releases.hashicorp.com/terraform-provider-name/<x.y.z>/terraform-provider-name_<x.y.z>_<os>_<arch>.<ext> 26 // 27 // The plugin protocol version will be saved with the release and returned in 28 // the header X-TERRAFORM_PROTOCOL_VERSION. 29 30 const protocolVersionHeader = "x-terraform-protocol-version" 31 32 var releaseHost = "https://releases.hashicorp.com" 33 34 var httpClient = cleanhttp.DefaultClient() 35 36 // An Installer maintains a local cache of plugins by downloading plugins 37 // from an online repository. 38 type Installer interface { 39 Get(name string, req Constraints) (PluginMeta, error) 40 PurgeUnused(used map[string]PluginMeta) (removed PluginMetaSet, err error) 41 } 42 43 // ProviderInstaller is an Installer implementation that knows how to 44 // download Terraform providers from the official HashiCorp releases service 45 // into a local directory. The files downloaded are compliant with the 46 // naming scheme expected by FindPlugins, so the target directory of a 47 // provider installer can be used as one of several plugin discovery sources. 48 type ProviderInstaller struct { 49 Dir string 50 51 PluginProtocolVersion uint 52 53 // OS and Arch specify the OS and architecture that should be used when 54 // installing plugins. These use the same labels as the runtime.GOOS and 55 // runtime.GOARCH variables respectively, and indeed the values of these 56 // are used as defaults if either of these is the empty string. 57 OS string 58 Arch string 59 60 // Skip checksum and signature verification 61 SkipVerify bool 62 63 Ui cli.Ui // Ui for output 64 } 65 66 // Get is part of an implementation of type Installer, and attempts to download 67 // and install a Terraform provider matching the given constraints. 68 // 69 // This method may return one of a number of sentinel errors from this 70 // package to indicate issues that are likely to be resolvable via user action: 71 // 72 // ErrorNoSuchProvider: no provider with the given name exists in the repository. 73 // ErrorNoSuitableVersion: the provider exists but no available version matches constraints. 74 // ErrorNoVersionCompatible: a plugin was found within the constraints but it is 75 // incompatible with the current Terraform version. 76 // 77 // These errors should be recognized and handled as special cases by the caller 78 // to present a suitable user-oriented error message. 79 // 80 // All other errors indicate an internal problem that is likely _not_ solvable 81 // through user action, or at least not within Terraform's scope. Error messages 82 // are produced under the assumption that if presented to the user they will 83 // be presented alongside context about what is being installed, and thus the 84 // error messages do not redundantly include such information. 85 func (i *ProviderInstaller) Get(provider string, req Constraints) (PluginMeta, error) { 86 versions, err := i.listProviderVersions(provider) 87 // TODO: return multiple errors 88 if err != nil { 89 return PluginMeta{}, err 90 } 91 92 if len(versions) == 0 { 93 return PluginMeta{}, ErrorNoSuitableVersion 94 } 95 96 versions = allowedVersions(versions, req) 97 if len(versions) == 0 { 98 return PluginMeta{}, ErrorNoSuitableVersion 99 } 100 101 // sort them newest to oldest 102 Versions(versions).Sort() 103 104 // take the first matching plugin we find 105 for _, v := range versions { 106 url := i.providerURL(provider, v.String()) 107 108 if !i.SkipVerify { 109 sha256, err := i.getProviderChecksum(provider, v.String()) 110 if err != nil { 111 return PluginMeta{}, err 112 } 113 114 // add the checksum parameter for go-getter to verify the download for us. 115 if sha256 != "" { 116 url = url + "?checksum=sha256:" + sha256 117 } 118 } 119 120 log.Printf("[DEBUG] fetching provider info for %s version %s", provider, v) 121 if checkPlugin(url, i.PluginProtocolVersion) { 122 i.Ui.Info(fmt.Sprintf("- Downloading plugin for provider %q (%s)...", provider, v.String())) 123 log.Printf("[DEBUG] getting provider %q version %q at %s", provider, v, url) 124 err := getter.Get(i.Dir, url) 125 if err != nil { 126 return PluginMeta{}, err 127 } 128 129 // Find what we just installed 130 // (This is weird, because go-getter doesn't directly return 131 // information about what was extracted, and we just extracted 132 // the archive directly into a shared dir here.) 133 log.Printf("[DEBUG] looking for the %s %s plugin we just installed", provider, v) 134 metas := FindPlugins("provider", []string{i.Dir}) 135 log.Printf("[DEBUG] all plugins found %#v", metas) 136 metas, _ = metas.ValidateVersions() 137 metas = metas.WithName(provider).WithVersion(v) 138 log.Printf("[DEBUG] filtered plugins %#v", metas) 139 if metas.Count() == 0 { 140 // This should never happen. Suggests that the release archive 141 // contains an executable file whose name doesn't match the 142 // expected convention. 143 return PluginMeta{}, fmt.Errorf( 144 "failed to find installed plugin version %s; this is a bug in Terraform and should be reported", 145 v, 146 ) 147 } 148 149 if metas.Count() > 1 { 150 // This should also never happen, and suggests that a 151 // particular version was re-released with a different 152 // executable filename. We consider releases as immutable, so 153 // this is an error. 154 return PluginMeta{}, fmt.Errorf( 155 "multiple plugins installed for version %s; this is a bug in Terraform and should be reported", 156 v, 157 ) 158 } 159 160 // By now we know we have exactly one meta, and so "Newest" will 161 // return that one. 162 return metas.Newest(), nil 163 } 164 165 log.Printf("[INFO] incompatible ProtocolVersion for %s version %s", provider, v) 166 } 167 168 return PluginMeta{}, ErrorNoVersionCompatible 169 } 170 171 func (i *ProviderInstaller) PurgeUnused(used map[string]PluginMeta) (PluginMetaSet, error) { 172 purge := make(PluginMetaSet) 173 174 present := FindPlugins("provider", []string{i.Dir}) 175 for meta := range present { 176 chosen, ok := used[meta.Name] 177 if !ok { 178 purge.Add(meta) 179 } 180 if chosen.Path != meta.Path { 181 purge.Add(meta) 182 } 183 } 184 185 removed := make(PluginMetaSet) 186 var errs error 187 for meta := range purge { 188 path := meta.Path 189 err := os.Remove(path) 190 if err != nil { 191 errs = multierror.Append(errs, fmt.Errorf( 192 "failed to remove unused provider plugin %s: %s", 193 path, err, 194 )) 195 } else { 196 removed.Add(meta) 197 } 198 } 199 200 return removed, errs 201 } 202 203 // Plugins are referred to by the short name, but all URLs and files will use 204 // the full name prefixed with terraform-<plugin_type>- 205 func (i *ProviderInstaller) providerName(name string) string { 206 return "terraform-provider-" + name 207 } 208 209 func (i *ProviderInstaller) providerFileName(name, version string) string { 210 os := i.OS 211 arch := i.Arch 212 if os == "" { 213 os = runtime.GOOS 214 } 215 if arch == "" { 216 arch = runtime.GOARCH 217 } 218 return fmt.Sprintf("%s_%s_%s_%s.zip", i.providerName(name), version, os, arch) 219 } 220 221 // providerVersionsURL returns the path to the released versions directory for the provider: 222 // https://releases.hashicorp.com/terraform-provider-name/ 223 func (i *ProviderInstaller) providerVersionsURL(name string) string { 224 return releaseHost + "/" + i.providerName(name) + "/" 225 } 226 227 // providerURL returns the full path to the provider file, using the current OS 228 // and ARCH: 229 // .../terraform-provider-name_<x.y.z>/terraform-provider-name_<x.y.z>_<os>_<arch>.<ext> 230 func (i *ProviderInstaller) providerURL(name, version string) string { 231 return fmt.Sprintf("%s%s/%s", i.providerVersionsURL(name), version, i.providerFileName(name, version)) 232 } 233 234 func (i *ProviderInstaller) providerChecksumURL(name, version string) string { 235 fileName := fmt.Sprintf("%s_%s_SHA256SUMS", i.providerName(name), version) 236 u := fmt.Sprintf("%s%s/%s", i.providerVersionsURL(name), version, fileName) 237 return u 238 } 239 240 func (i *ProviderInstaller) getProviderChecksum(name, version string) (string, error) { 241 checksums, err := getPluginSHA256SUMs(i.providerChecksumURL(name, version)) 242 if err != nil { 243 return "", err 244 } 245 246 return checksumForFile(checksums, i.providerFileName(name, version)), nil 247 } 248 249 // Return the plugin version by making a HEAD request to the provided url. 250 // If the header is not present, we assume the latest version will be 251 // compatible, and leave the check for discovery or execution. 252 func checkPlugin(url string, pluginProtocolVersion uint) bool { 253 resp, err := httpClient.Head(url) 254 if err != nil { 255 log.Printf("[ERROR] error fetching plugin headers: %s", err) 256 return false 257 } 258 259 if resp.StatusCode != http.StatusOK { 260 log.Println("[ERROR] non-200 status fetching plugin headers:", resp.Status) 261 return false 262 } 263 264 proto := resp.Header.Get(protocolVersionHeader) 265 if proto == "" { 266 // The header isn't present, but we don't make this error fatal since 267 // the latest version will probably work. 268 log.Printf("[WARNING] missing %s from: %s", protocolVersionHeader, url) 269 return true 270 } 271 272 protoVersion, err := strconv.Atoi(proto) 273 if err != nil { 274 log.Printf("[ERROR] invalid ProtocolVersion: %s", proto) 275 return false 276 } 277 278 return protoVersion == int(pluginProtocolVersion) 279 } 280 281 // list the version available for the named plugin 282 func (i *ProviderInstaller) listProviderVersions(name string) ([]Version, error) { 283 versions, err := listPluginVersions(i.providerVersionsURL(name)) 284 if err != nil { 285 // listPluginVersions returns a verbose error message indicating 286 // what was being accessed and what failed 287 return nil, err 288 } 289 return versions, nil 290 } 291 292 var errVersionNotFound = errors.New("version not found") 293 294 // take the list of available versions for a plugin, and filter out those that 295 // don't fit the constraints. 296 func allowedVersions(available []Version, required Constraints) []Version { 297 var allowed []Version 298 299 for _, v := range available { 300 if required.Allows(v) { 301 allowed = append(allowed, v) 302 } 303 } 304 305 return allowed 306 } 307 308 // return a list of the plugin versions at the given URL 309 func listPluginVersions(url string) ([]Version, error) { 310 resp, err := httpClient.Get(url) 311 if err != nil { 312 // http library produces a verbose error message that includes the 313 // URL being accessed, etc. 314 return nil, err 315 } 316 defer resp.Body.Close() 317 318 if resp.StatusCode != http.StatusOK { 319 body, _ := ioutil.ReadAll(resp.Body) 320 log.Printf("[ERROR] failed to fetch plugin versions from %s\n%s\n%s", url, resp.Status, body) 321 322 switch resp.StatusCode { 323 case http.StatusNotFound, http.StatusForbidden: 324 // These are treated as indicative of the given name not being 325 // a valid provider name at all. 326 return nil, ErrorNoSuchProvider 327 328 default: 329 // All other errors are assumed to be operational problems. 330 return nil, fmt.Errorf("error accessing %s: %s", url, resp.Status) 331 } 332 333 } 334 335 body, err := html.Parse(resp.Body) 336 if err != nil { 337 log.Fatal(err) 338 } 339 340 names := []string{} 341 342 // all we need to do is list links on the directory listing page that look like plugins 343 var f func(*html.Node) 344 f = func(n *html.Node) { 345 if n.Type == html.ElementNode && n.Data == "a" { 346 c := n.FirstChild 347 if c != nil && c.Type == html.TextNode && strings.HasPrefix(c.Data, "terraform-") { 348 names = append(names, c.Data) 349 return 350 } 351 } 352 for c := n.FirstChild; c != nil; c = c.NextSibling { 353 f(c) 354 } 355 } 356 f(body) 357 358 return versionsFromNames(names), nil 359 } 360 361 // parse the list of directory names into a sorted list of available versions 362 func versionsFromNames(names []string) []Version { 363 var versions []Version 364 for _, name := range names { 365 parts := strings.SplitN(name, "_", 2) 366 if len(parts) == 2 && parts[1] != "" { 367 v, err := VersionStr(parts[1]).Parse() 368 if err != nil { 369 // filter invalid versions scraped from the page 370 log.Printf("[WARN] invalid version found for %q: %s", name, err) 371 continue 372 } 373 374 versions = append(versions, v) 375 } 376 } 377 378 return versions 379 } 380 381 func checksumForFile(sums []byte, name string) string { 382 for _, line := range strings.Split(string(sums), "\n") { 383 parts := strings.Fields(line) 384 if len(parts) > 1 && parts[1] == name { 385 return parts[0] 386 } 387 } 388 return "" 389 } 390 391 // fetch the SHA256SUMS file provided, and verify its signature. 392 func getPluginSHA256SUMs(sumsURL string) ([]byte, error) { 393 sigURL := sumsURL + ".sig" 394 395 sums, err := getFile(sumsURL) 396 if err != nil { 397 return nil, fmt.Errorf("error fetching checksums: %s", err) 398 } 399 400 sig, err := getFile(sigURL) 401 if err != nil { 402 return nil, fmt.Errorf("error fetching checksums signature: %s", err) 403 } 404 405 if err := verifySig(sums, sig); err != nil { 406 return nil, err 407 } 408 409 return sums, nil 410 } 411 412 func getFile(url string) ([]byte, error) { 413 resp, err := httpClient.Get(url) 414 if err != nil { 415 return nil, err 416 } 417 defer resp.Body.Close() 418 419 if resp.StatusCode != http.StatusOK { 420 return nil, fmt.Errorf("%s", resp.Status) 421 } 422 423 data, err := ioutil.ReadAll(resp.Body) 424 if err != nil { 425 return data, err 426 } 427 return data, nil 428 } 429 430 func GetReleaseHost() string { 431 return releaseHost 432 }