github.com/loicalbertin/terraform@v0.6.15-0.20170626182346-8e2583055467/plugin/discovery/get.go (about) 1 package discovery 2 3 import ( 4 "errors" 5 "fmt" 6 "io/ioutil" 7 "log" 8 "net/http" 9 "os" 10 "runtime" 11 "strconv" 12 "strings" 13 14 "golang.org/x/net/html" 15 16 cleanhttp "github.com/hashicorp/go-cleanhttp" 17 getter "github.com/hashicorp/go-getter" 18 multierror "github.com/hashicorp/go-multierror" 19 ) 20 21 // Releases are located by parsing the html listing from releases.hashicorp.com. 22 // 23 // The URL for releases follows the pattern: 24 // https://releases.hashicorp.com/terraform-provider-name/<x.y.z>/terraform-provider-name_<x.y.z>_<os>_<arch>.<ext> 25 // 26 // The plugin protocol version will be saved with the release and returned in 27 // the header X-TERRAFORM_PROTOCOL_VERSION. 28 29 const protocolVersionHeader = "x-terraform-protocol-version" 30 31 var releaseHost = "https://releases.hashicorp.com" 32 33 var httpClient = cleanhttp.DefaultClient() 34 35 // Plugins are referred to by the short name, but all URLs and files will use 36 // the full name prefixed with terraform-<plugin_type>- 37 func providerName(name string) string { 38 return "terraform-provider-" + name 39 } 40 41 func providerFileName(name, version string) string { 42 return fmt.Sprintf("%s_%s_%s_%s.zip", providerName(name), version, runtime.GOOS, runtime.GOARCH) 43 } 44 45 // providerVersionsURL returns the path to the released versions directory for the provider: 46 // https://releases.hashicorp.com/terraform-provider-name/ 47 func providerVersionsURL(name string) string { 48 return releaseHost + "/" + providerName(name) + "/" 49 } 50 51 // providerURL returns the full path to the provider file, using the current OS 52 // and ARCH: 53 // .../terraform-provider-name_<x.y.z>/terraform-provider-name_<x.y.z>_<os>_<arch>.<ext> 54 func providerURL(name, version string) string { 55 return fmt.Sprintf("%s%s/%s", providerVersionsURL(name), version, providerFileName(name, version)) 56 } 57 58 func providerChecksumURL(name, version string) string { 59 fileName := fmt.Sprintf("%s_%s_SHA256SUMS", providerName(name), version) 60 u := fmt.Sprintf("%s%s/%s", providerVersionsURL(name), version, fileName) 61 return u 62 } 63 64 // An Installer maintains a local cache of plugins by downloading plugins 65 // from an online repository. 66 type Installer interface { 67 Get(name string, req Constraints) (PluginMeta, error) 68 PurgeUnused(used map[string]PluginMeta) (removed PluginMetaSet, err error) 69 } 70 71 // ProviderInstaller is an Installer implementation that knows how to 72 // download Terraform providers from the official HashiCorp releases service 73 // into a local directory. The files downloaded are compliant with the 74 // naming scheme expected by FindPlugins, so the target directory of a 75 // provider installer can be used as one of several plugin discovery sources. 76 type ProviderInstaller struct { 77 Dir string 78 79 PluginProtocolVersion uint 80 81 // Skip checksum and signature verification 82 SkipVerify bool 83 } 84 85 // Get is part of an implementation of type Installer, and attempts to download 86 // and install a Terraform provider matching the given constraints. 87 // 88 // This method may return one of a number of sentinel errors from this 89 // package to indicate issues that are likely to be resolvable via user action: 90 // 91 // ErrorNoSuchProvider: no provider with the given name exists in the repository. 92 // ErrorNoSuitableVersion: the provider exists but no available version matches constraints. 93 // ErrorNoVersionCompatible: a plugin was found within the constraints but it is 94 // incompatible with the current Terraform version. 95 // 96 // These errors should be recognized and handled as special cases by the caller 97 // to present a suitable user-oriented error message. 98 // 99 // All other errors indicate an internal problem that is likely _not_ solvable 100 // through user action, or at least not within Terraform's scope. Error messages 101 // are produced under the assumption that if presented to the user they will 102 // be presented alongside context about what is being installed, and thus the 103 // error messages do not redundantly include such information. 104 func (i *ProviderInstaller) Get(provider string, req Constraints) (PluginMeta, error) { 105 versions, err := listProviderVersions(provider) 106 // TODO: return multiple errors 107 if err != nil { 108 return PluginMeta{}, err 109 } 110 111 if len(versions) == 0 { 112 return PluginMeta{}, ErrorNoSuitableVersion 113 } 114 115 versions = allowedVersions(versions, req) 116 if len(versions) == 0 { 117 return PluginMeta{}, ErrorNoSuitableVersion 118 } 119 120 // sort them newest to oldest 121 Versions(versions).Sort() 122 123 // take the first matching plugin we find 124 for _, v := range versions { 125 url := providerURL(provider, v.String()) 126 127 if !i.SkipVerify { 128 sha256, err := getProviderChecksum(provider, v.String()) 129 if err != nil { 130 return PluginMeta{}, err 131 } 132 133 // add the checksum parameter for go-getter to verify the download for us. 134 if sha256 != "" { 135 url = url + "?checksum=sha256:" + sha256 136 } 137 } 138 139 log.Printf("[DEBUG] fetching provider info for %s version %s", provider, v) 140 if checkPlugin(url, i.PluginProtocolVersion) { 141 log.Printf("[DEBUG] getting provider %q version %q at %s", provider, v, url) 142 err := getter.Get(i.Dir, url) 143 if err != nil { 144 return PluginMeta{}, err 145 } 146 147 // Find what we just installed 148 // (This is weird, because go-getter doesn't directly return 149 // information about what was extracted, and we just extracted 150 // the archive directly into a shared dir here.) 151 log.Printf("[DEBUG] looking for the %s %s plugin we just installed", provider, v) 152 metas := FindPlugins("provider", []string{i.Dir}) 153 log.Printf("[DEBUG] all plugins found %#v", metas) 154 metas, _ = metas.ValidateVersions() 155 metas = metas.WithName(provider).WithVersion(v) 156 log.Printf("[DEBUG] filtered plugins %#v", metas) 157 if metas.Count() == 0 { 158 // This should never happen. Suggests that the release archive 159 // contains an executable file whose name doesn't match the 160 // expected convention. 161 return PluginMeta{}, fmt.Errorf( 162 "failed to find installed plugin version %s; this is a bug in Terraform and should be reported", 163 v, 164 ) 165 } 166 167 if metas.Count() > 1 { 168 // This should also never happen, and suggests that a 169 // particular version was re-released with a different 170 // executable filename. We consider releases as immutable, so 171 // this is an error. 172 return PluginMeta{}, fmt.Errorf( 173 "multiple plugins installed for version %s; this is a bug in Terraform and should be reported", 174 v, 175 ) 176 } 177 178 // By now we know we have exactly one meta, and so "Newest" will 179 // return that one. 180 return metas.Newest(), nil 181 } 182 183 log.Printf("[INFO] incompatible ProtocolVersion for %s version %s", provider, v) 184 } 185 186 return PluginMeta{}, ErrorNoVersionCompatible 187 } 188 189 func (i *ProviderInstaller) PurgeUnused(used map[string]PluginMeta) (PluginMetaSet, error) { 190 purge := make(PluginMetaSet) 191 192 present := FindPlugins("provider", []string{i.Dir}) 193 for meta := range present { 194 chosen, ok := used[meta.Name] 195 if !ok { 196 purge.Add(meta) 197 } 198 if chosen.Path != meta.Path { 199 purge.Add(meta) 200 } 201 } 202 203 removed := make(PluginMetaSet) 204 var errs error 205 for meta := range purge { 206 path := meta.Path 207 err := os.Remove(path) 208 if err != nil { 209 errs = multierror.Append(errs, fmt.Errorf( 210 "failed to remove unused provider plugin %s: %s", 211 path, err, 212 )) 213 } else { 214 removed.Add(meta) 215 } 216 } 217 218 return removed, errs 219 } 220 221 // Return the plugin version by making a HEAD request to the provided url 222 func checkPlugin(url string, pluginProtocolVersion uint) bool { 223 resp, err := httpClient.Head(url) 224 if err != nil { 225 log.Printf("[ERROR] error fetching plugin headers: %s", err) 226 return false 227 } 228 229 if resp.StatusCode != http.StatusOK { 230 log.Println("[ERROR] non-200 status fetching plugin headers:", resp.Status) 231 return false 232 } 233 234 proto := resp.Header.Get(protocolVersionHeader) 235 if proto == "" { 236 log.Printf("[WARNING] missing %s from: %s", protocolVersionHeader, url) 237 return false 238 } 239 240 protoVersion, err := strconv.Atoi(proto) 241 if err != nil { 242 log.Printf("[ERROR] invalid ProtocolVersion: %s", proto) 243 return false 244 } 245 246 return protoVersion == int(pluginProtocolVersion) 247 } 248 249 var errVersionNotFound = errors.New("version not found") 250 251 // take the list of available versions for a plugin, and filter out those that 252 // don't fit the constraints. 253 func allowedVersions(available []Version, required Constraints) []Version { 254 var allowed []Version 255 256 for _, v := range available { 257 if required.Allows(v) { 258 allowed = append(allowed, v) 259 } 260 } 261 262 return allowed 263 } 264 265 // list the version available for the named plugin 266 func listProviderVersions(name string) ([]Version, error) { 267 versions, err := listPluginVersions(providerVersionsURL(name)) 268 if err != nil { 269 // listPluginVersions returns a verbose error message indicating 270 // what was being accessed and what failed 271 return nil, err 272 } 273 return versions, nil 274 } 275 276 // return a list of the plugin versions at the given URL 277 func listPluginVersions(url string) ([]Version, error) { 278 resp, err := httpClient.Get(url) 279 if err != nil { 280 // http library produces a verbose error message that includes the 281 // URL being accessed, etc. 282 return nil, err 283 } 284 defer resp.Body.Close() 285 286 if resp.StatusCode != http.StatusOK { 287 body, _ := ioutil.ReadAll(resp.Body) 288 log.Printf("[ERROR] failed to fetch plugin versions from %s\n%s\n%s", url, resp.Status, body) 289 290 switch resp.StatusCode { 291 case http.StatusNotFound, http.StatusForbidden: 292 // These are treated as indicative of the given name not being 293 // a valid provider name at all. 294 return nil, ErrorNoSuchProvider 295 296 default: 297 // All other errors are assumed to be operational problems. 298 return nil, fmt.Errorf("error accessing %s: %s", url, resp.Status) 299 } 300 301 } 302 303 body, err := html.Parse(resp.Body) 304 if err != nil { 305 log.Fatal(err) 306 } 307 308 names := []string{} 309 310 // all we need to do is list links on the directory listing page that look like plugins 311 var f func(*html.Node) 312 f = func(n *html.Node) { 313 if n.Type == html.ElementNode && n.Data == "a" { 314 c := n.FirstChild 315 if c != nil && c.Type == html.TextNode && strings.HasPrefix(c.Data, "terraform-") { 316 names = append(names, c.Data) 317 return 318 } 319 } 320 for c := n.FirstChild; c != nil; c = c.NextSibling { 321 f(c) 322 } 323 } 324 f(body) 325 326 return versionsFromNames(names), nil 327 } 328 329 // parse the list of directory names into a sorted list of available versions 330 func versionsFromNames(names []string) []Version { 331 var versions []Version 332 for _, name := range names { 333 parts := strings.SplitN(name, "_", 2) 334 if len(parts) == 2 && parts[1] != "" { 335 v, err := VersionStr(parts[1]).Parse() 336 if err != nil { 337 // filter invalid versions scraped from the page 338 log.Printf("[WARN] invalid version found for %q: %s", name, err) 339 continue 340 } 341 342 versions = append(versions, v) 343 } 344 } 345 346 return versions 347 } 348 349 func getProviderChecksum(name, version string) (string, error) { 350 checksums, err := getPluginSHA256SUMs(providerChecksumURL(name, version)) 351 if err != nil { 352 return "", err 353 } 354 355 return checksumForFile(checksums, providerFileName(name, version)), nil 356 } 357 358 func checksumForFile(sums []byte, name string) string { 359 for _, line := range strings.Split(string(sums), "\n") { 360 parts := strings.Fields(line) 361 if len(parts) > 1 && parts[1] == name { 362 return parts[0] 363 } 364 } 365 return "" 366 } 367 368 // fetch the SHA256SUMS file provided, and verify its signature. 369 func getPluginSHA256SUMs(sumsURL string) ([]byte, error) { 370 sigURL := sumsURL + ".sig" 371 372 sums, err := getFile(sumsURL) 373 if err != nil { 374 return nil, fmt.Errorf("error fetching checksums: %s", err) 375 } 376 377 sig, err := getFile(sigURL) 378 if err != nil { 379 return nil, fmt.Errorf("error fetching checksums signature: %s", err) 380 } 381 382 if err := verifySig(sums, sig); err != nil { 383 return nil, err 384 } 385 386 return sums, nil 387 } 388 389 func getFile(url string) ([]byte, error) { 390 resp, err := httpClient.Get(url) 391 if err != nil { 392 return nil, err 393 } 394 defer resp.Body.Close() 395 396 if resp.StatusCode != http.StatusOK { 397 return nil, fmt.Errorf("%s", resp.Status) 398 } 399 400 data, err := ioutil.ReadAll(resp.Body) 401 if err != nil { 402 return data, err 403 } 404 return data, nil 405 }