github.com/kevinklinger/open_terraform@v1.3.6/noninternal/command/providers_mirror.go (about) 1 package command 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "io/ioutil" 7 "net/url" 8 "os" 9 "path/filepath" 10 11 "github.com/apparentlymart/go-versions/versions" 12 "github.com/hashicorp/go-getter" 13 "github.com/kevinklinger/open_terraform/noninternal/getproviders" 14 "github.com/kevinklinger/open_terraform/noninternal/httpclient" 15 "github.com/kevinklinger/open_terraform/noninternal/tfdiags" 16 ) 17 18 // ProvidersMirrorCommand is a Command implementation that implements the 19 // "terraform providers mirror" command, which populates a directory with 20 // local copies of provider plugins needed by the current configuration so 21 // that the mirror can be used to work offline, or similar. 22 type ProvidersMirrorCommand struct { 23 Meta 24 } 25 26 func (c *ProvidersMirrorCommand) Synopsis() string { 27 return "Save local copies of all required provider plugins" 28 } 29 30 func (c *ProvidersMirrorCommand) Run(args []string) int { 31 args = c.Meta.process(args) 32 cmdFlags := c.Meta.defaultFlagSet("providers mirror") 33 var optPlatforms FlagStringSlice 34 cmdFlags.Var(&optPlatforms, "platform", "target platform") 35 cmdFlags.Usage = func() { c.Ui.Error(c.Help()) } 36 if err := cmdFlags.Parse(args); err != nil { 37 c.Ui.Error(fmt.Sprintf("Error parsing command-line flags: %s\n", err.Error())) 38 return 1 39 } 40 41 var diags tfdiags.Diagnostics 42 43 args = cmdFlags.Args() 44 if len(args) != 1 { 45 diags = diags.Append(tfdiags.Sourceless( 46 tfdiags.Error, 47 "No output directory specified", 48 "The providers mirror command requires an output directory as a command-line argument.", 49 )) 50 c.showDiagnostics(diags) 51 return 1 52 } 53 outputDir := args[0] 54 55 var platforms []getproviders.Platform 56 if len(optPlatforms) == 0 { 57 platforms = []getproviders.Platform{getproviders.CurrentPlatform} 58 } else { 59 platforms = make([]getproviders.Platform, 0, len(optPlatforms)) 60 for _, platformStr := range optPlatforms { 61 platform, err := getproviders.ParsePlatform(platformStr) 62 if err != nil { 63 diags = diags.Append(tfdiags.Sourceless( 64 tfdiags.Error, 65 "Invalid target platform", 66 fmt.Sprintf("The string %q given in the -platform option is not a valid target platform: %s.", platformStr, err), 67 )) 68 continue 69 } 70 platforms = append(platforms, platform) 71 } 72 } 73 74 config, confDiags := c.loadConfig(".") 75 diags = diags.Append(confDiags) 76 reqs, moreDiags := config.ProviderRequirements() 77 diags = diags.Append(moreDiags) 78 79 // If we have any error diagnostics already then we won't proceed further. 80 if diags.HasErrors() { 81 c.showDiagnostics(diags) 82 return 1 83 } 84 85 // Unlike other commands, this command always consults the origin registry 86 // for every provider so that it can be used to update a local mirror 87 // directory without needing to first disable that local mirror 88 // in the CLI configuration. 89 source := getproviders.NewMemoizeSource( 90 getproviders.NewRegistrySource(c.Services), 91 ) 92 93 // Providers from registries always use HTTP, so we don't need the full 94 // generality of go-getter but it's still handy to use the HTTP getter 95 // as an easy way to download over HTTP into a file on disk. 96 httpGetter := getter.HttpGetter{ 97 Client: httpclient.New(), 98 Netrc: true, 99 XTerraformGetDisabled: true, 100 } 101 102 // The following logic is similar to that used by the provider installer 103 // in package providercache, but different in a few ways: 104 // - It produces the packed directory layout rather than the unpacked 105 // layout we require in provider cache directories. 106 // - It generates JSON index files that can be read by the 107 // getproviders.HTTPMirrorSource installation method if the result were 108 // copied into the docroot of an HTTP server. 109 // - It can mirror packages for potentially many different target platforms, 110 // so that we can construct a multi-platform mirror regardless of which 111 // platform we run this command on. 112 // - It ignores what's already present and just always downloads everything 113 // that the configuration requires. This is a command intended to be run 114 // infrequently to update a mirror, so it doesn't need to optimize away 115 // fetches of packages that might already be present. 116 117 ctx, cancel := c.InterruptibleContext() 118 defer cancel() 119 for provider, constraints := range reqs { 120 if provider.IsBuiltIn() { 121 c.Ui.Output(fmt.Sprintf("- Skipping %s because it is built in to Terraform CLI", provider.ForDisplay())) 122 continue 123 } 124 constraintsStr := getproviders.VersionConstraintsString(constraints) 125 c.Ui.Output(fmt.Sprintf("- Mirroring %s...", provider.ForDisplay())) 126 // First we'll look for the latest version that matches the given 127 // constraint, which we'll then try to mirror for each target platform. 128 acceptable := versions.MeetingConstraints(constraints) 129 avail, _, err := source.AvailableVersions(ctx, provider) 130 candidates := avail.Filter(acceptable) 131 if err == nil && len(candidates) == 0 { 132 err = fmt.Errorf("no releases match the given constraints %s", constraintsStr) 133 } 134 if err != nil { 135 diags = diags.Append(tfdiags.Sourceless( 136 tfdiags.Error, 137 "Provider not available", 138 fmt.Sprintf("Failed to download %s from its origin registry: %s.", provider.String(), err), 139 )) 140 continue 141 } 142 selected := candidates.Newest() 143 if len(constraintsStr) > 0 { 144 c.Ui.Output(fmt.Sprintf(" - Selected v%s to meet constraints %s", selected.String(), constraintsStr)) 145 } else { 146 c.Ui.Output(fmt.Sprintf(" - Selected v%s with no constraints", selected.String())) 147 } 148 for _, platform := range platforms { 149 c.Ui.Output(fmt.Sprintf(" - Downloading package for %s...", platform.String())) 150 meta, err := source.PackageMeta(ctx, provider, selected, platform) 151 if err != nil { 152 diags = diags.Append(tfdiags.Sourceless( 153 tfdiags.Error, 154 "Provider release not available", 155 fmt.Sprintf("Failed to download %s v%s for %s: %s.", provider.String(), selected.String(), platform.String(), err), 156 )) 157 continue 158 } 159 urlStr, ok := meta.Location.(getproviders.PackageHTTPURL) 160 if !ok { 161 // We don't expect to get non-HTTP locations here because we're 162 // using the registry source, so this seems like a bug in the 163 // registry source. 164 diags = diags.Append(tfdiags.Sourceless( 165 tfdiags.Error, 166 "Provider release not available", 167 fmt.Sprintf("Failed to download %s v%s for %s: Terraform's provider registry client returned unexpected location type %T. This is a bug in Terraform.", provider.String(), selected.String(), platform.String(), meta.Location), 168 )) 169 continue 170 } 171 urlObj, err := url.Parse(string(urlStr)) 172 if err != nil { 173 // We don't expect to get non-HTTP locations here because we're 174 // using the registry source, so this seems like a bug in the 175 // registry source. 176 diags = diags.Append(tfdiags.Sourceless( 177 tfdiags.Error, 178 "Invalid URL for provider release", 179 fmt.Sprintf("The origin registry for %s returned an invalid URL for v%s on %s: %s.", provider.String(), selected.String(), platform.String(), err), 180 )) 181 continue 182 } 183 // targetPath is the path where we ultimately want to place the 184 // downloaded archive, but we'll place it initially at stagingPath 185 // so we can verify its checksums and signatures before making 186 // it discoverable to mirror clients. (stagingPath intentionally 187 // does not follow the filesystem mirror file naming convention.) 188 targetPath := meta.PackedFilePath(outputDir) 189 stagingPath := filepath.Join(filepath.Dir(targetPath), "."+filepath.Base(targetPath)) 190 err = httpGetter.GetFile(stagingPath, urlObj) 191 if err != nil { 192 diags = diags.Append(tfdiags.Sourceless( 193 tfdiags.Error, 194 "Cannot download provider release", 195 fmt.Sprintf("Failed to download %s v%s for %s: %s.", provider.String(), selected.String(), platform.String(), err), 196 )) 197 continue 198 } 199 if meta.Authentication != nil { 200 result, err := meta.Authentication.AuthenticatePackage(getproviders.PackageLocalArchive(stagingPath)) 201 if err != nil { 202 diags = diags.Append(tfdiags.Sourceless( 203 tfdiags.Error, 204 "Invalid provider package", 205 fmt.Sprintf("Failed to authenticate %s v%s for %s: %s.", provider.String(), selected.String(), platform.String(), err), 206 )) 207 continue 208 } 209 c.Ui.Output(fmt.Sprintf(" - Package authenticated: %s", result)) 210 } 211 os.Remove(targetPath) // okay if it fails because we're going to try to rename over it next anyway 212 err = os.Rename(stagingPath, targetPath) 213 if err != nil { 214 diags = diags.Append(tfdiags.Sourceless( 215 tfdiags.Error, 216 "Cannot download provider release", 217 fmt.Sprintf("Failed to place %s package into mirror directory: %s.", provider.String(), err), 218 )) 219 continue 220 } 221 } 222 } 223 224 // Now we'll generate or update the JSON index files in the directory. 225 // We do this by scanning the directory to see what is present, rather than 226 // by relying on the selections we made above, because we want to still 227 // include in the indices any packages that were already present and 228 // not affected by the changes we just made. 229 available, err := getproviders.SearchLocalDirectory(outputDir) 230 if err != nil { 231 diags = diags.Append(tfdiags.Sourceless( 232 tfdiags.Error, 233 "Failed to update indexes", 234 fmt.Sprintf("Could not scan the output directory to get package metadata for the JSON indexes: %s.", err), 235 )) 236 available = nil // the following loop will be a no-op 237 } 238 for provider, metas := range available { 239 if len(metas) == 0 { 240 continue // should never happen, but we'll be resilient 241 } 242 // The index files live in the same directory as the package files, 243 // so to figure that out without duplicating the path-building logic 244 // we'll ask the getproviders package to build an archive filename 245 // for a fictitious package and then use the directory portion of it. 246 indexDir := filepath.Dir(getproviders.PackedFilePathForPackage( 247 outputDir, provider, versions.Unspecified, getproviders.CurrentPlatform, 248 )) 249 indexVersions := map[string]interface{}{} 250 indexArchives := map[getproviders.Version]map[string]interface{}{} 251 for _, meta := range metas { 252 archivePath, ok := meta.Location.(getproviders.PackageLocalArchive) 253 if !ok { 254 // only archive files are eligible to be included in JSON 255 // indices for a network mirror. 256 continue 257 } 258 archiveFilename := filepath.Base(string(archivePath)) 259 version := meta.Version 260 platform := meta.TargetPlatform 261 hash, err := meta.Hash() 262 if err != nil { 263 diags = diags.Append(tfdiags.Sourceless( 264 tfdiags.Error, 265 "Failed to update indexes", 266 fmt.Sprintf("Failed to determine a hash value for %s v%s on %s: %s.", provider, version, platform, err), 267 )) 268 continue 269 } 270 indexVersions[meta.Version.String()] = map[string]interface{}{} 271 if _, ok := indexArchives[version]; !ok { 272 indexArchives[version] = map[string]interface{}{} 273 } 274 indexArchives[version][platform.String()] = map[string]interface{}{ 275 "url": archiveFilename, // a relative URL from the index file's URL 276 "hashes": []string{hash.String()}, // an array to allow for additional hash formats in future 277 } 278 } 279 mainIndex := map[string]interface{}{ 280 "versions": indexVersions, 281 } 282 mainIndexJSON, err := json.MarshalIndent(mainIndex, "", " ") 283 if err != nil { 284 // Should never happen because the input here is entirely under 285 // our control. 286 panic(fmt.Sprintf("failed to encode main index: %s", err)) 287 } 288 // TODO: Ideally we would do these updates as atomic swap operations by 289 // creating a new file and then renaming it over the old one, in case 290 // this directory is the docroot of a live mirror. An atomic swap 291 // requires platform-specific code though: os.Rename alone can't do it 292 // when running on Windows as of Go 1.13. We should revisit this once 293 // we're supporting network mirrors, to avoid having them briefly 294 // become corrupted during updates. 295 err = ioutil.WriteFile(filepath.Join(indexDir, "index.json"), mainIndexJSON, 0644) 296 if err != nil { 297 diags = diags.Append(tfdiags.Sourceless( 298 tfdiags.Error, 299 "Failed to update indexes", 300 fmt.Sprintf("Failed to write an updated JSON index for %s: %s.", provider, err), 301 )) 302 } 303 for version, archiveIndex := range indexArchives { 304 versionIndex := map[string]interface{}{ 305 "archives": archiveIndex, 306 } 307 versionIndexJSON, err := json.MarshalIndent(versionIndex, "", " ") 308 if err != nil { 309 // Should never happen because the input here is entirely under 310 // our control. 311 panic(fmt.Sprintf("failed to encode version index: %s", err)) 312 } 313 err = ioutil.WriteFile(filepath.Join(indexDir, version.String()+".json"), versionIndexJSON, 0644) 314 if err != nil { 315 diags = diags.Append(tfdiags.Sourceless( 316 tfdiags.Error, 317 "Failed to update indexes", 318 fmt.Sprintf("Failed to write an updated JSON index for %s v%s: %s.", provider, version, err), 319 )) 320 } 321 } 322 } 323 324 c.showDiagnostics(diags) 325 if diags.HasErrors() { 326 return 1 327 } 328 return 0 329 } 330 331 func (c *ProvidersMirrorCommand) Help() string { 332 return ` 333 Usage: terraform [global options] providers mirror [options] <target-dir> 334 335 Populates a local directory with copies of the provider plugins needed for 336 the current configuration, so that the directory can be used either directly 337 as a filesystem mirror or as the basis for a network mirror and thus obtain 338 those providers without access to their origin registries in future. 339 340 The mirror directory will contain JSON index files that can be published 341 along with the mirrored packages on a static HTTP file server to produce 342 a network mirror. Those index files will be ignored if the directory is 343 used instead as a local filesystem mirror. 344 345 Options: 346 347 -platform=os_arch Choose which target platform to build a mirror for. 348 By default Terraform will obtain plugin packages 349 suitable for the platform where you run this command. 350 Use this flag multiple times to include packages for 351 multiple target systems. 352 353 Target names consist of an operating system and a CPU 354 architecture. For example, "linux_amd64" selects the 355 Linux operating system running on an AMD64 or x86_64 356 CPU. Each provider is available only for a limited 357 set of target platforms. 358 ` 359 }