github.com/cycloidio/terraform@v1.1.10-0.20220513142504-76d5c768dc63/command/providers_mirror.go (about) 1 package command 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "io/ioutil" 7 "net/url" 8 "os" 9 "path/filepath" 10 11 "github.com/apparentlymart/go-versions/versions" 12 "github.com/hashicorp/go-getter" 13 "github.com/cycloidio/terraform/getproviders" 14 "github.com/cycloidio/terraform/httpclient" 15 "github.com/cycloidio/terraform/tfdiags" 16 ) 17 18 // ProvidersMirrorCommand is a Command implementation that implements the 19 // "terraform providers mirror" command, which populates a directory with 20 // local copies of provider plugins needed by the current configuration so 21 // that the mirror can be used to work offline, or similar. 22 type ProvidersMirrorCommand struct { 23 Meta 24 } 25 26 func (c *ProvidersMirrorCommand) Synopsis() string { 27 return "Save local copies of all required provider plugins" 28 } 29 30 func (c *ProvidersMirrorCommand) Run(args []string) int { 31 args = c.Meta.process(args) 32 cmdFlags := c.Meta.defaultFlagSet("providers mirror") 33 var optPlatforms FlagStringSlice 34 cmdFlags.Var(&optPlatforms, "platform", "target platform") 35 cmdFlags.Usage = func() { c.Ui.Error(c.Help()) } 36 if err := cmdFlags.Parse(args); err != nil { 37 c.Ui.Error(fmt.Sprintf("Error parsing command-line flags: %s\n", err.Error())) 38 return 1 39 } 40 41 var diags tfdiags.Diagnostics 42 43 args = cmdFlags.Args() 44 if len(args) != 1 { 45 diags = diags.Append(tfdiags.Sourceless( 46 tfdiags.Error, 47 "No output directory specified", 48 "The providers mirror command requires an output directory as a command-line argument.", 49 )) 50 c.showDiagnostics(diags) 51 return 1 52 } 53 outputDir := args[0] 54 55 var platforms []getproviders.Platform 56 if len(optPlatforms) == 0 { 57 platforms = []getproviders.Platform{getproviders.CurrentPlatform} 58 } else { 59 platforms = make([]getproviders.Platform, 0, len(optPlatforms)) 60 for _, platformStr := range optPlatforms { 61 platform, err := getproviders.ParsePlatform(platformStr) 62 if err != nil { 63 diags = diags.Append(tfdiags.Sourceless( 64 tfdiags.Error, 65 "Invalid target platform", 66 fmt.Sprintf("The string %q given in the -platform option is not a valid target platform: %s.", platformStr, err), 67 )) 68 continue 69 } 70 platforms = append(platforms, platform) 71 } 72 } 73 74 config, confDiags := c.loadConfig(".") 75 diags = diags.Append(confDiags) 76 reqs, moreDiags := config.ProviderRequirements() 77 diags = diags.Append(moreDiags) 78 79 // If we have any error diagnostics already then we won't proceed further. 80 if diags.HasErrors() { 81 c.showDiagnostics(diags) 82 return 1 83 } 84 85 // Unlike other commands, this command always consults the origin registry 86 // for every provider so that it can be used to update a local mirror 87 // directory without needing to first disable that local mirror 88 // in the CLI configuration. 89 source := getproviders.NewMemoizeSource( 90 getproviders.NewRegistrySource(c.Services), 91 ) 92 93 // Providers from registries always use HTTP, so we don't need the full 94 // generality of go-getter but it's still handy to use the HTTP getter 95 // as an easy way to download over HTTP into a file on disk. 96 httpGetter := getter.HttpGetter{ 97 Client: httpclient.New(), 98 Netrc: true, 99 } 100 101 // The following logic is similar to that used by the provider installer 102 // in package providercache, but different in a few ways: 103 // - It produces the packed directory layout rather than the unpacked 104 // layout we require in provider cache directories. 105 // - It generates JSON index files that can be read by the 106 // getproviders.HTTPMirrorSource installation method if the result were 107 // copied into the docroot of an HTTP server. 108 // - It can mirror packages for potentially many different target platforms, 109 // so that we can construct a multi-platform mirror regardless of which 110 // platform we run this command on. 111 // - It ignores what's already present and just always downloads everything 112 // that the configuration requires. This is a command intended to be run 113 // infrequently to update a mirror, so it doesn't need to optimize away 114 // fetches of packages that might already be present. 115 116 ctx, cancel := c.InterruptibleContext() 117 defer cancel() 118 for provider, constraints := range reqs { 119 if provider.IsBuiltIn() { 120 c.Ui.Output(fmt.Sprintf("- Skipping %s because it is built in to Terraform CLI", provider.ForDisplay())) 121 continue 122 } 123 constraintsStr := getproviders.VersionConstraintsString(constraints) 124 c.Ui.Output(fmt.Sprintf("- Mirroring %s...", provider.ForDisplay())) 125 // First we'll look for the latest version that matches the given 126 // constraint, which we'll then try to mirror for each target platform. 127 acceptable := versions.MeetingConstraints(constraints) 128 avail, _, err := source.AvailableVersions(ctx, provider) 129 candidates := avail.Filter(acceptable) 130 if err == nil && len(candidates) == 0 { 131 err = fmt.Errorf("no releases match the given constraints %s", constraintsStr) 132 } 133 if err != nil { 134 diags = diags.Append(tfdiags.Sourceless( 135 tfdiags.Error, 136 "Provider not available", 137 fmt.Sprintf("Failed to download %s from its origin registry: %s.", provider.String(), err), 138 )) 139 continue 140 } 141 selected := candidates.Newest() 142 if len(constraintsStr) > 0 { 143 c.Ui.Output(fmt.Sprintf(" - Selected v%s to meet constraints %s", selected.String(), constraintsStr)) 144 } else { 145 c.Ui.Output(fmt.Sprintf(" - Selected v%s with no constraints", selected.String())) 146 } 147 for _, platform := range platforms { 148 c.Ui.Output(fmt.Sprintf(" - Downloading package for %s...", platform.String())) 149 meta, err := source.PackageMeta(ctx, provider, selected, platform) 150 if err != nil { 151 diags = diags.Append(tfdiags.Sourceless( 152 tfdiags.Error, 153 "Provider release not available", 154 fmt.Sprintf("Failed to download %s v%s for %s: %s.", provider.String(), selected.String(), platform.String(), err), 155 )) 156 continue 157 } 158 urlStr, ok := meta.Location.(getproviders.PackageHTTPURL) 159 if !ok { 160 // We don't expect to get non-HTTP locations here because we're 161 // using the registry source, so this seems like a bug in the 162 // registry source. 163 diags = diags.Append(tfdiags.Sourceless( 164 tfdiags.Error, 165 "Provider release not available", 166 fmt.Sprintf("Failed to download %s v%s for %s: Terraform's provider registry client returned unexpected location type %T. This is a bug in Terraform.", provider.String(), selected.String(), platform.String(), meta.Location), 167 )) 168 continue 169 } 170 urlObj, err := url.Parse(string(urlStr)) 171 if err != nil { 172 // We don't expect to get non-HTTP locations here because we're 173 // using the registry source, so this seems like a bug in the 174 // registry source. 175 diags = diags.Append(tfdiags.Sourceless( 176 tfdiags.Error, 177 "Invalid URL for provider release", 178 fmt.Sprintf("The origin registry for %s returned an invalid URL for v%s on %s: %s.", provider.String(), selected.String(), platform.String(), err), 179 )) 180 continue 181 } 182 // targetPath is the path where we ultimately want to place the 183 // downloaded archive, but we'll place it initially at stagingPath 184 // so we can verify its checksums and signatures before making 185 // it discoverable to mirror clients. (stagingPath intentionally 186 // does not follow the filesystem mirror file naming convention.) 187 targetPath := meta.PackedFilePath(outputDir) 188 stagingPath := filepath.Join(filepath.Dir(targetPath), "."+filepath.Base(targetPath)) 189 err = httpGetter.GetFile(stagingPath, urlObj) 190 if err != nil { 191 diags = diags.Append(tfdiags.Sourceless( 192 tfdiags.Error, 193 "Cannot download provider release", 194 fmt.Sprintf("Failed to download %s v%s for %s: %s.", provider.String(), selected.String(), platform.String(), err), 195 )) 196 continue 197 } 198 if meta.Authentication != nil { 199 result, err := meta.Authentication.AuthenticatePackage(getproviders.PackageLocalArchive(stagingPath)) 200 if err != nil { 201 diags = diags.Append(tfdiags.Sourceless( 202 tfdiags.Error, 203 "Invalid provider package", 204 fmt.Sprintf("Failed to authenticate %s v%s for %s: %s.", provider.String(), selected.String(), platform.String(), err), 205 )) 206 continue 207 } 208 c.Ui.Output(fmt.Sprintf(" - Package authenticated: %s", result)) 209 } 210 os.Remove(targetPath) // okay if it fails because we're going to try to rename over it next anyway 211 err = os.Rename(stagingPath, targetPath) 212 if err != nil { 213 diags = diags.Append(tfdiags.Sourceless( 214 tfdiags.Error, 215 "Cannot download provider release", 216 fmt.Sprintf("Failed to place %s package into mirror directory: %s.", provider.String(), err), 217 )) 218 continue 219 } 220 } 221 } 222 223 // Now we'll generate or update the JSON index files in the directory. 224 // We do this by scanning the directory to see what is present, rather than 225 // by relying on the selections we made above, because we want to still 226 // include in the indices any packages that were already present and 227 // not affected by the changes we just made. 228 available, err := getproviders.SearchLocalDirectory(outputDir) 229 if err != nil { 230 diags = diags.Append(tfdiags.Sourceless( 231 tfdiags.Error, 232 "Failed to update indexes", 233 fmt.Sprintf("Could not scan the output directory to get package metadata for the JSON indexes: %s.", err), 234 )) 235 available = nil // the following loop will be a no-op 236 } 237 for provider, metas := range available { 238 if len(metas) == 0 { 239 continue // should never happen, but we'll be resilient 240 } 241 // The index files live in the same directory as the package files, 242 // so to figure that out without duplicating the path-building logic 243 // we'll ask the getproviders package to build an archive filename 244 // for a fictitious package and then use the directory portion of it. 245 indexDir := filepath.Dir(getproviders.PackedFilePathForPackage( 246 outputDir, provider, versions.Unspecified, getproviders.CurrentPlatform, 247 )) 248 indexVersions := map[string]interface{}{} 249 indexArchives := map[getproviders.Version]map[string]interface{}{} 250 for _, meta := range metas { 251 archivePath, ok := meta.Location.(getproviders.PackageLocalArchive) 252 if !ok { 253 // only archive files are eligible to be included in JSON 254 // indices for a network mirror. 255 continue 256 } 257 archiveFilename := filepath.Base(string(archivePath)) 258 version := meta.Version 259 platform := meta.TargetPlatform 260 hash, err := meta.Hash() 261 if err != nil { 262 diags = diags.Append(tfdiags.Sourceless( 263 tfdiags.Error, 264 "Failed to update indexes", 265 fmt.Sprintf("Failed to determine a hash value for %s v%s on %s: %s.", provider, version, platform, err), 266 )) 267 continue 268 } 269 indexVersions[meta.Version.String()] = map[string]interface{}{} 270 if _, ok := indexArchives[version]; !ok { 271 indexArchives[version] = map[string]interface{}{} 272 } 273 indexArchives[version][platform.String()] = map[string]interface{}{ 274 "url": archiveFilename, // a relative URL from the index file's URL 275 "hashes": []string{hash.String()}, // an array to allow for additional hash formats in future 276 } 277 } 278 mainIndex := map[string]interface{}{ 279 "versions": indexVersions, 280 } 281 mainIndexJSON, err := json.MarshalIndent(mainIndex, "", " ") 282 if err != nil { 283 // Should never happen because the input here is entirely under 284 // our control. 285 panic(fmt.Sprintf("failed to encode main index: %s", err)) 286 } 287 // TODO: Ideally we would do these updates as atomic swap operations by 288 // creating a new file and then renaming it over the old one, in case 289 // this directory is the docroot of a live mirror. An atomic swap 290 // requires platform-specific code though: os.Rename alone can't do it 291 // when running on Windows as of Go 1.13. We should revisit this once 292 // we're supporting network mirrors, to avoid having them briefly 293 // become corrupted during updates. 294 err = ioutil.WriteFile(filepath.Join(indexDir, "index.json"), mainIndexJSON, 0644) 295 if err != nil { 296 diags = diags.Append(tfdiags.Sourceless( 297 tfdiags.Error, 298 "Failed to update indexes", 299 fmt.Sprintf("Failed to write an updated JSON index for %s: %s.", provider, err), 300 )) 301 } 302 for version, archiveIndex := range indexArchives { 303 versionIndex := map[string]interface{}{ 304 "archives": archiveIndex, 305 } 306 versionIndexJSON, err := json.MarshalIndent(versionIndex, "", " ") 307 if err != nil { 308 // Should never happen because the input here is entirely under 309 // our control. 310 panic(fmt.Sprintf("failed to encode version index: %s", err)) 311 } 312 err = ioutil.WriteFile(filepath.Join(indexDir, version.String()+".json"), versionIndexJSON, 0644) 313 if err != nil { 314 diags = diags.Append(tfdiags.Sourceless( 315 tfdiags.Error, 316 "Failed to update indexes", 317 fmt.Sprintf("Failed to write an updated JSON index for %s v%s: %s.", provider, version, err), 318 )) 319 } 320 } 321 } 322 323 c.showDiagnostics(diags) 324 if diags.HasErrors() { 325 return 1 326 } 327 return 0 328 } 329 330 func (c *ProvidersMirrorCommand) Help() string { 331 return ` 332 Usage: terraform [global options] providers mirror [options] <target-dir> 333 334 Populates a local directory with copies of the provider plugins needed for 335 the current configuration, so that the directory can be used either directly 336 as a filesystem mirror or as the basis for a network mirror and thus obtain 337 those providers without access to their origin registries in future. 338 339 The mirror directory will contain JSON index files that can be published 340 along with the mirrored packages on a static HTTP file server to produce 341 a network mirror. Those index files will be ignored if the directory is 342 used instead as a local filesystem mirror. 343 344 Options: 345 346 -platform=os_arch Choose which target platform to build a mirror for. 347 By default Terraform will obtain plugin packages 348 suitable for the platform where you run this command. 349 Use this flag multiple times to include packages for 350 multiple target systems. 351 352 Target names consist of an operating system and a CPU 353 architecture. For example, "linux_amd64" selects the 354 Linux operating system running on an AMD64 or x86_64 355 CPU. Each provider is available only for a limited 356 set of target platforms. 357 ` 358 }