github.com/opentofu/opentofu@v1.7.1/internal/command/providers_mirror.go (about) 1 // Copyright (c) The OpenTofu Authors 2 // SPDX-License-Identifier: MPL-2.0 3 // Copyright (c) 2023 HashiCorp, Inc. 4 // SPDX-License-Identifier: MPL-2.0 5 6 package command 7 8 import ( 9 "encoding/json" 10 "fmt" 11 "net/url" 12 "os" 13 "path/filepath" 14 15 "github.com/apparentlymart/go-versions/versions" 16 "github.com/hashicorp/go-getter" 17 18 "github.com/opentofu/opentofu/internal/getproviders" 19 "github.com/opentofu/opentofu/internal/httpclient" 20 "github.com/opentofu/opentofu/internal/tfdiags" 21 ) 22 23 // ProvidersMirrorCommand is a Command implementation that implements the 24 // "tofu providers mirror" command, which populates a directory with 25 // local copies of provider plugins needed by the current configuration so 26 // that the mirror can be used to work offline, or similar. 27 type ProvidersMirrorCommand struct { 28 Meta 29 } 30 31 func (c *ProvidersMirrorCommand) Synopsis() string { 32 return "Save local copies of all required provider plugins" 33 } 34 35 func (c *ProvidersMirrorCommand) Run(args []string) int { 36 args = c.Meta.process(args) 37 cmdFlags := c.Meta.defaultFlagSet("providers mirror") 38 var optPlatforms FlagStringSlice 39 cmdFlags.Var(&optPlatforms, "platform", "target platform") 40 cmdFlags.Usage = func() { c.Ui.Error(c.Help()) } 41 if err := cmdFlags.Parse(args); err != nil { 42 c.Ui.Error(fmt.Sprintf("Error parsing command-line flags: %s\n", err.Error())) 43 return 1 44 } 45 46 var diags tfdiags.Diagnostics 47 48 args = cmdFlags.Args() 49 if len(args) != 1 { 50 diags = diags.Append(tfdiags.Sourceless( 51 tfdiags.Error, 52 "No output directory specified", 53 "The providers mirror command requires an output directory as a command-line argument.", 54 )) 55 c.showDiagnostics(diags) 56 return 1 57 } 58 outputDir := args[0] 59 60 var platforms []getproviders.Platform 61 if len(optPlatforms) == 0 { 62 platforms = []getproviders.Platform{getproviders.CurrentPlatform} 63 } else { 64 platforms = make([]getproviders.Platform, 0, len(optPlatforms)) 65 for _, platformStr := range optPlatforms { 66 platform, err := getproviders.ParsePlatform(platformStr) 67 if err != nil { 68 diags = diags.Append(tfdiags.Sourceless( 69 tfdiags.Error, 70 "Invalid target platform", 71 fmt.Sprintf("The string %q given in the -platform option is not a valid target platform: %s.", platformStr, err), 72 )) 73 continue 74 } 75 platforms = append(platforms, platform) 76 } 77 } 78 79 // Installation steps can be cancelled by SIGINT and similar. 80 ctx, done := c.InterruptibleContext(c.CommandContext()) 81 defer done() 82 83 config, confDiags := c.loadConfig(".") 84 diags = diags.Append(confDiags) 85 reqs, moreDiags := config.ProviderRequirements() 86 diags = diags.Append(moreDiags) 87 88 // Read lock file 89 lockedDeps, lockedDepsDiags := c.Meta.lockedDependencies() 90 diags = diags.Append(lockedDepsDiags) 91 92 // If we have any error diagnostics already then we won't proceed further. 93 if diags.HasErrors() { 94 c.showDiagnostics(diags) 95 return 1 96 } 97 98 // If lock file is present, validate it against configuration 99 if !lockedDeps.Empty() { 100 if errs := config.VerifyDependencySelections(lockedDeps); len(errs) > 0 { 101 diags = diags.Append(tfdiags.Sourceless( 102 tfdiags.Error, 103 "Inconsistent dependency lock file", 104 fmt.Sprintf("To update the locked dependency selections to match a changed configuration, run:\n tofu init -upgrade\n got:%v", errs), 105 )) 106 } 107 } 108 109 // Unlike other commands, this command always consults the origin registry 110 // for every provider so that it can be used to update a local mirror 111 // directory without needing to first disable that local mirror 112 // in the CLI configuration. 113 source := getproviders.NewMemoizeSource( 114 getproviders.NewRegistrySource(c.Services), 115 ) 116 117 // Providers from registries always use HTTP, so we don't need the full 118 // generality of go-getter but it's still handy to use the HTTP getter 119 // as an easy way to download over HTTP into a file on disk. 120 httpGetter := getter.HttpGetter{ 121 Client: httpclient.New(), 122 Netrc: true, 123 XTerraformGetDisabled: true, 124 } 125 126 // The following logic is similar to that used by the provider installer 127 // in package providercache, but different in a few ways: 128 // - It produces the packed directory layout rather than the unpacked 129 // layout we require in provider cache directories. 130 // - It generates JSON index files that can be read by the 131 // getproviders.HTTPMirrorSource installation method if the result were 132 // copied into the docroot of an HTTP server. 133 // - It can mirror packages for potentially many different target platforms, 134 // so that we can construct a multi-platform mirror regardless of which 135 // platform we run this command on. 136 // - It ignores what's already present and just always downloads everything 137 // that the configuration requires. This is a command intended to be run 138 // infrequently to update a mirror, so it doesn't need to optimize away 139 // fetches of packages that might already be present. 140 141 for provider, constraints := range reqs { 142 if provider.IsBuiltIn() { 143 c.Ui.Output(fmt.Sprintf("- Skipping %s because it is built in to OpenTofu CLI", provider.ForDisplay())) 144 continue 145 } 146 constraintsStr := getproviders.VersionConstraintsString(constraints) 147 c.Ui.Output(fmt.Sprintf("- Mirroring %s...", provider.ForDisplay())) 148 // First we'll look for the latest version that matches the given 149 // constraint, which we'll then try to mirror for each target platform. 150 acceptable := versions.MeetingConstraints(constraints) 151 avail, _, err := source.AvailableVersions(ctx, provider) 152 candidates := avail.Filter(acceptable) 153 if err == nil && len(candidates) == 0 { 154 err = fmt.Errorf("no releases match the given constraints %s", constraintsStr) 155 } 156 if err != nil { 157 diags = diags.Append(tfdiags.Sourceless( 158 tfdiags.Error, 159 "Provider not available", 160 fmt.Sprintf("Failed to download %s from its origin registry: %s.", provider.String(), err), 161 )) 162 continue 163 } 164 selected := candidates.Newest() 165 if !lockedDeps.Empty() { 166 selected = lockedDeps.Provider(provider).Version() 167 c.Ui.Output(fmt.Sprintf(" - Selected v%s to match dependency lock file", selected.String())) 168 } else if len(constraintsStr) > 0 { 169 c.Ui.Output(fmt.Sprintf(" - Selected v%s to meet constraints %s", selected.String(), constraintsStr)) 170 } else { 171 c.Ui.Output(fmt.Sprintf(" - Selected v%s with no constraints", selected.String())) 172 } 173 for _, platform := range platforms { 174 c.Ui.Output(fmt.Sprintf(" - Downloading package for %s...", platform.String())) 175 meta, err := source.PackageMeta(ctx, provider, selected, platform) 176 if err != nil { 177 diags = diags.Append(tfdiags.Sourceless( 178 tfdiags.Error, 179 "Provider release not available", 180 fmt.Sprintf("Failed to download %s v%s for %s: %s.", provider.String(), selected.String(), platform.String(), err), 181 )) 182 continue 183 } 184 urlStr, ok := meta.Location.(getproviders.PackageHTTPURL) 185 if !ok { 186 // We don't expect to get non-HTTP locations here because we're 187 // using the registry source, so this seems like a bug in the 188 // registry source. 189 diags = diags.Append(tfdiags.Sourceless( 190 tfdiags.Error, 191 "Provider release not available", 192 fmt.Sprintf("Failed to download %s v%s for %s: OpenTofu's provider registry client returned unexpected location type %T. This is a bug in OpenTofu.", provider.String(), selected.String(), platform.String(), meta.Location), 193 )) 194 continue 195 } 196 urlObj, err := url.Parse(string(urlStr)) 197 if err != nil { 198 // We don't expect to get non-HTTP locations here because we're 199 // using the registry source, so this seems like a bug in the 200 // registry source. 201 diags = diags.Append(tfdiags.Sourceless( 202 tfdiags.Error, 203 "Invalid URL for provider release", 204 fmt.Sprintf("The origin registry for %s returned an invalid URL for v%s on %s: %s.", provider.String(), selected.String(), platform.String(), err), 205 )) 206 continue 207 } 208 // targetPath is the path where we ultimately want to place the 209 // downloaded archive, but we'll place it initially at stagingPath 210 // so we can verify its checksums and signatures before making 211 // it discoverable to mirror clients. (stagingPath intentionally 212 // does not follow the filesystem mirror file naming convention.) 213 targetPath := meta.PackedFilePath(outputDir) 214 stagingPath := filepath.Join(filepath.Dir(targetPath), "."+filepath.Base(targetPath)) 215 err = httpGetter.GetFile(stagingPath, urlObj) 216 if err != nil { 217 diags = diags.Append(tfdiags.Sourceless( 218 tfdiags.Error, 219 "Cannot download provider release", 220 fmt.Sprintf("Failed to download %s v%s for %s: %s.", provider.String(), selected.String(), platform.String(), err), 221 )) 222 continue 223 } 224 if meta.Authentication != nil { 225 result, err := meta.Authentication.AuthenticatePackage(getproviders.PackageLocalArchive(stagingPath)) 226 if err != nil { 227 diags = diags.Append(tfdiags.Sourceless( 228 tfdiags.Error, 229 "Invalid provider package", 230 fmt.Sprintf("Failed to authenticate %s v%s for %s: %s.", provider.String(), selected.String(), platform.String(), err), 231 )) 232 continue 233 } 234 c.Ui.Output(fmt.Sprintf(" - Package authenticated: %s", result)) 235 } 236 os.Remove(targetPath) // okay if it fails because we're going to try to rename over it next anyway 237 err = os.Rename(stagingPath, targetPath) 238 if err != nil { 239 diags = diags.Append(tfdiags.Sourceless( 240 tfdiags.Error, 241 "Cannot download provider release", 242 fmt.Sprintf("Failed to place %s package into mirror directory: %s.", provider.String(), err), 243 )) 244 continue 245 } 246 } 247 } 248 249 // Now we'll generate or update the JSON index files in the directory. 250 // We do this by scanning the directory to see what is present, rather than 251 // by relying on the selections we made above, because we want to still 252 // include in the indices any packages that were already present and 253 // not affected by the changes we just made. 254 available, err := getproviders.SearchLocalDirectory(outputDir) 255 if err != nil { 256 diags = diags.Append(tfdiags.Sourceless( 257 tfdiags.Error, 258 "Failed to update indexes", 259 fmt.Sprintf("Could not scan the output directory to get package metadata for the JSON indexes: %s.", err), 260 )) 261 available = nil // the following loop will be a no-op 262 } 263 for provider, metas := range available { 264 if len(metas) == 0 { 265 continue // should never happen, but we'll be resilient 266 } 267 // The index files live in the same directory as the package files, 268 // so to figure that out without duplicating the path-building logic 269 // we'll ask the getproviders package to build an archive filename 270 // for a fictitious package and then use the directory portion of it. 271 indexDir := filepath.Dir(getproviders.PackedFilePathForPackage( 272 outputDir, provider, versions.Unspecified, getproviders.CurrentPlatform, 273 )) 274 indexVersions := map[string]interface{}{} 275 indexArchives := map[getproviders.Version]map[string]interface{}{} 276 for _, meta := range metas { 277 archivePath, ok := meta.Location.(getproviders.PackageLocalArchive) 278 if !ok { 279 // only archive files are eligible to be included in JSON 280 // indices for a network mirror. 281 continue 282 } 283 archiveFilename := filepath.Base(string(archivePath)) 284 version := meta.Version 285 platform := meta.TargetPlatform 286 hash, err := meta.Hash() 287 if err != nil { 288 diags = diags.Append(tfdiags.Sourceless( 289 tfdiags.Error, 290 "Failed to update indexes", 291 fmt.Sprintf("Failed to determine a hash value for %s v%s on %s: %s.", provider, version, platform, err), 292 )) 293 continue 294 } 295 indexVersions[meta.Version.String()] = map[string]interface{}{} 296 if _, ok := indexArchives[version]; !ok { 297 indexArchives[version] = map[string]interface{}{} 298 } 299 indexArchives[version][platform.String()] = map[string]interface{}{ 300 "url": archiveFilename, // a relative URL from the index file's URL 301 "hashes": []string{hash.String()}, // an array to allow for additional hash formats in future 302 } 303 } 304 mainIndex := map[string]interface{}{ 305 "versions": indexVersions, 306 } 307 mainIndexJSON, err := json.MarshalIndent(mainIndex, "", " ") 308 if err != nil { 309 // Should never happen because the input here is entirely under 310 // our control. 311 panic(fmt.Sprintf("failed to encode main index: %s", err)) 312 } 313 // TODO: Ideally we would do these updates as atomic swap operations by 314 // creating a new file and then renaming it over the old one, in case 315 // this directory is the docroot of a live mirror. An atomic swap 316 // requires platform-specific code though: os.Rename alone can't do it 317 // when running on Windows as of Go 1.13. We should revisit this once 318 // we're supporting network mirrors, to avoid having them briefly 319 // become corrupted during updates. 320 err = os.WriteFile(filepath.Join(indexDir, "index.json"), mainIndexJSON, 0644) 321 if err != nil { 322 diags = diags.Append(tfdiags.Sourceless( 323 tfdiags.Error, 324 "Failed to update indexes", 325 fmt.Sprintf("Failed to write an updated JSON index for %s: %s.", provider, err), 326 )) 327 } 328 for version, archiveIndex := range indexArchives { 329 versionIndex := map[string]interface{}{ 330 "archives": archiveIndex, 331 } 332 versionIndexJSON, err := json.MarshalIndent(versionIndex, "", " ") 333 if err != nil { 334 // Should never happen because the input here is entirely under 335 // our control. 336 panic(fmt.Sprintf("failed to encode version index: %s", err)) 337 } 338 err = os.WriteFile(filepath.Join(indexDir, version.String()+".json"), versionIndexJSON, 0644) 339 if err != nil { 340 diags = diags.Append(tfdiags.Sourceless( 341 tfdiags.Error, 342 "Failed to update indexes", 343 fmt.Sprintf("Failed to write an updated JSON index for %s v%s: %s.", provider, version, err), 344 )) 345 } 346 } 347 } 348 349 c.showDiagnostics(diags) 350 if diags.HasErrors() { 351 return 1 352 } 353 return 0 354 } 355 356 func (c *ProvidersMirrorCommand) Help() string { 357 return ` 358 Usage: tofu [global options] providers mirror [options] <target-dir> 359 360 Populates a local directory with copies of the provider plugins needed for 361 the current configuration, so that the directory can be used either directly 362 as a filesystem mirror or as the basis for a network mirror and thus obtain 363 those providers without access to their origin registries in future. 364 365 The mirror directory will contain JSON index files that can be published 366 along with the mirrored packages on a static HTTP file server to produce 367 a network mirror. Those index files will be ignored if the directory is 368 used instead as a local filesystem mirror. 369 370 Options: 371 372 -platform=os_arch Choose which target platform to build a mirror for. 373 By default OpenTofu will obtain plugin packages 374 suitable for the platform where you run this command. 375 Use this flag multiple times to include packages for 376 multiple target systems. 377 378 Target names consist of an operating system and a CPU 379 architecture. For example, "linux_amd64" selects the 380 Linux operating system running on an AMD64 or x86_64 381 CPU. Each provider is available only for a limited 382 set of target platforms. 383 ` 384 }