github.com/opentofu/opentofu@v1.7.1/internal/command/providers_lock.go (about) 1 // Copyright (c) The OpenTofu Authors 2 // SPDX-License-Identifier: MPL-2.0 3 // Copyright (c) 2023 HashiCorp, Inc. 4 // SPDX-License-Identifier: MPL-2.0 5 6 package command 7 8 import ( 9 "fmt" 10 "net/url" 11 "os" 12 13 "github.com/opentofu/opentofu/internal/addrs" 14 "github.com/opentofu/opentofu/internal/depsfile" 15 "github.com/opentofu/opentofu/internal/getproviders" 16 "github.com/opentofu/opentofu/internal/providercache" 17 "github.com/opentofu/opentofu/internal/tfdiags" 18 ) 19 20 type providersLockChangeType string 21 22 const ( 23 providersLockChangeTypeNoChange providersLockChangeType = "providersLockChangeTypeNoChange" 24 providersLockChangeTypeNewProvider providersLockChangeType = "providersLockChangeTypeNewProvider" 25 providersLockChangeTypeNewHashes providersLockChangeType = "providersLockChangeTypeNewHashes" 26 ) 27 28 // ProvidersLockCommand is a Command implementation that implements the 29 // "tofu providers lock" command, which creates or updates the current 30 // configuration's dependency lock file using information from upstream 31 // registries, regardless of the provider installation configuration that 32 // is configured for normal provider installation. 33 type ProvidersLockCommand struct { 34 Meta 35 } 36 37 func (c *ProvidersLockCommand) Synopsis() string { 38 return "Write out dependency locks for the configured providers" 39 } 40 41 func (c *ProvidersLockCommand) Run(args []string) int { 42 args = c.Meta.process(args) 43 cmdFlags := c.Meta.defaultFlagSet("providers lock") 44 var optPlatforms FlagStringSlice 45 var fsMirrorDir string 46 var netMirrorURL string 47 cmdFlags.Var(&optPlatforms, "platform", "target platform") 48 cmdFlags.StringVar(&fsMirrorDir, "fs-mirror", "", "filesystem mirror directory") 49 cmdFlags.StringVar(&netMirrorURL, "net-mirror", "", "network mirror base URL") 50 cmdFlags.Usage = func() { c.Ui.Error(c.Help()) } 51 if err := cmdFlags.Parse(args); err != nil { 52 c.Ui.Error(fmt.Sprintf("Error parsing command-line flags: %s\n", err.Error())) 53 return 1 54 } 55 56 var diags tfdiags.Diagnostics 57 58 if fsMirrorDir != "" && netMirrorURL != "" { 59 diags = diags.Append(tfdiags.Sourceless( 60 tfdiags.Error, 61 "Invalid installation method options", 62 "The -fs-mirror and -net-mirror command line options are mutually-exclusive.", 63 )) 64 c.showDiagnostics(diags) 65 return 1 66 } 67 68 providerStrs := cmdFlags.Args() 69 70 var platforms []getproviders.Platform 71 if len(optPlatforms) == 0 { 72 platforms = []getproviders.Platform{getproviders.CurrentPlatform} 73 } else { 74 platforms = make([]getproviders.Platform, 0, len(optPlatforms)) 75 for _, platformStr := range optPlatforms { 76 platform, err := getproviders.ParsePlatform(platformStr) 77 if err != nil { 78 diags = diags.Append(tfdiags.Sourceless( 79 tfdiags.Error, 80 "Invalid target platform", 81 fmt.Sprintf("The string %q given in the -platform option is not a valid target platform: %s.", platformStr, err), 82 )) 83 continue 84 } 85 platforms = append(platforms, platform) 86 } 87 } 88 89 // Installation steps can be cancelled by SIGINT and similar. 90 ctx, done := c.InterruptibleContext(c.CommandContext()) 91 defer done() 92 93 // Unlike other commands, this command ignores the installation methods 94 // selected in the CLI configuration and instead chooses an installation 95 // method based on CLI options. 96 // 97 // This is so that folks who use a local mirror for everyday use can 98 // use this command to populate their lock files from upstream so 99 // subsequent "tofu init" calls can then verify the local mirror 100 // against the upstream checksums. 101 var source getproviders.Source 102 switch { 103 case fsMirrorDir != "": 104 source = getproviders.NewFilesystemMirrorSource(fsMirrorDir) 105 case netMirrorURL != "": 106 u, err := url.Parse(netMirrorURL) 107 if err != nil || u.Scheme != "https" { 108 diags = diags.Append(tfdiags.Sourceless( 109 tfdiags.Error, 110 "Invalid network mirror URL", 111 "The -net-mirror option requires a valid https: URL as the mirror base URL.", 112 )) 113 c.showDiagnostics(diags) 114 return 1 115 } 116 source = getproviders.NewHTTPMirrorSource(u, c.Services.CredentialsSource()) 117 default: 118 // With no special options we consult upstream registries directly, 119 // because that gives us the most information to produce as complete 120 // and portable as possible a lock entry. 121 source = getproviders.NewRegistrySource(c.Services) 122 } 123 124 config, confDiags := c.loadConfig(".") 125 diags = diags.Append(confDiags) 126 reqs, hclDiags := config.ProviderRequirements() 127 diags = diags.Append(hclDiags) 128 129 // If we have explicit provider selections on the command line then 130 // we'll modify "reqs" to only include those. Modifying this is okay 131 // because config.ProviderRequirements generates a fresh map result 132 // for each call. 133 if len(providerStrs) != 0 { 134 providers := map[addrs.Provider]struct{}{} 135 for _, raw := range providerStrs { 136 addr, moreDiags := addrs.ParseProviderSourceString(raw) 137 diags = diags.Append(moreDiags) 138 if moreDiags.HasErrors() { 139 continue 140 } 141 providers[addr] = struct{}{} 142 if _, exists := reqs[addr]; !exists { 143 // Can't request a provider that isn't required by the 144 // current configuration. 145 diags = diags.Append(tfdiags.Sourceless( 146 tfdiags.Error, 147 "Invalid provider argument", 148 fmt.Sprintf("The provider %s is not required by the current configuration.", addr.String()), 149 )) 150 } 151 } 152 153 for addr := range reqs { 154 if _, exists := providers[addr]; !exists { 155 delete(reqs, addr) 156 } 157 } 158 } 159 160 // We'll also ignore any providers that don't participate in locking. 161 for addr := range reqs { 162 if !depsfile.ProviderIsLockable(addr) { 163 delete(reqs, addr) 164 } 165 } 166 167 // We'll start our work with whatever locks we already have, so that 168 // we'll honor any existing version selections and just add additional 169 // hashes for them. 170 oldLocks, moreDiags := c.lockedDependencies() 171 diags = diags.Append(moreDiags) 172 173 // If we have any error diagnostics already then we won't proceed further. 174 if diags.HasErrors() { 175 c.showDiagnostics(diags) 176 return 1 177 } 178 179 // Our general strategy here is to install the requested providers into 180 // a separate temporary directory -- thus ensuring that the results won't 181 // ever be inadvertently executed by other OpenTofu commands -- and then 182 // use the results of that installation to update the lock file for the 183 // current working directory. Because we throwaway the packages we 184 // downloaded after completing our work, a subsequent "tofu init" will 185 // then respect the CLI configuration's provider installation strategies 186 // but will verify the packages against the hashes we found upstream. 187 188 // Because our Installer abstraction is a per-platform idea, we'll 189 // instantiate one for each of the platforms the user requested, and then 190 // merge all of the generated locks together at the end. 191 updatedLocks := map[getproviders.Platform]*depsfile.Locks{} 192 selectedVersions := map[addrs.Provider]getproviders.Version{} 193 for _, platform := range platforms { 194 tempDir, err := os.MkdirTemp("", "terraform-providers-lock") 195 if err != nil { 196 diags = diags.Append(tfdiags.Sourceless( 197 tfdiags.Error, 198 "Could not create temporary directory", 199 fmt.Sprintf("Failed to create a temporary directory for staging the requested provider packages: %s.", err), 200 )) 201 break 202 } 203 defer os.RemoveAll(tempDir) 204 205 evts := &providercache.InstallerEvents{ 206 // Our output from this command is minimal just to show that 207 // we're making progress, rather than just silently hanging. 208 FetchPackageBegin: func(provider addrs.Provider, version getproviders.Version, loc getproviders.PackageLocation) { 209 c.Ui.Output(fmt.Sprintf("- Fetching %s %s for %s...", provider.ForDisplay(), version, platform)) 210 if prevVersion, exists := selectedVersions[provider]; exists && version != prevVersion { 211 // This indicates a weird situation where we ended up 212 // selecting a different version for one platform than 213 // for another. We won't be able to merge the result 214 // in that case, so we'll generate an error. 215 // 216 // This could potentially happen if there's a provider 217 // we've not previously recorded in the lock file and 218 // the available versions change while we're running. To 219 // avoid that would require pre-locking all of the 220 // providers, which is complicated to do with the building 221 // blocks we have here, and so we'll wait to do it only 222 // if this situation arises often in practice. 223 diags = diags.Append(tfdiags.Sourceless( 224 tfdiags.Error, 225 "Inconsistent provider versions", 226 fmt.Sprintf( 227 "The version constraint for %s selected inconsistent versions for different platforms, which is unexpected.\n\nThe upstream registry may have changed its available versions during OpenTofu's work. If so, re-running this command may produce a successful result.", 228 provider, 229 ), 230 )) 231 } 232 selectedVersions[provider] = version 233 }, 234 FetchPackageSuccess: func(provider addrs.Provider, version getproviders.Version, localDir string, auth *getproviders.PackageAuthenticationResult) { 235 var keyID string 236 if auth != nil && auth.Signed() { 237 keyID = auth.KeyID 238 } 239 if keyID != "" { 240 keyID = c.Colorize().Color(fmt.Sprintf(", key ID [reset][bold]%s[reset]", keyID)) 241 } 242 c.Ui.Output(fmt.Sprintf("- Retrieved %s %s for %s (%s%s)", provider.ForDisplay(), version, platform, auth, keyID)) 243 }, 244 } 245 ctx := evts.OnContext(ctx) 246 247 dir := providercache.NewDirWithPlatform(tempDir, platform) 248 installer := providercache.NewInstaller(dir, source) 249 250 newLocks, err := installer.EnsureProviderVersions(ctx, oldLocks, reqs, providercache.InstallNewProvidersForce) 251 if err != nil { 252 diags = diags.Append(tfdiags.Sourceless( 253 tfdiags.Error, 254 "Could not retrieve providers for locking", 255 fmt.Sprintf("OpenTofu failed to fetch the requested providers for %s in order to calculate their checksums: %s.", platform, err), 256 )) 257 break 258 } 259 updatedLocks[platform] = newLocks 260 } 261 262 // If we have any error diagnostics from installation then we won't 263 // proceed to merging and updating the lock file on disk. 264 if diags.HasErrors() { 265 c.showDiagnostics(diags) 266 return 1 267 } 268 269 // Track whether we've made any changes to the lock file as part of this 270 // operation. We can customise the final message based on our actions. 271 madeAnyChange := false 272 273 // We now have a separate updated locks object for each platform. We need 274 // to merge those all together so that the final result has the union of 275 // all of the checksums we saw for each of the providers we've worked on. 276 // 277 // We'll copy the old locks first because we want to retain any existing 278 // locks for providers that we _didn't_ visit above. 279 newLocks := oldLocks.DeepCopy() 280 for provider := range reqs { 281 oldLock := oldLocks.Provider(provider) 282 283 var version getproviders.Version 284 var constraints getproviders.VersionConstraints 285 var hashes []getproviders.Hash 286 if oldLock != nil { 287 version = oldLock.Version() 288 constraints = oldLock.VersionConstraints() 289 hashes = append(hashes, oldLock.AllHashes()...) 290 } 291 for platform, platformLocks := range updatedLocks { 292 platformLock := platformLocks.Provider(provider) 293 if platformLock == nil { 294 continue // weird, but we'll tolerate it to avoid crashing 295 } 296 version = platformLock.Version() 297 constraints = platformLock.VersionConstraints() 298 299 // We don't make any effort to deduplicate hashes between different 300 // platforms here, because the SetProvider method we call below 301 // handles that automatically. 302 hashes = append(hashes, platformLock.AllHashes()...) 303 304 // At this point, we've merged all the hashes for this (provider, platform) 305 // combo into the combined hashes for this provider. Let's take this 306 // opportunity to print out a summary for this particular combination. 307 switch providersLockCalculateChangeType(oldLock, platformLock) { 308 case providersLockChangeTypeNewProvider: 309 madeAnyChange = true 310 c.Ui.Output( 311 fmt.Sprintf( 312 "- Obtained %s checksums for %s; This was a new provider and the checksums for this platform are now tracked in the lock file", 313 provider.ForDisplay(), 314 platform)) 315 case providersLockChangeTypeNewHashes: 316 madeAnyChange = true 317 c.Ui.Output( 318 fmt.Sprintf( 319 "- Obtained %s checksums for %s; Additional checksums for this platform are now tracked in the lock file", 320 provider.ForDisplay(), 321 platform)) 322 case providersLockChangeTypeNoChange: 323 c.Ui.Output( 324 fmt.Sprintf( 325 "- Obtained %s checksums for %s; All checksums for this platform were already tracked in the lock file", 326 provider.ForDisplay(), 327 platform)) 328 } 329 } 330 newLocks.SetProvider(provider, version, constraints, hashes) 331 } 332 333 moreDiags = c.replaceLockedDependencies(newLocks) 334 diags = diags.Append(moreDiags) 335 336 c.showDiagnostics(diags) 337 if diags.HasErrors() { 338 return 1 339 } 340 341 if madeAnyChange { 342 c.Ui.Output(c.Colorize().Color("\n[bold][green]Success![reset] [bold]OpenTofu has updated the lock file.[reset]")) 343 c.Ui.Output("\nReview the changes in .terraform.lock.hcl and then commit to your\nversion control system to retain the new checksums.\n") 344 } else { 345 c.Ui.Output(c.Colorize().Color("\n[bold][green]Success![reset] [bold]OpenTofu has validated the lock file and found no need for changes.[reset]")) 346 } 347 return 0 348 } 349 350 func (c *ProvidersLockCommand) Help() string { 351 return ` 352 Usage: tofu [global options] providers lock [options] [providers...] 353 354 Normally the dependency lock file (.terraform.lock.hcl) is updated 355 automatically by "tofu init", but the information available to the 356 normal provider installer can be constrained when you're installing providers 357 from filesystem or network mirrors, and so the generated lock file can end 358 up incomplete. 359 360 The "providers lock" subcommand addresses that by updating the lock file 361 based on the official packages available in the origin registry, ignoring 362 the currently-configured installation strategy. 363 364 After this command succeeds, the lock file will contain suitable checksums 365 to allow installation of the providers needed by the current configuration 366 on all of the selected platforms. 367 368 By default this command updates the lock file for every provider declared 369 in the configuration. You can override that behavior by providing one or 370 more provider source addresses on the command line. 371 372 Options: 373 374 -fs-mirror=dir Consult the given filesystem mirror directory instead 375 of the origin registry for each of the given providers. 376 377 This would be necessary to generate lock file entries for 378 a provider that is available only via a mirror, and not 379 published in an upstream registry. In this case, the set 380 of valid checksums will be limited only to what OpenTofu 381 can learn from the data in the mirror directory. 382 383 -net-mirror=url Consult the given network mirror (given as a base URL) 384 instead of the origin registry for each of the given 385 providers. 386 387 This would be necessary to generate lock file entries for 388 a provider that is available only via a mirror, and not 389 published in an upstream registry. In this case, the set 390 of valid checksums will be limited only to what OpenTofu 391 can learn from the data in the mirror indices. 392 393 -platform=os_arch Choose a target platform to request package checksums 394 for. 395 396 By default OpenTofu will request package checksums 397 suitable only for the platform where you run this 398 command. Use this option multiple times to include 399 checksums for multiple target systems. 400 401 Target names consist of an operating system and a CPU 402 architecture. For example, "linux_amd64" selects the 403 Linux operating system running on an AMD64 or x86_64 404 CPU. Each provider is available only for a limited 405 set of target platforms. 406 ` 407 } 408 409 // providersLockCalculateChangeType works out whether there is any difference 410 // between oldLock and newLock and returns a variable the main function can use 411 // to decide on which message to print. 412 // 413 // One assumption made here that is not obvious without the context from the 414 // main function is that while platformLock contains the lock information for a 415 // single platform after the current run, oldLock contains the combined 416 // information of all platforms from when the versions were last checked. A 417 // simple equality check is not sufficient for deciding on change as we expect 418 // that oldLock will be a superset of platformLock if no new hashes have been 419 // found. 420 // 421 // We've separated this function out so we can write unit tests around the 422 // logic. This function assumes the platformLock is not nil, as the main 423 // function explicitly checks this before calling this function. 424 func providersLockCalculateChangeType(oldLock *depsfile.ProviderLock, platformLock *depsfile.ProviderLock) providersLockChangeType { 425 if oldLock == nil { 426 return providersLockChangeTypeNewProvider 427 } 428 if oldLock.ContainsAll(platformLock) { 429 return providersLockChangeTypeNoChange 430 } 431 return providersLockChangeTypeNewHashes 432 }