github.com/kevinklinger/open_terraform@v1.3.6/noninternal/command/providers_lock.go (about) 1 package command 2 3 import ( 4 "fmt" 5 "io/ioutil" 6 "net/url" 7 "os" 8 9 "github.com/kevinklinger/open_terraform/noninternal/addrs" 10 "github.com/kevinklinger/open_terraform/noninternal/depsfile" 11 "github.com/kevinklinger/open_terraform/noninternal/getproviders" 12 "github.com/kevinklinger/open_terraform/noninternal/providercache" 13 "github.com/kevinklinger/open_terraform/noninternal/tfdiags" 14 ) 15 16 type providersLockChangeType string 17 18 const ( 19 providersLockChangeTypeNoChange providersLockChangeType = "providersLockChangeTypeNoChange" 20 providersLockChangeTypeNewProvider providersLockChangeType = "providersLockChangeTypeNewProvider" 21 providersLockChangeTypeNewHashes providersLockChangeType = "providersLockChangeTypeNewHashes" 22 ) 23 24 // ProvidersLockCommand is a Command implementation that implements the 25 // "terraform providers lock" command, which creates or updates the current 26 // configuration's dependency lock file using information from upstream 27 // registries, regardless of the provider installation configuration that 28 // is configured for normal provider installation. 29 type ProvidersLockCommand struct { 30 Meta 31 } 32 33 func (c *ProvidersLockCommand) Synopsis() string { 34 return "Write out dependency locks for the configured providers" 35 } 36 37 func (c *ProvidersLockCommand) Run(args []string) int { 38 args = c.Meta.process(args) 39 cmdFlags := c.Meta.defaultFlagSet("providers lock") 40 var optPlatforms FlagStringSlice 41 var fsMirrorDir string 42 var netMirrorURL string 43 cmdFlags.Var(&optPlatforms, "platform", "target platform") 44 cmdFlags.StringVar(&fsMirrorDir, "fs-mirror", "", "filesystem mirror directory") 45 cmdFlags.StringVar(&netMirrorURL, "net-mirror", "", "network mirror base URL") 46 cmdFlags.Usage = func() { c.Ui.Error(c.Help()) } 47 if err := cmdFlags.Parse(args); err != nil { 48 c.Ui.Error(fmt.Sprintf("Error parsing command-line flags: %s\n", err.Error())) 49 return 1 50 } 51 52 var diags tfdiags.Diagnostics 53 54 if fsMirrorDir != "" && netMirrorURL != "" { 55 diags = diags.Append(tfdiags.Sourceless( 56 tfdiags.Error, 57 "Invalid installation method options", 58 "The -fs-mirror and -net-mirror command line options are mutually-exclusive.", 59 )) 60 c.showDiagnostics(diags) 61 return 1 62 } 63 64 providerStrs := cmdFlags.Args() 65 66 var platforms []getproviders.Platform 67 if len(optPlatforms) == 0 { 68 platforms = []getproviders.Platform{getproviders.CurrentPlatform} 69 } else { 70 platforms = make([]getproviders.Platform, 0, len(optPlatforms)) 71 for _, platformStr := range optPlatforms { 72 platform, err := getproviders.ParsePlatform(platformStr) 73 if err != nil { 74 diags = diags.Append(tfdiags.Sourceless( 75 tfdiags.Error, 76 "Invalid target platform", 77 fmt.Sprintf("The string %q given in the -platform option is not a valid target platform: %s.", platformStr, err), 78 )) 79 continue 80 } 81 platforms = append(platforms, platform) 82 } 83 } 84 85 // Unlike other commands, this command ignores the installation methods 86 // selected in the CLI configuration and instead chooses an installation 87 // method based on CLI options. 88 // 89 // This is so that folks who use a local mirror for everyday use can 90 // use this command to populate their lock files from upstream so 91 // subsequent "terraform init" calls can then verify the local mirror 92 // against the upstream checksums. 93 var source getproviders.Source 94 switch { 95 case fsMirrorDir != "": 96 source = getproviders.NewFilesystemMirrorSource(fsMirrorDir) 97 case netMirrorURL != "": 98 u, err := url.Parse(netMirrorURL) 99 if err != nil || u.Scheme != "https" { 100 diags = diags.Append(tfdiags.Sourceless( 101 tfdiags.Error, 102 "Invalid network mirror URL", 103 "The -net-mirror option requires a valid https: URL as the mirror base URL.", 104 )) 105 c.showDiagnostics(diags) 106 return 1 107 } 108 source = getproviders.NewHTTPMirrorSource(u, c.Services.CredentialsSource()) 109 default: 110 // With no special options we consult upstream registries directly, 111 // because that gives us the most information to produce as complete 112 // and portable as possible a lock entry. 113 source = getproviders.NewRegistrySource(c.Services) 114 } 115 116 config, confDiags := c.loadConfig(".") 117 diags = diags.Append(confDiags) 118 reqs, hclDiags := config.ProviderRequirements() 119 diags = diags.Append(hclDiags) 120 121 // If we have explicit provider selections on the command line then 122 // we'll modify "reqs" to only include those. Modifying this is okay 123 // because config.ProviderRequirements generates a fresh map result 124 // for each call. 125 if len(providerStrs) != 0 { 126 providers := map[addrs.Provider]struct{}{} 127 for _, raw := range providerStrs { 128 addr, moreDiags := addrs.ParseProviderSourceString(raw) 129 diags = diags.Append(moreDiags) 130 if moreDiags.HasErrors() { 131 continue 132 } 133 providers[addr] = struct{}{} 134 if _, exists := reqs[addr]; !exists { 135 // Can't request a provider that isn't required by the 136 // current configuration. 137 diags = diags.Append(tfdiags.Sourceless( 138 tfdiags.Error, 139 "Invalid provider argument", 140 fmt.Sprintf("The provider %s is not required by the current configuration.", addr.String()), 141 )) 142 } 143 } 144 145 for addr := range reqs { 146 if _, exists := providers[addr]; !exists { 147 delete(reqs, addr) 148 } 149 } 150 } 151 152 // We'll also ignore any providers that don't participate in locking. 153 for addr := range reqs { 154 if !depsfile.ProviderIsLockable(addr) { 155 delete(reqs, addr) 156 } 157 } 158 159 // We'll start our work with whatever locks we already have, so that 160 // we'll honor any existing version selections and just add additional 161 // hashes for them. 162 oldLocks, moreDiags := c.lockedDependencies() 163 diags = diags.Append(moreDiags) 164 165 // If we have any error diagnostics already then we won't proceed further. 166 if diags.HasErrors() { 167 c.showDiagnostics(diags) 168 return 1 169 } 170 171 // Our general strategy here is to install the requested providers into 172 // a separate temporary directory -- thus ensuring that the results won't 173 // ever be inadvertently executed by other Terraform commands -- and then 174 // use the results of that installation to update the lock file for the 175 // current working directory. Because we throwaway the packages we 176 // downloaded after completing our work, a subsequent "terraform init" will 177 // then respect the CLI configuration's provider installation strategies 178 // but will verify the packages against the hashes we found upstream. 179 180 // Because our Installer abstraction is a per-platform idea, we'll 181 // instantiate one for each of the platforms the user requested, and then 182 // merge all of the generated locks together at the end. 183 updatedLocks := map[getproviders.Platform]*depsfile.Locks{} 184 selectedVersions := map[addrs.Provider]getproviders.Version{} 185 ctx, cancel := c.InterruptibleContext() 186 defer cancel() 187 for _, platform := range platforms { 188 tempDir, err := ioutil.TempDir("", "terraform-providers-lock") 189 if err != nil { 190 diags = diags.Append(tfdiags.Sourceless( 191 tfdiags.Error, 192 "Could not create temporary directory", 193 fmt.Sprintf("Failed to create a temporary directory for staging the requested provider packages: %s.", err), 194 )) 195 break 196 } 197 defer os.RemoveAll(tempDir) 198 199 evts := &providercache.InstallerEvents{ 200 // Our output from this command is minimal just to show that 201 // we're making progress, rather than just silently hanging. 202 FetchPackageBegin: func(provider addrs.Provider, version getproviders.Version, loc getproviders.PackageLocation) { 203 c.Ui.Output(fmt.Sprintf("- Fetching %s %s for %s...", provider.ForDisplay(), version, platform)) 204 if prevVersion, exists := selectedVersions[provider]; exists && version != prevVersion { 205 // This indicates a weird situation where we ended up 206 // selecting a different version for one platform than 207 // for another. We won't be able to merge the result 208 // in that case, so we'll generate an error. 209 // 210 // This could potentially happen if there's a provider 211 // we've not previously recorded in the lock file and 212 // the available versions change while we're running. To 213 // avoid that would require pre-locking all of the 214 // providers, which is complicated to do with the building 215 // blocks we have here, and so we'll wait to do it only 216 // if this situation arises often in practice. 217 diags = diags.Append(tfdiags.Sourceless( 218 tfdiags.Error, 219 "Inconsistent provider versions", 220 fmt.Sprintf( 221 "The version constraint for %s selected inconsistent versions for different platforms, which is unexpected.\n\nThe upstream registry may have changed its available versions during Terraform's work. If so, re-running this command may produce a successful result.", 222 provider, 223 ), 224 )) 225 } 226 selectedVersions[provider] = version 227 }, 228 FetchPackageSuccess: func(provider addrs.Provider, version getproviders.Version, localDir string, auth *getproviders.PackageAuthenticationResult) { 229 var keyID string 230 if auth != nil && auth.ThirdPartySigned() { 231 keyID = auth.KeyID 232 } 233 if keyID != "" { 234 keyID = c.Colorize().Color(fmt.Sprintf(", key ID [reset][bold]%s[reset]", keyID)) 235 } 236 c.Ui.Output(fmt.Sprintf("- Retrieved %s %s for %s (%s%s)", provider.ForDisplay(), version, platform, auth, keyID)) 237 }, 238 } 239 ctx := evts.OnContext(ctx) 240 241 dir := providercache.NewDirWithPlatform(tempDir, platform) 242 installer := providercache.NewInstaller(dir, source) 243 244 newLocks, err := installer.EnsureProviderVersions(ctx, oldLocks, reqs, providercache.InstallNewProvidersForce) 245 if err != nil { 246 diags = diags.Append(tfdiags.Sourceless( 247 tfdiags.Error, 248 "Could not retrieve providers for locking", 249 fmt.Sprintf("Terraform failed to fetch the requested providers for %s in order to calculate their checksums: %s.", platform, err), 250 )) 251 break 252 } 253 updatedLocks[platform] = newLocks 254 } 255 256 // If we have any error diagnostics from installation then we won't 257 // proceed to merging and updating the lock file on disk. 258 if diags.HasErrors() { 259 c.showDiagnostics(diags) 260 return 1 261 } 262 263 // Track whether we've made any changes to the lock file as part of this 264 // operation. We can customise the final message based on our actions. 265 madeAnyChange := false 266 267 // We now have a separate updated locks object for each platform. We need 268 // to merge those all together so that the final result has the union of 269 // all of the checksums we saw for each of the providers we've worked on. 270 // 271 // We'll copy the old locks first because we want to retain any existing 272 // locks for providers that we _didn't_ visit above. 273 newLocks := oldLocks.DeepCopy() 274 for provider := range reqs { 275 oldLock := oldLocks.Provider(provider) 276 277 var version getproviders.Version 278 var constraints getproviders.VersionConstraints 279 var hashes []getproviders.Hash 280 if oldLock != nil { 281 version = oldLock.Version() 282 constraints = oldLock.VersionConstraints() 283 hashes = append(hashes, oldLock.AllHashes()...) 284 } 285 for platform, platformLocks := range updatedLocks { 286 platformLock := platformLocks.Provider(provider) 287 if platformLock == nil { 288 continue // weird, but we'll tolerate it to avoid crashing 289 } 290 version = platformLock.Version() 291 constraints = platformLock.VersionConstraints() 292 293 // We don't make any effort to deduplicate hashes between different 294 // platforms here, because the SetProvider method we call below 295 // handles that automatically. 296 hashes = append(hashes, platformLock.AllHashes()...) 297 298 // At this point, we've merged all the hashes for this (provider, platform) 299 // combo into the combined hashes for this provider. Let's take this 300 // opportunity to print out a summary for this particular combination. 301 switch providersLockCalculateChangeType(oldLock, platformLock) { 302 case providersLockChangeTypeNewProvider: 303 madeAnyChange = true 304 c.Ui.Output( 305 fmt.Sprintf( 306 "- Obtained %s checksums for %s; This was a new provider and the checksums for this platform are now tracked in the lock file", 307 provider.ForDisplay(), 308 platform)) 309 case providersLockChangeTypeNewHashes: 310 madeAnyChange = true 311 c.Ui.Output( 312 fmt.Sprintf( 313 "- Obtained %s checksums for %s; Additional checksums for this platform are now tracked in the lock file", 314 provider.ForDisplay(), 315 platform)) 316 case providersLockChangeTypeNoChange: 317 c.Ui.Output( 318 fmt.Sprintf( 319 "- Obtained %s checksums for %s; All checksums for this platform were already tracked in the lock file", 320 provider.ForDisplay(), 321 platform)) 322 } 323 } 324 newLocks.SetProvider(provider, version, constraints, hashes) 325 } 326 327 moreDiags = c.replaceLockedDependencies(newLocks) 328 diags = diags.Append(moreDiags) 329 330 c.showDiagnostics(diags) 331 if diags.HasErrors() { 332 return 1 333 } 334 335 if madeAnyChange { 336 c.Ui.Output(c.Colorize().Color("\n[bold][green]Success![reset] [bold]Terraform has updated the lock file.[reset]")) 337 c.Ui.Output("\nReview the changes in .terraform.lock.hcl and then commit to your\nversion control system to retain the new checksums.\n") 338 } else { 339 c.Ui.Output(c.Colorize().Color("\n[bold][green]Success![reset] [bold]Terraform has validated the lock file and found no need for changes.[reset]")) 340 } 341 return 0 342 } 343 344 func (c *ProvidersLockCommand) Help() string { 345 return ` 346 Usage: terraform [global options] providers lock [options] [providers...] 347 348 Normally the dependency lock file (.terraform.lock.hcl) is updated 349 automatically by "terraform init", but the information available to the 350 normal provider installer can be constrained when you're installing providers 351 from filesystem or network mirrors, and so the generated lock file can end 352 up incomplete. 353 354 The "providers lock" subcommand addresses that by updating the lock file 355 based on the official packages available in the origin registry, ignoring 356 the currently-configured installation strategy. 357 358 After this command succeeds, the lock file will contain suitable checksums 359 to allow installation of the providers needed by the current configuration 360 on all of the selected platforms. 361 362 By default this command updates the lock file for every provider declared 363 in the configuration. You can override that behavior by providing one or 364 more provider source addresses on the command line. 365 366 Options: 367 368 -fs-mirror=dir Consult the given filesystem mirror directory instead 369 of the origin registry for each of the given providers. 370 371 This would be necessary to generate lock file entries for 372 a provider that is available only via a mirror, and not 373 published in an upstream registry. In this case, the set 374 of valid checksums will be limited only to what Terraform 375 can learn from the data in the mirror directory. 376 377 -net-mirror=url Consult the given network mirror (given as a base URL) 378 instead of the origin registry for each of the given 379 providers. 380 381 This would be necessary to generate lock file entries for 382 a provider that is available only via a mirror, and not 383 published in an upstream registry. In this case, the set 384 of valid checksums will be limited only to what Terraform 385 can learn from the data in the mirror indices. 386 387 -platform=os_arch Choose a target platform to request package checksums 388 for. 389 390 By default Terraform will request package checksums 391 suitable only for the platform where you run this 392 command. Use this option multiple times to include 393 checksums for multiple target systems. 394 395 Target names consist of an operating system and a CPU 396 architecture. For example, "linux_amd64" selects the 397 Linux operating system running on an AMD64 or x86_64 398 CPU. Each provider is available only for a limited 399 set of target platforms. 400 ` 401 } 402 403 // providersLockCalculateChangeType works out whether there is any difference 404 // between oldLock and newLock and returns a variable the main function can use 405 // to decide on which message to print. 406 // 407 // One assumption made here that is not obvious without the context from the 408 // main function is that while platformLock contains the lock information for a 409 // single platform after the current run, oldLock contains the combined 410 // information of all platforms from when the versions were last checked. A 411 // simple equality check is not sufficient for deciding on change as we expect 412 // that oldLock will be a superset of platformLock if no new hashes have been 413 // found. 414 // 415 // We've separated this function out so we can write unit tests around the 416 // logic. This function assumes the platformLock is not nil, as the main 417 // function explicitly checks this before calling this function. 418 func providersLockCalculateChangeType(oldLock *depsfile.ProviderLock, platformLock *depsfile.ProviderLock) providersLockChangeType { 419 if oldLock == nil { 420 return providersLockChangeTypeNewProvider 421 } 422 if oldLock.ContainsAll(platformLock) { 423 return providersLockChangeTypeNoChange 424 } 425 return providersLockChangeTypeNewHashes 426 }