github.com/opentofu/opentofu@v1.7.1/internal/initwd/from_module.go (about) 1 // Copyright (c) The OpenTofu Authors 2 // SPDX-License-Identifier: MPL-2.0 3 // Copyright (c) 2023 HashiCorp, Inc. 4 // SPDX-License-Identifier: MPL-2.0 5 6 package initwd 7 8 import ( 9 "context" 10 "fmt" 11 "log" 12 "os" 13 "path/filepath" 14 "sort" 15 "strings" 16 17 "github.com/hashicorp/hcl/v2" 18 "github.com/opentofu/opentofu/internal/addrs" 19 "github.com/opentofu/opentofu/internal/configs" 20 "github.com/opentofu/opentofu/internal/configs/configload" 21 "github.com/opentofu/opentofu/internal/copy" 22 "github.com/opentofu/opentofu/internal/getmodules" 23 24 version "github.com/hashicorp/go-version" 25 "github.com/opentofu/opentofu/internal/modsdir" 26 "github.com/opentofu/opentofu/internal/registry" 27 "github.com/opentofu/opentofu/internal/tfdiags" 28 ) 29 30 const initFromModuleRootCallName = "root" 31 const initFromModuleRootFilename = "<main configuration>" 32 const initFromModuleRootKeyPrefix = initFromModuleRootCallName + "." 33 34 // DirFromModule populates the given directory (which must exist and be 35 // empty) with the contents of the module at the given source address. 36 // 37 // It does this by installing the given module and all of its descendent 38 // modules in a temporary root directory and then copying the installed 39 // files into suitable locations. As a consequence, any diagnostics it 40 // generates will reveal the location of this temporary directory to the 41 // user. 42 // 43 // This rather roundabout installation approach is taken to ensure that 44 // installation proceeds in a manner identical to normal module installation. 45 // 46 // If the given source address specifies a sub-directory of the given 47 // package then only the sub-directory and its descendents will be copied 48 // into the given root directory, which will cause any relative module 49 // references using ../ from that module to be unresolvable. Error diagnostics 50 // are produced in that case, to prompt the user to rewrite the source strings 51 // to be absolute references to the original remote module. 52 func DirFromModule(ctx context.Context, loader *configload.Loader, rootDir, modulesDir, sourceAddrStr string, reg *registry.Client, hooks ModuleInstallHooks) tfdiags.Diagnostics { 53 54 var diags tfdiags.Diagnostics 55 56 // The way this function works is pretty ugly, but we accept it because 57 // -from-module is a less important case than normal module installation 58 // and so it's better to keep this ugly complexity out here rather than 59 // adding even more complexity to the normal module installer. 60 61 // The target directory must exist but be empty. 62 { 63 entries, err := os.ReadDir(rootDir) 64 if err != nil { 65 if os.IsNotExist(err) { 66 diags = diags.Append(tfdiags.Sourceless( 67 tfdiags.Error, 68 "Target directory does not exist", 69 fmt.Sprintf("Cannot initialize non-existent directory %s.", rootDir), 70 )) 71 } else { 72 diags = diags.Append(tfdiags.Sourceless( 73 tfdiags.Error, 74 "Failed to read target directory", 75 fmt.Sprintf("Error reading %s to ensure it is empty: %s.", rootDir, err), 76 )) 77 } 78 return diags 79 } 80 haveEntries := false 81 for _, entry := range entries { 82 if entry.Name() == "." || entry.Name() == ".." || entry.Name() == ".terraform" { 83 continue 84 } 85 haveEntries = true 86 } 87 if haveEntries { 88 diags = diags.Append(tfdiags.Sourceless( 89 tfdiags.Error, 90 "Can't populate non-empty directory", 91 fmt.Sprintf("The target directory %s is not empty, so it cannot be initialized with the -from-module=... option.", rootDir), 92 )) 93 return diags 94 } 95 } 96 97 instDir := filepath.Join(rootDir, ".terraform/init-from-module") 98 inst := NewModuleInstaller(instDir, loader, reg) 99 log.Printf("[DEBUG] installing modules in %s to initialize working directory from %q", instDir, sourceAddrStr) 100 os.RemoveAll(instDir) // if this fails then we'll fail on MkdirAll below too 101 err := os.MkdirAll(instDir, os.ModePerm) 102 if err != nil { 103 diags = diags.Append(tfdiags.Sourceless( 104 tfdiags.Error, 105 "Failed to create temporary directory", 106 fmt.Sprintf("Failed to create temporary directory %s: %s.", instDir, err), 107 )) 108 return diags 109 } 110 111 instManifest := make(modsdir.Manifest) 112 retManifest := make(modsdir.Manifest) 113 114 // -from-module allows relative paths but it's different than a normal 115 // module address where it'd be resolved relative to the module call 116 // (which is synthetic, here.) To address this, we'll just patch up any 117 // relative paths to be absolute paths before we run, ensuring we'll 118 // get the right result. This also, as an important side-effect, ensures 119 // that the result will be "downloaded" with go-getter (copied from the 120 // source location), rather than just recorded as a relative path. 121 { 122 maybePath := filepath.ToSlash(sourceAddrStr) 123 if maybePath == "." || strings.HasPrefix(maybePath, "./") || strings.HasPrefix(maybePath, "../") { 124 if wd, err := os.Getwd(); err == nil { 125 sourceAddrStr = filepath.Join(wd, sourceAddrStr) 126 log.Printf("[TRACE] -from-module relative path rewritten to absolute path %s", sourceAddrStr) 127 } 128 } 129 } 130 131 // Now we need to create an artificial root module that will seed our 132 // installation process. 133 sourceAddr, err := addrs.ParseModuleSource(sourceAddrStr) 134 if err != nil { 135 diags = diags.Append(tfdiags.Sourceless( 136 tfdiags.Error, 137 "Invalid module source address", 138 fmt.Sprintf("Failed to parse module source address: %s", err), 139 )) 140 } 141 fakeRootModule := &configs.Module{ 142 ModuleCalls: map[string]*configs.ModuleCall{ 143 initFromModuleRootCallName: { 144 Name: initFromModuleRootCallName, 145 SourceAddr: sourceAddr, 146 DeclRange: hcl.Range{ 147 Filename: initFromModuleRootFilename, 148 Start: hcl.InitialPos, 149 End: hcl.InitialPos, 150 }, 151 }, 152 }, 153 ProviderRequirements: &configs.RequiredProviders{}, 154 } 155 156 // wrapHooks filters hook notifications to only include Download calls 157 // and to trim off the initFromModuleRootCallName prefix. We'll produce 158 // our own Install notifications directly below. 159 wrapHooks := installHooksInitDir{ 160 Wrapped: hooks, 161 } 162 // Create a manifest record for the root module. This will be used if 163 // there are any relative-pathed modules in the root. 164 instManifest[""] = modsdir.Record{ 165 Key: "", 166 Dir: rootDir, 167 } 168 fetcher := getmodules.NewPackageFetcher() 169 170 walker := inst.moduleInstallWalker(ctx, instManifest, true, wrapHooks, fetcher) 171 _, cDiags := inst.installDescendentModules(fakeRootModule, instManifest, walker, true) 172 if cDiags.HasErrors() { 173 return diags.Append(cDiags) 174 } 175 176 // If all of that succeeded then we'll now migrate what was installed 177 // into the final directory structure. 178 err = os.MkdirAll(modulesDir, os.ModePerm) 179 if err != nil { 180 diags = diags.Append(tfdiags.Sourceless( 181 tfdiags.Error, 182 "Failed to create local modules directory", 183 fmt.Sprintf("Failed to create modules directory %s: %s.", modulesDir, err), 184 )) 185 return diags 186 } 187 188 recordKeys := make([]string, 0, len(instManifest)) 189 for k := range instManifest { 190 recordKeys = append(recordKeys, k) 191 } 192 sort.Strings(recordKeys) 193 194 for _, recordKey := range recordKeys { 195 record := instManifest[recordKey] 196 197 if record.Key == initFromModuleRootCallName { 198 // We've found the module the user requested, which we must 199 // now copy into rootDir so it can be used directly. 200 log.Printf("[TRACE] copying new root module from %s to %s", record.Dir, rootDir) 201 err := copy.CopyDir(rootDir, record.Dir) 202 if err != nil { 203 diags = diags.Append(tfdiags.Sourceless( 204 tfdiags.Error, 205 "Failed to copy root module", 206 fmt.Sprintf("Error copying root module %q from %s to %s: %s.", sourceAddrStr, record.Dir, rootDir, err), 207 )) 208 continue 209 } 210 211 // We'll try to load the newly-copied module here just so we can 212 // sniff for any module calls that ../ out of the root directory 213 // and must thus be rewritten to be absolute addresses again. 214 // For now we can't do this rewriting automatically, but we'll 215 // generate an error to help the user do it manually. 216 mod, _ := loader.Parser().LoadConfigDir(rootDir) // ignore diagnostics since we're just doing value-add here anyway 217 if mod != nil { 218 for _, mc := range mod.ModuleCalls { 219 if pathTraversesUp(mc.SourceAddrRaw) { 220 packageAddr, givenSubdir := getmodules.SplitPackageSubdir(sourceAddrStr) 221 newSubdir := filepath.Join(givenSubdir, mc.SourceAddrRaw) 222 if pathTraversesUp(newSubdir) { 223 // This should never happen in any reasonable 224 // configuration since this suggests a path that 225 // traverses up out of the package root. We'll just 226 // ignore this, since we'll fail soon enough anyway 227 // trying to resolve this path when this module is 228 // loaded. 229 continue 230 } 231 232 var newAddr = packageAddr 233 if newSubdir != "" { 234 newAddr = fmt.Sprintf("%s//%s", newAddr, filepath.ToSlash(newSubdir)) 235 } 236 diags = diags.Append(tfdiags.Sourceless( 237 tfdiags.Error, 238 "Root module references parent directory", 239 fmt.Sprintf("The requested module %q refers to a module via its parent directory. To use this as a new root module this source string must be rewritten as a remote source address, such as %q.", sourceAddrStr, newAddr), 240 )) 241 continue 242 } 243 } 244 } 245 246 retManifest[""] = modsdir.Record{ 247 Key: "", 248 Dir: rootDir, 249 } 250 continue 251 } 252 253 if !strings.HasPrefix(record.Key, initFromModuleRootKeyPrefix) { 254 // Ignore the *real* root module, whose key is empty, since 255 // we're only interested in the module named "root" and its 256 // descendents. 257 continue 258 } 259 260 newKey := record.Key[len(initFromModuleRootKeyPrefix):] 261 instPath := filepath.Join(modulesDir, newKey) 262 tempPath := filepath.Join(instDir, record.Key) 263 264 // tempPath won't be present for a module that was installed from 265 // a relative path, so in that case we just record the installation 266 // directory and assume it was already copied into place as part 267 // of its parent. 268 if _, err := os.Stat(tempPath); err != nil { 269 if !os.IsNotExist(err) { 270 diags = diags.Append(tfdiags.Sourceless( 271 tfdiags.Error, 272 "Failed to stat temporary module install directory", 273 fmt.Sprintf("Error from stat %s for module %s: %s.", instPath, newKey, err), 274 )) 275 continue 276 } 277 278 var parentKey string 279 if lastDot := strings.LastIndexByte(newKey, '.'); lastDot != -1 { 280 parentKey = newKey[:lastDot] 281 } 282 283 var parentOld modsdir.Record 284 // "" is the root module; all other modules get `root.` added as a prefix 285 if parentKey == "" { 286 parentOld = instManifest[parentKey] 287 } else { 288 parentOld = instManifest[initFromModuleRootKeyPrefix+parentKey] 289 } 290 parentNew := retManifest[parentKey] 291 292 // We need to figure out which portion of our directory is the 293 // parent package path and which portion is the subdirectory 294 // under that. 295 var baseDirRel string 296 baseDirRel, err = filepath.Rel(parentOld.Dir, record.Dir) 297 if err != nil { 298 // This error may occur when installing a local module with a 299 // relative path, for e.g. if the source is in a directory above 300 // the destination ("../") 301 if parentOld.Dir == "." { 302 absDir, err := filepath.Abs(parentOld.Dir) 303 if err != nil { 304 diags = diags.Append(tfdiags.Sourceless( 305 tfdiags.Error, 306 "Failed to determine module install directory", 307 fmt.Sprintf("Error determine relative source directory for module %s: %s.", newKey, err), 308 )) 309 continue 310 } 311 baseDirRel, err = filepath.Rel(absDir, record.Dir) 312 if err != nil { 313 diags = diags.Append(tfdiags.Sourceless( 314 tfdiags.Error, 315 "Failed to determine relative module source location", 316 fmt.Sprintf("Error determining relative source for module %s: %s.", newKey, err), 317 )) 318 continue 319 } 320 } else { 321 diags = diags.Append(tfdiags.Sourceless( 322 tfdiags.Error, 323 "Failed to determine relative module source location", 324 fmt.Sprintf("Error determining relative source for module %s: %s.", newKey, err), 325 )) 326 } 327 } 328 329 newDir := filepath.Join(parentNew.Dir, baseDirRel) 330 log.Printf("[TRACE] relative reference for %s rewritten from %s to %s", newKey, record.Dir, newDir) 331 newRecord := record // shallow copy 332 newRecord.Dir = newDir 333 newRecord.Key = newKey 334 retManifest[newKey] = newRecord 335 hooks.Install(newRecord.Key, newRecord.Version, newRecord.Dir) 336 continue 337 } 338 339 err = os.MkdirAll(instPath, os.ModePerm) 340 if err != nil { 341 diags = diags.Append(tfdiags.Sourceless( 342 tfdiags.Error, 343 "Failed to create module install directory", 344 fmt.Sprintf("Error creating directory %s for module %s: %s.", instPath, newKey, err), 345 )) 346 continue 347 } 348 349 // We copy rather than "rename" here because renaming between directories 350 // can be tricky in edge-cases like network filesystems, etc. 351 log.Printf("[TRACE] copying new module %s from %s to %s", newKey, record.Dir, instPath) 352 err := copy.CopyDir(instPath, tempPath) 353 if err != nil { 354 diags = diags.Append(tfdiags.Sourceless( 355 tfdiags.Error, 356 "Failed to copy descendent module", 357 fmt.Sprintf("Error copying module %q from %s to %s: %s.", newKey, tempPath, rootDir, err), 358 )) 359 continue 360 } 361 362 subDir, err := filepath.Rel(tempPath, record.Dir) 363 if err != nil { 364 // Should never happen, because we constructed both directories 365 // from the same base and so they must have a common prefix. 366 panic(err) 367 } 368 369 newRecord := record // shallow copy 370 newRecord.Dir = filepath.Join(instPath, subDir) 371 newRecord.Key = newKey 372 retManifest[newKey] = newRecord 373 hooks.Install(newRecord.Key, newRecord.Version, newRecord.Dir) 374 } 375 376 retManifest.WriteSnapshotToDir(modulesDir) 377 if err != nil { 378 diags = diags.Append(tfdiags.Sourceless( 379 tfdiags.Error, 380 "Failed to write module manifest", 381 fmt.Sprintf("Error writing module manifest: %s.", err), 382 )) 383 } 384 385 if !diags.HasErrors() { 386 // Try to clean up our temporary directory, but don't worry if we don't 387 // succeed since it shouldn't hurt anything. 388 os.RemoveAll(instDir) 389 } 390 391 return diags 392 } 393 394 func pathTraversesUp(path string) bool { 395 return strings.HasPrefix(filepath.ToSlash(path), "../") 396 } 397 398 // installHooksInitDir is an adapter wrapper for an InstallHooks that 399 // does some fakery to make downloads look like they are happening in their 400 // final locations, rather than in the temporary loader we use. 401 // 402 // It also suppresses "Install" calls entirely, since InitDirFromModule 403 // does its own installation steps after the initial installation pass 404 // has completed. 405 type installHooksInitDir struct { 406 Wrapped ModuleInstallHooks 407 ModuleInstallHooksImpl 408 } 409 410 func (h installHooksInitDir) Download(moduleAddr, packageAddr string, version *version.Version) { 411 if !strings.HasPrefix(moduleAddr, initFromModuleRootKeyPrefix) { 412 // We won't announce the root module, since hook implementations 413 // don't expect to see that and the caller will usually have produced 414 // its own user-facing notification about what it's doing anyway. 415 return 416 } 417 418 trimAddr := moduleAddr[len(initFromModuleRootKeyPrefix):] 419 h.Wrapped.Download(trimAddr, packageAddr, version) 420 }