github.com/kanishk98/terraform@v1.3.0-dev.0.20220917174235-661ca8088a6a/internal/initwd/from_module.go (about) 1 package initwd 2 3 import ( 4 "context" 5 "fmt" 6 "io/ioutil" 7 "log" 8 "os" 9 "path/filepath" 10 "sort" 11 "strings" 12 13 "github.com/hashicorp/terraform/internal/copy" 14 "github.com/hashicorp/terraform/internal/earlyconfig" 15 "github.com/hashicorp/terraform/internal/getmodules" 16 17 version "github.com/hashicorp/go-version" 18 "github.com/hashicorp/terraform-config-inspect/tfconfig" 19 "github.com/hashicorp/terraform/internal/modsdir" 20 "github.com/hashicorp/terraform/internal/registry" 21 "github.com/hashicorp/terraform/internal/tfdiags" 22 ) 23 24 const initFromModuleRootCallName = "root" 25 const initFromModuleRootKeyPrefix = initFromModuleRootCallName + "." 26 27 // DirFromModule populates the given directory (which must exist and be 28 // empty) with the contents of the module at the given source address. 29 // 30 // It does this by installing the given module and all of its descendent 31 // modules in a temporary root directory and then copying the installed 32 // files into suitable locations. As a consequence, any diagnostics it 33 // generates will reveal the location of this temporary directory to the 34 // user. 35 // 36 // This rather roundabout installation approach is taken to ensure that 37 // installation proceeds in a manner identical to normal module installation. 38 // 39 // If the given source address specifies a sub-directory of the given 40 // package then only the sub-directory and its descendents will be copied 41 // into the given root directory, which will cause any relative module 42 // references using ../ from that module to be unresolvable. Error diagnostics 43 // are produced in that case, to prompt the user to rewrite the source strings 44 // to be absolute references to the original remote module. 45 func DirFromModule(ctx context.Context, rootDir, modulesDir, sourceAddr string, reg *registry.Client, hooks ModuleInstallHooks) tfdiags.Diagnostics { 46 var diags tfdiags.Diagnostics 47 48 // The way this function works is pretty ugly, but we accept it because 49 // -from-module is a less important case than normal module installation 50 // and so it's better to keep this ugly complexity out here rather than 51 // adding even more complexity to the normal module installer. 52 53 // The target directory must exist but be empty. 54 { 55 entries, err := ioutil.ReadDir(rootDir) 56 if err != nil { 57 if os.IsNotExist(err) { 58 diags = diags.Append(tfdiags.Sourceless( 59 tfdiags.Error, 60 "Target directory does not exist", 61 fmt.Sprintf("Cannot initialize non-existent directory %s.", rootDir), 62 )) 63 } else { 64 diags = diags.Append(tfdiags.Sourceless( 65 tfdiags.Error, 66 "Failed to read target directory", 67 fmt.Sprintf("Error reading %s to ensure it is empty: %s.", rootDir, err), 68 )) 69 } 70 return diags 71 } 72 haveEntries := false 73 for _, entry := range entries { 74 if entry.Name() == "." || entry.Name() == ".." || entry.Name() == ".terraform" { 75 continue 76 } 77 haveEntries = true 78 } 79 if haveEntries { 80 diags = diags.Append(tfdiags.Sourceless( 81 tfdiags.Error, 82 "Can't populate non-empty directory", 83 fmt.Sprintf("The target directory %s is not empty, so it cannot be initialized with the -from-module=... option.", rootDir), 84 )) 85 return diags 86 } 87 } 88 89 instDir := filepath.Join(rootDir, ".terraform/init-from-module") 90 inst := NewModuleInstaller(instDir, reg) 91 log.Printf("[DEBUG] installing modules in %s to initialize working directory from %q", instDir, sourceAddr) 92 os.RemoveAll(instDir) // if this fails then we'll fail on MkdirAll below too 93 err := os.MkdirAll(instDir, os.ModePerm) 94 if err != nil { 95 diags = diags.Append(tfdiags.Sourceless( 96 tfdiags.Error, 97 "Failed to create temporary directory", 98 fmt.Sprintf("Failed to create temporary directory %s: %s.", instDir, err), 99 )) 100 return diags 101 } 102 103 instManifest := make(modsdir.Manifest) 104 retManifest := make(modsdir.Manifest) 105 106 fakeFilename := fmt.Sprintf("-from-module=%q", sourceAddr) 107 fakePos := tfconfig.SourcePos{ 108 Filename: fakeFilename, 109 Line: 1, 110 } 111 112 // -from-module allows relative paths but it's different than a normal 113 // module address where it'd be resolved relative to the module call 114 // (which is synthetic, here.) To address this, we'll just patch up any 115 // relative paths to be absolute paths before we run, ensuring we'll 116 // get the right result. This also, as an important side-effect, ensures 117 // that the result will be "downloaded" with go-getter (copied from the 118 // source location), rather than just recorded as a relative path. 119 { 120 maybePath := filepath.ToSlash(sourceAddr) 121 if maybePath == "." || strings.HasPrefix(maybePath, "./") || strings.HasPrefix(maybePath, "../") { 122 if wd, err := os.Getwd(); err == nil { 123 sourceAddr = filepath.Join(wd, sourceAddr) 124 log.Printf("[TRACE] -from-module relative path rewritten to absolute path %s", sourceAddr) 125 } 126 } 127 } 128 129 // Now we need to create an artificial root module that will seed our 130 // installation process. 131 fakeRootModule := &tfconfig.Module{ 132 ModuleCalls: map[string]*tfconfig.ModuleCall{ 133 initFromModuleRootCallName: { 134 Name: initFromModuleRootCallName, 135 Source: sourceAddr, 136 Pos: fakePos, 137 }, 138 }, 139 } 140 141 // wrapHooks filters hook notifications to only include Download calls 142 // and to trim off the initFromModuleRootCallName prefix. We'll produce 143 // our own Install notifications directly below. 144 wrapHooks := installHooksInitDir{ 145 Wrapped: hooks, 146 } 147 fetcher := getmodules.NewPackageFetcher() 148 _, instDiags := inst.installDescendentModules(ctx, fakeRootModule, rootDir, instManifest, true, wrapHooks, fetcher) 149 diags = append(diags, instDiags...) 150 if instDiags.HasErrors() { 151 return diags 152 } 153 154 // If all of that succeeded then we'll now migrate what was installed 155 // into the final directory structure. 156 err = os.MkdirAll(modulesDir, os.ModePerm) 157 if err != nil { 158 diags = diags.Append(tfdiags.Sourceless( 159 tfdiags.Error, 160 "Failed to create local modules directory", 161 fmt.Sprintf("Failed to create modules directory %s: %s.", modulesDir, err), 162 )) 163 return diags 164 } 165 166 recordKeys := make([]string, 0, len(instManifest)) 167 for k := range instManifest { 168 recordKeys = append(recordKeys, k) 169 } 170 sort.Strings(recordKeys) 171 172 for _, recordKey := range recordKeys { 173 record := instManifest[recordKey] 174 175 if record.Key == initFromModuleRootCallName { 176 // We've found the module the user requested, which we must 177 // now copy into rootDir so it can be used directly. 178 log.Printf("[TRACE] copying new root module from %s to %s", record.Dir, rootDir) 179 err := copy.CopyDir(rootDir, record.Dir) 180 if err != nil { 181 diags = diags.Append(tfdiags.Sourceless( 182 tfdiags.Error, 183 "Failed to copy root module", 184 fmt.Sprintf("Error copying root module %q from %s to %s: %s.", sourceAddr, record.Dir, rootDir, err), 185 )) 186 continue 187 } 188 189 // We'll try to load the newly-copied module here just so we can 190 // sniff for any module calls that ../ out of the root directory 191 // and must thus be rewritten to be absolute addresses again. 192 // For now we can't do this rewriting automatically, but we'll 193 // generate an error to help the user do it manually. 194 mod, _ := earlyconfig.LoadModule(rootDir) // ignore diagnostics since we're just doing value-add here anyway 195 if mod != nil { 196 for _, mc := range mod.ModuleCalls { 197 if pathTraversesUp(mc.Source) { 198 packageAddr, givenSubdir := getmodules.SplitPackageSubdir(sourceAddr) 199 newSubdir := filepath.Join(givenSubdir, mc.Source) 200 if pathTraversesUp(newSubdir) { 201 // This should never happen in any reasonable 202 // configuration since this suggests a path that 203 // traverses up out of the package root. We'll just 204 // ignore this, since we'll fail soon enough anyway 205 // trying to resolve this path when this module is 206 // loaded. 207 continue 208 } 209 210 var newAddr = packageAddr 211 if newSubdir != "" { 212 newAddr = fmt.Sprintf("%s//%s", newAddr, filepath.ToSlash(newSubdir)) 213 } 214 diags = diags.Append(tfdiags.Sourceless( 215 tfdiags.Error, 216 "Root module references parent directory", 217 fmt.Sprintf("The requested module %q refers to a module via its parent directory. To use this as a new root module this source string must be rewritten as a remote source address, such as %q.", sourceAddr, newAddr), 218 )) 219 continue 220 } 221 } 222 } 223 224 retManifest[""] = modsdir.Record{ 225 Key: "", 226 Dir: rootDir, 227 } 228 continue 229 } 230 231 if !strings.HasPrefix(record.Key, initFromModuleRootKeyPrefix) { 232 // Ignore the *real* root module, whose key is empty, since 233 // we're only interested in the module named "root" and its 234 // descendents. 235 continue 236 } 237 238 newKey := record.Key[len(initFromModuleRootKeyPrefix):] 239 instPath := filepath.Join(modulesDir, newKey) 240 tempPath := filepath.Join(instDir, record.Key) 241 242 // tempPath won't be present for a module that was installed from 243 // a relative path, so in that case we just record the installation 244 // directory and assume it was already copied into place as part 245 // of its parent. 246 if _, err := os.Stat(tempPath); err != nil { 247 if !os.IsNotExist(err) { 248 diags = diags.Append(tfdiags.Sourceless( 249 tfdiags.Error, 250 "Failed to stat temporary module install directory", 251 fmt.Sprintf("Error from stat %s for module %s: %s.", instPath, newKey, err), 252 )) 253 continue 254 } 255 256 var parentKey string 257 if lastDot := strings.LastIndexByte(newKey, '.'); lastDot != -1 { 258 parentKey = newKey[:lastDot] 259 } 260 261 var parentOld modsdir.Record 262 // "" is the root module; all other modules get `root.` added as a prefix 263 if parentKey == "" { 264 parentOld = instManifest[parentKey] 265 } else { 266 parentOld = instManifest[initFromModuleRootKeyPrefix+parentKey] 267 } 268 parentNew := retManifest[parentKey] 269 270 // We need to figure out which portion of our directory is the 271 // parent package path and which portion is the subdirectory 272 // under that. 273 var baseDirRel string 274 baseDirRel, err = filepath.Rel(parentOld.Dir, record.Dir) 275 if err != nil { 276 // This error may occur when installing a local module with a 277 // relative path, for e.g. if the source is in a directory above 278 // the destination ("../") 279 if parentOld.Dir == "." { 280 absDir, err := filepath.Abs(parentOld.Dir) 281 if err != nil { 282 diags = diags.Append(tfdiags.Sourceless( 283 tfdiags.Error, 284 "Failed to determine module install directory", 285 fmt.Sprintf("Error determine relative source directory for module %s: %s.", newKey, err), 286 )) 287 continue 288 } 289 baseDirRel, err = filepath.Rel(absDir, record.Dir) 290 if err != nil { 291 diags = diags.Append(tfdiags.Sourceless( 292 tfdiags.Error, 293 "Failed to determine relative module source location", 294 fmt.Sprintf("Error determining relative source for module %s: %s.", newKey, err), 295 )) 296 continue 297 } 298 } else { 299 diags = diags.Append(tfdiags.Sourceless( 300 tfdiags.Error, 301 "Failed to determine relative module source location", 302 fmt.Sprintf("Error determining relative source for module %s: %s.", newKey, err), 303 )) 304 } 305 } 306 307 newDir := filepath.Join(parentNew.Dir, baseDirRel) 308 log.Printf("[TRACE] relative reference for %s rewritten from %s to %s", newKey, record.Dir, newDir) 309 newRecord := record // shallow copy 310 newRecord.Dir = newDir 311 newRecord.Key = newKey 312 retManifest[newKey] = newRecord 313 hooks.Install(newRecord.Key, newRecord.Version, newRecord.Dir) 314 continue 315 } 316 317 err = os.MkdirAll(instPath, os.ModePerm) 318 if err != nil { 319 diags = diags.Append(tfdiags.Sourceless( 320 tfdiags.Error, 321 "Failed to create module install directory", 322 fmt.Sprintf("Error creating directory %s for module %s: %s.", instPath, newKey, err), 323 )) 324 continue 325 } 326 327 // We copy rather than "rename" here because renaming between directories 328 // can be tricky in edge-cases like network filesystems, etc. 329 log.Printf("[TRACE] copying new module %s from %s to %s", newKey, record.Dir, instPath) 330 err := copy.CopyDir(instPath, tempPath) 331 if err != nil { 332 diags = diags.Append(tfdiags.Sourceless( 333 tfdiags.Error, 334 "Failed to copy descendent module", 335 fmt.Sprintf("Error copying module %q from %s to %s: %s.", newKey, tempPath, rootDir, err), 336 )) 337 continue 338 } 339 340 subDir, err := filepath.Rel(tempPath, record.Dir) 341 if err != nil { 342 // Should never happen, because we constructed both directories 343 // from the same base and so they must have a common prefix. 344 panic(err) 345 } 346 347 newRecord := record // shallow copy 348 newRecord.Dir = filepath.Join(instPath, subDir) 349 newRecord.Key = newKey 350 retManifest[newKey] = newRecord 351 hooks.Install(newRecord.Key, newRecord.Version, newRecord.Dir) 352 } 353 354 retManifest.WriteSnapshotToDir(modulesDir) 355 if err != nil { 356 diags = diags.Append(tfdiags.Sourceless( 357 tfdiags.Error, 358 "Failed to write module manifest", 359 fmt.Sprintf("Error writing module manifest: %s.", err), 360 )) 361 } 362 363 if !diags.HasErrors() { 364 // Try to clean up our temporary directory, but don't worry if we don't 365 // succeed since it shouldn't hurt anything. 366 os.RemoveAll(instDir) 367 } 368 369 return diags 370 } 371 372 func pathTraversesUp(path string) bool { 373 return strings.HasPrefix(filepath.ToSlash(path), "../") 374 } 375 376 // installHooksInitDir is an adapter wrapper for an InstallHooks that 377 // does some fakery to make downloads look like they are happening in their 378 // final locations, rather than in the temporary loader we use. 379 // 380 // It also suppresses "Install" calls entirely, since InitDirFromModule 381 // does its own installation steps after the initial installation pass 382 // has completed. 383 type installHooksInitDir struct { 384 Wrapped ModuleInstallHooks 385 ModuleInstallHooksImpl 386 } 387 388 func (h installHooksInitDir) Download(moduleAddr, packageAddr string, version *version.Version) { 389 if !strings.HasPrefix(moduleAddr, initFromModuleRootKeyPrefix) { 390 // We won't announce the root module, since hook implementations 391 // don't expect to see that and the caller will usually have produced 392 // its own user-facing notification about what it's doing anyway. 393 return 394 } 395 396 trimAddr := moduleAddr[len(initFromModuleRootKeyPrefix):] 397 h.Wrapped.Download(trimAddr, packageAddr, version) 398 }