github.com/trawler/terraform@v0.10.8-0.20171106022149-4b1c7a1d9b48/config/module/tree.go (about) 1 package module 2 3 import ( 4 "bufio" 5 "bytes" 6 "fmt" 7 "log" 8 "path/filepath" 9 "strings" 10 "sync" 11 12 getter "github.com/hashicorp/go-getter" 13 "github.com/hashicorp/terraform/config" 14 ) 15 16 // RootName is the name of the root tree. 17 const RootName = "root" 18 19 // Tree represents the module import tree of configurations. 20 // 21 // This Tree structure can be used to get (download) new modules, load 22 // all the modules without getting, flatten the tree into something 23 // Terraform can use, etc. 24 type Tree struct { 25 name string 26 config *config.Config 27 children map[string]*Tree 28 path []string 29 lock sync.RWMutex 30 31 // version is the final version of the config loaded for the Tree's module 32 version string 33 // source is the "source" string used to load this module. It's possible 34 // for a module source to change, but the path remains the same, preventing 35 // it from being reloaded. 36 source string 37 // parent allows us to walk back up the tree and determine if there are any 38 // versioned ancestor modules which may effect the stored location of 39 // submodules 40 parent *Tree 41 } 42 43 // NewTree returns a new Tree for the given config structure. 44 func NewTree(name string, c *config.Config) *Tree { 45 return &Tree{config: c, name: name} 46 } 47 48 // NewEmptyTree returns a new tree that is empty (contains no configuration). 49 func NewEmptyTree() *Tree { 50 t := &Tree{config: &config.Config{}} 51 52 // We do this dummy load so that the tree is marked as "loaded". It 53 // should never fail because this is just about a no-op. If it does fail 54 // we panic so we can know its a bug. 55 if err := t.Load(&Storage{Mode: GetModeGet}); err != nil { 56 panic(err) 57 } 58 59 return t 60 } 61 62 // NewTreeModule is like NewTree except it parses the configuration in 63 // the directory and gives it a specific name. Use a blank name "" to specify 64 // the root module. 65 func NewTreeModule(name, dir string) (*Tree, error) { 66 c, err := config.LoadDir(dir) 67 if err != nil { 68 return nil, err 69 } 70 71 return NewTree(name, c), nil 72 } 73 74 // Config returns the configuration for this module. 75 func (t *Tree) Config() *config.Config { 76 return t.config 77 } 78 79 // Child returns the child with the given path (by name). 80 func (t *Tree) Child(path []string) *Tree { 81 if t == nil { 82 return nil 83 } 84 85 if len(path) == 0 { 86 return t 87 } 88 89 c := t.Children()[path[0]] 90 if c == nil { 91 return nil 92 } 93 94 return c.Child(path[1:]) 95 } 96 97 // Children returns the children of this tree (the modules that are 98 // imported by this root). 99 // 100 // This will only return a non-nil value after Load is called. 101 func (t *Tree) Children() map[string]*Tree { 102 t.lock.RLock() 103 defer t.lock.RUnlock() 104 return t.children 105 } 106 107 // DeepEach calls the provided callback for the receiver and then all of 108 // its descendents in the tree, allowing an operation to be performed on 109 // all modules in the tree. 110 // 111 // Parents will be visited before their children but otherwise the order is 112 // not defined. 113 func (t *Tree) DeepEach(cb func(*Tree)) { 114 t.lock.RLock() 115 defer t.lock.RUnlock() 116 t.deepEach(cb) 117 } 118 119 func (t *Tree) deepEach(cb func(*Tree)) { 120 cb(t) 121 for _, c := range t.children { 122 c.deepEach(cb) 123 } 124 } 125 126 // Loaded says whether or not this tree has been loaded or not yet. 127 func (t *Tree) Loaded() bool { 128 t.lock.RLock() 129 defer t.lock.RUnlock() 130 return t.children != nil 131 } 132 133 // Modules returns the list of modules that this tree imports. 134 // 135 // This is only the imports of _this_ level of the tree. To retrieve the 136 // full nested imports, you'll have to traverse the tree. 137 func (t *Tree) Modules() []*Module { 138 result := make([]*Module, len(t.config.Modules)) 139 for i, m := range t.config.Modules { 140 result[i] = &Module{ 141 Name: m.Name, 142 Version: m.Version, 143 Source: m.Source, 144 Providers: m.Providers, 145 } 146 } 147 148 return result 149 } 150 151 // Name returns the name of the tree. This will be "<root>" for the root 152 // tree and then the module name given for any children. 153 func (t *Tree) Name() string { 154 if t.name == "" { 155 return RootName 156 } 157 158 return t.name 159 } 160 161 // Load loads the configuration of the entire tree. 162 // 163 // The parameters are used to tell the tree where to find modules and 164 // whether it can download/update modules along the way. 165 // 166 // Calling this multiple times will reload the tree. 167 // 168 // Various semantic-like checks are made along the way of loading since 169 // module trees inherently require the configuration to be in a reasonably 170 // sane state: no circular dependencies, proper module sources, etc. A full 171 // suite of validations can be done by running Validate (after loading). 172 func (t *Tree) Load(s *Storage) error { 173 t.lock.Lock() 174 defer t.lock.Unlock() 175 176 children, err := t.getChildren(s) 177 if err != nil { 178 return err 179 } 180 181 // Go through all the children and load them. 182 for _, c := range children { 183 if err := c.Load(s); err != nil { 184 return err 185 } 186 } 187 188 // Set our tree up 189 t.children = children 190 191 // if we're the root module, we can now set the provider inheritance 192 if len(t.path) == 0 { 193 t.inheritProviderConfigs(nil) 194 } 195 196 return nil 197 } 198 199 func (t *Tree) getChildren(s *Storage) (map[string]*Tree, error) { 200 children := make(map[string]*Tree) 201 202 // Go through all the modules and get the directory for them. 203 for _, m := range t.Modules() { 204 if _, ok := children[m.Name]; ok { 205 return nil, fmt.Errorf( 206 "module %s: duplicated. module names must be unique", m.Name) 207 } 208 209 // Determine the path to this child 210 modPath := make([]string, len(t.path), len(t.path)+1) 211 copy(modPath, t.path) 212 modPath = append(modPath, m.Name) 213 214 log.Printf("[TRACE] module source: %q", m.Source) 215 216 // add the module path to help indicate where modules with relative 217 // paths are being loaded from 218 s.output(fmt.Sprintf("- module.%s", strings.Join(modPath, "."))) 219 220 // Lookup the local location of the module. 221 // dir is the local directory where the module is stored 222 mod, err := s.findRegistryModule(m.Source, m.Version) 223 if err != nil { 224 return nil, err 225 } 226 227 // The key is the string that will be used to uniquely id the Source in 228 // the local storage. The prefix digit can be incremented to 229 // invalidate the local module storage. 230 key := "1." + t.versionedPathKey(m) 231 if mod.Version != "" { 232 key += "." + mod.Version 233 } 234 235 // Check for the exact key if it's not a registry module 236 if !mod.registry { 237 mod.Dir, err = s.findModule(key) 238 if err != nil { 239 return nil, err 240 } 241 } 242 243 if mod.Dir != "" && s.Mode != GetModeUpdate { 244 // We found it locally, but in order to load the Tree we need to 245 // find out if there was another subDir stored from detection. 246 subDir, err := s.getModuleRoot(mod.Dir) 247 if err != nil { 248 // If there's a problem with the subdir record, we'll let the 249 // recordSubdir method fix it up. Any other filesystem errors 250 // will turn up again below. 251 log.Println("[WARN] error reading subdir record:", err) 252 } 253 254 fullDir := filepath.Join(mod.Dir, subDir) 255 256 child, err := NewTreeModule(m.Name, fullDir) 257 if err != nil { 258 return nil, fmt.Errorf("module %s: %s", m.Name, err) 259 } 260 child.path = modPath 261 child.parent = t 262 child.version = mod.Version 263 child.source = m.Source 264 children[m.Name] = child 265 continue 266 } 267 268 // Split out the subdir if we have one. 269 // Terraform keeps the entire requested tree, so that modules can 270 // reference sibling modules from the same archive or repo. 271 rawSource, subDir := getter.SourceDirSubdir(m.Source) 272 273 // we haven't found a source, so fallback to the go-getter detectors 274 source := mod.url 275 if source == "" { 276 source, err = getter.Detect(rawSource, t.config.Dir, getter.Detectors) 277 if err != nil { 278 return nil, fmt.Errorf("module %s: %s", m.Name, err) 279 } 280 } 281 282 log.Printf("[TRACE] detected module source %q", source) 283 284 // Check if the detector introduced something new. 285 // For example, the registry always adds a subdir of `//*`, 286 // indicating that we need to strip off the first component from the 287 // tar archive, though we may not yet know what it is called. 288 source, detectedSubDir := getter.SourceDirSubdir(source) 289 if detectedSubDir != "" { 290 subDir = filepath.Join(detectedSubDir, subDir) 291 } 292 293 output := "" 294 switch s.Mode { 295 case GetModeUpdate: 296 output = fmt.Sprintf(" Updating source %q", m.Source) 297 default: 298 output = fmt.Sprintf(" Getting source %q", m.Source) 299 } 300 s.output(output) 301 302 dir, ok, err := s.getStorage(key, source) 303 if err != nil { 304 return nil, err 305 } 306 if !ok { 307 return nil, fmt.Errorf("module %s: not found, may need to run 'terraform init'", m.Name) 308 } 309 310 log.Printf("[TRACE] %q stored in %q", source, dir) 311 312 // expand and record the subDir for later 313 fullDir := dir 314 if subDir != "" { 315 fullDir, err = getter.SubdirGlob(dir, subDir) 316 if err != nil { 317 return nil, err 318 } 319 320 // +1 to account for the pathsep 321 if len(dir)+1 > len(fullDir) { 322 return nil, fmt.Errorf("invalid module storage path %q", fullDir) 323 } 324 subDir = fullDir[len(dir)+1:] 325 } 326 327 // add new info to the module record 328 mod.Key = key 329 mod.Dir = dir 330 mod.Root = subDir 331 332 // record the module in our manifest 333 if err := s.recordModule(mod); err != nil { 334 return nil, err 335 } 336 337 child, err := NewTreeModule(m.Name, fullDir) 338 if err != nil { 339 return nil, fmt.Errorf("module %s: %s", m.Name, err) 340 } 341 child.path = modPath 342 child.parent = t 343 child.version = mod.Version 344 child.source = m.Source 345 children[m.Name] = child 346 } 347 348 return children, nil 349 } 350 351 // inheritProviderConfig resolves all provider config inheritance after the 352 // tree is loaded. 353 // 354 // If there is a provider block without a config, look in the parent's Module 355 // block for a provider, and fetch that provider's configuration. If that 356 // doesn't exist, assume a default empty config. Implicit providers can still 357 // inherit their config all the way up from the root, so walk up the tree and 358 // copy the first matching provider into the module. 359 func (t *Tree) inheritProviderConfigs(stack []*Tree) { 360 // the recursive calls only append, so we don't need to worry about copying 361 // this slice. 362 stack = append(stack, t) 363 for _, c := range t.children { 364 c.inheritProviderConfigs(stack) 365 } 366 367 if len(stack) == 1 { 368 return 369 } 370 371 providers := make(map[string]*config.ProviderConfig) 372 missingProviders := make(map[string]bool) 373 374 for _, p := range t.config.ProviderConfigs { 375 providers[p.FullName()] = p 376 } 377 378 for _, r := range t.config.Resources { 379 p := r.ProviderFullName() 380 if _, ok := providers[p]; !(ok || strings.Contains(p, ".")) { 381 missingProviders[p] = true 382 } 383 } 384 385 // get our parent's module config block 386 parent := stack[len(stack)-2] 387 var parentModule *config.Module 388 for _, m := range parent.config.Modules { 389 if m.Name == t.name { 390 parentModule = m 391 break 392 } 393 } 394 395 if parentModule == nil { 396 panic("can't be a module without a parent module config") 397 } 398 399 // now look for providers that need a config 400 for p, pc := range providers { 401 if len(pc.RawConfig.RawMap()) > 0 { 402 log.Printf("[TRACE] provider %q has a config, continuing", p) 403 continue 404 } 405 406 // this provider has no config yet, check for one being passed in 407 parentProviderName, ok := parentModule.Providers[p] 408 if !ok { 409 continue 410 } 411 412 var parentProvider *config.ProviderConfig 413 // there's a config for us in the parent module 414 for _, pp := range parent.config.ProviderConfigs { 415 if pp.FullName() == parentProviderName { 416 parentProvider = pp 417 break 418 } 419 } 420 421 if parentProvider == nil { 422 // no config found, assume defaults 423 continue 424 } 425 426 // Copy it in, but set an interpolation Scope. 427 // An interpolation Scope always need to have "root" 428 pc.Path = append([]string{RootName}, parent.path...) 429 pc.RawConfig = parentProvider.RawConfig 430 log.Printf("[TRACE] provider %q inheriting config from %q", 431 strings.Join(append(t.Path(), pc.FullName()), "."), 432 strings.Join(append(parent.Path(), parentProvider.FullName()), "."), 433 ) 434 } 435 436 } 437 438 // Path is the full path to this tree. 439 func (t *Tree) Path() []string { 440 return t.path 441 } 442 443 // String gives a nice output to describe the tree. 444 func (t *Tree) String() string { 445 var result bytes.Buffer 446 path := strings.Join(t.path, ", ") 447 if path != "" { 448 path = fmt.Sprintf(" (path: %s)", path) 449 } 450 result.WriteString(t.Name() + path + "\n") 451 452 cs := t.Children() 453 if cs == nil { 454 result.WriteString(" not loaded") 455 } else { 456 // Go through each child and get its string value, then indent it 457 // by two. 458 for _, c := range cs { 459 r := strings.NewReader(c.String()) 460 scanner := bufio.NewScanner(r) 461 for scanner.Scan() { 462 result.WriteString(" ") 463 result.WriteString(scanner.Text()) 464 result.WriteString("\n") 465 } 466 } 467 } 468 469 return result.String() 470 } 471 472 // Validate does semantic checks on the entire tree of configurations. 473 // 474 // This will call the respective config.Config.Validate() functions as well 475 // as verifying things such as parameters/outputs between the various modules. 476 // 477 // Load must be called prior to calling Validate or an error will be returned. 478 func (t *Tree) Validate() error { 479 if !t.Loaded() { 480 return fmt.Errorf("tree must be loaded before calling Validate") 481 } 482 483 // If something goes wrong, here is our error template 484 newErr := &treeError{Name: []string{t.Name()}} 485 486 // Terraform core does not handle root module children named "root". 487 // We plan to fix this in the future but this bug was brought up in 488 // the middle of a release and we don't want to introduce wide-sweeping 489 // changes at that time. 490 if len(t.path) == 1 && t.name == "root" { 491 return fmt.Errorf("root module cannot contain module named 'root'") 492 } 493 494 // Validate our configuration first. 495 if err := t.config.Validate(); err != nil { 496 newErr.Add(err) 497 } 498 499 // If we're the root, we do extra validation. This validation usually 500 // requires the entire tree (since children don't have parent pointers). 501 if len(t.path) == 0 { 502 if err := t.validateProviderAlias(); err != nil { 503 newErr.Add(err) 504 } 505 } 506 507 // Get the child trees 508 children := t.Children() 509 510 // Validate all our children 511 for _, c := range children { 512 err := c.Validate() 513 if err == nil { 514 continue 515 } 516 517 verr, ok := err.(*treeError) 518 if !ok { 519 // Unknown error, just return... 520 return err 521 } 522 523 // Append ourselves to the error and then return 524 verr.Name = append(verr.Name, t.Name()) 525 newErr.AddChild(verr) 526 } 527 528 // Go over all the modules and verify that any parameters are valid 529 // variables into the module in question. 530 for _, m := range t.config.Modules { 531 tree, ok := children[m.Name] 532 if !ok { 533 // This should never happen because Load watches us 534 panic("module not found in children: " + m.Name) 535 } 536 537 // Build the variables that the module defines 538 requiredMap := make(map[string]struct{}) 539 varMap := make(map[string]struct{}) 540 for _, v := range tree.config.Variables { 541 varMap[v.Name] = struct{}{} 542 543 if v.Required() { 544 requiredMap[v.Name] = struct{}{} 545 } 546 } 547 548 // Compare to the keys in our raw config for the module 549 for k, _ := range m.RawConfig.Raw { 550 if _, ok := varMap[k]; !ok { 551 newErr.Add(fmt.Errorf( 552 "module %s: %s is not a valid parameter", 553 m.Name, k)) 554 } 555 556 // Remove the required 557 delete(requiredMap, k) 558 } 559 560 // If we have any required left over, they aren't set. 561 for k, _ := range requiredMap { 562 newErr.Add(fmt.Errorf( 563 "module %s: required variable %q not set", 564 m.Name, k)) 565 } 566 } 567 568 // Go over all the variables used and make sure that any module 569 // variables represent outputs properly. 570 for source, vs := range t.config.InterpolatedVariables() { 571 for _, v := range vs { 572 mv, ok := v.(*config.ModuleVariable) 573 if !ok { 574 continue 575 } 576 577 tree, ok := children[mv.Name] 578 if !ok { 579 newErr.Add(fmt.Errorf( 580 "%s: undefined module referenced %s", 581 source, mv.Name)) 582 continue 583 } 584 585 found := false 586 for _, o := range tree.config.Outputs { 587 if o.Name == mv.Field { 588 found = true 589 break 590 } 591 } 592 if !found { 593 newErr.Add(fmt.Errorf( 594 "%s: %s is not a valid output for module %s", 595 source, mv.Field, mv.Name)) 596 } 597 } 598 } 599 600 return newErr.ErrOrNil() 601 } 602 603 // versionedPathKey returns a path string with every levels full name, version 604 // and source encoded. This is to provide a unique key for our module storage, 605 // since submodules need to know which versions of their ancestor modules they 606 // are loaded from. 607 // For example, if module A has a subdirectory B, if module A's source or 608 // version is updated B's storage key must reflect this change in order for the 609 // correct version of B's source to be loaded. 610 func (t *Tree) versionedPathKey(m *Module) string { 611 path := make([]string, len(t.path)+1) 612 path[len(path)-1] = m.Name + ";" + m.Source 613 // We're going to load these in order for easier reading and debugging, but 614 // in practice they only need to be unique and consistent. 615 616 p := t 617 i := len(path) - 2 618 for ; i >= 0; i-- { 619 if p == nil { 620 break 621 } 622 // we may have been loaded under a blank Tree, so always check for a name 623 // too. 624 if p.name == "" { 625 break 626 } 627 seg := p.name 628 if p.version != "" { 629 seg += "#" + p.version 630 } 631 632 if p.source != "" { 633 seg += ";" + p.source 634 } 635 636 path[i] = seg 637 p = p.parent 638 } 639 640 key := strings.Join(path, "|") 641 return key 642 } 643 644 // treeError is an error use by Tree.Validate to accumulates all 645 // validation errors. 646 type treeError struct { 647 Name []string 648 Errs []error 649 Children []*treeError 650 } 651 652 func (e *treeError) Add(err error) { 653 e.Errs = append(e.Errs, err) 654 } 655 656 func (e *treeError) AddChild(err *treeError) { 657 e.Children = append(e.Children, err) 658 } 659 660 func (e *treeError) ErrOrNil() error { 661 if len(e.Errs) > 0 || len(e.Children) > 0 { 662 return e 663 } 664 return nil 665 } 666 667 func (e *treeError) Error() string { 668 name := strings.Join(e.Name, ".") 669 var out bytes.Buffer 670 fmt.Fprintf(&out, "module %s: ", name) 671 672 if len(e.Errs) == 1 { 673 // single like error 674 out.WriteString(e.Errs[0].Error()) 675 } else { 676 // multi-line error 677 for _, err := range e.Errs { 678 fmt.Fprintf(&out, "\n %s", err) 679 } 680 } 681 682 if len(e.Children) > 0 { 683 // start the next error on a new line 684 out.WriteString("\n ") 685 } 686 for _, child := range e.Children { 687 out.WriteString(child.Error()) 688 } 689 690 return out.String() 691 }