github.com/opentofu/opentofu@v1.7.1/internal/instances/expander.go (about) 1 // Copyright (c) The OpenTofu Authors 2 // SPDX-License-Identifier: MPL-2.0 3 // Copyright (c) 2023 HashiCorp, Inc. 4 // SPDX-License-Identifier: MPL-2.0 5 6 package instances 7 8 import ( 9 "fmt" 10 "sort" 11 "sync" 12 13 "github.com/opentofu/opentofu/internal/addrs" 14 "github.com/zclconf/go-cty/cty" 15 ) 16 17 // Expander instances serve as a coordination point for gathering object 18 // repetition values (count and for_each in configuration) and then later 19 // making use of them to fully enumerate all of the instances of an object. 20 // 21 // The two repeatable object types in OpenTofu are modules and resources. 22 // Because resources belong to modules and modules can nest inside other 23 // modules, module expansion in particular has a recursive effect that can 24 // cause deep objects to expand exponentially. Expander assumes that all 25 // instances of a module have the same static objects inside, and that they 26 // differ only in the repetition count for some of those objects. 27 // 28 // Expander is a synchronized object whose methods can be safely called 29 // from concurrent threads of execution. However, it does expect a certain 30 // sequence of operations which is normally obtained by the caller traversing 31 // a dependency graph: each object must have its repetition mode set exactly 32 // once, and this must be done before any calls that depend on the repetition 33 // mode. In other words, the count or for_each expression value for a module 34 // must be provided before any object nested directly or indirectly inside 35 // that module can be expanded. If this ordering is violated, the methods 36 // will panic to enforce internal consistency. 37 // 38 // The Expand* methods of Expander only work directly with modules and with 39 // resources. Addresses for other objects that nest within modules but 40 // do not themselves support repetition can be obtained by calling ExpandModule 41 // with the containing module path and then producing one absolute instance 42 // address per module instance address returned. 43 type Expander struct { 44 mu sync.RWMutex 45 exps *expanderModule 46 } 47 48 // NewExpander initializes and returns a new Expander, empty and ready to use. 49 func NewExpander() *Expander { 50 return &Expander{ 51 exps: newExpanderModule(), 52 } 53 } 54 55 // SetModuleSingle records that the given module call inside the given parent 56 // module does not use any repetition arguments and is therefore a singleton. 57 func (e *Expander) SetModuleSingle(parentAddr addrs.ModuleInstance, callAddr addrs.ModuleCall) { 58 e.setModuleExpansion(parentAddr, callAddr, expansionSingleVal) 59 } 60 61 // SetModuleCount records that the given module call inside the given parent 62 // module instance uses the "count" repetition argument, with the given value. 63 func (e *Expander) SetModuleCount(parentAddr addrs.ModuleInstance, callAddr addrs.ModuleCall, count int) { 64 e.setModuleExpansion(parentAddr, callAddr, expansionCount(count)) 65 } 66 67 // SetModuleForEach records that the given module call inside the given parent 68 // module instance uses the "for_each" repetition argument, with the given 69 // map value. 70 // 71 // In the configuration language the for_each argument can also accept a set. 72 // It's the caller's responsibility to convert that into an identity map before 73 // calling this method. 74 func (e *Expander) SetModuleForEach(parentAddr addrs.ModuleInstance, callAddr addrs.ModuleCall, mapping map[string]cty.Value) { 75 e.setModuleExpansion(parentAddr, callAddr, expansionForEach(mapping)) 76 } 77 78 // SetResourceSingle records that the given resource inside the given module 79 // does not use any repetition arguments and is therefore a singleton. 80 func (e *Expander) SetResourceSingle(moduleAddr addrs.ModuleInstance, resourceAddr addrs.Resource) { 81 e.setResourceExpansion(moduleAddr, resourceAddr, expansionSingleVal) 82 } 83 84 // SetResourceCount records that the given resource inside the given module 85 // uses the "count" repetition argument, with the given value. 86 func (e *Expander) SetResourceCount(moduleAddr addrs.ModuleInstance, resourceAddr addrs.Resource, count int) { 87 e.setResourceExpansion(moduleAddr, resourceAddr, expansionCount(count)) 88 } 89 90 // SetResourceForEach records that the given resource inside the given module 91 // uses the "for_each" repetition argument, with the given map value. 92 // 93 // In the configuration language the for_each argument can also accept a set. 94 // It's the caller's responsibility to convert that into an identity map before 95 // calling this method. 96 func (e *Expander) SetResourceForEach(moduleAddr addrs.ModuleInstance, resourceAddr addrs.Resource, mapping map[string]cty.Value) { 97 e.setResourceExpansion(moduleAddr, resourceAddr, expansionForEach(mapping)) 98 } 99 100 // ExpandModule finds the exhaustive set of module instances resulting from 101 // the expansion of the given module and all of its ancestor modules. 102 // 103 // All of the modules on the path to the identified module must already have 104 // had their expansion registered using one of the SetModule* methods before 105 // calling, or this method will panic. 106 func (e *Expander) ExpandModule(addr addrs.Module) []addrs.ModuleInstance { 107 return e.expandModule(addr, false) 108 } 109 110 // expandModule allows skipping unexpanded module addresses by setting skipUnknown to true. 111 // This is used by instances.Set, which is only concerned with the expanded 112 // instances, and should not panic when looking up unknown addresses. 113 func (e *Expander) expandModule(addr addrs.Module, skipUnknown bool) []addrs.ModuleInstance { 114 if len(addr) == 0 { 115 // Root module is always a singleton. 116 return singletonRootModule 117 } 118 119 e.mu.RLock() 120 defer e.mu.RUnlock() 121 122 // We're going to be dynamically growing ModuleInstance addresses, so 123 // we'll preallocate some space to do it so that for typical shallow 124 // module trees we won't need to reallocate this. 125 // (moduleInstances does plenty of allocations itself, so the benefit of 126 // pre-allocating this is marginal but it's not hard to do.) 127 parentAddr := make(addrs.ModuleInstance, 0, 4) 128 ret := e.exps.moduleInstances(addr, parentAddr, skipUnknown) 129 sort.SliceStable(ret, func(i, j int) bool { 130 return ret[i].Less(ret[j]) 131 }) 132 return ret 133 } 134 135 // GetDeepestExistingModuleInstance is a funny specialized function for 136 // determining how many steps we can traverse through the given module instance 137 // address before encountering an undeclared instance of a declared module. 138 // 139 // The result is the longest prefix of the given address which steps only 140 // through module instances that exist. 141 // 142 // All of the modules on the given path must already have had their 143 // expansion registered using one of the SetModule* methods before calling, 144 // or this method will panic. 145 func (e *Expander) GetDeepestExistingModuleInstance(given addrs.ModuleInstance) addrs.ModuleInstance { 146 exps := e.exps // start with the root module expansions 147 for i := 0; i < len(given); i++ { 148 step := given[i] 149 callName := step.Name 150 if _, ok := exps.moduleCalls[addrs.ModuleCall{Name: callName}]; !ok { 151 // This is a bug in the caller, because it should always register 152 // expansions for an object and all of its ancestors before requesting 153 // expansion of it. 154 panic(fmt.Sprintf("no expansion has been registered for %s", given[:i].Child(callName, addrs.NoKey))) 155 } 156 157 var ok bool 158 exps, ok = exps.childInstances[step] 159 if !ok { 160 // We've found a non-existing instance, so we're done. 161 return given[:i] 162 } 163 } 164 165 // If we complete the loop above without returning early then the entire 166 // given address refers to a declared module instance. 167 return given 168 } 169 170 // ExpandModuleResource finds the exhaustive set of resource instances resulting from 171 // the expansion of the given resource and all of its containing modules. 172 // 173 // All of the modules on the path to the identified resource and the resource 174 // itself must already have had their expansion registered using one of the 175 // SetModule*/SetResource* methods before calling, or this method will panic. 176 func (e *Expander) ExpandModuleResource(moduleAddr addrs.Module, resourceAddr addrs.Resource) []addrs.AbsResourceInstance { 177 e.mu.RLock() 178 defer e.mu.RUnlock() 179 180 // We're going to be dynamically growing ModuleInstance addresses, so 181 // we'll preallocate some space to do it so that for typical shallow 182 // module trees we won't need to reallocate this. 183 // (moduleInstances does plenty of allocations itself, so the benefit of 184 // pre-allocating this is marginal but it's not hard to do.) 185 moduleInstanceAddr := make(addrs.ModuleInstance, 0, 4) 186 ret := e.exps.moduleResourceInstances(moduleAddr, resourceAddr, moduleInstanceAddr) 187 sort.SliceStable(ret, func(i, j int) bool { 188 return ret[i].Less(ret[j]) 189 }) 190 return ret 191 } 192 193 // ExpandResource finds the set of resource instances resulting from 194 // the expansion of the given resource within its module instance. 195 // 196 // All of the modules on the path to the identified resource and the resource 197 // itself must already have had their expansion registered using one of the 198 // SetModule*/SetResource* methods before calling, or this method will panic. 199 // 200 // ExpandModuleResource returns all instances of a resource across all 201 // instances of its containing module, whereas this ExpandResource function 202 // is more specific and only expands within a single module instance. If 203 // any of the module instances selected in the module path of the given address 204 // aren't valid for that module's expansion then ExpandResource returns an 205 // empty result, reflecting that a non-existing module instance can never 206 // contain any existing resource instances. 207 func (e *Expander) ExpandResource(resourceAddr addrs.AbsResource) []addrs.AbsResourceInstance { 208 e.mu.RLock() 209 defer e.mu.RUnlock() 210 211 moduleInstanceAddr := make(addrs.ModuleInstance, 0, 4) 212 ret := e.exps.resourceInstances(resourceAddr.Module, resourceAddr.Resource, moduleInstanceAddr) 213 sort.SliceStable(ret, func(i, j int) bool { 214 return ret[i].Less(ret[j]) 215 }) 216 return ret 217 } 218 219 // GetModuleInstanceRepetitionData returns an object describing the values 220 // that should be available for each.key, each.value, and count.index within 221 // the call block for the given module instance. 222 func (e *Expander) GetModuleInstanceRepetitionData(addr addrs.ModuleInstance) RepetitionData { 223 if len(addr) == 0 { 224 // The root module is always a singleton, so it has no repetition data. 225 return RepetitionData{} 226 } 227 228 e.mu.RLock() 229 defer e.mu.RUnlock() 230 231 parentMod := e.findModule(addr[:len(addr)-1]) 232 lastStep := addr[len(addr)-1] 233 exp, ok := parentMod.moduleCalls[addrs.ModuleCall{Name: lastStep.Name}] 234 if !ok { 235 panic(fmt.Sprintf("no expansion has been registered for %s", addr)) 236 } 237 return exp.repetitionData(lastStep.InstanceKey) 238 } 239 240 // GetResourceInstanceRepetitionData returns an object describing the values 241 // that should be available for each.key, each.value, and count.index within 242 // the definition block for the given resource instance. 243 func (e *Expander) GetResourceInstanceRepetitionData(addr addrs.AbsResourceInstance) RepetitionData { 244 e.mu.RLock() 245 defer e.mu.RUnlock() 246 247 parentMod := e.findModule(addr.Module) 248 exp, ok := parentMod.resources[addr.Resource.Resource] 249 if !ok { 250 panic(fmt.Sprintf("no expansion has been registered for %s", addr.ContainingResource())) 251 } 252 return exp.repetitionData(addr.Resource.Key) 253 } 254 255 // AllInstances returns a set of all of the module and resource instances known 256 // to the expander. 257 // 258 // It generally doesn't make sense to call this until everything has already 259 // been fully expanded by calling the SetModule* and SetResource* functions. 260 // After that, the returned set is a convenient small API only for querying 261 // whether particular instance addresses appeared as a result of those 262 // expansions. 263 func (e *Expander) AllInstances() Set { 264 return Set{e} 265 } 266 267 func (e *Expander) findModule(moduleInstAddr addrs.ModuleInstance) *expanderModule { 268 // We expect that all of the modules on the path to our module instance 269 // should already have expansions registered. 270 mod := e.exps 271 for i, step := range moduleInstAddr { 272 next, ok := mod.childInstances[step] 273 if !ok { 274 // Top-down ordering of registration is part of the contract of 275 // Expander, so this is always indicative of a bug in the caller. 276 panic(fmt.Sprintf("no expansion has been registered for ancestor module %s", moduleInstAddr[:i+1])) 277 } 278 mod = next 279 } 280 return mod 281 } 282 283 func (e *Expander) setModuleExpansion(parentAddr addrs.ModuleInstance, callAddr addrs.ModuleCall, exp expansion) { 284 e.mu.Lock() 285 defer e.mu.Unlock() 286 287 mod := e.findModule(parentAddr) 288 if _, exists := mod.moduleCalls[callAddr]; exists { 289 panic(fmt.Sprintf("expansion already registered for %s", parentAddr.Child(callAddr.Name, addrs.NoKey))) 290 } 291 // We'll also pre-register the child instances so that later calls can 292 // populate them as the caller traverses the configuration tree. 293 for _, key := range exp.instanceKeys() { 294 step := addrs.ModuleInstanceStep{Name: callAddr.Name, InstanceKey: key} 295 mod.childInstances[step] = newExpanderModule() 296 } 297 mod.moduleCalls[callAddr] = exp 298 } 299 300 func (e *Expander) setResourceExpansion(parentAddr addrs.ModuleInstance, resourceAddr addrs.Resource, exp expansion) { 301 e.mu.Lock() 302 defer e.mu.Unlock() 303 304 mod := e.findModule(parentAddr) 305 if _, exists := mod.resources[resourceAddr]; exists { 306 panic(fmt.Sprintf("expansion already registered for %s", resourceAddr.Absolute(parentAddr))) 307 } 308 mod.resources[resourceAddr] = exp 309 } 310 311 func (e *Expander) knowsModuleInstance(want addrs.ModuleInstance) bool { 312 if want.IsRoot() { 313 return true // root module instance is always present 314 } 315 316 e.mu.Lock() 317 defer e.mu.Unlock() 318 319 return e.exps.knowsModuleInstance(want) 320 } 321 322 func (e *Expander) knowsModuleCall(want addrs.AbsModuleCall) bool { 323 e.mu.Lock() 324 defer e.mu.Unlock() 325 326 return e.exps.knowsModuleCall(want) 327 } 328 329 func (e *Expander) knowsResourceInstance(want addrs.AbsResourceInstance) bool { 330 e.mu.Lock() 331 defer e.mu.Unlock() 332 333 return e.exps.knowsResourceInstance(want) 334 } 335 336 func (e *Expander) knowsResource(want addrs.AbsResource) bool { 337 e.mu.Lock() 338 defer e.mu.Unlock() 339 340 return e.exps.knowsResource(want) 341 } 342 343 type expanderModule struct { 344 moduleCalls map[addrs.ModuleCall]expansion 345 resources map[addrs.Resource]expansion 346 childInstances map[addrs.ModuleInstanceStep]*expanderModule 347 } 348 349 func newExpanderModule() *expanderModule { 350 return &expanderModule{ 351 moduleCalls: make(map[addrs.ModuleCall]expansion), 352 resources: make(map[addrs.Resource]expansion), 353 childInstances: make(map[addrs.ModuleInstanceStep]*expanderModule), 354 } 355 } 356 357 var singletonRootModule = []addrs.ModuleInstance{addrs.RootModuleInstance} 358 359 // if moduleInstances is being used to lookup known instances after all 360 // expansions have been done, set skipUnknown to true which allows addrs which 361 // may not have been seen to return with no instances rather than panicking. 362 func (m *expanderModule) moduleInstances(addr addrs.Module, parentAddr addrs.ModuleInstance, skipUnknown bool) []addrs.ModuleInstance { 363 callName := addr[0] 364 exp, ok := m.moduleCalls[addrs.ModuleCall{Name: callName}] 365 if !ok { 366 if skipUnknown { 367 return nil 368 } 369 // This is a bug in the caller, because it should always register 370 // expansions for an object and all of its ancestors before requesting 371 // expansion of it. 372 panic(fmt.Sprintf("no expansion has been registered for %s", parentAddr.Child(callName, addrs.NoKey))) 373 } 374 375 var ret []addrs.ModuleInstance 376 377 // If there's more than one step remaining then we need to traverse deeper. 378 if len(addr) > 1 { 379 for step, inst := range m.childInstances { 380 if step.Name != callName { 381 continue 382 } 383 instAddr := append(parentAddr, step) 384 ret = append(ret, inst.moduleInstances(addr[1:], instAddr, skipUnknown)...) 385 } 386 return ret 387 } 388 389 // Otherwise, we'll use the expansion from the final step to produce 390 // a sequence of addresses under this prefix. 391 for _, k := range exp.instanceKeys() { 392 // We're reusing the buffer under parentAddr as we recurse through 393 // the structure, so we need to copy it here to produce a final 394 // immutable slice to return. 395 full := make(addrs.ModuleInstance, 0, len(parentAddr)+1) 396 full = append(full, parentAddr...) 397 full = full.Child(callName, k) 398 ret = append(ret, full) 399 } 400 return ret 401 } 402 403 func (m *expanderModule) moduleResourceInstances(moduleAddr addrs.Module, resourceAddr addrs.Resource, parentAddr addrs.ModuleInstance) []addrs.AbsResourceInstance { 404 if len(moduleAddr) > 0 { 405 var ret []addrs.AbsResourceInstance 406 // We need to traverse through the module levels first, so we can 407 // then iterate resource expansions in the context of each module 408 // path leading to them. 409 callName := moduleAddr[0] 410 if _, ok := m.moduleCalls[addrs.ModuleCall{Name: callName}]; !ok { 411 // This is a bug in the caller, because it should always register 412 // expansions for an object and all of its ancestors before requesting 413 // expansion of it. 414 panic(fmt.Sprintf("no expansion has been registered for %s", parentAddr.Child(callName, addrs.NoKey))) 415 } 416 417 for step, inst := range m.childInstances { 418 if step.Name != callName { 419 continue 420 } 421 moduleInstAddr := append(parentAddr, step) 422 ret = append(ret, inst.moduleResourceInstances(moduleAddr[1:], resourceAddr, moduleInstAddr)...) 423 } 424 return ret 425 } 426 427 return m.onlyResourceInstances(resourceAddr, parentAddr) 428 } 429 430 func (m *expanderModule) resourceInstances(moduleAddr addrs.ModuleInstance, resourceAddr addrs.Resource, parentAddr addrs.ModuleInstance) []addrs.AbsResourceInstance { 431 if len(moduleAddr) > 0 { 432 // We need to traverse through the module levels first, using only the 433 // module instances for our specific resource, as the resource may not 434 // yet be expanded in all module instances. 435 step := moduleAddr[0] 436 callName := step.Name 437 if _, ok := m.moduleCalls[addrs.ModuleCall{Name: callName}]; !ok { 438 // This is a bug in the caller, because it should always register 439 // expansions for an object and all of its ancestors before requesting 440 // expansion of it. 441 panic(fmt.Sprintf("no expansion has been registered for %s", parentAddr.Child(callName, addrs.NoKey))) 442 } 443 444 if inst, ok := m.childInstances[step]; ok { 445 moduleInstAddr := append(parentAddr, step) 446 return inst.resourceInstances(moduleAddr[1:], resourceAddr, moduleInstAddr) 447 } else { 448 // If we have the module _call_ registered (as we checked above) 449 // but we don't have the given module _instance_ registered, that 450 // suggests that the module instance key in "step" is not declared 451 // by the current definition of this module call. That means the 452 // module instance doesn't exist at all, and therefore it can't 453 // possibly declare any resource instances either. 454 // 455 // For example, if we were asked about module.foo[0].aws_instance.bar 456 // but module.foo doesn't currently have count set, then there is no 457 // module.foo[0] at all, and therefore no aws_instance.bar 458 // instances inside it. 459 return nil 460 } 461 } 462 return m.onlyResourceInstances(resourceAddr, parentAddr) 463 } 464 465 func (m *expanderModule) onlyResourceInstances(resourceAddr addrs.Resource, parentAddr addrs.ModuleInstance) []addrs.AbsResourceInstance { 466 var ret []addrs.AbsResourceInstance 467 exp, ok := m.resources[resourceAddr] 468 if !ok { 469 panic(fmt.Sprintf("no expansion has been registered for %s", resourceAddr.Absolute(parentAddr))) 470 } 471 472 for _, k := range exp.instanceKeys() { 473 // We're reusing the buffer under parentAddr as we recurse through 474 // the structure, so we need to copy it here to produce a final 475 // immutable slice to return. 476 moduleAddr := make(addrs.ModuleInstance, len(parentAddr)) 477 copy(moduleAddr, parentAddr) 478 ret = append(ret, resourceAddr.Instance(k).Absolute(moduleAddr)) 479 } 480 return ret 481 } 482 483 func (m *expanderModule) getModuleInstance(want addrs.ModuleInstance) *expanderModule { 484 current := m 485 for _, step := range want { 486 next := current.childInstances[step] 487 if next == nil { 488 return nil 489 } 490 current = next 491 } 492 return current 493 } 494 495 func (m *expanderModule) knowsModuleInstance(want addrs.ModuleInstance) bool { 496 return m.getModuleInstance(want) != nil 497 } 498 499 func (m *expanderModule) knowsModuleCall(want addrs.AbsModuleCall) bool { 500 modInst := m.getModuleInstance(want.Module) 501 if modInst == nil { 502 return false 503 } 504 _, ret := modInst.moduleCalls[want.Call] 505 return ret 506 } 507 508 func (m *expanderModule) knowsResourceInstance(want addrs.AbsResourceInstance) bool { 509 modInst := m.getModuleInstance(want.Module) 510 if modInst == nil { 511 return false 512 } 513 resourceExp := modInst.resources[want.Resource.Resource] 514 if resourceExp == nil { 515 return false 516 } 517 for _, key := range resourceExp.instanceKeys() { 518 if key == want.Resource.Key { 519 return true 520 } 521 } 522 return false 523 } 524 525 func (m *expanderModule) knowsResource(want addrs.AbsResource) bool { 526 modInst := m.getModuleInstance(want.Module) 527 if modInst == nil { 528 return false 529 } 530 _, ret := modInst.resources[want.Resource] 531 return ret 532 }