github.com/opentofu/opentofu@v1.7.1/internal/instances/expander.go (about)

     1  // Copyright (c) The OpenTofu Authors
     2  // SPDX-License-Identifier: MPL-2.0
     3  // Copyright (c) 2023 HashiCorp, Inc.
     4  // SPDX-License-Identifier: MPL-2.0
     5  
     6  package instances
     7  
     8  import (
     9  	"fmt"
    10  	"sort"
    11  	"sync"
    12  
    13  	"github.com/opentofu/opentofu/internal/addrs"
    14  	"github.com/zclconf/go-cty/cty"
    15  )
    16  
    17  // Expander instances serve as a coordination point for gathering object
    18  // repetition values (count and for_each in configuration) and then later
    19  // making use of them to fully enumerate all of the instances of an object.
    20  //
    21  // The two repeatable object types in OpenTofu are modules and resources.
    22  // Because resources belong to modules and modules can nest inside other
    23  // modules, module expansion in particular has a recursive effect that can
    24  // cause deep objects to expand exponentially. Expander assumes that all
    25  // instances of a module have the same static objects inside, and that they
    26  // differ only in the repetition count for some of those objects.
    27  //
    28  // Expander is a synchronized object whose methods can be safely called
    29  // from concurrent threads of execution. However, it does expect a certain
    30  // sequence of operations which is normally obtained by the caller traversing
    31  // a dependency graph: each object must have its repetition mode set exactly
    32  // once, and this must be done before any calls that depend on the repetition
    33  // mode. In other words, the count or for_each expression value for a module
    34  // must be provided before any object nested directly or indirectly inside
    35  // that module can be expanded. If this ordering is violated, the methods
    36  // will panic to enforce internal consistency.
    37  //
    38  // The Expand* methods of Expander only work directly with modules and with
    39  // resources. Addresses for other objects that nest within modules but
    40  // do not themselves support repetition can be obtained by calling ExpandModule
    41  // with the containing module path and then producing one absolute instance
    42  // address per module instance address returned.
    43  type Expander struct {
    44  	mu   sync.RWMutex
    45  	exps *expanderModule
    46  }
    47  
    48  // NewExpander initializes and returns a new Expander, empty and ready to use.
    49  func NewExpander() *Expander {
    50  	return &Expander{
    51  		exps: newExpanderModule(),
    52  	}
    53  }
    54  
    55  // SetModuleSingle records that the given module call inside the given parent
    56  // module does not use any repetition arguments and is therefore a singleton.
    57  func (e *Expander) SetModuleSingle(parentAddr addrs.ModuleInstance, callAddr addrs.ModuleCall) {
    58  	e.setModuleExpansion(parentAddr, callAddr, expansionSingleVal)
    59  }
    60  
    61  // SetModuleCount records that the given module call inside the given parent
    62  // module instance uses the "count" repetition argument, with the given value.
    63  func (e *Expander) SetModuleCount(parentAddr addrs.ModuleInstance, callAddr addrs.ModuleCall, count int) {
    64  	e.setModuleExpansion(parentAddr, callAddr, expansionCount(count))
    65  }
    66  
    67  // SetModuleForEach records that the given module call inside the given parent
    68  // module instance uses the "for_each" repetition argument, with the given
    69  // map value.
    70  //
    71  // In the configuration language the for_each argument can also accept a set.
    72  // It's the caller's responsibility to convert that into an identity map before
    73  // calling this method.
    74  func (e *Expander) SetModuleForEach(parentAddr addrs.ModuleInstance, callAddr addrs.ModuleCall, mapping map[string]cty.Value) {
    75  	e.setModuleExpansion(parentAddr, callAddr, expansionForEach(mapping))
    76  }
    77  
    78  // SetResourceSingle records that the given resource inside the given module
    79  // does not use any repetition arguments and is therefore a singleton.
    80  func (e *Expander) SetResourceSingle(moduleAddr addrs.ModuleInstance, resourceAddr addrs.Resource) {
    81  	e.setResourceExpansion(moduleAddr, resourceAddr, expansionSingleVal)
    82  }
    83  
    84  // SetResourceCount records that the given resource inside the given module
    85  // uses the "count" repetition argument, with the given value.
    86  func (e *Expander) SetResourceCount(moduleAddr addrs.ModuleInstance, resourceAddr addrs.Resource, count int) {
    87  	e.setResourceExpansion(moduleAddr, resourceAddr, expansionCount(count))
    88  }
    89  
    90  // SetResourceForEach records that the given resource inside the given module
    91  // uses the "for_each" repetition argument, with the given map value.
    92  //
    93  // In the configuration language the for_each argument can also accept a set.
    94  // It's the caller's responsibility to convert that into an identity map before
    95  // calling this method.
    96  func (e *Expander) SetResourceForEach(moduleAddr addrs.ModuleInstance, resourceAddr addrs.Resource, mapping map[string]cty.Value) {
    97  	e.setResourceExpansion(moduleAddr, resourceAddr, expansionForEach(mapping))
    98  }
    99  
   100  // ExpandModule finds the exhaustive set of module instances resulting from
   101  // the expansion of the given module and all of its ancestor modules.
   102  //
   103  // All of the modules on the path to the identified module must already have
   104  // had their expansion registered using one of the SetModule* methods before
   105  // calling, or this method will panic.
   106  func (e *Expander) ExpandModule(addr addrs.Module) []addrs.ModuleInstance {
   107  	return e.expandModule(addr, false)
   108  }
   109  
   110  // expandModule allows skipping unexpanded module addresses by setting skipUnknown to true.
   111  // This is used by instances.Set, which is only concerned with the expanded
   112  // instances, and should not panic when looking up unknown addresses.
   113  func (e *Expander) expandModule(addr addrs.Module, skipUnknown bool) []addrs.ModuleInstance {
   114  	if len(addr) == 0 {
   115  		// Root module is always a singleton.
   116  		return singletonRootModule
   117  	}
   118  
   119  	e.mu.RLock()
   120  	defer e.mu.RUnlock()
   121  
   122  	// We're going to be dynamically growing ModuleInstance addresses, so
   123  	// we'll preallocate some space to do it so that for typical shallow
   124  	// module trees we won't need to reallocate this.
   125  	// (moduleInstances does plenty of allocations itself, so the benefit of
   126  	// pre-allocating this is marginal but it's not hard to do.)
   127  	parentAddr := make(addrs.ModuleInstance, 0, 4)
   128  	ret := e.exps.moduleInstances(addr, parentAddr, skipUnknown)
   129  	sort.SliceStable(ret, func(i, j int) bool {
   130  		return ret[i].Less(ret[j])
   131  	})
   132  	return ret
   133  }
   134  
   135  // GetDeepestExistingModuleInstance is a funny specialized function for
   136  // determining how many steps we can traverse through the given module instance
   137  // address before encountering an undeclared instance of a declared module.
   138  //
   139  // The result is the longest prefix of the given address which steps only
   140  // through module instances that exist.
   141  //
   142  // All of the modules on the given path must already have had their
   143  // expansion registered using one of the SetModule* methods before calling,
   144  // or this method will panic.
   145  func (e *Expander) GetDeepestExistingModuleInstance(given addrs.ModuleInstance) addrs.ModuleInstance {
   146  	exps := e.exps // start with the root module expansions
   147  	for i := 0; i < len(given); i++ {
   148  		step := given[i]
   149  		callName := step.Name
   150  		if _, ok := exps.moduleCalls[addrs.ModuleCall{Name: callName}]; !ok {
   151  			// This is a bug in the caller, because it should always register
   152  			// expansions for an object and all of its ancestors before requesting
   153  			// expansion of it.
   154  			panic(fmt.Sprintf("no expansion has been registered for %s", given[:i].Child(callName, addrs.NoKey)))
   155  		}
   156  
   157  		var ok bool
   158  		exps, ok = exps.childInstances[step]
   159  		if !ok {
   160  			// We've found a non-existing instance, so we're done.
   161  			return given[:i]
   162  		}
   163  	}
   164  
   165  	// If we complete the loop above without returning early then the entire
   166  	// given address refers to a declared module instance.
   167  	return given
   168  }
   169  
   170  // ExpandModuleResource finds the exhaustive set of resource instances resulting from
   171  // the expansion of the given resource and all of its containing modules.
   172  //
   173  // All of the modules on the path to the identified resource and the resource
   174  // itself must already have had their expansion registered using one of the
   175  // SetModule*/SetResource* methods before calling, or this method will panic.
   176  func (e *Expander) ExpandModuleResource(moduleAddr addrs.Module, resourceAddr addrs.Resource) []addrs.AbsResourceInstance {
   177  	e.mu.RLock()
   178  	defer e.mu.RUnlock()
   179  
   180  	// We're going to be dynamically growing ModuleInstance addresses, so
   181  	// we'll preallocate some space to do it so that for typical shallow
   182  	// module trees we won't need to reallocate this.
   183  	// (moduleInstances does plenty of allocations itself, so the benefit of
   184  	// pre-allocating this is marginal but it's not hard to do.)
   185  	moduleInstanceAddr := make(addrs.ModuleInstance, 0, 4)
   186  	ret := e.exps.moduleResourceInstances(moduleAddr, resourceAddr, moduleInstanceAddr)
   187  	sort.SliceStable(ret, func(i, j int) bool {
   188  		return ret[i].Less(ret[j])
   189  	})
   190  	return ret
   191  }
   192  
   193  // ExpandResource finds the set of resource instances resulting from
   194  // the expansion of the given resource within its module instance.
   195  //
   196  // All of the modules on the path to the identified resource and the resource
   197  // itself must already have had their expansion registered using one of the
   198  // SetModule*/SetResource* methods before calling, or this method will panic.
   199  //
   200  // ExpandModuleResource returns all instances of a resource across all
   201  // instances of its containing module, whereas this ExpandResource function
   202  // is more specific and only expands within a single module instance. If
   203  // any of the module instances selected in the module path of the given address
   204  // aren't valid for that module's expansion then ExpandResource returns an
   205  // empty result, reflecting that a non-existing module instance can never
   206  // contain any existing resource instances.
   207  func (e *Expander) ExpandResource(resourceAddr addrs.AbsResource) []addrs.AbsResourceInstance {
   208  	e.mu.RLock()
   209  	defer e.mu.RUnlock()
   210  
   211  	moduleInstanceAddr := make(addrs.ModuleInstance, 0, 4)
   212  	ret := e.exps.resourceInstances(resourceAddr.Module, resourceAddr.Resource, moduleInstanceAddr)
   213  	sort.SliceStable(ret, func(i, j int) bool {
   214  		return ret[i].Less(ret[j])
   215  	})
   216  	return ret
   217  }
   218  
   219  // GetModuleInstanceRepetitionData returns an object describing the values
   220  // that should be available for each.key, each.value, and count.index within
   221  // the call block for the given module instance.
   222  func (e *Expander) GetModuleInstanceRepetitionData(addr addrs.ModuleInstance) RepetitionData {
   223  	if len(addr) == 0 {
   224  		// The root module is always a singleton, so it has no repetition data.
   225  		return RepetitionData{}
   226  	}
   227  
   228  	e.mu.RLock()
   229  	defer e.mu.RUnlock()
   230  
   231  	parentMod := e.findModule(addr[:len(addr)-1])
   232  	lastStep := addr[len(addr)-1]
   233  	exp, ok := parentMod.moduleCalls[addrs.ModuleCall{Name: lastStep.Name}]
   234  	if !ok {
   235  		panic(fmt.Sprintf("no expansion has been registered for %s", addr))
   236  	}
   237  	return exp.repetitionData(lastStep.InstanceKey)
   238  }
   239  
   240  // GetResourceInstanceRepetitionData returns an object describing the values
   241  // that should be available for each.key, each.value, and count.index within
   242  // the definition block for the given resource instance.
   243  func (e *Expander) GetResourceInstanceRepetitionData(addr addrs.AbsResourceInstance) RepetitionData {
   244  	e.mu.RLock()
   245  	defer e.mu.RUnlock()
   246  
   247  	parentMod := e.findModule(addr.Module)
   248  	exp, ok := parentMod.resources[addr.Resource.Resource]
   249  	if !ok {
   250  		panic(fmt.Sprintf("no expansion has been registered for %s", addr.ContainingResource()))
   251  	}
   252  	return exp.repetitionData(addr.Resource.Key)
   253  }
   254  
   255  // AllInstances returns a set of all of the module and resource instances known
   256  // to the expander.
   257  //
   258  // It generally doesn't make sense to call this until everything has already
   259  // been fully expanded by calling the SetModule* and SetResource* functions.
   260  // After that, the returned set is a convenient small API only for querying
   261  // whether particular instance addresses appeared as a result of those
   262  // expansions.
   263  func (e *Expander) AllInstances() Set {
   264  	return Set{e}
   265  }
   266  
   267  func (e *Expander) findModule(moduleInstAddr addrs.ModuleInstance) *expanderModule {
   268  	// We expect that all of the modules on the path to our module instance
   269  	// should already have expansions registered.
   270  	mod := e.exps
   271  	for i, step := range moduleInstAddr {
   272  		next, ok := mod.childInstances[step]
   273  		if !ok {
   274  			// Top-down ordering of registration is part of the contract of
   275  			// Expander, so this is always indicative of a bug in the caller.
   276  			panic(fmt.Sprintf("no expansion has been registered for ancestor module %s", moduleInstAddr[:i+1]))
   277  		}
   278  		mod = next
   279  	}
   280  	return mod
   281  }
   282  
   283  func (e *Expander) setModuleExpansion(parentAddr addrs.ModuleInstance, callAddr addrs.ModuleCall, exp expansion) {
   284  	e.mu.Lock()
   285  	defer e.mu.Unlock()
   286  
   287  	mod := e.findModule(parentAddr)
   288  	if _, exists := mod.moduleCalls[callAddr]; exists {
   289  		panic(fmt.Sprintf("expansion already registered for %s", parentAddr.Child(callAddr.Name, addrs.NoKey)))
   290  	}
   291  	// We'll also pre-register the child instances so that later calls can
   292  	// populate them as the caller traverses the configuration tree.
   293  	for _, key := range exp.instanceKeys() {
   294  		step := addrs.ModuleInstanceStep{Name: callAddr.Name, InstanceKey: key}
   295  		mod.childInstances[step] = newExpanderModule()
   296  	}
   297  	mod.moduleCalls[callAddr] = exp
   298  }
   299  
   300  func (e *Expander) setResourceExpansion(parentAddr addrs.ModuleInstance, resourceAddr addrs.Resource, exp expansion) {
   301  	e.mu.Lock()
   302  	defer e.mu.Unlock()
   303  
   304  	mod := e.findModule(parentAddr)
   305  	if _, exists := mod.resources[resourceAddr]; exists {
   306  		panic(fmt.Sprintf("expansion already registered for %s", resourceAddr.Absolute(parentAddr)))
   307  	}
   308  	mod.resources[resourceAddr] = exp
   309  }
   310  
   311  func (e *Expander) knowsModuleInstance(want addrs.ModuleInstance) bool {
   312  	if want.IsRoot() {
   313  		return true // root module instance is always present
   314  	}
   315  
   316  	e.mu.Lock()
   317  	defer e.mu.Unlock()
   318  
   319  	return e.exps.knowsModuleInstance(want)
   320  }
   321  
   322  func (e *Expander) knowsModuleCall(want addrs.AbsModuleCall) bool {
   323  	e.mu.Lock()
   324  	defer e.mu.Unlock()
   325  
   326  	return e.exps.knowsModuleCall(want)
   327  }
   328  
   329  func (e *Expander) knowsResourceInstance(want addrs.AbsResourceInstance) bool {
   330  	e.mu.Lock()
   331  	defer e.mu.Unlock()
   332  
   333  	return e.exps.knowsResourceInstance(want)
   334  }
   335  
   336  func (e *Expander) knowsResource(want addrs.AbsResource) bool {
   337  	e.mu.Lock()
   338  	defer e.mu.Unlock()
   339  
   340  	return e.exps.knowsResource(want)
   341  }
   342  
   343  type expanderModule struct {
   344  	moduleCalls    map[addrs.ModuleCall]expansion
   345  	resources      map[addrs.Resource]expansion
   346  	childInstances map[addrs.ModuleInstanceStep]*expanderModule
   347  }
   348  
   349  func newExpanderModule() *expanderModule {
   350  	return &expanderModule{
   351  		moduleCalls:    make(map[addrs.ModuleCall]expansion),
   352  		resources:      make(map[addrs.Resource]expansion),
   353  		childInstances: make(map[addrs.ModuleInstanceStep]*expanderModule),
   354  	}
   355  }
   356  
   357  var singletonRootModule = []addrs.ModuleInstance{addrs.RootModuleInstance}
   358  
   359  // if moduleInstances is being used to lookup known instances after all
   360  // expansions have been done, set skipUnknown to true which allows addrs which
   361  // may not have been seen to return with no instances rather than panicking.
   362  func (m *expanderModule) moduleInstances(addr addrs.Module, parentAddr addrs.ModuleInstance, skipUnknown bool) []addrs.ModuleInstance {
   363  	callName := addr[0]
   364  	exp, ok := m.moduleCalls[addrs.ModuleCall{Name: callName}]
   365  	if !ok {
   366  		if skipUnknown {
   367  			return nil
   368  		}
   369  		// This is a bug in the caller, because it should always register
   370  		// expansions for an object and all of its ancestors before requesting
   371  		// expansion of it.
   372  		panic(fmt.Sprintf("no expansion has been registered for %s", parentAddr.Child(callName, addrs.NoKey)))
   373  	}
   374  
   375  	var ret []addrs.ModuleInstance
   376  
   377  	// If there's more than one step remaining then we need to traverse deeper.
   378  	if len(addr) > 1 {
   379  		for step, inst := range m.childInstances {
   380  			if step.Name != callName {
   381  				continue
   382  			}
   383  			instAddr := append(parentAddr, step)
   384  			ret = append(ret, inst.moduleInstances(addr[1:], instAddr, skipUnknown)...)
   385  		}
   386  		return ret
   387  	}
   388  
   389  	// Otherwise, we'll use the expansion from the final step to produce
   390  	// a sequence of addresses under this prefix.
   391  	for _, k := range exp.instanceKeys() {
   392  		// We're reusing the buffer under parentAddr as we recurse through
   393  		// the structure, so we need to copy it here to produce a final
   394  		// immutable slice to return.
   395  		full := make(addrs.ModuleInstance, 0, len(parentAddr)+1)
   396  		full = append(full, parentAddr...)
   397  		full = full.Child(callName, k)
   398  		ret = append(ret, full)
   399  	}
   400  	return ret
   401  }
   402  
   403  func (m *expanderModule) moduleResourceInstances(moduleAddr addrs.Module, resourceAddr addrs.Resource, parentAddr addrs.ModuleInstance) []addrs.AbsResourceInstance {
   404  	if len(moduleAddr) > 0 {
   405  		var ret []addrs.AbsResourceInstance
   406  		// We need to traverse through the module levels first, so we can
   407  		// then iterate resource expansions in the context of each module
   408  		// path leading to them.
   409  		callName := moduleAddr[0]
   410  		if _, ok := m.moduleCalls[addrs.ModuleCall{Name: callName}]; !ok {
   411  			// This is a bug in the caller, because it should always register
   412  			// expansions for an object and all of its ancestors before requesting
   413  			// expansion of it.
   414  			panic(fmt.Sprintf("no expansion has been registered for %s", parentAddr.Child(callName, addrs.NoKey)))
   415  		}
   416  
   417  		for step, inst := range m.childInstances {
   418  			if step.Name != callName {
   419  				continue
   420  			}
   421  			moduleInstAddr := append(parentAddr, step)
   422  			ret = append(ret, inst.moduleResourceInstances(moduleAddr[1:], resourceAddr, moduleInstAddr)...)
   423  		}
   424  		return ret
   425  	}
   426  
   427  	return m.onlyResourceInstances(resourceAddr, parentAddr)
   428  }
   429  
   430  func (m *expanderModule) resourceInstances(moduleAddr addrs.ModuleInstance, resourceAddr addrs.Resource, parentAddr addrs.ModuleInstance) []addrs.AbsResourceInstance {
   431  	if len(moduleAddr) > 0 {
   432  		// We need to traverse through the module levels first, using only the
   433  		// module instances for our specific resource, as the resource may not
   434  		// yet be expanded in all module instances.
   435  		step := moduleAddr[0]
   436  		callName := step.Name
   437  		if _, ok := m.moduleCalls[addrs.ModuleCall{Name: callName}]; !ok {
   438  			// This is a bug in the caller, because it should always register
   439  			// expansions for an object and all of its ancestors before requesting
   440  			// expansion of it.
   441  			panic(fmt.Sprintf("no expansion has been registered for %s", parentAddr.Child(callName, addrs.NoKey)))
   442  		}
   443  
   444  		if inst, ok := m.childInstances[step]; ok {
   445  			moduleInstAddr := append(parentAddr, step)
   446  			return inst.resourceInstances(moduleAddr[1:], resourceAddr, moduleInstAddr)
   447  		} else {
   448  			// If we have the module _call_ registered (as we checked above)
   449  			// but we don't have the given module _instance_ registered, that
   450  			// suggests that the module instance key in "step" is not declared
   451  			// by the current definition of this module call. That means the
   452  			// module instance doesn't exist at all, and therefore it can't
   453  			// possibly declare any resource instances either.
   454  			//
   455  			// For example, if we were asked about module.foo[0].aws_instance.bar
   456  			// but module.foo doesn't currently have count set, then there is no
   457  			// module.foo[0] at all, and therefore no aws_instance.bar
   458  			// instances inside it.
   459  			return nil
   460  		}
   461  	}
   462  	return m.onlyResourceInstances(resourceAddr, parentAddr)
   463  }
   464  
   465  func (m *expanderModule) onlyResourceInstances(resourceAddr addrs.Resource, parentAddr addrs.ModuleInstance) []addrs.AbsResourceInstance {
   466  	var ret []addrs.AbsResourceInstance
   467  	exp, ok := m.resources[resourceAddr]
   468  	if !ok {
   469  		panic(fmt.Sprintf("no expansion has been registered for %s", resourceAddr.Absolute(parentAddr)))
   470  	}
   471  
   472  	for _, k := range exp.instanceKeys() {
   473  		// We're reusing the buffer under parentAddr as we recurse through
   474  		// the structure, so we need to copy it here to produce a final
   475  		// immutable slice to return.
   476  		moduleAddr := make(addrs.ModuleInstance, len(parentAddr))
   477  		copy(moduleAddr, parentAddr)
   478  		ret = append(ret, resourceAddr.Instance(k).Absolute(moduleAddr))
   479  	}
   480  	return ret
   481  }
   482  
   483  func (m *expanderModule) getModuleInstance(want addrs.ModuleInstance) *expanderModule {
   484  	current := m
   485  	for _, step := range want {
   486  		next := current.childInstances[step]
   487  		if next == nil {
   488  			return nil
   489  		}
   490  		current = next
   491  	}
   492  	return current
   493  }
   494  
   495  func (m *expanderModule) knowsModuleInstance(want addrs.ModuleInstance) bool {
   496  	return m.getModuleInstance(want) != nil
   497  }
   498  
   499  func (m *expanderModule) knowsModuleCall(want addrs.AbsModuleCall) bool {
   500  	modInst := m.getModuleInstance(want.Module)
   501  	if modInst == nil {
   502  		return false
   503  	}
   504  	_, ret := modInst.moduleCalls[want.Call]
   505  	return ret
   506  }
   507  
   508  func (m *expanderModule) knowsResourceInstance(want addrs.AbsResourceInstance) bool {
   509  	modInst := m.getModuleInstance(want.Module)
   510  	if modInst == nil {
   511  		return false
   512  	}
   513  	resourceExp := modInst.resources[want.Resource.Resource]
   514  	if resourceExp == nil {
   515  		return false
   516  	}
   517  	for _, key := range resourceExp.instanceKeys() {
   518  		if key == want.Resource.Key {
   519  			return true
   520  		}
   521  	}
   522  	return false
   523  }
   524  
   525  func (m *expanderModule) knowsResource(want addrs.AbsResource) bool {
   526  	modInst := m.getModuleInstance(want.Module)
   527  	if modInst == nil {
   528  		return false
   529  	}
   530  	_, ret := modInst.resources[want.Resource]
   531  	return ret
   532  }