github.com/GoogleCloudPlatform/terraformer@v0.8.18/terraformutils/utils.go (about) 1 // Copyright 2018 The Terraformer Authors. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package terraformutils 16 17 import ( 18 "bytes" 19 "log" 20 "sync" 21 22 "github.com/GoogleCloudPlatform/terraformer/terraformutils/providerwrapper" 23 24 "github.com/hashicorp/terraform/terraform" 25 ) 26 27 type BaseResource struct { 28 Tags map[string]string `json:"tags,omitempty"` 29 } 30 31 func NewTfState(resources []Resource) *terraform.State { 32 tfstate := &terraform.State{ 33 Version: terraform.StateVersion, 34 TFVersion: terraform.VersionString(), //nolint 35 Serial: 1, 36 } 37 outputs := map[string]*terraform.OutputState{} 38 for _, r := range resources { 39 for k, v := range r.Outputs { 40 outputs[k] = v 41 } 42 } 43 tfstate.Modules = []*terraform.ModuleState{ 44 { 45 Path: []string{"root"}, 46 Resources: map[string]*terraform.ResourceState{}, 47 Outputs: outputs, 48 }, 49 } 50 for _, resource := range resources { 51 resourceState := &terraform.ResourceState{ 52 Type: resource.InstanceInfo.Type, 53 Primary: resource.InstanceState, 54 Provider: "provider." + resource.Provider, 55 } 56 tfstate.Modules[0].Resources[resource.InstanceInfo.Type+"."+resource.ResourceName] = resourceState 57 } 58 return tfstate 59 } 60 61 func PrintTfState(resources []Resource) ([]byte, error) { 62 state := NewTfState(resources) 63 var buf bytes.Buffer 64 err := terraform.WriteState(state, &buf) 65 return buf.Bytes(), err 66 } 67 68 func RefreshResources(resources []*Resource, provider *providerwrapper.ProviderWrapper, slowProcessingResources [][]*Resource) ([]*Resource, error) { 69 refreshedResources := []*Resource{} 70 input := make(chan *Resource, len(resources)) 71 var wg sync.WaitGroup 72 poolSize := 15 73 for i := range resources { 74 wg.Add(1) 75 input <- resources[i] 76 } 77 close(input) 78 79 for i := 0; i < poolSize; i++ { 80 go RefreshResourceWorker(input, &wg, provider) 81 } 82 83 spInputs := []chan *Resource{} 84 for i, resourceGroup := range slowProcessingResources { 85 spInputs = append(spInputs, make(chan *Resource, len(resourceGroup))) 86 for j := range resourceGroup { 87 spInputs[i] <- resourceGroup[j] 88 } 89 close(spInputs[i]) 90 } 91 92 for i := 0; i < len(spInputs); i++ { 93 wg.Add(len(slowProcessingResources[i])) 94 go RefreshResourceWorker(spInputs[i], &wg, provider) 95 } 96 97 wg.Wait() 98 for _, r := range resources { 99 if r.InstanceState != nil && r.InstanceState.ID != "" { 100 refreshedResources = append(refreshedResources, r) 101 } else { 102 log.Printf("ERROR: Unable to refresh resource %s", r.ResourceName) 103 } 104 } 105 106 for _, resourceGroup := range slowProcessingResources { 107 for i := range resourceGroup { 108 r := resourceGroup[i] 109 if r.InstanceState != nil && r.InstanceState.ID != "" { 110 refreshedResources = append(refreshedResources, r) 111 } else { 112 log.Printf("ERROR: Unable to refresh resource %s", r.ResourceName) 113 } 114 } 115 } 116 return refreshedResources, nil 117 } 118 119 func RefreshResourcesByProvider(providersMapping *ProvidersMapping, providerWrapper *providerwrapper.ProviderWrapper) error { 120 allResources := providersMapping.ShuffleResources() 121 slowProcessingResources := make(map[ProviderGenerator][]*Resource) 122 regularResources := []*Resource{} 123 for i := range allResources { 124 resource := allResources[i] 125 if resource.SlowQueryRequired { 126 provider := providersMapping.MatchProvider(resource) 127 if slowProcessingResources[provider] == nil { 128 slowProcessingResources[provider] = []*Resource{} 129 } 130 slowProcessingResources[provider] = append(slowProcessingResources[provider], resource) 131 } else { 132 regularResources = append(regularResources, resource) 133 } 134 } 135 136 var spResourcesList [][]*Resource 137 for p := range slowProcessingResources { 138 spResourcesList = append(spResourcesList, slowProcessingResources[p]) 139 } 140 141 refreshedResources, err := RefreshResources(regularResources, providerWrapper, spResourcesList) 142 if err != nil { 143 return err 144 } 145 146 providersMapping.SetResources(refreshedResources) 147 return nil 148 } 149 150 func RefreshResourceWorker(input chan *Resource, wg *sync.WaitGroup, provider *providerwrapper.ProviderWrapper) { 151 for r := range input { 152 log.Println("Refreshing state...", r.InstanceInfo.Id) 153 r.Refresh(provider) 154 wg.Done() 155 } 156 } 157 158 func IgnoreKeys(resourcesTypes []string, p *providerwrapper.ProviderWrapper) map[string][]string { 159 readOnlyAttributes, err := p.GetReadOnlyAttributes(resourcesTypes) 160 if err != nil { 161 log.Println("plugin error 2:", err) 162 return map[string][]string{} 163 } 164 return readOnlyAttributes 165 } 166 167 func ParseFilterValues(value string) []string { 168 var values []string 169 170 valueBuffering := true 171 wrapped := false 172 var valueBuffer []byte 173 for i := 0; i < len(value); i++ { 174 if value[i] == '\'' { 175 wrapped = !wrapped 176 continue 177 } else if value[i] == ':' { 178 if len(valueBuffer) == 0 { 179 continue 180 } else if valueBuffering && !wrapped { 181 values = append(values, string(valueBuffer)) 182 valueBuffering = false 183 valueBuffer = []byte{} 184 continue 185 } 186 } 187 valueBuffering = true 188 valueBuffer = append(valueBuffer, value[i]) 189 } 190 if len(valueBuffer) > 0 { 191 values = append(values, string(valueBuffer)) 192 } 193 194 return values 195 } 196 197 func FilterCleanup(s *Service, isInitial bool) { 198 if len(s.Filter) == 0 { 199 return 200 } 201 var newListOfResources []Resource 202 for _, resource := range s.Resources { 203 allPredicatesTrue := true 204 for _, filter := range s.Filter { 205 if filter.isInitial() == isInitial { 206 allPredicatesTrue = allPredicatesTrue && filter.Filter(resource) 207 } 208 } 209 if allPredicatesTrue && !ContainsResource(newListOfResources, resource) { 210 newListOfResources = append(newListOfResources, resource) 211 } 212 } 213 s.Resources = newListOfResources 214 } 215 216 func ContainsResource(s []Resource, e Resource) bool { 217 for _, a := range s { 218 if a.InstanceInfo.Id == e.InstanceInfo.Id { 219 return true 220 } 221 } 222 return false 223 }