github.com/jenkins-x/jx/v2@v2.1.155/pkg/helm/values_tree.go (about) 1 package helm 2 3 import ( 4 "bytes" 5 "fmt" 6 "io/ioutil" 7 "os" 8 "path/filepath" 9 "strings" 10 "text/template" 11 12 "github.com/jenkins-x/jx/v2/pkg/config" 13 "github.com/jenkins-x/jx/v2/pkg/secreturl" 14 "github.com/jenkins-x/jx/v2/pkg/util" 15 "github.com/pkg/errors" 16 "k8s.io/helm/pkg/chartutil" 17 "k8s.io/helm/pkg/engine" 18 19 "github.com/ghodss/yaml" 20 21 "github.com/jenkins-x/jx-logging/pkg/log" 22 ) 23 24 //DefaultValuesTreeIgnores is the default set of ignored files for collapsing the values tree which are used if 25 // ignores is nil 26 var DefaultValuesTreeIgnores = []string{ 27 "templates/*", 28 } 29 30 // GenerateValues will generate a values.yaml file in dir. It scans all subdirectories for values.yaml files, 31 // and merges them into the values.yaml in the root directory, 32 // creating a nested key structure that matches the directory structure. 33 // Any keys used that match files with the same name in the directory ( 34 // and have empty values) will be inlined as block scalars. 35 // Standard UNIX glob patterns can be passed to IgnoreFile directories. 36 func GenerateValues(requirements *config.RequirementsConfig, funcMap template.FuncMap, dir string, ignores []string, verbose bool, secretURLClient secreturl.Client) ([]byte, chartutil.Values, error) { 37 info, err := os.Stat(dir) 38 if err != nil { 39 return nil, nil, err 40 } else if os.IsNotExist(err) { 41 return nil, nil, fmt.Errorf("%s does not exist", dir) 42 } else if !info.IsDir() { 43 return nil, nil, fmt.Errorf("%s is not a directory", dir) 44 } 45 46 // load the parameter values if there are any 47 params, err := LoadParameters(dir, secretURLClient) 48 if err != nil { 49 return nil, params, err 50 } 51 if funcMap == nil { 52 funcMap = NewFunctionMap() 53 } 54 if ignores == nil { 55 ignores = DefaultValuesTreeIgnores 56 } 57 files := make(map[string]map[string]string) 58 values := make(map[string]interface{}) 59 err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { 60 rPath, err := filepath.Rel(dir, path) 61 if err != nil { 62 return err 63 } 64 // Check if should IgnoreFile the path 65 if ignore, err := util.IgnoreFile(rPath, ignores); err != nil { 66 return err 67 } else if !ignore { 68 rDir, file := filepath.Split(rPath) 69 // For the root dir we just consider directories (which the walk func does for us) 70 if rDir != "" { 71 // If it's values.tmpl.yaml, then evalate it as a go template and parse it 72 if file == ValuesTemplateFileName { 73 b, err := ReadValuesYamlFileTemplateOutput(path, params, funcMap, requirements) 74 if err != nil { 75 return err 76 } 77 v := make(map[string]interface{}) 78 79 err = yaml.Unmarshal(b, &v) 80 if err != nil { 81 return err 82 } 83 if values[rDir] != nil { 84 return fmt.Errorf("already has a nested values map at %s when processing file %s", rDir, rPath) 85 } 86 values[rDir] = v 87 } else if file == ValuesFileName { 88 b, err := ioutil.ReadFile(path) 89 if err != nil { 90 return err 91 } 92 v := make(map[string]interface{}) 93 94 err = yaml.Unmarshal(b, &v) 95 if err != nil { 96 return err 97 } 98 if values[rDir] != nil { 99 return fmt.Errorf("already has a nested values map at %s when processing file %s", rDir, rPath) 100 } 101 values[rDir] = v 102 } else { 103 // for other files, just store a reference 104 if _, ok := files[rDir]; !ok { 105 files[rDir] = make(map[string]string) 106 } 107 files[rDir][file] = path 108 } 109 } 110 } else { 111 if verbose { 112 log.Logger().Infof("Ignoring %s", rPath) 113 } 114 } 115 return nil 116 }) 117 if err != nil { 118 return nil, params, err 119 } 120 // Load the root values.yaml 121 rootData := []byte{} 122 123 rootValuesFileName := filepath.Join(dir, ValuesTemplateFileName) 124 exists, err := util.FileExists(rootValuesFileName) 125 if err != nil { 126 return nil, params, errors.Wrapf(err, "failed to find %s", rootValuesFileName) 127 } 128 if exists { 129 rootData, err = ReadValuesYamlFileTemplateOutput(rootValuesFileName, params, funcMap, requirements) 130 if err != nil { 131 return nil, params, errors.Wrapf(err, "failed to render template of file %s", rootValuesFileName) 132 } 133 } else { 134 rootValuesFileName = filepath.Join(dir, ValuesFileName) 135 exists, err = util.FileExists(rootValuesFileName) 136 if err != nil { 137 return nil, params, errors.Wrapf(err, "failed to find %s", rootValuesFileName) 138 } 139 if exists { 140 rootData, err = ioutil.ReadFile(rootValuesFileName) 141 if err != nil { 142 return nil, params, errors.Wrapf(err, "failed to load file %s", rootValuesFileName) 143 } 144 } 145 } 146 rootValues, err := LoadValues(rootData) 147 if err != nil { 148 return nil, params, err 149 } 150 151 // externalFileHandler is used to read any inline any files that match into the values.yaml 152 externalFileHandler := func(path string, element map[string]interface{}, key string) error { 153 b, err := ioutil.ReadFile(path) 154 if err != nil { 155 return err 156 } 157 element[key] = string(b) 158 return nil 159 } 160 for p, v := range values { 161 // First, do file substitution - but only if any files were actually found 162 if dirFiles := files[p]; dirFiles != nil && len(dirFiles) > 0 { 163 err := HandleExternalFileRefs(v, dirFiles, "", externalFileHandler) 164 if err != nil { 165 return nil, params, err 166 } 167 } 168 169 // Then, merge the values to the root file 170 keys := strings.Split(strings.TrimSuffix(p, "/"), string(os.PathSeparator)) 171 x := rootValues 172 jsonPath := "$" 173 for i, k := range keys { 174 k = strings.TrimSuffix(k, "/") 175 jsonPath = fmt.Sprintf("%s.%s", jsonPath, k) 176 v1, ok1 := x[k] 177 if i < len(keys)-1 { 178 // Create the nested file object structure 179 if !ok1 { 180 // Easy, just create the nested object! 181 new := make(map[string]interface{}) 182 x[k] = new 183 x = new 184 } else { 185 // Need to do a type check 186 v2, ok2 := v1.(map[string]interface{}) 187 188 if !ok2 { 189 return nil, params, fmt.Errorf("%s is not an associative array", jsonPath) 190 } 191 x = v2 192 } 193 } else { 194 // Apply 195 x[k] = v 196 } 197 } 198 } 199 data, err := yaml.Marshal(rootValues) 200 if err != nil { 201 return nil, nil, errors.WithStack(err) 202 } 203 var text string 204 if secretURLClient != nil { 205 text, err = secretURLClient.ReplaceURIs(string(data)) 206 if err != nil { 207 return nil, nil, errors.WithStack(err) 208 } 209 } else { 210 text = string(data) 211 } 212 213 return []byte(text), params, err 214 } 215 216 // NewFunctionMap creates a new function map for values.tmpl.yaml templating 217 func NewFunctionMap() template.FuncMap { 218 funcMap := engine.FuncMap() 219 funcMap["hashPassword"] = util.HashPassword 220 funcMap["removeScheme"] = util.RemoveScheme 221 return funcMap 222 } 223 224 // ReadValuesYamlFileTemplateOutput evaluates the given values.yaml file as a go template and returns the output data 225 func ReadValuesYamlFileTemplateOutput(templateFile string, params chartutil.Values, funcMap template.FuncMap, requirements *config.RequirementsConfig) ([]byte, error) { 226 tmpl, err := template.New(ValuesTemplateFileName).Option("missingkey=error").Funcs(funcMap).ParseFiles(templateFile) 227 if err != nil { 228 return nil, errors.Wrapf(err, "failed to parse Secrets template: %s", templateFile) 229 } 230 231 requirementsMap, err := requirements.ToMap() 232 if err != nil { 233 return nil, errors.Wrapf(err, "failed turn requirements into a map: %v", requirements) 234 } 235 236 templateData := map[string]interface{}{ 237 "Parameters": params, 238 "Requirements": chartutil.Values(requirementsMap), 239 "Environments": chartutil.Values(requirements.EnvironmentMap()), 240 } 241 var buf bytes.Buffer 242 err = tmpl.Execute(&buf, templateData) 243 if err != nil { 244 return nil, errors.Wrapf(err, "failed to execute Secrets template: %s", templateFile) 245 } 246 data := buf.Bytes() 247 return data, nil 248 } 249 250 // HandleExternalFileRefs recursively scans the element map structure, 251 // looking for nested maps. If it finds keys that match any key-value pair in possibles it will call the handler. 252 // The jsonPath is used for referencing the path in the map structure when reporting errors. 253 func HandleExternalFileRefs(element interface{}, possibles map[string]string, jsonPath string, 254 handler func(path string, element map[string]interface{}, key string) error) error { 255 if jsonPath == "" { 256 // set zero value 257 jsonPath = "$" 258 } 259 if e, ok := element.(map[string]interface{}); ok { 260 for k, v := range e { 261 if paths, ok := possibles[k]; ok { 262 if v == nil || util.IsZeroOfUnderlyingType(v) { 263 // There is a filename in the directory structure that matches this key, and it has no value, 264 // so we handle it 265 err := handler(paths, e, k) 266 if err != nil { 267 return err 268 } 269 } else { 270 return fmt.Errorf("value at %s must be empty but is %v", jsonPath, v) 271 } 272 } else { 273 // keep on recursing 274 jsonPath = fmt.Sprintf("%s.%s", jsonPath, k) 275 err := HandleExternalFileRefs(v, possibles, jsonPath, handler) 276 if err != nil { 277 return err 278 } 279 } 280 } 281 } 282 // If it's not an object, we can't do much with it 283 return nil 284 }