github.com/jaylevin/jenkins-library@v1.230.4/pkg/config/config.go (about) 1 package config 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "io" 7 "io/ioutil" 8 "net/http" 9 "net/url" 10 "os" 11 "reflect" 12 "regexp" 13 "strings" 14 15 piperhttp "github.com/SAP/jenkins-library/pkg/http" 16 "github.com/SAP/jenkins-library/pkg/log" 17 18 "github.com/ghodss/yaml" 19 "github.com/google/go-cmp/cmp" 20 "github.com/pkg/errors" 21 ) 22 23 // Config defines the structure of the config files 24 type Config struct { 25 CustomDefaults []string `json:"customDefaults,omitempty"` 26 General map[string]interface{} `json:"general"` 27 Stages map[string]map[string]interface{} `json:"stages"` 28 Steps map[string]map[string]interface{} `json:"steps"` 29 Hooks map[string]interface{} `json:"hooks,omitempty"` 30 defaults PipelineDefaults 31 initialized bool 32 accessTokens map[string]string 33 openFile func(s string, t map[string]string) (io.ReadCloser, error) 34 vaultCredentials VaultCredentials 35 } 36 37 // StepConfig defines the structure for merged step configuration 38 type StepConfig struct { 39 Config map[string]interface{} 40 HookConfig map[string]interface{} 41 } 42 43 // ReadConfig loads config and returns its content 44 func (c *Config) ReadConfig(configuration io.ReadCloser) error { 45 defer configuration.Close() 46 47 content, err := ioutil.ReadAll(configuration) 48 if err != nil { 49 return errors.Wrapf(err, "error reading %v", configuration) 50 } 51 52 err = yaml.Unmarshal(content, &c) 53 if err != nil { 54 return NewParseError(fmt.Sprintf("format of configuration is invalid %q: %v", content, err)) 55 } 56 return nil 57 } 58 59 // ApplyAliasConfig adds configuration values available on aliases to primary configuration parameters 60 func (c *Config) ApplyAliasConfig(parameters []StepParameters, secrets []StepSecrets, filters StepFilters, stageName, stepName string, stepAliases []Alias) { 61 // copy configuration from step alias to correct step 62 if len(stepAliases) > 0 { 63 c.copyStepAliasConfig(stepName, stepAliases) 64 } 65 for _, p := range parameters { 66 c.General = setParamValueFromAlias(stepName, c.General, filters.General, p.Name, p.Aliases) 67 if c.Stages[stageName] != nil { 68 c.Stages[stageName] = setParamValueFromAlias(stepName, c.Stages[stageName], filters.Stages, p.Name, p.Aliases) 69 } 70 if c.Steps[stepName] != nil { 71 c.Steps[stepName] = setParamValueFromAlias(stepName, c.Steps[stepName], filters.Steps, p.Name, p.Aliases) 72 } 73 } 74 for _, s := range secrets { 75 c.General = setParamValueFromAlias(stepName, c.General, filters.General, s.Name, s.Aliases) 76 if c.Stages[stageName] != nil { 77 c.Stages[stageName] = setParamValueFromAlias(stepName, c.Stages[stageName], filters.Stages, s.Name, s.Aliases) 78 } 79 if c.Steps[stepName] != nil { 80 c.Steps[stepName] = setParamValueFromAlias(stepName, c.Steps[stepName], filters.Steps, s.Name, s.Aliases) 81 } 82 } 83 } 84 85 func setParamValueFromAlias(stepName string, configMap map[string]interface{}, filter []string, name string, aliases []Alias) map[string]interface{} { 86 if configMap != nil && configMap[name] == nil && sliceContains(filter, name) { 87 for _, a := range aliases { 88 aliasVal := getDeepAliasValue(configMap, a.Name) 89 if aliasVal != nil { 90 configMap[name] = aliasVal 91 if a.Deprecated { 92 log.Entry().Warningf("[WARNING] The parameter '%v' is DEPRECATED, use '%v' instead. (%v/%v)", a.Name, name, log.LibraryName, stepName) 93 } 94 } 95 if configMap[name] != nil { 96 return configMap 97 } 98 } 99 } 100 return configMap 101 } 102 103 func getDeepAliasValue(configMap map[string]interface{}, key string) interface{} { 104 parts := strings.Split(key, "/") 105 if len(parts) > 1 { 106 if configMap[parts[0]] == nil { 107 return nil 108 } 109 110 paramValueType := reflect.ValueOf(configMap[parts[0]]) 111 if paramValueType.Kind() != reflect.Map { 112 log.Entry().Debugf("Ignoring alias '%v' as '%v' is not pointing to a map.", key, parts[0]) 113 return nil 114 } 115 return getDeepAliasValue(configMap[parts[0]].(map[string]interface{}), strings.Join(parts[1:], "/")) 116 } 117 return configMap[key] 118 } 119 120 func (c *Config) copyStepAliasConfig(stepName string, stepAliases []Alias) { 121 for _, stepAlias := range stepAliases { 122 if c.Steps[stepAlias.Name] != nil { 123 if stepAlias.Deprecated { 124 log.Entry().WithField("package", "SAP/jenkins-library/pkg/config").Warningf("DEPRECATION NOTICE: step configuration available for deprecated step '%v'. Please remove or move configuration to step '%v'!", stepAlias.Name, stepName) 125 } 126 for paramName, paramValue := range c.Steps[stepAlias.Name] { 127 if c.Steps[stepName] == nil { 128 c.Steps[stepName] = map[string]interface{}{} 129 } 130 if c.Steps[stepName][paramName] == nil { 131 c.Steps[stepName][paramName] = paramValue 132 } 133 } 134 } 135 } 136 } 137 138 // InitializeConfig prepares the config object, i.e. loading content, etc. 139 func (c *Config) InitializeConfig(configuration io.ReadCloser, defaults []io.ReadCloser, ignoreCustomDefaults bool) error { 140 if configuration != nil { 141 if err := c.ReadConfig(configuration); err != nil { 142 return errors.Wrap(err, "failed to parse custom pipeline configuration") 143 } 144 } 145 146 // consider custom defaults defined in config.yml unless told otherwise 147 if ignoreCustomDefaults { 148 log.Entry().Info("Ignoring custom defaults from pipeline config") 149 } else if c.CustomDefaults != nil && len(c.CustomDefaults) > 0 { 150 if c.openFile == nil { 151 c.openFile = OpenPiperFile 152 } 153 for _, f := range c.CustomDefaults { 154 fc, err := c.openFile(f, c.accessTokens) 155 if err != nil { 156 return errors.Wrapf(err, "getting default '%v' failed", f) 157 } 158 defaults = append(defaults, fc) 159 } 160 } 161 162 if err := c.defaults.ReadPipelineDefaults(defaults); err != nil { 163 return errors.Wrap(err, "failed to read default configuration") 164 } 165 c.initialized = true 166 return nil 167 } 168 169 // GetStepConfig provides merged step configuration using defaults, config, if available 170 func (c *Config) GetStepConfig(flagValues map[string]interface{}, paramJSON string, configuration io.ReadCloser, defaults []io.ReadCloser, ignoreCustomDefaults bool, filters StepFilters, metadata StepData, envParameters map[string]interface{}, stageName, stepName string) (StepConfig, error) { 171 parameters := metadata.Spec.Inputs.Parameters 172 secrets := metadata.Spec.Inputs.Secrets 173 stepAliases := metadata.Metadata.Aliases 174 175 var stepConfig StepConfig 176 var err error 177 178 if !c.initialized { 179 err = c.InitializeConfig(configuration, defaults, ignoreCustomDefaults) 180 if err != nil { 181 return StepConfig{}, err 182 } 183 } 184 185 c.ApplyAliasConfig(parameters, secrets, filters, stageName, stepName, stepAliases) 186 187 // initialize with defaults from step.yaml 188 stepConfig.mixInStepDefaults(parameters) 189 190 // merge parameters provided by Piper environment 191 stepConfig.mixIn(envParameters, filters.All) 192 stepConfig.mixIn(envParameters, ReportingParameters.getReportingFilter()) 193 194 // read defaults & merge general -> steps (-> general -> steps ...) 195 for _, def := range c.defaults.Defaults { 196 def.ApplyAliasConfig(parameters, secrets, filters, stageName, stepName, stepAliases) 197 stepConfig.mixIn(def.General, filters.General) 198 stepConfig.mixIn(def.Steps[stepName], filters.Steps) 199 stepConfig.mixIn(def.Stages[stageName], filters.Steps) 200 stepConfig.mixinVaultConfig(parameters, def.General, def.Steps[stepName], def.Stages[stageName]) 201 reportingConfig, err := cloneConfig(&def) 202 if err != nil { 203 return StepConfig{}, err 204 } 205 reportingConfig.ApplyAliasConfig(ReportingParameters.Parameters, []StepSecrets{}, ReportingParameters.getStepFilters(), stageName, stepName, []Alias{}) 206 stepConfig.mixinReportingConfig(reportingConfig.General, reportingConfig.Steps[stepName], reportingConfig.Stages[stageName]) 207 208 stepConfig.mixInHookConfig(def.Hooks) 209 } 210 211 // read config & merge - general -> steps -> stages 212 stepConfig.mixIn(c.General, filters.General) 213 stepConfig.mixIn(c.Steps[stepName], filters.Steps) 214 stepConfig.mixIn(c.Stages[stageName], filters.Stages) 215 216 // merge parameters provided via env vars 217 stepConfig.mixIn(envValues(filters.All), filters.All) 218 219 // if parameters are provided in JSON format merge them 220 if len(paramJSON) != 0 { 221 var params map[string]interface{} 222 err := json.Unmarshal([]byte(paramJSON), ¶ms) 223 if err != nil { 224 log.Entry().Warnf("failed to parse parameters from environment: %v", err) 225 } else { 226 // apply aliases 227 for _, p := range parameters { 228 params = setParamValueFromAlias(stepName, params, filters.Parameters, p.Name, p.Aliases) 229 } 230 for _, s := range secrets { 231 params = setParamValueFromAlias(stepName, params, filters.Parameters, s.Name, s.Aliases) 232 } 233 234 stepConfig.mixIn(params, filters.Parameters) 235 } 236 } 237 238 // merge command line flags 239 if flagValues != nil { 240 stepConfig.mixIn(flagValues, filters.Parameters) 241 } 242 243 if verbose, ok := stepConfig.Config["verbose"].(bool); ok && verbose { 244 log.SetVerbose(verbose) 245 } else if !ok && stepConfig.Config["verbose"] != nil { 246 log.Entry().Warnf("invalid value for parameter verbose: '%v'", stepConfig.Config["verbose"]) 247 } 248 249 stepConfig.mixinVaultConfig(parameters, c.General, c.Steps[stepName], c.Stages[stageName]) 250 251 reportingConfig, err := cloneConfig(c) 252 if err != nil { 253 return StepConfig{}, err 254 } 255 reportingConfig.ApplyAliasConfig(ReportingParameters.Parameters, []StepSecrets{}, ReportingParameters.getStepFilters(), stageName, stepName, []Alias{}) 256 stepConfig.mixinReportingConfig(reportingConfig.General, reportingConfig.Steps[stepName], reportingConfig.Stages[stageName]) 257 258 // check whether vault should be skipped 259 if skip, ok := stepConfig.Config["skipVault"].(bool); !ok || !skip { 260 // fetch secrets from vault 261 vaultClient, err := getVaultClientFromConfig(stepConfig, c.vaultCredentials) 262 if err != nil { 263 return StepConfig{}, err 264 } 265 if vaultClient != nil { 266 defer vaultClient.MustRevokeToken() 267 resolveAllVaultReferences(&stepConfig, vaultClient, append(parameters, ReportingParameters.Parameters...)) 268 resolveVaultTestCredentials(&stepConfig, vaultClient) 269 resolveVaultCredentials(&stepConfig, vaultClient) 270 } 271 } 272 273 // finally do the condition evaluation post processing 274 for _, p := range parameters { 275 if len(p.Conditions) > 0 { 276 for _, cond := range p.Conditions { 277 for _, param := range cond.Params { 278 // retrieve configuration value of condition parameter 279 dependentValue := stepConfig.Config[param.Name] 280 // check if configuration of condition parameter matches the value 281 // so far string-equals condition is assumed here 282 // if so and if no config applied yet, then try to apply the value 283 if cmp.Equal(dependentValue, param.Value) && stepConfig.Config[p.Name] == nil { 284 subMap, ok := stepConfig.Config[dependentValue.(string)].(map[string]interface{}) 285 if ok && subMap[p.Name] != nil { 286 stepConfig.Config[p.Name] = subMap[p.Name] 287 } 288 } 289 } 290 } 291 } 292 } 293 return stepConfig, nil 294 } 295 296 // SetVaultCredentials sets the appRoleID and the appRoleSecretID or the vaultTokento load additional 297 // configuration from vault 298 // Either appRoleID and appRoleSecretID or vaultToken must be specified. 299 func (c *Config) SetVaultCredentials(appRoleID, appRoleSecretID string, vaultToken string) { 300 c.vaultCredentials = VaultCredentials{ 301 AppRoleID: appRoleID, 302 AppRoleSecretID: appRoleSecretID, 303 VaultToken: vaultToken, 304 } 305 } 306 307 // GetStepConfigWithJSON provides merged step configuration using a provided stepConfigJSON with additional flags provided 308 func GetStepConfigWithJSON(flagValues map[string]interface{}, stepConfigJSON string, filters StepFilters) StepConfig { 309 var stepConfig StepConfig 310 311 stepConfigMap := map[string]interface{}{} 312 313 err := json.Unmarshal([]byte(stepConfigJSON), &stepConfigMap) 314 if err != nil { 315 log.Entry().Warnf("invalid stepConfig JSON: %v", err) 316 } 317 318 stepConfig.mixIn(stepConfigMap, filters.All) 319 320 // ToDo: mix in parametersJSON 321 322 if flagValues != nil { 323 stepConfig.mixIn(flagValues, filters.Parameters) 324 } 325 return stepConfig 326 } 327 328 func (c *Config) GetStageConfig(paramJSON string, configuration io.ReadCloser, defaults []io.ReadCloser, ignoreCustomDefaults bool, acceptedParams []string, stageName string) (StepConfig, error) { 329 330 filters := StepFilters{ 331 General: acceptedParams, 332 Steps: []string{}, 333 Stages: acceptedParams, 334 Parameters: acceptedParams, 335 Env: []string{}, 336 } 337 return c.GetStepConfig(map[string]interface{}{}, paramJSON, configuration, defaults, ignoreCustomDefaults, filters, StepData{}, map[string]interface{}{}, stageName, "") 338 } 339 340 // GetJSON returns JSON representation of an object 341 func GetJSON(data interface{}) (string, error) { 342 343 result, err := json.Marshal(data) 344 if err != nil { 345 return "", errors.Wrapf(err, "error marshalling json: %v", err) 346 } 347 return string(result), nil 348 } 349 350 // GetYAML returns YAML representation of an object 351 func GetYAML(data interface{}) (string, error) { 352 353 result, err := yaml.Marshal(data) 354 if err != nil { 355 return "", errors.Wrapf(err, "error marshalling yaml: %v", err) 356 } 357 return string(result), nil 358 } 359 360 // OpenPiperFile provides functionality to retrieve configuration via file or http 361 func OpenPiperFile(name string, accessTokens map[string]string) (io.ReadCloser, error) { 362 if len(name) == 0 { 363 return nil, errors.Wrap(os.ErrNotExist, "no filename provided") 364 } 365 366 if !strings.HasPrefix(name, "http://") && !strings.HasPrefix(name, "https://") { 367 return os.Open(name) 368 } 369 370 return httpReadFile(name, accessTokens) 371 } 372 373 func httpReadFile(name string, accessTokens map[string]string) (io.ReadCloser, error) { 374 375 u, err := url.Parse(name) 376 if err != nil { 377 return nil, fmt.Errorf("failed to read url: %w", err) 378 } 379 380 // support http(s) urls next to file path 381 client := piperhttp.Client{} 382 383 var header http.Header 384 if len(accessTokens[u.Host]) > 0 { 385 client.SetOptions(piperhttp.ClientOptions{Token: fmt.Sprintf("token %v", accessTokens[u.Host])}) 386 header = map[string][]string{"Accept": {"application/vnd.github.v3.raw"}} 387 } 388 389 response, err := client.SendRequest("GET", name, nil, header, nil) 390 if err != nil { 391 return nil, err 392 } 393 return response.Body, nil 394 } 395 396 func envValues(filter []string) map[string]interface{} { 397 vals := map[string]interface{}{} 398 for _, param := range filter { 399 if envVal := os.Getenv("PIPER_" + param); len(envVal) != 0 { 400 vals[param] = os.Getenv("PIPER_" + param) 401 } 402 } 403 return vals 404 } 405 406 func (s *StepConfig) mixIn(mergeData map[string]interface{}, filter []string) { 407 408 if s.Config == nil { 409 s.Config = map[string]interface{}{} 410 } 411 412 s.Config = merge(s.Config, filterMap(mergeData, filter)) 413 } 414 415 func (s *StepConfig) mixInHookConfig(mergeData map[string]interface{}) { 416 417 if s.HookConfig == nil { 418 s.HookConfig = map[string]interface{}{} 419 } 420 421 s.HookConfig = merge(s.HookConfig, mergeData) 422 } 423 424 func (s *StepConfig) mixInStepDefaults(stepParams []StepParameters) { 425 if s.Config == nil { 426 s.Config = map[string]interface{}{} 427 } 428 429 // conditional defaults need to be written to a sub map 430 // in order to prevent a "last one wins" situation 431 // this is then considered at the end of GetStepConfig once the complete configuration is known 432 for _, p := range stepParams { 433 if p.Default != nil { 434 if len(p.Conditions) == 0 { 435 s.Config[p.Name] = p.Default 436 } else { 437 for _, cond := range p.Conditions { 438 for _, param := range cond.Params { 439 s.Config[param.Value] = map[string]interface{}{p.Name: p.Default} 440 } 441 } 442 } 443 } 444 } 445 } 446 447 // ApplyContainerConditions evaluates conditions in step yaml container definitions 448 func ApplyContainerConditions(containers []Container, stepConfig *StepConfig) { 449 for _, container := range containers { 450 if len(container.Conditions) > 0 { 451 for _, param := range container.Conditions[0].Params { 452 if container.Conditions[0].ConditionRef == "strings-equal" && stepConfig.Config[param.Name] == param.Value { 453 var containerConf map[string]interface{} 454 if stepConfig.Config[param.Value] != nil { 455 containerConf = stepConfig.Config[param.Value].(map[string]interface{}) 456 for key, value := range containerConf { 457 if stepConfig.Config[key] == nil { 458 stepConfig.Config[key] = value 459 } 460 } 461 delete(stepConfig.Config, param.Value) 462 } 463 } 464 } 465 } 466 } 467 } 468 469 func filterMap(data map[string]interface{}, filter []string) map[string]interface{} { 470 result := map[string]interface{}{} 471 472 if data == nil { 473 data = map[string]interface{}{} 474 } 475 476 for key, value := range data { 477 if value != nil && (len(filter) == 0 || sliceContains(filter, key)) { 478 result[key] = value 479 } 480 } 481 return result 482 } 483 484 func merge(base, overlay map[string]interface{}) map[string]interface{} { 485 486 result := map[string]interface{}{} 487 488 if base == nil { 489 base = map[string]interface{}{} 490 } 491 492 for key, value := range base { 493 result[key] = value 494 } 495 496 for key, value := range overlay { 497 if val, ok := value.(map[string]interface{}); ok { 498 if valBaseKey, ok := base[key].(map[string]interface{}); !ok { 499 result[key] = merge(map[string]interface{}{}, val) 500 } else { 501 result[key] = merge(valBaseKey, val) 502 } 503 } else { 504 result[key] = value 505 } 506 } 507 return result 508 } 509 510 func sliceContains(slice []string, find string) bool { 511 for _, elem := range slice { 512 matches, _ := regexp.MatchString(elem, find) 513 if matches { 514 return true 515 } 516 } 517 return false 518 } 519 520 func cloneConfig(config *Config) (*Config, error) { 521 configJSON, err := json.Marshal(config) 522 if err != nil { 523 return nil, err 524 } 525 526 clone := &Config{} 527 if err = json.Unmarshal(configJSON, &clone); err != nil { 528 return nil, err 529 } 530 531 return clone, nil 532 }