github.com/jaylevin/jenkins-library@v1.230.4/pkg/config/vault.go (about) 1 package config 2 3 import ( 4 "io/ioutil" 5 "os" 6 "path" 7 "regexp" 8 "strings" 9 10 "github.com/SAP/jenkins-library/pkg/config/interpolation" 11 "github.com/SAP/jenkins-library/pkg/log" 12 CredentialUtils "github.com/SAP/jenkins-library/pkg/piperutils" 13 "github.com/SAP/jenkins-library/pkg/vault" 14 "github.com/hashicorp/vault/api" 15 ) 16 17 const ( 18 vaultRootPaths = "vaultRootPaths" 19 vaultTestCredentialPath = "vaultTestCredentialPath" 20 vaultCredentialPath = "vaultCredentialPath" 21 vaultTestCredentialKeys = "vaultTestCredentialKeys" 22 vaultCredentialKeys = "vaultCredentialKeys" 23 vaultAppRoleID = "vaultAppRoleID" 24 vaultAppRoleSecretID = "vaultAppRoleSecreId" 25 vaultServerUrl = "vaultServerUrl" 26 vaultNamespace = "vaultNamespace" 27 vaultBasePath = "vaultBasePath" 28 vaultPipelineName = "vaultPipelineName" 29 vaultPath = "vaultPath" 30 skipVault = "skipVault" 31 vaultDisableOverwrite = "vaultDisableOverwrite" 32 vaultTestCredentialEnvPrefix = "vaultTestCredentialEnvPrefix" 33 vaultCredentialEnvPrefix = "vaultCredentialEnvPrefix" 34 vaultTestCredentialEnvPrefixDefault = "PIPER_TESTCREDENTIAL_" 35 vaultCredentialEnvPrefixDefault = "PIPER_VAULTCREDENTIAL_" 36 vaultSecretName = ".+VaultSecretName$" 37 ) 38 39 var ( 40 vaultFilter = []string{ 41 vaultRootPaths, 42 vaultAppRoleID, 43 vaultAppRoleSecretID, 44 vaultServerUrl, 45 vaultNamespace, 46 vaultBasePath, 47 vaultPipelineName, 48 vaultPath, 49 skipVault, 50 vaultDisableOverwrite, 51 vaultTestCredentialPath, 52 vaultTestCredentialKeys, 53 vaultTestCredentialEnvPrefix, 54 vaultCredentialPath, 55 vaultCredentialKeys, 56 vaultCredentialEnvPrefix, 57 vaultSecretName, 58 } 59 60 // VaultRootPaths are the lookup paths piper tries to use during the vault lookup. 61 // A path is only used if it's variables can be interpolated from the config 62 VaultRootPaths = []string{ 63 "$(vaultPath)", 64 "$(vaultBasePath)/$(vaultPipelineName)", 65 "$(vaultBasePath)/GROUP-SECRETS", 66 } 67 68 // VaultSecretFileDirectory holds the directory for the current step run to temporarily store secret files fetched from vault 69 VaultSecretFileDirectory = "" 70 ) 71 72 // VaultCredentials hold all the auth information needed to fetch configuration from vault 73 type VaultCredentials struct { 74 AppRoleID string 75 AppRoleSecretID string 76 VaultToken string 77 } 78 79 // vaultClient interface for mocking 80 type vaultClient interface { 81 GetKvSecret(string) (map[string]string, error) 82 MustRevokeToken() 83 } 84 85 func (s *StepConfig) mixinVaultConfig(parameters []StepParameters, configs ...map[string]interface{}) { 86 for _, config := range configs { 87 s.mixIn(config, vaultFilter) 88 // when an empty filter is returned we skip the mixin call since an empty filter will allow everything 89 if referencesFilter := getFilterForResourceReferences(parameters); len(referencesFilter) > 0 { 90 s.mixIn(config, referencesFilter) 91 } 92 } 93 } 94 95 func getVaultClientFromConfig(config StepConfig, creds VaultCredentials) (vaultClient, error) { 96 address, addressOk := config.Config["vaultServerUrl"].(string) 97 // if vault isn't used it's not an error 98 99 if !addressOk || creds.VaultToken == "" && (creds.AppRoleID == "" || creds.AppRoleSecretID == "") { 100 log.Entry().Debug("Skipping fetching secrets from Vault since it is not configured") 101 return nil, nil 102 } 103 namespace := "" 104 // namespaces are only available in vault enterprise so using them should be optional 105 if config.Config["vaultNamespace"] != nil { 106 namespace = config.Config["vaultNamespace"].(string) 107 log.Entry().Debugf("Using Vault namespace %s", namespace) 108 } 109 110 var client vaultClient 111 var err error 112 clientConfig := &vault.Config{Config: &api.Config{Address: address}, Namespace: namespace} 113 if creds.VaultToken != "" { 114 log.Entry().Debugf("Using Vault Token Authentication") 115 client, err = vault.NewClient(clientConfig, creds.VaultToken) 116 } else { 117 log.Entry().Debugf("Using Vault AppRole Authentication") 118 client, err = vault.NewClientWithAppRole(clientConfig, creds.AppRoleID, creds.AppRoleSecretID) 119 } 120 if err != nil { 121 return nil, err 122 } 123 124 log.Entry().Infof("Fetching secrets from Vault at %s", address) 125 return client, nil 126 } 127 128 func resolveAllVaultReferences(config *StepConfig, client vaultClient, params []StepParameters) { 129 for _, param := range params { 130 if ref := param.GetReference("vaultSecret"); ref != nil { 131 resolveVaultReference(ref, config, client, param) 132 } 133 if ref := param.GetReference("vaultSecretFile"); ref != nil { 134 resolveVaultReference(ref, config, client, param) 135 } 136 } 137 } 138 139 func resolveVaultReference(ref *ResourceReference, config *StepConfig, client vaultClient, param StepParameters) { 140 vaultDisableOverwrite, _ := config.Config["vaultDisableOverwrite"].(bool) 141 if _, ok := config.Config[param.Name].(string); vaultDisableOverwrite && ok { 142 log.Entry().Debugf("Not fetching '%s' from Vault since it has already been set", param.Name) 143 return 144 } 145 146 var secretValue *string 147 for _, vaultPath := range getSecretReferencePaths(ref, config.Config) { 148 // it should be possible to configure the root path were the secret is stored 149 vaultPath, ok := interpolation.ResolveString(vaultPath, config.Config) 150 if !ok { 151 continue 152 } 153 154 secretValue = lookupPath(client, vaultPath, ¶m) 155 if secretValue != nil { 156 log.Entry().Debugf("Resolved param '%s' with Vault path '%s'", param.Name, vaultPath) 157 if ref.Type == "vaultSecret" { 158 config.Config[param.Name] = *secretValue 159 } else if ref.Type == "vaultSecretFile" { 160 filePath, err := createTemporarySecretFile(param.Name, *secretValue) 161 if err != nil { 162 log.Entry().WithError(err).Warnf("Couldn't create temporary secret file for '%s'", param.Name) 163 return 164 } 165 config.Config[param.Name] = filePath 166 } 167 break 168 } 169 } 170 if secretValue == nil { 171 log.Entry().Warnf("Could not resolve param '%s' from Vault", param.Name) 172 } 173 } 174 175 // resolve test credential keys and expose as environment variables 176 func resolveVaultTestCredentials(config *StepConfig, client vaultClient) { 177 credPath, pathOk := config.Config[vaultTestCredentialPath].(string) 178 keys := getTestCredentialKeys(config) 179 if !(pathOk && keys != nil) || credPath == "" || len(keys) == 0 { 180 log.Entry().Debugf("Not fetching test credentials from Vault since they are not (properly) configured") 181 return 182 } 183 184 lookupPath := make([]string, 3) 185 lookupPath[0] = "$(vaultPath)/" + credPath 186 lookupPath[1] = "$(vaultBasePath)/$(vaultPipelineName)/" + credPath 187 lookupPath[2] = "$(vaultBasePath)/GROUP-SECRETS/" + credPath 188 189 for _, path := range lookupPath { 190 vaultPath, ok := interpolation.ResolveString(path, config.Config) 191 if !ok { 192 continue 193 } 194 195 secret, err := client.GetKvSecret(vaultPath) 196 if err != nil { 197 log.Entry().WithError(err).Debugf("Couldn't fetch secret at '%s'", vaultPath) 198 continue 199 } 200 if secret == nil { 201 continue 202 } 203 secretsResolved := false 204 secretsResolved = populateTestCredentialsAsEnvs(config, secret, keys) 205 if secretsResolved { 206 // prevent overwriting resolved secrets 207 // only allows vault test credentials on one / the same vault path 208 break 209 } 210 } 211 } 212 213 func resolveVaultCredentials(config *StepConfig, client vaultClient) { 214 credPath, pathOk := config.Config[vaultCredentialPath].(string) 215 keys := getCredentialKeys(config) 216 if !(pathOk && keys != nil) || credPath == "" || len(keys) == 0 { 217 log.Entry().Debugf("Not fetching credentials from vault since they are not (properly) configured") 218 return 219 } 220 221 lookupPath := make([]string, 3) 222 lookupPath[0] = "$(vaultPath)/" + credPath 223 lookupPath[1] = "$(vaultBasePath)/$(vaultPipelineName)/" + credPath 224 lookupPath[2] = "$(vaultBasePath)/GROUP-SECRETS/" + credPath 225 226 for _, path := range lookupPath { 227 vaultPath, ok := interpolation.ResolveString(path, config.Config) 228 if !ok { 229 continue 230 } 231 232 secret, err := client.GetKvSecret(vaultPath) 233 if err != nil { 234 log.Entry().WithError(err).Debugf("Couldn't fetch secret at '%s'", vaultPath) 235 continue 236 } 237 if secret == nil { 238 continue 239 } 240 secretsResolved := false 241 secretsResolved = populateCredentialsAsEnvs(config, secret, keys) 242 if secretsResolved { 243 // prevent overwriting resolved secrets 244 // only allows vault test credentials on one / the same vault path 245 break 246 } 247 } 248 } 249 250 func populateTestCredentialsAsEnvs(config *StepConfig, secret map[string]string, keys []string) (matched bool) { 251 252 vaultTestCredentialEnvPrefix, ok := config.Config["vaultTestCredentialEnvPrefix"].(string) 253 if !ok || len(vaultTestCredentialEnvPrefix) == 0 { 254 vaultTestCredentialEnvPrefix = vaultTestCredentialEnvPrefixDefault 255 } 256 for secretKey, secretValue := range secret { 257 for _, key := range keys { 258 if secretKey == key { 259 log.RegisterSecret(secretValue) 260 envVariable := vaultTestCredentialEnvPrefix + convertEnvVar(secretKey) 261 log.Entry().Debugf("Exposing test credential '%v' as '%v'", key, envVariable) 262 os.Setenv(envVariable, secretValue) 263 matched = true 264 } 265 } 266 } 267 return 268 } 269 270 func populateCredentialsAsEnvs(config *StepConfig, secret map[string]string, keys []string) (matched bool) { 271 272 vaultCredentialEnvPrefix, ok := config.Config["vaultCredentialEnvPrefix"].(string) 273 isCredentialEnvPrefixDefault := false 274 275 if !ok || len(vaultCredentialEnvPrefix) == 0 { 276 vaultCredentialEnvPrefix = vaultCredentialEnvPrefixDefault 277 isCredentialEnvPrefixDefault = true 278 } 279 for secretKey, secretValue := range secret { 280 for _, key := range keys { 281 if secretKey == key { 282 log.RegisterSecret(secretValue) 283 envVariable := vaultCredentialEnvPrefix + convertEnvVar(secretKey) 284 log.Entry().Debugf("Exposing general purpose credential '%v' as '%v'", key, envVariable) 285 os.Setenv(envVariable, secretValue) 286 envVariable = vaultCredentialEnvPrefix + convertEnvVar(secretKey) + "_BASE64" 287 log.Entry().Debugf("Exposing general purpose base64 encoded credential '%v' as '%v'", key, envVariable) 288 os.Setenv(envVariable, CredentialUtils.EncodeString(secretValue)) 289 matched = true 290 } 291 } 292 } 293 294 // we always create a standard env variable with the default prefx so that 295 // we can always refer to it in steps if its to be hard-coded 296 if !isCredentialEnvPrefixDefault { 297 for secretKey, secretValue := range secret { 298 for _, key := range keys { 299 if secretKey == key { 300 log.RegisterSecret(secretValue) 301 envVariable := vaultCredentialEnvPrefixDefault + convertEnvVar(secretKey) 302 log.Entry().Debugf("Exposing general purpose credential '%v' as '%v'", key, envVariable) 303 os.Setenv(envVariable, secretValue) 304 envVariable = vaultCredentialEnvPrefixDefault + convertEnvVar(secretKey) + "_BASE64" 305 log.Entry().Debugf("Exposing general purpose base64 encoded credential '%v' as '%v'", key, envVariable) 306 os.Setenv(envVariable, CredentialUtils.EncodeString(secretValue)) 307 matched = true 308 } 309 } 310 } 311 } 312 return 313 } 314 315 func getTestCredentialKeys(config *StepConfig) []string { 316 keysRaw, ok := config.Config[vaultTestCredentialKeys].([]interface{}) 317 if !ok { 318 return nil 319 } 320 keys := make([]string, 0, len(keysRaw)) 321 for _, keyRaw := range keysRaw { 322 key, ok := keyRaw.(string) 323 if !ok { 324 log.Entry().Warnf("%s needs to be an array of strings", vaultTestCredentialKeys) 325 return nil 326 } 327 keys = append(keys, key) 328 } 329 return keys 330 } 331 332 func getCredentialKeys(config *StepConfig) []string { 333 keysRaw, ok := config.Config[vaultCredentialKeys].([]interface{}) 334 if !ok { 335 log.Entry().Debugf("Not fetching general purpose credentials from vault since they are not (properly) configured") 336 return nil 337 } 338 keys := make([]string, 0, len(keysRaw)) 339 for _, keyRaw := range keysRaw { 340 key, ok := keyRaw.(string) 341 if !ok { 342 log.Entry().Warnf("%s is needs to be an array of strings", vaultCredentialKeys) 343 return nil 344 } 345 keys = append(keys, key) 346 } 347 return keys 348 } 349 350 // converts to a valid environment variable string 351 func convertEnvVar(s string) string { 352 r := strings.ToUpper(s) 353 r = strings.ReplaceAll(r, "-", "_") 354 reg, err := regexp.Compile("[^a-zA-Z0-9_]*") 355 if err != nil { 356 log.Entry().Debugf("could not compile regex of convertEnvVar: %v", err) 357 } 358 replacedString := reg.ReplaceAllString(r, "") 359 return replacedString 360 } 361 362 // RemoveVaultSecretFiles removes all secret files that have been created during execution 363 func RemoveVaultSecretFiles() { 364 if VaultSecretFileDirectory != "" { 365 os.RemoveAll(VaultSecretFileDirectory) 366 } 367 } 368 369 func createTemporarySecretFile(namePattern string, content string) (string, error) { 370 if VaultSecretFileDirectory == "" { 371 var err error 372 VaultSecretFileDirectory, err = ioutil.TempDir("", "vault") 373 if err != nil { 374 return "", err 375 } 376 } 377 378 file, err := ioutil.TempFile(VaultSecretFileDirectory, namePattern) 379 if err != nil { 380 return "", err 381 } 382 defer file.Close() 383 _, err = file.WriteString(content) 384 if err != nil { 385 return "", err 386 } 387 return file.Name(), nil 388 } 389 390 func lookupPath(client vaultClient, path string, param *StepParameters) *string { 391 log.Entry().Debugf("Trying to resolve Vault parameter '%s' at '%s'", param.Name, path) 392 secret, err := client.GetKvSecret(path) 393 if err != nil { 394 log.Entry().WithError(err).Warnf("Couldn't fetch secret at '%s'", path) 395 return nil 396 } 397 if secret == nil { 398 return nil 399 } 400 401 field := secret[param.Name] 402 if field != "" { 403 log.RegisterSecret(field) 404 return &field 405 } 406 log.Entry().Debugf("Secret did not contain a field name '%s'", param.Name) 407 // try parameter aliases 408 for _, alias := range param.Aliases { 409 log.Entry().Debugf("Trying alias field name '%s'", alias.Name) 410 field := secret[alias.Name] 411 if field != "" { 412 log.RegisterSecret(field) 413 if alias.Deprecated { 414 log.Entry().WithField("package", "SAP/jenkins-library/pkg/config").Warningf("DEPRECATION NOTICE: old step config key '%s' used in Vault. Please switch to '%s'!", alias.Name, param.Name) 415 } 416 return &field 417 } 418 } 419 return nil 420 } 421 422 func getSecretReferencePaths(reference *ResourceReference, config map[string]interface{}) []string { 423 retPaths := make([]string, 0, len(VaultRootPaths)) 424 secretName := reference.Default 425 if providedName, ok := config[reference.Name].(string); ok && providedName != "" { 426 secretName = providedName 427 } 428 for _, rootPath := range VaultRootPaths { 429 fullPath := path.Join(rootPath, secretName) 430 retPaths = append(retPaths, fullPath) 431 } 432 return retPaths 433 } 434 435 func toStringSlice(interfaceSlice []interface{}) []string { 436 retSlice := make([]string, 0, len(interfaceSlice)) 437 for _, vRaw := range interfaceSlice { 438 if v, ok := vRaw.(string); ok { 439 retSlice = append(retSlice, v) 440 continue 441 } 442 log.Entry().Warnf("'%s' needs to be of type string or an array of strings but got %T (%[2]v)", vaultPath, vRaw) 443 } 444 return retSlice 445 }