github.com/jaylevin/jenkins-library@v1.230.4/cmd/kubernetesDeploy.go (about) 1 package cmd 2 3 import ( 4 "bytes" 5 "encoding/json" 6 "fmt" 7 "io" 8 "io/ioutil" 9 "os" 10 "path/filepath" 11 "regexp" 12 "sort" 13 "strconv" 14 "strings" 15 "text/template" 16 17 "github.com/SAP/jenkins-library/pkg/docker" 18 "github.com/SAP/jenkins-library/pkg/kubernetes" 19 "github.com/SAP/jenkins-library/pkg/log" 20 "github.com/SAP/jenkins-library/pkg/telemetry" 21 "github.com/pkg/errors" 22 "helm.sh/helm/v3/pkg/cli/values" 23 ) 24 25 func kubernetesDeploy(config kubernetesDeployOptions, telemetryData *telemetry.CustomData) { 26 customTLSCertificateLinks := []string{} 27 utils := kubernetes.NewDeployUtilsBundle(customTLSCertificateLinks) 28 29 // error situations stop execution through log.Entry().Fatal() call which leads to an os.Exit(1) in the end 30 err := runKubernetesDeploy(config, telemetryData, utils, log.Writer()) 31 if err != nil { 32 log.Entry().WithError(err).Fatal("step execution failed") 33 } 34 } 35 36 func runKubernetesDeploy(config kubernetesDeployOptions, telemetryData *telemetry.CustomData, utils kubernetes.DeployUtils, stdout io.Writer) error { 37 telemetryData.Custom1Label = "deployTool" 38 telemetryData.Custom1 = config.DeployTool 39 40 if config.DeployTool == "helm" || config.DeployTool == "helm3" { 41 return runHelmDeploy(config, utils, stdout) 42 } else if config.DeployTool == "kubectl" { 43 return runKubectlDeploy(config, utils, stdout) 44 } 45 return fmt.Errorf("Failed to execute deployments") 46 } 47 48 func runHelmDeploy(config kubernetesDeployOptions, utils kubernetes.DeployUtils, stdout io.Writer) error { 49 if len(config.ChartPath) <= 0 { 50 return fmt.Errorf("chart path has not been set, please configure chartPath parameter") 51 } 52 if len(config.DeploymentName) <= 0 { 53 return fmt.Errorf("deployment name has not been set, please configure deploymentName parameter") 54 } 55 _, containerRegistry, err := splitRegistryURL(config.ContainerRegistryURL) 56 if err != nil { 57 log.Entry().WithError(err).Fatalf("Container registry url '%v' incorrect", config.ContainerRegistryURL) 58 } 59 60 helmValues, err := defineDeploymentValues(config, containerRegistry) 61 if err != nil { 62 return errors.Wrap(err, "failed to process deployment values") 63 } 64 65 helmLogFields := map[string]interface{}{} 66 helmLogFields["Chart Path"] = config.ChartPath 67 helmLogFields["Namespace"] = config.Namespace 68 helmLogFields["Deployment Name"] = config.DeploymentName 69 helmLogFields["Context"] = config.KubeContext 70 helmLogFields["Kubeconfig"] = config.KubeConfig 71 log.Entry().WithFields(helmLogFields).Debug("Calling Helm") 72 73 helmEnv := []string{fmt.Sprintf("KUBECONFIG=%v", config.KubeConfig)} 74 if config.DeployTool == "helm" && len(config.TillerNamespace) > 0 { 75 helmEnv = append(helmEnv, fmt.Sprintf("TILLER_NAMESPACE=%v", config.TillerNamespace)) 76 } 77 log.Entry().Debugf("Helm SetEnv: %v", helmEnv) 78 utils.SetEnv(helmEnv) 79 utils.Stdout(stdout) 80 81 if config.DeployTool == "helm" { 82 initParams := []string{"init", "--client-only"} 83 if err := utils.RunExecutable("helm", initParams...); err != nil { 84 log.Entry().WithError(err).Fatal("Helm init call failed") 85 } 86 } 87 88 if len(config.ContainerRegistryUser) == 0 && len(config.ContainerRegistryPassword) == 0 { 89 log.Entry().Info("No/incomplete container registry credentials provided: skipping secret creation") 90 if len(config.ContainerRegistrySecret) > 0 { 91 helmValues.add("imagePullSecrets[0].name", config.ContainerRegistrySecret) 92 } 93 } else { 94 var dockerRegistrySecret bytes.Buffer 95 utils.Stdout(&dockerRegistrySecret) 96 err, kubeSecretParams := defineKubeSecretParams(config, containerRegistry, utils) 97 if err != nil { 98 log.Entry().WithError(err).Fatal("parameter definition for creating registry secret failed") 99 } 100 log.Entry().Infof("Calling kubectl create secret --dry-run=true ...") 101 log.Entry().Debugf("kubectl parameters %v", kubeSecretParams) 102 if err := utils.RunExecutable("kubectl", kubeSecretParams...); err != nil { 103 log.Entry().WithError(err).Fatal("Retrieving Docker config via kubectl failed") 104 } 105 106 var dockerRegistrySecretData struct { 107 Kind string `json:"kind"` 108 Data struct { 109 DockerConfJSON string `json:".dockerconfigjson"` 110 } `json:"data"` 111 Type string `json:"type"` 112 } 113 if err := json.Unmarshal(dockerRegistrySecret.Bytes(), &dockerRegistrySecretData); err != nil { 114 log.Entry().WithError(err).Fatal("Reading docker registry secret json failed") 115 } 116 // make sure that secret is hidden in log output 117 log.RegisterSecret(dockerRegistrySecretData.Data.DockerConfJSON) 118 119 log.Entry().Debugf("Secret created: %v", dockerRegistrySecret.String()) 120 121 // pass secret in helm default template way and in Piper backward compatible way 122 helmValues.add("secret.name", config.ContainerRegistrySecret) 123 helmValues.add("secret.dockerconfigjson", dockerRegistrySecretData.Data.DockerConfJSON) 124 helmValues.add("imagePullSecrets[0].name", config.ContainerRegistrySecret) 125 } 126 127 // Deprecated functionality 128 // only for backward compatible handling of ingress.hosts 129 // this requires an adoption of the default ingress.yaml template 130 // Due to the way helm is implemented it is currently not possible to overwrite a part of a list: 131 // see: https://github.com/helm/helm/issues/5711#issuecomment-636177594 132 // Recommended way is to use a custom values file which contains the appropriate data 133 for i, h := range config.IngressHosts { 134 helmValues.add(fmt.Sprintf("ingress.hosts[%v]", i), h) 135 } 136 137 upgradeParams := []string{ 138 "upgrade", 139 config.DeploymentName, 140 config.ChartPath, 141 } 142 143 for _, v := range config.HelmValues { 144 upgradeParams = append(upgradeParams, "--values", v) 145 } 146 147 err = helmValues.mapValues() 148 if err != nil { 149 return errors.Wrap(err, "failed to map values using 'valuesMapping' configuration") 150 } 151 152 upgradeParams = append( 153 upgradeParams, 154 "--install", 155 "--namespace", config.Namespace, 156 "--set", strings.Join(helmValues.marshal(), ","), 157 ) 158 159 if config.ForceUpdates { 160 upgradeParams = append(upgradeParams, "--force") 161 } 162 163 if config.DeployTool == "helm" { 164 upgradeParams = append(upgradeParams, "--wait", "--timeout", strconv.Itoa(config.HelmDeployWaitSeconds)) 165 } 166 167 if config.DeployTool == "helm3" { 168 upgradeParams = append(upgradeParams, "--wait", "--timeout", fmt.Sprintf("%vs", config.HelmDeployWaitSeconds)) 169 } 170 171 if !config.KeepFailedDeployments { 172 upgradeParams = append(upgradeParams, "--atomic") 173 } 174 175 if len(config.KubeContext) > 0 { 176 upgradeParams = append(upgradeParams, "--kube-context", config.KubeContext) 177 } 178 179 if len(config.AdditionalParameters) > 0 { 180 upgradeParams = append(upgradeParams, config.AdditionalParameters...) 181 } 182 183 utils.Stdout(stdout) 184 log.Entry().Info("Calling helm upgrade ...") 185 log.Entry().Debugf("Helm parameters %v", upgradeParams) 186 if err := utils.RunExecutable("helm", upgradeParams...); err != nil { 187 log.Entry().WithError(err).Fatal("Helm upgrade call failed") 188 } 189 190 testParams := []string{ 191 "test", 192 config.DeploymentName, 193 "--namespace", config.Namespace, 194 } 195 196 if config.ShowTestLogs { 197 testParams = append( 198 testParams, 199 "--logs", 200 ) 201 } 202 203 if config.RunHelmTests { 204 if err := utils.RunExecutable("helm", testParams...); err != nil { 205 log.Entry().WithError(err).Fatal("Helm test call failed") 206 } 207 } 208 209 return nil 210 } 211 212 func runKubectlDeploy(config kubernetesDeployOptions, utils kubernetes.DeployUtils, stdout io.Writer) error { 213 _, containerRegistry, err := splitRegistryURL(config.ContainerRegistryURL) 214 if err != nil { 215 log.Entry().WithError(err).Fatalf("Container registry url '%v' incorrect", config.ContainerRegistryURL) 216 } 217 218 kubeParams := []string{ 219 "--insecure-skip-tls-verify=true", 220 fmt.Sprintf("--namespace=%v", config.Namespace), 221 } 222 223 if len(config.KubeConfig) > 0 { 224 log.Entry().Info("Using KUBECONFIG environment for authentication.") 225 kubeEnv := []string{fmt.Sprintf("KUBECONFIG=%v", config.KubeConfig)} 226 utils.SetEnv(kubeEnv) 227 if len(config.KubeContext) > 0 { 228 kubeParams = append(kubeParams, fmt.Sprintf("--context=%v", config.KubeContext)) 229 } 230 231 } else { 232 log.Entry().Info("Using --token parameter for authentication.") 233 kubeParams = append(kubeParams, fmt.Sprintf("--server=%v", config.APIServer)) 234 kubeParams = append(kubeParams, fmt.Sprintf("--token=%v", config.KubeToken)) 235 } 236 237 utils.Stdout(stdout) 238 239 if len(config.ContainerRegistryUser) == 0 && len(config.ContainerRegistryPassword) == 0 { 240 log.Entry().Info("No/incomplete container registry credentials provided: skipping secret creation") 241 } else { 242 err, kubeSecretParams := defineKubeSecretParams(config, containerRegistry, utils) 243 if err != nil { 244 log.Entry().WithError(err).Fatal("parameter definition for creating registry secret failed") 245 } 246 var dockerRegistrySecret bytes.Buffer 247 utils.Stdout(&dockerRegistrySecret) 248 log.Entry().Infof("Creating container registry secret '%v'", config.ContainerRegistrySecret) 249 kubeSecretParams = append(kubeSecretParams, kubeParams...) 250 log.Entry().Debugf("Running kubectl with following parameters: %v", kubeSecretParams) 251 if err := utils.RunExecutable("kubectl", kubeSecretParams...); err != nil { 252 log.Entry().WithError(err).Fatal("Creating container registry secret failed") 253 } 254 255 var dockerRegistrySecretData map[string]interface{} 256 257 if err := json.Unmarshal(dockerRegistrySecret.Bytes(), &dockerRegistrySecretData); err != nil { 258 log.Entry().WithError(err).Fatal("Reading docker registry secret json failed") 259 } 260 261 // write the json output to a file 262 tmpFolder := getTempDirForKubeCtlJSON() 263 defer os.RemoveAll(tmpFolder) // clean up 264 jsonData, _ := json.Marshal(dockerRegistrySecretData) 265 ioutil.WriteFile(filepath.Join(tmpFolder, "secret.json"), jsonData, 0777) 266 267 kubeSecretApplyParams := []string{"apply", "-f", filepath.Join(tmpFolder, "secret.json")} 268 if err := utils.RunExecutable("kubectl", kubeSecretApplyParams...); err != nil { 269 log.Entry().WithError(err).Fatal("Creating container registry secret failed") 270 } 271 272 } 273 274 appTemplate, err := utils.FileRead(config.AppTemplate) 275 if err != nil { 276 log.Entry().WithError(err).Fatalf("Error when reading appTemplate '%v'", config.AppTemplate) 277 } 278 279 values, err := defineDeploymentValues(config, containerRegistry) 280 if err != nil { 281 return errors.Wrap(err, "failed to process deployment values") 282 } 283 err = values.mapValues() 284 if err != nil { 285 return errors.Wrap(err, "failed to map values using 'valuesMapping' configuration") 286 } 287 288 re := regexp.MustCompile(`image:[ ]*<image-name>`) 289 placeholderFound := re.Match(appTemplate) 290 291 if placeholderFound { 292 log.Entry().Warn("image placeholder '<image-name>' is deprecated and does not support multi-image replacement, please use Helm-like template syntax '{{ .Values.image.[image-name].reposotory }}:{{ .Values.image.[image-name].tag }}") 293 if values.singleImage { 294 // Update image name in deployment yaml, expects placeholder like 'image: <image-name>' 295 appTemplate = []byte(re.ReplaceAllString(string(appTemplate), fmt.Sprintf("image: %s:%s", values.get("image.repository"), values.get("image.tag")))) 296 } else { 297 return fmt.Errorf("multi-image replacement not supported for single image placeholder") 298 } 299 } 300 301 buf := bytes.NewBufferString("") 302 tpl, err := template.New("appTemplate").Parse(string(appTemplate)) 303 if err != nil { 304 return errors.Wrap(err, "failed to parse app-template file") 305 } 306 err = tpl.Execute(buf, values.asHelmValues()) 307 if err != nil { 308 return errors.Wrap(err, "failed to render app-template file") 309 } 310 311 err = utils.FileWrite(config.AppTemplate, buf.Bytes(), 0700) 312 if err != nil { 313 return errors.Wrapf(err, "Error when updating appTemplate '%v'", config.AppTemplate) 314 } 315 316 kubeParams = append(kubeParams, config.DeployCommand, "--filename", config.AppTemplate) 317 if config.ForceUpdates && config.DeployCommand == "replace" { 318 kubeParams = append(kubeParams, "--force") 319 } 320 321 if len(config.AdditionalParameters) > 0 { 322 kubeParams = append(kubeParams, config.AdditionalParameters...) 323 } 324 if err := utils.RunExecutable("kubectl", kubeParams...); err != nil { 325 log.Entry().Debugf("Running kubectl with following parameters: %v", kubeParams) 326 log.Entry().WithError(err).Fatal("Deployment with kubectl failed.") 327 } 328 return nil 329 } 330 331 type deploymentValues struct { 332 mapping map[string]interface{} 333 singleImage bool 334 values []struct { 335 key, value string 336 } 337 } 338 339 func (dv *deploymentValues) add(key, value string) { 340 dv.values = append(dv.values, struct { 341 key string 342 value string 343 }{ 344 key: key, 345 value: value, 346 }) 347 } 348 349 func (dv deploymentValues) get(key string) string { 350 for _, item := range dv.values { 351 if item.key == key { 352 return item.value 353 } 354 } 355 356 return "" 357 } 358 359 func (dv *deploymentValues) mapValues() error { 360 var keys []string 361 for k := range dv.mapping { 362 keys = append(keys, k) 363 } 364 sort.Strings(keys) 365 for _, dst := range keys { 366 srcString, ok := dv.mapping[dst].(string) 367 if !ok { 368 return fmt.Errorf("invalid path '%#v' is used for valuesMapping, only strings are supported", dv.mapping[dst]) 369 } 370 if val := dv.get(srcString); val != "" { 371 dv.add(dst, val) 372 } else { 373 log.Entry().Warnf("can not map '%s: %s', %s is not set", dst, dv.mapping[dst], dv.mapping[dst]) 374 } 375 } 376 377 return nil 378 } 379 380 func (dv deploymentValues) marshal() []string { 381 var result []string 382 for _, item := range dv.values { 383 result = append(result, fmt.Sprintf("%s=%s", item.key, item.value)) 384 } 385 return result 386 } 387 388 func (dv *deploymentValues) asHelmValues() map[string]interface{} { 389 valuesOpts := values.Options{ 390 Values: dv.marshal(), 391 } 392 mergedValues, err := valuesOpts.MergeValues(nil) 393 if err != nil { 394 log.Entry().WithError(err).Fatal("failed to process deployment values") 395 } 396 return map[string]interface{}{ 397 "Values": mergedValues, 398 } 399 } 400 401 func joinKey(parts ...string) string { 402 escapedParts := make([]string, 0, len(parts)) 403 replacer := strings.NewReplacer(".", "_", "-", "_") 404 for _, part := range parts { 405 escapedParts = append(escapedParts, replacer.Replace(part)) 406 } 407 return strings.Join(escapedParts, ".") 408 } 409 410 func getTempDirForKubeCtlJSON() string { 411 tmpFolder, err := ioutil.TempDir(".", "temp-") 412 if err != nil { 413 log.Entry().WithError(err).WithField("path", tmpFolder).Debug("creating temp directory failed") 414 } 415 return tmpFolder 416 } 417 418 func splitRegistryURL(registryURL string) (protocol, registry string, err error) { 419 parts := strings.Split(registryURL, "://") 420 if len(parts) != 2 || len(parts[1]) == 0 { 421 return "", "", fmt.Errorf("Failed to split registry url '%v'", registryURL) 422 } 423 return parts[0], parts[1], nil 424 } 425 426 func splitFullImageName(image string) (imageName, tag string, err error) { 427 parts := strings.Split(image, ":") 428 switch len(parts) { 429 case 0: 430 return "", "", fmt.Errorf("Failed to split image name '%v'", image) 431 case 1: 432 if len(parts[0]) > 0 { 433 return parts[0], "", nil 434 } 435 return "", "", fmt.Errorf("Failed to split image name '%v'", image) 436 case 2: 437 return parts[0], parts[1], nil 438 } 439 return "", "", fmt.Errorf("Failed to split image name '%v'", image) 440 } 441 442 func defineKubeSecretParams(config kubernetesDeployOptions, containerRegistry string, utils kubernetes.DeployUtils) (error, []string) { 443 targetPath := "" 444 if len(config.DockerConfigJSON) > 0 { 445 // first enhance config.json with additional pipeline-related credentials if they have been provided 446 if len(containerRegistry) > 0 && len(config.ContainerRegistryUser) > 0 && len(config.ContainerRegistryPassword) > 0 { 447 var err error 448 targetPath, err = docker.CreateDockerConfigJSON(containerRegistry, config.ContainerRegistryUser, config.ContainerRegistryPassword, "", config.DockerConfigJSON, utils) 449 if err != nil { 450 log.Entry().Warningf("failed to update Docker config.json: %v", err) 451 return err, []string{} 452 } 453 } 454 455 } else { 456 return fmt.Errorf("no docker config json file found to update credentials '%v'", config.DockerConfigJSON), []string{} 457 } 458 return nil, []string{ 459 "create", 460 "secret", 461 "generic", 462 config.ContainerRegistrySecret, 463 fmt.Sprintf("--from-file=.dockerconfigjson=%v", targetPath), 464 "--type=kubernetes.io/dockerconfigjson", 465 "--insecure-skip-tls-verify=true", 466 "--dry-run=client", 467 "--output=json", 468 } 469 } 470 471 func defineDeploymentValues(config kubernetesDeployOptions, containerRegistry string) (*deploymentValues, error) { 472 var err error 473 var useDigests bool 474 dv := &deploymentValues{ 475 mapping: config.ValuesMapping, 476 } 477 if len(config.ImageNames) > 0 { 478 if len(config.ImageNames) != len(config.ImageNameTags) { 479 log.SetErrorCategory(log.ErrorConfiguration) 480 return nil, fmt.Errorf("number of imageNames and imageNameTags must be equal") 481 } 482 if len(config.ImageDigests) > 0 { 483 if len(config.ImageDigests) != len(config.ImageNameTags) { 484 log.SetErrorCategory(log.ErrorConfiguration) 485 return nil, fmt.Errorf("number of imageDigests and imageNameTags must be equal") 486 } 487 488 useDigests = true 489 } 490 for i, key := range config.ImageNames { 491 name, tag, err := splitFullImageName(config.ImageNameTags[i]) 492 if err != nil { 493 log.Entry().WithError(err).Fatalf("Container image '%v' incorrect", config.ImageNameTags[i]) 494 } 495 496 if useDigests { 497 tag = fmt.Sprintf("%s@%s", tag, config.ImageDigests[i]) 498 } 499 500 dv.add(joinKey("image", key, "repository"), fmt.Sprintf("%v/%v", containerRegistry, name)) 501 dv.add(joinKey("image", key, "tag"), tag) 502 503 if len(config.ImageNames) == 1 { 504 dv.singleImage = true 505 dv.add("image.repository", fmt.Sprintf("%v/%v", containerRegistry, name)) 506 dv.add("image.tag", tag) 507 } 508 } 509 } else { 510 // support either image or containerImageName and containerImageTag 511 containerImageName := "" 512 containerImageTag := "" 513 dv.singleImage = true 514 515 if len(config.Image) > 0 { 516 containerImageName, containerImageTag, err = splitFullImageName(config.Image) 517 if err != nil { 518 log.Entry().WithError(err).Fatalf("Container image '%v' incorrect", config.Image) 519 } 520 } else if len(config.ContainerImageName) > 0 && len(config.ContainerImageTag) > 0 { 521 containerImageName = config.ContainerImageName 522 containerImageTag = config.ContainerImageTag 523 } else { 524 return nil, fmt.Errorf("image information not given - please either set image or containerImageName and containerImageTag") 525 } 526 dv.add("image.repository", fmt.Sprintf("%v/%v", containerRegistry, containerImageName)) 527 dv.add("image.tag", containerImageTag) 528 529 dv.add(joinKey("image", containerImageName, "repository"), fmt.Sprintf("%v/%v", containerRegistry, containerImageName)) 530 dv.add(joinKey("image", containerImageName, "tag"), containerImageTag) 531 } 532 533 return dv, nil 534 }