github.com/xgoffin/jenkins-library@v1.154.0/cmd/cnbBuild.go (about) 1 package cmd 2 3 import ( 4 "archive/zip" 5 "encoding/json" 6 "fmt" 7 "os" 8 "path" 9 "path/filepath" 10 11 "github.com/SAP/jenkins-library/pkg/certutils" 12 "github.com/SAP/jenkins-library/pkg/cnbutils" 13 "github.com/SAP/jenkins-library/pkg/cnbutils/bindings" 14 "github.com/SAP/jenkins-library/pkg/cnbutils/privacy" 15 "github.com/SAP/jenkins-library/pkg/cnbutils/project" 16 "github.com/SAP/jenkins-library/pkg/cnbutils/project/metadata" 17 "github.com/SAP/jenkins-library/pkg/command" 18 "github.com/SAP/jenkins-library/pkg/docker" 19 piperhttp "github.com/SAP/jenkins-library/pkg/http" 20 "github.com/SAP/jenkins-library/pkg/log" 21 "github.com/SAP/jenkins-library/pkg/piperutils" 22 23 "github.com/SAP/jenkins-library/pkg/telemetry" 24 "github.com/imdario/mergo" 25 "github.com/mitchellh/mapstructure" 26 "github.com/pkg/errors" 27 ignore "github.com/sabhiram/go-gitignore" 28 ) 29 30 const ( 31 creatorPath = "/cnb/lifecycle/creator" 32 platformPath = "/tmp/platform" 33 ) 34 35 type pathEnum string 36 37 const ( 38 pathEnumRoot = pathEnum("root") 39 pathEnumFolder = pathEnum("folder") 40 pathEnumArchive = pathEnum("archive") 41 ) 42 43 type cnbBuildUtilsBundle struct { 44 *command.Command 45 *piperutils.Files 46 *docker.Client 47 } 48 49 type cnbBuildTelemetry struct { 50 Version int `json:"version"` 51 Data []cnbBuildTelemetryData `json:"data"` 52 } 53 54 type cnbBuildTelemetryData struct { 55 ImageTag string `json:"imageTag"` 56 AdditionalTags []string `json:"additionalTags"` 57 BindingKeys []string `json:"bindingKeys"` 58 Path pathEnum `json:"path"` 59 BuildEnv cnbBuildTelemetryDataBuildEnv `json:"buildEnv"` 60 Buildpacks cnbBuildTelemetryDataBuildpacks `json:"buildpacks"` 61 ProjectDescriptor cnbBuildTelemetryDataProjectDescriptor `json:"projectDescriptor"` 62 BuildTool string `json:"buildTool"` 63 Builder string `json:"builder"` 64 } 65 66 type cnbBuildTelemetryDataBuildEnv struct { 67 KeysFromConfig []string `json:"keysFromConfig"` 68 KeysFromProjectDescriptor []string `json:"keysFromProjectDescriptor"` 69 KeysOverall []string `json:"keysOverall"` 70 JVMVersion string `json:"jvmVersion"` 71 KeyValues map[string]interface{} `json:"keyValues"` 72 } 73 74 type cnbBuildTelemetryDataBuildpacks struct { 75 FromConfig []string `json:"FromConfig"` 76 FromProjectDescriptor []string `json:"FromProjectDescriptor"` 77 Overall []string `json:"overall"` 78 } 79 80 type cnbBuildTelemetryDataProjectDescriptor struct { 81 Used bool `json:"used"` 82 IncludeUsed bool `json:"includeUsed"` 83 ExcludeUsed bool `json:"excludeUsed"` 84 } 85 86 func processConfigs(main cnbBuildOptions, multipleImages []map[string]interface{}) ([]cnbBuildOptions, error) { 87 var result []cnbBuildOptions 88 89 if len(multipleImages) == 0 { 90 result = append(result, main) 91 return result, nil 92 } 93 94 for _, conf := range multipleImages { 95 var structuredConf cnbBuildOptions 96 err := mapstructure.Decode(conf, &structuredConf) 97 if err != nil { 98 return nil, err 99 } 100 101 err = mergo.Merge(&structuredConf, main) 102 if err != nil { 103 return nil, err 104 } 105 106 result = append(result, structuredConf) 107 } 108 109 return result, nil 110 } 111 112 func setCustomBuildpacks(bpacks []string, dockerCreds string, utils cnbutils.BuildUtils) (string, string, error) { 113 buildpacksPath := "/tmp/buildpacks" 114 orderPath := "/tmp/buildpacks/order.toml" 115 newOrder, err := cnbutils.DownloadBuildpacks(buildpacksPath, bpacks, dockerCreds, utils) 116 if err != nil { 117 return "", "", err 118 } 119 120 err = newOrder.Save(orderPath) 121 if err != nil { 122 return "", "", err 123 } 124 125 return buildpacksPath, orderPath, nil 126 } 127 128 func newCnbBuildUtils() cnbutils.BuildUtils { 129 utils := cnbBuildUtilsBundle{ 130 Command: &command.Command{}, 131 Files: &piperutils.Files{}, 132 Client: &docker.Client{}, 133 } 134 utils.Stdout(log.Writer()) 135 utils.Stderr(log.Writer()) 136 return &utils 137 } 138 139 func cnbBuild(config cnbBuildOptions, telemetryData *telemetry.CustomData, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment) { 140 utils := newCnbBuildUtils() 141 142 client := &piperhttp.Client{} 143 144 err := callCnbBuild(&config, telemetryData, utils, commonPipelineEnvironment, client) 145 if err != nil { 146 log.Entry().WithError(err).Fatal("step execution failed") 147 } 148 } 149 150 func isBuilder(utils cnbutils.BuildUtils) error { 151 exists, err := utils.FileExists(creatorPath) 152 if err != nil { 153 return err 154 } 155 156 if !exists { 157 return fmt.Errorf("binary '%s' not found", creatorPath) 158 } 159 160 return nil 161 } 162 163 func isZip(path string) bool { 164 r, err := zip.OpenReader(path) 165 166 switch { 167 case err == nil: 168 _ = r.Close() 169 return true 170 case err == zip.ErrFormat: 171 return false 172 default: 173 return false 174 } 175 } 176 177 func cleanDir(dir string, utils cnbutils.BuildUtils) error { 178 dirContent, err := utils.Glob(filepath.Join(dir, "*")) 179 if err != nil { 180 return err 181 } 182 183 for _, obj := range dirContent { 184 err = utils.RemoveAll(obj) 185 if err != nil { 186 return err 187 } 188 } 189 190 return nil 191 } 192 193 func extractZip(source, target string) error { 194 if isZip(source) { 195 log.Entry().Infof("Extracting archive '%s' to '%s'", source, target) 196 _, err := piperutils.Unzip(source, target) 197 if err != nil { 198 log.SetErrorCategory(log.ErrorBuild) 199 return errors.Wrapf(err, "Extracting archive '%s' to '%s' failed", source, target) 200 } 201 } else { 202 log.SetErrorCategory(log.ErrorBuild) 203 return errors.New("application path must be a directory or zip") 204 } 205 206 return nil 207 } 208 209 func prepareDockerConfig(source string, utils cnbutils.BuildUtils) (string, error) { 210 if filepath.Base(source) != "config.json" { 211 log.Entry().Debugf("Renaming docker config file from '%s' to 'config.json'", filepath.Base(source)) 212 213 newPath := filepath.Join(filepath.Dir(source), "config.json") 214 alreadyExists, err := utils.FileExists(newPath) 215 if err != nil { 216 return "", err 217 } 218 if alreadyExists { 219 return newPath, nil 220 } 221 222 err = utils.FileRename(source, newPath) 223 if err != nil { 224 return "", err 225 } 226 227 return newPath, nil 228 } 229 230 return source, nil 231 } 232 233 func linkTargetFolder(utils cnbutils.BuildUtils, source, target string) error { 234 var err error 235 linkPath := filepath.Join(target, "target") 236 targetPath := filepath.Join(source, "target") 237 if ok, _ := utils.DirExists(targetPath); !ok { 238 err = utils.MkdirAll(targetPath, os.ModePerm) 239 if err != nil { 240 return err 241 } 242 } 243 244 if ok, _ := utils.DirExists(linkPath); ok { 245 err = utils.RemoveAll(linkPath) 246 if err != nil { 247 return err 248 } 249 } 250 251 return utils.Symlink(targetPath, linkPath) 252 } 253 254 func (config *cnbBuildOptions) mergeEnvVars(vars map[string]interface{}) { 255 if config.BuildEnvVars == nil { 256 config.BuildEnvVars = vars 257 258 return 259 } 260 261 for k, v := range vars { 262 _, exists := config.BuildEnvVars[k] 263 264 if !exists { 265 config.BuildEnvVars[k] = v 266 } 267 } 268 } 269 270 func (config *cnbBuildOptions) resolvePath(utils cnbutils.BuildUtils) (pathEnum, string, error) { 271 pwd, err := utils.Getwd() 272 if err != nil { 273 log.SetErrorCategory(log.ErrorBuild) 274 return "", "", errors.Wrap(err, "failed to get current working directory") 275 } 276 277 if config.Path == "" { 278 return pathEnumRoot, pwd, nil 279 } 280 source, err := utils.Abs(config.Path) 281 if err != nil { 282 log.SetErrorCategory(log.ErrorConfiguration) 283 return "", "", errors.Wrapf(err, "Failed to resolve absolute path for '%s'", config.Path) 284 } 285 286 dir, err := utils.DirExists(source) 287 if err != nil { 288 log.SetErrorCategory(log.ErrorBuild) 289 return "", "", errors.Wrapf(err, "Checking file info '%s' failed", source) 290 } 291 292 if dir { 293 return pathEnumFolder, source, nil 294 } else { 295 return pathEnumArchive, source, nil 296 } 297 } 298 299 func addConfigTelemetryData(utils cnbutils.BuildUtils, data *cnbBuildTelemetryData, config *cnbBuildOptions) { 300 var bindingKeys []string 301 for k := range config.Bindings { 302 bindingKeys = append(bindingKeys, k) 303 } 304 data.ImageTag = config.ContainerImageTag 305 data.AdditionalTags = config.AdditionalTags 306 data.BindingKeys = bindingKeys 307 data.Path, _, _ = config.resolvePath(utils) // ignore error here, telemetry problems should not fail the build 308 309 configKeys := data.BuildEnv.KeysFromConfig 310 overallKeys := data.BuildEnv.KeysOverall 311 for key := range config.BuildEnvVars { 312 configKeys = append(configKeys, key) 313 overallKeys = append(overallKeys, key) 314 } 315 data.BuildEnv.KeysFromConfig = configKeys 316 data.BuildEnv.KeysOverall = overallKeys 317 318 buildTool, _ := getBuildToolFromStageConfig("cnbBuild") // ignore error here, telemetry problems should not fail the build 319 data.BuildTool = buildTool 320 321 data.Buildpacks.FromConfig = privacy.FilterBuildpacks(config.Buildpacks) 322 323 dockerImage, err := GetDockerImageValue("cnbBuild") 324 if err != nil { 325 log.Entry().Warnf("Error while preparing telemetry: retrieving docker image failed: '%v'", err) 326 data.Builder = "" 327 } else { 328 data.Builder = privacy.FilterBuilder(dockerImage) 329 } 330 } 331 332 func addProjectDescriptorTelemetryData(data *cnbBuildTelemetryData, descriptor project.Descriptor) { 333 descriptorKeys := data.BuildEnv.KeysFromProjectDescriptor 334 overallKeys := data.BuildEnv.KeysOverall 335 for key := range descriptor.EnvVars { 336 descriptorKeys = append(descriptorKeys, key) 337 overallKeys = append(overallKeys, key) 338 } 339 data.BuildEnv.KeysFromProjectDescriptor = descriptorKeys 340 data.BuildEnv.KeysOverall = overallKeys 341 342 data.Buildpacks.FromProjectDescriptor = privacy.FilterBuildpacks(descriptor.Buildpacks) 343 344 data.ProjectDescriptor.Used = true 345 data.ProjectDescriptor.IncludeUsed = descriptor.Include != nil 346 data.ProjectDescriptor.ExcludeUsed = descriptor.Exclude != nil 347 } 348 349 func callCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment, httpClient piperhttp.Sender) error { 350 telemetry := &cnbBuildTelemetry{ 351 Version: 3, 352 } 353 mergedConfigs, err := processConfigs(*config, config.MultipleImages) 354 if err != nil { 355 return errors.Wrap(err, "failed to process config") 356 } 357 for _, c := range mergedConfigs { 358 err = runCnbBuild(&c, telemetryData, telemetry, utils, commonPipelineEnvironment, httpClient) 359 if err != nil { 360 return err 361 } 362 } 363 364 telemetryData.Custom1Label = "cnbBuildStepData" 365 customData, err := json.Marshal(telemetry) 366 if err != nil { 367 return errors.Wrap(err, "failed to marshal custom telemetry data") 368 } 369 telemetryData.Custom1 = string(customData) 370 return nil 371 } 372 373 func runCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, telemetry *cnbBuildTelemetry, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment, httpClient piperhttp.Sender) error { 374 err := cleanDir("/layers", utils) 375 if err != nil { 376 log.SetErrorCategory(log.ErrorBuild) 377 return errors.Wrap(err, "failed to clean up layers folder /layers") 378 } 379 380 err = cleanDir(platformPath, utils) 381 if err != nil { 382 log.SetErrorCategory(log.ErrorBuild) 383 return errors.Wrap(err, fmt.Sprintf("failed to clean up platform folder %s", platformPath)) 384 } 385 386 customTelemetryData := cnbBuildTelemetryData{} 387 addConfigTelemetryData(utils, &customTelemetryData, config) 388 389 err = isBuilder(utils) 390 if err != nil { 391 log.SetErrorCategory(log.ErrorConfiguration) 392 return errors.Wrap(err, "the provided dockerImage is not a valid builder") 393 } 394 395 include := ignore.CompileIgnoreLines("**/*") 396 exclude := ignore.CompileIgnoreLines("piper", ".pipeline") 397 398 projDescPath, err := project.ResolvePath(config.ProjectDescriptor, config.Path, utils) 399 if err != nil { 400 log.SetErrorCategory(log.ErrorConfiguration) 401 return errors.Wrap(err, "failed to check if project descriptor exists") 402 } 403 404 var projectID string 405 if projDescPath != "" { 406 descriptor, err := project.ParseDescriptor(projDescPath, utils, httpClient) 407 if err != nil { 408 log.SetErrorCategory(log.ErrorConfiguration) 409 return errors.Wrapf(err, "failed to parse %s", projDescPath) 410 } 411 addProjectDescriptorTelemetryData(&customTelemetryData, *descriptor) 412 413 config.mergeEnvVars(descriptor.EnvVars) 414 415 if (config.Buildpacks == nil || len(config.Buildpacks) == 0) && len(descriptor.Buildpacks) > 0 { 416 config.Buildpacks = descriptor.Buildpacks 417 } 418 419 if descriptor.Exclude != nil { 420 exclude = descriptor.Exclude 421 } 422 423 if descriptor.Include != nil { 424 include = descriptor.Include 425 } 426 427 projectID = descriptor.ProjectID 428 } 429 430 targetImage, err := cnbutils.GetTargetImage(config.ContainerRegistryURL, config.ContainerImageName, config.ContainerImageTag, projectID, GeneralConfig.EnvRootPath) 431 if err != nil { 432 log.SetErrorCategory(log.ErrorConfiguration) 433 return errors.Wrap(err, "failed to retrieve target image configuration") 434 } 435 customTelemetryData.Buildpacks.Overall = privacy.FilterBuildpacks(config.Buildpacks) 436 customTelemetryData.BuildEnv.KeyValues = privacy.FilterEnv(config.BuildEnvVars) 437 telemetry.Data = append(telemetry.Data, customTelemetryData) 438 439 if commonPipelineEnvironment.container.imageNameTag == "" { 440 commonPipelineEnvironment.container.registryURL = fmt.Sprintf("%s://%s", targetImage.ContainerRegistry.Scheme, targetImage.ContainerRegistry.Host) 441 commonPipelineEnvironment.container.imageNameTag = fmt.Sprintf("%v:%v", targetImage.ContainerImageName, targetImage.ContainerImageTag) 442 } 443 commonPipelineEnvironment.container.imageNameTags = append(commonPipelineEnvironment.container.imageNameTags, fmt.Sprintf("%v:%v", targetImage.ContainerImageName, targetImage.ContainerImageTag)) 444 commonPipelineEnvironment.container.imageNames = append(commonPipelineEnvironment.container.imageNames, targetImage.ContainerImageName) 445 446 if config.BuildEnvVars != nil && len(config.BuildEnvVars) > 0 { 447 log.Entry().Infof("Setting custom environment variables: '%v'", config.BuildEnvVars) 448 err = cnbutils.CreateEnvFiles(utils, platformPath, config.BuildEnvVars) 449 if err != nil { 450 log.SetErrorCategory(log.ErrorConfiguration) 451 return errors.Wrap(err, "failed to write environment variables to files") 452 } 453 } 454 455 err = bindings.ProcessBindings(utils, httpClient, platformPath, config.Bindings) 456 if err != nil { 457 log.SetErrorCategory(log.ErrorConfiguration) 458 return errors.Wrap(err, "failed process bindings") 459 } 460 461 dockerConfigFile := "" 462 if len(config.DockerConfigJSON) > 0 { 463 dockerConfigFile, err = prepareDockerConfig(config.DockerConfigJSON, utils) 464 if err != nil { 465 log.SetErrorCategory(log.ErrorConfiguration) 466 return errors.Wrapf(err, "failed to rename DockerConfigJSON file '%v'", config.DockerConfigJSON) 467 } 468 } 469 470 pathType, source, err := config.resolvePath(utils) 471 if err != nil { 472 log.SetErrorCategory(log.ErrorBuild) 473 return errors.Wrapf(err, "could no resolve path") 474 } 475 476 target := "/workspace" 477 err = cleanDir(target, utils) 478 if err != nil { 479 log.SetErrorCategory(log.ErrorBuild) 480 return errors.Wrapf(err, "failed to clean up target folder %s", target) 481 } 482 483 if pathType != pathEnumArchive { 484 err = cnbutils.CopyProject(source, target, include, exclude, utils) 485 if err != nil { 486 log.SetErrorCategory(log.ErrorBuild) 487 return errors.Wrapf(err, "Copying '%s' into '%s' failed", source, target) 488 } 489 } else { 490 err = extractZip(source, target) 491 if err != nil { 492 log.SetErrorCategory(log.ErrorBuild) 493 return errors.Wrapf(err, "Copying '%s' into '%s' failed", source, target) 494 } 495 } 496 497 if ok, _ := utils.FileExists(filepath.Join(target, "pom.xml")); ok { 498 err = linkTargetFolder(utils, source, target) 499 if err != nil { 500 log.SetErrorCategory(log.ErrorBuild) 501 return err 502 } 503 } 504 505 metadata.WriteProjectMetadata(GeneralConfig.EnvRootPath, utils) 506 507 var buildpacksPath = "/cnb/buildpacks" 508 var orderPath = "/cnb/order.toml" 509 510 if config.Buildpacks != nil && len(config.Buildpacks) > 0 { 511 log.Entry().Infof("Setting custom buildpacks: '%v'", config.Buildpacks) 512 buildpacksPath, orderPath, err = setCustomBuildpacks(config.Buildpacks, dockerConfigFile, utils) 513 defer utils.RemoveAll(buildpacksPath) 514 defer utils.RemoveAll(orderPath) 515 if err != nil { 516 log.SetErrorCategory(log.ErrorBuild) 517 return errors.Wrapf(err, "Setting custom buildpacks: %v", config.Buildpacks) 518 } 519 } 520 521 cnbRegistryAuth, err := cnbutils.GenerateCnbAuth(dockerConfigFile, utils) 522 if err != nil { 523 log.SetErrorCategory(log.ErrorConfiguration) 524 return errors.Wrap(err, "failed to generate CNB_REGISTRY_AUTH") 525 } 526 527 if len(config.CustomTLSCertificateLinks) > 0 { 528 caCertificates := "/tmp/ca-certificates.crt" 529 _, err := utils.Copy("/etc/ssl/certs/ca-certificates.crt", caCertificates) 530 if err != nil { 531 return errors.Wrap(err, "failed to copy certificates") 532 } 533 err = certutils.CertificateUpdate(config.CustomTLSCertificateLinks, httpClient, utils, caCertificates) 534 if err != nil { 535 return errors.Wrap(err, "failed to update certificates") 536 } 537 utils.AppendEnv([]string{fmt.Sprintf("SSL_CERT_FILE=%s", caCertificates)}) 538 } else { 539 log.Entry().Info("skipping certificates update") 540 } 541 542 utils.AppendEnv([]string{fmt.Sprintf("CNB_REGISTRY_AUTH=%s", cnbRegistryAuth)}) 543 utils.AppendEnv([]string{"CNB_PLATFORM_API=0.8"}) 544 545 creatorArgs := []string{ 546 "-no-color", 547 "-buildpacks", buildpacksPath, 548 "-order", orderPath, 549 "-platform", platformPath, 550 "-skip-restore", 551 } 552 553 if GeneralConfig.Verbose { 554 creatorArgs = append(creatorArgs, "-log-level", "debug") 555 } 556 557 containerImage := path.Join(targetImage.ContainerRegistry.Host, targetImage.ContainerImageName) 558 for _, tag := range config.AdditionalTags { 559 target := fmt.Sprintf("%s:%s", containerImage, tag) 560 if !piperutils.ContainsString(creatorArgs, target) { 561 creatorArgs = append(creatorArgs, "-tag", target) 562 } 563 } 564 565 creatorArgs = append(creatorArgs, fmt.Sprintf("%s:%s", containerImage, targetImage.ContainerImageTag)) 566 err = utils.RunExecutable(creatorPath, creatorArgs...) 567 if err != nil { 568 log.SetErrorCategory(log.ErrorBuild) 569 return errors.Wrapf(err, "execution of '%s' failed", creatorArgs) 570 } 571 572 digest, err := cnbutils.DigestFromReport(utils) 573 if err != nil { 574 log.SetErrorCategory(log.ErrorBuild) 575 return errors.Wrap(err, "failed to read image digest") 576 } 577 commonPipelineEnvironment.container.imageDigest = digest 578 commonPipelineEnvironment.container.imageDigests = append(commonPipelineEnvironment.container.imageDigests, digest) 579 580 if len(config.PreserveFiles) > 0 { 581 if pathType != pathEnumArchive { 582 err = cnbutils.CopyProject(target, source, ignore.CompileIgnoreLines(config.PreserveFiles...), nil, utils) 583 if err != nil { 584 log.SetErrorCategory(log.ErrorBuild) 585 return errors.Wrapf(err, "failed to preserve files using glob '%s'", config.PreserveFiles) 586 } 587 } else { 588 log.Entry().Warnf("skipping preserving files because the source '%s' is an archive", source) 589 } 590 } 591 592 return nil 593 }