github.com/ouraigua/jenkins-library@v0.0.0-20231028010029-fbeaf2f3aa9b/cmd/cnbBuild.go (about) 1 package cmd 2 3 import ( 4 "archive/zip" 5 "fmt" 6 "os" 7 "path" 8 "path/filepath" 9 10 "github.com/SAP/jenkins-library/pkg/buildpacks" 11 "github.com/SAP/jenkins-library/pkg/buildsettings" 12 "github.com/SAP/jenkins-library/pkg/certutils" 13 "github.com/SAP/jenkins-library/pkg/cnbutils" 14 "github.com/SAP/jenkins-library/pkg/cnbutils/bindings" 15 "github.com/SAP/jenkins-library/pkg/cnbutils/project" 16 "github.com/SAP/jenkins-library/pkg/cnbutils/project/metadata" 17 "github.com/SAP/jenkins-library/pkg/command" 18 "github.com/SAP/jenkins-library/pkg/docker" 19 piperhttp "github.com/SAP/jenkins-library/pkg/http" 20 "github.com/SAP/jenkins-library/pkg/log" 21 "github.com/SAP/jenkins-library/pkg/piperutils" 22 "github.com/SAP/jenkins-library/pkg/syft" 23 24 "github.com/SAP/jenkins-library/pkg/telemetry" 25 "github.com/imdario/mergo" 26 "github.com/mitchellh/mapstructure" 27 "github.com/pkg/errors" 28 ignore "github.com/sabhiram/go-gitignore" 29 ) 30 31 const ( 32 creatorPath = "/cnb/lifecycle/creator" 33 platformPath = "/tmp/platform" 34 platformAPIVersion = "0.11" 35 ) 36 37 type cnbBuildUtilsBundle struct { 38 *command.Command 39 *piperutils.Files 40 *docker.Client 41 } 42 43 func processConfigs(main cnbBuildOptions, multipleImages []map[string]interface{}) ([]cnbBuildOptions, error) { 44 var result []cnbBuildOptions 45 46 if len(multipleImages) == 0 { 47 result = append(result, main) 48 return result, nil 49 } 50 51 for _, conf := range multipleImages { 52 var structuredConf cnbBuildOptions 53 err := mapstructure.Decode(conf, &structuredConf) 54 if err != nil { 55 return nil, err 56 } 57 58 err = mergo.Merge(&structuredConf, main) 59 if err != nil { 60 return nil, err 61 } 62 63 result = append(result, structuredConf) 64 } 65 66 return result, nil 67 } 68 69 func setCustomBuildpacks(bpacks, preBuildpacks, postBuildpacks []string, dockerCreds string, utils cnbutils.BuildUtils) (string, string, error) { 70 buildpacksPath := "/tmp/buildpacks" 71 orderPath := "/tmp/buildpacks/order.toml" 72 err := cnbutils.DownloadBuildpacks(buildpacksPath, append(bpacks, append(preBuildpacks, postBuildpacks...)...), dockerCreds, utils) 73 if err != nil { 74 return "", "", err 75 } 76 77 if len(bpacks) == 0 && (len(postBuildpacks) > 0 || len(preBuildpacks) > 0) { 78 matches, err := utils.Glob("/cnb/buildpacks/*") 79 if err != nil { 80 return "", "", err 81 } 82 83 for _, match := range matches { 84 err = cnbutils.CreateVersionSymlinks(buildpacksPath, match, utils) 85 if err != nil { 86 return "", "", err 87 } 88 } 89 } 90 91 newOrder, err := cnbutils.CreateOrder(bpacks, preBuildpacks, postBuildpacks, dockerCreds, utils) 92 if err != nil { 93 return "", "", err 94 } 95 96 err = newOrder.Save(orderPath) 97 if err != nil { 98 return "", "", err 99 } 100 101 return buildpacksPath, orderPath, nil 102 } 103 104 func newCnbBuildUtils() cnbutils.BuildUtils { 105 utils := cnbBuildUtilsBundle{ 106 Command: &command.Command{ 107 StepName: "cnbBuild", 108 }, 109 Files: &piperutils.Files{}, 110 Client: &docker.Client{}, 111 } 112 utils.Stdout(log.Writer()) 113 utils.Stderr(log.Writer()) 114 return &utils 115 } 116 117 func cnbBuild(config cnbBuildOptions, telemetryData *telemetry.CustomData, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment) { 118 utils := newCnbBuildUtils() 119 120 client := &piperhttp.Client{} 121 122 err := callCnbBuild(&config, telemetryData, utils, commonPipelineEnvironment, client) 123 if err != nil { 124 log.Entry().WithError(err).Fatal("step execution failed") 125 } 126 } 127 128 func isBuilder(utils cnbutils.BuildUtils) error { 129 exists, err := utils.FileExists(creatorPath) 130 if err != nil { 131 return err 132 } 133 134 if !exists { 135 return fmt.Errorf("binary '%s' not found", creatorPath) 136 } 137 138 return nil 139 } 140 141 func isZip(path string) bool { 142 r, err := zip.OpenReader(path) 143 144 switch { 145 case err == nil: 146 _ = r.Close() 147 return true 148 case err == zip.ErrFormat: 149 return false 150 default: 151 return false 152 } 153 } 154 155 func cleanDir(dir string, utils cnbutils.BuildUtils) error { 156 dirContent, err := utils.Glob(filepath.Join(dir, "*")) 157 if err != nil { 158 return err 159 } 160 161 for _, obj := range dirContent { 162 err = utils.RemoveAll(obj) 163 if err != nil { 164 return err 165 } 166 } 167 168 return nil 169 } 170 171 func extractZip(source, target string) error { 172 if isZip(source) { 173 log.Entry().Infof("Extracting archive '%s' to '%s'", source, target) 174 _, err := piperutils.Unzip(source, target) 175 if err != nil { 176 log.SetErrorCategory(log.ErrorBuild) 177 return errors.Wrapf(err, "Extracting archive '%s' to '%s' failed", source, target) 178 } 179 } else { 180 log.SetErrorCategory(log.ErrorBuild) 181 return errors.New("application path must be a directory or zip") 182 } 183 184 return nil 185 } 186 187 func ensureDockerConfig(config *cnbBuildOptions, utils cnbutils.BuildUtils) error { 188 newFile := "/tmp/config.json" 189 if config.DockerConfigJSON == "" { 190 config.DockerConfigJSON = newFile 191 192 return utils.FileWrite(config.DockerConfigJSON, []byte("{}"), os.ModePerm) 193 } 194 195 log.Entry().Debugf("Copying docker config file from '%s' to '%s'", config.DockerConfigJSON, newFile) 196 _, err := utils.Copy(config.DockerConfigJSON, newFile) 197 if err != nil { 198 return err 199 } 200 201 err = utils.Chmod(newFile, 0644) 202 if err != nil { 203 return err 204 } 205 206 config.DockerConfigJSON = newFile 207 return nil 208 } 209 210 func linkTargetFolder(utils cnbutils.BuildUtils, source, target string) error { 211 var err error 212 linkPath := filepath.Join(target, "target") 213 targetPath := filepath.Join(source, "target") 214 if ok, _ := utils.DirExists(targetPath); !ok { 215 err = utils.MkdirAll(targetPath, os.ModePerm) 216 if err != nil { 217 return err 218 } 219 } 220 221 if ok, _ := utils.DirExists(linkPath); ok { 222 err = utils.RemoveAll(linkPath) 223 if err != nil { 224 return err 225 } 226 } 227 228 return utils.Symlink(targetPath, linkPath) 229 } 230 231 func (config *cnbBuildOptions) mergeEnvVars(vars map[string]interface{}) { 232 if config.BuildEnvVars == nil { 233 config.BuildEnvVars = vars 234 235 return 236 } 237 238 for k, v := range vars { 239 _, exists := config.BuildEnvVars[k] 240 241 if !exists { 242 config.BuildEnvVars[k] = v 243 } 244 } 245 } 246 247 func (config *cnbBuildOptions) resolvePath(utils cnbutils.BuildUtils) (buildpacks.PathEnum, string, error) { 248 pwd, err := utils.Getwd() 249 if err != nil { 250 log.SetErrorCategory(log.ErrorBuild) 251 return "", "", errors.Wrap(err, "failed to get current working directory") 252 } 253 254 if config.Path == "" { 255 return buildpacks.PathEnumRoot, pwd, nil 256 } 257 matches, err := utils.Glob(config.Path) 258 if err != nil { 259 log.SetErrorCategory(log.ErrorConfiguration) 260 return "", "", errors.Wrapf(err, "Failed to resolve glob for '%s'", config.Path) 261 } 262 numMatches := len(matches) 263 if numMatches != 1 { 264 log.SetErrorCategory(log.ErrorConfiguration) 265 return "", "", errors.Errorf("Failed to resolve glob for '%s', matching %d file(s)", config.Path, numMatches) 266 } 267 source, err := utils.Abs(matches[0]) 268 if err != nil { 269 log.SetErrorCategory(log.ErrorConfiguration) 270 return "", "", errors.Wrapf(err, "Failed to resolve absolute path for '%s'", matches[0]) 271 } 272 273 dir, err := utils.DirExists(source) 274 if err != nil { 275 log.SetErrorCategory(log.ErrorBuild) 276 return "", "", errors.Wrapf(err, "Checking file info '%s' failed", source) 277 } 278 279 if dir { 280 return buildpacks.PathEnumFolder, source, nil 281 } else { 282 return buildpacks.PathEnumArchive, source, nil 283 } 284 } 285 286 func callCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment, httpClient piperhttp.Sender) error { 287 stepName := "cnbBuild" 288 telemetry := buildpacks.NewTelemetry(telemetryData) 289 290 dockerImage, err := GetDockerImageValue(stepName) 291 if err != nil { 292 log.Entry().Warnf("failed to retrieve dockerImage configuration: '%v'", err) 293 } 294 295 telemetry.WithImage(dockerImage) 296 297 cnbBuildConfig := buildsettings.BuildOptions{ 298 CreateBOM: config.CreateBOM, 299 DockerImage: dockerImage, 300 BuildSettingsInfo: config.BuildSettingsInfo, 301 } 302 log.Entry().Debugf("creating build settings information...") 303 buildSettingsInfo, err := buildsettings.CreateBuildSettingsInfo(&cnbBuildConfig, stepName) 304 if err != nil { 305 log.Entry().Warnf("failed to create build settings info: %v", err) 306 } 307 commonPipelineEnvironment.custom.buildSettingsInfo = buildSettingsInfo 308 309 err = ensureDockerConfig(config, utils) 310 if err != nil { 311 log.SetErrorCategory(log.ErrorConfiguration) 312 return errors.Wrapf(err, "failed to create/rename DockerConfigJSON file") 313 } 314 315 if config.DockerConfigJSONCPE != "" { 316 log.Entry().Debugf("merging docker config file '%s' into '%s'", config.DockerConfigJSONCPE, config.DockerConfigJSON) 317 err = docker.MergeDockerConfigJSON(config.DockerConfigJSONCPE, config.DockerConfigJSON, utils) 318 if err != nil { 319 log.SetErrorCategory(log.ErrorConfiguration) 320 return errors.Wrapf(err, "failed to merge DockerConfigJSON files") 321 } 322 } 323 324 mergedConfigs, err := processConfigs(*config, config.MultipleImages) 325 if err != nil { 326 return errors.Wrap(err, "failed to process config") 327 } 328 329 buildSummary := cnbutils.NewBuildSummary(dockerImage, utils) 330 for _, c := range mergedConfigs { 331 imageSummary := &cnbutils.ImageSummary{} 332 err = runCnbBuild(&c, telemetry, imageSummary, utils, commonPipelineEnvironment, httpClient) 333 if err != nil { 334 return err 335 } 336 buildSummary.Images = append(buildSummary.Images, imageSummary) 337 } 338 339 buildSummary.Print() 340 341 if config.CreateBOM { 342 err = syft.GenerateSBOM(config.SyftDownloadURL, filepath.Dir(config.DockerConfigJSON), utils, utils, httpClient, commonPipelineEnvironment.container.registryURL, commonPipelineEnvironment.container.imageNameTags) 343 if err != nil { 344 log.SetErrorCategory(log.ErrorCompliance) 345 return errors.Wrap(err, "failed to create BOM file") 346 } 347 } 348 349 return telemetry.Export() 350 } 351 352 func runCnbBuild(config *cnbBuildOptions, telemetry *buildpacks.Telemetry, imageSummary *cnbutils.ImageSummary, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment, httpClient piperhttp.Sender) error { 353 err := cleanDir("/layers", utils) 354 if err != nil { 355 log.SetErrorCategory(log.ErrorBuild) 356 return errors.Wrap(err, "failed to clean up layers folder /layers") 357 } 358 359 err = cleanDir(platformPath, utils) 360 if err != nil { 361 log.SetErrorCategory(log.ErrorBuild) 362 return errors.Wrap(err, fmt.Sprintf("failed to clean up platform folder %s", platformPath)) 363 } 364 365 tempdir, err := os.MkdirTemp("", "cnbBuild-") 366 if err != nil { 367 return errors.Wrap(err, "failed to create tempdir") 368 } 369 defer os.RemoveAll(tempdir) 370 if config.BuildEnvVars == nil { 371 config.BuildEnvVars = map[string]interface{}{} 372 } 373 config.BuildEnvVars["TMPDIR"] = tempdir 374 375 telemetrySegment := createInitialTelemetrySegment(config, utils) 376 377 err = isBuilder(utils) 378 if err != nil { 379 log.SetErrorCategory(log.ErrorConfiguration) 380 return errors.Wrap(err, "the provided dockerImage is not a valid builder") 381 } 382 383 include := ignore.CompileIgnoreLines("**/*") 384 exclude := ignore.CompileIgnoreLines("piper", ".pipeline", ".git") 385 386 projDescPath, err := project.ResolvePath(config.ProjectDescriptor, config.Path, utils) 387 if err != nil { 388 log.SetErrorCategory(log.ErrorConfiguration) 389 return errors.Wrap(err, "failed to check if project descriptor exists") 390 } 391 imageSummary.ProjectDescriptor = projDescPath 392 393 var projectID string 394 if projDescPath != "" { 395 descriptor, err := project.ParseDescriptor(projDescPath, utils, httpClient) 396 if err != nil { 397 log.SetErrorCategory(log.ErrorConfiguration) 398 return errors.Wrapf(err, "failed to parse %s", projDescPath) 399 } 400 telemetrySegment.WithProjectDescriptor(descriptor) 401 402 config.mergeEnvVars(descriptor.EnvVars) 403 404 if len(config.Buildpacks) == 0 { 405 config.Buildpacks = descriptor.Buildpacks 406 } 407 408 if len(config.PreBuildpacks) == 0 { 409 config.PreBuildpacks = descriptor.PreBuildpacks 410 } 411 412 if len(config.PostBuildpacks) == 0 { 413 config.PostBuildpacks = descriptor.PostBuildpacks 414 } 415 416 if descriptor.Exclude != nil { 417 exclude = descriptor.Exclude 418 } 419 420 if descriptor.Include != nil { 421 include = descriptor.Include 422 } 423 424 projectID = descriptor.ProjectID 425 } 426 427 targetImage, err := cnbutils.GetTargetImage(config.ContainerRegistryURL, config.ContainerImageName, config.ContainerImageTag, projectID, GeneralConfig.EnvRootPath) 428 if err != nil { 429 log.SetErrorCategory(log.ErrorConfiguration) 430 return errors.Wrap(err, "failed to retrieve target image configuration") 431 } 432 433 telemetry.AddSegment(telemetrySegment.WithBuildpacksOverall(config.Buildpacks).WithKeyValues(config.BuildEnvVars)) 434 435 if commonPipelineEnvironment.container.imageNameTag == "" { 436 commonPipelineEnvironment.container.registryURL = fmt.Sprintf("%s://%s", targetImage.ContainerRegistry.Scheme, targetImage.ContainerRegistry.Host) 437 commonPipelineEnvironment.container.imageNameTag = fmt.Sprintf("%v:%v", targetImage.ContainerImageName, targetImage.ContainerImageTag) 438 } 439 commonPipelineEnvironment.container.imageNameTags = append(commonPipelineEnvironment.container.imageNameTags, fmt.Sprintf("%v:%v", targetImage.ContainerImageName, targetImage.ContainerImageTag)) 440 imageNameAlias := targetImage.ContainerImageName 441 if config.ContainerImageAlias != "" { 442 imageNameAlias = config.ContainerImageAlias 443 } 444 commonPipelineEnvironment.container.imageNames = append(commonPipelineEnvironment.container.imageNames, imageNameAlias) 445 446 if config.BuildEnvVars != nil && len(config.BuildEnvVars) > 0 { 447 log.Entry().Infof("Setting custom environment variables: '%v'", config.BuildEnvVars) 448 imageSummary.AddEnv(config.BuildEnvVars) 449 err = cnbutils.CreateEnvFiles(utils, platformPath, config.BuildEnvVars) 450 if err != nil { 451 log.SetErrorCategory(log.ErrorConfiguration) 452 return errors.Wrap(err, "failed to write environment variables to files") 453 } 454 } 455 456 err = bindings.ProcessBindings(utils, httpClient, platformPath, config.Bindings) 457 if err != nil { 458 log.SetErrorCategory(log.ErrorConfiguration) 459 return errors.Wrap(err, "failed process bindings") 460 } 461 462 pathType, source, err := config.resolvePath(utils) 463 if err != nil { 464 log.SetErrorCategory(log.ErrorBuild) 465 return errors.Wrapf(err, "could not resolve path") 466 } 467 468 target := "/workspace" 469 err = cleanDir(target, utils) 470 if err != nil { 471 log.SetErrorCategory(log.ErrorBuild) 472 return errors.Wrapf(err, "failed to clean up target folder %s", target) 473 } 474 475 if pathType != buildpacks.PathEnumArchive { 476 err = cnbutils.CopyProject(source, target, include, exclude, utils) 477 if err != nil { 478 log.SetErrorCategory(log.ErrorBuild) 479 return errors.Wrapf(err, "Copying '%s' into '%s' failed", source, target) 480 } 481 } else { 482 err = extractZip(source, target) 483 if err != nil { 484 log.SetErrorCategory(log.ErrorBuild) 485 return errors.Wrapf(err, "Copying '%s' into '%s' failed", source, target) 486 } 487 } 488 489 if ok, _ := utils.FileExists(filepath.Join(target, "pom.xml")); ok { 490 err = linkTargetFolder(utils, source, target) 491 if err != nil { 492 log.SetErrorCategory(log.ErrorBuild) 493 return err 494 } 495 } 496 497 metadata.WriteProjectMetadata(GeneralConfig.EnvRootPath, utils) 498 499 var buildpacksPath = "/cnb/buildpacks" 500 var orderPath = cnbutils.DefaultOrderPath 501 502 if len(config.Buildpacks) > 0 || len(config.PreBuildpacks) > 0 || len(config.PostBuildpacks) > 0 { 503 log.Entry().Infof("Setting custom buildpacks: '%v'", config.Buildpacks) 504 log.Entry().Infof("Pre-buildpacks: '%v'", config.PreBuildpacks) 505 log.Entry().Infof("Post-buildpacks: '%v'", config.PostBuildpacks) 506 buildpacksPath, orderPath, err = setCustomBuildpacks(config.Buildpacks, config.PreBuildpacks, config.PostBuildpacks, config.DockerConfigJSON, utils) 507 defer func() { _ = utils.RemoveAll(buildpacksPath) }() 508 defer func() { _ = utils.RemoveAll(orderPath) }() 509 if err != nil { 510 log.SetErrorCategory(log.ErrorBuild) 511 return errors.Wrapf(err, "Setting custom buildpacks: %v", config.Buildpacks) 512 } 513 } 514 515 cnbRegistryAuth, err := cnbutils.GenerateCnbAuth(config.DockerConfigJSON, utils) 516 if err != nil { 517 log.SetErrorCategory(log.ErrorConfiguration) 518 return errors.Wrap(err, "failed to generate CNB_REGISTRY_AUTH") 519 } 520 521 if len(config.CustomTLSCertificateLinks) > 0 { 522 caCertificates := "/tmp/ca-certificates.crt" 523 _, err := utils.Copy("/etc/ssl/certs/ca-certificates.crt", caCertificates) 524 if err != nil { 525 return errors.Wrap(err, "failed to copy certificates") 526 } 527 err = certutils.CertificateUpdate(config.CustomTLSCertificateLinks, httpClient, utils, caCertificates) 528 if err != nil { 529 return errors.Wrap(err, "failed to update certificates") 530 } 531 utils.AppendEnv([]string{fmt.Sprintf("SSL_CERT_FILE=%s", caCertificates)}) 532 } else { 533 log.Entry().Info("skipping certificates update") 534 } 535 536 utils.AppendEnv([]string{fmt.Sprintf("CNB_REGISTRY_AUTH=%s", cnbRegistryAuth)}) 537 utils.AppendEnv([]string{fmt.Sprintf("CNB_PLATFORM_API=%s", platformAPIVersion)}) 538 539 creatorArgs := []string{ 540 "-no-color", 541 "-buildpacks", buildpacksPath, 542 "-order", orderPath, 543 "-platform", platformPath, 544 "-skip-restore", 545 } 546 547 if GeneralConfig.Verbose { 548 creatorArgs = append(creatorArgs, "-log-level", "debug") 549 } 550 551 if config.RunImage != "" { 552 creatorArgs = append(creatorArgs, "-run-image", config.RunImage) 553 } 554 555 if config.DefaultProcess != "" { 556 creatorArgs = append(creatorArgs, "-process-type", config.DefaultProcess) 557 } 558 559 containerImage := path.Join(targetImage.ContainerRegistry.Host, targetImage.ContainerImageName) 560 for _, tag := range config.AdditionalTags { 561 target := fmt.Sprintf("%s:%s", containerImage, tag) 562 if !piperutils.ContainsString(creatorArgs, target) { 563 creatorArgs = append(creatorArgs, "-tag", target) 564 } 565 } 566 567 creatorArgs = append(creatorArgs, fmt.Sprintf("%s:%s", containerImage, targetImage.ContainerImageTag)) 568 err = utils.RunExecutable(creatorPath, creatorArgs...) 569 if err != nil { 570 log.SetErrorCategory(log.ErrorBuild) 571 return errors.Wrapf(err, "execution of '%s' failed", creatorArgs) 572 } 573 574 digest, err := cnbutils.DigestFromReport(utils) 575 if err != nil { 576 log.SetErrorCategory(log.ErrorBuild) 577 return errors.Wrap(err, "failed to read image digest") 578 } 579 commonPipelineEnvironment.container.imageDigest = digest 580 commonPipelineEnvironment.container.imageDigests = append(commonPipelineEnvironment.container.imageDigests, digest) 581 imageSummary.ImageRef = fmt.Sprintf("%s@%s", containerImage, digest) 582 583 if len(config.PreserveFiles) > 0 { 584 if pathType != buildpacks.PathEnumArchive { 585 err = cnbutils.CopyProject(target, source, ignore.CompileIgnoreLines(config.PreserveFiles...), nil, utils) 586 if err != nil { 587 log.SetErrorCategory(log.ErrorBuild) 588 return errors.Wrapf(err, "failed to preserve files using glob '%s'", config.PreserveFiles) 589 } 590 } else { 591 log.Entry().Warnf("skipping preserving files because the source '%s' is an archive", source) 592 } 593 } 594 595 return nil 596 } 597 598 func createInitialTelemetrySegment(config *cnbBuildOptions, utils cnbutils.BuildUtils) *buildpacks.Segment { 599 telemetrySegment := buildpacks.NewSegment() 600 projectPath, _, _ := config.resolvePath(utils) // ignore error here, telemetry problems should not fail the build 601 buildTool, _ := getBuildToolFromStageConfig("cnbBuild") // ignore error here, telemetry problems should not fail the build 602 603 return telemetrySegment.WithBindings(config.Bindings). 604 WithTags(config.ContainerImageTag, config.AdditionalTags). 605 WithPath(projectPath). 606 WithEnv(config.BuildEnvVars). 607 WithBuildTool(buildTool). 608 WithBuildpacksFromConfig(config.Buildpacks) 609 }