github.com/jaylevin/jenkins-library@v1.230.4/cmd/cnbBuild.go (about) 1 package cmd 2 3 import ( 4 "archive/zip" 5 "encoding/json" 6 "fmt" 7 "os" 8 "path" 9 "path/filepath" 10 11 "github.com/SAP/jenkins-library/pkg/certutils" 12 "github.com/SAP/jenkins-library/pkg/cnbutils" 13 "github.com/SAP/jenkins-library/pkg/cnbutils/bindings" 14 "github.com/SAP/jenkins-library/pkg/cnbutils/privacy" 15 "github.com/SAP/jenkins-library/pkg/cnbutils/project" 16 "github.com/SAP/jenkins-library/pkg/cnbutils/project/metadata" 17 "github.com/SAP/jenkins-library/pkg/command" 18 "github.com/SAP/jenkins-library/pkg/docker" 19 piperhttp "github.com/SAP/jenkins-library/pkg/http" 20 "github.com/SAP/jenkins-library/pkg/log" 21 "github.com/SAP/jenkins-library/pkg/piperutils" 22 23 "github.com/SAP/jenkins-library/pkg/telemetry" 24 "github.com/imdario/mergo" 25 "github.com/mitchellh/mapstructure" 26 "github.com/pkg/errors" 27 ignore "github.com/sabhiram/go-gitignore" 28 ) 29 30 const ( 31 creatorPath = "/cnb/lifecycle/creator" 32 platformPath = "/tmp/platform" 33 ) 34 35 type pathEnum string 36 37 const ( 38 pathEnumRoot = pathEnum("root") 39 pathEnumFolder = pathEnum("folder") 40 pathEnumArchive = pathEnum("archive") 41 ) 42 43 type cnbBuildUtilsBundle struct { 44 *command.Command 45 *piperutils.Files 46 *docker.Client 47 } 48 49 type cnbBuildTelemetry struct { 50 Version int `json:"version"` 51 Data []cnbBuildTelemetryData `json:"data"` 52 } 53 54 type cnbBuildTelemetryData struct { 55 ImageTag string `json:"imageTag"` 56 AdditionalTags []string `json:"additionalTags"` 57 BindingKeys []string `json:"bindingKeys"` 58 Path pathEnum `json:"path"` 59 BuildEnv cnbBuildTelemetryDataBuildEnv `json:"buildEnv"` 60 Buildpacks cnbBuildTelemetryDataBuildpacks `json:"buildpacks"` 61 ProjectDescriptor cnbBuildTelemetryDataProjectDescriptor `json:"projectDescriptor"` 62 BuildTool string `json:"buildTool"` 63 Builder string `json:"builder"` 64 } 65 66 type cnbBuildTelemetryDataBuildEnv struct { 67 KeysFromConfig []string `json:"keysFromConfig"` 68 KeysFromProjectDescriptor []string `json:"keysFromProjectDescriptor"` 69 KeysOverall []string `json:"keysOverall"` 70 JVMVersion string `json:"jvmVersion"` 71 KeyValues map[string]interface{} `json:"keyValues"` 72 } 73 74 type cnbBuildTelemetryDataBuildpacks struct { 75 FromConfig []string `json:"FromConfig"` 76 FromProjectDescriptor []string `json:"FromProjectDescriptor"` 77 Overall []string `json:"overall"` 78 } 79 80 type cnbBuildTelemetryDataProjectDescriptor struct { 81 Used bool `json:"used"` 82 IncludeUsed bool `json:"includeUsed"` 83 ExcludeUsed bool `json:"excludeUsed"` 84 } 85 86 func processConfigs(main cnbBuildOptions, multipleImages []map[string]interface{}) ([]cnbBuildOptions, error) { 87 var result []cnbBuildOptions 88 89 if len(multipleImages) == 0 { 90 result = append(result, main) 91 return result, nil 92 } 93 94 for _, conf := range multipleImages { 95 var structuredConf cnbBuildOptions 96 err := mapstructure.Decode(conf, &structuredConf) 97 if err != nil { 98 return nil, err 99 } 100 101 err = mergo.Merge(&structuredConf, main) 102 if err != nil { 103 return nil, err 104 } 105 106 result = append(result, structuredConf) 107 } 108 109 return result, nil 110 } 111 112 func setCustomBuildpacks(bpacks []string, dockerCreds string, utils cnbutils.BuildUtils) (string, string, error) { 113 buildpacksPath := "/tmp/buildpacks" 114 orderPath := "/tmp/buildpacks/order.toml" 115 newOrder, err := cnbutils.DownloadBuildpacks(buildpacksPath, bpacks, dockerCreds, utils) 116 if err != nil { 117 return "", "", err 118 } 119 120 err = newOrder.Save(orderPath) 121 if err != nil { 122 return "", "", err 123 } 124 125 return buildpacksPath, orderPath, nil 126 } 127 128 func newCnbBuildUtils() cnbutils.BuildUtils { 129 utils := cnbBuildUtilsBundle{ 130 Command: &command.Command{}, 131 Files: &piperutils.Files{}, 132 Client: &docker.Client{}, 133 } 134 utils.Stdout(log.Writer()) 135 utils.Stderr(log.Writer()) 136 return &utils 137 } 138 139 func cnbBuild(config cnbBuildOptions, telemetryData *telemetry.CustomData, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment) { 140 utils := newCnbBuildUtils() 141 142 client := &piperhttp.Client{} 143 144 err := callCnbBuild(&config, telemetryData, utils, commonPipelineEnvironment, client) 145 if err != nil { 146 log.Entry().WithError(err).Fatal("step execution failed") 147 } 148 } 149 150 func isBuilder(utils cnbutils.BuildUtils) error { 151 exists, err := utils.FileExists(creatorPath) 152 if err != nil { 153 return err 154 } 155 156 if !exists { 157 return fmt.Errorf("binary '%s' not found", creatorPath) 158 } 159 160 return nil 161 } 162 163 func isZip(path string) bool { 164 r, err := zip.OpenReader(path) 165 166 switch { 167 case err == nil: 168 _ = r.Close() 169 return true 170 case err == zip.ErrFormat: 171 return false 172 default: 173 return false 174 } 175 } 176 177 func cleanDir(dir string, utils cnbutils.BuildUtils) error { 178 dirContent, err := utils.Glob(filepath.Join(dir, "*")) 179 if err != nil { 180 return err 181 } 182 183 for _, obj := range dirContent { 184 err = utils.RemoveAll(obj) 185 if err != nil { 186 return err 187 } 188 } 189 190 return nil 191 } 192 193 func extractZip(source, target string) error { 194 if isZip(source) { 195 log.Entry().Infof("Extracting archive '%s' to '%s'", source, target) 196 _, err := piperutils.Unzip(source, target) 197 if err != nil { 198 log.SetErrorCategory(log.ErrorBuild) 199 return errors.Wrapf(err, "Extracting archive '%s' to '%s' failed", source, target) 200 } 201 } else { 202 log.SetErrorCategory(log.ErrorBuild) 203 return errors.New("application path must be a directory or zip") 204 } 205 206 return nil 207 } 208 209 func prepareDockerConfig(source string, utils cnbutils.BuildUtils) (string, error) { 210 if filepath.Base(source) != "config.json" { 211 log.Entry().Debugf("Renaming docker config file from '%s' to 'config.json'", filepath.Base(source)) 212 213 newPath := filepath.Join(filepath.Dir(source), "config.json") 214 alreadyExists, err := utils.FileExists(newPath) 215 if err != nil { 216 return "", err 217 } 218 if alreadyExists { 219 return newPath, nil 220 } 221 222 err = utils.FileRename(source, newPath) 223 if err != nil { 224 return "", err 225 } 226 227 return newPath, nil 228 } 229 230 return source, nil 231 } 232 233 func linkTargetFolder(utils cnbutils.BuildUtils, source, target string) error { 234 var err error 235 linkPath := filepath.Join(target, "target") 236 targetPath := filepath.Join(source, "target") 237 if ok, _ := utils.DirExists(targetPath); !ok { 238 err = utils.MkdirAll(targetPath, os.ModePerm) 239 if err != nil { 240 return err 241 } 242 } 243 244 if ok, _ := utils.DirExists(linkPath); ok { 245 err = utils.RemoveAll(linkPath) 246 if err != nil { 247 return err 248 } 249 } 250 251 return utils.Symlink(targetPath, linkPath) 252 } 253 254 func (config *cnbBuildOptions) mergeEnvVars(vars map[string]interface{}) { 255 if config.BuildEnvVars == nil { 256 config.BuildEnvVars = vars 257 258 return 259 } 260 261 for k, v := range vars { 262 _, exists := config.BuildEnvVars[k] 263 264 if !exists { 265 config.BuildEnvVars[k] = v 266 } 267 } 268 } 269 270 func (config *cnbBuildOptions) resolvePath(utils cnbutils.BuildUtils) (pathEnum, string, error) { 271 pwd, err := utils.Getwd() 272 if err != nil { 273 log.SetErrorCategory(log.ErrorBuild) 274 return "", "", errors.Wrap(err, "failed to get current working directory") 275 } 276 277 if config.Path == "" { 278 return pathEnumRoot, pwd, nil 279 } 280 matches, err := utils.Glob(config.Path) 281 if err != nil { 282 log.SetErrorCategory(log.ErrorConfiguration) 283 return "", "", errors.Wrapf(err, "Failed to resolve glob for '%s'", config.Path) 284 } 285 numMatches := len(matches) 286 if numMatches != 1 { 287 log.SetErrorCategory(log.ErrorConfiguration) 288 return "", "", errors.Errorf("Failed to resolve glob for '%s', matching %d file(s)", config.Path, numMatches) 289 } 290 source, err := utils.Abs(matches[0]) 291 if err != nil { 292 log.SetErrorCategory(log.ErrorConfiguration) 293 return "", "", errors.Wrapf(err, "Failed to resolve absolute path for '%s'", matches[0]) 294 } 295 296 dir, err := utils.DirExists(source) 297 if err != nil { 298 log.SetErrorCategory(log.ErrorBuild) 299 return "", "", errors.Wrapf(err, "Checking file info '%s' failed", source) 300 } 301 302 if dir { 303 return pathEnumFolder, source, nil 304 } else { 305 return pathEnumArchive, source, nil 306 } 307 } 308 309 func addConfigTelemetryData(utils cnbutils.BuildUtils, data *cnbBuildTelemetryData, config *cnbBuildOptions) { 310 var bindingKeys []string 311 for k := range config.Bindings { 312 bindingKeys = append(bindingKeys, k) 313 } 314 data.ImageTag = config.ContainerImageTag 315 data.AdditionalTags = config.AdditionalTags 316 data.BindingKeys = bindingKeys 317 data.Path, _, _ = config.resolvePath(utils) // ignore error here, telemetry problems should not fail the build 318 319 configKeys := data.BuildEnv.KeysFromConfig 320 overallKeys := data.BuildEnv.KeysOverall 321 for key := range config.BuildEnvVars { 322 configKeys = append(configKeys, key) 323 overallKeys = append(overallKeys, key) 324 } 325 data.BuildEnv.KeysFromConfig = configKeys 326 data.BuildEnv.KeysOverall = overallKeys 327 328 buildTool, _ := getBuildToolFromStageConfig("cnbBuild") // ignore error here, telemetry problems should not fail the build 329 data.BuildTool = buildTool 330 331 data.Buildpacks.FromConfig = privacy.FilterBuildpacks(config.Buildpacks) 332 333 dockerImage, err := GetDockerImageValue("cnbBuild") 334 if err != nil { 335 log.Entry().Warnf("Error while preparing telemetry: retrieving docker image failed: '%v'", err) 336 data.Builder = "" 337 } else { 338 data.Builder = privacy.FilterBuilder(dockerImage) 339 } 340 } 341 342 func addProjectDescriptorTelemetryData(data *cnbBuildTelemetryData, descriptor project.Descriptor) { 343 descriptorKeys := data.BuildEnv.KeysFromProjectDescriptor 344 overallKeys := data.BuildEnv.KeysOverall 345 for key := range descriptor.EnvVars { 346 descriptorKeys = append(descriptorKeys, key) 347 overallKeys = append(overallKeys, key) 348 } 349 data.BuildEnv.KeysFromProjectDescriptor = descriptorKeys 350 data.BuildEnv.KeysOverall = overallKeys 351 352 data.Buildpacks.FromProjectDescriptor = privacy.FilterBuildpacks(descriptor.Buildpacks) 353 354 data.ProjectDescriptor.Used = true 355 data.ProjectDescriptor.IncludeUsed = descriptor.Include != nil 356 data.ProjectDescriptor.ExcludeUsed = descriptor.Exclude != nil 357 } 358 359 func callCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment, httpClient piperhttp.Sender) error { 360 telemetry := &cnbBuildTelemetry{ 361 Version: 3, 362 } 363 mergedConfigs, err := processConfigs(*config, config.MultipleImages) 364 if err != nil { 365 return errors.Wrap(err, "failed to process config") 366 } 367 for _, c := range mergedConfigs { 368 err = runCnbBuild(&c, telemetryData, telemetry, utils, commonPipelineEnvironment, httpClient) 369 if err != nil { 370 return err 371 } 372 } 373 374 telemetryData.Custom1Label = "cnbBuildStepData" 375 customData, err := json.Marshal(telemetry) 376 if err != nil { 377 return errors.Wrap(err, "failed to marshal custom telemetry data") 378 } 379 telemetryData.Custom1 = string(customData) 380 return nil 381 } 382 383 func runCnbBuild(config *cnbBuildOptions, telemetryData *telemetry.CustomData, telemetry *cnbBuildTelemetry, utils cnbutils.BuildUtils, commonPipelineEnvironment *cnbBuildCommonPipelineEnvironment, httpClient piperhttp.Sender) error { 384 err := cleanDir("/layers", utils) 385 if err != nil { 386 log.SetErrorCategory(log.ErrorBuild) 387 return errors.Wrap(err, "failed to clean up layers folder /layers") 388 } 389 390 err = cleanDir(platformPath, utils) 391 if err != nil { 392 log.SetErrorCategory(log.ErrorBuild) 393 return errors.Wrap(err, fmt.Sprintf("failed to clean up platform folder %s", platformPath)) 394 } 395 396 customTelemetryData := cnbBuildTelemetryData{} 397 addConfigTelemetryData(utils, &customTelemetryData, config) 398 399 err = isBuilder(utils) 400 if err != nil { 401 log.SetErrorCategory(log.ErrorConfiguration) 402 return errors.Wrap(err, "the provided dockerImage is not a valid builder") 403 } 404 405 include := ignore.CompileIgnoreLines("**/*") 406 exclude := ignore.CompileIgnoreLines("piper", ".pipeline", ".git") 407 408 projDescPath, err := project.ResolvePath(config.ProjectDescriptor, config.Path, utils) 409 if err != nil { 410 log.SetErrorCategory(log.ErrorConfiguration) 411 return errors.Wrap(err, "failed to check if project descriptor exists") 412 } 413 414 var projectID string 415 if projDescPath != "" { 416 descriptor, err := project.ParseDescriptor(projDescPath, utils, httpClient) 417 if err != nil { 418 log.SetErrorCategory(log.ErrorConfiguration) 419 return errors.Wrapf(err, "failed to parse %s", projDescPath) 420 } 421 addProjectDescriptorTelemetryData(&customTelemetryData, *descriptor) 422 423 config.mergeEnvVars(descriptor.EnvVars) 424 425 if (config.Buildpacks == nil || len(config.Buildpacks) == 0) && len(descriptor.Buildpacks) > 0 { 426 config.Buildpacks = descriptor.Buildpacks 427 } 428 429 if descriptor.Exclude != nil { 430 exclude = descriptor.Exclude 431 } 432 433 if descriptor.Include != nil { 434 include = descriptor.Include 435 } 436 437 projectID = descriptor.ProjectID 438 } 439 440 targetImage, err := cnbutils.GetTargetImage(config.ContainerRegistryURL, config.ContainerImageName, config.ContainerImageTag, projectID, GeneralConfig.EnvRootPath) 441 if err != nil { 442 log.SetErrorCategory(log.ErrorConfiguration) 443 return errors.Wrap(err, "failed to retrieve target image configuration") 444 } 445 customTelemetryData.Buildpacks.Overall = privacy.FilterBuildpacks(config.Buildpacks) 446 customTelemetryData.BuildEnv.KeyValues = privacy.FilterEnv(config.BuildEnvVars) 447 telemetry.Data = append(telemetry.Data, customTelemetryData) 448 449 if commonPipelineEnvironment.container.imageNameTag == "" { 450 commonPipelineEnvironment.container.registryURL = fmt.Sprintf("%s://%s", targetImage.ContainerRegistry.Scheme, targetImage.ContainerRegistry.Host) 451 commonPipelineEnvironment.container.imageNameTag = fmt.Sprintf("%v:%v", targetImage.ContainerImageName, targetImage.ContainerImageTag) 452 } 453 commonPipelineEnvironment.container.imageNameTags = append(commonPipelineEnvironment.container.imageNameTags, fmt.Sprintf("%v:%v", targetImage.ContainerImageName, targetImage.ContainerImageTag)) 454 commonPipelineEnvironment.container.imageNames = append(commonPipelineEnvironment.container.imageNames, targetImage.ContainerImageName) 455 456 if config.BuildEnvVars != nil && len(config.BuildEnvVars) > 0 { 457 log.Entry().Infof("Setting custom environment variables: '%v'", config.BuildEnvVars) 458 err = cnbutils.CreateEnvFiles(utils, platformPath, config.BuildEnvVars) 459 if err != nil { 460 log.SetErrorCategory(log.ErrorConfiguration) 461 return errors.Wrap(err, "failed to write environment variables to files") 462 } 463 } 464 465 err = bindings.ProcessBindings(utils, httpClient, platformPath, config.Bindings) 466 if err != nil { 467 log.SetErrorCategory(log.ErrorConfiguration) 468 return errors.Wrap(err, "failed process bindings") 469 } 470 471 dockerConfigFile := "" 472 if len(config.DockerConfigJSON) > 0 { 473 dockerConfigFile, err = prepareDockerConfig(config.DockerConfigJSON, utils) 474 if err != nil { 475 log.SetErrorCategory(log.ErrorConfiguration) 476 return errors.Wrapf(err, "failed to rename DockerConfigJSON file '%v'", config.DockerConfigJSON) 477 } 478 } 479 480 pathType, source, err := config.resolvePath(utils) 481 if err != nil { 482 log.SetErrorCategory(log.ErrorBuild) 483 return errors.Wrapf(err, "could not resolve path") 484 } 485 486 target := "/workspace" 487 err = cleanDir(target, utils) 488 if err != nil { 489 log.SetErrorCategory(log.ErrorBuild) 490 return errors.Wrapf(err, "failed to clean up target folder %s", target) 491 } 492 493 if pathType != pathEnumArchive { 494 err = cnbutils.CopyProject(source, target, include, exclude, utils) 495 if err != nil { 496 log.SetErrorCategory(log.ErrorBuild) 497 return errors.Wrapf(err, "Copying '%s' into '%s' failed", source, target) 498 } 499 } else { 500 err = extractZip(source, target) 501 if err != nil { 502 log.SetErrorCategory(log.ErrorBuild) 503 return errors.Wrapf(err, "Copying '%s' into '%s' failed", source, target) 504 } 505 } 506 507 if ok, _ := utils.FileExists(filepath.Join(target, "pom.xml")); ok { 508 err = linkTargetFolder(utils, source, target) 509 if err != nil { 510 log.SetErrorCategory(log.ErrorBuild) 511 return err 512 } 513 } 514 515 metadata.WriteProjectMetadata(GeneralConfig.EnvRootPath, utils) 516 517 var buildpacksPath = "/cnb/buildpacks" 518 var orderPath = "/cnb/order.toml" 519 520 if config.Buildpacks != nil && len(config.Buildpacks) > 0 { 521 log.Entry().Infof("Setting custom buildpacks: '%v'", config.Buildpacks) 522 buildpacksPath, orderPath, err = setCustomBuildpacks(config.Buildpacks, dockerConfigFile, utils) 523 defer utils.RemoveAll(buildpacksPath) 524 defer utils.RemoveAll(orderPath) 525 if err != nil { 526 log.SetErrorCategory(log.ErrorBuild) 527 return errors.Wrapf(err, "Setting custom buildpacks: %v", config.Buildpacks) 528 } 529 } 530 531 cnbRegistryAuth, err := cnbutils.GenerateCnbAuth(dockerConfigFile, utils) 532 if err != nil { 533 log.SetErrorCategory(log.ErrorConfiguration) 534 return errors.Wrap(err, "failed to generate CNB_REGISTRY_AUTH") 535 } 536 537 if len(config.CustomTLSCertificateLinks) > 0 { 538 caCertificates := "/tmp/ca-certificates.crt" 539 _, err := utils.Copy("/etc/ssl/certs/ca-certificates.crt", caCertificates) 540 if err != nil { 541 return errors.Wrap(err, "failed to copy certificates") 542 } 543 err = certutils.CertificateUpdate(config.CustomTLSCertificateLinks, httpClient, utils, caCertificates) 544 if err != nil { 545 return errors.Wrap(err, "failed to update certificates") 546 } 547 utils.AppendEnv([]string{fmt.Sprintf("SSL_CERT_FILE=%s", caCertificates)}) 548 } else { 549 log.Entry().Info("skipping certificates update") 550 } 551 552 utils.AppendEnv([]string{fmt.Sprintf("CNB_REGISTRY_AUTH=%s", cnbRegistryAuth)}) 553 utils.AppendEnv([]string{"CNB_PLATFORM_API=0.8"}) 554 555 creatorArgs := []string{ 556 "-no-color", 557 "-buildpacks", buildpacksPath, 558 "-order", orderPath, 559 "-platform", platformPath, 560 "-skip-restore", 561 } 562 563 if GeneralConfig.Verbose { 564 creatorArgs = append(creatorArgs, "-log-level", "debug") 565 } 566 567 containerImage := path.Join(targetImage.ContainerRegistry.Host, targetImage.ContainerImageName) 568 for _, tag := range config.AdditionalTags { 569 target := fmt.Sprintf("%s:%s", containerImage, tag) 570 if !piperutils.ContainsString(creatorArgs, target) { 571 creatorArgs = append(creatorArgs, "-tag", target) 572 } 573 } 574 575 creatorArgs = append(creatorArgs, fmt.Sprintf("%s:%s", containerImage, targetImage.ContainerImageTag)) 576 err = utils.RunExecutable(creatorPath, creatorArgs...) 577 if err != nil { 578 log.SetErrorCategory(log.ErrorBuild) 579 return errors.Wrapf(err, "execution of '%s' failed", creatorArgs) 580 } 581 582 digest, err := cnbutils.DigestFromReport(utils) 583 if err != nil { 584 log.SetErrorCategory(log.ErrorBuild) 585 return errors.Wrap(err, "failed to read image digest") 586 } 587 commonPipelineEnvironment.container.imageDigest = digest 588 commonPipelineEnvironment.container.imageDigests = append(commonPipelineEnvironment.container.imageDigests, digest) 589 590 if len(config.PreserveFiles) > 0 { 591 if pathType != pathEnumArchive { 592 err = cnbutils.CopyProject(target, source, ignore.CompileIgnoreLines(config.PreserveFiles...), nil, utils) 593 if err != nil { 594 log.SetErrorCategory(log.ErrorBuild) 595 return errors.Wrapf(err, "failed to preserve files using glob '%s'", config.PreserveFiles) 596 } 597 } else { 598 log.Entry().Warnf("skipping preserving files because the source '%s' is an archive", source) 599 } 600 } 601 602 return nil 603 }