github.com/SAP/jenkins-library@v1.362.0/cmd/golangBuild.go (about) 1 package cmd 2 3 import ( 4 "bytes" 5 "fmt" 6 "net/http" 7 "os" 8 "path" 9 "path/filepath" 10 "strings" 11 12 "github.com/SAP/jenkins-library/pkg/buildsettings" 13 "github.com/SAP/jenkins-library/pkg/certutils" 14 "github.com/SAP/jenkins-library/pkg/command" 15 "github.com/SAP/jenkins-library/pkg/goget" 16 piperhttp "github.com/SAP/jenkins-library/pkg/http" 17 "github.com/SAP/jenkins-library/pkg/log" 18 "github.com/SAP/jenkins-library/pkg/piperenv" 19 "github.com/SAP/jenkins-library/pkg/piperutils" 20 "github.com/SAP/jenkins-library/pkg/telemetry" 21 22 "github.com/SAP/jenkins-library/pkg/multiarch" 23 "github.com/SAP/jenkins-library/pkg/versioning" 24 25 "golang.org/x/mod/modfile" 26 ) 27 28 const ( 29 coverageFile = "cover.out" 30 golangUnitTestOutput = "TEST-go.xml" 31 golangIntegrationTestOutput = "TEST-integration.xml" 32 unitJsonReport = "unit-report.out" 33 integrationJsonReport = "integration-report.out" 34 golangCoberturaPackage = "github.com/boumenot/gocover-cobertura@latest" 35 golangTestsumPackage = "gotest.tools/gotestsum@latest" 36 golangCycloneDXPackage = "github.com/CycloneDX/cyclonedx-gomod/cmd/cyclonedx-gomod@v1.4.0" 37 sbomFilename = "bom-golang.xml" 38 ) 39 40 type golangBuildUtils interface { 41 command.ExecRunner 42 goget.Client 43 44 piperutils.FileUtils 45 piperhttp.Uploader 46 47 getDockerImageValue(stepName string) (string, error) 48 GetExitCode() int 49 DownloadFile(url, filename string, header http.Header, cookies []*http.Cookie) error 50 Untar(src string, dest string, stripComponentLevel int) error 51 52 // Add more methods here, or embed additional interfaces, or remove/replace as required. 53 // The golangBuildUtils interface should be descriptive of your runtime dependencies, 54 // i.e. include everything you need to be able to mock in tests. 55 // Unit tests shall be executable in parallel (not depend on global state), and don't (re-)test dependencies. 56 } 57 58 type golangBuildUtilsBundle struct { 59 *command.Command 60 *piperutils.Files 61 piperhttp.Uploader 62 httpClient *piperhttp.Client 63 64 goget.Client 65 66 // Embed more structs as necessary to implement methods or interfaces you add to golangBuildUtils. 67 // Structs embedded in this way must each have a unique set of methods attached. 68 // If there is no struct which implements the method you need, attach the method to 69 // golangBuildUtilsBundle and forward to the implementation of the dependency. 70 } 71 72 func (g *golangBuildUtilsBundle) DownloadFile(url, filename string, header http.Header, cookies []*http.Cookie) error { 73 return g.httpClient.DownloadFile(url, filename, header, cookies) 74 } 75 76 func (g *golangBuildUtilsBundle) getDockerImageValue(stepName string) (string, error) { 77 return GetDockerImageValue(stepName) 78 } 79 80 func (g *golangBuildUtilsBundle) Untar(src string, dest string, stripComponentLevel int) error { 81 return piperutils.Untar(src, dest, stripComponentLevel) 82 } 83 84 func newGolangBuildUtils(config golangBuildOptions) golangBuildUtils { 85 httpClientOptions := piperhttp.ClientOptions{} 86 87 if len(config.CustomTLSCertificateLinks) > 0 { 88 httpClientOptions.TransportSkipVerification = false 89 httpClientOptions.TrustedCerts = config.CustomTLSCertificateLinks 90 } 91 92 httpClient := piperhttp.Client{} 93 httpClient.SetOptions(httpClientOptions) 94 95 utils := golangBuildUtilsBundle{ 96 Command: &command.Command{ 97 StepName: "golangBuild", 98 }, 99 Files: &piperutils.Files{}, 100 Uploader: &httpClient, 101 Client: &goget.ClientImpl{ 102 HTTPClient: &httpClient, 103 }, 104 httpClient: &httpClient, 105 } 106 // Reroute command output to logging framework 107 utils.Stdout(log.Writer()) 108 utils.Stderr(log.Writer()) 109 return &utils 110 } 111 112 func golangBuild(config golangBuildOptions, telemetryData *telemetry.CustomData, commonPipelineEnvironment *golangBuildCommonPipelineEnvironment) { 113 // Utils can be used wherever the command.ExecRunner interface is expected. 114 // It can also be used for example as a mavenExecRunner. 115 utils := newGolangBuildUtils(config) 116 117 // Error situations will be bubbled up until they reach the line below which will then stop execution 118 // through the log.Entry().Fatal() call leading to an os.Exit(1) in the end. 119 err := runGolangBuild(&config, telemetryData, utils, commonPipelineEnvironment) 120 if err != nil { 121 log.Entry().WithError(err).Fatal("execution of golang build failed") 122 } 123 } 124 125 func runGolangBuild(config *golangBuildOptions, telemetryData *telemetry.CustomData, utils golangBuildUtils, commonPipelineEnvironment *golangBuildCommonPipelineEnvironment) error { 126 goModFile, err := readGoModFile(utils) // returns nil if go.mod doesnt exist 127 if err != nil { 128 return err 129 } 130 131 if err = prepareGolangEnvironment(config, goModFile, utils); err != nil { 132 return err 133 } 134 135 // install test pre-requisites only in case testing should be performed 136 if config.RunTests || config.RunIntegrationTests { 137 if err := utils.RunExecutable("go", "install", golangTestsumPackage); err != nil { 138 return fmt.Errorf("failed to install pre-requisite: %w", err) 139 } 140 } 141 142 if config.CreateBOM { 143 if err := utils.RunExecutable("go", "install", golangCycloneDXPackage); err != nil { 144 return fmt.Errorf("failed to install pre-requisite: %w", err) 145 } 146 } 147 148 failedTests := false 149 150 if config.RunTests { 151 success, err := runGolangTests(config, utils) 152 if err != nil { 153 return err 154 } 155 failedTests = !success 156 } 157 158 if config.RunTests && config.ReportCoverage { 159 if err := reportGolangTestCoverage(config, utils); err != nil { 160 return err 161 } 162 } 163 164 if config.RunIntegrationTests { 165 success, err := runGolangIntegrationTests(config, utils) 166 if err != nil { 167 return err 168 } 169 failedTests = failedTests || !success 170 } 171 172 if failedTests { 173 log.SetErrorCategory(log.ErrorTest) 174 return fmt.Errorf("some tests failed") 175 } 176 177 if config.RunLint { 178 goPath := os.Getenv("GOPATH") 179 golangciLintDir := filepath.Join(goPath, "bin") 180 181 if err := retrieveGolangciLint(utils, golangciLintDir, config.GolangciLintURL); err != nil { 182 return err 183 } 184 185 // hardcode those for now 186 lintSettings := map[string]string{ 187 "reportStyle": "checkstyle", // readable by Sonar 188 "reportOutputPath": "golangci-lint-report.xml", 189 "additionalParams": "", 190 } 191 192 if err := runGolangciLint(utils, golangciLintDir, config.FailOnLintingError, lintSettings); err != nil { 193 return err 194 } 195 } 196 197 if config.CreateBOM { 198 if err := runBOMCreation(utils, sbomFilename); err != nil { 199 return err 200 } 201 } 202 203 ldflags := "" 204 205 if len(config.LdflagsTemplate) > 0 { 206 ldf, err := prepareLdflags(config, utils, GeneralConfig.EnvRootPath) 207 if err != nil { 208 return err 209 } 210 ldflags = (*ldf).String() 211 log.Entry().Infof("ldflags from template: '%v'", ldflags) 212 } 213 214 var binaries []string 215 platforms, err := multiarch.ParsePlatformStrings(config.TargetArchitectures) 216 if err != nil { 217 return err 218 } 219 220 for _, platform := range platforms { 221 binaryNames, err := runGolangBuildPerArchitecture(config, goModFile, utils, ldflags, platform) 222 if err != nil { 223 return err 224 } 225 226 if len(binaryNames) > 0 { 227 binaries = append(binaries, binaryNames...) 228 } 229 } 230 231 log.Entry().Debugf("creating build settings information...") 232 stepName := "golangBuild" 233 dockerImage, err := utils.getDockerImageValue(stepName) 234 if err != nil { 235 return err 236 } 237 238 buildConfig := buildsettings.BuildOptions{ 239 CreateBOM: config.CreateBOM, 240 Publish: config.Publish, 241 BuildSettingsInfo: config.BuildSettingsInfo, 242 DockerImage: dockerImage, 243 } 244 buildSettingsInfo, err := buildsettings.CreateBuildSettingsInfo(&buildConfig, stepName) 245 if err != nil { 246 log.Entry().Warnf("failed to create build settings info: %v", err) 247 } 248 commonPipelineEnvironment.custom.buildSettingsInfo = buildSettingsInfo 249 250 if config.Publish { 251 if len(config.TargetRepositoryURL) == 0 { 252 return fmt.Errorf("there's no target repository for binary publishing configured") 253 } 254 255 artifactVersion := config.ArtifactVersion 256 257 if len(artifactVersion) == 0 { 258 artifactOpts := versioning.Options{ 259 VersioningScheme: "library", 260 } 261 262 artifact, err := versioning.GetArtifact("golang", "", &artifactOpts, utils) 263 if err != nil { 264 return err 265 } 266 267 artifactVersion, err = artifact.GetVersion() 268 269 if err != nil { 270 return err 271 } 272 } 273 274 if goModFile == nil { 275 return fmt.Errorf("go.mod file not found") 276 } else if goModFile.Module == nil { 277 return fmt.Errorf("go.mod doesn't declare a module path") 278 } 279 280 repoClientOptions := piperhttp.ClientOptions{ 281 Username: config.TargetRepositoryUser, 282 Password: config.TargetRepositoryPassword, 283 TrustedCerts: config.CustomTLSCertificateLinks, 284 } 285 286 utils.SetOptions(repoClientOptions) 287 288 var binaryArtifacts piperenv.Artifacts 289 for _, binary := range binaries { 290 291 targetPath := fmt.Sprintf("go/%s/%s/%s", goModFile.Module.Mod.Path, artifactVersion, binary) 292 293 separator := "/" 294 295 if strings.HasSuffix(config.TargetRepositoryURL, "/") { 296 separator = "" 297 } 298 299 targetURL := fmt.Sprintf("%s%s%s", config.TargetRepositoryURL, separator, targetPath) 300 301 log.Entry().Infof("publishing artifact: %s", targetURL) 302 303 response, err := utils.UploadRequest(http.MethodPut, targetURL, binary, "", nil, nil, "binary") 304 if err != nil { 305 return fmt.Errorf("couldn't upload artifact: %w", err) 306 } 307 308 if !(response.StatusCode == 200 || response.StatusCode == 201) { 309 return fmt.Errorf("couldn't upload artifact, received status code %d", response.StatusCode) 310 } 311 312 binaryArtifacts = append(binaryArtifacts, piperenv.Artifact{ 313 Name: binary, 314 }) 315 } 316 commonPipelineEnvironment.custom.artifacts = binaryArtifacts 317 318 } 319 320 return nil 321 } 322 323 func prepareGolangEnvironment(config *golangBuildOptions, goModFile *modfile.File, utils golangBuildUtils) error { 324 // configure truststore 325 err := certutils.CertificateUpdate(config.CustomTLSCertificateLinks, utils, utils, "/etc/ssl/certs/ca-certificates.crt") // TODO reimplement 326 327 if config.PrivateModules == "" { 328 return nil 329 } 330 331 if config.PrivateModulesGitToken == "" { 332 return fmt.Errorf("please specify a token for fetching private git modules") 333 } 334 335 // pass private repos to go process 336 os.Setenv("GOPRIVATE", config.PrivateModules) 337 338 err = gitConfigurationForPrivateModules(config.PrivateModules, config.PrivateModulesGitToken, utils) 339 if err != nil { 340 return err 341 } 342 343 return nil 344 } 345 346 func runGolangTests(config *golangBuildOptions, utils golangBuildUtils) (bool, error) { 347 // execute gotestsum in order to have more output options 348 testOptions := []string{"--junitfile", golangUnitTestOutput, "--jsonfile", unitJsonReport, "--", fmt.Sprintf("-coverprofile=%v", coverageFile), "-tags=unit", "./..."} 349 testOptions = append(testOptions, config.TestOptions...) 350 if err := utils.RunExecutable("gotestsum", testOptions...); err != nil { 351 exists, fileErr := utils.FileExists(golangUnitTestOutput) 352 if !exists || fileErr != nil { 353 log.SetErrorCategory(log.ErrorBuild) 354 return false, fmt.Errorf("running tests failed - junit result missing: %w", err) 355 } 356 exists, fileErr = utils.FileExists(coverageFile) 357 if !exists || fileErr != nil { 358 log.SetErrorCategory(log.ErrorBuild) 359 return false, fmt.Errorf("running tests failed - coverage output missing: %w", err) 360 } 361 return false, nil 362 } 363 return true, nil 364 } 365 366 func runGolangIntegrationTests(config *golangBuildOptions, utils golangBuildUtils) (bool, error) { 367 // execute gotestsum in order to have more output options 368 // for integration tests coverage data is not meaningful and thus not being created 369 if err := utils.RunExecutable("gotestsum", "--junitfile", golangIntegrationTestOutput, "--jsonfile", integrationJsonReport, "--", "-tags=integration", "./..."); err != nil { 370 exists, fileErr := utils.FileExists(golangIntegrationTestOutput) 371 if !exists || fileErr != nil { 372 log.SetErrorCategory(log.ErrorBuild) 373 return false, fmt.Errorf("running tests failed: %w", err) 374 } 375 return false, nil 376 } 377 return true, nil 378 } 379 380 func reportGolangTestCoverage(config *golangBuildOptions, utils golangBuildUtils) error { 381 if config.CoverageFormat == "cobertura" { 382 // execute gocover-cobertura in order to create cobertura report 383 // install pre-requisites 384 if err := utils.RunExecutable("go", "install", golangCoberturaPackage); err != nil { 385 return fmt.Errorf("failed to install pre-requisite: %w", err) 386 } 387 388 coverageData, err := utils.FileRead(coverageFile) 389 if err != nil { 390 return fmt.Errorf("failed to read coverage file %v: %w", coverageFile, err) 391 } 392 utils.Stdin(bytes.NewBuffer(coverageData)) 393 394 coverageOutput := bytes.Buffer{} 395 utils.Stdout(&coverageOutput) 396 options := []string{} 397 if config.ExcludeGeneratedFromCoverage { 398 options = append(options, "-ignore-gen-files") 399 } 400 if err := utils.RunExecutable("gocover-cobertura", options...); err != nil { 401 log.SetErrorCategory(log.ErrorTest) 402 return fmt.Errorf("failed to convert coverage data to cobertura format: %w", err) 403 } 404 utils.Stdout(log.Writer()) 405 406 err = utils.FileWrite("cobertura-coverage.xml", coverageOutput.Bytes(), 0o666) 407 if err != nil { 408 return fmt.Errorf("failed to create cobertura coverage file: %w", err) 409 } 410 log.Entry().Info("created file cobertura-coverage.xml") 411 } else { 412 // currently only cobertura and html format supported, thus using html as fallback 413 if err := utils.RunExecutable("go", "tool", "cover", "-html", coverageFile, "-o", "coverage.html"); err != nil { 414 return fmt.Errorf("failed to create html coverage file: %w", err) 415 } 416 } 417 return nil 418 } 419 420 func retrieveGolangciLint(utils golangBuildUtils, golangciLintDir, golangciLintURL string) error { 421 archiveName := "golangci-lint.tar.gz" 422 err := utils.DownloadFile(golangciLintURL, archiveName, nil, nil) 423 if err != nil { 424 return fmt.Errorf("failed to download golangci-lint: %w", err) 425 } 426 427 err = utils.Untar(archiveName, golangciLintDir, 1) 428 if err != nil { 429 return fmt.Errorf("failed to install golangci-lint: %w", err) 430 } 431 432 return nil 433 } 434 435 func runGolangciLint(utils golangBuildUtils, golangciLintDir string, failOnError bool, lintSettings map[string]string) error { 436 binaryPath := filepath.Join(golangciLintDir, "golangci-lint") 437 438 var outputBuffer bytes.Buffer 439 utils.Stdout(&outputBuffer) 440 err := utils.RunExecutable(binaryPath, "run", "--out-format", lintSettings["reportStyle"]) 441 if err != nil && utils.GetExitCode() != 1 { 442 return fmt.Errorf("running golangci-lint failed: %w", err) 443 } 444 445 log.Entry().Infof("lint report: \n" + outputBuffer.String()) 446 log.Entry().Infof("writing lint report to %s", lintSettings["reportOutputPath"]) 447 err = utils.FileWrite(lintSettings["reportOutputPath"], outputBuffer.Bytes(), 0o644) 448 if err != nil { 449 return fmt.Errorf("writing golangci-lint report failed: %w", err) 450 } 451 452 if utils.GetExitCode() == 1 && failOnError { 453 return fmt.Errorf("golangci-lint found issues, see report above") 454 } 455 456 return nil 457 } 458 459 func prepareLdflags(config *golangBuildOptions, utils golangBuildUtils, envRootPath string) (*bytes.Buffer, error) { 460 cpe := piperenv.CPEMap{} 461 err := cpe.LoadFromDisk(path.Join(envRootPath, "commonPipelineEnvironment")) 462 if err != nil { 463 log.Entry().Warning("failed to load values from commonPipelineEnvironment") 464 } 465 466 log.Entry().Debugf("ldflagsTemplate in use: %v", config.LdflagsTemplate) 467 return cpe.ParseTemplate(config.LdflagsTemplate) 468 } 469 470 func runGolangBuildPerArchitecture(config *golangBuildOptions, goModFile *modfile.File, utils golangBuildUtils, ldflags string, architecture multiarch.Platform) ([]string, error) { 471 var binaryNames []string 472 473 envVars := os.Environ() 474 envVars = append(envVars, fmt.Sprintf("GOOS=%v", architecture.OS), fmt.Sprintf("GOARCH=%v", architecture.Arch)) 475 476 if !config.CgoEnabled { 477 envVars = append(envVars, "CGO_ENABLED=0") 478 } 479 utils.SetEnv(envVars) 480 481 buildOptions := []string{"build", "-trimpath"} 482 483 if len(config.Output) > 0 { 484 if len(config.Packages) > 1 { 485 binaries, outputDir, err := getOutputBinaries(config.Output, config.Packages, utils, architecture) 486 if err != nil { 487 log.SetErrorCategory(log.ErrorBuild) 488 return nil, fmt.Errorf("failed to calculate output binaries or directory, error: %s", err.Error()) 489 } 490 buildOptions = append(buildOptions, "-o", outputDir) 491 binaryNames = append(binaryNames, binaries...) 492 } else { 493 fileExtension := "" 494 if architecture.OS == "windows" { 495 fileExtension = ".exe" 496 } 497 binaryName := fmt.Sprintf("%s-%s.%s%s", strings.TrimRight(config.Output, string(os.PathSeparator)), architecture.OS, architecture.Arch, fileExtension) 498 buildOptions = append(buildOptions, "-o", binaryName) 499 binaryNames = append(binaryNames, binaryName) 500 } 501 } else { 502 // use default name in case no name is defined via Output 503 binaryName := path.Base(goModFile.Module.Mod.Path) 504 binaryNames = append(binaryNames, binaryName) 505 } 506 buildOptions = append(buildOptions, config.BuildFlags...) 507 if len(ldflags) > 0 { 508 buildOptions = append(buildOptions, "-ldflags", ldflags) 509 } 510 buildOptions = append(buildOptions, config.Packages...) 511 512 if err := utils.RunExecutable("go", buildOptions...); err != nil { 513 log.Entry().Debugf("buildOptions: %v", buildOptions) 514 log.SetErrorCategory(log.ErrorBuild) 515 return nil, fmt.Errorf("failed to run build for %v.%v: %w", architecture.OS, architecture.Arch, err) 516 } 517 518 return binaryNames, nil 519 } 520 521 func runBOMCreation(utils golangBuildUtils, outputFilename string) error { 522 if err := utils.RunExecutable("cyclonedx-gomod", "mod", "-licenses", fmt.Sprintf("-verbose=%t", GeneralConfig.Verbose), "-test", "-output", outputFilename, "-output-version", "1.4"); err != nil { 523 return fmt.Errorf("BOM creation failed: %w", err) 524 } 525 return nil 526 } 527 528 func readGoModFile(utils golangBuildUtils) (*modfile.File, error) { 529 modFilePath := "go.mod" 530 531 if modFileExists, err := utils.FileExists(modFilePath); err != nil { 532 return nil, err 533 } else if !modFileExists { 534 return nil, nil 535 } 536 537 modFileContent, err := utils.FileRead(modFilePath) 538 if err != nil { 539 return nil, err 540 } 541 542 return modfile.Parse(modFilePath, modFileContent, nil) 543 } 544 545 func getOutputBinaries(out string, packages []string, utils golangBuildUtils, architecture multiarch.Platform) ([]string, string, error) { 546 var binaries []string 547 outDir := fmt.Sprintf("%s-%s-%s%c", strings.TrimRight(out, string(os.PathSeparator)), architecture.OS, architecture.Arch, os.PathSeparator) 548 549 for _, pkg := range packages { 550 ok, err := isMainPackage(utils, pkg) 551 if err != nil { 552 return nil, "", err 553 } 554 555 if ok { 556 fileExt := "" 557 if architecture.OS == "windows" { 558 fileExt = ".exe" 559 } 560 binaries = append(binaries, filepath.Join(outDir, filepath.Base(pkg)+fileExt)) 561 } 562 } 563 564 return binaries, outDir, nil 565 } 566 567 func isMainPackage(utils golangBuildUtils, pkg string) (bool, error) { 568 outBuffer := bytes.NewBufferString("") 569 utils.Stdout(outBuffer) 570 utils.Stderr(outBuffer) 571 err := utils.RunExecutable("go", "list", "-f", "{{ .Name }}", pkg) 572 if err != nil { 573 return false, err 574 } 575 576 if outBuffer.String() != "main" { 577 return false, nil 578 } 579 580 return true, nil 581 } 582 583 func gitConfigurationForPrivateModules(privateMod string, token string, utils golangBuildUtils) error { 584 privateMod = strings.ReplaceAll(privateMod, "/*", "") 585 privateMod = strings.ReplaceAll(privateMod, "*.", "") 586 modules := strings.Split(privateMod, ",") 587 for _, v := range modules { 588 authenticatedRepoURL := fmt.Sprintf("https://%s@%s", token, v) 589 repoBaseURL := fmt.Sprintf("https://%s", v) 590 err := utils.RunExecutable("git", "config", "--global", fmt.Sprintf("url.%s.insteadOf", authenticatedRepoURL), repoBaseURL) 591 if err != nil { 592 return err 593 } 594 595 } 596 597 return nil 598 }