github.com/jaylevin/jenkins-library@v1.230.4/cmd/golangBuild.go (about) 1 package cmd 2 3 import ( 4 "bytes" 5 "fmt" 6 "net/http" 7 "os" 8 "path" 9 "path/filepath" 10 "regexp" 11 "strings" 12 "text/template" 13 14 "github.com/SAP/jenkins-library/pkg/buildsettings" 15 "github.com/SAP/jenkins-library/pkg/certutils" 16 "github.com/SAP/jenkins-library/pkg/command" 17 "github.com/SAP/jenkins-library/pkg/goget" 18 piperhttp "github.com/SAP/jenkins-library/pkg/http" 19 "github.com/SAP/jenkins-library/pkg/log" 20 "github.com/SAP/jenkins-library/pkg/piperenv" 21 "github.com/SAP/jenkins-library/pkg/piperutils" 22 "github.com/SAP/jenkins-library/pkg/telemetry" 23 24 "github.com/SAP/jenkins-library/pkg/multiarch" 25 "github.com/SAP/jenkins-library/pkg/versioning" 26 27 "golang.org/x/mod/modfile" 28 "golang.org/x/mod/module" 29 ) 30 31 const ( 32 coverageFile = "cover.out" 33 golangUnitTestOutput = "TEST-go.xml" 34 golangIntegrationTestOutput = "TEST-integration.xml" 35 golangCoberturaPackage = "github.com/boumenot/gocover-cobertura@latest" 36 golangTestsumPackage = "gotest.tools/gotestsum@latest" 37 golangCycloneDXPackage = "github.com/CycloneDX/cyclonedx-gomod/cmd/cyclonedx-gomod@latest" 38 sbomFilename = "bom.xml" 39 ) 40 41 type golangBuildUtils interface { 42 command.ExecRunner 43 goget.Client 44 45 piperutils.FileUtils 46 piperhttp.Uploader 47 48 DownloadFile(url, filename string, header http.Header, cookies []*http.Cookie) error 49 getDockerImageValue(stepName string) (string, error) 50 51 // Add more methods here, or embed additional interfaces, or remove/replace as required. 52 // The golangBuildUtils interface should be descriptive of your runtime dependencies, 53 // i.e. include everything you need to be able to mock in tests. 54 // Unit tests shall be executable in parallel (not depend on global state), and don't (re-)test dependencies. 55 } 56 57 type golangBuildUtilsBundle struct { 58 *command.Command 59 *piperutils.Files 60 piperhttp.Uploader 61 62 goget.Client 63 64 // Embed more structs as necessary to implement methods or interfaces you add to golangBuildUtils. 65 // Structs embedded in this way must each have a unique set of methods attached. 66 // If there is no struct which implements the method you need, attach the method to 67 // golangBuildUtilsBundle and forward to the implementation of the dependency. 68 } 69 70 func (g *golangBuildUtilsBundle) DownloadFile(url, filename string, header http.Header, cookies []*http.Cookie) error { 71 return fmt.Errorf("not implemented") 72 } 73 74 func (g *golangBuildUtilsBundle) getDockerImageValue(stepName string) (string, error) { 75 return GetDockerImageValue(stepName) 76 } 77 78 func newGolangBuildUtils(config golangBuildOptions) golangBuildUtils { 79 httpClientOptions := piperhttp.ClientOptions{} 80 81 if len(config.CustomTLSCertificateLinks) > 0 { 82 httpClientOptions.TransportSkipVerification = false 83 httpClientOptions.TrustedCerts = config.CustomTLSCertificateLinks 84 } 85 86 httpClient := piperhttp.Client{} 87 httpClient.SetOptions(httpClientOptions) 88 89 utils := golangBuildUtilsBundle{ 90 Command: &command.Command{}, 91 Files: &piperutils.Files{}, 92 Uploader: &httpClient, 93 Client: &goget.ClientImpl{ 94 HTTPClient: &httpClient, 95 }, 96 } 97 // Reroute command output to logging framework 98 utils.Stdout(log.Writer()) 99 utils.Stderr(log.Writer()) 100 return &utils 101 } 102 103 func golangBuild(config golangBuildOptions, telemetryData *telemetry.CustomData, commonPipelineEnvironment *golangBuildCommonPipelineEnvironment) { 104 // Utils can be used wherever the command.ExecRunner interface is expected. 105 // It can also be used for example as a mavenExecRunner. 106 utils := newGolangBuildUtils(config) 107 108 // Error situations will be bubbled up until they reach the line below which will then stop execution 109 // through the log.Entry().Fatal() call leading to an os.Exit(1) in the end. 110 err := runGolangBuild(&config, telemetryData, utils, commonPipelineEnvironment) 111 if err != nil { 112 log.Entry().WithError(err).Fatal("execution of golang build failed") 113 } 114 } 115 116 func runGolangBuild(config *golangBuildOptions, telemetryData *telemetry.CustomData, utils golangBuildUtils, commonPipelineEnvironment *golangBuildCommonPipelineEnvironment) error { 117 goModFile, err := readGoModFile(utils) // returns nil if go.mod doesnt exist 118 if err != nil { 119 return err 120 } 121 122 if err = prepareGolangEnvironment(config, goModFile, utils); err != nil { 123 return err 124 } 125 126 // install test pre-requisites only in case testing should be performed 127 if config.RunTests || config.RunIntegrationTests { 128 if err := utils.RunExecutable("go", "install", golangTestsumPackage); err != nil { 129 return fmt.Errorf("failed to install pre-requisite: %w", err) 130 } 131 } 132 133 if config.CreateBOM { 134 if err := utils.RunExecutable("go", "install", golangCycloneDXPackage); err != nil { 135 return fmt.Errorf("failed to install pre-requisite: %w", err) 136 } 137 } 138 139 failedTests := false 140 141 if config.RunTests { 142 success, err := runGolangTests(config, utils) 143 if err != nil { 144 return err 145 } 146 failedTests = !success 147 } 148 149 if config.RunTests && config.ReportCoverage { 150 if err := reportGolangTestCoverage(config, utils); err != nil { 151 return err 152 } 153 } 154 155 if config.RunIntegrationTests { 156 success, err := runGolangIntegrationTests(config, utils) 157 if err != nil { 158 return err 159 } 160 failedTests = failedTests || !success 161 } 162 163 if failedTests { 164 log.SetErrorCategory(log.ErrorTest) 165 return fmt.Errorf("some tests failed") 166 } 167 168 if config.CreateBOM { 169 if err := runBOMCreation(utils, sbomFilename); err != nil { 170 return err 171 } 172 } 173 174 ldflags := "" 175 176 if len(config.LdflagsTemplate) > 0 { 177 var err error 178 ldflags, err = prepareLdflags(config, utils, GeneralConfig.EnvRootPath) 179 if err != nil { 180 return err 181 } 182 log.Entry().Infof("ldflags from template: '%v'", ldflags) 183 } 184 185 var binaries []string 186 platforms, err := multiarch.ParsePlatformStrings(config.TargetArchitectures) 187 188 if err != nil { 189 return err 190 } 191 192 for _, platform := range platforms { 193 binaryNames, err := runGolangBuildPerArchitecture(config, utils, ldflags, platform) 194 195 if err != nil { 196 return err 197 } 198 199 if len(binaryNames) > 0 { 200 binaries = append(binaries, binaryNames...) 201 } 202 } 203 204 log.Entry().Debugf("creating build settings information...") 205 stepName := "golangBuild" 206 dockerImage, err := utils.getDockerImageValue(stepName) 207 if err != nil { 208 return err 209 } 210 211 buildConfig := buildsettings.BuildOptions{ 212 CreateBOM: config.CreateBOM, 213 Publish: config.Publish, 214 BuildSettingsInfo: config.BuildSettingsInfo, 215 DockerImage: dockerImage, 216 } 217 buildSettingsInfo, err := buildsettings.CreateBuildSettingsInfo(&buildConfig, stepName) 218 if err != nil { 219 log.Entry().Warnf("failed to create build settings info: %v", err) 220 } 221 commonPipelineEnvironment.custom.buildSettingsInfo = buildSettingsInfo 222 223 if config.Publish { 224 if len(config.TargetRepositoryURL) == 0 { 225 return fmt.Errorf("there's no target repository for binary publishing configured") 226 } 227 228 artifactVersion := config.ArtifactVersion 229 230 if len(artifactVersion) == 0 { 231 artifactOpts := versioning.Options{ 232 VersioningScheme: "library", 233 } 234 235 artifact, err := versioning.GetArtifact("golang", "", &artifactOpts, utils) 236 237 if err != nil { 238 return err 239 } 240 241 artifactVersion, err = artifact.GetVersion() 242 243 if err != nil { 244 return err 245 } 246 } 247 248 if goModFile == nil { 249 return fmt.Errorf("go.mod file not found") 250 } else if goModFile.Module == nil { 251 return fmt.Errorf("go.mod doesn't declare a module path") 252 } 253 254 repoClientOptions := piperhttp.ClientOptions{ 255 Username: config.TargetRepositoryUser, 256 Password: config.TargetRepositoryPassword, 257 TrustedCerts: config.CustomTLSCertificateLinks, 258 } 259 260 utils.SetOptions(repoClientOptions) 261 262 var binaryArtifacts piperenv.Artifacts 263 for _, binary := range binaries { 264 265 targetPath := fmt.Sprintf("go/%s/%s/%s", goModFile.Module.Mod.Path, config.ArtifactVersion, binary) 266 267 separator := "/" 268 269 if strings.HasSuffix(config.TargetRepositoryURL, "/") { 270 separator = "" 271 } 272 273 targetURL := fmt.Sprintf("%s%s%s", config.TargetRepositoryURL, separator, targetPath) 274 275 log.Entry().Infof("publishing artifact: %s", targetURL) 276 277 response, err := utils.UploadRequest(http.MethodPut, targetURL, binary, "", nil, nil, "binary") 278 279 if err != nil { 280 return fmt.Errorf("couldn't upload artifact: %w", err) 281 } 282 283 if !(response.StatusCode == 200 || response.StatusCode == 201) { 284 return fmt.Errorf("couldn't upload artifact, received status code %d", response.StatusCode) 285 } 286 287 binaryArtifacts = append(binaryArtifacts, piperenv.Artifact{ 288 Name: binary, 289 }) 290 } 291 commonPipelineEnvironment.custom.artifacts = binaryArtifacts 292 293 } 294 295 return nil 296 } 297 298 func prepareGolangEnvironment(config *golangBuildOptions, goModFile *modfile.File, utils golangBuildUtils) error { 299 // configure truststore 300 err := certutils.CertificateUpdate(config.CustomTLSCertificateLinks, utils, utils, "/etc/ssl/certs/ca-certificates.crt") // TODO reimplement 301 302 if config.PrivateModules == "" { 303 return nil 304 } 305 306 if config.PrivateModulesGitToken == "" { 307 return fmt.Errorf("please specify a token for fetching private git modules") 308 } 309 310 // pass private repos to go process 311 os.Setenv("GOPRIVATE", config.PrivateModules) 312 313 repoURLs, err := lookupGolangPrivateModulesRepositories(goModFile, config.PrivateModules, utils) 314 315 if err != nil { 316 return err 317 } 318 319 // configure credentials git shall use for pulling repos 320 for _, repoURL := range repoURLs { 321 if match, _ := regexp.MatchString("(?i)^https?://", repoURL); !match { 322 continue 323 } 324 325 authenticatedRepoURL := strings.Replace(repoURL, "://", fmt.Sprintf("://%s@", config.PrivateModulesGitToken), 1) 326 327 err = utils.RunExecutable("git", "config", "--global", fmt.Sprintf("url.%s.insteadOf", authenticatedRepoURL), fmt.Sprintf("%s", repoURL)) 328 if err != nil { 329 return err 330 } 331 } 332 333 return nil 334 } 335 336 func runGolangTests(config *golangBuildOptions, utils golangBuildUtils) (bool, error) { 337 // execute gotestsum in order to have more output options 338 if err := utils.RunExecutable("gotestsum", "--junitfile", golangUnitTestOutput, "--", fmt.Sprintf("-coverprofile=%v", coverageFile), "./..."); err != nil { 339 exists, fileErr := utils.FileExists(golangUnitTestOutput) 340 if !exists || fileErr != nil { 341 log.SetErrorCategory(log.ErrorBuild) 342 return false, fmt.Errorf("running tests failed - junit result missing: %w", err) 343 } 344 exists, fileErr = utils.FileExists(coverageFile) 345 if !exists || fileErr != nil { 346 log.SetErrorCategory(log.ErrorBuild) 347 return false, fmt.Errorf("running tests failed - coverage output missing: %w", err) 348 } 349 return false, nil 350 } 351 return true, nil 352 } 353 354 func runGolangIntegrationTests(config *golangBuildOptions, utils golangBuildUtils) (bool, error) { 355 // execute gotestsum in order to have more output options 356 // for integration tests coverage data is not meaningful and thus not being created 357 if err := utils.RunExecutable("gotestsum", "--junitfile", golangIntegrationTestOutput, "--", "-tags=integration", "./..."); err != nil { 358 exists, fileErr := utils.FileExists(golangIntegrationTestOutput) 359 if !exists || fileErr != nil { 360 log.SetErrorCategory(log.ErrorBuild) 361 return false, fmt.Errorf("running tests failed: %w", err) 362 } 363 return false, nil 364 } 365 return true, nil 366 } 367 368 func reportGolangTestCoverage(config *golangBuildOptions, utils golangBuildUtils) error { 369 if config.CoverageFormat == "cobertura" { 370 // execute gocover-cobertura in order to create cobertura report 371 // install pre-requisites 372 if err := utils.RunExecutable("go", "install", golangCoberturaPackage); err != nil { 373 return fmt.Errorf("failed to install pre-requisite: %w", err) 374 } 375 376 coverageData, err := utils.FileRead(coverageFile) 377 if err != nil { 378 return fmt.Errorf("failed to read coverage file %v: %w", coverageFile, err) 379 } 380 utils.Stdin(bytes.NewBuffer(coverageData)) 381 382 coverageOutput := bytes.Buffer{} 383 utils.Stdout(&coverageOutput) 384 options := []string{} 385 if config.ExcludeGeneratedFromCoverage { 386 options = append(options, "-ignore-gen-files") 387 } 388 if err := utils.RunExecutable("gocover-cobertura", options...); err != nil { 389 log.SetErrorCategory(log.ErrorTest) 390 return fmt.Errorf("failed to convert coverage data to cobertura format: %w", err) 391 } 392 utils.Stdout(log.Writer()) 393 394 err = utils.FileWrite("cobertura-coverage.xml", coverageOutput.Bytes(), 0666) 395 if err != nil { 396 return fmt.Errorf("failed to create cobertura coverage file: %w", err) 397 } 398 log.Entry().Info("created file cobertura-coverage.xml") 399 } else { 400 // currently only cobertura and html format supported, thus using html as fallback 401 if err := utils.RunExecutable("go", "tool", "cover", "-html", coverageFile, "-o", "coverage.html"); err != nil { 402 return fmt.Errorf("failed to create html coverage file: %w", err) 403 } 404 } 405 return nil 406 } 407 408 func prepareLdflags(config *golangBuildOptions, utils golangBuildUtils, envRootPath string) (string, error) { 409 cpe := piperenv.CPEMap{} 410 err := cpe.LoadFromDisk(path.Join(envRootPath, "commonPipelineEnvironment")) 411 if err != nil { 412 log.Entry().Warning("failed to load values from commonPipelineEnvironment") 413 } 414 415 log.Entry().Debugf("ldflagsTemplate in use: %v", config.LdflagsTemplate) 416 tmpl, err := template.New("ldflags").Parse(config.LdflagsTemplate) 417 if err != nil { 418 return "", fmt.Errorf("failed to parse ldflagsTemplate '%v': %w", config.LdflagsTemplate, err) 419 } 420 421 ldflagsParams := struct { 422 CPE map[string]interface{} 423 }{ 424 CPE: map[string]interface{}(cpe), 425 } 426 var generatedLdflags bytes.Buffer 427 err = tmpl.Execute(&generatedLdflags, ldflagsParams) 428 if err != nil { 429 return "", fmt.Errorf("failed to execute ldflagsTemplate '%v': %w", config.LdflagsTemplate, err) 430 } 431 432 return generatedLdflags.String(), nil 433 } 434 435 func runGolangBuildPerArchitecture(config *golangBuildOptions, utils golangBuildUtils, ldflags string, architecture multiarch.Platform) ([]string, error) { 436 var binaryNames []string 437 438 envVars := os.Environ() 439 envVars = append(envVars, fmt.Sprintf("GOOS=%v", architecture.OS), fmt.Sprintf("GOARCH=%v", architecture.Arch)) 440 441 if !config.CgoEnabled { 442 envVars = append(envVars, "CGO_ENABLED=0") 443 } 444 utils.SetEnv(envVars) 445 446 buildOptions := []string{"build", "-trimpath"} 447 448 if len(config.Output) > 0 { 449 if len(config.Packages) > 1 { 450 binaries, outputDir, err := getOutputBinaries(config.Output, config.Packages, utils, architecture) 451 if err != nil { 452 log.SetErrorCategory(log.ErrorBuild) 453 return nil, fmt.Errorf("failed to calculate output binaries or directory, error: %s", err.Error()) 454 } 455 buildOptions = append(buildOptions, "-o", outputDir) 456 binaryNames = append(binaryNames, binaries...) 457 } else { 458 fileExtension := "" 459 if architecture.OS == "windows" { 460 fileExtension = ".exe" 461 } 462 binaryName := fmt.Sprintf("%s-%s.%s%s", strings.TrimRight(config.Output, string(os.PathSeparator)), architecture.OS, architecture.Arch, fileExtension) 463 buildOptions = append(buildOptions, "-o", binaryName) 464 binaryNames = append(binaryNames, binaryName) 465 } 466 } 467 buildOptions = append(buildOptions, config.BuildFlags...) 468 if len(ldflags) > 0 { 469 buildOptions = append(buildOptions, "-ldflags", ldflags) 470 } 471 buildOptions = append(buildOptions, config.Packages...) 472 473 if err := utils.RunExecutable("go", buildOptions...); err != nil { 474 log.Entry().Debugf("buildOptions: %v", buildOptions) 475 log.SetErrorCategory(log.ErrorBuild) 476 return nil, fmt.Errorf("failed to run build for %v.%v: %w", architecture.OS, architecture.Arch, err) 477 } 478 479 return binaryNames, nil 480 } 481 482 // lookupPrivateModulesRepositories returns a slice of all modules that match the given glob pattern 483 func lookupGolangPrivateModulesRepositories(goModFile *modfile.File, globPattern string, utils golangBuildUtils) ([]string, error) { 484 if globPattern == "" { 485 return []string{}, nil 486 } 487 488 if goModFile == nil { 489 return nil, fmt.Errorf("couldn't find go.mod file") 490 } else if goModFile.Require == nil { 491 return []string{}, nil // no modules referenced, nothing to do 492 } 493 494 privateModules := []string{} 495 496 for _, goModule := range goModFile.Require { 497 if !module.MatchPrefixPatterns(globPattern, goModule.Mod.Path) { 498 continue 499 } 500 501 repo, err := utils.GetRepositoryURL(goModule.Mod.Path) 502 503 if err != nil { 504 return nil, err 505 } 506 507 privateModules = append(privateModules, repo) 508 } 509 return privateModules, nil 510 } 511 512 func runBOMCreation(utils golangBuildUtils, outputFilename string) error { 513 if err := utils.RunExecutable("cyclonedx-gomod", "mod", "-licenses", "-test", "-output", outputFilename); err != nil { 514 return fmt.Errorf("BOM creation failed: %w", err) 515 } 516 return nil 517 } 518 519 func readGoModFile(utils golangBuildUtils) (*modfile.File, error) { 520 modFilePath := "go.mod" 521 522 if modFileExists, err := utils.FileExists(modFilePath); err != nil { 523 return nil, err 524 } else if !modFileExists { 525 return nil, nil 526 } 527 528 modFileContent, err := utils.FileRead(modFilePath) 529 if err != nil { 530 return nil, err 531 } 532 533 return modfile.Parse(modFilePath, modFileContent, nil) 534 } 535 536 func getOutputBinaries(out string, packages []string, utils golangBuildUtils, architecture multiarch.Platform) ([]string, string, error) { 537 var binaries []string 538 outDir := fmt.Sprintf("%s-%s-%s%c", strings.TrimRight(out, string(os.PathSeparator)), architecture.OS, architecture.Arch, os.PathSeparator) 539 540 for _, pkg := range packages { 541 ok, err := isMainPackage(utils, pkg) 542 if err != nil { 543 return nil, "", err 544 } 545 546 if ok { 547 fileExt := "" 548 if architecture.OS == "windows" { 549 fileExt = ".exe" 550 } 551 binaries = append(binaries, filepath.Join(outDir, filepath.Base(pkg)+fileExt)) 552 } 553 } 554 555 return binaries, outDir, nil 556 } 557 558 func isMainPackage(utils golangBuildUtils, pkg string) (bool, error) { 559 outBuffer := bytes.NewBufferString("") 560 utils.Stdout(outBuffer) 561 utils.Stderr(outBuffer) 562 err := utils.RunExecutable("go", "list", "-f", "{{ .Name }}", pkg) 563 if err != nil { 564 return false, err 565 } 566 567 if outBuffer.String() != "main" { 568 return false, nil 569 } 570 571 return true, nil 572 }