github.com/xgoffin/jenkins-library@v1.154.0/cmd/golangBuild.go (about) 1 package cmd 2 3 import ( 4 "bytes" 5 "fmt" 6 "net/http" 7 "os" 8 "path" 9 "regexp" 10 "strings" 11 "text/template" 12 13 "github.com/SAP/jenkins-library/pkg/buildsettings" 14 "github.com/SAP/jenkins-library/pkg/certutils" 15 "github.com/SAP/jenkins-library/pkg/command" 16 "github.com/SAP/jenkins-library/pkg/goget" 17 piperhttp "github.com/SAP/jenkins-library/pkg/http" 18 "github.com/SAP/jenkins-library/pkg/log" 19 "github.com/SAP/jenkins-library/pkg/piperenv" 20 "github.com/SAP/jenkins-library/pkg/piperutils" 21 22 "github.com/SAP/jenkins-library/pkg/telemetry" 23 24 "github.com/SAP/jenkins-library/pkg/multiarch" 25 "github.com/SAP/jenkins-library/pkg/versioning" 26 27 "golang.org/x/mod/modfile" 28 "golang.org/x/mod/module" 29 ) 30 31 const ( 32 coverageFile = "cover.out" 33 golangUnitTestOutput = "TEST-go.xml" 34 golangIntegrationTestOutput = "TEST-integration.xml" 35 golangCoberturaPackage = "github.com/boumenot/gocover-cobertura@latest" 36 golangTestsumPackage = "gotest.tools/gotestsum@latest" 37 golangCycloneDXPackage = "github.com/CycloneDX/cyclonedx-gomod/cmd/cyclonedx-gomod@latest" 38 sbomFilename = "bom.xml" 39 ) 40 41 type golangBuildUtils interface { 42 command.ExecRunner 43 goget.Client 44 45 piperutils.FileUtils 46 piperhttp.Uploader 47 48 DownloadFile(url, filename string, header http.Header, cookies []*http.Cookie) error 49 getDockerImageValue(stepName string) (string, error) 50 51 // Add more methods here, or embed additional interfaces, or remove/replace as required. 52 // The golangBuildUtils interface should be descriptive of your runtime dependencies, 53 // i.e. include everything you need to be able to mock in tests. 54 // Unit tests shall be executable in parallel (not depend on global state), and don't (re-)test dependencies. 55 } 56 57 type golangBuildUtilsBundle struct { 58 *command.Command 59 *piperutils.Files 60 piperhttp.Uploader 61 62 goget.Client 63 64 // Embed more structs as necessary to implement methods or interfaces you add to golangBuildUtils. 65 // Structs embedded in this way must each have a unique set of methods attached. 66 // If there is no struct which implements the method you need, attach the method to 67 // golangBuildUtilsBundle and forward to the implementation of the dependency. 68 } 69 70 func (g *golangBuildUtilsBundle) DownloadFile(url, filename string, header http.Header, cookies []*http.Cookie) error { 71 return fmt.Errorf("not implemented") 72 } 73 74 func (g *golangBuildUtilsBundle) getDockerImageValue(stepName string) (string, error) { 75 return GetDockerImageValue(stepName) 76 } 77 78 func newGolangBuildUtils(config golangBuildOptions) golangBuildUtils { 79 httpClientOptions := piperhttp.ClientOptions{} 80 81 if len(config.CustomTLSCertificateLinks) > 0 { 82 httpClientOptions.TransportSkipVerification = false 83 httpClientOptions.TrustedCerts = config.CustomTLSCertificateLinks 84 } 85 86 httpClient := piperhttp.Client{} 87 httpClient.SetOptions(httpClientOptions) 88 89 utils := golangBuildUtilsBundle{ 90 Command: &command.Command{}, 91 Files: &piperutils.Files{}, 92 Uploader: &httpClient, 93 Client: &goget.ClientImpl{ 94 HTTPClient: &httpClient, 95 }, 96 } 97 // Reroute command output to logging framework 98 utils.Stdout(log.Writer()) 99 utils.Stderr(log.Writer()) 100 return &utils 101 } 102 103 func golangBuild(config golangBuildOptions, telemetryData *telemetry.CustomData, commonPipelineEnvironment *golangBuildCommonPipelineEnvironment) { 104 // Utils can be used wherever the command.ExecRunner interface is expected. 105 // It can also be used for example as a mavenExecRunner. 106 utils := newGolangBuildUtils(config) 107 108 // Error situations will be bubbled up until they reach the line below which will then stop execution 109 // through the log.Entry().Fatal() call leading to an os.Exit(1) in the end. 110 err := runGolangBuild(&config, telemetryData, utils, commonPipelineEnvironment) 111 if err != nil { 112 log.Entry().WithError(err).Fatal("execution of golang build failed") 113 } 114 } 115 116 func runGolangBuild(config *golangBuildOptions, telemetryData *telemetry.CustomData, utils golangBuildUtils, commonPipelineEnvironment *golangBuildCommonPipelineEnvironment) error { 117 goModFile, err := readGoModFile(utils) // returns nil if go.mod doesnt exist 118 if err != nil { 119 return err 120 } 121 122 if err = prepareGolangEnvironment(config, goModFile, utils); err != nil { 123 return err 124 } 125 126 // install test pre-requisites only in case testing should be performed 127 if config.RunTests || config.RunIntegrationTests { 128 if err := utils.RunExecutable("go", "install", golangTestsumPackage); err != nil { 129 return fmt.Errorf("failed to install pre-requisite: %w", err) 130 } 131 } 132 133 if config.CreateBOM { 134 if err := utils.RunExecutable("go", "install", golangCycloneDXPackage); err != nil { 135 return fmt.Errorf("failed to install pre-requisite: %w", err) 136 } 137 } 138 139 failedTests := false 140 141 if config.RunTests { 142 success, err := runGolangTests(config, utils) 143 if err != nil { 144 return err 145 } 146 failedTests = !success 147 } 148 149 if config.RunTests && config.ReportCoverage { 150 if err := reportGolangTestCoverage(config, utils); err != nil { 151 return err 152 } 153 } 154 155 if config.RunIntegrationTests { 156 success, err := runGolangIntegrationTests(config, utils) 157 if err != nil { 158 return err 159 } 160 failedTests = failedTests || !success 161 } 162 163 if failedTests { 164 log.SetErrorCategory(log.ErrorTest) 165 return fmt.Errorf("some tests failed") 166 } 167 168 if config.CreateBOM { 169 if err := runBOMCreation(utils, sbomFilename); err != nil { 170 return err 171 } 172 } 173 174 ldflags := "" 175 176 if len(config.LdflagsTemplate) > 0 { 177 var err error 178 ldflags, err = prepareLdflags(config, utils, GeneralConfig.EnvRootPath) 179 if err != nil { 180 return err 181 } 182 log.Entry().Infof("ldflags from template: '%v'", ldflags) 183 } 184 185 binaries := []string{} 186 187 platforms, err := multiarch.ParsePlatformStrings(config.TargetArchitectures) 188 189 if err != nil { 190 return err 191 } 192 193 for _, platform := range platforms { 194 binary, err := runGolangBuildPerArchitecture(config, utils, ldflags, platform) 195 196 if err != nil { 197 return err 198 } 199 200 if len(binary) > 0 { 201 binaries = append(binaries, binary) 202 } 203 } 204 205 log.Entry().Debugf("creating build settings information...") 206 stepName := "golangBuild" 207 dockerImage, err := utils.getDockerImageValue(stepName) 208 if err != nil { 209 return err 210 } 211 212 buildConfig := buildsettings.BuildOptions{ 213 CreateBOM: config.CreateBOM, 214 Publish: config.Publish, 215 BuildSettingsInfo: config.BuildSettingsInfo, 216 DockerImage: dockerImage, 217 } 218 buildSettingsInfo, err := buildsettings.CreateBuildSettingsInfo(&buildConfig, stepName) 219 if err != nil { 220 log.Entry().Warnf("failed to create build settings info: %v", err) 221 } 222 commonPipelineEnvironment.custom.buildSettingsInfo = buildSettingsInfo 223 224 if config.Publish { 225 if len(config.TargetRepositoryURL) == 0 { 226 return fmt.Errorf("there's no target repository for binary publishing configured") 227 } 228 229 artifactVersion := config.ArtifactVersion 230 231 if len(artifactVersion) == 0 { 232 artifactOpts := versioning.Options{ 233 VersioningScheme: "library", 234 } 235 236 artifact, err := versioning.GetArtifact("golang", "", &artifactOpts, utils) 237 238 if err != nil { 239 return err 240 } 241 242 artifactVersion, err = artifact.GetVersion() 243 244 if err != nil { 245 return err 246 } 247 } 248 249 if goModFile == nil { 250 return fmt.Errorf("go.mod file not found") 251 } else if goModFile.Module == nil { 252 return fmt.Errorf("go.mod doesn't declare a module path") 253 } 254 255 repoClientOptions := piperhttp.ClientOptions{ 256 Username: config.TargetRepositoryUser, 257 Password: config.TargetRepositoryPassword, 258 TrustedCerts: config.CustomTLSCertificateLinks, 259 } 260 261 utils.SetOptions(repoClientOptions) 262 263 for _, binary := range binaries { 264 targetPath := fmt.Sprintf("go/%s/%s/%s", goModFile.Module.Mod.Path, config.ArtifactVersion, binary) 265 266 separator := "/" 267 268 if strings.HasSuffix(config.TargetRepositoryURL, "/") { 269 separator = "" 270 } 271 272 targetURL := fmt.Sprintf("%s%s%s", config.TargetRepositoryURL, separator, targetPath) 273 274 log.Entry().Infof("publishing artifact: %s", targetURL) 275 276 response, err := utils.UploadRequest(http.MethodPut, targetURL, binary, "", nil, nil, "binary") 277 278 if err != nil { 279 return fmt.Errorf("couldn't upload artifact: %w", err) 280 } 281 282 if !(response.StatusCode == 200 || response.StatusCode == 201) { 283 return fmt.Errorf("couldn't upload artifact, received status code %d", response.StatusCode) 284 } 285 } 286 } 287 288 return nil 289 } 290 291 func prepareGolangEnvironment(config *golangBuildOptions, goModFile *modfile.File, utils golangBuildUtils) error { 292 // configure truststore 293 err := certutils.CertificateUpdate(config.CustomTLSCertificateLinks, utils, utils, "/etc/ssl/certs/ca-certificates.crt") // TODO reimplement 294 295 if config.PrivateModules == "" { 296 return nil 297 } 298 299 if config.PrivateModulesGitToken == "" { 300 return fmt.Errorf("please specify a token for fetching private git modules") 301 } 302 303 // pass private repos to go process 304 os.Setenv("GOPRIVATE", config.PrivateModules) 305 306 repoURLs, err := lookupGolangPrivateModulesRepositories(goModFile, config.PrivateModules, utils) 307 308 if err != nil { 309 return err 310 } 311 312 // configure credentials git shall use for pulling repos 313 for _, repoURL := range repoURLs { 314 if match, _ := regexp.MatchString("(?i)^https?://", repoURL); !match { 315 continue 316 } 317 318 authenticatedRepoURL := strings.Replace(repoURL, "://", fmt.Sprintf("://%s@", config.PrivateModulesGitToken), 1) 319 320 err = utils.RunExecutable("git", "config", "--global", fmt.Sprintf("url.%s.insteadOf", authenticatedRepoURL), fmt.Sprintf("%s", repoURL)) 321 if err != nil { 322 return err 323 } 324 } 325 326 return nil 327 } 328 329 func runGolangTests(config *golangBuildOptions, utils golangBuildUtils) (bool, error) { 330 // execute gotestsum in order to have more output options 331 if err := utils.RunExecutable("gotestsum", "--junitfile", golangUnitTestOutput, "--", fmt.Sprintf("-coverprofile=%v", coverageFile), "./..."); err != nil { 332 exists, fileErr := utils.FileExists(golangUnitTestOutput) 333 if !exists || fileErr != nil { 334 log.SetErrorCategory(log.ErrorBuild) 335 return false, fmt.Errorf("running tests failed - junit result missing: %w", err) 336 } 337 exists, fileErr = utils.FileExists(coverageFile) 338 if !exists || fileErr != nil { 339 log.SetErrorCategory(log.ErrorBuild) 340 return false, fmt.Errorf("running tests failed - coverage output missing: %w", err) 341 } 342 return false, nil 343 } 344 return true, nil 345 } 346 347 func runGolangIntegrationTests(config *golangBuildOptions, utils golangBuildUtils) (bool, error) { 348 // execute gotestsum in order to have more output options 349 // for integration tests coverage data is not meaningful and thus not being created 350 if err := utils.RunExecutable("gotestsum", "--junitfile", golangIntegrationTestOutput, "--", "-tags=integration", "./..."); err != nil { 351 exists, fileErr := utils.FileExists(golangIntegrationTestOutput) 352 if !exists || fileErr != nil { 353 log.SetErrorCategory(log.ErrorBuild) 354 return false, fmt.Errorf("running tests failed: %w", err) 355 } 356 return false, nil 357 } 358 return true, nil 359 } 360 361 func reportGolangTestCoverage(config *golangBuildOptions, utils golangBuildUtils) error { 362 if config.CoverageFormat == "cobertura" { 363 // execute gocover-cobertura in order to create cobertura report 364 // install pre-requisites 365 if err := utils.RunExecutable("go", "install", golangCoberturaPackage); err != nil { 366 return fmt.Errorf("failed to install pre-requisite: %w", err) 367 } 368 369 coverageData, err := utils.FileRead(coverageFile) 370 if err != nil { 371 return fmt.Errorf("failed to read coverage file %v: %w", coverageFile, err) 372 } 373 utils.Stdin(bytes.NewBuffer(coverageData)) 374 375 coverageOutput := bytes.Buffer{} 376 utils.Stdout(&coverageOutput) 377 options := []string{} 378 if config.ExcludeGeneratedFromCoverage { 379 options = append(options, "-ignore-gen-files") 380 } 381 if err := utils.RunExecutable("gocover-cobertura", options...); err != nil { 382 log.SetErrorCategory(log.ErrorTest) 383 return fmt.Errorf("failed to convert coverage data to cobertura format: %w", err) 384 } 385 utils.Stdout(log.Writer()) 386 387 err = utils.FileWrite("cobertura-coverage.xml", coverageOutput.Bytes(), 0666) 388 if err != nil { 389 return fmt.Errorf("failed to create cobertura coverage file: %w", err) 390 } 391 log.Entry().Info("created file cobertura-coverage.xml") 392 } else { 393 // currently only cobertura and html format supported, thus using html as fallback 394 if err := utils.RunExecutable("go", "tool", "cover", "-html", coverageFile, "-o", "coverage.html"); err != nil { 395 return fmt.Errorf("failed to create html coverage file: %w", err) 396 } 397 } 398 return nil 399 } 400 401 func prepareLdflags(config *golangBuildOptions, utils golangBuildUtils, envRootPath string) (string, error) { 402 cpe := piperenv.CPEMap{} 403 err := cpe.LoadFromDisk(path.Join(envRootPath, "commonPipelineEnvironment")) 404 if err != nil { 405 log.Entry().Warning("failed to load values from commonPipelineEnvironment") 406 } 407 408 log.Entry().Debugf("ldflagsTemplate in use: %v", config.LdflagsTemplate) 409 tmpl, err := template.New("ldflags").Parse(config.LdflagsTemplate) 410 if err != nil { 411 return "", fmt.Errorf("failed to parse ldflagsTemplate '%v': %w", config.LdflagsTemplate, err) 412 } 413 414 ldflagsParams := struct { 415 CPE map[string]interface{} 416 }{ 417 CPE: map[string]interface{}(cpe), 418 } 419 var generatedLdflags bytes.Buffer 420 err = tmpl.Execute(&generatedLdflags, ldflagsParams) 421 if err != nil { 422 return "", fmt.Errorf("failed to execute ldflagsTemplate '%v': %w", config.LdflagsTemplate, err) 423 } 424 425 return generatedLdflags.String(), nil 426 } 427 428 func runGolangBuildPerArchitecture(config *golangBuildOptions, utils golangBuildUtils, ldflags string, architecture multiarch.Platform) (string, error) { 429 var binaryName string 430 431 envVars := os.Environ() 432 envVars = append(envVars, fmt.Sprintf("GOOS=%v", architecture.OS), fmt.Sprintf("GOARCH=%v", architecture.Arch)) 433 434 if !config.CgoEnabled { 435 envVars = append(envVars, "CGO_ENABLED=0") 436 } 437 utils.SetEnv(envVars) 438 439 buildOptions := []string{"build", "-trimpath"} 440 if len(config.Output) > 0 { 441 fileExtension := "" 442 if architecture.OS == "windows" { 443 fileExtension = ".exe" 444 } 445 binaryName = fmt.Sprintf("%v-%v.%v%v", config.Output, architecture.OS, architecture.Arch, fileExtension) 446 buildOptions = append(buildOptions, "-o", binaryName) 447 } 448 buildOptions = append(buildOptions, config.BuildFlags...) 449 if len(ldflags) > 0 { 450 buildOptions = append(buildOptions, "-ldflags", ldflags) 451 } 452 buildOptions = append(buildOptions, config.Packages...) 453 454 if err := utils.RunExecutable("go", buildOptions...); err != nil { 455 log.Entry().Debugf("buildOptions: %v", buildOptions) 456 log.SetErrorCategory(log.ErrorBuild) 457 return "", fmt.Errorf("failed to run build for %v.%v: %w", architecture.OS, architecture.Arch, err) 458 } 459 return binaryName, nil 460 } 461 462 // lookupPrivateModulesRepositories returns a slice of all modules that match the given glob pattern 463 func lookupGolangPrivateModulesRepositories(goModFile *modfile.File, globPattern string, utils golangBuildUtils) ([]string, error) { 464 if globPattern == "" { 465 return []string{}, nil 466 } 467 468 if goModFile == nil { 469 return nil, fmt.Errorf("couldn't find go.mod file") 470 } else if goModFile.Require == nil { 471 return []string{}, nil // no modules referenced, nothing to do 472 } 473 474 privateModules := []string{} 475 476 for _, goModule := range goModFile.Require { 477 if !module.MatchPrefixPatterns(globPattern, goModule.Mod.Path) { 478 continue 479 } 480 481 repo, err := utils.GetRepositoryURL(goModule.Mod.Path) 482 483 if err != nil { 484 return nil, err 485 } 486 487 privateModules = append(privateModules, repo) 488 } 489 return privateModules, nil 490 } 491 492 func runBOMCreation(utils golangBuildUtils, outputFilename string) error { 493 if err := utils.RunExecutable("cyclonedx-gomod", "mod", "-licenses", "-test", "-output", outputFilename); err != nil { 494 return fmt.Errorf("BOM creation failed: %w", err) 495 } 496 return nil 497 } 498 499 func readGoModFile(utils golangBuildUtils) (*modfile.File, error) { 500 modFilePath := "go.mod" 501 502 if modFileExists, err := utils.FileExists(modFilePath); err != nil { 503 return nil, err 504 } else if !modFileExists { 505 return nil, nil 506 } 507 508 modFileContent, err := utils.FileRead(modFilePath) 509 if err != nil { 510 return nil, err 511 } 512 513 return modfile.Parse(modFilePath, modFileContent, nil) 514 }