github.com/SAP/jenkins-library@v1.362.0/cmd/sonarExecuteScan.go (about) 1 package cmd 2 3 import ( 4 "fmt" 5 "net" 6 "net/url" 7 "os" 8 "os/exec" 9 "path" 10 "path/filepath" 11 "strconv" 12 "strings" 13 "time" 14 15 "github.com/bmatcuk/doublestar" 16 "github.com/pkg/errors" 17 18 "github.com/SAP/jenkins-library/pkg/command" 19 piperhttp "github.com/SAP/jenkins-library/pkg/http" 20 keytool "github.com/SAP/jenkins-library/pkg/java" 21 "github.com/SAP/jenkins-library/pkg/log" 22 "github.com/SAP/jenkins-library/pkg/orchestrator" 23 "github.com/SAP/jenkins-library/pkg/piperutils" 24 SonarUtils "github.com/SAP/jenkins-library/pkg/sonar" 25 "github.com/SAP/jenkins-library/pkg/telemetry" 26 "github.com/SAP/jenkins-library/pkg/versioning" 27 ) 28 29 type sonarSettings struct { 30 workingDir string 31 binary string 32 environment []string 33 options []string 34 } 35 36 func (s *sonarSettings) addEnvironment(element string) { 37 s.environment = append(s.environment, element) 38 } 39 40 func (s *sonarSettings) addOption(element string) { 41 s.options = append(s.options, element) 42 } 43 44 var ( 45 sonar sonarSettings 46 47 execLookPath = exec.LookPath 48 fileUtilsExists = piperutils.FileExists 49 fileUtilsUnzip = piperutils.Unzip 50 osRename = os.Rename 51 osStat = os.Stat 52 doublestarGlob = doublestar.Glob 53 ) 54 55 const ( 56 javaBinaries = "sonar.java.binaries=" 57 javaLibraries = "sonar.java.libraries=" 58 coverageExclusions = "sonar.coverage.exclusions=" 59 pomXMLPattern = "**/pom.xml" 60 ) 61 62 func sonarExecuteScan(config sonarExecuteScanOptions, _ *telemetry.CustomData, influx *sonarExecuteScanInflux) { 63 runner := command.Command{ 64 ErrorCategoryMapping: map[string][]string{ 65 log.ErrorConfiguration.String(): { 66 "You must define the following mandatory properties for '*': *", 67 "org.sonar.java.AnalysisException: Your project contains .java files, please provide compiled classes with sonar.java.binaries property, or exclude them from the analysis with sonar.exclusions property.", 68 "ERROR: Invalid value for *", 69 "java.lang.IllegalStateException: No files nor directories matching '*'", 70 }, 71 log.ErrorInfrastructure.String(): { 72 "ERROR: SonarQube server [*] can not be reached", 73 "Caused by: java.net.SocketTimeoutException: timeout", 74 "java.lang.IllegalStateException: Fail to request *", 75 "java.lang.IllegalStateException: Fail to download plugin [*] into *", 76 }, 77 }, 78 } 79 // reroute command output to logging framework 80 runner.Stdout(log.Writer()) 81 runner.Stderr(log.Writer()) 82 // client for downloading the sonar-scanner 83 downloadClient := &piperhttp.Client{} 84 downloadClient.SetOptions(piperhttp.ClientOptions{TransportTimeout: 20 * time.Second}) 85 // client for talking to the SonarQube API 86 apiClient := &piperhttp.Client{} 87 proxy := config.Proxy 88 if proxy != "" { 89 transportProxy, err := url.Parse(proxy) 90 if err != nil { 91 log.Entry().WithError(err).Fatalf("Failed to parse proxy string %v into a URL structure", proxy) 92 } 93 host, port, err := net.SplitHostPort(transportProxy.Host) 94 if err != nil { 95 log.Entry().WithError(err).Fatalf("Failed to retrieve host and port from the proxy URL") 96 } 97 // provide proxy setting for Java based Sonar scanner 98 javaToolOptions := fmt.Sprintf("-Dhttp.proxyHost=%v -Dhttp.proxyPort=%v", host, port) 99 os.Setenv("JAVA_TOOL_OPTIONS", javaToolOptions) 100 101 apiClient.SetOptions(piperhttp.ClientOptions{TransportProxy: transportProxy, TransportSkipVerification: true}) 102 log.Entry().Infof("HTTP client instructed to use %v proxy", proxy) 103 104 } else { 105 //TODO: implement certificate handling 106 apiClient.SetOptions(piperhttp.ClientOptions{TransportSkipVerification: true}) 107 } 108 109 sonar = sonarSettings{ 110 workingDir: "./", 111 binary: "sonar-scanner", 112 environment: []string{}, 113 options: []string{}, 114 } 115 116 influx.step_data.fields.sonar = false 117 fileUtils := piperutils.Files{} 118 if err := runSonar(config, downloadClient, &runner, apiClient, &fileUtils, influx); err != nil { 119 if log.GetErrorCategory() == log.ErrorUndefined && runner.GetExitCode() == 2 { 120 // see https://github.com/SonarSource/sonar-scanner-cli/blob/adb67d645c3bcb9b46f29dea06ba082ebec9ba7a/src/main/java/org/sonarsource/scanner/cli/Exit.java#L25 121 log.SetErrorCategory(log.ErrorConfiguration) 122 } 123 log.Entry().WithError(err).Fatal("Execution failed") 124 } 125 influx.step_data.fields.sonar = true 126 } 127 128 func runSonar(config sonarExecuteScanOptions, client piperhttp.Downloader, runner command.ExecRunner, apiClient SonarUtils.Sender, utils piperutils.FileUtils, influx *sonarExecuteScanInflux) error { 129 // Set config based on orchestrator-specific environment variables 130 detectParametersFromCI(&config) 131 132 if len(config.ServerURL) > 0 { 133 sonar.addEnvironment("SONAR_HOST_URL=" + config.ServerURL) 134 } 135 if len(config.Token) == 0 { 136 log.Entry().Warn("sonar token not set") 137 // use token provided by sonar-scanner-jenkins plugin 138 // https://github.com/SonarSource/sonar-scanner-jenkins/blob/441ef2f485884758b60767bed2ef8a1a0a7fc863/src/main/java/hudson/plugins/sonar/SonarBuildWrapper.java#L132 139 if len(os.Getenv("SONAR_AUTH_TOKEN")) > 0 { 140 log.Entry().Info("using token from env var SONAR_AUTH_TOKEN") 141 config.Token = os.Getenv("SONAR_AUTH_TOKEN") 142 } 143 } 144 if len(config.Token) > 0 { 145 sonar.addEnvironment("SONAR_TOKEN=" + config.Token) 146 } 147 if len(config.Organization) > 0 { 148 sonar.addOption("sonar.organization=" + config.Organization) 149 } 150 if len(config.Version) > 0 { 151 version := config.CustomScanVersion 152 if len(version) > 0 { 153 log.Entry().Infof("Using custom version: %v", version) 154 } else { 155 version = versioning.ApplyVersioningModel(config.VersioningModel, config.Version) 156 } 157 sonar.addOption("sonar.projectVersion=" + version) 158 } 159 if GeneralConfig.Verbose { 160 sonar.addOption("sonar.verbose=true") 161 } 162 if len(config.ProjectKey) > 0 { 163 sonar.addOption("sonar.projectKey=" + config.ProjectKey) 164 } 165 if len(config.M2Path) > 0 && config.InferJavaLibraries { 166 sonar.addOption(javaLibraries + filepath.Join(config.M2Path, "**")) 167 } 168 if len(config.CoverageExclusions) > 0 && !isInOptions(config, coverageExclusions) { 169 sonar.addOption(coverageExclusions + strings.Join(config.CoverageExclusions, ",")) 170 } 171 if config.InferJavaBinaries && !isInOptions(config, javaBinaries) { 172 addJavaBinaries() 173 } 174 if config.WaitForQualityGate { 175 sonar.addOption("sonar.qualitygate.wait=true") 176 } 177 if err := handlePullRequest(config); err != nil { 178 log.SetErrorCategory(log.ErrorConfiguration) 179 return err 180 } 181 if err := loadSonarScanner(config.SonarScannerDownloadURL, client); err != nil { 182 log.SetErrorCategory(log.ErrorInfrastructure) 183 return err 184 } 185 if err := loadCertificates(config.CustomTLSCertificateLinks, client, runner); err != nil { 186 log.SetErrorCategory(log.ErrorInfrastructure) 187 return err 188 } 189 190 if len(config.Options) > 0 { 191 sonar.options = append(sonar.options, config.Options...) 192 } 193 194 sonar.options = piperutils.PrefixIfNeeded(piperutils.Trim(sonar.options), "-D") 195 196 log.Entry(). 197 WithField("command", sonar.binary). 198 WithField("options", sonar.options). 199 WithField("environment", sonar.environment). 200 Debug("Executing sonar scan command") 201 // execute scan 202 runner.SetEnv(sonar.environment) 203 err := runner.RunExecutable(sonar.binary, sonar.options...) 204 if err != nil { 205 return err 206 } 207 208 // as PRs are handled locally for legacy SonarQube systems, no measurements will be fetched. 209 if len(config.ChangeID) > 0 && config.LegacyPRHandling { 210 return nil 211 } 212 213 // load task results 214 taskReport, err := SonarUtils.ReadTaskReport(sonar.workingDir) 215 if err != nil { 216 log.Entry().WithError(err).Warning("no scan report found") 217 return nil 218 } 219 220 var serverUrl string 221 222 if len(config.Proxy) > 0 { 223 serverUrl = config.ServerURL 224 } else { 225 serverUrl = taskReport.ServerURL 226 } 227 // write reports JSON 228 reports := []piperutils.Path{ 229 { 230 Target: "sonarscan.json", 231 Mandatory: false, 232 }, 233 } 234 // write links JSON 235 links := []piperutils.Path{ 236 { 237 Target: taskReport.DashboardURL, 238 Name: "Sonar Web UI", 239 }, 240 } 241 piperutils.PersistReportsAndLinks("sonarExecuteScan", sonar.workingDir, utils, reports, links) 242 243 if len(config.Token) == 0 { 244 log.Entry().Warn("no measurements are fetched due to missing credentials") 245 return nil 246 } 247 taskService := SonarUtils.NewTaskService(serverUrl, config.Token, taskReport.TaskID, apiClient) 248 // wait for analysis task to complete 249 err = taskService.WaitForTask() 250 if err != nil { 251 return err 252 } 253 // fetch number of issues by severity 254 issueService := SonarUtils.NewIssuesService(serverUrl, config.Token, taskReport.ProjectKey, config.Organization, config.BranchName, config.ChangeID, apiClient) 255 influx.sonarqube_data.fields.blocker_issues, err = issueService.GetNumberOfBlockerIssues() 256 if err != nil { 257 return err 258 } 259 influx.sonarqube_data.fields.critical_issues, err = issueService.GetNumberOfCriticalIssues() 260 if err != nil { 261 return err 262 } 263 influx.sonarqube_data.fields.major_issues, err = issueService.GetNumberOfMajorIssues() 264 if err != nil { 265 return err 266 } 267 influx.sonarqube_data.fields.minor_issues, err = issueService.GetNumberOfMinorIssues() 268 if err != nil { 269 return err 270 } 271 influx.sonarqube_data.fields.info_issues, err = issueService.GetNumberOfInfoIssues() 272 if err != nil { 273 return err 274 } 275 276 reportData := SonarUtils.ReportData{ 277 ServerURL: taskReport.ServerURL, 278 ProjectKey: taskReport.ProjectKey, 279 TaskID: taskReport.TaskID, 280 ChangeID: config.ChangeID, 281 BranchName: config.BranchName, 282 Organization: config.Organization, 283 NumberOfIssues: SonarUtils.Issues{ 284 Blocker: influx.sonarqube_data.fields.blocker_issues, 285 Critical: influx.sonarqube_data.fields.critical_issues, 286 Major: influx.sonarqube_data.fields.major_issues, 287 Minor: influx.sonarqube_data.fields.minor_issues, 288 Info: influx.sonarqube_data.fields.info_issues, 289 }} 290 291 componentService := SonarUtils.NewMeasuresComponentService(serverUrl, config.Token, taskReport.ProjectKey, config.Organization, config.BranchName, config.ChangeID, apiClient) 292 cov, err := componentService.GetCoverage() 293 if err != nil { 294 log.Entry().Warnf("failed to retrieve sonar coverage data: %v", err) 295 } else { 296 reportData.Coverage = cov 297 } 298 299 loc, err := componentService.GetLinesOfCode() 300 if err != nil { 301 log.Entry().Warnf("failed to retrieve sonar lines of code data: %v", err) 302 } else { 303 reportData.LinesOfCode = loc 304 } 305 306 log.Entry().Debugf("Influx values: %v", influx.sonarqube_data.fields) 307 308 err = SonarUtils.WriteReport(reportData, sonar.workingDir, os.WriteFile) 309 310 if err != nil { 311 return err 312 } 313 return nil 314 } 315 316 // isInOptions returns true, if the given property is already provided in config.Options. 317 func isInOptions(config sonarExecuteScanOptions, property string) bool { 318 property = strings.TrimSuffix(property, "=") 319 return piperutils.ContainsStringPart(config.Options, property) 320 } 321 322 func addJavaBinaries() { 323 pomFiles, err := doublestarGlob(pomXMLPattern) 324 if err != nil { 325 log.Entry().Warnf("failed to glob for pom modules: %v", err) 326 return 327 } 328 var binaries []string 329 330 var classesDirs = []string{"classes", "test-classes"} 331 332 for _, pomFile := range pomFiles { 333 module := filepath.Dir(pomFile) 334 for _, classDir := range classesDirs { 335 classesPath := filepath.Join(module, "target", classDir) 336 _, err := osStat(classesPath) 337 if err == nil { 338 binaries = append(binaries, classesPath) 339 } 340 } 341 } 342 if len(binaries) > 0 { 343 sonar.addOption(javaBinaries + strings.Join(binaries, ",")) 344 } 345 } 346 347 func handlePullRequest(config sonarExecuteScanOptions) error { 348 if len(config.ChangeID) > 0 { 349 if config.LegacyPRHandling { 350 // see https://docs.sonarqube.org/display/PLUG/GitHub+Plugin 351 sonar.addOption("sonar.analysis.mode=preview") 352 sonar.addOption("sonar.github.pullRequest=" + config.ChangeID) 353 if len(config.GithubAPIURL) > 0 { 354 sonar.addOption("sonar.github.endpoint=" + config.GithubAPIURL) 355 } 356 if len(config.GithubToken) > 0 { 357 sonar.addOption("sonar.github.oauth=" + config.GithubToken) 358 } 359 if len(config.Owner) > 0 && len(config.Repository) > 0 { 360 sonar.addOption("sonar.github.repository=" + config.Owner + "/" + config.Repository) 361 } 362 if config.DisableInlineComments { 363 sonar.addOption("sonar.github.disableInlineComments=" + strconv.FormatBool(config.DisableInlineComments)) 364 } 365 } else { 366 // see https://sonarcloud.io/documentation/analysis/pull-request/ 367 provider := strings.ToLower(config.PullRequestProvider) 368 if provider == "github" { 369 if len(config.Owner) > 0 && len(config.Repository) > 0 { 370 sonar.addOption("sonar.pullrequest.github.repository=" + config.Owner + "/" + config.Repository) 371 } 372 } else { 373 return errors.New("Pull-Request provider '" + provider + "' is not supported!") 374 } 375 sonar.addOption("sonar.pullrequest.key=" + config.ChangeID) 376 sonar.addOption("sonar.pullrequest.base=" + config.ChangeTarget) 377 sonar.addOption("sonar.pullrequest.branch=" + config.ChangeBranch) 378 sonar.addOption("sonar.pullrequest.provider=" + provider) 379 } 380 } else if len(config.BranchName) > 0 { 381 sonar.addOption("sonar.branch.name=" + config.BranchName) 382 } 383 return nil 384 } 385 386 func loadSonarScanner(url string, client piperhttp.Downloader) error { 387 if scannerPath, err := execLookPath(sonar.binary); err == nil { 388 // using existing sonar-scanner 389 log.Entry().WithField("path", scannerPath).Debug("Using local sonar-scanner") 390 } else if len(url) != 0 { 391 // download sonar-scanner-cli into TEMP folder 392 log.Entry().WithField("url", url).Debug("Downloading sonar-scanner") 393 tmpFolder := getTempDir() 394 defer os.RemoveAll(tmpFolder) // clean up 395 archive := filepath.Join(tmpFolder, path.Base(url)) 396 if err := client.DownloadFile(url, archive, nil, nil); err != nil { 397 return errors.Wrap(err, "Download of sonar-scanner failed") 398 } 399 // unzip sonar-scanner-cli 400 log.Entry().WithField("source", archive).WithField("target", tmpFolder).Debug("Extracting sonar-scanner") 401 if _, err := fileUtilsUnzip(archive, tmpFolder); err != nil { 402 return errors.Wrap(err, "Extraction of sonar-scanner failed") 403 } 404 // move sonar-scanner-cli to .sonar-scanner/ 405 toolPath := ".sonar-scanner" 406 foldername := strings.ReplaceAll(strings.ReplaceAll(archive, ".zip", ""), "cli-", "") 407 log.Entry().WithField("source", foldername).WithField("target", toolPath).Debug("Moving sonar-scanner") 408 if err := osRename(foldername, toolPath); err != nil { 409 return errors.Wrap(err, "Moving of sonar-scanner failed") 410 } 411 // update binary path 412 sonar.binary = filepath.Join(getWorkingDir(), toolPath, "bin", sonar.binary) 413 log.Entry().Debug("Download completed") 414 } 415 return nil 416 } 417 418 func loadCertificates(certificateList []string, client piperhttp.Downloader, runner command.ExecRunner) error { 419 truststorePath := filepath.Join(getWorkingDir(), ".certificates") 420 truststoreFile := filepath.Join(truststorePath, "cacerts") 421 422 if exists, _ := fileUtilsExists(truststoreFile); exists { 423 // use local existing trust store 424 sonar.addEnvironment("SONAR_SCANNER_OPTS=" + keytool.GetMavenOpts(truststoreFile)) 425 log.Entry().WithField("trust store", truststoreFile).Info("Using local trust store") 426 } else if len(certificateList) > 0 { 427 // create download temp dir 428 tmpFolder := getTempDir() 429 defer os.RemoveAll(tmpFolder) // clean up 430 if err := os.MkdirAll(truststorePath, 0777); err != nil { 431 log.Entry().Warningf("failed to create directory %v: %v", truststorePath, err) 432 } 433 // copying existing truststore 434 defaultTruststorePath := keytool.GetDefaultTruststorePath() 435 if exists, _ := fileUtilsExists(defaultTruststorePath); exists { 436 if err := keytool.ImportTruststore(runner, truststoreFile, defaultTruststorePath); err != nil { 437 return errors.Wrap(err, "Copying existing keystore failed") 438 } 439 } 440 // use local created trust store with downloaded certificates 441 for _, certificate := range certificateList { 442 target := filepath.Join(tmpFolder, path.Base(certificate)) 443 log.Entry().WithField("source", certificate).WithField("target", target).Info("Downloading TLS certificate") 444 // download certificate 445 if err := client.DownloadFile(certificate, target, nil, nil); err != nil { 446 return errors.Wrapf(err, "Download of TLS certificate failed") 447 } 448 // add certificate to keystore 449 if err := keytool.ImportCert(runner, truststoreFile, target); err != nil { 450 log.Entry().Warnf("Adding certificate to keystore failed") 451 // return errors.Wrap(err, "Adding certificate to keystore failed") 452 } 453 } 454 sonar.addEnvironment("SONAR_SCANNER_OPTS=" + keytool.GetMavenOpts(truststoreFile)) 455 log.Entry().WithField("trust store", truststoreFile).Info("Using local trust store") 456 } else { 457 log.Entry().Debug("Download of TLS certificates skipped") 458 } 459 return nil 460 } 461 462 func getWorkingDir() string { 463 workingDir, err := os.Getwd() 464 if err != nil { 465 log.Entry().WithError(err).WithField("path", workingDir).Debug("Retrieving of work directory failed") 466 } 467 return workingDir 468 } 469 470 func getTempDir() string { 471 tmpFolder, err := os.MkdirTemp(".", "temp-") 472 if err != nil { 473 log.Entry().WithError(err).WithField("path", tmpFolder).Debug("Creating temp directory failed") 474 } 475 return tmpFolder 476 } 477 478 // Fetches parameters from environment variables and updates the options accordingly (only if not already set) 479 func detectParametersFromCI(options *sonarExecuteScanOptions) { 480 provider, err := orchestrator.GetOrchestratorConfigProvider(nil) 481 if err != nil { 482 log.Entry().WithError(err).Warning("Cannot infer config from CI environment") 483 return 484 } 485 486 if provider.IsPullRequest() { 487 config := provider.PullRequestConfig() 488 if len(options.ChangeBranch) == 0 { 489 log.Entry().Info("Inferring parameter changeBranch from environment: " + config.Branch) 490 options.ChangeBranch = config.Branch 491 } 492 if len(options.ChangeTarget) == 0 { 493 log.Entry().Info("Inferring parameter changeTarget from environment: " + config.Base) 494 options.ChangeTarget = config.Base 495 } 496 if len(options.ChangeID) == 0 { 497 log.Entry().Info("Inferring parameter changeId from environment: " + config.Key) 498 options.ChangeID = config.Key 499 } 500 } else { 501 branch := provider.Branch() 502 if options.InferBranchName && len(options.BranchName) == 0 { 503 log.Entry().Info("Inferring parameter branchName from environment: " + branch) 504 options.BranchName = branch 505 } 506 } 507 }