github.com/jaylevin/jenkins-library@v1.230.4/cmd/sonarExecuteScan.go (about) 1 package cmd 2 3 import ( 4 "io/ioutil" 5 "os" 6 "os/exec" 7 "path" 8 "path/filepath" 9 "strconv" 10 "strings" 11 "time" 12 13 "github.com/bmatcuk/doublestar" 14 "github.com/pkg/errors" 15 16 "github.com/SAP/jenkins-library/pkg/command" 17 piperhttp "github.com/SAP/jenkins-library/pkg/http" 18 keytool "github.com/SAP/jenkins-library/pkg/java" 19 "github.com/SAP/jenkins-library/pkg/log" 20 "github.com/SAP/jenkins-library/pkg/orchestrator" 21 FileUtils "github.com/SAP/jenkins-library/pkg/piperutils" 22 SliceUtils "github.com/SAP/jenkins-library/pkg/piperutils" 23 StepResults "github.com/SAP/jenkins-library/pkg/piperutils" 24 SonarUtils "github.com/SAP/jenkins-library/pkg/sonar" 25 "github.com/SAP/jenkins-library/pkg/telemetry" 26 "github.com/SAP/jenkins-library/pkg/versioning" 27 ) 28 29 type sonarSettings struct { 30 workingDir string 31 binary string 32 environment []string 33 options []string 34 } 35 36 func (s *sonarSettings) addEnvironment(element string) { 37 s.environment = append(s.environment, element) 38 } 39 40 func (s *sonarSettings) addOption(element string) { 41 s.options = append(s.options, element) 42 } 43 44 var ( 45 sonar sonarSettings 46 47 execLookPath = exec.LookPath 48 fileUtilsExists = FileUtils.FileExists 49 fileUtilsUnzip = FileUtils.Unzip 50 osRename = os.Rename 51 osStat = os.Stat 52 doublestarGlob = doublestar.Glob 53 ) 54 55 const ( 56 coverageReportPaths = "sonar.coverage.jacoco.xmlReportPaths=" 57 javaBinaries = "sonar.java.binaries=" 58 javaLibraries = "sonar.java.libraries=" 59 coverageExclusions = "sonar.coverage.exclusions=" 60 pomXMLPattern = "**/pom.xml" 61 ) 62 63 func sonarExecuteScan(config sonarExecuteScanOptions, _ *telemetry.CustomData, influx *sonarExecuteScanInflux) { 64 runner := command.Command{ 65 ErrorCategoryMapping: map[string][]string{ 66 log.ErrorConfiguration.String(): { 67 "You must define the following mandatory properties for '*': *", 68 "org.sonar.java.AnalysisException: Your project contains .java files, please provide compiled classes with sonar.java.binaries property, or exclude them from the analysis with sonar.exclusions property.", 69 "ERROR: Invalid value for *", 70 "java.lang.IllegalStateException: No files nor directories matching '*'", 71 }, 72 log.ErrorInfrastructure.String(): { 73 "ERROR: SonarQube server [*] can not be reached", 74 "Caused by: java.net.SocketTimeoutException: timeout", 75 "java.lang.IllegalStateException: Fail to request *", 76 "java.lang.IllegalStateException: Fail to download plugin [*] into *", 77 }, 78 }, 79 } 80 // reroute command output to logging framework 81 runner.Stdout(log.Writer()) 82 runner.Stderr(log.Writer()) 83 // client for downloading the sonar-scanner 84 downloadClient := &piperhttp.Client{} 85 downloadClient.SetOptions(piperhttp.ClientOptions{TransportTimeout: 20 * time.Second}) 86 // client for talking to the SonarQube API 87 apiClient := &piperhttp.Client{} 88 //TODO: implement certificate handling 89 apiClient.SetOptions(piperhttp.ClientOptions{TransportSkipVerification: true}) 90 91 sonar = sonarSettings{ 92 workingDir: "./", 93 binary: "sonar-scanner", 94 environment: []string{}, 95 options: []string{}, 96 } 97 98 influx.step_data.fields.sonar = false 99 if err := runSonar(config, downloadClient, &runner, apiClient, influx); err != nil { 100 if log.GetErrorCategory() == log.ErrorUndefined && runner.GetExitCode() == 2 { 101 // see https://github.com/SonarSource/sonar-scanner-cli/blob/adb67d645c3bcb9b46f29dea06ba082ebec9ba7a/src/main/java/org/sonarsource/scanner/cli/Exit.java#L25 102 log.SetErrorCategory(log.ErrorConfiguration) 103 } 104 log.Entry().WithError(err).Fatal("Execution failed") 105 } 106 influx.step_data.fields.sonar = true 107 } 108 109 func runSonar(config sonarExecuteScanOptions, client piperhttp.Downloader, runner command.ExecRunner, apiClient SonarUtils.Sender, influx *sonarExecuteScanInflux) error { 110 // Set config based on orchestrator-specific environment variables 111 detectParametersFromCI(&config) 112 113 if len(config.ServerURL) > 0 { 114 sonar.addEnvironment("SONAR_HOST_URL=" + config.ServerURL) 115 } 116 if len(config.Token) == 0 { 117 log.Entry().Warn("sonar token not set") 118 // use token provided by sonar-scanner-jenkins plugin 119 // https://github.com/SonarSource/sonar-scanner-jenkins/blob/441ef2f485884758b60767bed2ef8a1a0a7fc863/src/main/java/hudson/plugins/sonar/SonarBuildWrapper.java#L132 120 if len(os.Getenv("SONAR_AUTH_TOKEN")) > 0 { 121 log.Entry().Info("using token from env var SONAR_AUTH_TOKEN") 122 config.Token = os.Getenv("SONAR_AUTH_TOKEN") 123 } 124 } 125 if len(config.Token) > 0 { 126 sonar.addEnvironment("SONAR_TOKEN=" + config.Token) 127 } 128 if len(config.Organization) > 0 { 129 sonar.addOption("sonar.organization=" + config.Organization) 130 } 131 if len(config.Version) > 0 { 132 version := config.CustomScanVersion 133 if len(version) > 0 { 134 log.Entry().Infof("Using custom version: %v", version) 135 } else { 136 version = versioning.ApplyVersioningModel(config.VersioningModel, config.Version) 137 } 138 sonar.addOption("sonar.projectVersion=" + version) 139 } 140 if GeneralConfig.Verbose { 141 sonar.addOption("sonar.verbose=true") 142 } 143 if len(config.ProjectKey) > 0 { 144 sonar.addOption("sonar.projectKey=" + config.ProjectKey) 145 } 146 if len(config.M2Path) > 0 && config.InferJavaLibraries { 147 sonar.addOption(javaLibraries + filepath.Join(config.M2Path, "**")) 148 } 149 if len(config.CoverageExclusions) > 0 && !isInOptions(config, coverageExclusions) { 150 sonar.addOption(coverageExclusions + strings.Join(config.CoverageExclusions, ",")) 151 } 152 if config.InferJavaBinaries && !isInOptions(config, javaBinaries) { 153 addJavaBinaries() 154 } 155 if err := handlePullRequest(config); err != nil { 156 log.SetErrorCategory(log.ErrorConfiguration) 157 return err 158 } 159 if err := loadSonarScanner(config.SonarScannerDownloadURL, client); err != nil { 160 log.SetErrorCategory(log.ErrorInfrastructure) 161 return err 162 } 163 if err := loadCertificates(config.CustomTLSCertificateLinks, client, runner); err != nil { 164 log.SetErrorCategory(log.ErrorInfrastructure) 165 return err 166 } 167 168 if len(config.Options) > 0 { 169 sonar.options = append(sonar.options, config.Options...) 170 } 171 172 sonar.options = SliceUtils.PrefixIfNeeded(SliceUtils.Trim(sonar.options), "-D") 173 174 log.Entry(). 175 WithField("command", sonar.binary). 176 WithField("options", sonar.options). 177 WithField("environment", sonar.environment). 178 Debug("Executing sonar scan command") 179 // execute scan 180 runner.SetEnv(sonar.environment) 181 err := runner.RunExecutable(sonar.binary, sonar.options...) 182 if err != nil { 183 return err 184 } 185 186 // as PRs are handled locally for legacy SonarQube systems, no measurements will be fetched. 187 if len(config.ChangeID) > 0 && config.LegacyPRHandling { 188 return nil 189 } 190 191 // load task results 192 taskReport, err := SonarUtils.ReadTaskReport(sonar.workingDir) 193 if err != nil { 194 log.Entry().WithError(err).Warning("no scan report found") 195 return nil 196 } 197 // write links JSON 198 links := []StepResults.Path{ 199 { 200 Target: taskReport.DashboardURL, 201 Name: "Sonar Web UI", 202 }, 203 } 204 StepResults.PersistReportsAndLinks("sonarExecuteScan", sonar.workingDir, nil, links) 205 206 if len(config.Token) == 0 { 207 log.Entry().Warn("no measurements are fetched due to missing credentials") 208 return nil 209 } 210 taskService := SonarUtils.NewTaskService(taskReport.ServerURL, config.Token, taskReport.TaskID, apiClient) 211 // wait for analysis task to complete 212 err = taskService.WaitForTask() 213 if err != nil { 214 return err 215 } 216 // fetch number of issues by severity 217 issueService := SonarUtils.NewIssuesService(taskReport.ServerURL, config.Token, taskReport.ProjectKey, config.Organization, config.BranchName, config.ChangeID, apiClient) 218 influx.sonarqube_data.fields.blocker_issues, err = issueService.GetNumberOfBlockerIssues() 219 if err != nil { 220 return err 221 } 222 influx.sonarqube_data.fields.critical_issues, err = issueService.GetNumberOfCriticalIssues() 223 if err != nil { 224 return err 225 } 226 influx.sonarqube_data.fields.major_issues, err = issueService.GetNumberOfMajorIssues() 227 if err != nil { 228 return err 229 } 230 influx.sonarqube_data.fields.minor_issues, err = issueService.GetNumberOfMinorIssues() 231 if err != nil { 232 return err 233 } 234 influx.sonarqube_data.fields.info_issues, err = issueService.GetNumberOfInfoIssues() 235 if err != nil { 236 return err 237 } 238 239 reportData := SonarUtils.ReportData{ 240 ServerURL: taskReport.ServerURL, 241 ProjectKey: taskReport.ProjectKey, 242 TaskID: taskReport.TaskID, 243 ChangeID: config.ChangeID, 244 BranchName: config.BranchName, 245 Organization: config.Organization, 246 NumberOfIssues: SonarUtils.Issues{ 247 Blocker: influx.sonarqube_data.fields.blocker_issues, 248 Critical: influx.sonarqube_data.fields.critical_issues, 249 Major: influx.sonarqube_data.fields.major_issues, 250 Minor: influx.sonarqube_data.fields.minor_issues, 251 Info: influx.sonarqube_data.fields.info_issues, 252 }} 253 254 componentService := SonarUtils.NewMeasuresComponentService(taskReport.ServerURL, config.Token, taskReport.ProjectKey, config.Organization, config.BranchName, config.ChangeID, apiClient) 255 cov, err := componentService.GetCoverage() 256 if err != nil { 257 log.Entry().Warnf("failed to retrieve sonar coverage data: %v", err) 258 } else { 259 reportData.Coverage = cov 260 } 261 262 loc, err := componentService.GetLinesOfCode() 263 if err != nil { 264 log.Entry().Warnf("failed to retrieve sonar lines of code data: %v", err) 265 } else { 266 reportData.LinesOfCode = loc 267 } 268 269 log.Entry().Debugf("Influx values: %v", influx.sonarqube_data.fields) 270 271 err = SonarUtils.WriteReport(reportData, sonar.workingDir, ioutil.WriteFile) 272 273 if err != nil { 274 return err 275 } 276 return nil 277 } 278 279 // isInOptions returns true, if the given property is already provided in config.Options. 280 func isInOptions(config sonarExecuteScanOptions, property string) bool { 281 property = strings.TrimSuffix(property, "=") 282 return SliceUtils.ContainsStringPart(config.Options, property) 283 } 284 285 func addJavaBinaries() { 286 pomFiles, err := doublestarGlob(pomXMLPattern) 287 if err != nil { 288 log.Entry().Warnf("failed to glob for pom modules: %v", err) 289 return 290 } 291 var binaries []string 292 293 var classesDirs = []string{"classes", "test-classes"} 294 295 for _, pomFile := range pomFiles { 296 module := filepath.Dir(pomFile) 297 for _, classDir := range classesDirs { 298 classesPath := filepath.Join(module, "target", classDir) 299 _, err := osStat(classesPath) 300 if err == nil { 301 binaries = append(binaries, classesPath) 302 } 303 } 304 } 305 if len(binaries) > 0 { 306 sonar.addOption(javaBinaries + strings.Join(binaries, ",")) 307 } 308 } 309 310 func handlePullRequest(config sonarExecuteScanOptions) error { 311 if len(config.ChangeID) > 0 { 312 if config.LegacyPRHandling { 313 // see https://docs.sonarqube.org/display/PLUG/GitHub+Plugin 314 sonar.addOption("sonar.analysis.mode=preview") 315 sonar.addOption("sonar.github.pullRequest=" + config.ChangeID) 316 if len(config.GithubAPIURL) > 0 { 317 sonar.addOption("sonar.github.endpoint=" + config.GithubAPIURL) 318 } 319 if len(config.GithubToken) > 0 { 320 sonar.addOption("sonar.github.oauth=" + config.GithubToken) 321 } 322 if len(config.Owner) > 0 && len(config.Repository) > 0 { 323 sonar.addOption("sonar.github.repository=" + config.Owner + "/" + config.Repository) 324 } 325 if config.DisableInlineComments { 326 sonar.addOption("sonar.github.disableInlineComments=" + strconv.FormatBool(config.DisableInlineComments)) 327 } 328 } else { 329 // see https://sonarcloud.io/documentation/analysis/pull-request/ 330 provider := strings.ToLower(config.PullRequestProvider) 331 if provider == "github" { 332 if len(config.Owner) > 0 && len(config.Repository) > 0 { 333 sonar.addOption("sonar.pullrequest.github.repository=" + config.Owner + "/" + config.Repository) 334 } 335 } else { 336 return errors.New("Pull-Request provider '" + provider + "' is not supported!") 337 } 338 sonar.addOption("sonar.pullrequest.key=" + config.ChangeID) 339 sonar.addOption("sonar.pullrequest.base=" + config.ChangeTarget) 340 sonar.addOption("sonar.pullrequest.branch=" + config.ChangeBranch) 341 sonar.addOption("sonar.pullrequest.provider=" + provider) 342 } 343 } else if len(config.BranchName) > 0 { 344 sonar.addOption("sonar.branch.name=" + config.BranchName) 345 } 346 return nil 347 } 348 349 func loadSonarScanner(url string, client piperhttp.Downloader) error { 350 if scannerPath, err := execLookPath(sonar.binary); err == nil { 351 // using existing sonar-scanner 352 log.Entry().WithField("path", scannerPath).Debug("Using local sonar-scanner") 353 } else if len(url) != 0 { 354 // download sonar-scanner-cli into TEMP folder 355 log.Entry().WithField("url", url).Debug("Downloading sonar-scanner") 356 tmpFolder := getTempDir() 357 defer os.RemoveAll(tmpFolder) // clean up 358 archive := filepath.Join(tmpFolder, path.Base(url)) 359 if err := client.DownloadFile(url, archive, nil, nil); err != nil { 360 return errors.Wrap(err, "Download of sonar-scanner failed") 361 } 362 // unzip sonar-scanner-cli 363 log.Entry().WithField("source", archive).WithField("target", tmpFolder).Debug("Extracting sonar-scanner") 364 if _, err := fileUtilsUnzip(archive, tmpFolder); err != nil { 365 return errors.Wrap(err, "Extraction of sonar-scanner failed") 366 } 367 // move sonar-scanner-cli to .sonar-scanner/ 368 toolPath := ".sonar-scanner" 369 foldername := strings.ReplaceAll(strings.ReplaceAll(archive, ".zip", ""), "cli-", "") 370 log.Entry().WithField("source", foldername).WithField("target", toolPath).Debug("Moving sonar-scanner") 371 if err := osRename(foldername, toolPath); err != nil { 372 return errors.Wrap(err, "Moving of sonar-scanner failed") 373 } 374 // update binary path 375 sonar.binary = filepath.Join(getWorkingDir(), toolPath, "bin", sonar.binary) 376 log.Entry().Debug("Download completed") 377 } 378 return nil 379 } 380 381 func loadCertificates(certificateList []string, client piperhttp.Downloader, runner command.ExecRunner) error { 382 truststorePath := filepath.Join(getWorkingDir(), ".certificates") 383 truststoreFile := filepath.Join(truststorePath, "cacerts") 384 385 if exists, _ := fileUtilsExists(truststoreFile); exists { 386 // use local existing trust store 387 sonar.addEnvironment("SONAR_SCANNER_OPTS=" + keytool.GetMavenOpts(truststoreFile)) 388 log.Entry().WithField("trust store", truststoreFile).Info("Using local trust store") 389 } else if len(certificateList) > 0 { 390 // create download temp dir 391 tmpFolder := getTempDir() 392 defer os.RemoveAll(tmpFolder) // clean up 393 os.MkdirAll(truststorePath, 0777) 394 // copying existing truststore 395 defaultTruststorePath := keytool.GetDefaultTruststorePath() 396 if exists, _ := fileUtilsExists(defaultTruststorePath); exists { 397 if err := keytool.ImportTruststore(runner, truststoreFile, defaultTruststorePath); err != nil { 398 return errors.Wrap(err, "Copying existing keystore failed") 399 } 400 } 401 // use local created trust store with downloaded certificates 402 for _, certificate := range certificateList { 403 target := filepath.Join(tmpFolder, path.Base(certificate)) 404 log.Entry().WithField("source", certificate).WithField("target", target).Info("Downloading TLS certificate") 405 // download certificate 406 if err := client.DownloadFile(certificate, target, nil, nil); err != nil { 407 return errors.Wrapf(err, "Download of TLS certificate failed") 408 } 409 // add certificate to keystore 410 if err := keytool.ImportCert(runner, truststoreFile, target); err != nil { 411 log.Entry().Warnf("Adding certificate to keystore failed") 412 // return errors.Wrap(err, "Adding certificate to keystore failed") 413 } 414 } 415 sonar.addEnvironment("SONAR_SCANNER_OPTS=" + keytool.GetMavenOpts(truststoreFile)) 416 log.Entry().WithField("trust store", truststoreFile).Info("Using local trust store") 417 } else { 418 log.Entry().Debug("Download of TLS certificates skipped") 419 } 420 return nil 421 } 422 423 func getWorkingDir() string { 424 workingDir, err := os.Getwd() 425 if err != nil { 426 log.Entry().WithError(err).WithField("path", workingDir).Debug("Retrieving of work directory failed") 427 } 428 return workingDir 429 } 430 431 func getTempDir() string { 432 tmpFolder, err := ioutil.TempDir(".", "temp-") 433 if err != nil { 434 log.Entry().WithError(err).WithField("path", tmpFolder).Debug("Creating temp directory failed") 435 } 436 return tmpFolder 437 } 438 439 // Fetches parameters from environment variables and updates the options accordingly (only if not already set) 440 func detectParametersFromCI(options *sonarExecuteScanOptions) { 441 provider, err := orchestrator.NewOrchestratorSpecificConfigProvider() 442 if err != nil { 443 log.Entry().WithError(err).Warning("Cannot infer config from CI environment") 444 return 445 } 446 447 if provider.IsPullRequest() { 448 config := provider.GetPullRequestConfig() 449 if len(options.ChangeBranch) == 0 { 450 log.Entry().Info("Inferring parameter changeBranch from environment: " + config.Branch) 451 options.ChangeBranch = config.Branch 452 } 453 if len(options.ChangeTarget) == 0 { 454 log.Entry().Info("Inferring parameter changeTarget from environment: " + config.Base) 455 options.ChangeTarget = config.Base 456 } 457 if len(options.ChangeID) == 0 { 458 log.Entry().Info("Inferring parameter changeId from environment: " + config.Key) 459 options.ChangeID = config.Key 460 } 461 } else { 462 branch := provider.GetBranch() 463 if options.InferBranchName && len(options.BranchName) == 0 { 464 log.Entry().Info("Inferring parameter branchName from environment: " + branch) 465 options.BranchName = branch 466 } 467 } 468 }